Harvest all stores from wp-store-locator and generate GeoJSON for Google Earth

get-stores.sh

#! /usr/bin/env bash
# shellcheck disable=SC1112
#
# Harvest all stores from wp-store-locator and generate GeoJSON for Google Earth.
#
# Assumes continental US: latitude range 23° to 50° North and longitude range 67° to 126° West.
#
set -o errexit
set -o errtrace
set -o pipefail
set -o nounset
# set -o xtrace
shopt -s inherit_errexit
shopt -s nullglob

SITE_NAME="${1:-}"
if [[ -z "$SITE_NAME" ]]
then
    echo "Usage: $0 SITE_NAME [SITE_URL]"
    exit 1
fi >&2
SITE_URL="${SITE_URL:-${2:-https://$SITE_NAME.com}}"
RADIUS=50
SCF_STORE_SEARCH="$SITE_URL/wp-admin/admin-ajax.php?action=store_search&search_radius=$RADIUS&max_results=500"
printf -v NOW '%(%FT%H%M)T'

main() {
    archive_json

    trap 'kill $(jobs -p) 2>/dev/null' EXIT
    local lat_n_deg
    for lat_n_deg in {23..50}
    do  chk_lons "$lat_n_deg" &
        sleep 0.1
    done
    wait

    grep -h '"id"' ./*.json |sort >"$SITE_NAME.store-ids.$RADIUS.$NOW.txt"
    uniq -c "$SITE_NAME.store-ids.$RADIUS.$NOW.txt" |
        sort -n >"$SITE_NAME.store-ids.$RADIUS.$NOW.uniq"
    awk '{print$1}' "$SITE_NAME.store-ids.$RADIUS.$NOW.uniq" |uniq -c
    uniq "$SITE_NAME.store-ids.$RADIUS.$NOW.txt" |wc -l
    find . -maxdepth 1 -name "$SITE_NAME.*.json" |wc -l

    js2gj
    jq --compact-output . "$SITE_NAME-stores.dedupe.geojson" >"$SITE_NAME-stores.geojson"

    archive_json
}

archive_json() {
    local files=(./*.*json)
    ((${#files[@]})) ||return 0

    tar --create --xz --file "stores.$NOW.tar.xz" -- "${files[@]}"
    rm --force -- ./*.json
}

chk_lons() {
    local lat_n_deg="$1"
    local lon_w_deg outfile size

    for lon_w_deg in {67..126}
    do  outfile="$SITE_NAME-stores.$lat_n_deg.-$lon_w_deg.json"

        curl --silent --show-error --fail \
            "$SCF_STORE_SEARCH&lat=$lat_n_deg.0&lng=-$lon_w_deg.0" |
            jq . >"$outfile"

        size=$(jq '.|length' "$outfile")
        if [[ $size == 0 ]]
        then rm "$outfile"
        elif [[ $size -gt 500 ]]
        then
            >&2 echo "ERROR: Found $size stores at $lat_n_deg,-$lon_w_deg (exceeds max_results=$RADIUS)"
            return 1
        else echo "Found $size stores at $lat_n_deg,-$lon_w_deg"
        fi

    done
}

js2gj() {
    jq --slurp --arg site_name "$SITE_NAME" '

        # flatten list-of-arrays -> array of store objects
        map(.[])

        # de-dupe by id (keep first occurrence)
        |unique_by(.id)

        # GeoJSON
        | {
            type: "FeatureCollection",
            name: ($site_name + "-stores"),
            features: map({
                type: "Feature",
                geometry: {
                    type: "Point",
                    coordinates: [ (.lng|tonumber), (.lat|tonumber) ]
                },
                properties: (
                    .state |= ascii_upcase
                    |.id = (
                        (.state + ": " + .store + ", " + .city)
                        |gsub("&"; "&")
                        |gsub("&"; "&")
                        |gsub("–"; "–")
                        |gsub("‘"; "‘")
                        |gsub("’"; "’")
                        |gsub("“"; "“")
                        |gsub("”"; "”")
                    )
                    |del(.thumb, .country, .lat, .lng, .fax, .email, .hours) # empty/redundant fields
                )
            })
        }

        # sort by formatted store name
        |.features |= sort_by(.properties.id)

    ' "$SITE_NAME"-stores.*.json >"$SITE_NAME-stores.dedupe.geojson"
}

time main "$@"
exit $?