54 lines
2.1 KiB
Bash
54 lines
2.1 KiB
Bash
#!/usr/bin/env bash
|
|
echo -e "\n-========================================================================-"
|
|
echo -e "-=- -=-"
|
|
echo -e "-=- SCRAPE_AGIS.SH: Downloads ArcGIS maps -=-"
|
|
echo -e "-=- -=-"
|
|
echo -e "-=- Lillian Skinner -=-"
|
|
echo -e "-=- -=-"
|
|
echo -e "-========================================================================-"
|
|
|
|
WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
|
|
|
|
ARGIS_URL="https://maps.london.ca/server/rest/services"
|
|
|
|
TMP="./tmp"
|
|
TMP_STAGING="./tmp/layers"
|
|
SERVICELIST_JSON="$TMP/servicelist.json"
|
|
FOLDER_JSON="$TMP/folder.json"
|
|
SERVICE_JSON="$TMP/service.json"
|
|
LAYERQUERY_JSON="$TMP/layer_query.json"
|
|
|
|
mkdir "$TMP"
|
|
mkdir "$TMP_STAGING"
|
|
|
|
wget "$ARGIS_URL?f=json" --user-agent="$WGET_UA" -O "$SERVICELIST_JSON" -q
|
|
|
|
jq -r '.folders[]?' "$SERVICELIST_JSON" | while read -r FOLDER; do
|
|
wget "$ARGIS_URL/$FOLDER?f=json" --user-agent="$WGET_UA" -O "$FOLDER_JSON" -q
|
|
echo "Looking in $FOLDER"
|
|
jq -r '.services[]
|
|
| select(.type=="MapServer")
|
|
| .name' "$FOLDER_JSON" | while read -r SERVICE; do
|
|
echo "Found $SERVICE"
|
|
SERVICE_PATH="$FOLDER/$SERVICE"
|
|
echo "$ARGIS_URL/$SERVICE/MapServer"
|
|
wget "$ARGIS_URL/$SERVICE/MapServer?f=json" --user-agent="$WGET_UA" -O "$SERVICE_JSON" -q
|
|
|
|
rm -r "$TMP_STAGING"
|
|
mkdir "$TMP_STAGING"
|
|
jq -r '.layers[]? | "\(.id)|\(.name)"' "$SERVICE_JSON" | while IFS='|' read -r LAYERID LAYERNAME; do
|
|
|
|
echo "Downloading $LAYERID-$LAYERNAME..."
|
|
curl -s \
|
|
"$ARGIS_URL/$SERVICE/MapServer/$LAYERID/query\
|
|
?where=1=1\
|
|
&outFields=*\
|
|
&returnGeometry=true\
|
|
&f=geojson" \
|
|
-o "$TMP_STAGING/layer${LAYERID}-${LAYERNAME}.geojson"
|
|
done
|
|
mkdir -p "LondonArchive/ArcGIS/${FOLDER}/${SERVICE}"
|
|
7z a "LondonArchive/ArcGIS/${FOLDER}/${SERVICE}/layers.7z" "$TMP_STAGING"
|
|
done
|
|
done
|