#!/usr/bin/env bash
echo -e "\n-========================================================================-"
echo -e "-=-                                                                    -=-"
echo -e "-=-      SCRAPE_AGIS.SH: Downloads ArcGIS maps                         -=-"
echo -e "-=-                                                                    -=-"
echo -e "-=-     Lillian Skinner                                                -=-"
echo -e "-=-                                                                    -=-"
echo -e "-========================================================================-"

WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"

ARGIS_URL="https://maps.london.ca/server/rest/services"

TMP="./tmp"
TMP_STAGING="./tmp/layers"
SERVICELIST_JSON="$TMP/servicelist.json"
FOLDER_JSON="$TMP/folder.json"
SERVICE_JSON="$TMP/service.json"
LAYERQUERY_JSON="$TMP/layer_query.json"

mkdir "$TMP"
mkdir "$TMP_STAGING"

wget "$ARGIS_URL?f=json" --user-agent="$WGET_UA" -O "$SERVICELIST_JSON" -q

jq -r '.folders[]?' "$SERVICELIST_JSON" | while read -r FOLDER; do
  wget "$ARGIS_URL/$FOLDER?f=json" --user-agent="$WGET_UA" -O "$FOLDER_JSON" -q
  echo "Looking in $FOLDER"
  jq -r '.services[]
         | select(.type=="MapServer")
         | .name' "$FOLDER_JSON" | while read -r SERVICE; do
    echo "Found $SERVICE"
    SERVICE_PATH="$FOLDER/$SERVICE"
    echo "$ARGIS_URL/$SERVICE/MapServer"
	wget "$ARGIS_URL/$SERVICE/MapServer?f=json" --user-agent="$WGET_UA" -O "$SERVICE_JSON" -q

    rm -r "$TMP_STAGING"
    mkdir "$TMP_STAGING"
    jq -r '.layers[]? | "\(.id)|\(.name)"' "$SERVICE_JSON" | while IFS='|' read -r LAYERID LAYERNAME; do
    
      echo "Downloading $LAYERID-$LAYERNAME..."
      curl -s \
        "$ARGIS_URL/$SERVICE/MapServer/$LAYERID/query\
?where=1=1\
&outFields=*\
&returnGeometry=true\
&f=geojson" \
        -o "$TMP_STAGING/layer${LAYERID}-${LAYERNAME}.geojson"
    done
    mkdir -p "LondonArchive/ArcGIS/${FOLDER}/${SERVICE}"
    7z a "LondonArchive/ArcGIS/${FOLDER}/${SERVICE}/layers.7z" "$TMP_STAGING"
  done
done