$]*>$\?$[^<]*$.*/\2/p' | sed 's/&/\&/g' | sed 's/'/'\''/g' | uniq) if [[ $PROJECT_INFO_IS_ITEMS == "" ]] && [[ $PROJECT_INFO_IS_BAD == "" ]] && [[ $PROJECT_INFO_ITEM != "" ]]; then # We'll check to see if a non-info item made it in. Sometimes attachments will get caught, but can be detected by "visually-hidden" PROJECT_INFO_LABEL_BAD=$(echo $PROJECT_INFO_LABEL | grep "visually-hidden") if [[ $PROJECT_INFO_LABEL_BAD == "" ]]; then printf "%-17s: %s\n" "$PROJECT_INFO_LABEL" "$PROJECT_INFO_ITEM" >> $PROJECT_INFO if [[ $PROJECT_INFO_LABEL == "File Number" ]]; then PROJECT_FILE_NUM_2="" PROJECT_FILE_NUM_TYPE_2="" PROJECT_FILE_NUM_IS_MULTI="" # Multiple file numbers may be listed. We will always use the first one as it is contained in PDF names. # I think it takes priority. Anyways, here are the possible formats: # XX-##### # XX-#####/XX-##### # XX-##### / XX-##### # XX-##### and XX-##### # XX-##### & XX-##### # # I think the city is allergic to standardization... PROJECT_FILE_NUM_IS_MULTI=$(echo $PROJECT_INFO_ITEM | grep -e "and" -e "/" -e "&") PROJECT_FILE_NUM=$(echo $PROJECT_INFO_ITEM | sed 's|/.*||' | sed 's| and .*||' | sed 's| & .*||' | sed 's/^[[:space:]]*//g' | sed 's/[[:space:]]*$//g' | uniq) PROJECT_FILE_NUM_TYPE=$(echo "$PROJECT_FILE_NUM" | sed 's/^$[^-]*$-.*$/\1/') if [[ "$PROJECT_FILE_NUM_TYPE" == "Line of Sight" ]]; then PROJECT_FILE_TYPE="Line of Sight" elif [[ "$PROJECT_FILE_NUM_TYPE" == "O" ]]; then PROJECT_FILE_TYPE="Official Plan Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE" == "Z" ]]; then PROJECT_FILE_TYPE="Zoning By-law Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE" == "OZ" ]]; then PROJECT_FILE_TYPE="Official Plan and Zoning By-law Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE" == "TZ" ]]; then PROJECT_FILE_TYPE="Temporary Zoning By-law Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE" == "39T" ]]; then PROJECT_FILE_TYPE="Draft Plan of Subdivision" elif [[ "$PROJECT_FILE_NUM_TYPE" == "39CD" ]]; then PROJECT_FILE_TYPE="Draft Plan of Condominium" elif [[ "$PROJECT_FILE_NUM_TYPE" =~ ^SPA2[0-9]+$ ]]; then PROJECT_FILE_TYPE="Site Plan Control Application" else PROJECT_FILE_TYPE="BAD RECORD TYPE" fi echo "SCRAPE_PLANAPPS: Found file# : $PROJECT_FILE_NUM ($PROJECT_FILE_TYPE)" if [[ "$PROJECT_FILE_NUM_IS_MULTI" != "" ]]; then # It isn't great, but if a project has 2 file numbers then we'll save it as both. # I'm not sure how to get around this since I don't have a way to tag files. PROJECT_FILE_NUM_2=$(echo $PROJECT_INFO_ITEM | sed 's|.*/||' | sed 's|.* and ||' | sed 's|.* & ||' | sed 's/^[[:space:]]*//g' | sed 's/[[:space:]]*$//g' | uniq) PROJECT_FILE_NUM_TYPE_2=$(echo "$PROJECT_FILE_NUM_2" | sed 's/^$[^-]*$-.*$/\1/') if [[ "$PROJECT_FILE_NUM_TYPE_2" == "Line of Sight" ]]; then PROJECT_FILE_TYPE_2="Line of Sight" elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "O" ]]; then PROJECT_FILE_TYPE_2="Official Plan Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "Z" ]]; then PROJECT_FILE_TYPE_2="Zoning By-law Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "OZ" ]]; then PROJECT_FILE_TYPE_2="Official Plan and Zoning By-law Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "TZ" ]]; then PROJECT_FILE_TYPE_2="Temporary Zoning By-law Amendment" elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "39T" ]]; then PROJECT_FILE_TYPE_2="Draft Plan of Subdivision" elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "39CD" ]]; then PROJECT_FILE_TYPE_2="Draft Plan of Condominium" elif [[ "$PROJECT_FILE_NUM_TYPE_2" =~ ^SPA2[0-9]+$ ]]; then PROJECT_FILE_TYPE_2="Site Plan Control Application" else PROJECT_FILE_TYPE_2="BAD RECORD TYPE" fi echo "SCRAPE_PLANAPPS: Also filed as: $PROJECT_FILE_NUM_2 ($PROJECT_FILE_TYPE_2)" fi fi PROJECT_FOUND_TIME=$(echo $PLINE | grep "datetime") if [[ $PROJECT_FOUND_TIME != "" ]]; then PROJECT_TIME_YEAR=$(echo $PLINE | sed 's/.*$[0-9]\{4\}$.*/\1/p' | uniq) PROJECT_TIME_MONTH_WORD=$(echo $PLINE | sed 's/.*,\s*$[A-Za-z]*$\s[0-9]\{1,2\},.*/\1/p' | uniq) PROJECT_TIME_DAY_SHORT=$(echo $PLINE | sed 's/.*,\s*[A-Za-z]*\s$[0-9]\{1,2\}$,.*/\1/p' | uniq) PROJECT_TIME_DAY=$(printf "%02d" $PROJECT_TIME_DAY_SHORT) case "$PROJECT_TIME_MONTH_WORD" in January) PROJECT_TIME_MONTH="01" ;; February) PROJECT_TIME_MONTH="02" ;; March) PROJECT_TIME_MONTH="03" ;; April) PROJECT_TIME_MONTH="04" ;; May) PROJECT_TIME_MONTH="05" ;; June) PROJECT_TIME_MONTH="06" ;; July) PROJECT_TIME_MONTH="07" ;; August) PROJECT_TIME_MONTH="08" ;; September) PROJECT_TIME_MONTH="09" ;; October) PROJECT_TIME_MONTH="10" ;; November) PROJECT_TIME_MONTH="11" ;; December) PROJECT_TIME_MONTH="12" ;; *) PROJECT_TIME_MONTH="--" ;; esac echo "SCRAPE_PLANAPPS: Found date : $PROJECT_TIME_YEAR/$PROJECT_TIME_MONTH/$PROJECT_TIME_DAY" fi fi fi fi if [[ "$NEXT_LINE_IMAGE" == "TRUE" ]]; then NEXT_LINE_IMAGE="FALSE" PROJECT_IMAGE_URL=$(echo $PLINE | sed 's/.*]*src="$[^"]*$".*/\1/p' | sed 's/?.*//' | uniq) PROJECT_IMAGE_URL_SHORT=$(echo $PLINE | grep "https://london.ca") if [[ $PROJECT_IMAGE_URL_SHORT == "" ]];then PROJECT_IMAGE_URL=$(echo "https://london.ca"$PROJECT_IMAGE_URL) fi PROJECT_IMAGE_NAME=$(echo $PROJECT_IMAGE_URL | sed 's#.*/##p' | uniq) echo $PROJECT_IMAGE_URL >> $PROJECT_IMAGE_URLS echo $PROJECT_IMAGE_NAME >> $PROJECT_IMAGE_NAMES fi PROJECT_FOUND_FILE=$(echo $PLINE | grep "file--mime-application-") if [[ $PROJECT_FOUND_FILE != "" ]]; then PROJECT_ATTACH_URL=$(echo $PLINE | sed 's/.*href="$[^"]*$".*/\1/p' | uniq) # Newer links are relative paths, so we must add back the domain PROJECT_ATTACH_URL_SHORT=$(echo $PLINE | grep "https://london.ca") if [[ $PROJECT_ATTACH_URL_SHORT == "" ]];then PROJECT_ATTACH_URL=$(echo "https://london.ca"$PROJECT_ATTACH_URL) fi PROJECT_ATTACH_NAME=$(echo $PLINE | sed 's/.*title="$[^"]*$".*/\1/p' | sed 's/&/\&/g' | sed 's/'/'\''/g' | uniq) echo $PROJECT_ATTACH_URL >> $PROJECT_ATTACH_URLS echo $PROJECT_ATTACH_NAME >> $PROJECT_ATTACH_NAMES fi PROJECT_FOUND_FLABEL=$(echo $PLINE | grep "field__label") if [[ "$PROJECT_FOUND_FLABEL" != "" ]]; then PROJECT_INFO_LABEL=$(echo $PLINE | sed 's/.*

$]*>$\?$[^<]*$.*/\2/p' | uniq) NEXT_LINE_FITEM="TRUE" # Info boxes will always have a label on one line, then the contents in the next. (except for contact info lol) # We're setting a flag to let the script know if an upcoming line is contents. fi PROJECT_FOUND_IMAGE=$(echo $PLINE | grep "field__label visually-hidden" | grep "Image") if [[ "$PROJECT_FOUND_IMAGE" != "" ]]; then NEXT_LINE_IMAGE="TRUE" # Same idea as before but for the image shown on the main page. fi PROJECT_FOUND_EMAIL=$(echo $PLINE | grep "field--name-field-email" | sed 's/.*href="$[^"]*$".*/\1/p' | sed 's|^mailto:||' | uniq) if [[ "$PROJECT_FOUND_EMAIL" != "" ]]; then printf "%-17s: %s\n" "Email" "$PROJECT_FOUND_EMAIL" >> $PROJECT_INFO fi PROJECT_FOUND_PLANNER=$(echo $PLINE | grep "field--name-name" | sed 's/.*]*>$[^<]*$<[\/:-]div>.*/\1/p' | uniq) if [[ "$PROJECT_FOUND_PLANNER" != "" ]]; then printf "\n%-17s: %s\n" "Planner" "$PROJECT_FOUND_PLANNER" >> $PROJECT_INFO fi done < $PROJECT_PAGE echo "SCRAPE_PLANAPPS: Filing away all the datas..." mkdir "./LondonArchive" mkdir "./LondonArchive/Planning Applications" if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/" fi if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE == $PROJECT_FILE_TYPE_2 ]]; then if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME/" fi if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments" fi PROJECT_SAVE_PATH="./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME" else if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME/" fi if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME/Attachments" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME/Attachments" fi PROJECT_SAVE_PATH="./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME" fi if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/" fi if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME/" fi if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments" ]; then mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments" fi PROJECT_SAVE_PATH_2="./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME" fi echo "SCRAPE_PLANAPPS: Saving attachments:" while IFS= read -r LINEA1 && IFS= read -r LINEA2 <&3; do #echo " - $LINEA1" echo "SCRAPE_PLANAPPS: - $LINEA2" wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH/Attachments/$LINEA2" --timestamping -q #--show-progress if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH_2/Attachments/$LINEA2" --timestamping -q #--show-progress fi done < $PROJECT_ATTACH_URLS 3< $PROJECT_ATTACH_NAMES echo "SCRAPE_PLANAPPS: All attachments saved." if [[ "$PROJECT_IMAGE_URL" != "" ]]; then PROJECT_IMAGE_URL="" echo "SCRAPE_PLANAPPS: Saving photos:" while IFS= read -r LINEA1 && IFS= read -r LINEA2 <&3; do #echo " - $LINEA1" echo "SCRAPE_PLANAPPS: - $LINEA2" wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH/$LINEA2" --timestamping -q #--show-progress if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH_2/$LINEA2" --timestamping -q #--show-progress fi done < $PROJECT_IMAGE_URLS 3< $PROJECT_IMAGE_NAMES echo "SCRAPE_PLANAPPS: All photos saved." fi echo "SCRAPE_PLANAPPS: Extracted info summary:" cat $PROJECT_INFO > "$PROJECT_SAVE_PATH/Info.txt" if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then cat $PROJECT_INFO > "$PROJECT_SAVE_PATH_2/Info.txt" fi cat $PROJECT_INFO fi done < $SEARCH_PAGE else SEARCH_END="TRUE" echo "SCRAPE_PLANAPPS: No more pages!" fi else SEARCH_END="TRUE" echo "SCRAPE_PLANAPPS: No more pages!" fi ((j++)) done echo "Done job: SCRAPE_PLANAPPS: $(date)"