330 lines
17 KiB
Bash
Executable File
330 lines
17 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
echo -e "\n-========================================================================-"
|
||
echo -e "-=- -=-"
|
||
echo -e "-=- SCRAPE_MPaS.SH: Scrape London Master Plans and Strategies -=-"
|
||
echo -e "-=- -=-"
|
||
echo -e "-=- Lillian Skinner -=-"
|
||
echo -e "-=- -=-"
|
||
echo -e "-========================================================================-"
|
||
|
||
source ./functions/.functions
|
||
|
||
# Todo:
|
||
# - Save updates (see bradley-ave)
|
||
# - Order, title, and collapse each scraped modal
|
||
|
||
# London seems to have recently blocked unusual user agents. Can't use wget or even ping. Thankfully pretend to be a real person!
|
||
WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
|
||
|
||
TEMP_DIR="./tmp/"
|
||
SEARCH_PAGE="./tmp/index_ginv.html"
|
||
PROJECT_PAGE="./tmp/project_ginv.html"
|
||
WORK_HTML="./tmp/tmp.html"
|
||
CUSTOM_HTML="./tmp/custom_ginv.html"
|
||
CUSTOM_HTML_LINKS="./tmp/custom_link_ginv.html"
|
||
CUSTOM_HTML_PHOTOS="./tmp/custom_photo_ginv.html"
|
||
CUSTOM_HTML_FAQ="./tmp/custom_faq_ginv.html"
|
||
CUSTOM_HTML_PROFILE="./tmp/custom_profile_ginv.html"
|
||
CUSTOM_HTML_TIMELINE="./tmp/custom_timeline_ginv.html"
|
||
CUSTOM_HTML_KEYDATES="./tmp/custom_keydates_ginv.html"
|
||
CUSTOM_HTML_SLIDER="./tmp/custom_slider_ginv.html"
|
||
FULLDUMP="./tmp/.fulldump.txt"
|
||
|
||
current_year=$(date +%Y)
|
||
current_month=$(date +%m)
|
||
current_day=$(date +%d)
|
||
|
||
rm -f $SEARCH_PAGE
|
||
|
||
mkdir $TEMP_DIR
|
||
|
||
SEARCH_URL="https://getinvolved.london.ca/projects"
|
||
|
||
set_metadata() {
|
||
tmp=$(echo "$1" | sed 's/&/\&/g' | sed 's/"//g' | sed 's/'/'\''/g' | sed 's/\[/''/g' | sed 's/\]/''/g')
|
||
PROJECT_NAME=$(_utils_fix_dashes "$(echo $tmp | sed 's/.*data-project-name="\([^"]*\)".*/\1/' | sed 's/‘//g' | sed 's/’//g' | sed 's/'\''//g' | sed 's/://g')")
|
||
PROJECT_CATS=$(echo "$tmp" | sed 's/.*data-project-category="\([^"]*\)".*/\1/')
|
||
PROJECT_LOCATION=$(echo "$tmp" | sed 's/.*data-project-location="\([^"]*\)".*/\1/')
|
||
}
|
||
|
||
wget --user-agent="$WGET_UA" "$SEARCH_URL" -O $SEARCH_PAGE --timestamping -q #--show-progress
|
||
if [ $? -ne 8 ]; then
|
||
while IFS= read -r LINE; do
|
||
|
||
if (( FOUND_DATE )) && [[ "$LAST_LINE" == "" ]] && (( 10#$ITEM_YEAR >= 10#$current_year )) && (( 10#$ITEM_MONTH >= $((10#$current_month - 1)) )); then
|
||
FOUND_DATE=0
|
||
echo $PROJECT_URL
|
||
echo $PROJECT_NAME
|
||
wget --user-agent="$WGET_UA" "$PROJECT_URL" -O $PROJECT_PAGE --timestamping -q #--show-progress
|
||
# Now we can work on the actual project page.
|
||
rm -f $CUSTOM_HTML_LINKS
|
||
rm -f $CUSTOM_HTML_PHOTOS
|
||
rm -f $CUSTOM_HTML_FAQ
|
||
rm -f $CUSTOM_HTML_PROFILE
|
||
rm -f $CUSTOM_HTML_TIMELINE
|
||
rm -f $CUSTOM_HTML_KEYDATES
|
||
rm -f $CUSTOM_HTML_SLIDER
|
||
rm -f $FULLDUMP
|
||
|
||
cat ./template/default_getinvolved.html > $CUSTOM_HTML
|
||
echo "<h1>$PROJECT_NAME</h1>" >> $CUSTOM_HTML
|
||
while IFS= read -r LINE_PROJ; do
|
||
if (( NEXT_LINE_CONTENT )); then
|
||
# Next hive-block marks end of current item
|
||
if [[ "$LINE_PROJ" == *"hive-block"* ]] || [[ "$LINE_PROJ" == "" ]]; then
|
||
NEXT_LINE_CONTENT=0
|
||
echo "End of current content."
|
||
else
|
||
# Ignore boring notices
|
||
if [[ "$LINE_PROJ" != *"</h1>"* ]] &&
|
||
[[ "$LINE_PROJ" != *"City of London Land Acknowledgement"* ]] &&
|
||
[[ "$LINE_PROJ" != *"Ongoing Site Specific Planning Applications"* ]] &&
|
||
[[ "$LINE_PROJ" != *"This site is owned and operated by the City of London using software licensed from Social Pinpoint"* ]] &&
|
||
[[ "$LINE_PROJ" != *"Social Pinpoint has been commissioned by City of London (Canada) to collect and display user content on their behalf"* ]] &&
|
||
[[ "$LINE_PROJ" != *"Notice of Collection of Personal Information"* ]] &&
|
||
[[ "$LINE_PROJ" != *'href="/register"'* ]] &&
|
||
[[ "$LINE_PROJ" != *'href="/login"'* ]] &&
|
||
[[ "$LINE_PROJ" != *"Users have the right to access, correct, or delete their personal information"* ]] &&
|
||
[[ "$LINE_PROJ" != *"This privacy policy may change from time to time"* ]] &&
|
||
#[[ "$LINE_PROJ" != *"Share your feedback"* ]] &&
|
||
[[ "$LINE_PROJ" != *"Notice of Collection"* ]] &&
|
||
#[[ "$LINE_PROJ" != *"Subscribe for project updates"* ]] &&
|
||
[[ "$LINE_PROJ" != *"Ready to have your say?"* ]]; then
|
||
# seds to replace youtube iframe with a normal <a href=""> link. wkhtmltopdf obviously can't embed youtube videos.
|
||
if (( FIRST_CONTENT )); then
|
||
echo "<!-- LondonArchive_GINV_Body -->" >> $FULLDUMP
|
||
FIRST_CONTENT=0
|
||
fi
|
||
echo $(echo " $LINE_PROJ" | sed 's/src="https:\/\/www\.youtube\.com\/embed/href="https:\/\/www\.youtube\.com\/watch/' | sed 's/<iframe/<a/' | sed 's/<\/iframe>/YouTube Link<\/a><\/br>/') >> $CUSTOM_HTML
|
||
echo $(echo " $LINE_PROJ" | sed 's/src="https:\/\/www\.youtube\.com\/embed/href="https:\/\/www\.youtube\.com\/watch/' | sed 's/<iframe/<a/' | sed 's/<\/iframe>/YouTube Link<\/a><\/br>/') >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_DOC_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"modal-footer"* ]]; then
|
||
IS_DOC_BLOCK=0
|
||
FOUND_LINK=$(cat "$CUSTOM_HTML_LINKS" | grep "/download_file/")
|
||
if [[ "$FOUND_LINK" == "" ]]; then
|
||
rm -f $CUSTOM_HTML_LINKS
|
||
fi
|
||
echo "End of current documents."
|
||
else
|
||
if [[ "$LINE_PROJ" != *"btn btn-close btn-inverse close"* ]] && [[ "$LINE_PROJ" != *"</h1>"* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML_LINKS
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_PHOTO_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"<!-- end foreach -->"* ]]; then
|
||
IS_PHOTO_BLOCK=0
|
||
FOUND_LINK=$(cat "$CUSTOM_HTML_PHOTOS" | grep "amazonaws")
|
||
if [[ "$FOUND_LINK" == "" ]]; then
|
||
rm -f $CUSTOM_HTML_PHOTOS
|
||
fi
|
||
echo "End of current photos."
|
||
else
|
||
if [[ "$LINE_PROJ" == *'aria-label="'* ]] && [[ "$LINE_PROJ" != *"</h1>"* ]]; then
|
||
echo $(echo $LINE_PROJ | sed 's/.*href="\([^"]*\)".*/\1/') >> $CUSTOM_HTML_PHOTOS
|
||
echo $(echo $LINE_PROJ | sed 's/.*href="\([^"]*\)".*/\1/') >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_FAQ_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"modal-footer"* ]]; then
|
||
IS_FAQ_BLOCK=0
|
||
echo "End of current FAQ."
|
||
else
|
||
if [[ "$LINE_PROJ" != *"btn btn-close btn-inverse close"* ]]; then
|
||
# I don't care that this is invalid HTML. All you'll see in the end is a nicely formatted PDF.
|
||
if [[ "$LINE_PROJ" == *"hive-block-faq mod-reverse"* ]]; then
|
||
echo $(echo $LINE_PROJ | sed 's/<a role/<h3 role/g' | sed 's/<\/a>/<\/h3>/g') >> $CUSTOM_HTML
|
||
elif [[ "$LINE_PROJ" != *"</h1>"* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_PROFILE_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"<script>"* ]]; then
|
||
IS_PROFILE_BLOCK=0
|
||
echo "End of current profile."
|
||
else
|
||
if [[ "$LINE_PROJ" != *"btn btn-close btn-inverse close"* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_TIMELINE_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"btn-unfill btn-primary"* ]]; then
|
||
IS_TIMELINE_BLOCK=0
|
||
echo "End of current timeline."
|
||
else
|
||
if [[ "$LINE_PROJ" != *"btn-unfill btn-primary"* ]] && [[ "$LINE_PROJ" != *'class="sr-only"'* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_KEYDATES_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"modal-footer"* ]]; then
|
||
IS_KEYDATES_BLOCK=0
|
||
echo "End of current key dates."
|
||
else
|
||
if [[ "$LINE_PROJ" != *"btn btn-default"* ]] && [[ "$LINE_PROJ" != *"btn-close btn-inverse close"* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_SLIDER_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == *"<!-- Controls -->"* ]]; then
|
||
IS_SLIDER_BLOCK=0
|
||
echo "End of current key dates."
|
||
else
|
||
if [[ "$LINE_PROJ" != *"btn btn-default"* ]] && [[ "$LINE_PROJ" != *"</h3"* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML_SLIDER
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if (( IS_SINGLE_IMAGE_BLOCK )); then
|
||
if [[ "$LINE_PROJ" == "" ]]; then
|
||
IS_SINGLE_IMAGE_BLOCK=0
|
||
FOUND_LINK=$(cat "$CUSTOM_HTML_PHOTOS" | grep "amazonaws")
|
||
if [[ "$FOUND_LINK" == "" ]]; then
|
||
rm -f $CUSTOM_HTML_PHOTOS
|
||
else
|
||
cat "$CUSTOM_HTML_PHOTOS"
|
||
fi
|
||
echo "End of current single image."
|
||
else
|
||
if [[ "$LINE_PROJ" == *'class="hive-image"'* ]]; then
|
||
echo $LINE_PROJ >> $CUSTOM_HTML
|
||
echo $LINE_PROJ >> $FULLDUMP
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if [[ "$LINE_PROJ" == *"hive-block hive-block-content ljs"* ]]; then
|
||
NEXT_LINE_CONTENT=1
|
||
FIRST_CONTENT=1
|
||
# We'll write the LA comment inside of the content block.
|
||
# There we can ensure that the comment is only written if content does exist.
|
||
echo "Found content start."
|
||
elif [[ "$LINE_PROJ" == *"docLibModal hive-block-document-library"* ]]; then
|
||
IS_DOC_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_Documents -->" >> $FULLDUMP
|
||
echo "Found documents start."
|
||
elif [[ "$LINE_PROJ" == *"hive-block-media hive-block"* ]]; then
|
||
IS_PHOTO_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_Photos -->" >> $FULLDUMP
|
||
echo "Found photos start."
|
||
elif [[ "$LINE_PROJ" == *"hive-modal faqModal hive-block-faq"* ]]; then
|
||
IS_FAQ_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_FAQ -->" >> $FULLDUMP
|
||
echo "Found FAQ start."
|
||
elif [[ "$LINE_PROJ" == *"hive-block-bio hive-block"* ]]; then
|
||
IS_PROFILE_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_Bio -->" >> $FULLDUMP
|
||
echo "Found profile start."
|
||
elif [[ "$LINE_PROJ" == *"hive-block-timeline hive-block"* ]]; then
|
||
IS_TIMELINE_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_Timeline -->" >> $FULLDUMP
|
||
echo "Found timeline start."
|
||
elif [[ "$LINE_PROJ" == *"hive-modal dateModal"* ]]; then
|
||
IS_KEYDATES_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_Date -->" >> $FULLDUMP
|
||
echo "Found key dates start."
|
||
elif [[ "$LINE_PROJ" == *"<!-- Wrapper for slider -->"* ]]; then
|
||
IS_SLIDER_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_Slider -->" >> $FULLDUMP
|
||
echo "Found slider start."
|
||
elif [[ "$LINE_PROJ" == *"hive-block hive-block-image"* ]]; then
|
||
IS_SINGLE_IMAGE_BLOCK=1
|
||
echo "<!-- LondonArchive_GINV_SingleImage -->" >> $FULLDUMP
|
||
echo "Found single image start."
|
||
fi
|
||
done < $PROJECT_PAGE
|
||
|
||
#cat "$CUSTOM_HTML_FAQ" >> "$CUSTOM_HTML"
|
||
#cat "$CUSTOM_HTML_LINKS" >> "$CUSTOM_HTML"
|
||
|
||
mkdir -p "./LondonArchive/GetInvolved/$PROJECT_NAME/"
|
||
|
||
if [ -e "$CUSTOM_HTML_LINKS" ] && [ -s "$CUSTOM_HTML_LINKS" ]; then
|
||
while IFS= read -r LINE_DOC; do
|
||
if [[ "$LINE_DOC" == *"download_file"* ]]; then
|
||
mkdir -p "./LondonArchive/GetInvolved/$PROJECT_NAME/Attachments"
|
||
DOC_NAME=$(curl -s -L -I "$(echo $LINE_DOC | sed 's/.*href="\([^"]*\)".*/\1/')" | grep "location:" | sed 's/location: //' | sed 's/.*\///' | sed 's/^[0-9a-f]\{32\}_//' | sed 's/\.pdf./\.pdf/')
|
||
echo $DOC_NAME
|
||
_utils_download_helper "$(echo $LINE_DOC | sed 's/.*href="\([^"]*\)".*/\1/')" "./LondonArchive/GetInvolved/$PROJECT_NAME/Attachments/$DOC_NAME"
|
||
fi
|
||
done < $CUSTOM_HTML_LINKS
|
||
fi
|
||
|
||
if [ -e "$CUSTOM_HTML_PHOTOS" ] && [ -s "$CUSTOM_HTML_PHOTOS" ]; then
|
||
while IFS= read -r LINE_DOC; do
|
||
if [[ "$LINE_DOC" == *"amazonaws"* ]]; then
|
||
mkdir -p "./LondonArchive/GetInvolved/$PROJECT_NAME/Attachments"
|
||
DOC_NAME=$(echo $LINE_DOC | sed 's/.*\///' | sed 's/^[0-9a-f]\{32\}_//')
|
||
echo $DOC_NAME
|
||
_utils_download_helper "$LINE_DOC" "./LondonArchive/GetInvolved/$PROJECT_NAME/Attachments/$DOC_NAME"
|
||
fi
|
||
done < $CUSTOM_HTML_PHOTOS
|
||
fi
|
||
|
||
if [ -e "$CUSTOM_HTML_SLIDER" ] && [ -s "$CUSTOM_HTML_SLIDER" ]; then
|
||
mkdir -p "./LondonArchive/GetInvolved/$PROJECT_NAME/Attachments"
|
||
cat ./template/default_getinvolved.html > $WORK_HTML
|
||
echo "<h1>$PROJECT_NAME Photo Gallery</h1>" >> $WORK_HTML
|
||
cat "$CUSTOM_HTML_SLIDER" >> $WORK_HTML
|
||
echo "<br><br><small><i>Automatically generated for the London Archive on $(date)</i></small>" >> $WORK_HTML
|
||
wkhtmltopdf --image-quality 100 "$WORK_HTML" "./LondonArchive/GetInvolved/$PROJECT_NAME/Attachments/${PROJECT_NAME}_GALLERY.pdf"
|
||
fi
|
||
echo "<br><br><small><i>Automatically generated for the London Archive on $(date)</i></small>" >> $CUSTOM_HTML
|
||
wkhtmltopdf --image-quality 100 "$CUSTOM_HTML" "./LondonArchive/GetInvolved/$PROJECT_NAME/Main.pdf"
|
||
cp "$FULLDUMP" "./LondonArchive/GetInvolved/$PROJECT_NAME/.backup.txt"
|
||
fi
|
||
|
||
if (( NEXT_LINE_URL )); then
|
||
NEXT_LINE_URL=0
|
||
PROJECT_URL=$(echo $LINE | sed 's/.*href="\([^"]*\)".*/\1/')
|
||
fi
|
||
|
||
if [[ "$LAST_LINE" != "" ]]; then
|
||
set_metadata "$LAST_LINE$LINE"
|
||
LAST_LINE=""
|
||
NEXT_LINE_URL=1
|
||
elif [[ "$LINE" == *"h-entry project card"* ]] && [[ "$LINE" == *"data-project-name"* ]] && [[ "$LINE" != *"<%-"* ]]; then
|
||
#echo $LINE
|
||
if [[ "$LINE" != *"data-project-category"* ]]; then
|
||
# Sometimes lines are split, so we'll combine the pieces over time.
|
||
LAST_LINE=$LINE
|
||
echo "Line is split!"
|
||
else
|
||
LAST_LINE=""
|
||
set_metadata "$LINE"
|
||
NEXT_LINE_URL=1
|
||
fi
|
||
elif [[ "$LINE" == *'time class="dt-updated"'* ]]; then
|
||
PROJECT_DATE=$(echo $LINE | sed 's/.*<time[^>]*>\([^<]*\)<[\/:-]time>.*/\1/g')
|
||
echo $PROJECT_DATE
|
||
_time_parse_monddyyyy "$PROJECT_DATE"
|
||
echo "$ITEM_YEAR$ITEM_MONTH$ITEM_DAY"
|
||
FOUND_DATE=1
|
||
fi
|
||
|
||
done < $SEARCH_PAGE
|
||
fi
|