Fix some municipalities having slightly different attachment HTML.
This commit is contained in:
parent
438ae651c2
commit
4ec67bc5c0
@ -112,72 +112,72 @@ download_agendas() {
|
||||
if [[ $AGENDA_REVISE_PDF_URL != "" ]] || [[ $AGENDA_PDF_URL != "" ]]; then
|
||||
if [[ $AGENDA_REVISE_PDF_URL != "" ]]; then
|
||||
echo "Saving revised agenda as PDF..."
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_REVISE_PDF_URL" -O "$1/Agenda_Revised.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_REVISE_PDF_URL" -O "$1/Agenda_Revised.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $AGENDA_PDF_URL != "" ]]; then
|
||||
echo "Saving regular agenda as PDF..."
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_PDF_URL" -O "$1/Agenda.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_PDF_URL" -O "$1/Agenda.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
elif [[ $AGENDA_REVISE_HTML_URL != "" ]] || [[ $AGENDA_HTML_URL != "" ]]; then
|
||||
if [[ $AGENDA_REVISE_HTML_URL != "" ]]; then
|
||||
echo "Saving revised agenda as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_REVISE_HTML_URL" -O "$1/Agenda_Revised.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_REVISE_HTML_URL" -O "$1/Agenda_Revised.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $AGENDA_HTML_URL != "" ]]; then
|
||||
echo "Saving regular agenda as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_HTML_URL" -O "$1/Agenda.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_HTML_URL" -O "$1/Agenda.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
elif [[ $AGENDA_FULL_PDF_URL != "" ]] || [[ $AGENDA_FULL_HTML_URL != "" ]]; then
|
||||
if [[ $AGENDA_FULL_PDF_URL != "" ]]; then
|
||||
echo "Saving full package agenda as PDF... (no HTML found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_FULL_PDF_URL" -O "$1/Agenda_FullPackage.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_FULL_PDF_URL" -O "$1/Agenda_FullPackage.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $AGENDA_FULL_HTML_URL != "" ]]; then
|
||||
echo "Saving full package agenda as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_FULL_HTML_URL" -O "$1/Agenda_FullPackage.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_FULL_HTML_URL" -O "$1/Agenda_FullPackage.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
elif [[ $AGENDA_POST_PDF_URL != "" ]] || [[ $AGENDA_POST_HTML_URL != "" ]]; then
|
||||
if [[ $AGENDA_POST_PDF_URL != "" ]]; then
|
||||
echo "Saving post agenda as HTML... (no HTML found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_POST_PDF_URL" -O "$1/Agenda_Post.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_POST_PDF_URL" -O "$1/Agenda_Post.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $AGENDA_POST_HTML_URL != "" ]]; then
|
||||
echo "Saving post agenda as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_POST_HTML_URL" -O "$1/Agenda_Post.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_POST_HTML_URL" -O "$1/Agenda_Post.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $MINUTES_ATTACH_PDF_URL != "" ]] || [[ $MINUTES_PDF_URL != "" ]]; then
|
||||
if [[ $MINUTES_ATTACH_PDF_URL != "" ]]; then
|
||||
echo "Saving minutes with attachments as PDF..."
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$MINUTES_ATTACH_PDF_URL" -O "$1/Minutes_With_Attachments.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$MINUTES_ATTACH_PDF_URL" -O "$1/Minutes_With_Attachments.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $MINUTES_PDF_URL != "" ]]; then
|
||||
echo "Saving minutes as PDF..."
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$MINUTES_PDF_URL" -O "$1/Minutes.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$MINUTES_PDF_URL" -O "$1/Minutes.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
else
|
||||
if [[ $MINUTES_HTML_URL != "" ]]; then
|
||||
echo "Saving minutes as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$MINUTES_HTML_URL" -O "$1/Minutes.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$MINUTES_HTML_URL" -O "$1/Minutes.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $AGENDA_COVER_PDF_URL != "" ]]; then
|
||||
echo "Saving cover agenda as PDF... (no HTML found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_COVER_PDF_URL" -O "$1/Agenda_Cover.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_COVER_PDF_URL" -O "$1/Agenda_Cover.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $AGENDA_COVER_HTML_URL != "" ]]; then
|
||||
echo "Saving cover agenda as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_COVER_HTML_URL" -O "$1/Agenda_Cover.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_COVER_HTML_URL" -O "$1/Agenda_Cover.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $ADDENDUM_PDF_URL != "" ]]; then
|
||||
echo "Saving addendum as PDF... (no HTML found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$ADDENDUM_PDF_URL" -O "$1/Addendum.pdf" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$ADDENDUM_PDF_URL" -O "$1/Addendum.pdf" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
if [[ $ADDENDUM_HTML_URL != "" ]]; then
|
||||
echo "Saving addendum as HTML... (no PDF found!)"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$ADDENDUM_HTML_URL" -O "$1/Addendum.html" -N -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$ADDENDUM_HTML_URL" -O "$1/Addendum.html" --no-hsts -N -q #--show-progress
|
||||
fi
|
||||
}
|
||||
|
||||
@ -217,7 +217,7 @@ while read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
|
||||
INDEX_END="FALSE"
|
||||
while [[ $INDEX_END == "FALSE" ]]; do
|
||||
echo "SCRAPE_ESCRIBE: Downloading eScribe index..."
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" $INDEX_URL -O $INDEX_PAGE --show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" $INDEX_URL -O $INDEX_PAGE --no-hsts --show-progress
|
||||
if [ $? -ne 8 ]; then
|
||||
FOUNDLIST="FALSE"
|
||||
while IFS= read -r LINE; do
|
||||
@ -329,21 +329,21 @@ while read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
|
||||
ADDENDUM_ERROR="FALSE"
|
||||
echo "Downloading agenda HTML..."
|
||||
if [[ $AGENDA_REVISE_HTML_URL != "" ]]; then
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_REVISE_HTML_URL" -O $AGENDA_HTML -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_REVISE_HTML_URL" -O $AGENDA_HTML --no-hsts -q #--show-progress
|
||||
elif [[ $AGENDA_HTML_URL != "" ]]; then
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_HTML_URL" -O $AGENDA_HTML -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_HTML_URL" -O $AGENDA_HTML --no-hsts -q #--show-progress
|
||||
elif [[ $AGENDA_FULL_HTML_URL != "" ]]; then
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_FULL_HTML_URL" -O $AGENDA_HTML -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_FULL_HTML_URL" -O $AGENDA_HTML --no-hsts -q #--show-progress
|
||||
elif [[ $AGENDA_POST_HTML_URL != "" ]]; then
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_POST_HTML_URL" -O $AGENDA_HTML -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_POST_HTML_URL" -O $AGENDA_HTML --no-hsts -q #--show-progress
|
||||
elif [[ $AGENDA_COVER_HTML_URL != "" ]]; then
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_COVER_HTML_URL" -O $AGENDA_HTML -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$AGENDA_COVER_HTML_URL" -O $AGENDA_HTML --no-hsts -q #--show-progress
|
||||
else
|
||||
ERROR="TRUE"
|
||||
fi
|
||||
|
||||
if [[ $ADDENDUM_HTML_URL != "" ]]; then
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$ADDENDUM_HTML_URL" -O $ADDENDUM_HTML -q #--show-progress
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$ADDENDUM_HTML_URL" -O $ADDENDUM_HTML --no-hsts -q #--show-progress
|
||||
else
|
||||
ADDENDUM_ERROR="TRUE"
|
||||
fi
|
||||
@ -373,20 +373,21 @@ while read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
|
||||
fi
|
||||
|
||||
# Get attachment links
|
||||
cat $AGENDA_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/href=.filestream\.ashx/\nhref="filestream\.ashx/g' | grep 'filestream.ashx' | sed 's/. data-toggle/\" data-toggle/p' | sed 's/href=.\([^/]*\)".*/\1/p' | awk '!x[$0]++' > "./tmp/attachment_urls"
|
||||
cat $AGENDA_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/^.*AgendaHeaderTitle/AgendaHeaderTitle/' | sed 's/href=.[Ff]ile[Ss]tream\.ashx/\nhref="filestream\.ashx/g' | grep -i 'filestream.ashx' | sed 's/. data-toggle/\" data-toggle/p' | sed 's/href=.\([^"]*\)".*/\1/p' | awk '!x[$0]++' > "./tmp/attachment_urls"
|
||||
# Get attachment names
|
||||
cat $AGENDA_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/data-original-title=./\ndata-original-title='\''/g' | grep 'data-original-title' | sed 's/data-original-title=.//p' | sed 's/.pdf['\'':"].*/.pdf/g' | awk '!x[$0]++' > "./tmp/attachment_names"
|
||||
cat $AGENDA_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed -E "s/data-original-title=['\"]/\\ndata-original-title='/g" | grep 'data-original-title' | sed 's/data-original-title=.//p' | sed 's/.pdf['\'':"].*/.pdf/g' | awk '!x[$0]++' > "./tmp/attachment_names"
|
||||
if [[ "$ADDENDUM_ERROR" == "FALSE" ]]; then
|
||||
# Get attachment links
|
||||
cat $ADDENDUM_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/href=.filestream\.ashx/\nhref="filestream\.ashx/g' | grep 'filestream.ashx' | sed 's/. data-toggle/\" data-toggle/p' | sed 's/href=.\([^/]*\)".*/\1/p' | awk '!x[$0]++' > "./tmp/attachment_urls"
|
||||
cat $ADDENDUM_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/^.*AgendaHeaderTitle/AgendaHeaderTitle/' | sed 's/href=.[Ff]ile[Ss]tream\.ashx/\nhref="filestream\.ashx/g' | grep -i 'filestream.ashx' | sed 's/. data-toggle/\" data-toggle/p' | sed 's/href=.\([^"]*\)".*/\1/p' | awk '!x[$0]++' > "./tmp/attachment_urls"
|
||||
# Get attachment names
|
||||
cat $ADDENDUM_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/data-original-title=./\ndata-original-title='\''/g' | grep 'data-original-title' | sed 's/data-original-title=.//p' | sed 's/.pdf['\'':"].*/.pdf/g' | awk '!x[$0]++' > "./tmp/attachment_names"
|
||||
cat $ADDENDUM_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed -E "s/data-original-title=['\"]/\\ndata-original-title='/g" | grep 'data-original-title' | sed 's/data-original-title=.//p' | sed 's/.pdf['\'':"].*/.pdf/g' | awk '!x[$0]++' > "./tmp/attachment_names"
|
||||
fi
|
||||
# Download attachment and use the name grabbed above
|
||||
echo "Found the following agenda attachments:"
|
||||
while IFS= read -r LINEA1 && IFS= read -r LINEA2 <&3; do
|
||||
echo "- $LINEA2"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "https://pub-london.escribemeetings.com/$LINEA1" -O "$MEETING_DIR/Attachments/$LINEA2" -N -q #--show-progress
|
||||
echo "- $LINEA2 / $LINEA1"
|
||||
wget --no-check-certificate --user-agent="$WGET_UA" "$INDEX_URL$LINEA1" -O "$MEETING_DIR/Attachments/$LINEA2" --no-hsts -N -q #--show-progress
|
||||
# [ ! -s "$MEETING_DIR/Attachments/$LINEA2" ] && rm -f "$MEETING_DIR/Attachments/$LINEA2"
|
||||
done < ./tmp/attachment_urls 3< ./tmp/attachment_names
|
||||
echo "All attachments saved."
|
||||
|
||||
@ -399,6 +400,9 @@ while read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
|
||||
fi
|
||||
|
||||
echo "All files from this meeting have been saved."
|
||||
find "$MEETING_DIR" -type f -size 0 -delete
|
||||
echo "Cleaning PDFs for archive.org..."
|
||||
find "$MEETING_DIR" -type f -name '*.pdf' -print0 | xargs -0 -n1 qpdf --replace-input
|
||||
fi
|
||||
|
||||
((i++))
|
||||
@ -419,4 +423,3 @@ while read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
|
||||
fi
|
||||
done
|
||||
done < websites.csv
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user