#!/usr/bin/env bash
echo -e "\n-========================================================================-"
echo -e "-=-                                                                    -=-"
echo -e "-=-  SCRAPE_LONDON.SH: Downloads committee videos and agendas          -=-"
echo -e "-=-                                                                    -=-"
echo -e "-=-   Lillian Skinner                                                  -=-"
echo -e "-=-                                                                    -=-"
echo -e "-========================================================================-"

source ./functions/.functions

# Warning to all who read this script:
# It is bad. I know it is bad, but I am tired okay, and sometimes sloppy just works.

# London seems to have recently blocked unusual user agents. Can't use wget or even ping. Thankfully pretend to be a real person!
WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"

TEMP_DIR="./tmp/"
INDEX_PAGE="./tmp/index.html"
SEARCH_PAGE="./tmp/search.html"
AGENDA_HTML="./tmp/work.html"
ADDENDUM_HTML="./tmp/addendum.html"
#VIDEO_TIMESTAMP_JSON="./tmp/time.json"

current_year=$(date +%Y)
current_month=$(date +%m)
current_day=$(date +%d)

SUPPORT_PAST=""

if [ -d "$TEMP_DIR" ]; then
    rm -r $TEMP_DIR
fi
rm -f $INDEX_PAGE
rm -f $SEARCH_PAGE
rm -f $AGENDA_HTML

mkdir $TEMP_DIR

while IFS="," read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
    INDEX_URL=$(echo "$INDEX_URL_PRE" | sed 's/\"//g' | sed 's/,//g' | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//')
    CITY_ARCHIVE_NAME=$(echo "$CITY_ARCHIVE_NAME_PRE" | sed 's/\"//g' | sed 's/\,//g' | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//')
    CALENDAR_NAME=$(echo "$CALENDAR_NAME_PRE" | sed 's/\"//g' | sed 's/\,//g' | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//')
    INDEX_END="FALSE"
    while [[ $INDEX_END == "FALSE" ]]; do
        echo "SCRAPE_ESCRIBE: Downloading eScribe index..."
        wget --no-check-certificate --user-agent="$WGET_UA" $INDEX_URL -O $INDEX_PAGE --no-hsts --show-progress
        if [ $? -ne 8 ]; then
            FOUNDLIST="FALSE"
            while IFS= read -r LINE; do
                if [[ "TRUE" == $FOUNDLIST ]]; then
                    GREPENDLIST=$(echo $LINE | grep '<option ')
                    if [[ "$GREPENDLIST" == "" ]]; then
                        echo "SCRAPE_ESCRIBE: End of list."
                        INDEX_END="TRUE"
                        break
                    else
                        MEETING_NAME=$(echo $LINE | sed 's/.*<option[^>]*>\([^<]*\)<[\/:-]option>.*/\1/g')
                        echo "-========================================================================-"
                        echo "- $MEETING_NAME"

                        if [[ "$MEETING_NAME" == "CANCELLED"* ]]; then
                            MEETING_NAME=$(echo $MEETING_NAME | sed 's/^[^ ]* //' | sed 's/^[^ ]* //')
                            echo "- Corrected to: $MEETING_NAME"
                        fi
                        # Pages start at 1. Ew.
                        x=1
                        curl -s -d "{'type': '$MEETING_NAME', 'pageNumber': $x}" -H "Content-Type: application/json" -X POST "$INDEX_URL"MeetingsCalendarView.aspx/PastMeetings --insecure | jq . >"${TEMP_DIR}escribe.json"
                        #cat "${TEMP_DIR}escribe.json" > debug.json

                        y=0
                        i=0
                        NUM_MEETINGS=$(cat "${TEMP_DIR}escribe.json" | jq '.d.TotalCount')
                        while (true); do
                            NUM_IN_JSON=$(cat "${TEMP_DIR}escribe.json" | jq '.d.Meetings | length')

                            if [[ "$NUM_IN_JSON" == "" ]]; then
                                break
                            fi

                            # Decrease in the meeting count == we're on the final page.
                            if (($i >= $NUM_IN_JSON)) && ((10#$NUM_IN_JSON >= 50)); then
                                ((x++))
                                i=0
                                curl -s -d "{'type': '$MEETING_NAME', 'pageNumber': $x}" -H "Content-Type: application/json" -X POST "$INDEX_URL"MeetingsCalendarView.aspx/PastMeetings --insecure | jq . >"${TEMP_DIR}escribe.json"
                            elif (($i >= 10#$NUM_IN_JSON)); then
                                break
                            fi

                            echo "$(($i + 1)) of $NUM_IN_JSON ($NUM_MEETINGS total) in page $x"

                            # Boost speed by extracting a single meeting from the large JSON, then working on the extract.
                            # No need to cat the entire file every time.
                            cat "${TEMP_DIR}escribe.json" | jq --argjson i "$i" '.d.Meetings.[$i]' >"${TEMP_DIR}escribe_short.json"

                            #echo "> Meeting ID"
                            #cat "${TEMP_DIR}escribe_short.json" | jq --argjson i "$i" '.Id'
                            #echo "> Meeting Attachments"
                            NUM_ATTACHMENTS=$(cat "${TEMP_DIR}escribe_short.json" | jq --argjson i "$i" '.MeetingLinks | length')
                            # Will go in a loop, collecting links/types like with the earlier SCRAPE_MEET script.

                            clear_agenda_url
                            for ((j = 0; j <= (($NUM_ATTACHMENTS - 1)); j++)); do
                                set_agenda_url "$(cat "${TEMP_DIR}escribe_short.json" | jq --argjson i "$i" --argjson j "$j" '.MeetingLinks.[$j].Title')" "$INDEX_URL$(cat "${TEMP_DIR}escribe_short.json" | jq --argjson i "$i" --argjson j "$j" '.MeetingLinks.[$j].Url')"
                            done

                            _time_parse_helper "$(cat ${TEMP_DIR}escribe_short.json | jq --argjson i "$i" '.DateMedium' | sed 's/\"//g')"

                            INPAST=""
                            if ((10#$ITEM_YEAR >= 10#$current_year)) && ((10#$ITEM_MONTH >= $((10#$current_month - 1)))); then
                                echo "NAME  : $MEETING_NAME"
                                echo "DATE  : $ITEM_YEAR/$ITEM_MONTH/$ITEM_DAY"
                                echo "A (H) : $AGENDA_HTML_URL"
                                echo "A (P) : $AGENDA_PDF_URL"
                                echo "AR(H) : $AGENDA_REVISE_HTML_URL"
                                echo "AR(P) : $AGENDA_REVISE_PDF_URL"
                                echo "AF(H) : $AGENDA_FULL_HTML_URL"
                                echo "AF(P) : $AGENDA_FULL_PDF_URL"
                                echo "AC(H) : $AGENDA_COVER_HTML_URL"
                                echo "AC(P) : $AGENDA_COVER_PDF_URL"
                                echo "AP(H) : $AGENDA_POST_HTML_URL"
                                echo "AP(P) : $AGENDA_POST_PDF_URL"
                                echo "M (H) : $MINUTES_HTML_URL"
                                echo "M (P) : $MINUTES_PDF_URL"
                                echo "MA(P) : $MINUTES_ATTACH_PDF_URL"
                                echo "AD(H) : $ADDENDUM_HTML_URL"
                                echo "AD(P) : $ADDENDUM_PDF_URL"
                            else
                                echo "Dates are in the past!"
                                echo "DATE  : $ITEM_YEAR/$ITEM_MONTH/$ITEM_DAY"
                                INPAST="TRUE"
                            fi

                            # I think "break" broke when I did nested loops. idk I'm too drunk for this.
                            if [[ "$INPAST" == "TRUE" ]] && [[ "$SUPPORT_PAST" != "TRUE" ]]; then
                                echo "Abort."
                                break
                            fi

                            #echo "> Meeting Video"
                            #cat "${TEMP_DIR}escribe_short.json" | jq --argjson i "$i" '.VideoLink.[].HasVideo'
                            VIDEOURL="$INDEX_URL$(cat "${TEMP_DIR}escribe_short.json" | jq --argjson i "$i" '.VideoLink.[].Url' | sed 's/\"//g')"

                            ERROR="FALSE"
                            ADDENDUM_ERROR="FALSE"
                            echo "Downloading agenda HTML..."

                            if [[ -n $AGENDA_REVISE_HTML_URL ]]; then
                                _utils_download_helper "$AGENDA_REVISE_HTML_URL" "$AGENDA_HTML"
                            elif [[ -n $AGENDA_HTML_URL ]]; then
                                _utils_download_helper "$AGENDA_HTML_URL" "$AGENDA_HTML"

                            elif [[ -n $AGENDA_FULL_HTML_URL ]]; then
                                _utils_download_helper "$AGENDA_FULL_HTML_URL" "$AGENDA_HTML"

                            elif [[ -n $AGENDA_POST_HTML_URL ]]; then
                                _utils_download_helper "$AGENDA_POST_HTML_URL" "$AGENDA_HTML"

                            elif [[ -n $AGENDA_COVER_HTML_URL ]]; then
                                _utils_download_helper "$AGENDA_COVER_HTML_URL" "$AGENDA_HTML"
                            else
                                ERROR="TRUE"
                            fi

                            if [[ -n $ADDENDUM_HTML_URL ]]; then
                                _utils_download_helper "$ADDENDUM_HTML_URL" "$ADDENDUM_HTML"
                            else
                                ADDENDUM_ERROR="TRUE"
                            fi

                            if [[ "$ERROR" == "FALSE" ]]; then

                                mkdir "./$CITY_ARCHIVE_NAME"
                                mkdir "./$CITY_ARCHIVE_NAME/Meetings"

                                if [ ! -d "./$CITY_ARCHIVE_NAME/Meetings/$MEETING_NAME" ]; then
                                    mkdir "./$CITY_ARCHIVE_NAME/Meetings/$MEETING_NAME/"
                                fi
                                if [ ! -d "./$CITY_ARCHIVE_NAME/Meetings/$MEETING_NAME/$ITEM_YEAR" ]; then
                                    mkdir "./$CITY_ARCHIVE_NAME/Meetings/$MEETING_NAME/$ITEM_YEAR/"
                                fi
                                MEETING_DIR=$(printf "./$CITY_ARCHIVE_NAME/Meetings/%s/%s/%s-%s" "$MEETING_NAME" "$ITEM_YEAR" "$ITEM_MONTH" "$ITEM_DAY")
                                if [ ! -d "$MEETING_DIR" ]; then
                                    mkdir "$MEETING_DIR/"
                                fi
                                if [ ! -d "$MEETING_DIR/Attachments" ]; then
                                    mkdir "$MEETING_DIR/Attachments/"
                                fi

                                if [[ $VIDEO_URL != "" ]]; then
                                    echo "Saving recording URL..."
                                    echo "https://video.isilive.ca/london/"$VIDEO_URL >"$MEETING_DIR/RecordingLink.txt"
                                fi

                                # Get attachment links
                                cat $AGENDA_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/^.*AgendaHeaderTitle/AgendaHeaderTitle/' | sed 's/href=.[Ff]ile[Ss]tream\.ashx/\nhref="filestream\.ashx/g' | grep -i 'filestream.ashx' | sed 's/. data-toggle/\" data-toggle/p' | sed 's/href=.\([^"]*\)".*/\1/p' | awk '!x[$0]++' >"./tmp/attachment_urls"
                                # Get attachment names
                                cat $AGENDA_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed -E "s/data-original-title=['\"]/\\ndata-original-title='/g" | grep 'data-original-title' | sed 's/data-original-title=.//p' | sed 's/.pdf['\'':"].*/.pdf/g' | awk '!x[$0]++' >"./tmp/attachment_names"
                                if [[ "$ADDENDUM_ERROR" == "FALSE" ]]; then
                                    # Get attachment links
                                    cat $ADDENDUM_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed 's/^.*AgendaHeaderTitle/AgendaHeaderTitle/' | sed 's/href=.[Ff]ile[Ss]tream\.ashx/\nhref="filestream\.ashx/g' | grep -i 'filestream.ashx' | sed 's/. data-toggle/\" data-toggle/p' | sed 's/href=.\([^"]*\)".*/\1/p' | awk '!x[$0]++' >"./tmp/attachment_urls"
                                    # Get attachment names
                                    cat $ADDENDUM_HTML | grep "AgendaItemAttachment AgendaItemAttachmentNotSelected" | sed -E "s/data-original-title=['\"]/\\ndata-original-title='/g" | grep 'data-original-title' | sed 's/data-original-title=.//p' | sed 's/.pdf['\'':"].*/.pdf/g' | awk '!x[$0]++' >"./tmp/attachment_names"
                                fi
                                # Download attachment and use the name grabbed above
                                echo "Found the following agenda attachments:"
                                while IFS= read -r LINEA1 && IFS= read -r LINEA2 <&3; do
                                    echo "- $LINEA2 / $LINEA1"
                                    _utils_download_helper "$INDEX_URL$LINEA1" "$MEETING_DIR/Attachments/$LINEA2"
                                    # [ ! -s "$MEETING_DIR/Attachments/$LINEA2" ] && rm -f "$MEETING_DIR/Attachments/$LINEA2"
                                done < ./tmp/attachment_urls 3<./tmp/attachment_names
                                echo "All attachments saved."

                                download_agendas "$MEETING_DIR"

                                if find "$MEETING_DIR/Attachments" -mindepth 1 -maxdepth 1 | read; then
                                    echo "dir not empty" >>/dev/null
                                else
                                    rm -r "$MEETING_DIR/Attachments"
                                fi

                                echo "All files from this meeting have been saved."
                                find "$MEETING_DIR" -type f -size 0 -delete
                                echo "Cleaning PDFs for archive.org..."
                                find "$MEETING_DIR" -type f -name '*.pdf' -print0 | xargs -0 -n1 qpdf --replace-input
                                # qpdf repairs and leaves garbage original PDFs
                                find "$MEETING_DIR" -type f -name '*~qpdf-orig' -delete -print
                            fi

                            ((i++))
                            ((y++))
                        done
                    fi
                fi

                GREPLIST=$(echo $LINE | grep 'class="MeetingTypeListbox"')
                if [[ "$GREPLIST" != "" ]]; then
                    echo "SCRAPE_ESCRIBE: Found meeting type list."
                    FOUNDLIST="TRUE"
                fi
            done < $INDEX_PAGE
        else
            INDEX_END="TRUE"
            echo "SCRAPE_ESCRIBE: Couldn't save index!"
        fi
    done
done < websites.csv