Bypass eScribe's new anti-bot measures

SCRAPE_MEET.SH

@@ -15,6 +15,9 @@ source ./functions/.functions
 # London seems to have recently blocked unusual user agents. Can't use wget or even ping. Thankfully pretend to be a real person!
 WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
 
+# See _utils_download_escribe() and .functions.utils.playwright.py
+UTILS_DLMODE="escribe"
+
 TEMP_DIR="./tmp/"
 INDEX_PAGE="./tmp/index.html"
 SEARCH_PAGE="./tmp/search.html"
@@ -244,4 +247,4 @@ while IFS="," read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
             echo "SCRAPE_ESCRIBE: Couldn't save index!"
         fi
     done
-done < websites.csv
+done < websites.csv

functions/.functions

@@ -1,5 +1,7 @@
+#!/bin/bash
 sdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+export "$sdir"
 # General
 source "$sdir/.functions.time"
 source "$sdir/.functions.utils"

functions/.functions.escribe

@@ -1,3 +1,4 @@
+#!/bin/bash
 set_agenda_url() {
     case "$1" in
         '"Agenda (HTML)"')
@@ -130,4 +131,4 @@ download_agendas() {
     echo "Saving addendum as HTML... (no PDF found!)"
     _utils_download_helper "$ADDENDUM_HTML_URL" "$outdir/Addendum.html"
   fi
-}
+}

functions/.functions.filepro

@@ -1,3 +1,4 @@
+#!/bin/bash
 _filepro_download_folder() {
     if [ "$#" -eq 0 ]; then
       echo "Usage: <input string>"
@@ -31,4 +32,4 @@ _filepro_download_folder() {
   done  < "$tmp_index"
 
   rm -f $tmp_index
-}
+}

functions/.functions.time

@@ -1,7 +1,8 @@
+#!/bin/bash
 _time_parse_helper() {
   if [ "$#" -eq 0 ]; then
     echo "Usage: <date>"
-    exit 1
+    return 1
   fi
 
   echo $1
@@ -19,7 +20,7 @@ _time_parse_helper() {
 _time_parse_monddyyyy() {
   if [ "$#" -eq 0 ]; then
     echo "Usage: <date in mon dd yyyy>"
-    exit 1
+    return 1
   fi
 
   ITEM_MONTH_WORD=$(echo "$1" | sed -E 's/^([A-Za-z]+) .*/\1/' | sed 's/^[ \t]*//;s/[ \t]*$//')
@@ -46,7 +47,7 @@ _time_parse_monddyyyy() {
 _time_parse_ddmonyyyy() {   
   if [ "$#" -eq 0 ]; then
     echo "Usage: <date in dd mon yyyy>"
-    exit 1
+    return 1
   fi
    
   ITEM_MONTH_WORD=$(echo "$1" | sed 's/^[^ ]* //' | sed 's/ .*//')

functions/.functions.utils

@@ -1,7 +1,8 @@
+#!/bin/bash
 _utils_ocrmypdf() {
   if [ "$#" -eq 0 ]; then
     echo "Usage: <in.pdf> <out.pdf>"
-    exit 1
+    return 1
   fi
   
   # https://stackoverflow.com/questions/7997399/bash-script-to-check-pdfs-are-ocrd
@@ -11,7 +12,7 @@ _utils_ocrmypdf() {
   else 
       echo "$1 is already OCRed. Saving as is."
       cp "$1" "$2"
-      exit 0
+      return 0
   fi 
 
   in="$1"
@@ -53,7 +54,7 @@ _utils_ocrmypdf() {
 _utils_fix_dashes() {
     if [ "$#" -eq 0 ]; then
       echo "Usage: <input string>"
-      exit 1
+      return 1
     fi
     
     perl -CSDA -MURI::Escape -MUnicode::Normalize -e '
@@ -76,9 +77,19 @@ _utils_fix_dashes() {
 }
 
 _utils_download_helper() {
+	if [ "$UTILS_DLMODE" == "escribe" ]; then
+       # echo "eScribe download mode."
+		_utils_download_escribe "$1" "$2"
+	else
+		_utils_download_standard "$1" "$2"
+	fi
+}
+
+
+_utils_download_standard() {
   if [ "$#" -eq 0 ]; then
     echo "Usage: <url> <outfile>"
-    exit 1
+    return 1
   fi
  
   WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
@@ -102,3 +113,31 @@ _utils_download_helper() {
       ;;
   esac
 }
+
+# eScribe started adding anti-bot measures in 2026/09, so we must use a headless browser like playwright.
+# Not a fan of using a tool intended for AI, but sacrifices must be made in this dystopian hellscape that is the modern internet...
+_utils_download_escribe() {
+    if [ "$#" -eq 0 ]; then
+        echo "Usage: <url> <outfile>"
+        return 1
+    fi
+
+    local url="$1"
+    local out="$2"
+
+    # Ehhhh probably don't need xvfb since we're running firefox headless?
+    xvfb-run -a \
+        python $sdir/.functions.utils.playwright.py \
+        "$url" \
+        "$out"
+    local result=$?
+
+    if [ "$result" -eq 0 ]; then
+        # echo "Downloaded."
+        return 0
+    fi
+
+    echo "FAILED! $out | $url" >&2
+    rm -f "$out"
+    return 1
+}

functions/.functions.utils.playwright.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+from playwright.sync_api import sync_playwright
+
+def fail(msg):
+    print(f"FAILED! {msg}")
+    sys.exit(1)
+
+if len(sys.argv) != 3:
+    fail("Usage: <url> <outfile>")
+
+url = sys.argv[1]
+outfile = sys.argv[2]
+# eScribe doesn't use consistent cases between municipalities.
+# I don't know, maybe python doesn't care? I don't know python lol, this entire thing was blogs and stack overflow glued together.
+url_lowercase = url.lower()
+
+try:
+    with sync_playwright() as p:
+        # Firefox headless is okay I guess. I tried chrome but 403.
+        browser = p.firefox.launch(
+            headless=True
+        )
+        context = browser.new_context(
+            accept_downloads=True
+        )
+        page = context.new_page()
+
+        # The PDFs
+        if "filestream.ashx" in url_lowercase:
+            page.set_content(
+                f'<a id="download" href="{url}">Too bad eScribe, your files are mine. Click here!</a>'
+            )
+            with page.expect_download(
+                timeout=60000
+            ) as d:
+                page.locator("#download").click()
+            download = d.value
+            os.makedirs(
+                os.path.dirname(outfile) or ".",
+                exist_ok=True
+            )
+            download.save_as(outfile)
+            browser.close()
+            sys.exit(0)
+
+        # The HTMLs
+        if "meeting.aspx" in url_lowercase:
+            response = page.goto(
+                url,
+                timeout=50000
+            )
+            if response is None:
+                fail("No response")
+            body = response.body()
+            os.makedirs(
+                os.path.dirname(outfile) or ".",
+                exist_ok=True
+            )
+            with open(outfile, "wb") as fp:
+                fp.write(body)
+            browser.close()
+            sys.exit(0)
+        browser.close()
+
+        # Not filestream or meeting. I do not know what this could be.
+        fail(
+            f"New or unsupported eScribe URL: {url}"
+        )
+
+except Exception as e:
+
+    fail(
+        f"{type(e).__name__}: {e}"
+    )