From 030d51149e53c7d0f51d50255f86804e186649dc Mon Sep 17 00:00:00 2001
From: Lillian Skinner <56081713+rvtr@users.noreply.github.com>
Date: Fri, 30 Oct 2026 11:26:56 -0400
Subject: [PATCH] Bypass eScribe's new anti-bot measures
---
SCRAPE_MEET.SH | 5 +-
functions/.functions | 2 +
functions/.functions.escribe | 3 +-
functions/.functions.filepro | 3 +-
functions/.functions.time | 7 ++-
functions/.functions.utils | 47 +++++++++++++--
functions/.functions.utils.playwright.py | 77 ++++++++++++++++++++++++
7 files changed, 134 insertions(+), 10 deletions(-)
create mode 100644 functions/.functions.utils.playwright.py
diff --git a/SCRAPE_MEET.SH b/SCRAPE_MEET.SH
index 86ca1c6..a538a2f 100755
--- a/SCRAPE_MEET.SH
+++ b/SCRAPE_MEET.SH
@@ -15,6 +15,9 @@ source ./functions/.functions
# London seems to have recently blocked unusual user agents. Can't use wget or even ping. Thankfully pretend to be a real person!
WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
+# See _utils_download_escribe() and .functions.utils.playwright.py
+UTILS_DLMODE="escribe"
+
TEMP_DIR="./tmp/"
INDEX_PAGE="./tmp/index.html"
SEARCH_PAGE="./tmp/search.html"
@@ -244,4 +247,4 @@ while IFS="," read -r INDEX_URL_PRE CITY_ARCHIVE_NAME_PRE CALENDAR_NAME_PRE; do
echo "SCRAPE_ESCRIBE: Couldn't save index!"
fi
done
-done < websites.csv
+done < websites.csv
\ No newline at end of file
diff --git a/functions/.functions b/functions/.functions
index 148aa9b..4c8f44b 100644
--- a/functions/.functions
+++ b/functions/.functions
@@ -1,5 +1,7 @@
+#!/bin/bash
sdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export "$sdir"
# General
source "$sdir/.functions.time"
source "$sdir/.functions.utils"
diff --git a/functions/.functions.escribe b/functions/.functions.escribe
index b2b035f..4e79e1c 100644
--- a/functions/.functions.escribe
+++ b/functions/.functions.escribe
@@ -1,3 +1,4 @@
+#!/bin/bash
set_agenda_url() {
case "$1" in
'"Agenda (HTML)"')
@@ -130,4 +131,4 @@ download_agendas() {
echo "Saving addendum as HTML... (no PDF found!)"
_utils_download_helper "$ADDENDUM_HTML_URL" "$outdir/Addendum.html"
fi
-}
\ No newline at end of file
+}
diff --git a/functions/.functions.filepro b/functions/.functions.filepro
index d453627..5841802 100644
--- a/functions/.functions.filepro
+++ b/functions/.functions.filepro
@@ -1,3 +1,4 @@
+#!/bin/bash
_filepro_download_folder() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
@@ -31,4 +32,4 @@ _filepro_download_folder() {
done < "$tmp_index"
rm -f $tmp_index
-}
\ No newline at end of file
+}
diff --git a/functions/.functions.time b/functions/.functions.time
index 768f0c1..18f5f48 100644
--- a/functions/.functions.time
+++ b/functions/.functions.time
@@ -1,7 +1,8 @@
+#!/bin/bash
_time_parse_helper() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
- exit 1
+ return 1
fi
echo $1
@@ -19,7 +20,7 @@ _time_parse_helper() {
_time_parse_monddyyyy() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
- exit 1
+ return 1
fi
ITEM_MONTH_WORD=$(echo "$1" | sed -E 's/^([A-Za-z]+) .*/\1/' | sed 's/^[ \t]*//;s/[ \t]*$//')
@@ -46,7 +47,7 @@ _time_parse_monddyyyy() {
_time_parse_ddmonyyyy() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
- exit 1
+ return 1
fi
ITEM_MONTH_WORD=$(echo "$1" | sed 's/^[^ ]* //' | sed 's/ .*//')
diff --git a/functions/.functions.utils b/functions/.functions.utils
index 24419ec..da7b67e 100644
--- a/functions/.functions.utils
+++ b/functions/.functions.utils
@@ -1,7 +1,8 @@
+#!/bin/bash
_utils_ocrmypdf() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
- exit 1
+ return 1
fi
# https://stackoverflow.com/questions/7997399/bash-script-to-check-pdfs-are-ocrd
@@ -11,7 +12,7 @@ _utils_ocrmypdf() {
else
echo "$1 is already OCRed. Saving as is."
cp "$1" "$2"
- exit 0
+ return 0
fi
in="$1"
@@ -53,7 +54,7 @@ _utils_ocrmypdf() {
_utils_fix_dashes() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
- exit 1
+ return 1
fi
perl -CSDA -MURI::Escape -MUnicode::Normalize -e '
@@ -76,9 +77,19 @@ _utils_fix_dashes() {
}
_utils_download_helper() {
+ if [ "$UTILS_DLMODE" == "escribe" ]; then
+ # echo "eScribe download mode."
+ _utils_download_escribe "$1" "$2"
+ else
+ _utils_download_standard "$1" "$2"
+ fi
+}
+
+
+_utils_download_standard() {
if [ "$#" -eq 0 ]; then
echo "Usage: "
- exit 1
+ return 1
fi
WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
@@ -102,3 +113,31 @@ _utils_download_helper() {
;;
esac
}
+
+# eScribe started adding anti-bot measures in 2026/09, so we must use a headless browser like playwright.
+# Not a fan of using a tool intended for AI, but sacrifices must be made in this dystopian hellscape that is the modern internet...
+_utils_download_escribe() {
+ if [ "$#" -eq 0 ]; then
+ echo "Usage: "
+ return 1
+ fi
+
+ local url="$1"
+ local out="$2"
+
+ # Ehhhh probably don't need xvfb since we're running firefox headless?
+ xvfb-run -a \
+ python $sdir/.functions.utils.playwright.py \
+ "$url" \
+ "$out"
+ local result=$?
+
+ if [ "$result" -eq 0 ]; then
+ # echo "Downloaded."
+ return 0
+ fi
+
+ echo "FAILED! $out | $url" >&2
+ rm -f "$out"
+ return 1
+}
\ No newline at end of file
diff --git a/functions/.functions.utils.playwright.py b/functions/.functions.utils.playwright.py
new file mode 100644
index 0000000..d333b70
--- /dev/null
+++ b/functions/.functions.utils.playwright.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+from playwright.sync_api import sync_playwright
+
+def fail(msg):
+ print(f"FAILED! {msg}")
+ sys.exit(1)
+
+if len(sys.argv) != 3:
+ fail("Usage: ")
+
+url = sys.argv[1]
+outfile = sys.argv[2]
+# eScribe doesn't use consistent cases between municipalities.
+# I don't know, maybe python doesn't care? I don't know python lol, this entire thing was blogs and stack overflow glued together.
+url_lowercase = url.lower()
+
+try:
+ with sync_playwright() as p:
+ # Firefox headless is okay I guess. I tried chrome but 403.
+ browser = p.firefox.launch(
+ headless=True
+ )
+ context = browser.new_context(
+ accept_downloads=True
+ )
+ page = context.new_page()
+
+ # The PDFs
+ if "filestream.ashx" in url_lowercase:
+ page.set_content(
+ f'Too bad eScribe, your files are mine. Click here!'
+ )
+ with page.expect_download(
+ timeout=60000
+ ) as d:
+ page.locator("#download").click()
+ download = d.value
+ os.makedirs(
+ os.path.dirname(outfile) or ".",
+ exist_ok=True
+ )
+ download.save_as(outfile)
+ browser.close()
+ sys.exit(0)
+
+ # The HTMLs
+ if "meeting.aspx" in url_lowercase:
+ response = page.goto(
+ url,
+ timeout=50000
+ )
+ if response is None:
+ fail("No response")
+ body = response.body()
+ os.makedirs(
+ os.path.dirname(outfile) or ".",
+ exist_ok=True
+ )
+ with open(outfile, "wb") as fp:
+ fp.write(body)
+ browser.close()
+ sys.exit(0)
+ browser.close()
+
+ # Not filestream or meeting. I do not know what this could be.
+ fail(
+ f"New or unsupported eScribe URL: {url}"
+ )
+
+except Exception as e:
+
+ fail(
+ f"{type(e).__name__}: {e}"
+ )
\ No newline at end of file