robertoandClaude Sonnet 4.6
dcd980988d
Downloads images from multiple Wayback Machine snapshot dates using page-crawl extraction (im_ modifier). Adds recover_assets3.py and recover_content.py for targeted multi-date recovery. Remaining ~107 content images (2021 step-by-step recipe photos for verduras, arroz negro, arrozabanda, fideua) are confirmed absent from all web archives — pages were crawled but images were lazy-loaded. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
250 lines
11 KiB
Python
250 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Phase 3: For each remaining missing image:
|
||
1. Query CDX for every timestamp at which that exact URL was captured
|
||
2. Try each timestamp with im_ modifier
|
||
3. Also crawl additional WBM page snapshots from the period images were
|
||
published (e.g. 2021/03 verduras images → fetch verduras page from 2021)
|
||
"""
|
||
|
||
import os, re, json, time, subprocess, tempfile, urllib.parse
|
||
from pathlib import Path
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from threading import Lock
|
||
|
||
SITE = Path(__file__).parent / "site"
|
||
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||
DELAY = 0.2
|
||
lock = Lock()
|
||
|
||
IMG_PAT = re.compile(
|
||
r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
|
||
r'https?://(?:www\.)?lapaella\.net'
|
||
r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))',
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
# Extra WBM page snapshots to mine — specifically from around the publication
|
||
# date of each image batch we're still missing
|
||
EXTRA_WBM_PAGES = [
|
||
# paellanet_verduras_* and arroz_negro images published ~2021/03
|
||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/",
|
||
"https://web.archive.org/web/20211001000000*/https://lapaella.net/receta-paella-de-verduras/",
|
||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/",
|
||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/",
|
||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||
"https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||
# paso_fideua and paso_arrozabanda published ~2021/09
|
||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||
"https://web.archive.org/web/20220101000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||
"https://web.archive.org/web/20220101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||
# receta-de-fideua (missing page itself – try multiple dates)
|
||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-fideua/",
|
||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/",
|
||
"https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/",
|
||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-fideua/",
|
||
# coca + IMG-WA images – published ~2020/05
|
||
"https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/",
|
||
"https://web.archive.org/web/20201201000000/https://lapaella.net/coca-valenciana/",
|
||
"https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/",
|
||
# mejilones / fideuadepato-2 – clochina page
|
||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
|
||
"https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/",
|
||
# banner / slide images – home page older snapshots
|
||
"https://web.archive.org/web/20201201000000/https://lapaella.net/",
|
||
"https://web.archive.org/web/20210201000000/https://lapaella.net/",
|
||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/",
|
||
# paella bogavante / arroz negro (old)
|
||
"https://web.archive.org/web/20200901000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||
"https://web.archive.org/web/20210201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||
# paella marisco (old flag icons)
|
||
"https://web.archive.org/web/20130101000000/http://www.lapaella.net/receta-paella-de-marisco/",
|
||
"https://web.archive.org/web/20150101000000/http://www.lapaella.net/receta-paella-de-marisco/",
|
||
"https://web.archive.org/web/20120601000000/http://www.lapaella.net/receta-paella-valenciana/",
|
||
"https://web.archive.org/web/20130601000000/http://www.lapaella.net/receta-paella-valenciana/",
|
||
]
|
||
|
||
def log(msg):
|
||
with lock:
|
||
print(msg, flush=True)
|
||
|
||
def curl_get(url, min_size=200):
|
||
tmp = None
|
||
try:
|
||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
|
||
tmp.close()
|
||
r = subprocess.run(
|
||
["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
|
||
"-w", "\n__S__%{http_code}", "-o", tmp.name, url],
|
||
capture_output=True, text=True, timeout=35,
|
||
)
|
||
status = 0
|
||
for line in r.stdout.splitlines():
|
||
if line.startswith("__S__"):
|
||
try: status = int(line[5:].strip())
|
||
except: pass
|
||
if status in (404, 403, 410, 0):
|
||
return None
|
||
with open(tmp.name, "rb") as f:
|
||
data = f.read()
|
||
return data if len(data) >= min_size else None
|
||
except Exception:
|
||
return None
|
||
finally:
|
||
if tmp:
|
||
try: os.unlink(tmp.name)
|
||
except: pass
|
||
|
||
def cdx_timestamps(orig_path):
|
||
"""Return all CDX-indexed timestamps for a specific image URL."""
|
||
for base in ("lapaella.net", "www.lapaella.net"):
|
||
cdx = (
|
||
"https://web.archive.org/cdx/search/cdx"
|
||
f"?url={base}{urllib.parse.quote(orig_path)}"
|
||
"&output=json&fl=timestamp&filter=statuscode:200&limit=20"
|
||
)
|
||
data = curl_get(cdx, min_size=5)
|
||
if data:
|
||
try:
|
||
rows = json.loads(data.decode())
|
||
ts = [r[0] for r in rows[1:]]
|
||
if ts:
|
||
return ts
|
||
except Exception:
|
||
pass
|
||
time.sleep(DELAY)
|
||
return []
|
||
|
||
def local_path_for(orig_path):
|
||
decoded = urllib.parse.unquote(orig_path).lstrip("/")
|
||
if not decoded or decoded.endswith("/"):
|
||
return SITE / decoded / "index.html"
|
||
p = Path(decoded)
|
||
return SITE / p / "index.html" if not p.suffix else SITE / p
|
||
|
||
def try_download(orig_path, timestamps):
|
||
"""Try all timestamps × (im_ / plain) for an image."""
|
||
local = local_path_for(orig_path)
|
||
for ts in timestamps:
|
||
for modifier in ("im_", ""):
|
||
url = f"https://web.archive.org/web/{ts}{modifier}/https://lapaella.net{orig_path}"
|
||
data = curl_get(url, min_size=500)
|
||
if data:
|
||
local.parent.mkdir(parents=True, exist_ok=True)
|
||
local.write_bytes(data)
|
||
return ts, modifier
|
||
time.sleep(DELAY)
|
||
# Also try www variant
|
||
url = f"https://web.archive.org/web/{ts}im_/http://www.lapaella.net{orig_path}"
|
||
data = curl_get(url, min_size=500)
|
||
if data:
|
||
local.parent.mkdir(parents=True, exist_ok=True)
|
||
local.write_bytes(data)
|
||
return ts, "www+im_"
|
||
time.sleep(DELAY)
|
||
return None, None
|
||
|
||
def extract_images_from_page(wbm_url):
|
||
data = curl_get(wbm_url)
|
||
if not data:
|
||
return {}
|
||
html = data.decode("utf-8", errors="replace")
|
||
result = {}
|
||
for ts, path in IMG_PAT.findall(html):
|
||
path = re.split(r'["\'\s&?]', path)[0]
|
||
result[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}"
|
||
return result
|
||
|
||
def missing_images():
|
||
refs = set()
|
||
for f in SITE.rglob("*.html"):
|
||
t = f.read_text(errors="replace")
|
||
for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp|svg))', t, re.I):
|
||
if '?' not in m and '*' not in m:
|
||
refs.add(m)
|
||
return sorted(r for r in refs if not local_path_for(r).exists())
|
||
|
||
def main():
|
||
missing = missing_images()
|
||
print(f"Missing images: {len(missing)}\n")
|
||
|
||
# ── Phase A: mine extra WBM page snapshots ───────────────────────────────
|
||
print("Phase A: mining extra WBM page snapshots...")
|
||
page_found = {}
|
||
for page_url in EXTRA_WBM_PAGES:
|
||
found = extract_images_from_page(page_url)
|
||
new = {k: v for k, v in found.items()
|
||
if k in missing and k not in page_found}
|
||
if new:
|
||
label = page_url.split('/web/')[1][:60]
|
||
print(f" {label} +{len(new)}")
|
||
page_found.update({k: v for k, v in found.items() if k in missing})
|
||
time.sleep(0.5)
|
||
|
||
print(f" Found WBM URLs for {len(page_found)} of {len(missing)} missing images\n")
|
||
|
||
# ── Phase B: CDX lookup for images NOT found via pages ───────────────────
|
||
still_missing = [p for p in missing if p not in page_found]
|
||
print(f"Phase B: CDX timestamp search for {len(still_missing)} remaining images...")
|
||
cdx_found = {}
|
||
for orig in still_missing:
|
||
ts_list = cdx_timestamps(orig)
|
||
if ts_list:
|
||
cdx_found[orig] = ts_list
|
||
print(f" CDX hit: {orig} ({ts_list[0][:8]}…)")
|
||
time.sleep(DELAY)
|
||
|
||
# ── Phase C: download everything ─────────────────────────────────────────
|
||
print(f"\nPhase C: downloading...")
|
||
|
||
def do_download(orig):
|
||
if local_path_for(orig).exists():
|
||
return "skip", orig
|
||
# Try page-discovered URL first (exact timestamp known to work)
|
||
if orig in page_found:
|
||
wbm_url = page_found[orig]
|
||
ts = re.search(r'/web/(\d+)', wbm_url).group(1)
|
||
found_ts, mod = try_download(orig, [ts])
|
||
if found_ts:
|
||
log(f" ✓ page/{found_ts[:8]} {orig}")
|
||
return "ok", orig
|
||
# Try CDX timestamps
|
||
if orig in cdx_found:
|
||
found_ts, mod = try_download(orig, cdx_found[orig])
|
||
if found_ts:
|
||
log(f" ✓ cdx/{found_ts[:8]} {orig}")
|
||
return "ok", orig
|
||
# Brute-force: try a broad range of timestamps
|
||
broad = [
|
||
"20251013025800", "20241130065929", "20241014051856",
|
||
"20230601000000", "20220601000000", "20210603053819",
|
||
"20200601000000", "20190601000000", "20180101000000",
|
||
"20160101000000", "20140101000000", "20120601000000",
|
||
]
|
||
found_ts, mod = try_download(orig, broad)
|
||
if found_ts:
|
||
log(f" ✓ brute/{found_ts[:8]} {orig}")
|
||
return "ok", orig
|
||
log(f" ✗ {orig}")
|
||
return "fail", orig
|
||
|
||
ok = fail = skip = 0
|
||
with ThreadPoolExecutor(max_workers=3) as ex:
|
||
futures = {ex.submit(do_download, p): p for p in missing}
|
||
for fut in as_completed(futures):
|
||
status, _ = fut.result()
|
||
if status == "ok": ok += 1
|
||
elif status == "fail": fail += 1
|
||
else: skip += 1
|
||
|
||
print(f"\n{'='*55}")
|
||
print(f"Recovered: {ok} Failed: {fail} Skipped: {skip}")
|
||
print(f"{'='*55}")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|