#!/usr/bin/env python3 """ Phase 3: For each remaining missing image: 1. Query CDX for every timestamp at which that exact URL was captured 2. Try each timestamp with im_ modifier 3. Also crawl additional WBM page snapshots from the period images were published (e.g. 2021/03 verduras images → fetch verduras page from 2021) """ import os, re, json, time, subprocess, tempfile, urllib.parse from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock SITE = Path(__file__).parent / "site" UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" DELAY = 0.2 lock = Lock() IMG_PAT = re.compile( r'https://web\.archive\.org/web/(\d+)(?:im_)?/' r'https?://(?:www\.)?lapaella\.net' r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))', re.IGNORECASE, ) # Extra WBM page snapshots to mine — specifically from around the publication # date of each image batch we're still missing EXTRA_WBM_PAGES = [ # paellanet_verduras_* and arroz_negro images published ~2021/03 "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20211001000000*/https://lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", # paso_fideua and paso_arrozabanda published ~2021/09 "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-a-banda/", "https://web.archive.org/web/20220101000000/https://lapaella.net/receta-de-arroz-a-banda/", "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/", "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/", "https://web.archive.org/web/20220101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/", # receta-de-fideua (missing page itself – try multiple dates) "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-fideua/", "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/", "https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/", "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-fideua/", # coca + IMG-WA images – published ~2020/05 "https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/", "https://web.archive.org/web/20201201000000/https://lapaella.net/coca-valenciana/", "https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/", # mejilones / fideuadepato-2 – clochina page "https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/", "https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/", # banner / slide images – home page older snapshots "https://web.archive.org/web/20201201000000/https://lapaella.net/", "https://web.archive.org/web/20210201000000/https://lapaella.net/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/", # paella bogavante / arroz negro (old) "https://web.archive.org/web/20200901000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20210201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/", # paella marisco (old flag icons) "https://web.archive.org/web/20130101000000/http://www.lapaella.net/receta-paella-de-marisco/", "https://web.archive.org/web/20150101000000/http://www.lapaella.net/receta-paella-de-marisco/", "https://web.archive.org/web/20120601000000/http://www.lapaella.net/receta-paella-valenciana/", "https://web.archive.org/web/20130601000000/http://www.lapaella.net/receta-paella-valenciana/", ] def log(msg): with lock: print(msg, flush=True) def curl_get(url, min_size=200): tmp = None try: tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") tmp.close() r = subprocess.run( ["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA, "-w", "\n__S__%{http_code}", "-o", tmp.name, url], capture_output=True, text=True, timeout=35, ) status = 0 for line in r.stdout.splitlines(): if line.startswith("__S__"): try: status = int(line[5:].strip()) except: pass if status in (404, 403, 410, 0): return None with open(tmp.name, "rb") as f: data = f.read() return data if len(data) >= min_size else None except Exception: return None finally: if tmp: try: os.unlink(tmp.name) except: pass def cdx_timestamps(orig_path): """Return all CDX-indexed timestamps for a specific image URL.""" for base in ("lapaella.net", "www.lapaella.net"): cdx = ( "https://web.archive.org/cdx/search/cdx" f"?url={base}{urllib.parse.quote(orig_path)}" "&output=json&fl=timestamp&filter=statuscode:200&limit=20" ) data = curl_get(cdx, min_size=5) if data: try: rows = json.loads(data.decode()) ts = [r[0] for r in rows[1:]] if ts: return ts except Exception: pass time.sleep(DELAY) return [] def local_path_for(orig_path): decoded = urllib.parse.unquote(orig_path).lstrip("/") if not decoded or decoded.endswith("/"): return SITE / decoded / "index.html" p = Path(decoded) return SITE / p / "index.html" if not p.suffix else SITE / p def try_download(orig_path, timestamps): """Try all timestamps × (im_ / plain) for an image.""" local = local_path_for(orig_path) for ts in timestamps: for modifier in ("im_", ""): url = f"https://web.archive.org/web/{ts}{modifier}/https://lapaella.net{orig_path}" data = curl_get(url, min_size=500) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) return ts, modifier time.sleep(DELAY) # Also try www variant url = f"https://web.archive.org/web/{ts}im_/http://www.lapaella.net{orig_path}" data = curl_get(url, min_size=500) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) return ts, "www+im_" time.sleep(DELAY) return None, None def extract_images_from_page(wbm_url): data = curl_get(wbm_url) if not data: return {} html = data.decode("utf-8", errors="replace") result = {} for ts, path in IMG_PAT.findall(html): path = re.split(r'["\'\s&?]', path)[0] result[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}" return result def missing_images(): refs = set() for f in SITE.rglob("*.html"): t = f.read_text(errors="replace") for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp|svg))', t, re.I): if '?' not in m and '*' not in m: refs.add(m) return sorted(r for r in refs if not local_path_for(r).exists()) def main(): missing = missing_images() print(f"Missing images: {len(missing)}\n") # ── Phase A: mine extra WBM page snapshots ─────────────────────────────── print("Phase A: mining extra WBM page snapshots...") page_found = {} for page_url in EXTRA_WBM_PAGES: found = extract_images_from_page(page_url) new = {k: v for k, v in found.items() if k in missing and k not in page_found} if new: label = page_url.split('/web/')[1][:60] print(f" {label} +{len(new)}") page_found.update({k: v for k, v in found.items() if k in missing}) time.sleep(0.5) print(f" Found WBM URLs for {len(page_found)} of {len(missing)} missing images\n") # ── Phase B: CDX lookup for images NOT found via pages ─────────────────── still_missing = [p for p in missing if p not in page_found] print(f"Phase B: CDX timestamp search for {len(still_missing)} remaining images...") cdx_found = {} for orig in still_missing: ts_list = cdx_timestamps(orig) if ts_list: cdx_found[orig] = ts_list print(f" CDX hit: {orig} ({ts_list[0][:8]}…)") time.sleep(DELAY) # ── Phase C: download everything ───────────────────────────────────────── print(f"\nPhase C: downloading...") def do_download(orig): if local_path_for(orig).exists(): return "skip", orig # Try page-discovered URL first (exact timestamp known to work) if orig in page_found: wbm_url = page_found[orig] ts = re.search(r'/web/(\d+)', wbm_url).group(1) found_ts, mod = try_download(orig, [ts]) if found_ts: log(f" ✓ page/{found_ts[:8]} {orig}") return "ok", orig # Try CDX timestamps if orig in cdx_found: found_ts, mod = try_download(orig, cdx_found[orig]) if found_ts: log(f" ✓ cdx/{found_ts[:8]} {orig}") return "ok", orig # Brute-force: try a broad range of timestamps broad = [ "20251013025800", "20241130065929", "20241014051856", "20230601000000", "20220601000000", "20210603053819", "20200601000000", "20190601000000", "20180101000000", "20160101000000", "20140101000000", "20120601000000", ] found_ts, mod = try_download(orig, broad) if found_ts: log(f" ✓ brute/{found_ts[:8]} {orig}") return "ok", orig log(f" ✗ {orig}") return "fail", orig ok = fail = skip = 0 with ThreadPoolExecutor(max_workers=3) as ex: futures = {ex.submit(do_download, p): p for p in missing} for fut in as_completed(futures): status, _ = fut.result() if status == "ok": ok += 1 elif status == "fail": fail += 1 else: skip += 1 print(f"\n{'='*55}") print(f"Recovered: {ok} Failed: {fail} Skipped: {skip}") print(f"{'='*55}") if __name__ == "__main__": main()