#!/usr/bin/env python3 """ Recover missing images and pages from lapaella.net mirror. Strategy: 1. Scan all local HTML for wp-content/uploads/* paths not present on disk 2. For each missing image try: a. Direct live URL (site may still exist in cache/CDN) b. Wayback im_ modifier with CDX-best timestamp c. Wayback im_ with known timestamps in order 3. Download missing HTML pages via fetch_missing-style CDX lookup 4. Report final stats Run from the lapaella-mirror directory. """ import os, re, sys, time, subprocess, tempfile, urllib.parse, json from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock SITE = Path(__file__).parent / "site" UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" BASE = "https://lapaella.net" BASE_OLD = "http://www.lapaella.net" # Most useful timestamps for this site TS_LIST = [ "20241130065929", "20241014051856", "20210603053819", "20220101000000", "20200601000000", "20180101000000", "20150101000000", "20120601000000", ] DELAY = 0.25 # per-worker delay between requests print_lock = Lock() # ── Missing pages to recover ───────────────────────────────────────────────── MISSING_PAGES = [ "/otras-recetas-valencianas/all-i-oli/", "/arroz-de-frutos-secos-garbanzos-y-nisperos/", "/clochina-o-mejillon/", "/receta-de-arroz-negro/", "/receta-de-fideua", "/otras-recetas-valencianas/", "/enlaces/", "/valencian-paella-introduction/", "/paella-valenciana-recipe-english/", "/contacto/", ] # ── Helpers ────────────────────────────────────────────────────────────────── def log(msg): with print_lock: print(msg, flush=True) def curl_fetch(url, is_image=False): tmp = None try: tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") tmp.close() cmd = [ "/usr/bin/curl", "-s", "-L", "--max-time", "20", "-A", UA, "-w", "\n__S__%{http_code}", "-o", tmp.name, url, ] r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) status = 0 for line in r.stdout.splitlines(): if line.startswith("__S__"): try: status = int(line[5:].strip()) except: pass if status in (404, 403, 410, 0): return None with open(tmp.name, "rb") as f: data = f.read() min_size = 200 if is_image else 500 return data if len(data) >= min_size else None except Exception: return None finally: if tmp: try: os.unlink(tmp.name) except: pass def cdx_best_ts(orig_path, from_year="2010"): """Ask CDX for the best (closest to latest) 200-status snapshot timestamp.""" try: url = ( "https://web.archive.org/cdx/search/cdx" f"?url=lapaella.net{urllib.parse.quote(orig_path)}" "&output=json&limit=5&fl=timestamp&filter=statuscode:200" f"&from={from_year}0101&to=20251231&fastLatest=true" ) data = curl_fetch(url) if not data: return None rows = json.loads(data.decode()) # rows[0] is the header ["timestamp"] if len(rows) > 1: return rows[1][0] # best timestamp except Exception: pass return None def local_path_for(orig_path): decoded = urllib.parse.unquote(orig_path).lstrip("/") if not decoded or decoded.endswith("/"): return SITE / decoded / "index.html" p = Path(decoded) return SITE / p / "index.html" if not p.suffix else SITE / p def wbm_image_url(orig_path, ts): return f"https://web.archive.org/web/{ts}im_/{BASE}{orig_path}" def wbm_page_url(orig_path, ts, old=False): base = BASE_OLD if old else BASE return f"https://web.archive.org/web/{ts}/{base}{orig_path}" # ── Image recovery ─────────────────────────────────────────────────────────── def recover_image(orig_path): local = local_path_for(orig_path) if local.exists(): return "skip", orig_path # 1) Try live site first (fast, no WBM rate limits) live_url = f"{BASE}{orig_path}" data = curl_fetch(live_url, is_image=True) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) log(f" ✓ live {orig_path}") return "ok", orig_path # 2) CDX best timestamp best_ts = cdx_best_ts(orig_path) ts_candidates = ([best_ts] if best_ts else []) + [ ts for ts in TS_LIST if ts != best_ts ] for ts in ts_candidates: url = wbm_image_url(orig_path, ts) data = curl_fetch(url, is_image=True) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) log(f" ✓ wbm/{ts[:8]} {orig_path}") return "ok", orig_path time.sleep(DELAY) log(f" ✗ not found {orig_path}") return "fail", orig_path # ── Page recovery ──────────────────────────────────────────────────────────── def orig_path_from_wbm(url): m = re.match( r'https?://web\.archive\.org/web/\d+(?:im_|cs_|js_|oe_|if_|mp_)?/' r'https?://(?:www\.)?lapaella\.net(/[^"\'<>\s#]*)?', url, ) return (m.group(1) or "/") if m else None def rewrite_html(html, local_file): html = re.sub(r'.*?', '', html, flags=re.DOTALL) html = re.sub(r']*(?:archive\.org|wombat|athena|bundle-playback|ruffle)[^>]*>.*?', '', html, flags=re.DOTALL|re.IGNORECASE) html = re.sub(r']*(?:archive\.org|wombat|athena|bundle-playback|ruffle)[^>]*/?>', '', html, flags=re.IGNORECASE) html = re.sub(r']*(?:archive\.org|iconochive|banner-styles)[^>]*/?>', '', html, flags=re.IGNORECASE) html = re.sub(r']*>\s*__wm\b.*?', '', html, flags=re.DOTALL) def _rewrite(url): if "web.archive.org" not in url: return url orig = orig_path_from_wbm(url) if not orig: return "#" rel = os.path.relpath(str(local_path_for(orig)), str(local_file.parent)) return rel def _attr(m): attr, q, url, q2 = m.group(1), m.group(2), m.group(3), m.group(4) return f'{attr}={q}{_rewrite(url)}{q2}' html = re.sub(r'((?:src|href|action|data-src)=)(["\'])([^"\']+)(["\'])', _attr, html) def _css_url(m): url = m.group(1).strip().strip("'\"") return f"url('{_rewrite(url)}')" html = re.sub(r'url\(([^)]+)\)', _css_url, html) return html def recover_page(orig_path): local = local_path_for(orig_path) if local.exists(): return "skip", orig_path best_ts = cdx_best_ts(orig_path) ts_candidates = ([best_ts] if best_ts else []) + [ ts for ts in TS_LIST if ts != best_ts ] for ts in ts_candidates: for old in (False, True): url = wbm_page_url(orig_path, ts, old=old) data = curl_fetch(url) if data: html = data.decode("utf-8", errors="replace") local.parent.mkdir(parents=True, exist_ok=True) html = rewrite_html(html, local) local.write_text(html, encoding="utf-8") log(f" ✓ page/{ts[:8]} {orig_path}") return "ok", orig_path time.sleep(DELAY) log(f" ✗ page not found {orig_path}") return "fail", orig_path # ── Main ───────────────────────────────────────────────────────────────────── def collect_missing_images(): refs = set() for f in SITE.rglob("*.html"): txt = f.read_text(errors="replace") for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+)', txt): # skip CSS query-string variants and wildcards if "?" in m or "*" in m or not re.search(r'\.(jpe?g|png|gif|webp|svg|ico|bmp)$', m, re.I): continue refs.add(m) missing = [r for r in sorted(refs) if not (SITE / r.lstrip("/")).exists()] return missing def main(): missing_images = collect_missing_images() log(f"\n{'='*60}") log(f"Missing images : {len(missing_images)}") log(f"Missing pages : {len(MISSING_PAGES)}") log(f"{'='*60}\n") img_ok = img_fail = 0 pg_ok = pg_fail = 0 # Images — parallel with 4 workers (be polite to WBM) log("── Images ──────────────────────────────────────────────────") with ThreadPoolExecutor(max_workers=4) as ex: futures = {ex.submit(recover_image, p): p for p in missing_images} for fut in as_completed(futures): status, _ = fut.result() if status == "ok": img_ok += 1 elif status == "fail": img_fail += 1 # Pages — sequential (each page may cascade more asset fetches) log("\n── Pages ───────────────────────────────────────────────────") for orig_path in MISSING_PAGES: status, _ = recover_page(orig_path) if status == "ok": pg_ok += 1 elif status == "fail": pg_fail += 1 time.sleep(1.0) log(f"\n{'='*60}") log(f"Images recovered: {img_ok} failed: {img_fail}") log(f"Pages recovered: {pg_ok} failed: {pg_fail}") log(f"{'='*60}") if __name__ == "__main__": main()