Recover 28 missing images + add phase-2 recovery scripts

Downloads 28 previously missing images from Wayback Machine using
im_ modifier and CDX-lookup timestamps. Adds recover_assets.py
(CDX-based) and recover_assets2.py (page-crawl-based) for continued
recovery when WBM rate limit lifts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

lapaella-mirror/recover_assets.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""
+Recover missing images and pages from lapaella.net mirror.
+
+Strategy:
+  1. Scan all local HTML for wp-content/uploads/* paths not present on disk
+  2. For each missing image try:
+       a. Direct live URL  (site may still exist in cache/CDN)
+       b. Wayback im_ modifier with CDX-best timestamp
+       c. Wayback im_ with known timestamps in order
+  3. Download missing HTML pages via fetch_missing-style CDX lookup
+  4. Report final stats
+
+Run from the lapaella-mirror directory.
+"""
+
+import os, re, sys, time, subprocess, tempfile, urllib.parse, json
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
+
+SITE      = Path(__file__).parent / "site"
+UA        = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+BASE      = "https://lapaella.net"
+BASE_OLD  = "http://www.lapaella.net"
+# Most useful timestamps for this site
+TS_LIST   = [
+    "20241130065929",
+    "20241014051856",
+    "20210603053819",
+    "20220101000000",
+    "20200601000000",
+    "20180101000000",
+    "20150101000000",
+    "20120601000000",
+]
+DELAY     = 0.25   # per-worker delay between requests
+print_lock = Lock()
+
+# ── Missing pages to recover ─────────────────────────────────────────────────
+MISSING_PAGES = [
+    "/otras-recetas-valencianas/all-i-oli/",
+    "/arroz-de-frutos-secos-garbanzos-y-nisperos/",
+    "/clochina-o-mejillon/",
+    "/receta-de-arroz-negro/",
+    "/receta-de-fideua",
+    "/otras-recetas-valencianas/",
+    "/enlaces/",
+    "/valencian-paella-introduction/",
+    "/paella-valenciana-recipe-english/",
+    "/contacto/",
+]
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+def log(msg):
+    with print_lock:
+        print(msg, flush=True)
+
+def curl_fetch(url, is_image=False):
+    tmp = None
+    try:
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
+        tmp.close()
+        cmd = [
+            "/usr/bin/curl", "-s", "-L", "--max-time", "20",
+            "-A", UA,
+            "-w", "\n__S__%{http_code}",
+            "-o", tmp.name,
+            url,
+        ]
+        r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+        status = 0
+        for line in r.stdout.splitlines():
+            if line.startswith("__S__"):
+                try: status = int(line[5:].strip())
+                except: pass
+        if status in (404, 403, 410, 0):
+            return None
+        with open(tmp.name, "rb") as f:
+            data = f.read()
+        min_size = 200 if is_image else 500
+        return data if len(data) >= min_size else None
+    except Exception:
+        return None
+    finally:
+        if tmp:
+            try: os.unlink(tmp.name)
+            except: pass
+
+def cdx_best_ts(orig_path, from_year="2010"):
+    """Ask CDX for the best (closest to latest) 200-status snapshot timestamp."""
+    try:
+        url = (
+            "https://web.archive.org/cdx/search/cdx"
+            f"?url=lapaella.net{urllib.parse.quote(orig_path)}"
+            "&output=json&limit=5&fl=timestamp&filter=statuscode:200"
+            f"&from={from_year}0101&to=20251231&fastLatest=true"
+        )
+        data = curl_fetch(url)
+        if not data:
+            return None
+        rows = json.loads(data.decode())
+        # rows[0] is the header ["timestamp"]
+        if len(rows) > 1:
+            return rows[1][0]  # best timestamp
+    except Exception:
+        pass
+    return None
+
+def local_path_for(orig_path):
+    decoded = urllib.parse.unquote(orig_path).lstrip("/")
+    if not decoded or decoded.endswith("/"):
+        return SITE / decoded / "index.html"
+    p = Path(decoded)
+    return SITE / p / "index.html" if not p.suffix else SITE / p
+
+def wbm_image_url(orig_path, ts):
+    return f"https://web.archive.org/web/{ts}im_/{BASE}{orig_path}"
+
+def wbm_page_url(orig_path, ts, old=False):
+    base = BASE_OLD if old else BASE
+    return f"https://web.archive.org/web/{ts}/{base}{orig_path}"
+
+# ── Image recovery ───────────────────────────────────────────────────────────
+
+def recover_image(orig_path):
+    local = local_path_for(orig_path)
+    if local.exists():
+        return "skip", orig_path
+
+    # 1) Try live site first (fast, no WBM rate limits)
+    live_url = f"{BASE}{orig_path}"
+    data = curl_fetch(live_url, is_image=True)
+    if data:
+        local.parent.mkdir(parents=True, exist_ok=True)
+        local.write_bytes(data)
+        log(f"  ✓ live  {orig_path}")
+        return "ok", orig_path
+
+    # 2) CDX best timestamp
+    best_ts = cdx_best_ts(orig_path)
+    ts_candidates = ([best_ts] if best_ts else []) + [
+        ts for ts in TS_LIST if ts != best_ts
+    ]
+
+    for ts in ts_candidates:
+        url = wbm_image_url(orig_path, ts)
+        data = curl_fetch(url, is_image=True)
+        if data:
+            local.parent.mkdir(parents=True, exist_ok=True)
+            local.write_bytes(data)
+            log(f"  ✓ wbm/{ts[:8]}  {orig_path}")
+            return "ok", orig_path
+        time.sleep(DELAY)
+
+    log(f"  ✗ not found  {orig_path}")
+    return "fail", orig_path
+
+# ── Page recovery ────────────────────────────────────────────────────────────
+
+def orig_path_from_wbm(url):
+    m = re.match(
+        r'https?://web\.archive\.org/web/\d+(?:im_|cs_|js_|oe_|if_|mp_)?/'
+        r'https?://(?:www\.)?lapaella\.net(/[^"\'<>\s#]*)?',
+        url,
+    )
+    return (m.group(1) or "/") if m else None
+
+def rewrite_html(html, local_file):
+    html = re.sub(r'<!-- BEGIN WAYBACK TOOLBAR INSERT -->.*?<!-- END WAYBACK TOOLBAR INSERT -->', '', html, flags=re.DOTALL)
+    html = re.sub(r'<script[^>]*(?:archive\.org|wombat|athena|bundle-playback|ruffle)[^>]*>.*?</script>', '', html, flags=re.DOTALL|re.IGNORECASE)
+    html = re.sub(r'<script[^>]*(?:archive\.org|wombat|athena|bundle-playback|ruffle)[^>]*/?>',   '', html, flags=re.IGNORECASE)
+    html = re.sub(r'<link[^>]*(?:archive\.org|iconochive|banner-styles)[^>]*/?>',                 '', html, flags=re.IGNORECASE)
+    html = re.sub(r'<script[^>]*>\s*__wm\b.*?</script>', '', html, flags=re.DOTALL)
+
+    def _rewrite(url):
+        if "web.archive.org" not in url:
+            return url
+        orig = orig_path_from_wbm(url)
+        if not orig:
+            return "#"
+        rel = os.path.relpath(str(local_path_for(orig)), str(local_file.parent))
+        return rel
+
+    def _attr(m):
+        attr, q, url, q2 = m.group(1), m.group(2), m.group(3), m.group(4)
+        return f'{attr}={q}{_rewrite(url)}{q2}'
+    html = re.sub(r'((?:src|href|action|data-src)=)(["\'])([^"\']+)(["\'])', _attr, html)
+
+    def _css_url(m):
+        url = m.group(1).strip().strip("'\"")
+        return f"url('{_rewrite(url)}')"
+    html = re.sub(r'url\(([^)]+)\)', _css_url, html)
+    return html
+
+def recover_page(orig_path):
+    local = local_path_for(orig_path)
+    if local.exists():
+        return "skip", orig_path
+
+    best_ts = cdx_best_ts(orig_path)
+    ts_candidates = ([best_ts] if best_ts else []) + [
+        ts for ts in TS_LIST if ts != best_ts
+    ]
+
+    for ts in ts_candidates:
+        for old in (False, True):
+            url = wbm_page_url(orig_path, ts, old=old)
+            data = curl_fetch(url)
+            if data:
+                html = data.decode("utf-8", errors="replace")
+                local.parent.mkdir(parents=True, exist_ok=True)
+                html = rewrite_html(html, local)
+                local.write_text(html, encoding="utf-8")
+                log(f"  ✓ page/{ts[:8]}  {orig_path}")
+                return "ok", orig_path
+            time.sleep(DELAY)
+
+    log(f"  ✗ page not found  {orig_path}")
+    return "fail", orig_path
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+
+def collect_missing_images():
+    refs = set()
+    for f in SITE.rglob("*.html"):
+        txt = f.read_text(errors="replace")
+        for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+)', txt):
+            # skip CSS query-string variants and wildcards
+            if "?" in m or "*" in m or not re.search(r'\.(jpe?g|png|gif|webp|svg|ico|bmp)$', m, re.I):
+                continue
+            refs.add(m)
+    missing = [r for r in sorted(refs) if not (SITE / r.lstrip("/")).exists()]
+    return missing
+
+def main():
+    missing_images = collect_missing_images()
+    log(f"\n{'='*60}")
+    log(f"Missing images : {len(missing_images)}")
+    log(f"Missing pages  : {len(MISSING_PAGES)}")
+    log(f"{'='*60}\n")
+
+    img_ok = img_fail = 0
+    pg_ok  = pg_fail  = 0
+
+    # Images — parallel with 4 workers (be polite to WBM)
+    log("── Images ──────────────────────────────────────────────────")
+    with ThreadPoolExecutor(max_workers=4) as ex:
+        futures = {ex.submit(recover_image, p): p for p in missing_images}
+        for fut in as_completed(futures):
+            status, _ = fut.result()
+            if status == "ok":   img_ok   += 1
+            elif status == "fail": img_fail += 1
+
+    # Pages — sequential (each page may cascade more asset fetches)
+    log("\n── Pages ───────────────────────────────────────────────────")
+    for orig_path in MISSING_PAGES:
+        status, _ = recover_page(orig_path)
+        if status == "ok":   pg_ok   += 1
+        elif status == "fail": pg_fail += 1
+        time.sleep(1.0)
+
+    log(f"\n{'='*60}")
+    log(f"Images  recovered: {img_ok}  failed: {img_fail}")
+    log(f"Pages   recovered: {pg_ok}  failed: {pg_fail}")
+    log(f"{'='*60}")
+
+if __name__ == "__main__":
+    main()

lapaella-mirror/recover_assets2.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""
+Phase 2 recovery: fetch RAW WBM HTML pages, extract the actual embedded
+WBM image URLs (which carry the correct serving timestamp), download each.
+"""
+
+import os, re, subprocess, tempfile, time, urllib.parse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
+
+SITE     = Path(__file__).parent / "site"
+UA       = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+DELAY    = 0.3
+lock     = Lock()
+
+# All WBM snapshot URLs to crawl for image references
+WBM_PAGES = [
+    # Primary snapshot
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-valenciana/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-marisco/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-al-horno-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-allipebrat-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-bledes-con-acelgas/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-fesols-i-naps-con-alubias-y-nabos/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-con-pasas-y-garbanzos/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-de-pato-setas-y-trufa-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-semicaldoso-con-brocheta-de-calamarcitos-y-gambas-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-langosta-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-marisco-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-pato-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-verduras-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-morena-2/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/all-i-pebre/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/angulas-al-ajillo/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/clochinas-al-vapor/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/espardenya/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/lubina-a-la-sal/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/pato-al-pebre-picant/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/blog/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/clochina-o-mejillon/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/coca-valenciana/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/fideua-de-pato-boletus-y-puerro/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/paella-de-coliflor-y-bacalao/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-a-banda/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-negro/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/fotos/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/ingredientes/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/recipiente/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/trucos-y-consejos/",
+    # Fallback snapshot for older content
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-valenciana/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/fideua-de-pato-boletus-y-puerro/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
+]
+
+IMG_PAT = re.compile(
+    r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
+    r'https?://(?:www\.)?lapaella\.net'
+    r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))',
+    re.IGNORECASE,
+)
+
+def log(msg):
+    with lock:
+        print(msg, flush=True)
+
+def curl_get(url):
+    tmp = None
+    try:
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
+        tmp.close()
+        r = subprocess.run(
+            ["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
+             "-w", "\n__S__%{http_code}", "-o", tmp.name, url],
+            capture_output=True, text=True, timeout=35,
+        )
+        status = 0
+        for line in r.stdout.splitlines():
+            if line.startswith("__S__"):
+                try: status = int(line[5:].strip())
+                except: pass
+        if status in (404, 403, 410, 0):
+            return None
+        with open(tmp.name, "rb") as f:
+            data = f.read()
+        return data if len(data) > 200 else None
+    except Exception:
+        return None
+    finally:
+        if tmp:
+            try: os.unlink(tmp.name)
+            except: pass
+
+def local_path_for(orig_path):
+    decoded = urllib.parse.unquote(orig_path).lstrip("/")
+    if not decoded or decoded.endswith("/"):
+        return SITE / decoded / "index.html"
+    p = Path(decoded)
+    return SITE / p / "index.html" if not p.suffix else SITE / p
+
+def extract_image_urls(wbm_page_url):
+    """Fetch a raw WBM page and extract all embedded WBM image URLs."""
+    data = curl_get(wbm_page_url)
+    if not data:
+        return {}
+    html = data.decode("utf-8", errors="replace")
+    result = {}
+    for ts, orig_path in IMG_PAT.findall(html):
+        # clean trailing junk (query strings, fragments, extra chars)
+        orig_path = re.split(r'["\'\s&?]', orig_path)[0]
+        wbm_img = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig_path}"
+        result[orig_path] = wbm_img
+    return result
+
+def download_image(orig_path, wbm_url):
+    local = local_path_for(orig_path)
+    if local.exists():
+        return "skip", orig_path
+    data = curl_get(wbm_url)
+    if data:
+        local.parent.mkdir(parents=True, exist_ok=True)
+        local.write_bytes(data)
+        log(f"  ✓  {orig_path}")
+        return "ok", orig_path
+    # Fallback: try without im_ modifier
+    wbm_plain = wbm_url.replace("im_/", "/")
+    data = curl_get(wbm_plain)
+    if data:
+        local.parent.mkdir(parents=True, exist_ok=True)
+        local.write_bytes(data)
+        log(f"  ✓  {orig_path} (plain)")
+        return "ok", orig_path
+    log(f"  ✗  {orig_path}")
+    return "fail", orig_path
+
+def main():
+    print("Phase 1: crawling WBM pages for embedded image URLs...")
+    all_imgs = {}  # orig_path -> wbm_url
+    for page_url in WBM_PAGES:
+        found = extract_image_urls(page_url)
+        new = {k: v for k, v in found.items() if k not in all_imgs}
+        if new:
+            print(f"  {page_url.split('/web/')[1][:50]:50s}  +{len(new)} images")
+        all_imgs.update(found)
+        time.sleep(DELAY)
+
+    # Only bother with images not yet on disk
+    to_fetch = {k: v for k, v in all_imgs.items()
+                if not local_path_for(k).exists()}
+    print(f"\nTotal unique image URLs found : {len(all_imgs)}")
+    print(f"Already on disk              : {len(all_imgs) - len(to_fetch)}")
+    print(f"To download                  : {len(to_fetch)}")
+    print("\nPhase 2: downloading images...")
+
+    ok = fail = 0
+    with ThreadPoolExecutor(max_workers=4) as ex:
+        futures = {ex.submit(download_image, k, v): k
+                   for k, v in to_fetch.items()}
+        for fut in as_completed(futures):
+            status, _ = fut.result()
+            if status == "ok":   ok   += 1
+            elif status == "fail": fail += 1
+
+    print(f"\n{'='*50}")
+    print(f"Recovered: {ok}   Failed: {fail}")
+    print(f"{'='*50}")
+
+if __name__ == "__main__":
+    main()