#!/usr/bin/env python3 """ Focused recovery of content images only (recipe step photos, food photos). Mines specific WBM page snapshots known to contain these images, then downloads only what's missing. No CDX lookups, no tiny icons. """ import os, re, time, subprocess, tempfile, urllib.parse from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock SITE = Path(__file__).parent / "site" UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" lock = Lock() IMG_PAT = re.compile( r'https://web\.archive\.org/web/(\d+)(?:im_)?/' r'https?://(?:www\.)?lapaella\.net' r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp))', re.IGNORECASE, ) # Pages known to contain the missing content images, with timestamps # that were confirmed to have embedded image URLs in Phase A SOURCE_PAGES = [ # verduras step-by-step (paellanet_verduras_*) "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/", # arroz negro step-by-step (arroz_negro_lapaella_net_*) "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", # paso_arrozabanda + paso_fideua "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/", "https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-arroz-a-banda/", "https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/", "https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/", "https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/", "https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/", # coca valenciana (lapaellanet_coca_*) "https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/", "https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/coca-valenciana/", # paella marisco main photo + mejilones "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-marisco/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/", # clochina / mejillones / fideuadepato-2 "https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/", "https://web.archive.org/web/20211201000000/https://lapaella.net/fideua-de-pato-boletus-y-puerro/", # arroz negro top image + paella bogavante "https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-negro/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/", "https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/", # arroz frutoss secos "https://web.archive.org/web/20211201000000/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/", # nosotros (carlosgomezsenent photo) "https://web.archive.org/web/20211201000000/https://lapaella.net/nosotros/", "https://web.archive.org/web/20220601000000/https://lapaella.net/nosotros/", # old paella photos + home "https://web.archive.org/web/20120601000000/http://www.lapaella.net/", "https://web.archive.org/web/20130601000000/http://www.lapaella.net/", "https://web.archive.org/web/20150601000000/http://www.lapaella.net/", ] def log(msg): with lock: print(msg, flush=True) def curl_get(url, min_size=300): tmp = None try: tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") tmp.close() r = subprocess.run( ["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA, "-w", "\n__S__%{http_code}", "-o", tmp.name, url], capture_output=True, text=True, timeout=35, ) status = 0 for line in r.stdout.splitlines(): if line.startswith("__S__"): try: status = int(line[5:].strip()) except: pass if status in (404, 403, 410, 0): return None with open(tmp.name, "rb") as f: data = f.read() return data if len(data) >= min_size else None except Exception: return None finally: if tmp: try: os.unlink(tmp.name) except: pass def local_path_for(orig_path): decoded = urllib.parse.unquote(orig_path).lstrip("/") p = Path(decoded) return SITE / p / "index.html" if not p.suffix else SITE / p def is_content(path): name = path.split("/")[-1].lower() if re.search(r"-\d{1,2}x\d{1,2}\.", name): return False skip = ["favicon","cropped","banner","slide","italia","china","ingles", "bandera","botonera","slider","maqueta","logo"] return not any(w in name for w in skip) def missing_content_images(): refs = set() for f in SITE.rglob("*.html"): t = f.read_text(errors="replace") for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp))', t, re.I): if "?" not in m and "*" not in m: refs.add(m) return sorted(r for r in refs if not local_path_for(r).exists() and is_content(r)) def mine_page(wbm_url, wanted): data = curl_get(wbm_url, min_size=1000) if not data: return {} html = data.decode("utf-8", errors="replace") found = {} for ts, path in IMG_PAT.findall(html): path = re.split(r'["\'\s&?]', path)[0] if path in wanted and path not in found: found[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}" return found def download(orig_path, wbm_url): local = local_path_for(orig_path) if local.exists(): return "skip" for url in [wbm_url, wbm_url.replace("im_/", "/")]: data = curl_get(url, min_size=1000) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) return "ok" return "fail" def main(): missing = missing_content_images() wanted = set(missing) print(f"Content images to recover: {len(missing)}\n") print("Mining WBM pages for image URLs...") img_map = {} # orig_path -> wbm_url for page in SOURCE_PAGES: found = mine_page(page, wanted) new = {k: v for k, v in found.items() if k not in img_map} if new: label = page.split("/web/")[1][:65] print(f" {label} +{len(new)}") img_map.update(new) time.sleep(0.3) # For anything not found via pages, try brute-force timestamps directly not_found = [p for p in missing if p not in img_map] if not_found: print(f"\n{len(not_found)} images not found via pages, trying direct timestamps...") TIMESTAMPS = [ "20251013025800", "20241130065929", "20241014051856", "20230601000000", "20220601000000", "20211201000000", "20210603053819", "20200601000000", "20190601000000", ] for orig in not_found: for ts in TIMESTAMPS: url = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig}" data = curl_get(url, min_size=1000) if data: local = local_path_for(orig) local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) print(f" ✓ brute/{ts[:8]} {orig}") img_map[orig] = url break time.sleep(0.2) # Download everything found via page-mining to_dl = {k: v for k, v in img_map.items() if not local_path_for(k).exists()} print(f"\nDownloading {len(to_dl)} images...") ok = fail = 0 with ThreadPoolExecutor(max_workers=5) as ex: futures = {ex.submit(download, k, v): k for k, v in to_dl.items()} for fut in as_completed(futures): orig = futures[fut] status = fut.result() if status == "ok": ok += 1 log(f" ✓ {orig}") elif status == "fail": fail += 1 log(f" ✗ {orig}") still = len(missing_content_images()) print(f"\n{'='*55}") print(f"Downloaded: {ok} Failed: {fail}") print(f"Still missing: {still} content images") print(f"{'='*55}") if __name__ == "__main__": main()