#!/usr/bin/env python3 """ Phase 2 recovery: fetch RAW WBM HTML pages, extract the actual embedded WBM image URLs (which carry the correct serving timestamp), download each. """ import os, re, subprocess, tempfile, time, urllib.parse from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock SITE = Path(__file__).parent / "site" UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" DELAY = 0.3 lock = Lock() # All WBM snapshot URLs to crawl for image references WBM_PAGES = [ # Primary snapshot "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-valenciana/", "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-marisco/", "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-verduras/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-al-horno-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-allipebrat-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-bledes-con-acelgas/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-fesols-i-naps-con-alubias-y-nabos/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-con-pasas-y-garbanzos/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-de-pato-setas-y-trufa-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-semicaldoso-con-brocheta-de-calamarcitos-y-gambas-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-langosta-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-marisco-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-pato-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-verduras-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-morena-2/", "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/all-i-pebre/", "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/angulas-al-ajillo/", "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/clochinas-al-vapor/", "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/espardenya/", "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/lubina-a-la-sal/", "https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/pato-al-pebre-picant/", "https://web.archive.org/web/20241130065929/https://lapaella.net/blog/", "https://web.archive.org/web/20241130065929/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/", "https://web.archive.org/web/20241130065929/https://lapaella.net/clochina-o-mejillon/", "https://web.archive.org/web/20241130065929/https://lapaella.net/coca-valenciana/", "https://web.archive.org/web/20241130065929/https://lapaella.net/fideua-de-pato-boletus-y-puerro/", "https://web.archive.org/web/20241130065929/https://lapaella.net/paella-de-coliflor-y-bacalao/", "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-a-banda/", "https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-negro/", "https://web.archive.org/web/20241130065929/https://lapaella.net/", "https://web.archive.org/web/20241130065929/https://lapaella.net/fotos/", "https://web.archive.org/web/20241130065929/https://lapaella.net/ingredientes/", "https://web.archive.org/web/20241130065929/https://lapaella.net/recipiente/", "https://web.archive.org/web/20241130065929/https://lapaella.net/trucos-y-consejos/", # Fallback snapshot for older content "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-valenciana/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/fideua-de-pato-boletus-y-puerro/", "https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/", ] IMG_PAT = re.compile( r'https://web\.archive\.org/web/(\d+)(?:im_)?/' r'https?://(?:www\.)?lapaella\.net' r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))', re.IGNORECASE, ) def log(msg): with lock: print(msg, flush=True) def curl_get(url): tmp = None try: tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") tmp.close() r = subprocess.run( ["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA, "-w", "\n__S__%{http_code}", "-o", tmp.name, url], capture_output=True, text=True, timeout=35, ) status = 0 for line in r.stdout.splitlines(): if line.startswith("__S__"): try: status = int(line[5:].strip()) except: pass if status in (404, 403, 410, 0): return None with open(tmp.name, "rb") as f: data = f.read() return data if len(data) > 200 else None except Exception: return None finally: if tmp: try: os.unlink(tmp.name) except: pass def local_path_for(orig_path): decoded = urllib.parse.unquote(orig_path).lstrip("/") if not decoded or decoded.endswith("/"): return SITE / decoded / "index.html" p = Path(decoded) return SITE / p / "index.html" if not p.suffix else SITE / p def extract_image_urls(wbm_page_url): """Fetch a raw WBM page and extract all embedded WBM image URLs.""" data = curl_get(wbm_page_url) if not data: return {} html = data.decode("utf-8", errors="replace") result = {} for ts, orig_path in IMG_PAT.findall(html): # clean trailing junk (query strings, fragments, extra chars) orig_path = re.split(r'["\'\s&?]', orig_path)[0] wbm_img = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig_path}" result[orig_path] = wbm_img return result def download_image(orig_path, wbm_url): local = local_path_for(orig_path) if local.exists(): return "skip", orig_path data = curl_get(wbm_url) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) log(f" ✓ {orig_path}") return "ok", orig_path # Fallback: try without im_ modifier wbm_plain = wbm_url.replace("im_/", "/") data = curl_get(wbm_plain) if data: local.parent.mkdir(parents=True, exist_ok=True) local.write_bytes(data) log(f" ✓ {orig_path} (plain)") return "ok", orig_path log(f" ✗ {orig_path}") return "fail", orig_path def main(): print("Phase 1: crawling WBM pages for embedded image URLs...") all_imgs = {} # orig_path -> wbm_url for page_url in WBM_PAGES: found = extract_image_urls(page_url) new = {k: v for k, v in found.items() if k not in all_imgs} if new: print(f" {page_url.split('/web/')[1][:50]:50s} +{len(new)} images") all_imgs.update(found) time.sleep(DELAY) # Only bother with images not yet on disk to_fetch = {k: v for k, v in all_imgs.items() if not local_path_for(k).exists()} print(f"\nTotal unique image URLs found : {len(all_imgs)}") print(f"Already on disk : {len(all_imgs) - len(to_fetch)}") print(f"To download : {len(to_fetch)}") print("\nPhase 2: downloading images...") ok = fail = 0 with ThreadPoolExecutor(max_workers=4) as ex: futures = {ex.submit(download_image, k, v): k for k, v in to_fetch.items()} for fut in as_completed(futures): status, _ = fut.result() if status == "ok": ok += 1 elif status == "fail": fail += 1 print(f"\n{'='*50}") print(f"Recovered: {ok} Failed: {fail}") print(f"{'='*50}") if __name__ == "__main__": main()