roberto/Paella
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
Paella/lapaella-mirror/recover_assets2.py
T
robertoandClaude Sonnet 4.6 05cfdc22eb Recover 28 missing images + add phase-2 recovery scripts 
Downloads 28 previously missing images from Wayback Machine using
im_ modifier and CDX-lookup timestamps. Adds recover_assets.py
(CDX-based) and recover_assets2.py (page-crawl-based) for continued
recovery when WBM rate limit lifts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-01 09:40:32 +02:00

184 lines
9.5 KiB
Python
Raw Permalink Blame History

#!/usr/bin/env python3
"""
Phase 2 recovery: fetch RAW WBM HTML pages, extract the actual embedded
WBM image URLs (which carry the correct serving timestamp), download each.
"""
import os, re, subprocess, tempfile, time, urllib.parse
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock
SITE = Path(__file__).parent / "site"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
DELAY = 0.3
lock = Lock()
# All WBM snapshot URLs to crawl for image references
WBM_PAGES = [
# Primary snapshot
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-valenciana/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-marisco/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-verduras/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-al-horno-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-allipebrat-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-bledes-con-acelgas/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-fesols-i-naps-con-alubias-y-nabos/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-con-pasas-y-garbanzos/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-de-pato-setas-y-trufa-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-semicaldoso-con-brocheta-de-calamarcitos-y-gambas-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-langosta-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-marisco-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-pato-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-verduras-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-morena-2/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/all-i-pebre/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/angulas-al-ajillo/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/clochinas-al-vapor/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/espardenya/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/lubina-a-la-sal/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/pato-al-pebre-picant/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/blog/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/clochina-o-mejillon/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/coca-valenciana/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/fideua-de-pato-boletus-y-puerro/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/paella-de-coliflor-y-bacalao/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-a-banda/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-negro/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/fotos/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/ingredientes/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/recipiente/",
"https://web.archive.org/web/20241130065929/https://lapaella.net/trucos-y-consejos/",
# Fallback snapshot for older content
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-valenciana/",
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/",
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/",
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/fideua-de-pato-boletus-y-puerro/",
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
]
IMG_PAT = re.compile(
r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
r'https?://(?:www\.)?lapaella\.net'
r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))',
re.IGNORECASE,
)
def log(msg):
with lock:
print(msg, flush=True)
def curl_get(url):
tmp = None
try:
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
tmp.close()
r = subprocess.run(
["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
"-w", "\n__S__%{http_code}", "-o", tmp.name, url],
capture_output=True, text=True, timeout=35,
)
status = 0
for line in r.stdout.splitlines():
if line.startswith("__S__"):
try: status = int(line[5:].strip())
except: pass
if status in (404, 403, 410, 0):
return None
with open(tmp.name, "rb") as f:
data = f.read()
return data if len(data) > 200 else None
except Exception:
return None
finally:
if tmp:
try: os.unlink(tmp.name)
except: pass
def local_path_for(orig_path):
decoded = urllib.parse.unquote(orig_path).lstrip("/")
if not decoded or decoded.endswith("/"):
return SITE / decoded / "index.html"
p = Path(decoded)
return SITE / p / "index.html" if not p.suffix else SITE / p
def extract_image_urls(wbm_page_url):
"""Fetch a raw WBM page and extract all embedded WBM image URLs."""
data = curl_get(wbm_page_url)
if not data:
return {}
html = data.decode("utf-8", errors="replace")
result = {}
for ts, orig_path in IMG_PAT.findall(html):
# clean trailing junk (query strings, fragments, extra chars)
orig_path = re.split(r'["\'\s&?]', orig_path)[0]
wbm_img = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig_path}"
result[orig_path] = wbm_img
return result
def download_image(orig_path, wbm_url):
local = local_path_for(orig_path)
if local.exists():
return "skip", orig_path
data = curl_get(wbm_url)
if data:
local.parent.mkdir(parents=True, exist_ok=True)
local.write_bytes(data)
log(f" ✓ {orig_path}")
return "ok", orig_path
# Fallback: try without im_ modifier
wbm_plain = wbm_url.replace("im_/", "/")
data = curl_get(wbm_plain)
if data:
local.parent.mkdir(parents=True, exist_ok=True)
local.write_bytes(data)
log(f" ✓ {orig_path} (plain)")
return "ok", orig_path
log(f" ✗ {orig_path}")
return "fail", orig_path
def main():
print("Phase 1: crawling WBM pages for embedded image URLs...")
all_imgs = {} # orig_path -> wbm_url
for page_url in WBM_PAGES:
found = extract_image_urls(page_url)
new = {k: v for k, v in found.items() if k not in all_imgs}
if new:
print(f" {page_url.split('/web/')[1][:50]:50s} +{len(new)} images")
all_imgs.update(found)
time.sleep(DELAY)
# Only bother with images not yet on disk
to_fetch = {k: v for k, v in all_imgs.items()
if not local_path_for(k).exists()}
print(f"\nTotal unique image URLs found : {len(all_imgs)}")
print(f"Already on disk : {len(all_imgs) - len(to_fetch)}")
print(f"To download : {len(to_fetch)}")
print("\nPhase 2: downloading images...")
ok = fail = 0
with ThreadPoolExecutor(max_workers=4) as ex:
futures = {ex.submit(download_image, k, v): k
for k, v in to_fetch.items()}
for fut in as_completed(futures):
status, _ = fut.result()
if status == "ok": ok += 1
elif status == "fail": fail += 1
print(f"\n{'='*50}")
print(f"Recovered: {ok} Failed: {fail}")
print(f"{'='*50}")
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink