robertoandClaude Sonnet 4.6
dcd980988d
Downloads images from multiple Wayback Machine snapshot dates using page-crawl extraction (im_ modifier). Adds recover_assets3.py and recover_content.py for targeted multi-date recovery. Remaining ~107 content images (2021 step-by-step recipe photos for verduras, arroz negro, arrozabanda, fideua) are confirmed absent from all web archives — pages were crawled but images were lazy-loaded. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
207 lines
9.1 KiB
Python
207 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Focused recovery of content images only (recipe step photos, food photos).
|
|
Mines specific WBM page snapshots known to contain these images,
|
|
then downloads only what's missing. No CDX lookups, no tiny icons.
|
|
"""
|
|
|
|
import os, re, time, subprocess, tempfile, urllib.parse
|
|
from pathlib import Path
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from threading import Lock
|
|
|
|
SITE = Path(__file__).parent / "site"
|
|
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
|
lock = Lock()
|
|
|
|
IMG_PAT = re.compile(
|
|
r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
|
|
r'https?://(?:www\.)?lapaella\.net'
|
|
r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp))',
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
# Pages known to contain the missing content images, with timestamps
|
|
# that were confirmed to have embedded image URLs in Phase A
|
|
SOURCE_PAGES = [
|
|
# verduras step-by-step (paellanet_verduras_*)
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/",
|
|
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/",
|
|
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/",
|
|
# arroz negro step-by-step (arroz_negro_lapaella_net_*)
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
|
"https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
|
"https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
|
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
|
# paso_arrozabanda + paso_fideua
|
|
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
|
"https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
|
"https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
|
"https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
|
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/",
|
|
"https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/",
|
|
# coca valenciana (lapaellanet_coca_*)
|
|
"https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/",
|
|
"https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/",
|
|
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/coca-valenciana/",
|
|
# paella marisco main photo + mejilones
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-marisco/",
|
|
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/",
|
|
# clochina / mejillones / fideuadepato-2
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/",
|
|
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/fideua-de-pato-boletus-y-puerro/",
|
|
# arroz negro top image + paella bogavante
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-negro/",
|
|
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
|
# arroz frutoss secos
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/",
|
|
# nosotros (carlosgomezsenent photo)
|
|
"https://web.archive.org/web/20211201000000/https://lapaella.net/nosotros/",
|
|
"https://web.archive.org/web/20220601000000/https://lapaella.net/nosotros/",
|
|
# old paella photos + home
|
|
"https://web.archive.org/web/20120601000000/http://www.lapaella.net/",
|
|
"https://web.archive.org/web/20130601000000/http://www.lapaella.net/",
|
|
"https://web.archive.org/web/20150601000000/http://www.lapaella.net/",
|
|
]
|
|
|
|
def log(msg):
|
|
with lock:
|
|
print(msg, flush=True)
|
|
|
|
def curl_get(url, min_size=300):
|
|
tmp = None
|
|
try:
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
|
|
tmp.close()
|
|
r = subprocess.run(
|
|
["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
|
|
"-w", "\n__S__%{http_code}", "-o", tmp.name, url],
|
|
capture_output=True, text=True, timeout=35,
|
|
)
|
|
status = 0
|
|
for line in r.stdout.splitlines():
|
|
if line.startswith("__S__"):
|
|
try: status = int(line[5:].strip())
|
|
except: pass
|
|
if status in (404, 403, 410, 0):
|
|
return None
|
|
with open(tmp.name, "rb") as f:
|
|
data = f.read()
|
|
return data if len(data) >= min_size else None
|
|
except Exception:
|
|
return None
|
|
finally:
|
|
if tmp:
|
|
try: os.unlink(tmp.name)
|
|
except: pass
|
|
|
|
def local_path_for(orig_path):
|
|
decoded = urllib.parse.unquote(orig_path).lstrip("/")
|
|
p = Path(decoded)
|
|
return SITE / p / "index.html" if not p.suffix else SITE / p
|
|
|
|
def is_content(path):
|
|
name = path.split("/")[-1].lower()
|
|
if re.search(r"-\d{1,2}x\d{1,2}\.", name): return False
|
|
skip = ["favicon","cropped","banner","slide","italia","china","ingles",
|
|
"bandera","botonera","slider","maqueta","logo"]
|
|
return not any(w in name for w in skip)
|
|
|
|
def missing_content_images():
|
|
refs = set()
|
|
for f in SITE.rglob("*.html"):
|
|
t = f.read_text(errors="replace")
|
|
for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp))', t, re.I):
|
|
if "?" not in m and "*" not in m:
|
|
refs.add(m)
|
|
return sorted(r for r in refs if not local_path_for(r).exists() and is_content(r))
|
|
|
|
def mine_page(wbm_url, wanted):
|
|
data = curl_get(wbm_url, min_size=1000)
|
|
if not data:
|
|
return {}
|
|
html = data.decode("utf-8", errors="replace")
|
|
found = {}
|
|
for ts, path in IMG_PAT.findall(html):
|
|
path = re.split(r'["\'\s&?]', path)[0]
|
|
if path in wanted and path not in found:
|
|
found[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}"
|
|
return found
|
|
|
|
def download(orig_path, wbm_url):
|
|
local = local_path_for(orig_path)
|
|
if local.exists():
|
|
return "skip"
|
|
for url in [wbm_url, wbm_url.replace("im_/", "/")]:
|
|
data = curl_get(url, min_size=1000)
|
|
if data:
|
|
local.parent.mkdir(parents=True, exist_ok=True)
|
|
local.write_bytes(data)
|
|
return "ok"
|
|
return "fail"
|
|
|
|
def main():
|
|
missing = missing_content_images()
|
|
wanted = set(missing)
|
|
print(f"Content images to recover: {len(missing)}\n")
|
|
|
|
print("Mining WBM pages for image URLs...")
|
|
img_map = {} # orig_path -> wbm_url
|
|
for page in SOURCE_PAGES:
|
|
found = mine_page(page, wanted)
|
|
new = {k: v for k, v in found.items() if k not in img_map}
|
|
if new:
|
|
label = page.split("/web/")[1][:65]
|
|
print(f" {label} +{len(new)}")
|
|
img_map.update(new)
|
|
time.sleep(0.3)
|
|
|
|
# For anything not found via pages, try brute-force timestamps directly
|
|
not_found = [p for p in missing if p not in img_map]
|
|
if not_found:
|
|
print(f"\n{len(not_found)} images not found via pages, trying direct timestamps...")
|
|
TIMESTAMPS = [
|
|
"20251013025800", "20241130065929", "20241014051856",
|
|
"20230601000000", "20220601000000", "20211201000000",
|
|
"20210603053819", "20200601000000", "20190601000000",
|
|
]
|
|
for orig in not_found:
|
|
for ts in TIMESTAMPS:
|
|
url = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig}"
|
|
data = curl_get(url, min_size=1000)
|
|
if data:
|
|
local = local_path_for(orig)
|
|
local.parent.mkdir(parents=True, exist_ok=True)
|
|
local.write_bytes(data)
|
|
print(f" ✓ brute/{ts[:8]} {orig}")
|
|
img_map[orig] = url
|
|
break
|
|
time.sleep(0.2)
|
|
|
|
# Download everything found via page-mining
|
|
to_dl = {k: v for k, v in img_map.items() if not local_path_for(k).exists()}
|
|
print(f"\nDownloading {len(to_dl)} images...")
|
|
ok = fail = 0
|
|
with ThreadPoolExecutor(max_workers=5) as ex:
|
|
futures = {ex.submit(download, k, v): k for k, v in to_dl.items()}
|
|
for fut in as_completed(futures):
|
|
orig = futures[fut]
|
|
status = fut.result()
|
|
if status == "ok":
|
|
ok += 1
|
|
log(f" ✓ {orig}")
|
|
elif status == "fail":
|
|
fail += 1
|
|
log(f" ✗ {orig}")
|
|
|
|
still = len(missing_content_images())
|
|
print(f"\n{'='*55}")
|
|
print(f"Downloaded: {ok} Failed: {fail}")
|
|
print(f"Still missing: {still} content images")
|
|
print(f"{'='*55}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|