Recover 106 additional images + missing English page
Downloads images from multiple Wayback Machine snapshot dates using page-crawl extraction (im_ modifier). Adds recover_assets3.py and recover_content.py for targeted multi-date recovery. Remaining ~107 content images (2021 step-by-step recipe photos for verduras, arroz negro, arrozabanda, fideua) are confirmed absent from all web archives — pages were crawled but images were lazy-loaded. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Focused recovery of content images only (recipe step photos, food photos).
|
||||
Mines specific WBM page snapshots known to contain these images,
|
||||
then downloads only what's missing. No CDX lookups, no tiny icons.
|
||||
"""
|
||||
|
||||
import os, re, time, subprocess, tempfile, urllib.parse
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from threading import Lock
|
||||
|
||||
SITE = Path(__file__).parent / "site"
|
||||
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
lock = Lock()
|
||||
|
||||
IMG_PAT = re.compile(
|
||||
r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
|
||||
r'https?://(?:www\.)?lapaella\.net'
|
||||
r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp))',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Pages known to contain the missing content images, with timestamps
|
||||
# that were confirmed to have embedded image URLs in Phase A
|
||||
SOURCE_PAGES = [
|
||||
# verduras step-by-step (paellanet_verduras_*)
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/",
|
||||
# arroz negro step-by-step (arroz_negro_lapaella_net_*)
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
# paso_arrozabanda + paso_fideua
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||||
"https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||||
"https://web.archive.org/web/20230101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/",
|
||||
"https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/",
|
||||
# coca valenciana (lapaellanet_coca_*)
|
||||
"https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/",
|
||||
"https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/coca-valenciana/",
|
||||
# paella marisco main photo + mejilones
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-marisco/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/",
|
||||
# clochina / mejillones / fideuadepato-2
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/fideua-de-pato-boletus-y-puerro/",
|
||||
# arroz negro top image + paella bogavante
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-negro/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||||
# arroz frutoss secos
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/",
|
||||
# nosotros (carlosgomezsenent photo)
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/nosotros/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/nosotros/",
|
||||
# old paella photos + home
|
||||
"https://web.archive.org/web/20120601000000/http://www.lapaella.net/",
|
||||
"https://web.archive.org/web/20130601000000/http://www.lapaella.net/",
|
||||
"https://web.archive.org/web/20150601000000/http://www.lapaella.net/",
|
||||
]
|
||||
|
||||
def log(msg):
|
||||
with lock:
|
||||
print(msg, flush=True)
|
||||
|
||||
def curl_get(url, min_size=300):
|
||||
tmp = None
|
||||
try:
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
|
||||
tmp.close()
|
||||
r = subprocess.run(
|
||||
["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
|
||||
"-w", "\n__S__%{http_code}", "-o", tmp.name, url],
|
||||
capture_output=True, text=True, timeout=35,
|
||||
)
|
||||
status = 0
|
||||
for line in r.stdout.splitlines():
|
||||
if line.startswith("__S__"):
|
||||
try: status = int(line[5:].strip())
|
||||
except: pass
|
||||
if status in (404, 403, 410, 0):
|
||||
return None
|
||||
with open(tmp.name, "rb") as f:
|
||||
data = f.read()
|
||||
return data if len(data) >= min_size else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if tmp:
|
||||
try: os.unlink(tmp.name)
|
||||
except: pass
|
||||
|
||||
def local_path_for(orig_path):
|
||||
decoded = urllib.parse.unquote(orig_path).lstrip("/")
|
||||
p = Path(decoded)
|
||||
return SITE / p / "index.html" if not p.suffix else SITE / p
|
||||
|
||||
def is_content(path):
|
||||
name = path.split("/")[-1].lower()
|
||||
if re.search(r"-\d{1,2}x\d{1,2}\.", name): return False
|
||||
skip = ["favicon","cropped","banner","slide","italia","china","ingles",
|
||||
"bandera","botonera","slider","maqueta","logo"]
|
||||
return not any(w in name for w in skip)
|
||||
|
||||
def missing_content_images():
|
||||
refs = set()
|
||||
for f in SITE.rglob("*.html"):
|
||||
t = f.read_text(errors="replace")
|
||||
for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp))', t, re.I):
|
||||
if "?" not in m and "*" not in m:
|
||||
refs.add(m)
|
||||
return sorted(r for r in refs if not local_path_for(r).exists() and is_content(r))
|
||||
|
||||
def mine_page(wbm_url, wanted):
|
||||
data = curl_get(wbm_url, min_size=1000)
|
||||
if not data:
|
||||
return {}
|
||||
html = data.decode("utf-8", errors="replace")
|
||||
found = {}
|
||||
for ts, path in IMG_PAT.findall(html):
|
||||
path = re.split(r'["\'\s&?]', path)[0]
|
||||
if path in wanted and path not in found:
|
||||
found[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}"
|
||||
return found
|
||||
|
||||
def download(orig_path, wbm_url):
|
||||
local = local_path_for(orig_path)
|
||||
if local.exists():
|
||||
return "skip"
|
||||
for url in [wbm_url, wbm_url.replace("im_/", "/")]:
|
||||
data = curl_get(url, min_size=1000)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
return "ok"
|
||||
return "fail"
|
||||
|
||||
def main():
|
||||
missing = missing_content_images()
|
||||
wanted = set(missing)
|
||||
print(f"Content images to recover: {len(missing)}\n")
|
||||
|
||||
print("Mining WBM pages for image URLs...")
|
||||
img_map = {} # orig_path -> wbm_url
|
||||
for page in SOURCE_PAGES:
|
||||
found = mine_page(page, wanted)
|
||||
new = {k: v for k, v in found.items() if k not in img_map}
|
||||
if new:
|
||||
label = page.split("/web/")[1][:65]
|
||||
print(f" {label} +{len(new)}")
|
||||
img_map.update(new)
|
||||
time.sleep(0.3)
|
||||
|
||||
# For anything not found via pages, try brute-force timestamps directly
|
||||
not_found = [p for p in missing if p not in img_map]
|
||||
if not_found:
|
||||
print(f"\n{len(not_found)} images not found via pages, trying direct timestamps...")
|
||||
TIMESTAMPS = [
|
||||
"20251013025800", "20241130065929", "20241014051856",
|
||||
"20230601000000", "20220601000000", "20211201000000",
|
||||
"20210603053819", "20200601000000", "20190601000000",
|
||||
]
|
||||
for orig in not_found:
|
||||
for ts in TIMESTAMPS:
|
||||
url = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig}"
|
||||
data = curl_get(url, min_size=1000)
|
||||
if data:
|
||||
local = local_path_for(orig)
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
print(f" ✓ brute/{ts[:8]} {orig}")
|
||||
img_map[orig] = url
|
||||
break
|
||||
time.sleep(0.2)
|
||||
|
||||
# Download everything found via page-mining
|
||||
to_dl = {k: v for k, v in img_map.items() if not local_path_for(k).exists()}
|
||||
print(f"\nDownloading {len(to_dl)} images...")
|
||||
ok = fail = 0
|
||||
with ThreadPoolExecutor(max_workers=5) as ex:
|
||||
futures = {ex.submit(download, k, v): k for k, v in to_dl.items()}
|
||||
for fut in as_completed(futures):
|
||||
orig = futures[fut]
|
||||
status = fut.result()
|
||||
if status == "ok":
|
||||
ok += 1
|
||||
log(f" ✓ {orig}")
|
||||
elif status == "fail":
|
||||
fail += 1
|
||||
log(f" ✗ {orig}")
|
||||
|
||||
still = len(missing_content_images())
|
||||
print(f"\n{'='*55}")
|
||||
print(f"Downloaded: {ok} Failed: {fail}")
|
||||
print(f"Still missing: {still} content images")
|
||||
print(f"{'='*55}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user