Recover 106 additional images + missing English page
Downloads images from multiple Wayback Machine snapshot dates using page-crawl extraction (im_ modifier). Adds recover_assets3.py and recover_content.py for targeted multi-date recovery. Remaining ~107 content images (2021 step-by-step recipe photos for verduras, arroz negro, arrozabanda, fideua) are confirmed absent from all web archives — pages were crawled but images were lazy-loaded. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 3: For each remaining missing image:
|
||||
1. Query CDX for every timestamp at which that exact URL was captured
|
||||
2. Try each timestamp with im_ modifier
|
||||
3. Also crawl additional WBM page snapshots from the period images were
|
||||
published (e.g. 2021/03 verduras images → fetch verduras page from 2021)
|
||||
"""
|
||||
|
||||
import os, re, json, time, subprocess, tempfile, urllib.parse
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from threading import Lock
|
||||
|
||||
SITE = Path(__file__).parent / "site"
|
||||
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
DELAY = 0.2
|
||||
lock = Lock()
|
||||
|
||||
IMG_PAT = re.compile(
|
||||
r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
|
||||
r'https?://(?:www\.)?lapaella\.net'
|
||||
r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Extra WBM page snapshots to mine — specifically from around the publication
|
||||
# date of each image batch we're still missing
|
||||
EXTRA_WBM_PAGES = [
|
||||
# paellanet_verduras_* and arroz_negro images published ~2021/03
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20211001000000*/https://lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
# paso_fideua and paso_arrozabanda published ~2021/09
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||||
"https://web.archive.org/web/20220101000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-arroz-a-banda/",
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||||
"https://web.archive.org/web/20220101000000/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||||
# receta-de-fideua (missing page itself – try multiple dates)
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/receta-de-fideua/",
|
||||
"https://web.archive.org/web/20220601000000/https://lapaella.net/receta-de-fideua/",
|
||||
"https://web.archive.org/web/20230101000000/https://lapaella.net/receta-de-fideua/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-fideua/",
|
||||
# coca + IMG-WA images – published ~2020/05
|
||||
"https://web.archive.org/web/20200901000000/https://lapaella.net/coca-valenciana/",
|
||||
"https://web.archive.org/web/20201201000000/https://lapaella.net/coca-valenciana/",
|
||||
"https://web.archive.org/web/20210201000000/https://lapaella.net/coca-valenciana/",
|
||||
# mejilones / fideuadepato-2 – clochina page
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
|
||||
"https://web.archive.org/web/20211201000000/https://lapaella.net/clochina-o-mejillon/",
|
||||
# banner / slide images – home page older snapshots
|
||||
"https://web.archive.org/web/20201201000000/https://lapaella.net/",
|
||||
"https://web.archive.org/web/20210201000000/https://lapaella.net/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/",
|
||||
# paella bogavante / arroz negro (old)
|
||||
"https://web.archive.org/web/20200901000000/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20210201000000/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||||
# paella marisco (old flag icons)
|
||||
"https://web.archive.org/web/20130101000000/http://www.lapaella.net/receta-paella-de-marisco/",
|
||||
"https://web.archive.org/web/20150101000000/http://www.lapaella.net/receta-paella-de-marisco/",
|
||||
"https://web.archive.org/web/20120601000000/http://www.lapaella.net/receta-paella-valenciana/",
|
||||
"https://web.archive.org/web/20130601000000/http://www.lapaella.net/receta-paella-valenciana/",
|
||||
]
|
||||
|
||||
def log(msg):
|
||||
with lock:
|
||||
print(msg, flush=True)
|
||||
|
||||
def curl_get(url, min_size=200):
|
||||
tmp = None
|
||||
try:
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
|
||||
tmp.close()
|
||||
r = subprocess.run(
|
||||
["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
|
||||
"-w", "\n__S__%{http_code}", "-o", tmp.name, url],
|
||||
capture_output=True, text=True, timeout=35,
|
||||
)
|
||||
status = 0
|
||||
for line in r.stdout.splitlines():
|
||||
if line.startswith("__S__"):
|
||||
try: status = int(line[5:].strip())
|
||||
except: pass
|
||||
if status in (404, 403, 410, 0):
|
||||
return None
|
||||
with open(tmp.name, "rb") as f:
|
||||
data = f.read()
|
||||
return data if len(data) >= min_size else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if tmp:
|
||||
try: os.unlink(tmp.name)
|
||||
except: pass
|
||||
|
||||
def cdx_timestamps(orig_path):
|
||||
"""Return all CDX-indexed timestamps for a specific image URL."""
|
||||
for base in ("lapaella.net", "www.lapaella.net"):
|
||||
cdx = (
|
||||
"https://web.archive.org/cdx/search/cdx"
|
||||
f"?url={base}{urllib.parse.quote(orig_path)}"
|
||||
"&output=json&fl=timestamp&filter=statuscode:200&limit=20"
|
||||
)
|
||||
data = curl_get(cdx, min_size=5)
|
||||
if data:
|
||||
try:
|
||||
rows = json.loads(data.decode())
|
||||
ts = [r[0] for r in rows[1:]]
|
||||
if ts:
|
||||
return ts
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(DELAY)
|
||||
return []
|
||||
|
||||
def local_path_for(orig_path):
|
||||
decoded = urllib.parse.unquote(orig_path).lstrip("/")
|
||||
if not decoded or decoded.endswith("/"):
|
||||
return SITE / decoded / "index.html"
|
||||
p = Path(decoded)
|
||||
return SITE / p / "index.html" if not p.suffix else SITE / p
|
||||
|
||||
def try_download(orig_path, timestamps):
|
||||
"""Try all timestamps × (im_ / plain) for an image."""
|
||||
local = local_path_for(orig_path)
|
||||
for ts in timestamps:
|
||||
for modifier in ("im_", ""):
|
||||
url = f"https://web.archive.org/web/{ts}{modifier}/https://lapaella.net{orig_path}"
|
||||
data = curl_get(url, min_size=500)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
return ts, modifier
|
||||
time.sleep(DELAY)
|
||||
# Also try www variant
|
||||
url = f"https://web.archive.org/web/{ts}im_/http://www.lapaella.net{orig_path}"
|
||||
data = curl_get(url, min_size=500)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
return ts, "www+im_"
|
||||
time.sleep(DELAY)
|
||||
return None, None
|
||||
|
||||
def extract_images_from_page(wbm_url):
|
||||
data = curl_get(wbm_url)
|
||||
if not data:
|
||||
return {}
|
||||
html = data.decode("utf-8", errors="replace")
|
||||
result = {}
|
||||
for ts, path in IMG_PAT.findall(html):
|
||||
path = re.split(r'["\'\s&?]', path)[0]
|
||||
result[path] = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{path}"
|
||||
return result
|
||||
|
||||
def missing_images():
|
||||
refs = set()
|
||||
for f in SITE.rglob("*.html"):
|
||||
t = f.read_text(errors="replace")
|
||||
for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+\.(?:jpe?g|png|gif|webp|svg))', t, re.I):
|
||||
if '?' not in m and '*' not in m:
|
||||
refs.add(m)
|
||||
return sorted(r for r in refs if not local_path_for(r).exists())
|
||||
|
||||
def main():
|
||||
missing = missing_images()
|
||||
print(f"Missing images: {len(missing)}\n")
|
||||
|
||||
# ── Phase A: mine extra WBM page snapshots ───────────────────────────────
|
||||
print("Phase A: mining extra WBM page snapshots...")
|
||||
page_found = {}
|
||||
for page_url in EXTRA_WBM_PAGES:
|
||||
found = extract_images_from_page(page_url)
|
||||
new = {k: v for k, v in found.items()
|
||||
if k in missing and k not in page_found}
|
||||
if new:
|
||||
label = page_url.split('/web/')[1][:60]
|
||||
print(f" {label} +{len(new)}")
|
||||
page_found.update({k: v for k, v in found.items() if k in missing})
|
||||
time.sleep(0.5)
|
||||
|
||||
print(f" Found WBM URLs for {len(page_found)} of {len(missing)} missing images\n")
|
||||
|
||||
# ── Phase B: CDX lookup for images NOT found via pages ───────────────────
|
||||
still_missing = [p for p in missing if p not in page_found]
|
||||
print(f"Phase B: CDX timestamp search for {len(still_missing)} remaining images...")
|
||||
cdx_found = {}
|
||||
for orig in still_missing:
|
||||
ts_list = cdx_timestamps(orig)
|
||||
if ts_list:
|
||||
cdx_found[orig] = ts_list
|
||||
print(f" CDX hit: {orig} ({ts_list[0][:8]}…)")
|
||||
time.sleep(DELAY)
|
||||
|
||||
# ── Phase C: download everything ─────────────────────────────────────────
|
||||
print(f"\nPhase C: downloading...")
|
||||
|
||||
def do_download(orig):
|
||||
if local_path_for(orig).exists():
|
||||
return "skip", orig
|
||||
# Try page-discovered URL first (exact timestamp known to work)
|
||||
if orig in page_found:
|
||||
wbm_url = page_found[orig]
|
||||
ts = re.search(r'/web/(\d+)', wbm_url).group(1)
|
||||
found_ts, mod = try_download(orig, [ts])
|
||||
if found_ts:
|
||||
log(f" ✓ page/{found_ts[:8]} {orig}")
|
||||
return "ok", orig
|
||||
# Try CDX timestamps
|
||||
if orig in cdx_found:
|
||||
found_ts, mod = try_download(orig, cdx_found[orig])
|
||||
if found_ts:
|
||||
log(f" ✓ cdx/{found_ts[:8]} {orig}")
|
||||
return "ok", orig
|
||||
# Brute-force: try a broad range of timestamps
|
||||
broad = [
|
||||
"20251013025800", "20241130065929", "20241014051856",
|
||||
"20230601000000", "20220601000000", "20210603053819",
|
||||
"20200601000000", "20190601000000", "20180101000000",
|
||||
"20160101000000", "20140101000000", "20120601000000",
|
||||
]
|
||||
found_ts, mod = try_download(orig, broad)
|
||||
if found_ts:
|
||||
log(f" ✓ brute/{found_ts[:8]} {orig}")
|
||||
return "ok", orig
|
||||
log(f" ✗ {orig}")
|
||||
return "fail", orig
|
||||
|
||||
ok = fail = skip = 0
|
||||
with ThreadPoolExecutor(max_workers=3) as ex:
|
||||
futures = {ex.submit(do_download, p): p for p in missing}
|
||||
for fut in as_completed(futures):
|
||||
status, _ = fut.result()
|
||||
if status == "ok": ok += 1
|
||||
elif status == "fail": fail += 1
|
||||
else: skip += 1
|
||||
|
||||
print(f"\n{'='*55}")
|
||||
print(f"Recovered: {ok} Failed: {fail} Skipped: {skip}")
|
||||
print(f"{'='*55}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user