Recover 28 missing images + add phase-2 recovery scripts
Downloads 28 previously missing images from Wayback Machine using im_ modifier and CDX-lookup timestamps. Adds recover_assets.py (CDX-based) and recover_assets2.py (page-crawl-based) for continued recovery when WBM rate limit lifts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 2 recovery: fetch RAW WBM HTML pages, extract the actual embedded
|
||||
WBM image URLs (which carry the correct serving timestamp), download each.
|
||||
"""
|
||||
|
||||
import os, re, subprocess, tempfile, time, urllib.parse
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from threading import Lock
|
||||
|
||||
SITE = Path(__file__).parent / "site"
|
||||
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
DELAY = 0.3
|
||||
lock = Lock()
|
||||
|
||||
# All WBM snapshot URLs to crawl for image references
|
||||
WBM_PAGES = [
|
||||
# Primary snapshot
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-valenciana/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-marisco/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-paella-de-verduras/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-al-horno-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-negro/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-allipebrat-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-bledes-con-acelgas/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-amb-fesols-i-naps-con-alubias-y-nabos/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-con-pasas-y-garbanzos/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-de-pato-setas-y-trufa-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/arroz-meloso-semicaldoso-con-brocheta-de-calamarcitos-y-gambas-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-de-marisco/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/fideua-negreta/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-bogavante-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-langosta-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-marisco-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-pato-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-de-verduras-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroces-tipicos-de-valencia/paella-morena-2/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/all-i-pebre/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/angulas-al-ajillo/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/clochinas-al-vapor/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/espardenya/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/lubina-a-la-sal/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/otras-recetas-valencianas/pato-al-pebre-picant/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/blog/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/arroz-de-frutos-secos-garbanzos-y-nisperos/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/clochina-o-mejillon/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/coca-valenciana/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/fideua-de-pato-boletus-y-puerro/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/paella-de-coliflor-y-bacalao/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-a-banda/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/receta-de-arroz-negro/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/fotos/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/ingredientes/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/recipiente/",
|
||||
"https://web.archive.org/web/20241130065929/https://lapaella.net/trucos-y-consejos/",
|
||||
# Fallback snapshot for older content
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-valenciana/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/receta-paella-de-marisco/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/arroces-tipicos-de-valencia/arroz-a-banda/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/fideua-de-pato-boletus-y-puerro/",
|
||||
"https://web.archive.org/web/20210603053819/http://www.lapaella.net/clochina-o-mejillon/",
|
||||
]
|
||||
|
||||
IMG_PAT = re.compile(
|
||||
r'https://web\.archive\.org/web/(\d+)(?:im_)?/'
|
||||
r'https?://(?:www\.)?lapaella\.net'
|
||||
r'(/wp-content/uploads/[^\s"\'<>)]+\.(?:jpe?g|jpeg|png|gif|webp|svg))',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def log(msg):
|
||||
with lock:
|
||||
print(msg, flush=True)
|
||||
|
||||
def curl_get(url):
|
||||
tmp = None
|
||||
try:
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
|
||||
tmp.close()
|
||||
r = subprocess.run(
|
||||
["/usr/bin/curl", "-s", "-L", "--max-time", "25", "-A", UA,
|
||||
"-w", "\n__S__%{http_code}", "-o", tmp.name, url],
|
||||
capture_output=True, text=True, timeout=35,
|
||||
)
|
||||
status = 0
|
||||
for line in r.stdout.splitlines():
|
||||
if line.startswith("__S__"):
|
||||
try: status = int(line[5:].strip())
|
||||
except: pass
|
||||
if status in (404, 403, 410, 0):
|
||||
return None
|
||||
with open(tmp.name, "rb") as f:
|
||||
data = f.read()
|
||||
return data if len(data) > 200 else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if tmp:
|
||||
try: os.unlink(tmp.name)
|
||||
except: pass
|
||||
|
||||
def local_path_for(orig_path):
|
||||
decoded = urllib.parse.unquote(orig_path).lstrip("/")
|
||||
if not decoded or decoded.endswith("/"):
|
||||
return SITE / decoded / "index.html"
|
||||
p = Path(decoded)
|
||||
return SITE / p / "index.html" if not p.suffix else SITE / p
|
||||
|
||||
def extract_image_urls(wbm_page_url):
|
||||
"""Fetch a raw WBM page and extract all embedded WBM image URLs."""
|
||||
data = curl_get(wbm_page_url)
|
||||
if not data:
|
||||
return {}
|
||||
html = data.decode("utf-8", errors="replace")
|
||||
result = {}
|
||||
for ts, orig_path in IMG_PAT.findall(html):
|
||||
# clean trailing junk (query strings, fragments, extra chars)
|
||||
orig_path = re.split(r'["\'\s&?]', orig_path)[0]
|
||||
wbm_img = f"https://web.archive.org/web/{ts}im_/https://lapaella.net{orig_path}"
|
||||
result[orig_path] = wbm_img
|
||||
return result
|
||||
|
||||
def download_image(orig_path, wbm_url):
|
||||
local = local_path_for(orig_path)
|
||||
if local.exists():
|
||||
return "skip", orig_path
|
||||
data = curl_get(wbm_url)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
log(f" ✓ {orig_path}")
|
||||
return "ok", orig_path
|
||||
# Fallback: try without im_ modifier
|
||||
wbm_plain = wbm_url.replace("im_/", "/")
|
||||
data = curl_get(wbm_plain)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
log(f" ✓ {orig_path} (plain)")
|
||||
return "ok", orig_path
|
||||
log(f" ✗ {orig_path}")
|
||||
return "fail", orig_path
|
||||
|
||||
def main():
|
||||
print("Phase 1: crawling WBM pages for embedded image URLs...")
|
||||
all_imgs = {} # orig_path -> wbm_url
|
||||
for page_url in WBM_PAGES:
|
||||
found = extract_image_urls(page_url)
|
||||
new = {k: v for k, v in found.items() if k not in all_imgs}
|
||||
if new:
|
||||
print(f" {page_url.split('/web/')[1][:50]:50s} +{len(new)} images")
|
||||
all_imgs.update(found)
|
||||
time.sleep(DELAY)
|
||||
|
||||
# Only bother with images not yet on disk
|
||||
to_fetch = {k: v for k, v in all_imgs.items()
|
||||
if not local_path_for(k).exists()}
|
||||
print(f"\nTotal unique image URLs found : {len(all_imgs)}")
|
||||
print(f"Already on disk : {len(all_imgs) - len(to_fetch)}")
|
||||
print(f"To download : {len(to_fetch)}")
|
||||
print("\nPhase 2: downloading images...")
|
||||
|
||||
ok = fail = 0
|
||||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||||
futures = {ex.submit(download_image, k, v): k
|
||||
for k, v in to_fetch.items()}
|
||||
for fut in as_completed(futures):
|
||||
status, _ = fut.result()
|
||||
if status == "ok": ok += 1
|
||||
elif status == "fail": fail += 1
|
||||
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Recovered: {ok} Failed: {fail}")
|
||||
print(f"{'='*50}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user