Recover 28 missing images + add phase-2 recovery scripts
Downloads 28 previously missing images from Wayback Machine using im_ modifier and CDX-lookup timestamps. Adds recover_assets.py (CDX-based) and recover_assets2.py (page-crawl-based) for continued recovery when WBM rate limit lifts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Recover missing images and pages from lapaella.net mirror.
|
||||
|
||||
Strategy:
|
||||
1. Scan all local HTML for wp-content/uploads/* paths not present on disk
|
||||
2. For each missing image try:
|
||||
a. Direct live URL (site may still exist in cache/CDN)
|
||||
b. Wayback im_ modifier with CDX-best timestamp
|
||||
c. Wayback im_ with known timestamps in order
|
||||
3. Download missing HTML pages via fetch_missing-style CDX lookup
|
||||
4. Report final stats
|
||||
|
||||
Run from the lapaella-mirror directory.
|
||||
"""
|
||||
|
||||
import os, re, sys, time, subprocess, tempfile, urllib.parse, json
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from threading import Lock
|
||||
|
||||
SITE = Path(__file__).parent / "site"
|
||||
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
BASE = "https://lapaella.net"
|
||||
BASE_OLD = "http://www.lapaella.net"
|
||||
# Most useful timestamps for this site
|
||||
TS_LIST = [
|
||||
"20241130065929",
|
||||
"20241014051856",
|
||||
"20210603053819",
|
||||
"20220101000000",
|
||||
"20200601000000",
|
||||
"20180101000000",
|
||||
"20150101000000",
|
||||
"20120601000000",
|
||||
]
|
||||
DELAY = 0.25 # per-worker delay between requests
|
||||
print_lock = Lock()
|
||||
|
||||
# ── Missing pages to recover ─────────────────────────────────────────────────
|
||||
MISSING_PAGES = [
|
||||
"/otras-recetas-valencianas/all-i-oli/",
|
||||
"/arroz-de-frutos-secos-garbanzos-y-nisperos/",
|
||||
"/clochina-o-mejillon/",
|
||||
"/receta-de-arroz-negro/",
|
||||
"/receta-de-fideua",
|
||||
"/otras-recetas-valencianas/",
|
||||
"/enlaces/",
|
||||
"/valencian-paella-introduction/",
|
||||
"/paella-valenciana-recipe-english/",
|
||||
"/contacto/",
|
||||
]
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def log(msg):
|
||||
with print_lock:
|
||||
print(msg, flush=True)
|
||||
|
||||
def curl_fetch(url, is_image=False):
|
||||
tmp = None
|
||||
try:
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp")
|
||||
tmp.close()
|
||||
cmd = [
|
||||
"/usr/bin/curl", "-s", "-L", "--max-time", "20",
|
||||
"-A", UA,
|
||||
"-w", "\n__S__%{http_code}",
|
||||
"-o", tmp.name,
|
||||
url,
|
||||
]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
status = 0
|
||||
for line in r.stdout.splitlines():
|
||||
if line.startswith("__S__"):
|
||||
try: status = int(line[5:].strip())
|
||||
except: pass
|
||||
if status in (404, 403, 410, 0):
|
||||
return None
|
||||
with open(tmp.name, "rb") as f:
|
||||
data = f.read()
|
||||
min_size = 200 if is_image else 500
|
||||
return data if len(data) >= min_size else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if tmp:
|
||||
try: os.unlink(tmp.name)
|
||||
except: pass
|
||||
|
||||
def cdx_best_ts(orig_path, from_year="2010"):
|
||||
"""Ask CDX for the best (closest to latest) 200-status snapshot timestamp."""
|
||||
try:
|
||||
url = (
|
||||
"https://web.archive.org/cdx/search/cdx"
|
||||
f"?url=lapaella.net{urllib.parse.quote(orig_path)}"
|
||||
"&output=json&limit=5&fl=timestamp&filter=statuscode:200"
|
||||
f"&from={from_year}0101&to=20251231&fastLatest=true"
|
||||
)
|
||||
data = curl_fetch(url)
|
||||
if not data:
|
||||
return None
|
||||
rows = json.loads(data.decode())
|
||||
# rows[0] is the header ["timestamp"]
|
||||
if len(rows) > 1:
|
||||
return rows[1][0] # best timestamp
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def local_path_for(orig_path):
|
||||
decoded = urllib.parse.unquote(orig_path).lstrip("/")
|
||||
if not decoded or decoded.endswith("/"):
|
||||
return SITE / decoded / "index.html"
|
||||
p = Path(decoded)
|
||||
return SITE / p / "index.html" if not p.suffix else SITE / p
|
||||
|
||||
def wbm_image_url(orig_path, ts):
|
||||
return f"https://web.archive.org/web/{ts}im_/{BASE}{orig_path}"
|
||||
|
||||
def wbm_page_url(orig_path, ts, old=False):
|
||||
base = BASE_OLD if old else BASE
|
||||
return f"https://web.archive.org/web/{ts}/{base}{orig_path}"
|
||||
|
||||
# ── Image recovery ───────────────────────────────────────────────────────────
|
||||
|
||||
def recover_image(orig_path):
|
||||
local = local_path_for(orig_path)
|
||||
if local.exists():
|
||||
return "skip", orig_path
|
||||
|
||||
# 1) Try live site first (fast, no WBM rate limits)
|
||||
live_url = f"{BASE}{orig_path}"
|
||||
data = curl_fetch(live_url, is_image=True)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
log(f" ✓ live {orig_path}")
|
||||
return "ok", orig_path
|
||||
|
||||
# 2) CDX best timestamp
|
||||
best_ts = cdx_best_ts(orig_path)
|
||||
ts_candidates = ([best_ts] if best_ts else []) + [
|
||||
ts for ts in TS_LIST if ts != best_ts
|
||||
]
|
||||
|
||||
for ts in ts_candidates:
|
||||
url = wbm_image_url(orig_path, ts)
|
||||
data = curl_fetch(url, is_image=True)
|
||||
if data:
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
local.write_bytes(data)
|
||||
log(f" ✓ wbm/{ts[:8]} {orig_path}")
|
||||
return "ok", orig_path
|
||||
time.sleep(DELAY)
|
||||
|
||||
log(f" ✗ not found {orig_path}")
|
||||
return "fail", orig_path
|
||||
|
||||
# ── Page recovery ────────────────────────────────────────────────────────────
|
||||
|
||||
def orig_path_from_wbm(url):
|
||||
m = re.match(
|
||||
r'https?://web\.archive\.org/web/\d+(?:im_|cs_|js_|oe_|if_|mp_)?/'
|
||||
r'https?://(?:www\.)?lapaella\.net(/[^"\'<>\s#]*)?',
|
||||
url,
|
||||
)
|
||||
return (m.group(1) or "/") if m else None
|
||||
|
||||
def rewrite_html(html, local_file):
|
||||
html = re.sub(r'<!-- BEGIN WAYBACK TOOLBAR INSERT -->.*?<!-- END WAYBACK TOOLBAR INSERT -->', '', html, flags=re.DOTALL)
|
||||
html = re.sub(r'<script[^>]*(?:archive\.org|wombat|athena|bundle-playback|ruffle)[^>]*>.*?</script>', '', html, flags=re.DOTALL|re.IGNORECASE)
|
||||
html = re.sub(r'<script[^>]*(?:archive\.org|wombat|athena|bundle-playback|ruffle)[^>]*/?>', '', html, flags=re.IGNORECASE)
|
||||
html = re.sub(r'<link[^>]*(?:archive\.org|iconochive|banner-styles)[^>]*/?>', '', html, flags=re.IGNORECASE)
|
||||
html = re.sub(r'<script[^>]*>\s*__wm\b.*?</script>', '', html, flags=re.DOTALL)
|
||||
|
||||
def _rewrite(url):
|
||||
if "web.archive.org" not in url:
|
||||
return url
|
||||
orig = orig_path_from_wbm(url)
|
||||
if not orig:
|
||||
return "#"
|
||||
rel = os.path.relpath(str(local_path_for(orig)), str(local_file.parent))
|
||||
return rel
|
||||
|
||||
def _attr(m):
|
||||
attr, q, url, q2 = m.group(1), m.group(2), m.group(3), m.group(4)
|
||||
return f'{attr}={q}{_rewrite(url)}{q2}'
|
||||
html = re.sub(r'((?:src|href|action|data-src)=)(["\'])([^"\']+)(["\'])', _attr, html)
|
||||
|
||||
def _css_url(m):
|
||||
url = m.group(1).strip().strip("'\"")
|
||||
return f"url('{_rewrite(url)}')"
|
||||
html = re.sub(r'url\(([^)]+)\)', _css_url, html)
|
||||
return html
|
||||
|
||||
def recover_page(orig_path):
|
||||
local = local_path_for(orig_path)
|
||||
if local.exists():
|
||||
return "skip", orig_path
|
||||
|
||||
best_ts = cdx_best_ts(orig_path)
|
||||
ts_candidates = ([best_ts] if best_ts else []) + [
|
||||
ts for ts in TS_LIST if ts != best_ts
|
||||
]
|
||||
|
||||
for ts in ts_candidates:
|
||||
for old in (False, True):
|
||||
url = wbm_page_url(orig_path, ts, old=old)
|
||||
data = curl_fetch(url)
|
||||
if data:
|
||||
html = data.decode("utf-8", errors="replace")
|
||||
local.parent.mkdir(parents=True, exist_ok=True)
|
||||
html = rewrite_html(html, local)
|
||||
local.write_text(html, encoding="utf-8")
|
||||
log(f" ✓ page/{ts[:8]} {orig_path}")
|
||||
return "ok", orig_path
|
||||
time.sleep(DELAY)
|
||||
|
||||
log(f" ✗ page not found {orig_path}")
|
||||
return "fail", orig_path
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def collect_missing_images():
|
||||
refs = set()
|
||||
for f in SITE.rglob("*.html"):
|
||||
txt = f.read_text(errors="replace")
|
||||
for m in re.findall(r'(/wp-content/uploads/[^\"\'\s<>\)]+)', txt):
|
||||
# skip CSS query-string variants and wildcards
|
||||
if "?" in m or "*" in m or not re.search(r'\.(jpe?g|png|gif|webp|svg|ico|bmp)$', m, re.I):
|
||||
continue
|
||||
refs.add(m)
|
||||
missing = [r for r in sorted(refs) if not (SITE / r.lstrip("/")).exists()]
|
||||
return missing
|
||||
|
||||
def main():
|
||||
missing_images = collect_missing_images()
|
||||
log(f"\n{'='*60}")
|
||||
log(f"Missing images : {len(missing_images)}")
|
||||
log(f"Missing pages : {len(MISSING_PAGES)}")
|
||||
log(f"{'='*60}\n")
|
||||
|
||||
img_ok = img_fail = 0
|
||||
pg_ok = pg_fail = 0
|
||||
|
||||
# Images — parallel with 4 workers (be polite to WBM)
|
||||
log("── Images ──────────────────────────────────────────────────")
|
||||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||||
futures = {ex.submit(recover_image, p): p for p in missing_images}
|
||||
for fut in as_completed(futures):
|
||||
status, _ = fut.result()
|
||||
if status == "ok": img_ok += 1
|
||||
elif status == "fail": img_fail += 1
|
||||
|
||||
# Pages — sequential (each page may cascade more asset fetches)
|
||||
log("\n── Pages ───────────────────────────────────────────────────")
|
||||
for orig_path in MISSING_PAGES:
|
||||
status, _ = recover_page(orig_path)
|
||||
if status == "ok": pg_ok += 1
|
||||
elif status == "fail": pg_fail += 1
|
||||
time.sleep(1.0)
|
||||
|
||||
log(f"\n{'='*60}")
|
||||
log(f"Images recovered: {img_ok} failed: {img_fail}")
|
||||
log(f"Pages recovered: {pg_ok} failed: {pg_fail}")
|
||||
log(f"{'='*60}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user