Paella/lapaella-mirror/recover_deleted_recipes.py

#!/usr/bin/env python3
"""
Recover ONLY the Spanish recipes that are no longer on the live lapaella.net,
from the Wayback Machine, into  wayback-mirror/ .

Targets (decided with the user):
  1. Pimientos rojos rellenos de arroz  - modern WordPress recipe page, now 404
  2. Arroz con pollo, pato y caracoles   - /web1/arrozpollo.html  (~2003 static site)
  3. Paella de habas                     - /web1/paellahabas.html
  4. Paella mixta                        - /web1/paellamixta.html

Each page is saved with the Wayback toolbar stripped and its images pulled in
alongside it (links rewritten to the local copies).
"""

import json, os, re, subprocess, tempfile, time, urllib.parse
from pathlib import Path

ROOT = Path(__file__).parent
OUT  = ROOT / "wayback-mirror"
UA   = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"

TARGETS = [
    # (original url, local dir under OUT, local html name)
    ("https://lapaella.net/arroces-tipicos-de-valencia/pimientos-rojos-rellenos-de-arroz/",
     "pimientos-rojos-rellenos-de-arroz", "index.html"),
    ("http://www.lapaella.net/web1/arrozpollo.html",  "web1", "arrozpollo.html"),
    ("http://www.lapaella.net/web1/paellahabas.html", "web1", "paellahabas.html"),
    ("http://www.lapaella.net/web1/paellamixta.html", "web1", "paellamixta.html"),
]

def curl(url, binary=True, tries=5):
    for a in range(tries):
        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp"); tmp.close()
        try:
            r = subprocess.run(
                ["/usr/bin/curl", "-s", "-L", "--max-time", "60", "-A", UA,
                 "-w", "%{http_code}", "-o", tmp.name, url],
                capture_output=True, text=True, timeout=90)
            code = (r.stdout or "").strip()[-3:]
            data = Path(tmp.name).read_bytes()
            if code == "200" and data:
                return data if binary else data.decode("utf-8", "replace")
            if code in ("429", "403", "503", "000", "520", "521"):
                wait = 20 * (a + 1)
                print(f"    {code} on {url[:70]} - backoff {wait}s")
                time.sleep(wait); continue
            print(f"    {code} on {url[:80]}")
            return None
        finally:
            try: os.unlink(tmp.name)
            except OSError: pass
    return None

def cdx_best(original):
    """Return the newest 200 snapshot timestamp for a URL, or None."""
    q = ("http://web.archive.org/cdx/search/cdx?"
         + urllib.parse.urlencode({"url": original, "output": "json",
                                   "filter": "statuscode:200", "limit": "-8"}))
    raw = curl(q, binary=False)
    if not raw:
        return None
    try:
        rows = json.loads(raw)
    except json.JSONDecodeError:
        return None
    if len(rows) < 2:
        return None
    # rows[0] is the header; take the last (newest) row's timestamp
    return rows[-1][1]

TOOLBAR = [
    (re.compile(r'<!-- BEGIN WAYBACK TOOLBAR INSERT -->.*?<!-- END WAYBACK TOOLBAR INSERT -->', re.S), ''),
    (re.compile(r'<script[^>]*(?:archive\.org|wombat\.js|bundle-playback|wm-|ruffle)[^>]*>.*?</script>', re.S | re.I), ''),
    (re.compile(r'<script[^>]*(?:archive\.org|bundle-playback|wombat)[^>]*/?>', re.I), ''),
    (re.compile(r'<link[^>]*(?:archive\.org|banner-styles|iconochive)[^>]*/?>', re.I), ''),
    (re.compile(r'<!--\s*End Wayback Rewrite JS Include\s*-->', re.I), ''),
]

WB_ASSET = re.compile(
    r'https?://web\.archive\.org/web/\d+(?:\w\w_)?/'
    r'(https?://(?:www\.)?lapaella\.net[^"\'\s)>]+)', re.I)

def strip_and_collect(html):
    for pat, rep in TOOLBAR:
        html = pat.sub(rep, html)
    assets = set(m.group(1) for m in WB_ASSET.finditer(html))
    # keep only real media
    assets = {a for a in assets
              if re.search(r'\.(jpe?g|png|gif|webp|svg|css)(\?|$)', a, re.I)}
    return html, assets

def localize(html, asset_map, html_local):
    # replace any wayback-wrapped url whose target we downloaded
    def repl(m):
        target = m.group(1)
        key = target.split("?")[0]
        if key in asset_map:
            rel = os.path.relpath(asset_map[key], html_local.parent)
            return rel
        return target  # de-wayback even if not downloaded
    html = WB_ASSET.sub(repl, html)
    return html

def asset_local(original_url):
    p = urllib.parse.urlparse(original_url)
    path = urllib.parse.unquote(p.path).lstrip("/")
    path = "/".join(re.sub(r'[^\w.\-]+', "_", s) or "_" for s in path.split("/"))
    return OUT / "_assets" / path

def main():
    OUT.mkdir(parents=True, exist_ok=True)
    manifest = []
    for original, subdir, name in TARGETS:
        print(f"\n=== {original}")
        ts = cdx_best(original)
        if not ts:
            print("    no snapshot found - SKIP")
            continue
        page_url = f"https://web.archive.org/web/{ts}/{original}"
        raw = curl(page_url, binary=False)
        if not raw:
            print("    could not fetch page - SKIP")
            continue
        html, assets = strip_and_collect(raw)

        # sanity check the web1 pages actually contain a recipe
        text = re.sub(r"<[^>]+>", " ", html)
        if subdir == "web1" and not re.search(
                r"ingredientes|elaboraci|receta|arroz|paella", text, re.I):
            print("    !! no recipe markers - fetched anyway, review manually")

        html_local = OUT / subdir / name
        html_local.parent.mkdir(parents=True, exist_ok=True)

        asset_map = {}
        for a in sorted(assets):
            dest = asset_local(a)
            if dest.exists() and dest.stat().st_size:
                asset_map[a.split("?")[0]] = dest
                continue
            # images: use the raw im_ snapshot; css: cs_
            mod = "cs_" if a.lower().split("?")[0].endswith(".css") else "im_"
            data = curl(f"https://web.archive.org/web/{ts}{mod}/{a}")
            if not data:
                data = curl(f"https://web.archive.org/web/{ts}/{a}")
            if data:
                dest.parent.mkdir(parents=True, exist_ok=True)
                dest.write_bytes(data)
                asset_map[a.split("?")[0]] = dest
                print(f"    asset  {a.split('/')[-1][:60]}")
            else:
                print(f"    MISS   {a}")
            time.sleep(1)

        html = localize(html, asset_map, html_local)
        html_local.write_text(html, encoding="utf-8")
        title = (re.search(r"<title>([^<]*)</title>", raw, re.I) or [None, "?"])[1].strip()
        manifest.append(f"{subdir}/{name}\t{ts}\t{title}\t<- {original}")
        print(f"    saved  {html_local.relative_to(ROOT)}  ({len(assets)} assets)")
        time.sleep(2)

    (OUT / "RECOVERED.txt").write_text(
        "Spanish recipes recovered from the Wayback Machine because they are no "
        "longer on the live site.\n\n" + "\n".join(manifest) + "\n")
    print("\n--- manifest ---")
    print("\n".join(manifest))

if __name__ == "__main__":
    main()