Paella/lapaella-mirror/mirror_originalpaella_recetas.py

#!/usr/bin/env python3
"""
Mirror ONLY the "Recetas de paellas y arroces para preparar en casa" blog
category from originalpaella.es (smartblog category id 9), plus every photo
used by those recipe posts.

Scope guard: we only ever fetch
  - the category listing pages  /blog/9-recetas-...?page=N
  - individual post pages       /blog/<id>_<slug>.html   linked from those listings
  - image files on originalpaella.es referenced by those pages
No other links are followed.

Output: originalpaella-recetas/
"""

import os, re, sys, subprocess, tempfile, urllib.parse
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock

BASE   = "https://originalpaella.es"
HOST   = "originalpaella.es"
CAT    = "/blog/9-recetas-de-paellas-y-arroces-para-preparar-en-casa"
OUT    = Path(__file__).parent / "originalpaella-recetas"
UA     = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
          "(KHTML, like Gecko) Chrome/125 Safari/537.36")
IMG_EXT = (".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg", ".avif")
POST_RE = re.compile(r'https://originalpaella\.es/blog/(\d+)_([a-z0-9\-]+)\.html', re.I)
LOCK = Lock()

def log(m):
    with LOCK:
        print(m, flush=True)

def enc(url):
    """Percent-encode the path/query so curl accepts spaces & parens."""
    p = urllib.parse.urlsplit(url)
    path = urllib.parse.quote(urllib.parse.unquote(p.path), safe="/%")
    query = urllib.parse.quote(urllib.parse.unquote(p.query), safe="=&%")
    return urllib.parse.urlunsplit((p.scheme, p.netloc, path, query, ""))

def curl(url, dest=None):
    url = enc(url)
    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp"); tmp.close()
    try:
        r = subprocess.run(
            ["/usr/bin/curl", "-s", "-L", "--max-time", "60",
             "--retry", "3", "--retry-delay", "3", "--retry-all-errors",
             "-A", UA, "-w", "%{http_code}", "-o", tmp.name, url],
            capture_output=True, text=True, timeout=120)
        code = (r.stdout or "").strip()[-3:]
        data = Path(tmp.name).read_bytes()
        if code == "200" and data:
            if dest:
                dest.parent.mkdir(parents=True, exist_ok=True)
                dest.write_bytes(data)
            return data
        return None
    except Exception as e:
        log(f"    curl err {e}"); return None
    finally:
        try: os.unlink(tmp.name)
        except OSError: pass

def local_for(url):
    """Map an originalpaella.es URL to a path under OUT."""
    p = urllib.parse.urlparse(url)
    path = urllib.parse.unquote(p.path).lstrip("/")
    if not path:
        path = "index.html"
    if p.query:
        q = re.sub(r'[^\w.=-]+', "_", urllib.parse.unquote(p.query))
        base, dot, ext = path.rpartition(".")
        path = f"{base}__{q}.{ext}" if dot else f"{path}__{q}.html"
    if path.endswith("/"):
        path += "index.html"
    # sanitise
    path = "/".join(re.sub(r'[<>:"\\|?*]+', "_", seg).strip() or "_" for seg in path.split("/"))
    return OUT / path

def absolutise(src, page_url):
    src = src.strip().strip('"\'')
    if src.startswith("//"):
        return "https:" + src
    if src.startswith(("http://", "https://")):
        return src
    if src.startswith("/"):
        return BASE + src
    return urllib.parse.urljoin(page_url, src)

def extract_imgs(html, page_url):
    urls = set()
    for m in re.findall(r'(?:src|data-src|data-original|content|href)=["\']([^"\']+)["\']', html, re.I):
        u = absolutise(m, page_url)
        pu = urllib.parse.urlparse(u)
        if pu.netloc.endswith(HOST) and pu.path.lower().rsplit("?", 1)[0].endswith(IMG_EXT):
            urls.add(u.split("#")[0])
    for m in re.findall(r'srcset=["\']([^"\']+)["\']', html, re.I):
        for part in m.split(","):
            cand = part.strip().split(" ")[0]
            if not cand:
                continue
            u = absolutise(cand, page_url)
            pu = urllib.parse.urlparse(u)
            if pu.netloc.endswith(HOST) and pu.path.lower().endswith(IMG_EXT):
                urls.add(u.split("#")[0])
    return urls

def rewrite(html, page_file, asset_urls):
    """Point originalpaella.es img/asset refs at local relative paths."""
    def repl(m):
        attr, q, val = m.group(1), m.group(2), m.group(3)
        u = absolutise(val, BASE + "/")
        if u.split("#")[0] in asset_urls:
            rel = os.path.relpath(local_for(u), page_file.parent)
            return f'{attr}={q}{rel}{q}'
        return m.group(0)
    html = re.sub(r'(src|data-src|href)=(["\'])([^"\']+)\2', repl, html)
    return html

def main():
    OUT.mkdir(parents=True, exist_ok=True)
    # ---- 1. walk the category pagination ----
    post_urls = {}          # id -> url
    listing_pages = {}       # page_no -> html
    page = 1
    while True:
        url = f"{BASE}{CAT}?page={page}"
        html = curl(url)
        if not html:
            break
        html = html.decode("utf-8", "replace")
        found = {int(i): f"https://originalpaella.es/blog/{i}_{s}.html"
                 for i, s in POST_RE.findall(html)}
        new = {k: v for k, v in found.items() if k not in post_urls}
        listing_pages[page] = html
        log(f"listing page {page}: {len(found)} posts ({len(new)} new)")
        if not new and page > 1:
            break
        post_urls.update(found)
        page += 1
        if page > 40:
            break

    log(f"\nTotal recipe posts: {len(post_urls)}")
    (OUT / "_post_urls.txt").write_text("\n".join(v for _, v in sorted(post_urls.items())))

    # ---- 2. fetch each post, collect images ----
    all_assets = set()
    page_assets = {}         # local_file -> (html, asset_urls)

    def handle_page(url, html=None):
        if html is None:
            data = curl(url)
            if not data:
                log(f"  MISS {url}")
                return
            html = data.decode("utf-8", "replace")
        imgs = extract_imgs(html, url)
        lf = local_for(url)
        page_assets[lf] = (html, imgs)
        with LOCK:
            all_assets.update(imgs)
        log(f"  post {url.rsplit('/',1)[-1]}  ({len(imgs)} imgs)")

    # listing pages
    for pno, html in listing_pages.items():
        u = f"{BASE}{CAT}?page={pno}"
        handle_page(u, html)

    with ThreadPoolExecutor(max_workers=4) as ex:
        list(as_completed([ex.submit(handle_page, u) for u in post_urls.values()]))

    # ---- 3. download images ----
    log(f"\nDownloading {len(all_assets)} images ...")
    ok = fail = 0
    def dl(u):
        dest = local_for(u)
        if dest.exists() and dest.stat().st_size:
            return "skip"
        return "ok" if curl(u, dest) else "fail"
    with ThreadPoolExecutor(max_workers=6) as ex:
        futs = {ex.submit(dl, u): u for u in sorted(all_assets)}
        for f in as_completed(futs):
            r = f.result()
            if r == "ok":   ok += 1
            elif r == "fail":
                fail += 1; log(f"  img FAIL {futs[f]}")
    log(f"images: ok={ok} fail={fail} (skip={len(all_assets)-ok-fail})")

    # ---- 4. write rewritten HTML ----
    for lf, (html, imgs) in page_assets.items():
        lf.parent.mkdir(parents=True, exist_ok=True)
        lf.write_text(rewrite(html, lf, all_assets), encoding="utf-8")
    log(f"\nWrote {len(page_assets)} HTML pages -> {OUT}")

if __name__ == "__main__":
    main()