#!/usr/bin/env python3 """ Mirror ONLY the "Recetas de paellas y arroces para preparar en casa" blog category from originalpaella.es (smartblog category id 9), plus every photo used by those recipe posts. Scope guard: we only ever fetch - the category listing pages /blog/9-recetas-...?page=N - individual post pages /blog/_.html linked from those listings - image files on originalpaella.es referenced by those pages No other links are followed. Output: originalpaella-recetas/ """ import os, re, sys, subprocess, tempfile, urllib.parse from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from threading import Lock BASE = "https://originalpaella.es" HOST = "originalpaella.es" CAT = "/blog/9-recetas-de-paellas-y-arroces-para-preparar-en-casa" OUT = Path(__file__).parent / "originalpaella-recetas" UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/125 Safari/537.36") IMG_EXT = (".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg", ".avif") POST_RE = re.compile(r'https://originalpaella\.es/blog/(\d+)_([a-z0-9\-]+)\.html', re.I) LOCK = Lock() def log(m): with LOCK: print(m, flush=True) def enc(url): """Percent-encode the path/query so curl accepts spaces & parens.""" p = urllib.parse.urlsplit(url) path = urllib.parse.quote(urllib.parse.unquote(p.path), safe="/%") query = urllib.parse.quote(urllib.parse.unquote(p.query), safe="=&%") return urllib.parse.urlunsplit((p.scheme, p.netloc, path, query, "")) def curl(url, dest=None): url = enc(url) tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp"); tmp.close() try: r = subprocess.run( ["/usr/bin/curl", "-s", "-L", "--max-time", "60", "--retry", "3", "--retry-delay", "3", "--retry-all-errors", "-A", UA, "-w", "%{http_code}", "-o", tmp.name, url], capture_output=True, text=True, timeout=120) code = (r.stdout or "").strip()[-3:] data = Path(tmp.name).read_bytes() if code == "200" and data: if dest: dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(data) return data return None except Exception as e: log(f" curl err {e}"); return None finally: try: os.unlink(tmp.name) except OSError: pass def local_for(url): """Map an originalpaella.es URL to a path under OUT.""" p = urllib.parse.urlparse(url) path = urllib.parse.unquote(p.path).lstrip("/") if not path: path = "index.html" if p.query: q = re.sub(r'[^\w.=-]+', "_", urllib.parse.unquote(p.query)) base, dot, ext = path.rpartition(".") path = f"{base}__{q}.{ext}" if dot else f"{path}__{q}.html" if path.endswith("/"): path += "index.html" # sanitise path = "/".join(re.sub(r'[<>:"\\|?*]+', "_", seg).strip() or "_" for seg in path.split("/")) return OUT / path def absolutise(src, page_url): src = src.strip().strip('"\'') if src.startswith("//"): return "https:" + src if src.startswith(("http://", "https://")): return src if src.startswith("/"): return BASE + src return urllib.parse.urljoin(page_url, src) def extract_imgs(html, page_url): urls = set() for m in re.findall(r'(?:src|data-src|data-original|content|href)=["\']([^"\']+)["\']', html, re.I): u = absolutise(m, page_url) pu = urllib.parse.urlparse(u) if pu.netloc.endswith(HOST) and pu.path.lower().rsplit("?", 1)[0].endswith(IMG_EXT): urls.add(u.split("#")[0]) for m in re.findall(r'srcset=["\']([^"\']+)["\']', html, re.I): for part in m.split(","): cand = part.strip().split(" ")[0] if not cand: continue u = absolutise(cand, page_url) pu = urllib.parse.urlparse(u) if pu.netloc.endswith(HOST) and pu.path.lower().endswith(IMG_EXT): urls.add(u.split("#")[0]) return urls def rewrite(html, page_file, asset_urls): """Point originalpaella.es img/asset refs at local relative paths.""" def repl(m): attr, q, val = m.group(1), m.group(2), m.group(3) u = absolutise(val, BASE + "/") if u.split("#")[0] in asset_urls: rel = os.path.relpath(local_for(u), page_file.parent) return f'{attr}={q}{rel}{q}' return m.group(0) html = re.sub(r'(src|data-src|href)=(["\'])([^"\']+)\2', repl, html) return html def main(): OUT.mkdir(parents=True, exist_ok=True) # ---- 1. walk the category pagination ---- post_urls = {} # id -> url listing_pages = {} # page_no -> html page = 1 while True: url = f"{BASE}{CAT}?page={page}" html = curl(url) if not html: break html = html.decode("utf-8", "replace") found = {int(i): f"https://originalpaella.es/blog/{i}_{s}.html" for i, s in POST_RE.findall(html)} new = {k: v for k, v in found.items() if k not in post_urls} listing_pages[page] = html log(f"listing page {page}: {len(found)} posts ({len(new)} new)") if not new and page > 1: break post_urls.update(found) page += 1 if page > 40: break log(f"\nTotal recipe posts: {len(post_urls)}") (OUT / "_post_urls.txt").write_text("\n".join(v for _, v in sorted(post_urls.items()))) # ---- 2. fetch each post, collect images ---- all_assets = set() page_assets = {} # local_file -> (html, asset_urls) def handle_page(url, html=None): if html is None: data = curl(url) if not data: log(f" MISS {url}") return html = data.decode("utf-8", "replace") imgs = extract_imgs(html, url) lf = local_for(url) page_assets[lf] = (html, imgs) with LOCK: all_assets.update(imgs) log(f" post {url.rsplit('/',1)[-1]} ({len(imgs)} imgs)") # listing pages for pno, html in listing_pages.items(): u = f"{BASE}{CAT}?page={pno}" handle_page(u, html) with ThreadPoolExecutor(max_workers=4) as ex: list(as_completed([ex.submit(handle_page, u) for u in post_urls.values()])) # ---- 3. download images ---- log(f"\nDownloading {len(all_assets)} images ...") ok = fail = 0 def dl(u): dest = local_for(u) if dest.exists() and dest.stat().st_size: return "skip" return "ok" if curl(u, dest) else "fail" with ThreadPoolExecutor(max_workers=6) as ex: futs = {ex.submit(dl, u): u for u in sorted(all_assets)} for f in as_completed(futs): r = f.result() if r == "ok": ok += 1 elif r == "fail": fail += 1; log(f" img FAIL {futs[f]}") log(f"images: ok={ok} fail={fail} (skip={len(all_assets)-ok-fail})") # ---- 4. write rewritten HTML ---- for lf, (html, imgs) in page_assets.items(): lf.parent.mkdir(parents=True, exist_ok=True) lf.write_text(rewrite(html, lf, all_assets), encoding="utf-8") log(f"\nWrote {len(page_assets)} HTML pages -> {OUT}") if __name__ == "__main__": main()