Add live-site mirror, deleted-recipe recovery, and originalpaella.es recipes
lapaella.net is back online. Collected via the apex domain (www. host has a bad TLS cert): - live-mirror/: complete wget mirror of the current site (145 pages + requisites, links converted for offline use), covering all of sitemap_index.xml incl. the blog and its recipe posts. - wayback-mirror/: only the Spanish recipes no longer on the live site, recovered from the Wayback Machine (toolbar stripped, links de-waybacked): Pimientos rojos rellenos de arroz, plus c.2009 static-site recipes Arroz con pollo/pato/caracoles, Paella de habas, Paella mixta. - originalpaella-recetas/: originalpaella.es "Recetas de paellas y arroces" blog category (id 9) only - 103 posts, 12 listing pages, 1,255 photos. - inventory.md: folder map + apex-domain note. - Scripts: recover_deleted_recipes.py, mirror_originalpaella_recetas.py. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FAteBZwRGH7tcBWNptKFjs
This commit is contained in:
robertoandClaude Sonnet 5 committed
1 parent
aa662e7fdc
commit
75cd28ea8e
1665 files changed
+520157
No files matched your search
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Mirror ONLY the "Recetas de paellas y arroces para preparar en casa" blog
|
||||
category from originalpaella.es (smartblog category id 9), plus every photo
|
||||
used by those recipe posts.
|
||||
|
||||
Scope guard: we only ever fetch
|
||||
- the category listing pages /blog/9-recetas-...?page=N
|
||||
- individual post pages /blog/<id>_<slug>.html linked from those listings
|
||||
- image files on originalpaella.es referenced by those pages
|
||||
No other links are followed.
|
||||
|
||||
Output: originalpaella-recetas/
|
||||
"""
|
||||
|
||||
import os, re, sys, subprocess, tempfile, urllib.parse
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from threading import Lock
|
||||
|
||||
BASE = "https://originalpaella.es"
|
||||
HOST = "originalpaella.es"
|
||||
CAT = "/blog/9-recetas-de-paellas-y-arroces-para-preparar-en-casa"
|
||||
OUT = Path(__file__).parent / "originalpaella-recetas"
|
||||
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125 Safari/537.36")
|
||||
IMG_EXT = (".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg", ".avif")
|
||||
POST_RE = re.compile(r'https://originalpaella\.es/blog/(\d+)_([a-z0-9\-]+)\.html', re.I)
|
||||
LOCK = Lock()
|
||||
|
||||
def log(m):
|
||||
with LOCK:
|
||||
print(m, flush=True)
|
||||
|
||||
def enc(url):
|
||||
"""Percent-encode the path/query so curl accepts spaces & parens."""
|
||||
p = urllib.parse.urlsplit(url)
|
||||
path = urllib.parse.quote(urllib.parse.unquote(p.path), safe="/%")
|
||||
query = urllib.parse.quote(urllib.parse.unquote(p.query), safe="=&%")
|
||||
return urllib.parse.urlunsplit((p.scheme, p.netloc, path, query, ""))
|
||||
|
||||
def curl(url, dest=None):
|
||||
url = enc(url)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp"); tmp.close()
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["/usr/bin/curl", "-s", "-L", "--max-time", "60",
|
||||
"--retry", "3", "--retry-delay", "3", "--retry-all-errors",
|
||||
"-A", UA, "-w", "%{http_code}", "-o", tmp.name, url],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
code = (r.stdout or "").strip()[-3:]
|
||||
data = Path(tmp.name).read_bytes()
|
||||
if code == "200" and data:
|
||||
if dest:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(data)
|
||||
return data
|
||||
return None
|
||||
except Exception as e:
|
||||
log(f" curl err {e}"); return None
|
||||
finally:
|
||||
try: os.unlink(tmp.name)
|
||||
except OSError: pass
|
||||
|
||||
def local_for(url):
|
||||
"""Map an originalpaella.es URL to a path under OUT."""
|
||||
p = urllib.parse.urlparse(url)
|
||||
path = urllib.parse.unquote(p.path).lstrip("/")
|
||||
if not path:
|
||||
path = "index.html"
|
||||
if p.query:
|
||||
q = re.sub(r'[^\w.=-]+', "_", urllib.parse.unquote(p.query))
|
||||
base, dot, ext = path.rpartition(".")
|
||||
path = f"{base}__{q}.{ext}" if dot else f"{path}__{q}.html"
|
||||
if path.endswith("/"):
|
||||
path += "index.html"
|
||||
# sanitise
|
||||
path = "/".join(re.sub(r'[<>:"\\|?*]+', "_", seg).strip() or "_" for seg in path.split("/"))
|
||||
return OUT / path
|
||||
|
||||
def absolutise(src, page_url):
|
||||
src = src.strip().strip('"\'')
|
||||
if src.startswith("//"):
|
||||
return "https:" + src
|
||||
if src.startswith(("http://", "https://")):
|
||||
return src
|
||||
if src.startswith("/"):
|
||||
return BASE + src
|
||||
return urllib.parse.urljoin(page_url, src)
|
||||
|
||||
def extract_imgs(html, page_url):
|
||||
urls = set()
|
||||
for m in re.findall(r'(?:src|data-src|data-original|content|href)=["\']([^"\']+)["\']', html, re.I):
|
||||
u = absolutise(m, page_url)
|
||||
pu = urllib.parse.urlparse(u)
|
||||
if pu.netloc.endswith(HOST) and pu.path.lower().rsplit("?", 1)[0].endswith(IMG_EXT):
|
||||
urls.add(u.split("#")[0])
|
||||
for m in re.findall(r'srcset=["\']([^"\']+)["\']', html, re.I):
|
||||
for part in m.split(","):
|
||||
cand = part.strip().split(" ")[0]
|
||||
if not cand:
|
||||
continue
|
||||
u = absolutise(cand, page_url)
|
||||
pu = urllib.parse.urlparse(u)
|
||||
if pu.netloc.endswith(HOST) and pu.path.lower().endswith(IMG_EXT):
|
||||
urls.add(u.split("#")[0])
|
||||
return urls
|
||||
|
||||
def rewrite(html, page_file, asset_urls):
|
||||
"""Point originalpaella.es img/asset refs at local relative paths."""
|
||||
def repl(m):
|
||||
attr, q, val = m.group(1), m.group(2), m.group(3)
|
||||
u = absolutise(val, BASE + "/")
|
||||
if u.split("#")[0] in asset_urls:
|
||||
rel = os.path.relpath(local_for(u), page_file.parent)
|
||||
return f'{attr}={q}{rel}{q}'
|
||||
return m.group(0)
|
||||
html = re.sub(r'(src|data-src|href)=(["\'])([^"\']+)\2', repl, html)
|
||||
return html
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
# ---- 1. walk the category pagination ----
|
||||
post_urls = {} # id -> url
|
||||
listing_pages = {} # page_no -> html
|
||||
page = 1
|
||||
while True:
|
||||
url = f"{BASE}{CAT}?page={page}"
|
||||
html = curl(url)
|
||||
if not html:
|
||||
break
|
||||
html = html.decode("utf-8", "replace")
|
||||
found = {int(i): f"https://originalpaella.es/blog/{i}_{s}.html"
|
||||
for i, s in POST_RE.findall(html)}
|
||||
new = {k: v for k, v in found.items() if k not in post_urls}
|
||||
listing_pages[page] = html
|
||||
log(f"listing page {page}: {len(found)} posts ({len(new)} new)")
|
||||
if not new and page > 1:
|
||||
break
|
||||
post_urls.update(found)
|
||||
page += 1
|
||||
if page > 40:
|
||||
break
|
||||
|
||||
log(f"\nTotal recipe posts: {len(post_urls)}")
|
||||
(OUT / "_post_urls.txt").write_text("\n".join(v for _, v in sorted(post_urls.items())))
|
||||
|
||||
# ---- 2. fetch each post, collect images ----
|
||||
all_assets = set()
|
||||
page_assets = {} # local_file -> (html, asset_urls)
|
||||
|
||||
def handle_page(url, html=None):
|
||||
if html is None:
|
||||
data = curl(url)
|
||||
if not data:
|
||||
log(f" MISS {url}")
|
||||
return
|
||||
html = data.decode("utf-8", "replace")
|
||||
imgs = extract_imgs(html, url)
|
||||
lf = local_for(url)
|
||||
page_assets[lf] = (html, imgs)
|
||||
with LOCK:
|
||||
all_assets.update(imgs)
|
||||
log(f" post {url.rsplit('/',1)[-1]} ({len(imgs)} imgs)")
|
||||
|
||||
# listing pages
|
||||
for pno, html in listing_pages.items():
|
||||
u = f"{BASE}{CAT}?page={pno}"
|
||||
handle_page(u, html)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||||
list(as_completed([ex.submit(handle_page, u) for u in post_urls.values()]))
|
||||
|
||||
# ---- 3. download images ----
|
||||
log(f"\nDownloading {len(all_assets)} images ...")
|
||||
ok = fail = 0
|
||||
def dl(u):
|
||||
dest = local_for(u)
|
||||
if dest.exists() and dest.stat().st_size:
|
||||
return "skip"
|
||||
return "ok" if curl(u, dest) else "fail"
|
||||
with ThreadPoolExecutor(max_workers=6) as ex:
|
||||
futs = {ex.submit(dl, u): u for u in sorted(all_assets)}
|
||||
for f in as_completed(futs):
|
||||
r = f.result()
|
||||
if r == "ok": ok += 1
|
||||
elif r == "fail":
|
||||
fail += 1; log(f" img FAIL {futs[f]}")
|
||||
log(f"images: ok={ok} fail={fail} (skip={len(all_assets)-ok-fail})")
|
||||
|
||||
# ---- 4. write rewritten HTML ----
|
||||
for lf, (html, imgs) in page_assets.items():
|
||||
lf.parent.mkdir(parents=True, exist_ok=True)
|
||||
lf.write_text(rewrite(html, lf, all_assets), encoding="utf-8")
|
||||
log(f"\nWrote {len(page_assets)} HTML pages -> {OUT}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user