Skip re-downloading site MP3s; reuse existing Gooya audio on PVC.

Crawl only ayah text/notes/messages; API falls back to episode audio.

scripts/crawl-gharaati.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-"""Crawl Tafsir Noor from gharaati.ir into AUDIO_DIR/gharaati + content JSON."""
+"""Crawl Tafsir Noor text/notes/messages from gharaati.ir (JSON only by default).
+
+Existing PVC audio (Tafsir Gooya RARs) is kept; do not re-download site MP3s
+unless --with-audio is passed.
+"""
 from __future__ import annotations
 
 import argparse
@@ -8,7 +12,6 @@ import os
 import re
 import sys
 import time
-import urllib.error
 import urllib.request
 from html import unescape
 from pathlib import Path
@@ -38,20 +41,15 @@ def strip_tags(html: str) -> str:
 
 
 def section_after_title(html: str, title: str) -> str | None:
-    # Match heading text then first content cell with class tafsir or tafsir_ayeh
     pattern = (
         rf'<span class="post-title h-text">\s*{re.escape(title)}\s*</span>.*?'
         rf'<td[^>]*class=[\'"]?(?:tafsir_ayeh|tafsir)[\'"]?[^>]*>(.*?)</td>'
     )
     m = re.search(pattern, html, flags=re.I | re.S)
-    if not m:
-        # alternate spellings
-        return None
-    return m.group(1).strip()
+    return m.group(1).strip() if m else None
 
 
 def parse_show3(html: str) -> dict:
-    # Normalize common title variants
     ayah_html = section_after_title(html, "آيه") or section_after_title(html, "آیه")
     tarjome = section_after_title(html, "ترجمه")
     nokteh = section_after_title(html, "نکته ها") or section_after_title(html, "نكته ها")
@@ -98,18 +96,18 @@ def download_mp3(surah: int, ayah: int, dest: Path) -> bool:
         return False
 
 
-def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float) -> dict:
+def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float, with_audio: bool) -> dict:
     html = fetch(f"{BASE}/show3.php?page=tafsir3&numsooreh={surah}&numayeh={ayah}").decode(
         "utf-8", "replace"
     )
     parsed = parse_show3(html)
-    rel = f"gharaati/{surah}/{ayah}.mp3"
-    mp3_path = audio_dir / rel
-    download_mp3(surah, ayah, mp3_path)
-    if mp3_path.exists():
-        parsed["audioPath"] = rel
-    else:
-        parsed["audioPath"] = None
+    parsed["audioPath"] = None
+    if with_audio:
+        rel = f"gharaati/{surah}/{ayah}.mp3"
+        mp3_path = audio_dir / rel
+        download_mp3(surah, ayah, mp3_path)
+        if mp3_path.exists():
+            parsed["audioPath"] = rel
     parsed["surah"] = surah
     parsed["ayah"] = ayah
     content_path = audio_dir / "gharaati" / "content" / str(surah) / f"{ayah}.json"
@@ -124,8 +122,13 @@ def main() -> int:
     ap.add_argument("audio_dir", nargs="?", default=os.environ.get("AUDIO_DIR", "/data/audio"))
     ap.add_argument("--surah-from", type=int, default=1)
     ap.add_argument("--surah-to", type=int, default=114)
-    ap.add_argument("--delay", type=float, default=0.35)
+    ap.add_argument("--delay", type=float, default=0.2)
     ap.add_argument("--skip-existing-json", action="store_true")
+    ap.add_argument(
+        "--with-audio",
+        action="store_true",
+        help="Also download archive/tafsir/{s}/{a}.mp3 (usually unnecessary; PVC already has Gooya audio)",
+    )
     args = ap.parse_args()
     audio_dir = Path(args.audio_dir)
     audio_dir.mkdir(parents=True, exist_ok=True)
@@ -136,24 +139,20 @@ def main() -> int:
         if not ayahs:
             print(f"surah {s}: no ayah list", file=sys.stderr)
             continue
-        print(f"surah {s}: {len(ayahs)} ayahs")
+        print(f"surah {s}: {len(ayahs)} ayahs", flush=True)
         for a in ayahs:
             jpath = audio_dir / "gharaati" / "content" / str(s) / f"{a}.json"
             if args.skip_existing_json and jpath.exists():
-                # still try mp3 if missing
-                mp3 = audio_dir / "gharaati" / str(s) / f"{a}.mp3"
-                if not mp3.exists():
-                    download_mp3(s, a, mp3)
                 continue
             try:
-                crawl_one(s, a, audio_dir, args.delay)
+                crawl_one(s, a, audio_dir, args.delay, args.with_audio)
                 total += 1
-                if total % 25 == 0:
-                    print(f"  crawled {total} ayahs (at {s}:{a})")
+                if total % 50 == 0:
+                    print(f"  crawled {total} ayahs (at {s}:{a})", flush=True)
             except Exception as ex:  # noqa: BLE001
                 print(f"FAIL {s}:{a} {ex}", file=sys.stderr)
                 time.sleep(2)
-    print(f"done. crawled={total}")
+    print(f"done. crawled={total}", flush=True)
     return 0