Skip re-downloading site MP3s; reuse existing Gooya audio on PVC.

Crawl only ayah text/notes/messages; API falls back to episode audio.

k8s/gharaati-crawl-job.yaml

@@ -24,7 +24,8 @@ spec:
               set -euo pipefail
               apt-get update -qq
               DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ca-certificates >/dev/null
-              python3 /scripts/crawl-gharaati.py /data/audio --skip-existing-json --delay 0.25
+              # Content only — PVC already has ~2997 Gooya MP3s; skip site audio re-download.
+              python3 /scripts/crawl-gharaati.py /data/audio --skip-existing-json --delay 0.15
           volumeMounts:
             - name: audio
               mountPath: /data/audio

scripts/crawl-gharaati.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-"""Crawl Tafsir Noor from gharaati.ir into AUDIO_DIR/gharaati + content JSON."""
+"""Crawl Tafsir Noor text/notes/messages from gharaati.ir (JSON only by default).
+
+Existing PVC audio (Tafsir Gooya RARs) is kept; do not re-download site MP3s
+unless --with-audio is passed.
+"""
 from __future__ import annotations
 
 import argparse
@@ -8,7 +12,6 @@ import os
 import re
 import sys
 import time
-import urllib.error
 import urllib.request
 from html import unescape
 from pathlib import Path
@@ -38,20 +41,15 @@ def strip_tags(html: str) -> str:
 
 
 def section_after_title(html: str, title: str) -> str | None:
-    # Match heading text then first content cell with class tafsir or tafsir_ayeh
     pattern = (
         rf'<span class="post-title h-text">\s*{re.escape(title)}\s*</span>.*?'
         rf'<td[^>]*class=[\'"]?(?:tafsir_ayeh|tafsir)[\'"]?[^>]*>(.*?)</td>'
     )
     m = re.search(pattern, html, flags=re.I | re.S)
-    if not m:
-        # alternate spellings
-        return None
-    return m.group(1).strip()
+    return m.group(1).strip() if m else None
 
 
 def parse_show3(html: str) -> dict:
-    # Normalize common title variants
     ayah_html = section_after_title(html, "آيه") or section_after_title(html, "آیه")
     tarjome = section_after_title(html, "ترجمه")
     nokteh = section_after_title(html, "نکته ها") or section_after_title(html, "نكته ها")
@@ -98,18 +96,18 @@ def download_mp3(surah: int, ayah: int, dest: Path) -> bool:
         return False
 
 
-def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float) -> dict:
+def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float, with_audio: bool) -> dict:
     html = fetch(f"{BASE}/show3.php?page=tafsir3&numsooreh={surah}&numayeh={ayah}").decode(
         "utf-8", "replace"
     )
     parsed = parse_show3(html)
-    rel = f"gharaati/{surah}/{ayah}.mp3"
-    mp3_path = audio_dir / rel
-    download_mp3(surah, ayah, mp3_path)
-    if mp3_path.exists():
-        parsed["audioPath"] = rel
-    else:
-        parsed["audioPath"] = None
+    parsed["audioPath"] = None
+    if with_audio:
+        rel = f"gharaati/{surah}/{ayah}.mp3"
+        mp3_path = audio_dir / rel
+        download_mp3(surah, ayah, mp3_path)
+        if mp3_path.exists():
+            parsed["audioPath"] = rel
     parsed["surah"] = surah
     parsed["ayah"] = ayah
     content_path = audio_dir / "gharaati" / "content" / str(surah) / f"{ayah}.json"
@@ -124,8 +122,13 @@ def main() -> int:
     ap.add_argument("audio_dir", nargs="?", default=os.environ.get("AUDIO_DIR", "/data/audio"))
     ap.add_argument("--surah-from", type=int, default=1)
     ap.add_argument("--surah-to", type=int, default=114)
-    ap.add_argument("--delay", type=float, default=0.35)
+    ap.add_argument("--delay", type=float, default=0.2)
     ap.add_argument("--skip-existing-json", action="store_true")
+    ap.add_argument(
+        "--with-audio",
+        action="store_true",
+        help="Also download archive/tafsir/{s}/{a}.mp3 (usually unnecessary; PVC already has Gooya audio)",
+    )
     args = ap.parse_args()
     audio_dir = Path(args.audio_dir)
     audio_dir.mkdir(parents=True, exist_ok=True)
@@ -136,24 +139,20 @@ def main() -> int:
         if not ayahs:
             print(f"surah {s}: no ayah list", file=sys.stderr)
             continue
-        print(f"surah {s}: {len(ayahs)} ayahs")
+        print(f"surah {s}: {len(ayahs)} ayahs", flush=True)
         for a in ayahs:
             jpath = audio_dir / "gharaati" / "content" / str(s) / f"{a}.json"
             if args.skip_existing_json and jpath.exists():
-                # still try mp3 if missing
-                mp3 = audio_dir / "gharaati" / str(s) / f"{a}.mp3"
-                if not mp3.exists():
-                    download_mp3(s, a, mp3)
                 continue
             try:
-                crawl_one(s, a, audio_dir, args.delay)
+                crawl_one(s, a, audio_dir, args.delay, args.with_audio)
                 total += 1
-                if total % 25 == 0:
-                    print(f"  crawled {total} ayahs (at {s}:{a})")
+                if total % 50 == 0:
+                    print(f"  crawled {total} ayahs (at {s}:{a})", flush=True)
             except Exception as ex:  # noqa: BLE001
                 print(f"FAIL {s}:{a} {ex}", file=sys.stderr)
                 time.sleep(2)
-    print(f"done. crawled={total}")
+    print(f"done. crawled={total}", flush=True)
     return 0
 
 

src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs

@@ -97,18 +97,26 @@ public static class TafsirEndpoints
                 }).ToList();
             }
 
+                var episodeAyahs = (await db.Episodes.AsNoTracking()
+                    .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id)
+                    .Select(e => e.AyahNumber)
+                    .Distinct()
+                    .ToListAsync(ct)).ToHashSet();
+
             var list = ayahs.Select(a =>
             {
                 contents.TryGetValue(a.Number, out var c);
                 var preview = a.TextFa;
                 if (!string.IsNullOrWhiteSpace(preview) && preview.Length > 80)
                     preview = preview[..80] + "…";
+                var hasAudio = !string.IsNullOrWhiteSpace(c?.AudioPath) || episodeAyahs.Contains(a.Number)
+                               || episodeAyahs.Count > 0;
                 return new AyahListItemDto(
                     surah.Number,
                     surah.NameFa,
                     a.Number,
                     preview,
-                    !string.IsNullOrWhiteSpace(c?.AudioPath),
+                    hasAudio,
                     bookmarks.Contains(a.Number),
                     favorites.Contains(a.Number));
             }).ToList();
@@ -156,7 +164,24 @@ public static class TafsirEndpoints
 
             string? audioUrl = null;
             if (!string.IsNullOrWhiteSpace(content?.AudioPath))
+            {
                 audioUrl = $"/api/audio/ayah/{surahNumber}/{ayahNumber}?tafsirId={tafsir.Id}";
+            }
+            else
+            {
+                // Fall back to existing Gooya episode audio already on PVC (best-effort ayah match).
+                var episode = await db.Episodes.AsNoTracking()
+                    .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id && e.AyahNumber == ayahNumber)
+                    .OrderBy(e => e.SortOrder)
+                    .FirstOrDefaultAsync(ct);
+                episode ??= await db.Episodes.AsNoTracking()
+                    .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id)
+                    .OrderBy(e => Math.Abs(e.AyahNumber - ayahNumber))
+                    .ThenBy(e => e.SortOrder)
+                    .FirstOrDefaultAsync(ct);
+                if (episode is not null)
+                    audioUrl = $"/api/audio/{episode.Id}";
+            }
 
             return Results.Ok(new AyahDetailDto(
                 tafsir.Id,