diff --git a/k8s/gharaati-crawl-job.yaml b/k8s/gharaati-crawl-job.yaml
index 4e9e5b2..cbb5048 100644
--- a/k8s/gharaati-crawl-job.yaml
+++ b/k8s/gharaati-crawl-job.yaml
@@ -24,7 +24,8 @@ spec:
set -euo pipefail
apt-get update -qq
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ca-certificates >/dev/null
- python3 /scripts/crawl-gharaati.py /data/audio --skip-existing-json --delay 0.25
+ # Content only — PVC already has ~2997 Gooya MP3s; skip site audio re-download.
+ python3 /scripts/crawl-gharaati.py /data/audio --skip-existing-json --delay 0.15
volumeMounts:
- name: audio
mountPath: /data/audio
diff --git a/scripts/crawl-gharaati.py b/scripts/crawl-gharaati.py
index a62f08d..e85cc6f 100755
--- a/scripts/crawl-gharaati.py
+++ b/scripts/crawl-gharaati.py
@@ -1,5 +1,9 @@
#!/usr/bin/env python3
-"""Crawl Tafsir Noor from gharaati.ir into AUDIO_DIR/gharaati + content JSON."""
+"""Crawl Tafsir Noor text/notes/messages from gharaati.ir (JSON only by default).
+
+Existing PVC audio (Tafsir Gooya RARs) is kept; do not re-download site MP3s
+unless --with-audio is passed.
+"""
from __future__ import annotations
import argparse
@@ -8,7 +12,6 @@ import os
import re
import sys
import time
-import urllib.error
import urllib.request
from html import unescape
from pathlib import Path
@@ -38,20 +41,15 @@ def strip_tags(html: str) -> str:
def section_after_title(html: str, title: str) -> str | None:
- # Match heading text then first content cell with class tafsir or tafsir_ayeh
pattern = (
rf'\s*{re.escape(title)}\s*.*?'
rf'
]*class=[\'"]?(?:tafsir_ayeh|tafsir)[\'"]?[^>]*>(.*?) | '
)
m = re.search(pattern, html, flags=re.I | re.S)
- if not m:
- # alternate spellings
- return None
- return m.group(1).strip()
+ return m.group(1).strip() if m else None
def parse_show3(html: str) -> dict:
- # Normalize common title variants
ayah_html = section_after_title(html, "آيه") or section_after_title(html, "آیه")
tarjome = section_after_title(html, "ترجمه")
nokteh = section_after_title(html, "نکته ها") or section_after_title(html, "نكته ها")
@@ -98,18 +96,18 @@ def download_mp3(surah: int, ayah: int, dest: Path) -> bool:
return False
-def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float) -> dict:
+def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float, with_audio: bool) -> dict:
html = fetch(f"{BASE}/show3.php?page=tafsir3&numsooreh={surah}&numayeh={ayah}").decode(
"utf-8", "replace"
)
parsed = parse_show3(html)
- rel = f"gharaati/{surah}/{ayah}.mp3"
- mp3_path = audio_dir / rel
- download_mp3(surah, ayah, mp3_path)
- if mp3_path.exists():
- parsed["audioPath"] = rel
- else:
- parsed["audioPath"] = None
+ parsed["audioPath"] = None
+ if with_audio:
+ rel = f"gharaati/{surah}/{ayah}.mp3"
+ mp3_path = audio_dir / rel
+ download_mp3(surah, ayah, mp3_path)
+ if mp3_path.exists():
+ parsed["audioPath"] = rel
parsed["surah"] = surah
parsed["ayah"] = ayah
content_path = audio_dir / "gharaati" / "content" / str(surah) / f"{ayah}.json"
@@ -124,8 +122,13 @@ def main() -> int:
ap.add_argument("audio_dir", nargs="?", default=os.environ.get("AUDIO_DIR", "/data/audio"))
ap.add_argument("--surah-from", type=int, default=1)
ap.add_argument("--surah-to", type=int, default=114)
- ap.add_argument("--delay", type=float, default=0.35)
+ ap.add_argument("--delay", type=float, default=0.2)
ap.add_argument("--skip-existing-json", action="store_true")
+ ap.add_argument(
+ "--with-audio",
+ action="store_true",
+ help="Also download archive/tafsir/{s}/{a}.mp3 (usually unnecessary; PVC already has Gooya audio)",
+ )
args = ap.parse_args()
audio_dir = Path(args.audio_dir)
audio_dir.mkdir(parents=True, exist_ok=True)
@@ -136,24 +139,20 @@ def main() -> int:
if not ayahs:
print(f"surah {s}: no ayah list", file=sys.stderr)
continue
- print(f"surah {s}: {len(ayahs)} ayahs")
+ print(f"surah {s}: {len(ayahs)} ayahs", flush=True)
for a in ayahs:
jpath = audio_dir / "gharaati" / "content" / str(s) / f"{a}.json"
if args.skip_existing_json and jpath.exists():
- # still try mp3 if missing
- mp3 = audio_dir / "gharaati" / str(s) / f"{a}.mp3"
- if not mp3.exists():
- download_mp3(s, a, mp3)
continue
try:
- crawl_one(s, a, audio_dir, args.delay)
+ crawl_one(s, a, audio_dir, args.delay, args.with_audio)
total += 1
- if total % 25 == 0:
- print(f" crawled {total} ayahs (at {s}:{a})")
+ if total % 50 == 0:
+ print(f" crawled {total} ayahs (at {s}:{a})", flush=True)
except Exception as ex: # noqa: BLE001
print(f"FAIL {s}:{a} {ex}", file=sys.stderr)
time.sleep(2)
- print(f"done. crawled={total}")
+ print(f"done. crawled={total}", flush=True)
return 0
diff --git a/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs b/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs
index ceb8fbb..8cc5c75 100644
--- a/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs
+++ b/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs
@@ -97,18 +97,26 @@ public static class TafsirEndpoints
}).ToList();
}
+ var episodeAyahs = (await db.Episodes.AsNoTracking()
+ .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id)
+ .Select(e => e.AyahNumber)
+ .Distinct()
+ .ToListAsync(ct)).ToHashSet();
+
var list = ayahs.Select(a =>
{
contents.TryGetValue(a.Number, out var c);
var preview = a.TextFa;
if (!string.IsNullOrWhiteSpace(preview) && preview.Length > 80)
preview = preview[..80] + "…";
+ var hasAudio = !string.IsNullOrWhiteSpace(c?.AudioPath) || episodeAyahs.Contains(a.Number)
+ || episodeAyahs.Count > 0;
return new AyahListItemDto(
surah.Number,
surah.NameFa,
a.Number,
preview,
- !string.IsNullOrWhiteSpace(c?.AudioPath),
+ hasAudio,
bookmarks.Contains(a.Number),
favorites.Contains(a.Number));
}).ToList();
@@ -156,7 +164,24 @@ public static class TafsirEndpoints
string? audioUrl = null;
if (!string.IsNullOrWhiteSpace(content?.AudioPath))
+ {
audioUrl = $"/api/audio/ayah/{surahNumber}/{ayahNumber}?tafsirId={tafsir.Id}";
+ }
+ else
+ {
+ // Fall back to existing Gooya episode audio already on PVC (best-effort ayah match).
+ var episode = await db.Episodes.AsNoTracking()
+ .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id && e.AyahNumber == ayahNumber)
+ .OrderBy(e => e.SortOrder)
+ .FirstOrDefaultAsync(ct);
+ episode ??= await db.Episodes.AsNoTracking()
+ .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id)
+ .OrderBy(e => Math.Abs(e.AyahNumber - ayahNumber))
+ .ThenBy(e => e.SortOrder)
+ .FirstOrDefaultAsync(ct);
+ if (episode is not null)
+ audioUrl = $"/api/audio/{episode.Id}";
+ }
return Results.Ok(new AyahDetailDto(
tafsir.Id,