diff --git a/k8s/gharaati-crawl-job.yaml b/k8s/gharaati-crawl-job.yaml index 4e9e5b2..cbb5048 100644 --- a/k8s/gharaati-crawl-job.yaml +++ b/k8s/gharaati-crawl-job.yaml @@ -24,7 +24,8 @@ spec: set -euo pipefail apt-get update -qq DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ca-certificates >/dev/null - python3 /scripts/crawl-gharaati.py /data/audio --skip-existing-json --delay 0.25 + # Content only — PVC already has ~2997 Gooya MP3s; skip site audio re-download. + python3 /scripts/crawl-gharaati.py /data/audio --skip-existing-json --delay 0.15 volumeMounts: - name: audio mountPath: /data/audio diff --git a/scripts/crawl-gharaati.py b/scripts/crawl-gharaati.py index a62f08d..e85cc6f 100755 --- a/scripts/crawl-gharaati.py +++ b/scripts/crawl-gharaati.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Crawl Tafsir Noor from gharaati.ir into AUDIO_DIR/gharaati + content JSON.""" +"""Crawl Tafsir Noor text/notes/messages from gharaati.ir (JSON only by default). + +Existing PVC audio (Tafsir Gooya RARs) is kept; do not re-download site MP3s +unless --with-audio is passed. +""" from __future__ import annotations import argparse @@ -8,7 +12,6 @@ import os import re import sys import time -import urllib.error import urllib.request from html import unescape from pathlib import Path @@ -38,20 +41,15 @@ def strip_tags(html: str) -> str: def section_after_title(html: str, title: str) -> str | None: - # Match heading text then first content cell with class tafsir or tafsir_ayeh pattern = ( rf'\s*{re.escape(title)}\s*.*?' rf']*class=[\'"]?(?:tafsir_ayeh|tafsir)[\'"]?[^>]*>(.*?)' ) m = re.search(pattern, html, flags=re.I | re.S) - if not m: - # alternate spellings - return None - return m.group(1).strip() + return m.group(1).strip() if m else None def parse_show3(html: str) -> dict: - # Normalize common title variants ayah_html = section_after_title(html, "آيه") or section_after_title(html, "آیه") tarjome = section_after_title(html, "ترجمه") nokteh = section_after_title(html, "نکته ها") or section_after_title(html, "نكته ها") @@ -98,18 +96,18 @@ def download_mp3(surah: int, ayah: int, dest: Path) -> bool: return False -def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float) -> dict: +def crawl_one(surah: int, ayah: int, audio_dir: Path, delay: float, with_audio: bool) -> dict: html = fetch(f"{BASE}/show3.php?page=tafsir3&numsooreh={surah}&numayeh={ayah}").decode( "utf-8", "replace" ) parsed = parse_show3(html) - rel = f"gharaati/{surah}/{ayah}.mp3" - mp3_path = audio_dir / rel - download_mp3(surah, ayah, mp3_path) - if mp3_path.exists(): - parsed["audioPath"] = rel - else: - parsed["audioPath"] = None + parsed["audioPath"] = None + if with_audio: + rel = f"gharaati/{surah}/{ayah}.mp3" + mp3_path = audio_dir / rel + download_mp3(surah, ayah, mp3_path) + if mp3_path.exists(): + parsed["audioPath"] = rel parsed["surah"] = surah parsed["ayah"] = ayah content_path = audio_dir / "gharaati" / "content" / str(surah) / f"{ayah}.json" @@ -124,8 +122,13 @@ def main() -> int: ap.add_argument("audio_dir", nargs="?", default=os.environ.get("AUDIO_DIR", "/data/audio")) ap.add_argument("--surah-from", type=int, default=1) ap.add_argument("--surah-to", type=int, default=114) - ap.add_argument("--delay", type=float, default=0.35) + ap.add_argument("--delay", type=float, default=0.2) ap.add_argument("--skip-existing-json", action="store_true") + ap.add_argument( + "--with-audio", + action="store_true", + help="Also download archive/tafsir/{s}/{a}.mp3 (usually unnecessary; PVC already has Gooya audio)", + ) args = ap.parse_args() audio_dir = Path(args.audio_dir) audio_dir.mkdir(parents=True, exist_ok=True) @@ -136,24 +139,20 @@ def main() -> int: if not ayahs: print(f"surah {s}: no ayah list", file=sys.stderr) continue - print(f"surah {s}: {len(ayahs)} ayahs") + print(f"surah {s}: {len(ayahs)} ayahs", flush=True) for a in ayahs: jpath = audio_dir / "gharaati" / "content" / str(s) / f"{a}.json" if args.skip_existing_json and jpath.exists(): - # still try mp3 if missing - mp3 = audio_dir / "gharaati" / str(s) / f"{a}.mp3" - if not mp3.exists(): - download_mp3(s, a, mp3) continue try: - crawl_one(s, a, audio_dir, args.delay) + crawl_one(s, a, audio_dir, args.delay, args.with_audio) total += 1 - if total % 25 == 0: - print(f" crawled {total} ayahs (at {s}:{a})") + if total % 50 == 0: + print(f" crawled {total} ayahs (at {s}:{a})", flush=True) except Exception as ex: # noqa: BLE001 print(f"FAIL {s}:{a} {ex}", file=sys.stderr) time.sleep(2) - print(f"done. crawled={total}") + print(f"done. crawled={total}", flush=True) return 0 diff --git a/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs b/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs index ceb8fbb..8cc5c75 100644 --- a/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs +++ b/src/TafsirQuran.Api/Endpoints/TafsirEndpoints.cs @@ -97,18 +97,26 @@ public static class TafsirEndpoints }).ToList(); } + var episodeAyahs = (await db.Episodes.AsNoTracking() + .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id) + .Select(e => e.AyahNumber) + .Distinct() + .ToListAsync(ct)).ToHashSet(); + var list = ayahs.Select(a => { contents.TryGetValue(a.Number, out var c); var preview = a.TextFa; if (!string.IsNullOrWhiteSpace(preview) && preview.Length > 80) preview = preview[..80] + "…"; + var hasAudio = !string.IsNullOrWhiteSpace(c?.AudioPath) || episodeAyahs.Contains(a.Number) + || episodeAyahs.Count > 0; return new AyahListItemDto( surah.Number, surah.NameFa, a.Number, preview, - !string.IsNullOrWhiteSpace(c?.AudioPath), + hasAudio, bookmarks.Contains(a.Number), favorites.Contains(a.Number)); }).ToList(); @@ -156,7 +164,24 @@ public static class TafsirEndpoints string? audioUrl = null; if (!string.IsNullOrWhiteSpace(content?.AudioPath)) + { audioUrl = $"/api/audio/ayah/{surahNumber}/{ayahNumber}?tafsirId={tafsir.Id}"; + } + else + { + // Fall back to existing Gooya episode audio already on PVC (best-effort ayah match). + var episode = await db.Episodes.AsNoTracking() + .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id && e.AyahNumber == ayahNumber) + .OrderBy(e => e.SortOrder) + .FirstOrDefaultAsync(ct); + episode ??= await db.Episodes.AsNoTracking() + .Where(e => e.TafsirId == tafsir.Id && e.SurahId == surah.Id) + .OrderBy(e => Math.Abs(e.AyahNumber - ayahNumber)) + .ThenBy(e => e.SortOrder) + .FirstOrDefaultAsync(ct); + if (episode is not null) + audioUrl = $"/api/audio/{episode.Id}"; + } return Results.Ok(new AyahDetailDto( tafsir.Id,