#!/usr/bin/env python3 """Paper of the day: pick the single most relevant unfeatured paper from the research feed, read the FULL paper (not the abstract), and produce a deep-dive podcast episode ending with what it means for the Lakota language work. Selection and summarization both run on the local model; the paper PDF comes from arXiv. featured.json prevents repeats. """ import os, sys, json, re, urllib.request, subprocess from datetime import datetime, timezone HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) sys.path.insert(0, os.path.dirname(HERE)) from make_episode import tts_to_mp3, rebuild_rss, llm_script, EPISODES # noqa: E402 from build_feed import RESEARCH_STATEMENT # noqa: E402 LLM_URL = "http://localhost:8000/v1/chat/completions" LLM = "gemma4-26b-a4b" BASE_URL = "https://edgexpert-83b2.tail6663b0.ts.net" DEEP_DIVE_PROMPT = """You are writing a spoken deep-dive podcast script (one host, 5 to 8 minutes) about ONE research paper, for a listener who builds community-sovereign language AI for Lakota (OCR of legacy texts, RAG grounded in community-approved books, per-stage evaluation, community-governed data). Cover, in plain spoken prose with no markdown and no URLs: 1. The problem the paper tackles and why it exists. 2. How their approach works — mechanics, not buzzwords. 3. The concrete findings, with the actual numbers. 4. Honest limitations. 5. Close with the most useful segment: what this means for the Lakota pipeline — what to adopt, what to test, what to ignore, and why. CRITICAL: never attempt to pronounce Lakota or other Indigenous-language words; describe them or spell them letter by letter, naming diacritics. PAPER TITLE: {title} WHY IT WAS FLAGGED: {why} PAPER TEXT (truncated): {text} """ def llm(prompt, max_tokens=600, temperature=0): body = {"model": LLM, "temperature": temperature, "max_tokens": max_tokens, "messages": [{"role": "user", "content": prompt}]} req = urllib.request.Request(LLM_URL, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=300) as r: return json.load(r)["choices"][0]["message"]["content"].strip() def pick_paper(candidates): """Ask the local model to choose the one paper most relevant to the work.""" listing = "\n".join(f"[{i}] {c['title']} — {c['why']}" for i, c in enumerate(candidates)) raw = llm(f"RESEARCH STATEMENT:\n{RESEARCH_STATEMENT}\n\nPAPERS:\n{listing}\n\n" "Which ONE paper is most worth a deep-dive for this researcher today? " 'Reply ONLY with JSON: {"index": , "reason": ""}', max_tokens=120) d = json.loads(re.search(r"\{.*\}", raw, re.S).group(0)) return candidates[int(d["index"])], d.get("reason", "") def fetch_paper_text(arxiv_url): """arXiv abs URL -> full text, or None if the paper has no fetchable body (some arXiv entries are abstract-only). Tries versioned PDF, unversioned PDF, then the HTML rendering.""" https = arxiv_url.replace("http://", "https://") tries = [https.replace("/abs/", "/pdf/"), re.sub(r"v\d+$", "", https.replace("/abs/", "/pdf/")), https.replace("/abs/", "/html/")] pdf_path = os.path.join(HERE, "_paper.pdf") for url in tries: try: req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=60) as r: data = r.read() except Exception: continue if url.startswith(https.replace("/abs/", "/html/")[:30]) and b"]+>", " ", data.decode("utf-8", "ignore")) txt = re.sub(r"\s+", " ", txt) else: open(pdf_path, "wb").write(data) try: txt = subprocess.run(["pdftotext", pdf_path, "-"], capture_output=True, text=True, check=True).stdout except subprocess.CalledProcessError: continue finally: if os.path.exists(pdf_path): os.remove(pdf_path) if len(txt) > 3000: return txt[:42000] + ("\n[...]\n" + txt[-12000:] if len(txt) > 54000 else "") return None def main(): history = json.load(open(os.path.join(os.path.dirname(HERE), "history.json"))) feat_path = os.path.join(HERE, "featured.json") featured = set(json.load(open(feat_path))) if os.path.exists(feat_path) else set() candidates = [h for h in history if h["id"] not in featured][:25] if not candidates: print("[paper] nothing unfeatured — skipping") return paper = text = reason = None for _ in range(5): if not candidates: break paper, reason = pick_paper(candidates) print(f"[paper] selected: {paper['title'][:70]} ({reason[:80]})", file=sys.stderr) text = fetch_paper_text(paper["id"]) if text: break print("[paper] no fetchable full text — skipping permanently", file=sys.stderr) featured.add(paper["id"]) # abstract-only: never worth retrying candidates = [c for c in candidates if c["id"] != paper["id"]] paper = None if not paper or not text: json.dump(sorted(featured), open(feat_path, "w")) print("[paper] no fetchable paper today") return script = llm(DEEP_DIVE_PROMPT.format(title=paper["title"], why=paper["why"], text=text), max_tokens=2200, temperature=0.3) stamp = datetime.now(timezone.utc).strftime("%Y%m%d") mp3 = os.path.join(EPISODES, f"{stamp}_paper_of_the_day.mp3") dur = tts_to_mp3(script, mp3) json.dump({"title": f"Paper of the day — {paper['title'][:80]}", "description": f"{reason} {paper['id']}"}, open(mp3.replace(".mp3", ".json"), "w")) featured.add(paper["id"]) json.dump(sorted(featured), open(feat_path, "w")) n = rebuild_rss(BASE_URL) print(f"[paper] episode ready ({dur}s) · feed has {n} items") if __name__ == "__main__": main()