release: v2.6.7 — Reduce enrichment CI time from 12min to ~2min

perf: reduce enrichment from 12min to ~2min

src/enrichment.py

@@ -13,10 +13,10 @@ from src.logger import log_event
 # ---------------------------------------------------------------------------
 # Constants
 # ---------------------------------------------------------------------------
-CNCF_LANDSCAPE_URL = "https://landscape.cncf.io/api/items"
+CNCF_LANDSCAPE_URL = "https://landscape.cncf.io/api/items"  # Legacy, now SPA — fallback to GitHub topic search
 GITHUB_API_BASE = "https://api.github.com"
-GITHUB_RATE_DELAY = 0.75  # seconds between GitHub API calls to stay under 5000/hr
-MAX_REPOS_DEFAULT = 500
+GITHUB_RATE_DELAY = 0.5   # seconds between GitHub API calls (5000/hr limit = ~1.4/s safe)
+MAX_REPOS_DEFAULT = 200   # cap per run — 200 × 0.5s = ~100s, well within CI timeout
 ACTIVITY_STALENESS_DAYS = 30
 
 # Community health thresholds
@@ -74,38 +74,37 @@ def _is_activity_stale(entry: Dict) -> bool:
 # ---------------------------------------------------------------------------
 
 async def fetch_cncf_landscape() -> Dict[str, str]:
-    """Fetch CNCF project graduation status.
+    """Fetch CNCF project graduation status via GitHub topic search.
 
-    Returns dict mapping repo_url (normalized) -> maturity
-    ("sandbox" | "incubating" | "graduated" | "archived").
+    The legacy landscape.cncf.io/api/items endpoint is now a SPA and no
+    longer returns JSON.  Instead, we search GitHub for repos with CNCF
+    maturity topics (cncf-sandbox, cncf-incubating, cncf-graduated).
+
+    Returns dict mapping repo_url (normalized) -> maturity.
     """
     result: Dict[str, str] = {}
+    headers = _github_headers()
+    maturity_queries = {
+        "graduated": "topic:cncf-graduated",
+        "incubating": "topic:cncf-incubating",
+        "sandbox": "topic:cncf-sandbox",
+    }
+
     async with httpx.AsyncClient(timeout=30.0) as client:
-        try:
-            resp = await client.get(CNCF_LANDSCAPE_URL)
-            resp.raise_for_status()
-            items = resp.json()
-        except Exception as e:
-            log_event(f"[WARN] Failed to fetch CNCF landscape: {str(e)[:200]}")
-            return result
-
-    if not isinstance(items, list):
-        log_event("[WARN] CNCF landscape response is not a list; skipping")
-        return result
-
-    for item in items:
-        repo_url = item.get("repo_url") or ""
-        project = item.get("project")
-        if not repo_url or not project:
-            continue
-        maturity = project if isinstance(project, str) else project.get("maturity", "")
-        if not maturity:
-            continue
-        maturity = maturity.lower()
-        if maturity not in ("sandbox", "incubating", "graduated", "archived"):
-            continue
-        normalized = _normalize_repo_url(repo_url)
-        result[normalized] = maturity
+        for maturity, query in maturity_queries.items():
+            try:
+                url = f"{GITHUB_API_BASE}/search/repositories?q={query}&per_page=100"
+                resp = await client.get(url, headers=headers)
+                resp.raise_for_status()
+                data = resp.json()
+                for repo in data.get("items", []):
+                    repo_url = repo.get("html_url", "")
+                    if repo_url:
+                        result[_normalize_repo_url(repo_url)] = maturity
+                log_event(f"  [CNCF] {maturity}: {len(data.get('items', []))} repos found")
+            except Exception as e:
+                log_event(f"[WARN] CNCF {maturity} search failed: {str(e)[:100]}")
+            await asyncio.sleep(GITHUB_RATE_DELAY)
 
     log_event(f"[CNCF] Fetched {len(result)} projects from CNCF landscape")
     return result
@@ -156,15 +155,15 @@ async def _fetch_repo_activity(
     client: httpx.AsyncClient,
     owner_repo: str,
 ) -> Tuple[int, int]:
-    """Fetch open issue count and recent open PR count for a single repo.
+    """Fetch open issue + PR count for a single repo in one API call.
 
-    Returns (open_issues_count, open_prs_count).
+    GitHub's open_issues_count includes PRs, so a single /repos endpoint
+    call is sufficient. Returns (open_issues_count, 0) — the caller uses
+    the combined metric for health classification.
     """
     headers = _github_headers()
     open_issues = 0
-    open_prs = 0
 
-    # Fetch repo-level stats (open_issues_count includes PRs on GitHub)
     try:
         resp = await client.get(
             f"{GITHUB_API_BASE}/repos/{owner_repo}",
@@ -174,37 +173,15 @@ async def _fetch_repo_activity(
         if resp.status_code == 200:
             data = resp.json()
             open_issues = data.get("open_issues_count", 0)
+        elif resp.status_code == 429:
+            # Rate limited — back off
+            await asyncio.sleep(5.0)
     except Exception as e:
         log_event(f"[WARN] GitHub repo fetch failed for {owner_repo}: {str(e)[:120]}")
 
     await asyncio.sleep(GITHUB_RATE_DELAY)
 
-    # Fetch open PRs (page 1 only — we use total_count from search or headers)
-    try:
-        resp = await client.get(
-            f"{GITHUB_API_BASE}/repos/{owner_repo}/pulls",
-            headers=headers,
-            params={"state": "open", "sort": "created", "per_page": 1},
-            timeout=15.0,
-        )
-        if resp.status_code == 200:
-            # The total count of open PRs is not directly in the body for the
-            # list endpoint, but we can parse the Link header for the last page.
-            # As a simpler approach, the body is a list; if it has items the repo
-            # has open PRs. We use the repo-level open_issues_count as the
-            # combined metric (GitHub counts PRs as issues).
-            pr_data = resp.json()
-            if isinstance(pr_data, list) and len(pr_data) > 0:
-                # Parse Link header for total pages
-                link_header = resp.headers.get("Link", "")
-                last_match = re.search(r'page=(\d+)>;\s*rel="last"', link_header)
-                open_prs = int(last_match.group(1)) if last_match else len(pr_data)
-    except Exception as e:
-        log_event(f"[WARN] GitHub PRs fetch failed for {owner_repo}: {str(e)[:120]}")
-
-    await asyncio.sleep(GITHUB_RATE_DELAY)
-
-    return open_issues, open_prs
+    return open_issues, 0
 
 
 def _classify_health(total_activity: int) -> str:

src/news_digest.py

@@ -145,6 +145,12 @@ class NewsDigestEngine:
         "12_months": 365,
     }
 
+    ITEMS_PER_PERIOD: Dict[str, int] = {
+        "3_months": 10,
+        "6_months": 15,
+        "12_months": 20,
+    }
+
     # ------------------------------------------------------------------ #
 
     def __init__(self) -> None:
@@ -185,17 +191,20 @@ class NewsDigestEngine:
 
     @staticmethod
     def _is_within_period(entry: dict, cutoff_iso: str) -> bool:
-        """Return *True* when the entry's ``discovered_at`` is on or after
-        the ISO-formatted *cutoff_iso* string.  ISO 8601 strings sort
-        lexicographically so a simple ``>=`` comparison is sufficient.
-        """
+        """Check if entry falls within the time period using discovered_at,
+        with year field as fallback for backfilled entries."""
         discovered = entry.get("discovered_at", "")
-        if not discovered:
-            return False
-        try:
-            return discovered >= cutoff_iso
-        except Exception:
-            return False
+        if discovered:
+            try:
+                if discovered >= cutoff_iso:
+                    return True
+            except Exception:
+                pass
+        year = entry.get("year", "")
+        if year and isinstance(year, str) and year.isdigit():
+            cutoff_year = cutoff_iso[:4] if len(cutoff_iso) >= 4 else "2020"
+            return year >= cutoff_year
+        return False
 
     # ------------------------------------------------------------------ #
     # Prompt builder                                                      #
@@ -253,8 +262,8 @@ class NewsDigestEngine:
                 "url": e["url"],
                 "title": e.get("title", "Unknown"),
                 "date": e.get("discovered_at", "")[:10],
-                "stars": e.get("stars", 0),
-                "impact": "high" if e.get("stars", 0) >= 4 else "medium",
+                "stars": e.get("stars") or 0,
+                "impact": "high" if (e.get("stars") or 0) >= 4 else "medium",
                 "why": (e.get("ai_summary", "") or "")[:200],
                 "category": cat_name,
             }
@@ -316,14 +325,14 @@ class NewsDigestEngine:
                     reverse=True,
                 )
 
+                max_items = self.ITEMS_PER_PERIOD.get(period_name, 10)
+
                 if len(entries) < 3:
-                    # Too few entries – include all without AI ranking
                     digest[period_name][cat_name] = self._fallback_items(
-                        entries, cat_name
+                        entries, cat_name, limit=max_items
                     )
                     continue
 
-                # Ask Gemini to rank
                 try:
                     prompt = self._build_ranking_prompt(
                         cat_name, entries, period_name
@@ -351,7 +360,7 @@ class NewsDigestEngine:
                                 }
                             )
 
-                    digest[period_name][cat_name] = ranked[:10]
+                    digest[period_name][cat_name] = ranked[:max_items]
                     log_event(
                         f"  [Digest] {period_name}/{cat_name}: "
                         f"{len(ranked)} items ranked"
@@ -364,7 +373,7 @@ class NewsDigestEngine:
                         "using star-based fallback"
                     )
                     digest[period_name][cat_name] = self._fallback_items(
-                        entries, cat_name
+                        entries, cat_name, limit=max_items
                     )
 
                 # Respect Gemini rate limits

src/v2_optimizer.py

@@ -964,13 +964,14 @@ class V2VisionEngine:
                     items = period_data.get(cat, [])
                     if not items:
                         continue
-                    md += f"    ## {cat}\n\n"
+                    md += f"    **{cat}**\n\n"
                     md += "    | Date | Resource | Impact | Why It Matters |\n"
                     md += "    | :--- | :--- | :---: | :--- |\n"
                     for item in items:
                         impact_badge = {"critical": "🔴", "high": "🟡", "medium": "🔵"}.get(item.get("impact", "medium"), "🔵")
                         t = nuclear_strip(item.get("title", "Unknown"))
-                        md += f'    | {item.get("date", "")} | [{t}]({item.get("url", "#")}) | {impact_badge} {item.get("impact", "medium")} | {item.get("why", "")} |\n'
+                        why = (item.get("why", "") or "").replace("|", "-").replace("\n", " ")
+                        md += f'    | {item.get("date", "")} | [{t}]({item.get("url", "#")}) | {impact_badge} {item.get("impact", "medium")} | {why} |\n'
                     md += "\n"
                 md += "\n"
             return md