github/awesome-kubernetes
1
0
Fork 0
mirror of https://github.com/nubenetes/awesome-kubernetes.git synced 2026-07-12 18:00:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: incremental digest engine — reuse Gemini results when inventory unchanged

Browse Source 
Each (category × period) cell in news_digest.json now carries a hash of its
entry pool (sorted URLs). Before calling Gemini, the engine checks:
  1. Hash matches stored hash (same inventory entries)
  2. Last analyzed within MAX_STALENESS_DAYS=30

Both conditions true → reuse existing ranked list, 0 Gemini API calls.
Only cells where the inventory actually changed (new/removed entries) trigger
a fresh Gemini ranking.

Expected savings: re-runs with code-only changes → 0 API calls (down from 66).
New ingestion batch → only affected categories call Gemini (5-10 of 26 typical).
Stale refresh (>30d) → full refresh at most once per month.

Also:
- Remove day-level GitHub Actions cache for digest (redundant: news_digest.json
  is committed to the repo and is the persistent store for _meta)
- Always run news_digest.py in both 04.1 and 09 workflows (it's cheap when 0
  Gemini calls are needed)
- Trending badge now reads _meta.last_updated (actual analysis date) instead of
  file mtime (which changes on every render/commit)
- Remove invalid extra_head key from v2-mkdocs.yml (left over from rollback)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Nubenetes Bot
2026-06-19 14:09:11 +02:00
parent 72bb3c5150
commit e773676a70
5 changed files with 165 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/04.1.agentic_v2_publish.yml vendored
View File

@@ -61,19 +61,6 @@ jobs:
run: |
pip install --no-cache-dir pydantic PyGithub httpx fake-useragent pytz python-dotenv pyyaml tenacity
- name: Get current date for digest cache key
id: digest-date
run: echo "date=$(date -u +%Y-%m-%d)" >> $GITHUB_OUTPUT
- name: Restore News Digest Cache
id: cache-digest
uses: actions/cache/restore@v5
with:
path: data/news_digest.json
key: news-digest-${{ steps.digest-date.outputs.date }}
restore-keys: |
news-digest-
- name: Execute Video Portal Generator
env:
PYTHONPATH: ${{ github.workspace }}
@@ -99,13 +86,16 @@ jobs:
run: |
python -u -m src.enrichment || echo "Enrichment pipeline skipped (no token or error)"
- name: Generate News Digest
if: steps.cache-digest.outputs.cache-hit != 'true'
- name: Generate News Digest (Incremental)
env:
PYTHONPATH: ${{ github.workspace }}
GEMINI_API_KEY_1: ${{ secrets.GEMINI_API_KEY_1 }}
GEMINI_API_KEY_2: ${{ secrets.GEMINI_API_KEY_2 }}
run: |
# The engine loads news_digest.json from the repo, checks per-cell
# hashes, and only calls Gemini for categories with new entries or
# results older than MAX_STALENESS_DAYS (30). Typical re-runs with
# no inventory changes cost 0 Gemini API calls.
python -u -m src.news_digest || echo "News digest generation skipped (no API key or error)"
- name: Generate RSS Feed
@@ -114,13 +104,6 @@ jobs:
run: |
python -u -m src.rss_generator || echo "RSS generation skipped"
- name: Save News Digest Cache
if: steps.cache-digest.outputs.cache-hit != 'true' && hashFiles('data/news_digest.json') != ''
uses: actions/cache/save@v5
with:
path: data/news_digest.json
key: news-digest-${{ steps.digest-date.outputs.date }}
- name: Run V2 Publisher (Render-Only)
env:
PYTHONPATH: ${{ github.workspace }}

26
.github/workflows/09.weekly_digest.yml vendored
View File

@@ -33,35 +33,17 @@ jobs:
- name: Install Dependencies
run: pip install -r requirements.txt
- name: Get current date for digest cache key
id: digest-date
run: echo "date=$(date -u +%Y-%m-%d)" >> $GITHUB_OUTPUT
- name: Restore News Digest Cache
id: cache-digest
uses: actions/cache/restore@v5
with:
path: data/news_digest.json
key: news-digest-${{ steps.digest-date.outputs.date }}
restore-keys: |
news-digest-
- name: Generate News Digest (Gemini)
if: steps.cache-digest.outputs.cache-hit != 'true'
- name: Generate News Digest (Incremental)
env:
PYTHONPATH: ${{ github.workspace }}
GEMINI_API_KEY_1: ${{ secrets.GEMINI_API_KEY_1 }}
GEMINI_API_KEY_2: ${{ secrets.GEMINI_API_KEY_2 }}
run: |
# Incremental engine: reuses stored results for unchanged categories.
# On Monday, only categories with new entries since the last run
# (or older than MAX_STALENESS_DAYS=30) trigger Gemini API calls.
python -u -m src.news_digest
- name: Save News Digest Cache
if: steps.cache-digest.outputs.cache-hit != 'true' && hashFiles('data/news_digest.json') != ''
uses: actions/cache/save@v5
with:
path: data/news_digest.json
key: news-digest-${{ steps.digest-date.outputs.date }}
- name: Generate RSS Feed
env:
PYTHONPATH: ${{ github.workspace }}

169
src/news_digest.py
View File

@@ -2,9 +2,10 @@ from __future__ import annotations
import os
import json
import hashlib
import asyncio
from datetime import datetime, timedelta
from typing import Dict, List, Any
from typing import Dict, List, Any, Tuple
from src.inventory_manager import load_inventory
from src.gemini_utils import call_gemini_with_retry
@@ -13,6 +14,10 @@ from src.logger import log_event
DIGEST_OUTPUT_PATH = "data/news_digest.json"
# Refresh a cell even when hash matches if last_analyzed is older than this.
# Ensures Gemini rankings stay fresh even for stable categories.
MAX_STALENESS_DAYS = 30
class NewsDigestEngine:
"""Generates a curated news digest by filtering inventory entries by
@@ -297,52 +302,120 @@ class NewsDigestEngine:
for e in entries[:limit]
]
# ------------------------------------------------------------------ #
# Incremental cache helpers #
# ------------------------------------------------------------------ #
@staticmethod
def _compute_pool_hash(entries: List[dict]) -> str:
"""Stable fingerprint of a category entry pool.
Uses sorted URLs so the hash only changes when the actual set of
entries changes — not when their order or metadata is updated.
Returns first 16 hex chars (64-bit fingerprint, collision-safe).
"""
sorted_urls = sorted(e["url"] for e in entries if e.get("url"))
return hashlib.sha256("|".join(sorted_urls).encode()).hexdigest()[:16]
def _load_existing_digest(self) -> Tuple[dict, dict]:
"""Load existing digest + meta block from disk.
Returns ``(periods_dict, meta_dict)``. Both are empty dicts if the
file does not exist or cannot be parsed. The ``_meta`` key is
stripped from *periods_dict* so callers only see period data.
"""
if not os.path.exists(DIGEST_OUTPUT_PATH):
return {}, {}
try:
with open(DIGEST_OUTPUT_PATH, encoding="utf-8") as f:
raw = json.load(f)
meta = raw.pop("_meta", {})
return raw, meta
except Exception as e:
log_event(
f"[Digest WARN] Could not load existing digest: {str(e)[:100]}"
)
return {}, {}
def _is_cache_valid(
self, period: str, cat: str, pool_hash: str, meta: dict
) -> bool:
"""Return True when the stored result can be reused without Gemini.
Conditions (both must hold):
1. Entry-pool hash matches (same URLs, same period).
2. Result was analysed no more than MAX_STALENESS_DAYS ago.
"""
cell = meta.get(period, {}).get(cat)
if not cell:
return False
if cell.get("entry_hash") != pool_hash:
return False
last = cell.get("last_analyzed", "")
if not last:
return False
try:
age = (
datetime.now(MADRID_TZ) - datetime.fromisoformat(last)
).days
return age <= MAX_STALENESS_DAYS
except Exception:
return False
# ------------------------------------------------------------------ #
# Core generation loop #
# ------------------------------------------------------------------ #
async def generate_digest(self) -> dict:
"""Generate the full digest for all categories and time periods.
"""Incrementally generate the digest for all categories / periods.
For each (category, period) cell the engine checks whether the
inventory pool hash has changed since the last run. If the hash
is identical *and* the result is not stale, the existing ranked
list is reused with zero Gemini API calls. Gemini is only invoked
for cells whose entry pool has actually changed or exceeded
MAX_STALENESS_DAYS.
Returns a nested dict::
{
"3_months": { "Kubernetes & Orchestration": [...], ... },
"6_months": { ... },
"12_months": { ... }
"12_months": { ... },
"_meta": { ... } ← persistence metadata, internal use
}
"""
existing, old_meta = self._load_existing_digest()
new_meta: Dict[str, Dict[str, dict]] = {}
digest: Dict[str, Dict[str, List[dict]]] = {}
n_cached = n_refreshed = n_fallback = 0
for period_name, days in self.PERIODS.items():
cutoff = (
datetime.now(MADRID_TZ) - timedelta(days=days)
).isoformat()
digest[period_name] = {}
new_meta[period_name] = {}
# Bucket entries into their digest categories
# Build per-category entry pools for this period
category_pools: Dict[str, List[dict]] = {}
for url, entry in self.inventory.items():
if not isinstance(entry, dict):
continue
if not self._is_within_period(entry, cutoff):
continue
# Tech / topic category
cat = self._get_entry_category(entry)
if cat:
category_pools.setdefault(cat, []).append(
dict(entry, url=url)
)
# Geo category (entry may belong to both)
geo = self._get_entry_geo(entry)
if geo:
category_pools.setdefault(geo, []).append(
dict(entry, url=url)
)
# Rank each category pool
for cat_name, entries in category_pools.items():
entries.sort(
key=lambda x: (
@@ -351,25 +424,42 @@ class NewsDigestEngine:
),
reverse=True,
)
max_items = self.ITEMS_PER_PERIOD.get(period_name, 10)
pool_hash = self._compute_pool_hash(entries)
# ── Cache hit: reuse without calling Gemini ──
if self._is_cache_valid(period_name, cat_name, pool_hash, old_meta):
cached_items = existing.get(period_name, {}).get(cat_name, [])
if cached_items:
digest[period_name][cat_name] = cached_items
new_meta[period_name][cat_name] = (
old_meta[period_name][cat_name]
)
n_cached += 1
continue
# ── Fewer than 3 entries: star-based fallback (no AI) ──
if len(entries) < 3:
digest[period_name][cat_name] = self._fallback_items(
entries, cat_name, limit=max_items
)
new_meta[period_name][cat_name] = {
"last_analyzed": datetime.now(MADRID_TZ).isoformat(),
"entry_hash": pool_hash,
"entry_count": len(entries),
"method": "fallback_small",
}
n_fallback += 1
continue
# ── Gemini ranking ──
try:
prompt = self._build_ranking_prompt(
cat_name, entries, period_name
)
result = await call_gemini_with_retry(
prompt,
prefer_flash=True,
role="Digest-Analyst",
prompt, prefer_flash=True, role="Digest-Analyst"
)
ranked: List[dict] = []
for item in result.get("items", []):
idx = int(item.get("idx", -1))
@@ -386,26 +476,43 @@ class NewsDigestEngine:
"category": cat_name,
}
)
digest[period_name][cat_name] = ranked[:max_items]
new_meta[period_name][cat_name] = {
"last_analyzed": datetime.now(MADRID_TZ).isoformat(),
"entry_hash": pool_hash,
"entry_count": len(entries),
"method": "gemini",
}
n_refreshed += 1
log_event(
f" [Digest] {period_name}/{cat_name}: "
f"{len(ranked)} items ranked"
f"{len(ranked)} items ranked (Gemini)"
)
except Exception as exc:
log_event(
f" [Digest WARN] {period_name}/{cat_name}: "
f"Gemini failed ({str(exc)[:80]}), "
"using star-based fallback"
f"Gemini failed ({str(exc)[:80]}), using fallback"
)
digest[period_name][cat_name] = self._fallback_items(
entries, cat_name, limit=max_items
)
new_meta[period_name][cat_name] = {
"last_analyzed": datetime.now(MADRID_TZ).isoformat(),
"entry_hash": pool_hash,
"entry_count": len(entries),
"method": "fallback_error",
}
n_fallback += 1
# Respect Gemini rate limits
await asyncio.sleep(1.0)
log_event(
f"[Digest] {n_cached} cells reused (0 API calls), "
f"{n_refreshed} refreshed via Gemini, "
f"{n_fallback} fallback"
)
new_meta["last_updated"] = datetime.now(MADRID_TZ).isoformat()
digest["_meta"] = new_meta
return digest
# ------------------------------------------------------------------ #
@@ -414,11 +521,29 @@ class NewsDigestEngine:
@staticmethod
def save_digest(digest: dict) -> None:
"""Serialise *digest* to ``data/news_digest.json``."""
"""Serialise digest (including ``_meta``) to ``data/news_digest.json``."""
os.makedirs(os.path.dirname(DIGEST_OUTPUT_PATH), exist_ok=True)
with open(DIGEST_OUTPUT_PATH, "w", encoding="utf-8") as fh:
json.dump(digest, fh, indent=2, ensure_ascii=False)
log_event(f"[Digest] Saved to {DIGEST_OUTPUT_PATH}")
# Count only period keys for the total-items stat
total_items = sum(
len(items)
for key, period in digest.items()
if key != "_meta" and isinstance(period, dict)
for items in period.values()
if isinstance(items, list)
)
gemini_cells = sum(
1
for period_meta in digest.get("_meta", {}).values()
if isinstance(period_meta, dict)
for cell in period_meta.values()
if isinstance(cell, dict) and cell.get("method") == "gemini"
)
log_event(
f"[Digest] Saved to {DIGEST_OUTPUT_PATH} — "
f"{total_items} items, {gemini_cells} Gemini calls this run"
)
# ====================================================================== #

11
src/v2_optimizer.py
View File

@@ -1021,9 +1021,16 @@ class V2VisionEngine:
impact_icons = {"critical": "🔴", "high": "🟡", "medium": "🔵"}
try:
digest_mtime = os.path.getmtime(digest_path)
from datetime import datetime as _dt
digest_updated = _dt.fromtimestamp(digest_mtime).strftime("%b %d, %Y")
# Prefer _meta.last_updated (tracks actual Gemini analysis date)
# over file mtime (which changes on every commit/render).
raw_ts = digest_data.get("_meta", {}).get("last_updated", "")
if raw_ts:
digest_updated = _dt.fromisoformat(raw_ts).strftime("%b %d, %Y")
else:
digest_updated = _dt.fromtimestamp(
os.path.getmtime(digest_path)
).strftime("%b %d, %Y")
except Exception:
digest_updated = ""
updated_badge = f'<span class="trending-section__updated">Updated {digest_updated}</span>' if digest_updated else ""

3
v2-mkdocs.yml
View File

@@ -96,9 +96,6 @@ extra:
version:
provider: mike # Ready for version switching
extra_head:
- '<link rel="alternate" type="application/rss+xml" title="Nubenetes Intelligence Digest" href="/feed.xml"/>'
extra_css:
- https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap
- static/extra.css