From 28e07c83dd7b982e79bc9310092474adce20d544 Mon Sep 17 00:00:00 2001 From: Nubenetes Bot Date: Tue, 19 May 2026 19:14:05 +0200 Subject: [PATCH] feat(v2): implement on-demand metadata enrichment and GH stars fetching --- .github/workflows/agentic_v2_builder.yml | 7 +++++- src/gemini_utils.py | 29 ++++++++++++++++++++++++ src/v2_optimizer.py | 13 ++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/agentic_v2_builder.yml b/.github/workflows/agentic_v2_builder.yml index 12f481c7..c908ef85 100644 --- a/.github/workflows/agentic_v2_builder.yml +++ b/.github/workflows/agentic_v2_builder.yml @@ -7,6 +7,10 @@ on: description: 'Force AI re-evaluation (ignores cache for tags/years)' type: boolean default: false + enrich_metadata: + description: 'Enrich GitHub Metadata (fetch stars/license for V2 logic)' + type: boolean + default: false activate_backup_key: description: 'Activate Identity B (Subscription) as backup/rotation' type: boolean @@ -59,7 +63,8 @@ jobs: GEMINI_API_KEY_2: ${{ secrets.GEMINI_API_KEY_2 }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ACTIVATE_BACKUP_KEY: ${{ github.event.inputs.activate_backup_key || 'false' }} - FORCE_EVAL: ${{ github.event.inputs.force_reevaluate }} + FORCE_EVAL: ${{ github.event.inputs.force_reevaluate || 'false' }} + ENRICH_METADATA: ${{ github.event.inputs.enrich_metadata || 'false' }} PYTHONPATH: . run: | python -u src/v2_optimizer.py diff --git a/src/gemini_utils.py b/src/gemini_utils.py index 715f749d..905e0a26 100644 --- a/src/gemini_utils.py +++ b/src/gemini_utils.py @@ -180,6 +180,35 @@ def clean_toc_text(text: str) -> str: text = re.sub(r'[^\w\s\-.]', '', text) return text.strip() +async def get_github_activity(url: str) -> Dict: + """ + Mandate 15: Fetch real-time GitHub metadata (stars, license, last push). + """ + match = re.search(r'github\.com/([^/]+/[^/]+)', url) + if not match: return {} + repo = match.group(1) + api_url = f"https://api.github.com/repos/{repo}" + + # Import GH_TOKEN from config locally to avoid circular dependencies + try: + from src.config import GH_TOKEN + headers = {"Authorization": f"token {GH_TOKEN}"} if GH_TOKEN else {} + except: + headers = {} + + try: + async with httpx.AsyncClient() as client: + resp = await client.get(api_url, headers=headers, timeout=10.0) + if resp.status_code == 200: + data = resp.json() + return { + "gh_stars": data.get("stargazers_count", 0), + "gh_pushed": data.get("pushed_at", "N/A"), + "gh_license": data.get("license", {}).get("spdx_id", "N/A") + } + except: pass + return {} + def sanitize_trailing_slashes(url: str) -> str: """ Mandate 34: Enforces a ZERO trailing slash policy. diff --git a/src/v2_optimizer.py b/src/v2_optimizer.py index b68586a0..e1c64c3c 100644 --- a/src/v2_optimizer.py +++ b/src/v2_optimizer.py @@ -7,7 +7,7 @@ import httpx from datetime import datetime from typing import List, Dict, Set, Any, Tuple from src.config import GEMINI_API_KEYS, GH_TOKEN, TARGET_REPO, MADRID_TZ, INVENTORY_PATH -from src.gemini_utils import call_gemini_with_retry, normalize_url, clean_toc_text +from src.gemini_utils import call_gemini_with_retry, normalize_url, clean_toc_text, get_github_activity from src.logger import log_event V1_DIR = "docs" @@ -194,6 +194,7 @@ class V2VisionEngine: to_evaluate = [] project_registry = {} force_eval = os.getenv("FORCE_EVAL", "false").lower() == "true" + enrich_metadata = os.getenv("ENRICH_METADATA", "false").lower() == "true" special_files = [sa["file"] for sa in self.special_assets_rules.get("special_assets", [])] for l in links: @@ -207,6 +208,16 @@ class V2VisionEngine: match = re.search(r'github\.com/([^/]+/[^/]+)', norm_url) if match: project_id = match.group(1).lower() + # Mandate 15: If enrichment is ON and gh_metadata is missing, we must fetch it + if enrich_metadata and "github.com" in norm_url: + cached = self.inventory.get(norm_url, {}) + if not cached.get("gh_stars"): + log_event(f" [METADATA] Enrichment: Fetching GH Activity for {norm_url}") + gh_data = await get_github_activity(norm_url) + if gh_data: + if norm_url not in self.inventory: self.inventory[norm_url] = {} + self.inventory[norm_url].update(gh_data) + if not force_eval and norm_url in self.inventory and "stars" in self.inventory[norm_url]: cached = self.inventory[norm_url] item.update(cached)