mirror of
https://github.com/nubenetes/awesome-kubernetes.git
synced 2026-07-12 18:00:37 +00:00
54 lines
2.1 KiB
Python
54 lines
2.1 KiB
Python
import os
|
|
import asyncio
|
|
from src.v2_optimizer import V2VisionEngine
|
|
from src.logger import log_event
|
|
from src.gemini_utils import normalize_url, get_github_activity
|
|
|
|
async def run_metadata_enrichment():
|
|
engine = V2VisionEngine(render_only=False)
|
|
log_event("STARTING V2 SPECIALIZED METADATA ENGINE", section_break=True)
|
|
|
|
all_v1_links, _, _ = await engine._gather_all_v1_content()
|
|
log_event(f"[*] Discovery: Found {len(all_v1_links)} resources for metadata enrichment.")
|
|
|
|
# Mandate 15 & 43: Proactive Enrichment for V2
|
|
processed_gh_metadata = set()
|
|
gh_fetch_count = 0
|
|
force_full = os.getenv("ENRICH_METADATA", "false").lower() == "true"
|
|
|
|
gh_tasks = []
|
|
gh_sem = asyncio.Semaphore(20) # Increased concurrency for final sprint
|
|
|
|
async def _fetch_gh_with_sem(url: str):
|
|
async with gh_sem:
|
|
return url, await get_github_activity(url)
|
|
|
|
for l in all_v1_links:
|
|
norm_url = normalize_url(l["url"])
|
|
if "github.com" not in norm_url: continue
|
|
|
|
cached = engine.inventory.get(norm_url, {})
|
|
# Enrich if missing or if forced
|
|
if (force_full or cached.get("gh_stars") is None) and norm_url not in processed_gh_metadata:
|
|
processed_gh_metadata.add(norm_url)
|
|
gh_tasks.append(_fetch_gh_with_sem(norm_url))
|
|
|
|
if gh_tasks:
|
|
log_event(f" [METADATA] Pulse: Fetching {len(gh_tasks)} GitHub profiles in parallel...")
|
|
gh_results = await asyncio.gather(*gh_tasks)
|
|
for norm_url, gh_data in gh_results:
|
|
if gh_data:
|
|
if norm_url not in engine.inventory: engine.inventory[norm_url] = {}
|
|
engine.inventory[norm_url].update(gh_data)
|
|
gh_fetch_count += 1
|
|
|
|
if gh_fetch_count % 100 == 0:
|
|
engine._save_inventory()
|
|
log_event(f" [💾] Periodic Save: {gh_fetch_count} fetches...")
|
|
|
|
engine._save_inventory()
|
|
log_event(f"V2 METADATA ENGINE COMPLETED. Processed {gh_fetch_count} repos.")
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(run_metadata_enrichment())
|