diff --git a/.github/workflows/agentic_cron.yml b/.github/workflows/agentic_cron.yml index e72014f4..603c6edc 100644 --- a/.github/workflows/agentic_cron.yml +++ b/.github/workflows/agentic_cron.yml @@ -1,5 +1,6 @@ name: Nubenetes Automated Agentic Curation +# Last Sync: 2026-05-15 - Triggering Refresh on: schedule: - cron: '0 5 1 * *' # Monthly: 1st day of the month at 05:00 UTC diff --git a/.gitignore b/.gitignore index 20f2963e..dafd0e72 100644 --- a/.gitignore +++ b/.gitignore @@ -351,6 +351,7 @@ MigrationBackup/ # AutomatizaciΓ³n Nubenetes src/__pycache__/ *.json +!data/v2_cache.json .env nubenetes_agent_env/ .venv/ diff --git a/GEMINI.md b/GEMINI.md index 1e0095c0..b032dcd0 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -17,6 +17,9 @@ This file contains the accumulated instructions and long-term vision for the aut 11. **Workflow-Config Synchronization**: The GitHub Actions curation workflow form (`agentic_cron.yml`) MUST remain perfectly synchronized with the curation sources configuration file (`data/curation_sources.yaml`). Any addition, removal, or renaming of topics/categories in the configuration file requires a corresponding update to the workflow's input fields (checkboxes) to ensure users can toggle those sources manually. This maintains consistency between data-driven sources and the UI trigger. 12. **V2 Elite Maintenance**: The Nubenetes V2 (Agentic Elite) edition is a derived view of the V1 archive. It is managed via the `src/v2_optimizer.py` script and stored in the `v2-docs/` directory. AI agents MUST NOT modify `v2-docs/` directly via standard curation workflows; they must only use the `agentic_v2_builder.yml` workflow to perform the periodic "Elite Selection" process. Standard curation and cleaning workflows must always target the `docs/` directory as the primary source of truth. 13. **Detailed Logging for V2**: When running the V2 Optimizer, agents MUST use unbuffered logging and detailed output messages. If the optimizer returns '0 links kept', the agent MUST investigate the logs to determine if it was due to AI selection or a parsing/API error. +14. **Persistent V2 Caching**: The V2 Optimizer MUST use a persistent cache file (`data/v2_cache.json`) to store AI evaluations (year, quality, category). This is mandatory to minimize API costs and ensure execution speed across 15k+ links. +15. **GitHub Metadata Enrichment**: For all `github.com` resources, the bot MUST attempt to fetch real-time metadata (stars, last commit) using the GitHub API. This data must be included in the V2 rendering to provide current context. +16. **Asynchronous Link Health**: Every V2 generation cycle MUST perform asynchronous health checks (HTTP HEAD) on all links. Broken links (404) should be excluded or flagged as offline to maintain the library's technical integrity. ## πŸ› οΈ Structural Evolution & Navigation ... diff --git a/data/v2_cache.json b/data/v2_cache.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/data/v2_cache.json @@ -0,0 +1 @@ +{} diff --git a/src/v2_optimizer.py b/src/v2_optimizer.py index faf3f75f..4f7f480c 100644 --- a/src/v2_optimizer.py +++ b/src/v2_optimizer.py @@ -3,6 +3,7 @@ import re import json import asyncio import yaml +import httpx from datetime import datetime from typing import List, Dict, Set, Any from src.config import GEMINI_API_KEYS, GH_TOKEN, TARGET_REPO, MADRID_TZ @@ -11,6 +12,7 @@ from src.logger import log_event V1_DIR = "docs" V2_DIR = "v2-docs" +V2_CACHE_PATH = "data/v2_cache.json" class V2VisionEngine: def __init__(self): @@ -31,92 +33,215 @@ class V2VisionEngine: self.library_criteria = ( "You are a Technical Librarian in 2026. Your mission is to build a high-density, professional reference library.\n" - "PHASE 1: TECHNICAL PRESERVATION\n" - "- Be INCLUSIVE. Do not discard useful technical content just because it's a few years old.\n" - "- KEEP: Technically solid tools, guides, and documentation. Even if 'classic', they are part of the library.\n" - "- DISCARD ONLY: Broken links, 404s, obvious spam, non-technical jokes, and personal anecdotes.\n" - "- ALWAYS keep 'Awesome' repositories.\n\n" - "PHASE 2: TEMPORAL ANALYSIS\n" - "- For EACH kept resource, identify or estimate the PUBLICATION YEAR.\n" - "- Use URL patterns (e.g., /2023/...), content clues, or tool era to decide the year.\n" - "- If totally unknown, use 2024 as default for recently active looking sites, or 'N/A'.\n" + "PHASE 1: TECHNICAL PRESERVATION (HIGH INCLUSIVITY)\n" + "- KEEP >90% of technical resources. Only discard 404s, obvious spam, or non-technical content.\n" + "- 'Awesome' repositories, official documentation, and deep technical guides are mandatory.\n" + "- YouTube videos are HIGH-VALUE resources; keep them as technical references.\n\n" + "PHASE 2: TEMPORAL & QUALITY SYNTHESIS\n" + "- Identify/estimate PUBLICATION YEAR.\n" + "- Assign QUALITY level (1-3 stars):\n" + " * 3 stars (🌟🌟🌟): Masterpieces, foundational standards, definitive 'Awesome' lists.\n" + " * 2 stars (🌟🌟): Production-grade tools, deep tutorials, highly recommended videos.\n" + " * 1 star (🌟): Solid technical references.\n" + "- Identify if a resource is a 'YouTube Video/Playlist' for special rendering.\n" ) + self.cache = self._load_cache() + + def _load_cache(self) -> Dict: + if os.path.exists(V2_CACHE_PATH): + try: + with open(V2_CACHE_PATH, "r") as f: return json.load(f) + except: return {} + return {} + + def _save_cache(self): + os.makedirs(os.path.dirname(V2_CACHE_PATH), exist_ok=True) + with open(V2_CACHE_PATH, "w") as f: json.dump(self.cache, f, indent=2) async def analyze_and_cluster(self): log_event("STARTING V2 HIGH-DENSITY CHRONOLOGICAL LIBRARY GENERATION", section_break=True) - all_v1_links = await self._gather_all_v1_content() + all_v1_links, mosaic_html, videos_html = await self._gather_all_v1_content() log_event(f"[*] Discovery: Found {len(all_v1_links)} resources in V1 archive.") - log_event("[*] Phase 1: Library Evaluation & Year Extraction...") - library_inventory = await self._evaluate_and_date_resources(all_v1_links) - log_event(f"[*] Inventory Refined: {len(library_inventory)} high-quality resources kept.") + log_event("[*] Phase 1: Health Check & Metadata Enrichment...") + # Rapid Async Health Check + health_inventory = await self._verify_link_health(all_v1_links) + log_event(f"[*] Health Check Complete. {len(health_inventory)}/{len(all_v1_links)} links are online.") - log_event("[*] Phase 2: Dimensional Clustering & Chronological Sorting...") + log_event("[*] Phase 2: Library Evaluation, Year Extraction & Quality Scoring...") + library_inventory = await self._evaluate_and_score_resources(health_inventory) + log_event(f"[*] Inventory Refined: {len(library_inventory)} resources kept.") + + log_event("[*] Phase 3: Dimensional Clustering & Chronological Sorting...") v2_data = await self._rebuild_structure(library_inventory) - log_event("[*] Phase 3: Generating Premium Portal Pages...") + log_event("[*] Phase 4: Generating Premium Portal Pages...") os.makedirs(V2_DIR, exist_ok=True) - await self._write_premium_files(v2_data) + await self._write_premium_files(v2_data, mosaic_html, videos_html) await self._sync_enterprise_navigation(v2_data) + self._save_cache() log_event("V2 LIBRARY GENERATION COMPLETED.", section_break=True) - async def _gather_all_v1_content(self) -> List[Dict]: + async def _gather_all_v1_content(self) -> (List[Dict], str, str): all_links = [] + mosaic_html = "" + videos_html = "" + + if os.path.exists("docs/index.md"): + with open("docs/index.md", "r") as f: + idx_content = f.read() + mosaic_match = re.search(r'
\s*(\[!\[.*?)\s*
', idx_content, re.DOTALL) + if mosaic_match: mosaic_html = mosaic_match.group(1) + videos_match = re.search(r'\?\?\? note "Top Videos & Clips.*?\n(.*?\n)\s*', idx_content, re.DOTALL) + if videos_match: videos_html = videos_match.group(1) + for root, _, files in os.walk(V1_DIR): for file in files: if not file.endswith(".md") or file == "index.md": continue path = os.path.join(root, file) with open(path, "r") as f: content = f.read() - matches = re.findall(r'^\s*-\s*\[([^\]]+)\]\(([^\)]+)\)(.*)', content, re.MULTILINE) - for title, url, desc in matches: - all_links.append({"title": title, "url": url, "description": desc.strip(), "original_file": file}) - return all_links + matches = re.finditer(r'^\s*-\s*\[([^\]]+)\]\(([^\)]+)\)(.*?(?:\n\s{2,}.*)*)', content, re.MULTILINE) + for m in matches: + title, url, full_desc = m.groups() + all_links.append({ + "title": title, + "url": url, + "description": full_desc.strip(), + "original_file": file + }) + return all_links, mosaic_html, videos_html - async def _evaluate_and_date_resources(self, links: List[Dict]) -> List[Dict]: + async def _verify_link_health(self, links: List[Dict]) -> List[Dict]: + online_links = [] + BATCH_SIZE = 100 + async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client: + for i in range(0, len(links), BATCH_SIZE): + batch = links[i:i+BATCH_SIZE] + tasks = [self._check_single_link(client, l) for l in batch] + results = await asyncio.gather(*tasks) + online_links.extend([r for r in results if r is not None]) + if i % 500 == 0: + log_event(f" [Health] Verified {i}/{len(links)} links...") + return online_links + + async def _check_single_link(self, client, link: Dict) -> Dict: + url = link["url"] + # Skip health check if cached as healthy recently (simple heuristic) + if url in self.cache and self.cache[url].get("status") == "online": + return link + + try: + resp = await client.head(url) + if resp.status_code < 400: + self.cache.setdefault(url, {})["status"] = "online" + return link + # Fallback to GET for some servers that block HEAD + resp = await client.get(url) + if resp.status_code < 400: + self.cache.setdefault(url, {})["status"] = "online" + return link + except: pass + + # If it was foundational, keep even if down (maybe temporary) + if "awesome" in link["title"].lower(): + return link + + return None + + async def _evaluate_and_score_resources(self, links: List[Dict]) -> List[Dict]: refined = [] + to_evaluate = [] + + # Pull from cache first + for l in links: + url = l["url"] + if url in self.cache and "year" in self.cache[url]: + item = l.copy() + item.update(self.cache[url]) + # Refresh GitHub metadata if it's a GH link + if "github.com" in url: + gh_meta = await self._fetch_github_metadata(url) + item.update(gh_meta) + refined.append(item) + else: + to_evaluate.append(l) + + if not to_evaluate: return refined + BATCH_SIZE = 50 - for i in range(0, len(links), BATCH_SIZE): - batch = links[i:i+BATCH_SIZE] + for i in range(0, len(to_evaluate), BATCH_SIZE): + batch = to_evaluate[i:i+BATCH_SIZE] batch_num = i//BATCH_SIZE + 1 - log_event(f" [>] Processing Batch {batch_num}...") + log_event(f" [>] Processing Batch {batch_num} with AI...") prompt = ( f"{self.library_criteria}\n" - "Respond ONLY with a JSON object: {\"keep_indices\": [int, ...], \"years\": {\"index\": \"YYYY\"}}\n\n" + "Respond ONLY with a JSON object: {\"results\": [{\"idx\": int, \"year\": \"YYYY\", \"stars\": int, \"is_video\": bool}, ...]}\n\n" "LINKS:\n" + "\n".join([f"{idx}. {l['title']} ({l['url']})" for idx, l in enumerate(batch)]) ) try: data = await call_gemini_with_retry(prompt) - indices = data.get("keep_indices", []) - years_map = data.get("years", {}) + results = data.get("results", []) - for idx in indices: + for res in results: try: - idx_int = int(idx) - if idx_int < len(batch): - item = batch[idx_int].copy() - year_val = years_map.get(str(idx), "2024") - item["year"] = str(year_val) - # Tag based on year + idx = int(res["idx"]) + if idx < len(batch): + item = batch[idx].copy() + eval_data = { + "year": str(res.get("year", "2024")), + "stars": min(max(int(res.get("stars", 1)), 1), 3), + "is_video": res.get("is_video", False) + } + item.update(eval_data) + + if "github.com" in item["url"]: + gh_meta = await self._fetch_github_metadata(item["url"]) + item.update(gh_meta) + eval_data.update(gh_meta) + + # Save to cache + self.cache[item["url"]] = eval_data + if item["year"].isdigit() and int(item["year"]) >= 2025: item["tag"] = "[CUTTING-EDGE]" elif "awesome" in item["title"].lower(): item["tag"] = "[FOUNDATIONAL]" else: item["tag"] = "[PRODUCTION-READY]" + refined.append(item) except: continue - log_event(f" [Batch {batch_num}] Kept {len(indices)}/{len(batch)}") except: - # Conservative fallback: keep and mark as 2024 for l in batch: item = l.copy() - item["year"] = "2024" + item["year"], item["stars"], item["is_video"] = "2024", 1, "youtube" in l["url"] item["tag"] = "[FOUNDATIONAL]" if "awesome" in l["title"].lower() else "[PRODUCTION-READY]" refined.append(item) - await asyncio.sleep(0.5) + await asyncio.sleep(0.3) return refined + async def _fetch_github_metadata(self, url: str) -> Dict: + match = re.search(r'github\.com/([^/]+)/([^/]+)', url) + if not match: return {} + owner, repo = match.groups() + repo = repo.split("#")[0].split("?")[0] # Clean up + + headers = {"Authorization": f"token {GH_TOKEN}"} if GH_TOKEN else {} + api_url = f"https://api.github.com/repos/{owner}/{repo}" + + try: + async with httpx.AsyncClient(timeout=5.0) as client: + resp = await client.get(api_url, headers=headers) + if resp.status_code == 200: + data = resp.json() + return { + "gh_stars": data.get("stargazers_count", 0), + "gh_updated": data.get("updated_at", "").split("T")[0] + } + except: pass + return {} + async def _rebuild_structure(self, inventory: List[Dict]) -> Dict[str, Dict]: v2_structure = {dim: {"summary": "", "categories": {}} for dim in self.dimensions.keys()} file_to_dim = {} @@ -130,57 +255,73 @@ class V2VisionEngine: v2_structure[dim]["categories"][cat_name] = [] v2_structure[dim]["categories"][cat_name].append(item) - # Sort within each category by year (descending) for dim in v2_structure.keys(): if not v2_structure[dim]["categories"]: continue for cat in v2_structure[dim]["categories"]: - v2_structure[dim]["categories"][cat].sort(key=lambda x: x.get("year", "0"), reverse=True) + v2_structure[dim]["categories"][cat].sort(key=lambda x: (x.get("year", "0"), -x.get("stars", 0))) - # Dimension summary - prompt = f"Write a 1-sentence executive summary for section '{dim}'. Professional 2026 tone. Respond ONLY with the sentence." + prompt = f"Write a professional 2026 executive summary for '{dim}'. Focus on high-density value. 1 sentence only." try: v2_structure[dim]["summary"] = await call_gemini_with_retry(prompt, response_format="text") except: - v2_structure[dim]["summary"] = f"Curated high-density chronological resources for {dim}." + v2_structure[dim]["summary"] = f"Comprehensive chronological reference library for {dim}." return v2_structure - async def _write_premium_files(self, data: Dict[str, Dict]): - # Home - mermaid_code = "graph TD\n" - for dim in data.keys(): - if data[dim]["categories"]: - mermaid_code += f" V2[Nubenetes V2] --> {dim.replace(' ', '_').replace('&', 'and').replace('(', '').replace(')', '')}[{dim}]\n" + async def _write_premium_files(self, data: Dict[str, Dict], mosaic_html: str, videos_html: str): + master_selection = [] + for dim in data.values(): + for cat_links in dim["categories"].values(): + master_selection.extend([l for l in cat_links if l.get("stars", 1) == 3]) + master_selection.sort(key=lambda x: (x.get("year", "0"), x["title"])) index_md = ( "# Nubenetes V2 | The High-Density Library (2026)\n\n" "![Banner](https://raw.githubusercontent.com/nubenetes/awesome-kubernetes/master/docs/images/logo.png)\n\n" - "!!! quote \"Chronological Excellence\"\n" - " This portal is a time-indexed reference for Cloud Native engineering. Resources are sorted by publication year " - " to ensure you have both the latest innovations and foundational classics at your fingertips.\n\n" - "## System Architecture View\n\n" - "```mermaid\n" + mermaid_code + "```\n\n" - "## Strategic Dimensions\n\n" + "!!! quote \"The Library of 2026\"\n" + " A meticulously curated reference of over 15,000 resources. This V2 portal preserves technical depth while providing " + " chronological clarity and expert quality synthesis.\n\n" + f"
\n{mosaic_html}\n
\n\n" + "## 🌟 Master Selection (Top-Tier Gems)\n" + "A global selection of the most impactful resources across all dimensions.\n\n" ) + for l in master_selection[:100]: + gh_info = f" `[⭐ {l['gh_stars']}]`" if "gh_stars" in l else "" + index_md += f"- **({l['year']})** [{l['title']}]({l['url']}){gh_info} 🌟🌟🌟\n" + + index_md += "\n??? note \"Elite Video Selection - Click to expand!\"\n" + index_md += f"
\n{videos_html}\n
\n\n" + + index_md += "## Strategic Dimensions\n" for dim, content in data.items(): if not content["categories"]: continue slug = dim.lower().replace(" ", "-").replace("&", "and").replace("(", "").replace(")", "").replace(" ", "-") index_md += f"- **[{dim}](./{slug}.md)**: {content['summary']}\n" + with open(os.path.join(V2_DIR, "index.md"), "w") as f: f.write(index_md) - # Dimension pages for dim, content in data.items(): if not content["categories"]: continue slug = dim.lower().replace(" ", "-").replace("&", "and").replace("(", "").replace(")", "").replace(" ", "-") md = f"# {dim}\n\n" - md += f"!!! info \"Executive Overview\"\n {content['summary']}\n\n" + md += f"!!! info \"Architectural Context\"\n {content['summary']}\n\n" for cat, links in content["categories"].items(): md += f"## {cat}\n" for l in links: - year = l.get("year", "N/A") + year, stars = l.get("year", "N/A"), "🌟" * l.get("stars", 1) tag = l.get("tag", "[PRODUCTION-READY]") color = "success" if "FOUNDATIONAL" in tag else "info" if "PRODUCTION" in tag else "warning" - md += f" - **({year})** [{l['title']}]({l['url']}) {l['description']} {tag}\n" + title_display = f"**{l['title']}**" if l.get("stars", 1) >= 2 else l['title'] + + gh_info = f" ⭐ {l['gh_stars']}" if "gh_stars" in l else "" + icon = " πŸŽ₯" if l.get("is_video") else "" + md += f" - **({year})** [{title_display}]({l['url']}){icon}{gh_info} {stars} {tag}\n" + if l['description']: + desc = l['description'] + if "\n" in desc: + md += "\n" + "\n".join([" " + line for line in desc.split("\n")]) + "\n\n" + else: + md += f" {desc}\n" md += "\n" with open(os.path.join(V2_DIR, f"{slug}.md"), "w") as f: f.write(md)