diff --git a/README.md b/README.md index 05f7e12e..40ed1815 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,13 @@ * [5.5. Multi-Language Support Policy](#55-multi-language-support-policy) 6. [6. The Unified Agentic Database (Knowledge Graph)](#6-the-unified-agentic-database-knowledge-graph) * [6.1. Database Components](#61-database-components) - * [6.2. The 'Database-First' Reasoning Protocol](#62-the-database-first-reasoning-protocol) + * [6.2. The 'Database-First' Reasoning Protocol (Zero-Redundancy)](#62-the-database-first-reasoning-protocol-zero-redundancy) * [6.3. Database Lifecycle and Hygiene](#63-database-lifecycle-and-hygiene) * [6.4. Multi-Format Synchronization Logic](#64-multi-format-synchronization-logic) * [6.5. Dynamic AI Discovery and Optimization](#65-dynamic-ai-discovery-and-optimization) * [6.6. AI Intelligence and Observability (Transparency)](#66-ai-intelligence-and-observability-transparency) + * [6.7. Platinum Operational Tier (2026 Standards)](#67-platinum-operational-tier-2026-standards) + * [6.8. Platinum Capability Matrix](#68-platinum-capability-matrix) 7. [7. AI Economic Architecture and Cost Analysis](#7-ai-economic-architecture-and-cost-analysis) * [7.1. Comprehensive Economic Projections (2026 Inception)](#71-comprehensive-economic-projections-2026-inception) * [7.2. Efficiency and Performance Metrics](#72-efficiency-and-performance-metrics) @@ -60,6 +62,7 @@ * [9.8. Workflow UI Auto-Sync](#98-workflow-ui-auto-sync) 10. [10. Branching Strategy and Lifecycle](#10-branching-strategy-and-lifecycle) 11. [11. Contributing to the Archive](#11-contributing-to-the-archive) + * [How to Contribute](#how-to-contribute) 12. [12. Developer Experience and VSCode Setup](#12-developer-experience-and-vscode-setup) * [12.1. Optimized "Power User" Environment](#121-optimized-power-user-environment) * [12.2. Extension Recommendations (Legacy/General)](#122-extension-recommendations-legacygeneral) @@ -423,7 +426,7 @@ To maintain a high-performance "Single Source of Truth", Nubenetes implements au - **Universal Rescue Protocol (The Resurrection Rule)**: For ALL technical resources, the engine refuses to delete a link immediately upon a 404 or generic redirect. Instead, it triggers a "Technical Resurrection" cycle using **Real-time Web Grounding** to identify specific paths on destination domains. This is essential for preserving legendary content during massive corporate site migrations (e.g., **Nginx** to **F5**, or the **Ansible Blog** move to personal domains). - **High-Value Preservation (The 'Review Required' Rule)**: Resources identified as **High-Value** (marked with 🌟 or bold formatting) are exempt from automatic deletion. If rescue fails, they are marked as `status: review_required` for manual verification, ensuring no significant technical assets are lost during autonomous cleaning. -#### 🕵️ Intelligent Cleaning Observability +#### Intelligent Cleaning Observability ```log # 1. PROGRESS TRACKING & PARALLEL EXECUTION @@ -820,7 +823,7 @@ Maintains **Mandate 11** by detecting new categories and alerting maintainers to Nubenetes thrives on a **Hybrid Human-AI Collaboration** model. Community contributions are the lifeblood of the V1 archive. -### 🤝 How to Contribute +### How to Contribute 1. **Target Branch**: Always create your Pull Requests against the `develop` branch. 2. **Source of Truth (V1)**: Only add or edit files in the `docs/` directory. **Do not manually edit [`v2-docs/`](v2-docs/)**. 3. **Manual Link Format**: Use the standard format: ` - [Title](URL) - Your descriptive summary.` diff --git a/src/gemini_utils.py b/src/gemini_utils.py index 27d75546..ab640b18 100644 --- a/src/gemini_utils.py +++ b/src/gemini_utils.py @@ -310,11 +310,16 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max for key_offset in range(total_keys): current_idx = (CURRENT_KEY_INDEX + key_offset) % total_keys api_key = GEMINI_API_KEYS[current_idx] + key_label = GEMINI_API_KEYS_DATA[current_idx]["label"] async with httpx.AsyncClient() as client: - for model in models: + # Limit the number of models to try per key to avoid excessive timeouts + for model in models[:5]: if THROTTLED_MODELS.get(f"{current_idx}_{model}", 0) > time.time(): continue + + # Mandate 13: Detailed tracing for long-running workflows + # log_event(f" [AI] Attempt: Key {current_idx+1} ({key_label}) | Model: {model}") full_model_name = f"models/{model}" api_url = f"https://generativelanguage.googleapis.com/{GEMINI_API_VERSION}/{full_model_name}:generateContent?key={api_key}" @@ -325,7 +330,7 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max "contents": [{"parts": [{"text": prompt}]}], "tools": [{"google_search_retrieval": {}}] if use_grounding else [] } - response = await client.post(api_url, json=payload, timeout=50) + response = await client.post(api_url, json=payload, timeout=60) resp_json = {} try: resp_json = response.json() @@ -335,6 +340,7 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max SESSION_TRACKER.track_call(current_idx, model, response.status_code, usage, role=role) if response.status_code == 200: + log_event(f" [AI] Success: Key {current_idx+1} | Model: {model} | Role: {role}") CURRENT_KEY_INDEX = current_idx if 'candidates' in resp_json and resp_json['candidates']: text_resp = resp_json['candidates'][0]['content']['parts'][0]['text'] diff --git a/src/v2_optimizer.py b/src/v2_optimizer.py index f8ca28f8..76c310c0 100644 --- a/src/v2_optimizer.py +++ b/src/v2_optimizer.py @@ -292,25 +292,32 @@ class V2VisionEngine: # 1. Has AI summary (previous run) # 2. Is GitHub and has stars (metadata present) # 3. Has decent manual description (> 40 chars) + # 4. Is already in inventory (we have title/category context) has_ai_summary = l.get("ai_summary") is not None and len(l.get("ai_summary")) > 50 has_stars = l.get("gh_stars") is not None has_desc = len(l.get("description", "")) > 40 - - if has_ai_summary or has_stars or has_desc: + is_known = nu in self.inventory + + if has_ai_summary or has_stars or has_desc or is_known: fast_track.append(l) else: # Grounded-Track is ONLY for "Unknown" resources with zero context grounded_track.append(l) - + log_event(f"[*] Agent Phase 1: Analyst Evaluation ({len(to_evaluate)} resources)...") log_event(f" [>] Fast-Track: {len(fast_track)} | Grounded-Track: {len(grounded_track)}") - + analyst_results = [] - + # 1.1 Fast-Track: Large Batches, NO GROUNDING (Fast) - BATCH_SIZE_FAST = 25 - for i in range(0, len(fast_track), BATCH_SIZE_FAST): + BATCH_SIZE_FAST = 40 # Increased from 25 + total_fast = len(fast_track) + for i in range(0, total_fast, BATCH_SIZE_FAST): batch = fast_track[i:i+BATCH_SIZE_FAST] + batch_num = (i // BATCH_SIZE_FAST) + 1 + total_batches = (total_fast + BATCH_SIZE_FAST - 1) // BATCH_SIZE_FAST + log_event(f" [>] Fast-Track: Processing Batch {batch_num}/{total_batches}...") + prompt = ( f"You are the Nubenetes Technical Analyst (2026).\n" f"{dynamic_mandates}\n" @@ -328,7 +335,7 @@ class V2VisionEngine: item = batch[idx].copy() eval_data = { "year": str(res.get("year", "N/A")), "stars": min(max(int(res.get("stars", 0)), 0), 5), - "ai_summary": res.get("summary", item.get("ai_summary", "")), + "ai_summary": res.get("summary", item.get("ai_summary", "")), "language": res.get("language", "English"), "resource_type": res.get("type", "Reference"), "complexity": res.get("complexity", "Intermediate"), "hierarchy": res.get("hierarchy", ["General"]), "tags": res.get("tags", []), @@ -337,14 +344,19 @@ class V2VisionEngine: } item.update(eval_data) analyst_results.append(item) - except: + except: for l in batch: analyst_results.append(l) - await asyncio.sleep(0.5) + await asyncio.sleep(0.5) # 1.2 Grounded-Track: Small Batches, WITH GROUNDING (Slower but precise) - BATCH_SIZE_GROUNDED = 5 # Reduced batch for grounding to avoid 429 - for i in range(0, len(grounded_track), BATCH_SIZE_GROUNDED): + BATCH_SIZE_GROUNDED = 15 # Increased from 5 + total_grounded = len(grounded_track) + for i in range(0, total_grounded, BATCH_SIZE_GROUNDED): batch = grounded_track[i:i+BATCH_SIZE_GROUNDED] + batch_num = (i // BATCH_SIZE_GROUNDED) + 1 + total_batches = (total_grounded + BATCH_SIZE_GROUNDED - 1) // BATCH_SIZE_GROUNDED + log_event(f" [🌟] Grounded-Track: Processing Batch {batch_num}/{total_batches} (Grounding active)...") + prompt = ( f"You are the Nubenetes Technical Analyst (2026).\n" f"{dynamic_mandates}\n" @@ -370,11 +382,9 @@ class V2VisionEngine: } item.update(eval_data) analyst_results.append(item) - except: + except: for l in batch: analyst_results.append(l) - await asyncio.sleep(5.0) # Grounding is very expensive in terms of quota - - # --- AGENT PHASE 2: SELECTIVE AUDIT (MCP-Grounded) --- + await asyncio.sleep(2.0) # Reduced from 5.0 to improve throughput # --- AGENT PHASE 2: SELECTIVE AUDIT (MCP-Grounded) --- # Identify candidates for high-trust verification audit_candidates = [l for l in analyst_results if "[DE FACTO STANDARD]" in l.get("tags", []) or "[ENTERPRISE-STABLE]" in l.get("tags", [])] @@ -562,7 +572,7 @@ class V2VisionEngine: async def _write_premium_files(self, data: Dict[str, Dict], mosaic_html: str, videos_html: str): # 1. Update Index with Pulse trending_pool = sorted([dict(meta, url=url) for url, meta in self.inventory.items() if isinstance(meta, dict) and meta.get("stars", 0) >= 4], key=lambda x: (x.get("pub_date", "0000"), -x.get("stars", 0)), reverse=True) - pulse_md = "## ⚡ The Agentic Pulse\n" + "\n".join([f"- **({l.get('pub_date', 'N/A')[:10]})** [**=={l['title']}==**]({l['url']}) {'🌟'*l.get('stars',3)}" for l in trending_pool[:5]]) + pulse_md = "## The Agentic Pulse\n" + "\n".join([f"- **({l.get('pub_date', 'N/A')[:10]})** [**=={l['title']}==**]({l['url']}) {'🌟'*l.get('stars',3)}" for l in trending_pool[:5]]) index_md = ( "# Nubenetes Elite Portal (V2) | Nubenetes: Awesome Kubernetes & Cloud [![Awesome](https://cdn.jsdelivr.net/gh/sindresorhus/awesome@d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome)\n\n" @@ -594,7 +604,7 @@ class V2VisionEngine: index_md += ( "\n***\n\n" - "## 💎 The Maturity Taxonomy\n\n" + "## The Maturity Taxonomy\n\n" "To ensure industrial-grade precision, every resource in V2 is classified using our proprietary 5-tier maturity system:\n\n" "| Tag | Description | Engineering Context |\n" "| :--- | :--- | :--- |\n" @@ -603,7 +613,7 @@ class V2VisionEngine: "| **`[EMERGING]`** | The cutting edge. | High-potential tools and patterns (e.g., AI Agents, MCP) shaping the future. |\n" "| **`[GUIDE]`** | Strategic knowledge. | High-quality tutorials, architectural deep-dives, and decision matrices. |\n" "| **`[LEGACY]`** | Historical context. | Established tools that are being replaced or are primarily for maintaining older stacks. |\n\n" - "## 🌟 Technical Impact (Relevance Score)\n\n" + "## Technical Impact (Relevance Score)\n\n" "The stars accompanying each resource represent its **Technical Impact** and **Architectural Relevance** for a 2026 Senior Architect:\n\n" "| Impact | Level | Meaning | Visual Code |\n" "| :---: | :--- | :--- | :--- |\n" diff --git a/v2-docs/index.md b/v2-docs/index.md index 7925db20..ef4cdcba 100644 --- a/v2-docs/index.md +++ b/v2-docs/index.md @@ -28,7 +28,7 @@ [![Azure Terraformer](images/azure-terraformer.jpg){: style="width:7%"}](https://www.youtube.com/@azure-terraformer) [![Ned in the Cloud](images/nedinthecloud.jpg){: style="width:7%"}](https://www.youtube.com/@NedintheCloud) [![netbox](images/netboxlabs_logo.jpg){: style="width:7%"}](https://www.youtube.com/@NetBoxLabs) [![Tech with Helen](images/techwithhelen.jpg){: style="width:7%"}](https://www.youtube.com/@techwithhelen) [![bytebytego](images/bytebytego.jpg){: style="width:7%"}](https://www.youtube.com/@ByteByteGo) [![dotcsv](images/dotcsv.jpg){: style="width:7%"}](https://www.youtube.com/@DotCSV) [![midulive](images/midulive.jpg){: style="width:7%"}](https://www.youtube.com/@midulive) [![returngis](images/returngis_logo.jpg){: style="width:7%"}](https://www.youtube.com/@returngis) [![kubefm](images/kubefm_logo.jpg){: style="width:7%"}](https://www.youtube.com/@kubefm) -## ⚡ The Agentic Pulse +## The Agentic Pulse ## Strategic Dimensions @@ -202,7 +202,7 @@ *** -## 💎 The Maturity Taxonomy +## The Maturity Taxonomy To ensure industrial-grade precision, every resource in V2 is classified using our proprietary 5-tier maturity system: @@ -214,7 +214,7 @@ To ensure industrial-grade precision, every resource in V2 is classified using o | **`[GUIDE]`** | Strategic knowledge. | High-quality tutorials, architectural deep-dives, and decision matrices. | | **`[LEGACY]`** | Historical context. | Established tools that are being replaced or are primarily for maintaining older stacks. | -## 🌟 Technical Impact (Relevance Score) +## Technical Impact (Relevance Score) The stars accompanying each resource represent its **Technical Impact** and **Architectural Relevance** for a 2026 Senior Architect: