arch: decouple V2 monolith into specialized micro-workflows (Health, Metadata, AI, Publish)

.github/workflows/agentic_v2_ai.yml

@@ -0,0 +1,69 @@
+name: Nubenetes V2 AI Curator
+
+on:
+  workflow_dispatch:
+    inputs:
+      force_reevaluate:
+        description: 'Force AI re-evaluation (ignores cache for tags/years)'
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: v2-ai-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  ai-curation:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Repository Synchronization
+        uses: actions/checkout@v6
+        with:
+          ref: develop
+
+      - name: Python 3.11 Environment Provisioning
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Restore Incremental Inventory Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: data/inventory.yaml
+          key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            inventory-v2-
+
+      - name: Installation of Dependencies
+        run: |
+          pip install --no-cache-dir pydantic PyGithub httpx fake-useragent pytz python-dotenv pyyaml tenacity
+          
+      - name: Run V2 AI Curator
+        env:
+          GEMINI_API_KEY_1: ${{ secrets.GEMINI_API_KEY_1 }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FORCE_EVAL: ${{ github.event.inputs.force_reevaluate || 'false' }}
+          PYTHONPATH: .
+          PYTHONUNBUFFERED: "1"
+        run: python -u src/v2_ai.py
+
+      - name: Create Pull Request for Inventory Update
+        uses: peter-evans/create-pull-request@v8
+        with:
+          branch: bot/v2-ai-sync
+          base: develop
+          title: "V2 Data: AI Curator Sync"
+          commit-message: "chore: update inventory AI analysis [skip ci]"
+          labels: "v2-data, ai-curator"
+
+      - name: Persist Incremental Inventory to Cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: data/inventory.yaml
+          key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/agentic_v2_health.yml

@@ -0,0 +1,69 @@
+name: Nubenetes V2 Health Monitor
+
+on:
+  schedule:
+    - cron: '0 0 * * 0' # Weekly on Sunday
+  workflow_dispatch:
+    inputs:
+      force_full_check:
+        description: 'Force full re-validation (bypasses cache)'
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: v2-health-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  health-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Repository Synchronization
+        uses: actions/checkout@v6
+        with:
+          ref: develop
+
+      - name: Python 3.11 Environment Provisioning
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Restore Incremental Inventory Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: data/inventory.yaml
+          key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            inventory-v2-
+
+      - name: Installation of Dependencies
+        run: |
+          pip install --no-cache-dir pydantic PyGithub httpx fake-useragent pytz python-dotenv pyyaml tenacity
+          
+      - name: Run V2 Health Monitor
+        env:
+          FORCE_FULL_CHECK: ${{ github.event.inputs.force_full_check || 'false' }}
+          PYTHONPATH: .
+          PYTHONUNBUFFERED: "1"
+        run: python -u src/v2_health.py
+
+      - name: Create Pull Request for Inventory Update
+        uses: peter-evans/create-pull-request@v8
+        with:
+          branch: bot/v2-health-sync
+          base: develop
+          title: "V2 Data: Health Monitor Sync"
+          commit-message: "chore: update inventory health status [skip ci]"
+          labels: "v2-data, health-monitor"
+
+      - name: Persist Incremental Inventory to Cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: data/inventory.yaml
+          key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/agentic_v2_metadata.yml

@@ -0,0 +1,70 @@
+name: Nubenetes V2 Metadata Engine
+
+on:
+  schedule:
+    - cron: '0 0 1,15 * *' # Bi-weekly
+  workflow_dispatch:
+    inputs:
+      enrich_metadata:
+        description: 'Enrich GitHub Metadata (fetch stars/license)'
+        type: boolean
+        default: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: v2-metadata-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  enrich-metadata:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Repository Synchronization
+        uses: actions/checkout@v6
+        with:
+          ref: develop
+
+      - name: Python 3.11 Environment Provisioning
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Restore Incremental Inventory Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: data/inventory.yaml
+          key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            inventory-v2-
+
+      - name: Installation of Dependencies
+        run: |
+          pip install --no-cache-dir pydantic PyGithub httpx fake-useragent pytz python-dotenv pyyaml tenacity
+          
+      - name: Run V2 Metadata Engine
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ENRICH_METADATA: ${{ github.event.inputs.enrich_metadata || 'true' }}
+          PYTHONPATH: .
+          PYTHONUNBUFFERED: "1"
+        run: python -u src/v2_metadata.py
+
+      - name: Create Pull Request for Inventory Update
+        uses: peter-evans/create-pull-request@v8
+        with:
+          branch: bot/v2-metadata-sync
+          base: develop
+          title: "V2 Data: Metadata Engine Sync"
+          commit-message: "chore: update inventory github metadata [skip ci]"
+          labels: "v2-data, metadata-engine"
+
+      - name: Persist Incremental Inventory to Cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: data/inventory.yaml
+          key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/agentic_v2_publish.yml

@@ -1,36 +1,28 @@
-name: Nubenetes V2 Agentic Builder
+name: Nubenetes V2 Publisher
 
 on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'data/inventory.yaml'
+      - 'docs/**'
   workflow_dispatch:
-    inputs:
-      force_full_check:
-        description: 'Force full re-validation (bypasses cache)'
-        type: boolean
-        default: false
-      force_reevaluate:
-        description: 'Force AI re-evaluation (ignores cache for tags/years)'
-        type: boolean
-        default: false
-      enrich_metadata:
-        description: 'Enrich GitHub Metadata (fetch stars/license for V2 logic)'
-        type: boolean
-        default: false
 
 permissions:
   contents: write
   pull-requests: write
 
 concurrency:
-  group: v2-builder-${{ github.ref }}
+  group: v2-publisher-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
-  build-and-pr:
+  publish-v2:
     runs-on: ubuntu-latest
     if: |
       github.event_name == 'workflow_dispatch' || 
       (github.event_name == 'push' && 
-       github.ref == 'refs/heads/develop' && 
        !contains(github.event.head_commit.message, '[skip ci]') &&
        !contains(github.event.head_commit.message, 'bot/v2-elite-sync'))
     steps:
@@ -38,7 +30,6 @@ jobs:
         uses: actions/checkout@v6
         with:
           ref: develop
-          fetch-depth: 0
 
       - name: Python 3.11 Environment Provisioning
         uses: actions/setup-python@v6
@@ -46,9 +37,8 @@ jobs:
           python-version: '3.11'
           cache: 'pip'
 
-      - name: Restore Incremental Inventory Cache (Mandate 22)
+      - name: Restore Incremental Inventory Cache
         uses: actions/cache/restore@v4
-        id: inventory-cache-restore
         with:
           path: data/inventory.yaml
           key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }}
@@ -57,38 +47,25 @@ jobs:
 
       - name: Installation of Dependencies
         run: |
-          python -m pip install --upgrade pip
           pip install --no-cache-dir pydantic PyGithub httpx fake-useragent pytz python-dotenv pyyaml tenacity
           
-      - name: Run V2 Agentic Optimizer (Sequential Fast-Track)
+      - name: Run V2 Publisher (Render-Only)
         env:
-          GEMINI_API_KEY_1: ${{ secrets.GEMINI_API_KEY_1 }}
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          FORCE_FULL_CHECK: ${{ github.event.inputs.force_full_check || 'false' }}
-          FORCE_EVAL: ${{ github.event.inputs.force_reevaluate || 'false' }}
-          ENRICH_METADATA: ${{ github.event.inputs.enrich_metadata || 'false' }}
           PYTHONPATH: .
           PYTHONUNBUFFERED: "1"
         run: |
-          python -u src/v2_optimizer.py
+          python -u src/v2_optimizer.py --render-only
           if [ -f v2_safety_report.md ]; then
             cat v2_safety_report.md >> pr_description.md
           else
             echo "No safety report generated." > pr_description.md
           fi
 
-      - name: Consolidate README Metrics (Integrated)
-        env:
-          PYTHONPATH: .
+      - name: Consolidate README Metrics
         run: |
           python src/readme_updater.py
           python src/safety_readme.py
 
-      - name: Safety Reset for Workflow Files (Security)
-        run: |
-          # Prevent security rejection by ensuring no workflow YAMLs are part of the PR
-          git checkout HEAD -- .github/workflows/
-
       - name: Create Pull Request for V2 Elite Update
         id: cpr
         uses: peter-evans/create-pull-request@v8
@@ -117,14 +94,13 @@ jobs:
           PR_NUMBER: ${{ steps.cpr.outputs.pull-request-number }}
         run: |
           if [ -f v2_decision_matrix.md ]; then
-            # Split decision matrix if too large for a single comment (approx 60k chars)
             split -b 60000 v2_decision_matrix.md matrix_part_
             for part in matrix_part_*; do
               gh pr comment $PR_NUMBER --body-file $part
             done
           fi
 
-      - name: Persist Incremental Inventory to Cache (Always)
+      - name: Persist Incremental Inventory to Cache
         if: always()
         uses: actions/cache/save@v4
         with:

GEMINI.md

@@ -140,11 +140,14 @@ This file contains the accumulated instructions and long-term vision for the aut
 44. **Agentic Presubmit Safeguards (PR Guardian)**: All PRs to `develop` MUST be analyzed by the `PR Guardian` AI agent to ensure compliance with Nubenetes standards (No emojis in headers, valid Markdown, correct URL normalization, high-density descriptions).
 45. **Resilient Quota Management (Circuit Breakers)**: AI workflows MUST implement circuit breaker logic (Exit Code 42) to gracefully pause processing and disable the workflow when API quotas (e.g., 429 Too Many Requests) are exhausted, preventing infinite loop failures.
 46. **Markdown Linting Continuity**: All files in `docs/` and `v2-docs/` MUST pass the automated `markdownlint` validation to ensure pristine HTML rendering within MkDocs.
-47. **Zero-Redundancy Agentic Pipeline (Performance Standard)**: To maintain the **30-minute execution standard**, the V2 Optimizer MUST NOT perform redundant network health checks or grounding-heavy AI calls for resources already validated by the `IntelligentLinkCleaner`.
-    - **Metadata Inheritance**: V2 MUST prioritize `gh_stars`, `gh_license`, and `gh_pushed` from the inventory to bypass AI grounding during maturity tagging.
-    - **Smart Grounding**: Google Search Retrieval is strictly reserved for resources missing critical metadata or those flagged as `needs_ai_refresh`.
-    - **Linear Knowledge Flow**: The workflow follows a strict sequence: 1. Health/Metadata (Cleaner) -> 2. Distributed Inventory -> 3. Fast-Track Optimization (V2).
+- **Zero-Redundancy Agentic Pipeline (Performance Standard)**: To maintain the **30-minute execution standard**, the V2 ecosystem utilizes a decoupled micro-workflow architecture:
+    - **V2 Health Monitor**: Weekly network validation of the link archive.
+    - **V2 Metadata Engine**: Bi-weekly extraction of GitHub stars and licenses.
+    - **V2 AI Curator**: On-demand deep architectural analysis and hierarchical indexing.
+    - **V2 Publisher**: Fast-track rendering of the Elite portal from pre-enriched metadata.
+    - **Linear Knowledge Flow**: The workflow follows a strict sequence: 1. Health/Metadata (Decoupled) -> 2. Distributed Inventory -> 3. Fast-Track Optimization (V2 Publisher).
     - **Manual Override Control**: All agents must respect the manual workflow flags (`FORCE_FULL_CHECK`, `FORCE_EVAL`, `ENRICH_METADATA`). When disabled (Standard Run), the system MUST strictly enforce the cache-first policy for maximum efficiency.
+
 37. **Linguistic Uniformity**: All core documentation (index, README, GEMINI.md) and V2 portal summaries MUST be written in **Professional Technical English**. V1 descriptions remain in their native language (Mandate 10).
 48. **Flash-First High-Density Curation (Scale Mandate)**: For mass processing (>1,000 resources), the system MUST prioritize **Gemini Flash/Lite** models for the Analyst phase. This ensures high RPM/TPM throughput while maintaining cost efficiency. Pro models are strictly reserved for the Auditor phase or high-value resource verification.
 49. **Robust Batch Processing & Rate-Limit Resilience**: Large-scale curation MUST use batch sizes of **50 resources** for Fast-Track processing with a mandatory **2-second safety delay** between batches. 

src/v2_ai.py

@@ -0,0 +1,25 @@
+import asyncio
+from src.v2_optimizer import V2VisionEngine
+from src.logger import log_event
+from src.gemini_utils import normalize_url
+
+async def run_ai_curation():
+    engine = V2VisionEngine(render_only=False)
+    log_event("STARTING V2 SPECIALIZED AI CURATOR", section_break=True)
+    
+    all_v1_links, _, _ = await engine._gather_all_v1_content()
+    log_event(f"[*] Discovery: Found {len(all_v1_links)} resources for AI analysis.")
+    
+    # Filter for online links to save tokens
+    online_links = [l for l in all_v1_links if engine.inventory.get(normalize_url(l["url"]), {}).get("status") == "online"]
+    log_event(f"[*] Analysis: Processing {len(online_links)} online resources.")
+    
+    # Run the evaluation phase (Analyst + Auditor)
+    # This method already handles incremental persistence and batching
+    await engine._evaluate_and_score_resources(online_links)
+    
+    engine._save_inventory()
+    log_event("V2 AI CURATOR COMPLETED SUCCESSFULLY.")
+
+if __name__ == "__main__":
+    asyncio.run(run_ai_curation())

src/v2_health.py

@@ -0,0 +1,19 @@
+import asyncio
+from src.v2_optimizer import V2VisionEngine
+from src.logger import log_event
+
+async def run_health_check():
+    engine = V2VisionEngine(render_only=False)
+    log_event("STARTING V2 SPECIALIZED HEALTH MONITOR", section_break=True)
+    
+    all_v1_links, _, _ = await engine._gather_all_v1_content()
+    log_event(f"[*] Discovery: Found {len(all_v1_links)} resources to validate.")
+    
+    # We only run the health check phase
+    await engine._verify_link_health(all_v1_links)
+    
+    engine._save_inventory()
+    log_event("V2 HEALTH MONITOR COMPLETED SUCCESSFULLY.")
+
+if __name__ == "__main__":
+    asyncio.run(run_health_check())

src/v2_metadata.py

@@ -0,0 +1,42 @@
+import os
+import asyncio
+from src.v2_optimizer import V2VisionEngine
+from src.logger import log_event
+from src.gemini_utils import normalize_url, get_github_activity
+
+async def run_metadata_enrichment():
+    engine = V2VisionEngine(render_only=False)
+    log_event("STARTING V2 SPECIALIZED METADATA ENGINE", section_break=True)
+    
+    all_v1_links, _, _ = await engine._gather_all_v1_content()
+    log_event(f"[*] Discovery: Found {len(all_v1_links)} resources for metadata enrichment.")
+    
+    # Mandate 15 & 43: Proactive Enrichment for V2
+    processed_gh_metadata = set()
+    gh_fetch_count = 0
+    force_full = os.getenv("ENRICH_METADATA", "false").lower() == "true"
+
+    for l in all_v1_links:
+        norm_url = normalize_url(l["url"])
+        if "github.com" not in norm_url: continue
+        
+        cached = engine.inventory.get(norm_url, {})
+        # Enrich if missing or if forced
+        if (force_full or not cached.get("gh_stars")) and norm_url not in processed_gh_metadata:
+            log_event(f"  [METADATA] Fetching GH Activity for {norm_url}")
+            processed_gh_metadata.add(norm_url)
+            gh_data = await get_github_activity(norm_url)
+            if gh_data:
+                if norm_url not in engine.inventory: engine.inventory[norm_url] = {}
+                engine.inventory[norm_url].update(gh_data)
+                
+            gh_fetch_count += 1
+            if gh_fetch_count % 100 == 0:
+                engine._save_inventory()
+                log_event(f"    [💾] Periodic Save: {gh_fetch_count} fetches...")
+
+    engine._save_inventory()
+    log_event(f"V2 METADATA ENGINE COMPLETED. Processed {gh_fetch_count} repos.")
+
+if __name__ == "__main__":
+    asyncio.run(run_metadata_enrichment())