From dff19eff1042089b6f2ff28991edc48ace721cf3 Mon Sep 17 00:00:00 2001 From: Nubenetes Bot Date: Sun, 17 May 2026 13:22:32 +0200 Subject: [PATCH] feat(ops): unify metadata architecture and restore critical database mandates in GEMINI.md --- GEMINI.md | 14 ++++---- README.md | 3 -- src/agentic_curator.py | 13 ------- src/intelligent_health_checker.py | 60 ++++++++++++++----------------- src/v2_optimizer.py | 31 ++++++++-------- 5 files changed, 50 insertions(+), 71 deletions(-) diff --git a/GEMINI.md b/GEMINI.md index 0eedb217..5a2d7836 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -44,13 +44,12 @@ This file contains the accumulated instructions and long-term vision for the aut - **Flat Asset Routing**: To avoid depth-related path breakage, both V1 (`mkdocs.yml`) and V2 (`v2-mkdocs.yml`) MUST have `use_directory_urls: false`. This ensures relative paths (e.g., `images/img.png`) resolve correctly regardless of the page depth. 20. **V2 Navigation Design**: The V2 top navigation bar MUST maintain a flat structure. All dimensions and categories must be top-level tabs in `v2-mkdocs.yml` to ensure direct discoverability and avoid nested groupings like "Categories". 21. **V2 Impact-Driven Sorting**: The V2 portal MUST prioritize **relevance (Impact) over dates** within sections to provide high-density technical value. Sorting MUST follow: 1. Stars/Relevance (DESC), 2. Year (DESC). The mission statement and descriptions MUST reflect this impact-driven synthesis. -22. **Unified Metadata Database (Local Storage & Persistence)**: All link metadata MUST be managed via the local YAML database in `data/`. - - **`inventory.yaml`**: The primary source of truth for years, stars (0-5), and descriptions. - - **`structure_map.yaml`**: Tracks link locations and visual formatting (bold/highlight) across V1 and V2. - - **Persistence (MANDATORY)**: Every AI agent and workflow MUST include these YAML files in their Pull Requests if any change is detected. Discarding the database during a workflow run is a CRITICAL FAILURE. All workflows must load the DB, update it, and INJECT the modified YAML files into the final PR payload. - - **Exhaustive Initialization**: The system supports a `FORCE_FULL_CHECK` environment variable to bypass all caches (e.g., 21-day health cache) and force a full re-validation and re-enrichment of the entire 17k+ link archive. +22. **Unified Database Architecture (Single Source of Truth)**: All resource metadata MUST be managed via the centralized [`data/inventory.yaml`](data/inventory.yaml). + - **Persistence (MANDATORY)**: Every AI agent and workflow MUST load this file at startup, update it, and INJECT the modified YAML into the final PR payload. Discarding the database during a workflow run is a CRITICAL FAILURE. + - **Lifecycle Metadata**: The inventory MUST track the physical locations of every link in the project (fields: `v1_locations` and `v2_locations`) and its visual formatting. This eliminates the need for external mapping files (like the legacy `structure_map.yaml`) and ensures full visibility across workflows. + - **Exhaustive Initialization**: The system supports a `FORCE_FULL_CHECK` environment variable to bypass all caches and force a full re-validation and re-enrichment of the entire 17k+ link archive. - **No Trusted Bypassing**: All domains, including high-trust ones (GitHub, Google, AWS), MUST be verified for link validity. Trusted status only grants a lower priority for aggressive scraper rotation, not a bypass for existence checks. - - **Manual Priority**: AI agents MUST NOT overwrite existing manual descriptions in the V1 archive files. Enrichment is strictly for `inventory.yaml` and the V2 portal. + - **Manual Priority**: AI agents MUST NOT overwrite existing manual descriptions or stars in the V1 archive files. Enrichment is strictly for the YAML database and the V2 portal. 23. **Canonical URL Normalization & Semantic Deduplication**: To prevent duplication and fragmented metadata, all agents MUST normalize URLs before any inventory operation. - **Tracking Stripping**: Systematically remove UTM parameters, social media trackers (X.com, LinkedIn), and URL fragments (except technical ones). - **Technical Preservation (V1)**: Normalization MUST **preserve line anchors** (e.g., `#L123`) and **respect URL path case-sensitivity**. Technical fragmentation is preferred over data loss for deep-links. @@ -75,7 +74,8 @@ This file contains the accumulated instructions and long-term vision for the aut - **AI Curation Discovery**: The discovery engine MUST actively search for new high-quality curation sources (e.g., "Awesome" repos) and suggest them for inclusion in `curation_sources.yaml`. 28. **Sophisticated V2 Knowledge Architecture**: The V2 Portal MUST be structured like an advanced O'Reilly technical book: - **Deep Hierarchical Classification**: Resources MUST be organized using the `hierarchy` metadata field (a list of up to 10 strings: Area > Topic > Subtopics). This structure is mandatory for both V1 reorganization and V2 generation to ensure perfect consistency. - - **Structural Intelligence Persistence**: All agents MUST store and reuse the `hierarchy` metadata in the centralized inventory. This ensures zero-cost structural updates and industrial-grade scalability. + - **Location-Aware Automation**: Workflows MUST utilize the location metadata in the inventory (`v1_locations`, `v2_locations`) to perform surgical updates without redundant repository scans. + - **Structural Intelligence Persistence**: All agents MUST store and reuse the `hierarchy` and location metadata in the centralized inventory. This ensures zero-cost structural updates and industrial-grade scalability. - **O'Reilly Learning Flow**: The organization must facilitate knowledge assimilation, moving from foundational theory to advanced engineering internals in a logical, ordered sequence. - **Dynamic Indexing**: Every V2 page MUST include a Table of Contents (TOC) with clickable anchors for all technical sub-sections. - **AI Dimension Naming**: Prioritize industry-standard terms (e.g., "AI and Artificial Intelligence" instead of internal jargon) for top-level navigation. diff --git a/README.md b/README.md index 0572caee..22d02137 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,6 @@ Nubenetes now utilizes a **Unified Metadata Architecture** to maintain consisten ### 6.1. Database Components 1. **Central Inventory (`data/inventory.yaml`)**: Stores global technical metadata. * `title`, `year`, `stars` (0-5), `description` (V1), `ai_summary` (V2 Elite), `category`, and `related_categories`. -2. **Structure Map (`data/structure_map.yaml`)**: Tracks the physical presence and formatting of links. * Tracks which `.md` pages contain the link in V1 and V2. * Stores visual state: `is_bold`, `is_highlighted` (`==`). @@ -432,7 +431,6 @@ graph TD subgraph Local Storage DB1[inventory.yaml] - DB2[structure_map.yaml] end ``` @@ -644,7 +642,6 @@ To maintain transparency and ease of navigation, all key configuration, database ### 13.2. Centralized Metadata Databases - **Global Inventory:** [`data/inventory.yaml`](data/inventory.yaml) - The "System Memory" containing all link metadata (years, stars, descriptions, and audit history). -- **Structure Map:** [`data/structure_map.yaml`](data/structure_map.yaml) - Tracks link presence and formatting across all Markdown pages. ### 13.3. Autonomous Workflows - **Discovery & Curation:** [`.github/workflows/agentic_cron.yml`](.github/workflows/agentic_cron.yml) diff --git a/src/agentic_curator.py b/src/agentic_curator.py index 952f0f35..3cc1068b 100644 --- a/src/agentic_curator.py +++ b/src/agentic_curator.py @@ -195,7 +195,6 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: return evaluations INVENTORY_PATH = "data/inventory.yaml" -STRUCTURE_MAP_PATH = "data/structure_map.yaml" class AgenticCurator: def __init__(self): @@ -205,7 +204,6 @@ class AgenticCurator: self.index_path = "docs/index.md" self.stats = {"orphans_linked": 0} self.inventory = self._load_inventory() - self.structure_map = self._load_structure_map() def _load_inventory(self) -> dict: if os.path.exists(INVENTORY_PATH): @@ -218,17 +216,6 @@ class AgenticCurator: os.makedirs(os.path.dirname(INVENTORY_PATH), exist_ok=True) with open(INVENTORY_PATH, "w") as f: yaml.dump(self.inventory, f, sort_keys=False, allow_unicode=True) - def _load_structure_map(self) -> dict: - if os.path.exists(STRUCTURE_MAP_PATH): - try: - with open(STRUCTURE_MAP_PATH, "r") as f: return yaml.safe_load(f) or {} - except: return {} - return {} - - def _save_structure_map(self): - os.makedirs(os.path.dirname(STRUCTURE_MAP_PATH), exist_ok=True) - with open(STRUCTURE_MAP_PATH, "w") as f: yaml.dump(self.structure_map, f, sort_keys=False, allow_unicode=True) - async def _rebuild_toc(self, content: str) -> str: lines = content.splitlines() headers = [] diff --git a/src/intelligent_health_checker.py b/src/intelligent_health_checker.py index 839afd8b..70218e70 100644 --- a/src/intelligent_health_checker.py +++ b/src/intelligent_health_checker.py @@ -17,7 +17,6 @@ from src.gemini_utils import normalize_url CORE_FILES = ["docs/index.md", "README.md"] MEMORY_FILE = "src/memory/health_learning.json" INVENTORY_PATH = "data/inventory.yaml" -STRUCTURE_MAP_PATH = "data/structure_map.yaml" class IntelligentLinkCleaner: def __init__(self): @@ -30,7 +29,6 @@ class IntelligentLinkCleaner: self.description_updates: Dict[str, str] = {} self.learning_data = self._load_memory() self.inventory = self._load_inventory() - self.structure_map = self._load_structure_map() self.action_log: List[Dict] = [] self.detailed_stats = { "total_scanned": 0, @@ -67,21 +65,6 @@ class IntelligentLinkCleaner: import yaml yaml.dump(self.inventory, f, sort_keys=False, allow_unicode=True) - def _load_structure_map(self) -> dict: - if os.path.exists(STRUCTURE_MAP_PATH): - try: - with open(STRUCTURE_MAP_PATH, "r") as f: - import yaml - return yaml.safe_load(f) or {} - except: return {} - return {} - - def _save_structure_map(self): - os.makedirs(os.path.dirname(STRUCTURE_MAP_PATH), exist_ok=True) - with open(STRUCTURE_MAP_PATH, "w") as f: - import yaml - yaml.dump(self.structure_map, f, sort_keys=False, allow_unicode=True) - async def _fetch_github_metadata(self, url: str) -> Dict: match = re.search(r'github\.com/([^/]+)/([^/]+)', url) if not match: return {} @@ -366,30 +349,41 @@ class IntelligentLinkCleaner: async def prune_orphaned_metadata(self): """ DATABASE GARBAGE COLLECTOR: Removes metadata for links no longer present in any .md file. - Ensures inventory.yaml and structure_map.yaml remain lean and professional. + Updates 'v1_locations' in inventory for all active links. """ log_event("RUNNING DATABASE GARBAGE COLLECTION...", section_break=True) initial_inv = len(self.inventory) - initial_struct = len(self.structure_map) - # Identify valid links from registry (those actually found in docs/) - valid_urls = {normalize_url(u) for u in self.link_registry.keys()} + # 1. Gather all current links in V1 + valid_urls_map = {} # {url: [paths]} + for root, _, files in os.walk("docs"): + for file in files: + if file.endswith(".md"): + path = os.path.join(root, file) + with open(path, "r") as f: content = f.read() + urls = re.findall(r'\[.*?\]\((https?://.*?)\)', content) + for u in urls: + nu = normalize_url(u) + if path not in valid_urls_map.get(nu, []): + valid_urls_map.setdefault(nu, []).append(path) - # Prune Inventory - self.inventory = {u: m for u, m in self.inventory.items() if u in valid_urls} - # Prune Structure Map - self.structure_map = {u: m for u, m in self.structure_map.items() if u in valid_urls} + # 2. Prune and Update Locations + new_inventory = {} + for u, m in self.inventory.items(): + if u.startswith("INTRO:"): + new_inventory[u] = m + continue + if u in valid_urls_map: + m["v1_locations"] = sorted(list(set(valid_urls_map[u]))) + new_inventory[u] = m + self.inventory = new_inventory pruned_inv = initial_inv - len(self.inventory) - pruned_struct = initial_struct - len(self.structure_map) - if pruned_inv > 0 or pruned_struct > 0: - log_event(f" [OK] Pruned {pruned_inv} orphaned inventory entries.") - log_event(f" [OK] Pruned {pruned_struct} orphaned structure mappings.") - self._save_inventory() - self._save_structure_map() + if pruned_inv > 0: + log_event(f" [OK] Pruned {pruned_inv} orphaned records from inventory.") else: - log_event(" [OK] Database is already lean. No orphans found.") + log_event(" [OK] Database is clean. No orphans found.") async def _enrich_description_batch(self, urls: List[str]): """ @@ -532,14 +526,12 @@ class IntelligentLinkCleaner: # 3. Ensure BBDD YAML Persistence: Include database files in the PR payload await self.prune_orphaned_metadata() # GC first self._save_inventory() - self._save_structure_map() final_payload = {p: "".join([l for l in lines if l is not None]) for p, lines in file_updates.items()} # Load fresh YAML content for the PR import yaml with open(INVENTORY_PATH, "r") as f: final_payload[INVENTORY_PATH] = f.read() - with open(STRUCTURE_MAP_PATH, "r") as f: final_payload[STRUCTURE_MAP_PATH] = f.read() if orphans_linked > 0: with open(getattr(self.curator, "index_path", "docs/index.md"), 'r') as f: diff --git a/src/v2_optimizer.py b/src/v2_optimizer.py index a26e7378..6f9cb730 100644 --- a/src/v2_optimizer.py +++ b/src/v2_optimizer.py @@ -6,7 +6,7 @@ import yaml import httpx from datetime import datetime from typing import List, Dict, Set, Any, Tuple -from src.config import GEMINI_API_KEYS, GH_TOKEN, TARGET_REPO, MADRID_TZ, INVENTORY_PATH, STRUCTURE_MAP_PATH +from src.config import GEMINI_API_KEYS, GH_TOKEN, TARGET_REPO, MADRID_TZ, INVENTORY_PATH from src.gemini_utils import call_gemini_with_retry, normalize_url from src.logger import log_event @@ -49,7 +49,6 @@ class V2VisionEngine: "- If 'Current Desc' is empty, generate a professional summary. Style: O'Reilly technical.\n" ) self.inventory = self._load_inventory() - self.structure_map = self._load_structure_map() self.maturity_audit = [] def _load_special_assets(self) -> Dict: @@ -80,18 +79,6 @@ class V2VisionEngine: with open(INVENTORY_PATH, "w") as f: yaml.dump(self.inventory, f, sort_keys=False, allow_unicode=True) - def _load_structure_map(self) -> dict: - if os.path.exists(STRUCTURE_MAP_PATH): - try: - with open(STRUCTURE_MAP_PATH, "r") as f: return yaml.safe_load(f) or {} - except: return {} - return {} - - def _save_structure_map(self): - os.makedirs(os.path.dirname(STRUCTURE_MAP_PATH), exist_ok=True) - with open(STRUCTURE_MAP_PATH, "w") as f: - yaml.dump(self.structure_map, f, sort_keys=False, allow_unicode=True) - async def analyze_and_cluster(self): log_event("STARTING V2 HIGH-DENSITY O'REILLY LIBRARY GENERATION", section_break=True) all_v1_links, mosaic_html, videos_html = await self._gather_all_v1_content() @@ -318,6 +305,22 @@ class V2VisionEngine: for dim, content in data.items(): if not content["categories"]: continue slug = dim.lower().replace(" ", "-").replace("&", "and").replace("(", "").replace(")", "") + v2_page = f"{slug}.md" + + # --- TRACK V2 LOCATIONS IN INVENTORY --- + def track_v2(node, page_path): + if "__links__" in node: + for l in node["__links__"]: + nu = normalize_url(l["url"]) + if nu in self.inventory: + locs = self.inventory[nu].get("v2_locations", []) + if page_path not in locs: + self.inventory[nu].setdefault("v2_locations", []).append(page_path) + for k, v in node.items(): + if k != "__links__": track_v2(v, page_path) + + for cat_topics in content["categories"].values(): track_v2(cat_topics, v2_page) + md = f"# {dim}\n\n!!! info \"Architectural Context\"\n {content['summary']}\n\n## Table of Contents\n" for cat, topics in content["categories"].items(): cat_slug = cat.lower().replace(" ", "-")