diff --git a/GEMINI.md b/GEMINI.md index d300c726..17b4f2c2 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -73,10 +73,12 @@ This file contains the accumulated instructions and long-term vision for the aut - **High-Precision Reorganization (V1)**: These files MUST use nested semantic grouping (## and ###) to organize links without ever deleting technically valid content. - **Exhaustive Inclusion (V2)**: Unlike standard categories, V2 pages for Special Assets MUST include 100% of the ALIVE links from V1. - **AI Curation Discovery**: The discovery engine MUST actively search for new high-quality curation sources (e.g., "Awesome" repos) and suggest them for inclusion in `curation_sources.yaml`. -28. **Zero-to-Hero V2 Architecture**: The V2 Portal MUST be structured as a learning journey: - - **Complexity Hierarchy**: Resources MUST be grouped by level: Fundamentals -> Intermediate -> Advanced -> Architect. - - **Strategic Dimensions**: The "AI and Artificial Intelligence" dimension is the primary entry point for agentic innovation. Dimension naming MUST prioritize industry-standard terms over internal terminology. - - **Clickable Navigational Maps**: Every V2 page MUST include a Table of Contents (TOC) with nested anchors for all complexity levels. +28. **Sophisticated V2 Knowledge Architecture**: The V2 Portal MUST be structured like an advanced O'Reilly technical book: + - **Contextual Hierarchy**: Resources MUST NOT be flat lists. They must be organized into a sophisticated hierarchy of nested sections (##) and subsections (###) based on technical areas, topics, and subtopics. + - **Structured Learning**: The organization must facilitate knowledge assimilation, moving from foundations to advanced implementation details in a logical, ordered sequence. + - **Dynamic Indexing**: Every V2 page MUST include a Table of Contents (TOC) with clickable anchors for all technical sub-sections. + - **AI Dimension Naming**: Prioritize industry-standard terms (e.g., "AI and Artificial Intelligence" instead of internal jargon) for top-level navigation. + - **Content Extraction**: High-value sub-topics may be extracted into dedicated documents (e.g., "Microservices" content extracted from `introduction.md` into `microservices.md`) to maintain high-density focus. ## 🛠️ Structural Evolution & Navigation ... diff --git a/README.md b/README.md index 9c16cd80..798fc4a4 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ 12. [12. Developer Experience and VSCode Setup](#12-developer-experience-and-vscode-setup) 14. [14. Special Assets and Learning Paths](#14-special-assets-and-learning-paths) * [14.1. Special Assets Management](#141-special-assets-management) - * [14.2. Zero-to-Hero Learning Architecture](#142-zero-to-hero-learning-architecture) + * [14.2. O'Reilly-style Knowledge Architecture](#142-oreilly-style-knowledge-architecture) * [12.1. Extension Recommendations](#121-extension-recommendations) * [12.2. Recommended settings.json](#122-recommended-settingsjson) 13. [13. Repository Inventory and Configuration](#13-repository-inventory-and-configuration) @@ -673,17 +673,16 @@ Nubenetes prioritizes high-value technical documents through a specialized prese ### 📚 Special Assets Management Certain files are designated as **Special Assets** (defined in [`data/special_assets.yaml`](data/special_assets.yaml)) due to their foundational importance. These include: -- **Introduction & Fundamentals**: Specialized grouping to ensure a perfect "Day 0" experience. -- **YAML & JSON Ecosystem**: Exhaustive technical references for configuration languages. +- **Introduction and Fundamentals**: High-impact fundamental selection for V2, with 100% preservation in V1. +- **Microservices Ecosystem**: A dedicated V2 document (`microservices.md`) extracted from the introduction to maintain architectural focus. +- **YAML and JSON Ecosystem**: Exhaustive technical references for configuration languages. - **Awesome Repositories**: Preserved curation lists that act as gateways to specialized sub-ecosystems. **Rules of Engagement:** -1. **Exhaustive V2 Inclusion**: 100% of ALIVE links from these V1 files are migrated to the V2 Elite portal, bypassing standard impact filters. -2. **High-Precision Grouping**: AI agents use nested hierarchies (Sections & Subsections) to organize these files without losing any technically valid reference. +1. **High-Precision Grouping**: AI agents use nested hierarchies (Areas -> Topics -> Subtopics) to organize these files without losing any technically valid reference, following a **Professional Technical Book** (O'Reilly style) structure. +2. **Elite Curation**: For the V2 Portal, `introduction.md` undergoes a specialized "Elite selection" (Impact ≥ 4) to ensure a high-density entry point for global users. -### 🎓 Zero-to-Hero Learning Architecture -The V2 Portal is structured as an educational journey rather than a flat list. Resources are programmatically classified into four expertise tiers: -- **Fundamentals**: Core concepts and "Getting Started" material. -- **Intermediate**: Practical implementations and standard tooling. -- **Advanced**: Performance optimization and complex technical internals. -- **Architect**: System design, trade-offs, and long-term strategic direction. +### 🎓 O'Reilly-style Knowledge Architecture +The V2 Portal is structured as a sophisticated technical reference guide. Resources are programmatically classified and grouped by technical topics and subtopics, transforming the portal into a structured educational journey for Cloud Native engineering. +- **Structured Assimilation**: Information is ordered sections-by-section to facilitate learning from foundational theory to advanced implementation. +- **Contextual Hierarchy**: Every page features a clickable Table of Contents (TOC) with nested anchors for precise navigation across all technical sub-domains. diff --git a/data/special_assets.yaml b/data/special_assets.yaml index ef34592c..cffe4884 100644 --- a/data/special_assets.yaml +++ b/data/special_assets.yaml @@ -13,9 +13,9 @@ special_assets: keywords: ["yaml", "json", "schema"] - file: "introduction.md" - v1_rule: "Semantic clustering: Group by 'Fundamentals', 'Getting Started', and 'Advanced Concepts'." - v2_rule: "Elite curated view: Highlight only high-impact fundamental resources." - keywords: ["introduction", "getting started", "fundamentals"] + v1_rule: "Semantic clustering: Group by technical sub-topics. Preserve 100% of links." + v2_rule: "Elite curated selection (Impact >= 4). EXTRACT all Microservices-related links into a new dedicated V2 document: 'microservices.md'." + keywords: ["introduction", "getting started", "fundamentals", "microservices"] # Navigation Renaming dimension_renaming: diff --git a/src/agentic_curator.py b/src/agentic_curator.py index a235e3ab..9adf02f1 100644 --- a/src/agentic_curator.py +++ b/src/agentic_curator.py @@ -332,8 +332,10 @@ class AgenticCurator: log_event(f" [!] REORGANIZING: {file} ({'Special' if is_special else 'Standard'})") depth_instruction = ( - "SOPHISTICATED HIERARCHY: Create nested sections (##) and subsections (###). " - "Group links by technical theme. Maintain all existing links. No deletions." + "SOPHISTICATED O'REILLY HIERARCHY: Create nested sections (##) and subsections (###). " + "Group links by technical AREAS, TOPICS, and SUBTOPICS (e.g. 'Foundations' -> 'Container Orchestration'). " + "The structure must facilitate knowledge assimilation like a professional technical book. " + "Ensure every valid link is preserved. Maintain professional titles." if is_special else "Group into logical sections (##)." ) diff --git a/src/v2_optimizer.py b/src/v2_optimizer.py index 01b8f487..9640bdb4 100644 --- a/src/v2_optimizer.py +++ b/src/v2_optimizer.py @@ -36,19 +36,19 @@ class V2VisionEngine: } self.library_criteria = ( - "You are a Senior Technical Librarian and Architect in 2026. Your mission is to build a high-density, professional reference library.\n" - "PHASE 1: TECHNICAL PRESERVATION (HIGH INCLUSIVITY)\n" - "- KEEP >90% of technical resources.\n" - "PHASE 2: SOPHISTICATED SYNTHESIS & DATING\n" - "- Extract precise PUBLICATION DATE (YYYY-MM-DD or YYYY).\n" - "- Detect source content LANGUAGE.\n" - "- Identify RESOURCE_TYPE and complexity LEVEL.\n" - "PHASE 3: ZERO-TO-HERO CLASSIFICATION\n" - "- Categorize into: 'Fundamentals', 'Intermediate', 'Advanced', or 'Architect' level.\n" - "- For special curation lists (e.g. Awesome repos), identify the primary curation topic.\n" - "PHASE 4: MANDATORY DESCRIPTIONS (V1 PRIORITY)\n" - "- If 'Current Desc' is empty or too short, generate a professional 1-2 sentence summary.\n" - "- Style: Technical, neutral, and informative. Language: English only.\n" + "You are a Senior Technical Content Architect in 2026. Your mission is to organize a high-density technical reference portal " + "with the structure and logical flow of an advanced O'Reilly technical book.\n" + "PHASE 1: TECHNICAL PRESERVATION & CURATION\n" + "- KEEP >90% of technical resources (except for 'introduction.md' where only high-impact links are kept).\n" + "PHASE 2: SOPHISTICATED HIERARCHICAL CLASSIFICATION\n" + "- Analyze the deep context of each resource to identify its TECHNICAL AREA (e.g., 'Storage Protocols', 'Observability Patterns').\n" + "- Identify a TOPIC and SUBTOPIC for precise nested grouping.\n" + "- For 'introduction.md', identify links related to MICROSERVICES for extraction.\n" + "PHASE 3: KNOWLEDGE ASSIMILATION FLOW\n" + "- Order topics to facilitate a structured learning journey: from architectural foundations to advanced engineering internals.\n" + "- Assign QUALITY level (0-5 stars) and complexity LEVEL.\n" + "PHASE 4: MANDATORY DESCRIPTIONS\n" + "- If 'Current Desc' is empty, generate a professional 1-2 sentence summary. Style: O'Reilly technical, neutral, and informative.\n" ) self.inventory = self._load_inventory() self.structure_map = self._load_structure_map() @@ -314,7 +314,7 @@ class V2VisionEngine: prompt = ( f"{self.library_criteria}\n" - "Respond ONLY with a JSON object: {\"results\": [{\"idx\": int, \"year\": \"YYYY\", \"stars\": 0-5, \"is_video\": bool, \"tag\": \"[TAG]\", \"summary\": \"1-2 sentences description\", \"language\": \"...\", \"type\": \"...\", \"level\": \"Fundamentals|Intermediate|Advanced|Architect\"}, ...]}\n\n" + "Respond ONLY with a JSON object: {\"results\": [{\"idx\": int, \"year\": \"YYYY\", \"stars\": 0-5, \"is_video\": bool, \"tag\": \"[TAG]\", \"summary\": \"...\", \"language\": \"...\", \"type\": \"...\", \"level\": \"...\", \"area\": \"...\", \"topic\": \"...\", \"subtopic\": \"...\", \"is_microservice\": bool}, ...]}\n\n" "LINKS:\n" + "\n".join([f"{idx}. {l['title']} ({l['url']}) - Desc: {l['description'][:60]}" for idx, l in enumerate(batch)]) ) @@ -330,7 +330,7 @@ class V2VisionEngine: norm_url = normalize_url(item["url"]) old_tag = self.inventory.get(norm_url, {}).get("tag") - # SPECIAL ASSET BYPASS: If file is special, force 5 stars or preservation + # SPECIAL ASSET BYPASS is_special = item["original_file"] in special_files eval_data = { @@ -341,7 +341,11 @@ class V2VisionEngine: "ai_summary": res.get("summary", ""), "language": res.get("language", "English"), "resource_type": res.get("type", "Reference"), - "complexity": res.get("level", "Intermediate") + "complexity": res.get("level", "Intermediate"), + "area": res.get("area", "General"), + "topic": res.get("topic", "Uncategorized"), + "subtopic": res.get("subtopic", ""), + "is_microservice": bool(res.get("is_microservice", False)) } item.update(eval_data) @@ -370,7 +374,9 @@ class V2VisionEngine: "is_video": item["is_video"], "ai_summary": item["ai_summary"], "language": item["language"], "resource_type": item["resource_type"], "complexity": item["complexity"], "tag": item["tag"], "status": "online", - "original_file": item["original_file"] + "original_file": item["original_file"], "area": item["area"], + "topic": item["topic"], "subtopic": item["subtopic"], + "is_microservice": item["is_microservice"] } if "gh_stars" in item: self.inventory[norm_url]["gh_stars"] = item["gh_stars"] if "gh_updated" in item: self.inventory[norm_url]["gh_updated"] = item["gh_updated"] @@ -435,8 +441,14 @@ class V2VisionEngine: return {} async def _rebuild_structure(self, library_inventory: List[Dict]) -> Dict[str, Dict]: - special_files = [sa["file"] for sa in self.special_assets_rules.get("special_assets", [])] + """ + REBUILD STRUCTURE: O'REILLY TECHNICAL BOOK ARCHITECTURE. + Implements a sophisticated hierarchy: Dimension -> Category -> Topic -> Subtopic. + Handles Special Asset rules and Microservices extraction. + """ + special_rules = {sa["file"]: sa for sa in self.special_assets_rules.get("special_assets", [])} v2_structure = {dim: {"summary": "", "categories": {}} for dim in self.dimensions.keys()} + file_to_dim = {} for dim, files in self.dimensions.items(): for f in files: file_to_dim[f + ".md"] = dim @@ -445,39 +457,62 @@ class V2VisionEngine: orig_file = item.get("original_file", "unknown.md") dim = file_to_dim.get(orig_file, "Architectural Foundations") cat_name = orig_file.replace(".md", "").replace("-", " ").title() - is_special = orig_file in special_files - # Filtering: Keep if stars >= 3 OR if it's a Special Asset - if not is_special and item.get("stars", 0) < 3: + # --- RULE: Microservices Extraction --- + if item.get("is_microservice"): + cat_name = "Microservices" + # If from introduction, move to Foundations dimension, else keep in current + if orig_file == "introduction.md": dim = "Architectural Foundations" + + # --- RULE: Special Asset Handling --- + rule = special_rules.get(orig_file, {}) + is_special = orig_file in special_rules + + # Elite Curation for Introduction + if orig_file == "introduction.md" and item.get("stars", 0) < 4 and not item.get("is_microservice"): + continue + + # General Filtering for V2 + if not is_special and item.get("stars", 0) < 3 and not item.get("is_microservice"): continue + # Hierarchy: Dimension -> Category -> Topic -> Subtopic if cat_name not in v2_structure[dim]["categories"]: - v2_structure[dim]["categories"][cat_name] = { - "Fundamentals": [], "Intermediate": [], "Advanced": [], "Architect": [] - } + v2_structure[dim]["categories"][cat_name] = {} - level = item.get("complexity", "Intermediate") - if level not in v2_structure[dim]["categories"][cat_name]: level = "Intermediate" - v2_structure[dim]["categories"][cat_name][level].append(item) + topic = item.get("topic", "General Concepts") + if topic not in v2_structure[dim]["categories"][cat_name]: + v2_structure[dim]["categories"][cat_name][topic] = {} + + subtopic = item.get("subtopic", "Other Resources") or "General" + if subtopic not in v2_structure[dim]["categories"][cat_name][topic]: + v2_structure[dim]["categories"][cat_name][topic][subtopic] = [] + + v2_structure[dim]["categories"][cat_name][topic][subtopic].append(item) + # Cleanup and Sorting for dim in v2_structure: for cat in list(v2_structure[dim]["categories"].keys()): - has_content = False - for level in v2_structure[dim]["categories"][cat]: - if v2_structure[dim]["categories"][cat][level]: - has_content = True - v2_structure[dim]["categories"][cat][level].sort( + for topic in v2_structure[dim]["categories"][cat]: + for subtopic in v2_structure[dim]["categories"][cat][topic]: + # Sort by Impact and Year + v2_structure[dim]["categories"][cat][topic][subtopic].sort( key=lambda x: (-x.get("stars", 1), -(int(x["year"]) if str(x.get("year", "")).isdigit() else 0)) ) - if not has_content: + + if not v2_structure[dim]["categories"][cat]: del v2_structure[dim]["categories"][cat] else: - # Maintain Executive Summary for Dimension - prompt = f"Write a professional 2026 executive summary for '{dim}'. Focus on high-density value. 1 sentence only." - try: - v2_structure[dim]["summary"] = await call_gemini_with_retry(prompt, response_format="text", prefer_flash=True) - except: - v2_structure[dim]["summary"] = f"Impact-driven reference library for {dim}." + # Update Executive Summary + cache_key = f"INTRO:{dim}" + if cache_key in self.inventory: + v2_structure[dim]["summary"] = self.inventory[cache_key].get("ai_summary") + else: + prompt = f"Write a professional 1-sentence executive summary for the '{dim}' technical dimension." + try: + v2_structure[dim]["summary"] = await call_gemini_with_retry(prompt, response_format="text", prefer_flash=True) + except: + v2_structure[dim]["summary"] = f"Strategic reference library for {dim}." return v2_structure @@ -588,81 +623,70 @@ class V2VisionEngine: slug = dim.lower().replace(" ", "-").replace("&", "and").replace("(", "").replace(")", "").replace(" ", "-") md = f"# {dim}\n\n" md += f"!!! info \"Architectural Context\"\n {content['summary']}\n\n" - + # --- Table of Contents for the Page --- md += "## Table of Contents\n" - for cat in content["categories"].keys(): + for cat, topics in content["categories"].items(): cat_slug = cat.lower().replace(" ", "-") md += f"- [{cat}](#{cat_slug})\n" - for level, level_links in content["categories"][cat].items(): - if level_links: - level_slug = f"{cat_slug}-{level.lower()}" - md += f" - [{level}](#{level_slug})\n" + for topic, subtopics in topics.items(): + topic_slug = f"{cat_slug}-{topic.lower().replace(' ', '-')}" + md += f" - [{topic}](#{topic_slug})\n" md += "\n---\n\n" - for cat, levels in content["categories"].items(): + for cat, topics in content["categories"].items(): cat_slug = cat.lower().replace(" ", "-") md += f"## {cat}\n\n" - for level, links in levels.items(): - if not links: continue - level_slug = f"{cat_slug}-{level.lower()}" - md += f"### {cat} - {level}\n" - for l in links: - year, stars_val = l.get("year", "N/A"), l.get("stars", 0) - stars = ("🌟" * stars_val) if stars_val > 0 else "" - tag = l.get("tag", "[ENTERPRISE-STABLE]") - - # Determine color mapping - if "STANDARD" in tag or "FOUNDATIONAL" in tag: color = "success" - elif "EMERGING" in tag: color = "warning" - elif "LEGACY" in tag: color = "critical" - elif "STABLE" in tag: color = "info" - else: color = "primary" - - title_clean = l['title'].replace("==", "") - if stars_val >= 3 or "STANDARD" in tag: - title_display = f"**=={title_clean}==**" - elif stars_val == 2: - title_display = f"**{title_clean}**" - else: - title_display = title_clean - - year_prefix = f"**({year})** " if year and year != "N/A" else "" - gh_info = f" ⭐ {l['gh_stars']}" if "gh_stars" in l else "" - icon = " 🎥" if l.get("is_video") else "" - - # Language Tagging - lang = l.get("language", "English") - lang_tag = "" - if lang.lower() != "english": - lang_tag = f" [{lang.upper()} CONTENT]" - - # Complexity Tagging - l_val = l.get("complexity", "Intermediate") - level_tag = "" - if l_val.lower() in ["architect", "advanced"]: - level_tag = f" [{l_val.upper()} LEVEL]" - - # Resource Type Tagging - res_type = l.get("resource_type", "Reference") - type_tag = "" - if res_type.lower() in ["case study", "guide", "documentation"]: - type_tag = f" [{res_type.upper()}]" - # Rich Metadata - rich_tags = "" - if l.get("author"): rich_tags += f" by **{l['author']}**" - if l.get("duration"): rich_tags += f" ⏱️ {l['duration']}" - if l.get("reading_time"): rich_tags += f" 📖 {l['reading_time']}" + for topic, subtopics in topics.items(): + topic_slug = f"{cat_slug}-{topic.lower().replace(' ', '-')}" + md += f"### {topic}\n\n" - md += f" - {year_prefix}[{title_display}]({l['url']}){icon}{gh_info}{lang_tag}{level_tag}{type_tag}{rich_tags} {stars} {tag}\n" - if l['description']: - desc = l['description'] - if "\n" in desc: - md += "\n" + "\n".join([" " + line for line in desc.split("\n")]) + "\n\n" - else: - md += f" {desc}\n" + for subtopic, links in subtopics.items(): + if subtopic and subtopic != "General": + md += f"#### {subtopic}\n" + + for l in links: + year, stars_val = l.get("year", "N/A"), l.get("stars", 0) + stars = ("🌟" * stars_val) if stars_val > 0 else "" + tag = l.get("tag", "[ENTERPRISE-STABLE]") + + # Color mapping + if "STANDARD" in tag or "FOUNDATIONAL" in tag: color = "success" + elif "EMERGING" in tag: color = "warning" + elif "LEGACY" in tag: color = "critical" + elif "STABLE" in tag: color = "info" + else: color = "primary" + + title_clean = l['title'].replace("==", "") + title_display = f"**=={title_clean}==**" if stars_val >= 3 else title_clean + + year_prefix = f"**({year})** " if year and year != "N/A" else "" + gh_info = f" ⭐ {l['gh_stars']}" if "gh_stars" in l else "" + icon = " 🎥" if l.get("is_video") else "" + + lang = l.get("language", "English") + lang_tag = f" [{lang.upper()} CONTENT]" if lang.lower() != "english" else "" + + complexity = l.get("complexity", "Intermediate") + level_tag = f" [{complexity.upper()} LEVEL]" if complexity.lower() in ["architect", "advanced"] else "" + + res_type = l.get("resource_type", "Reference") + type_tag = f" [{res_type.upper()}]" if res_type.lower() in ["case study", "guide", "documentation"] else "" + + # Rich Metadata + rich_tags = "" + if l.get("author"): rich_tags += f" by **{l['author']}**" + if l.get("duration"): rich_tags += f" ⏱️ {l['duration']}" + if l.get("reading_time"): rich_tags += f" 📖 {l['reading_time']}" + + md += f" - {year_prefix}[{title_display}]({l['url']}){icon}{gh_info}{lang_tag}{level_tag}{type_tag}{rich_tags} {stars} {tag}\n" + if l['description']: + desc = l['description'] + md += f"\n {desc}\n\n" + md += "\n" md += "\n" + with open(os.path.join(V2_DIR, f"{slug}.md"), "w") as f: f.write(md) async def _sync_enterprise_navigation(self, data: Dict[str, Dict]):