diff --git a/.github/workflows/intelligent_link_cleaner.yml b/.github/workflows/intelligent_link_cleaner.yml index bbe8a8c5..7624a2cb 100644 --- a/.github/workflows/intelligent_link_cleaner.yml +++ b/.github/workflows/intelligent_link_cleaner.yml @@ -4,6 +4,11 @@ on: schedule: - cron: '0 0 1 * *' workflow_dispatch: + inputs: + force_full_check: + description: 'Force full re-validation (bypasses 21-day cache)' + type: boolean + default: false permissions: contents: write @@ -37,6 +42,7 @@ jobs: GEMINI_API_KEY_1: ${{ secrets.GEMINI_API_KEY_1 }} GEMINI_API_KEY_2: ${{ secrets.GEMINI_API_KEY_2 }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FORCE_FULL_CHECK: ${{ github.event.inputs.force_full_check || 'false' }} PYTHONPATH: ${{ github.workspace }} PYTHONUNBUFFERED: 1 run: | diff --git a/GEMINI.md b/GEMINI.md index af570696..a705a0a3 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -36,10 +36,12 @@ This file contains the accumulated instructions and long-term vision for the aut - **Flat Asset Routing**: To avoid depth-related path breakage, both V1 (`mkdocs.yml`) and V2 (`v2-mkdocs.yml`) MUST have `use_directory_urls: false`. This ensures relative paths (e.g., `images/img.png`) resolve correctly regardless of the page depth. 20. **V2 Navigation Design**: The V2 top navigation bar MUST maintain a flat structure. All dimensions and categories must be top-level tabs in `v2-mkdocs.yml` to ensure direct discoverability and avoid nested groupings like "Categories". 21. **V2 Impact-Driven Sorting**: The V2 portal MUST prioritize **relevance (Impact) over dates** within sections to provide high-density technical value. Sorting MUST follow: 1. Stars/Relevance (DESC), 2. Year (DESC). The mission statement and descriptions MUST reflect this impact-driven synthesis. -22. **Unified Metadata Database (Local Storage)**: All link metadata MUST be managed via the local YAML database in `data/`. +22. **Unified Metadata Database (Local Storage & Persistence)**: All link metadata MUST be managed via the local YAML database in `data/`. - **`inventory.yaml`**: The primary source of truth for years, stars (0-5), and descriptions. - **`structure_map.yaml`**: Tracks link locations and visual formatting (bold/highlight) across V1 and V2. - - **Persistence**: Every agent MUST load these files at startup and save any modifications immediately to ensure state continuity across workflows. + - **Persistence (MANDATORY)**: Every AI agent and workflow MUST include these YAML files in their Pull Requests if any change is detected. Discarding the database during a workflow run is a CRITICAL FAILURE. All workflows must load the DB, update it, and INJECT the modified YAML files into the final PR payload. + - **Exhaustive Initialization**: The system supports a `FORCE_FULL_CHECK` environment variable to bypass all caches (e.g., 21-day health cache) and force a full re-validation and re-enrichment of the entire 17k+ link archive. + - **No Trusted Bypassing**: All domains, including high-trust ones (GitHub, Google, AWS), MUST be verified for link validity. Trusted status only grants a lower priority for aggressive scraper rotation, not a bypass for existence checks. - **Manual Priority**: AI agents MUST NOT overwrite existing manual descriptions in the V1 archive files. Enrichment is strictly for `inventory.yaml` and the V2 portal. 23. **Canonical URL Normalization**: To prevent duplication and fragmented metadata, all agents MUST normalize URLs before any inventory operation. - **Tracking Stripping**: Systematically remove UTM parameters, social media trackers (X.com, LinkedIn), and URL fragments (`#`). diff --git a/src/intelligent_health_checker.py b/src/intelligent_health_checker.py index 12f8ebe8..10182c70 100644 --- a/src/intelligent_health_checker.py +++ b/src/intelligent_health_checker.py @@ -117,23 +117,19 @@ class IntelligentLinkCleaner: async def _check_url_with_retries(self, url: str, max_retries=5) -> Tuple[str, bool, Optional[str], str]: now = datetime.now().timestamp() + force_full = os.getenv("FORCE_FULL_CHECK", "false").lower() == "true" # 0. Policy Enforcement: No archive.org links allowed if "archive.org" in url.lower(): return url, False, None, "Archive.org link (Forbidden by policy)" - # 1. NOTE: V1 Exhaustiveness Mandate - # We fetch GitHub metadata for logging/metrics, but we DO NOT delete based on activity. - # Only definitively dead links are removed in V1. - if "github.com" in url: - gh_meta = await self._fetch_github_metadata(url) - # Metadata is stored in cache/logs but not used for deletion here. - - cache_entry = self.learning_data.get("link_cache", {}).get(url) - if cache_entry and cache_entry.get("status") == "ALIVE": - if now - cache_entry.get("last_checked", 0) < (21 * 24 * 3600): - self.detailed_stats["skipped_recent"] += 1 - return url, True, None, "Cached (Recent)" + # 1. Skip cache if FORCE_FULL_CHECK is active + if not force_full: + cache_entry = self.learning_data.get("link_cache", {}).get(url) + if cache_entry and cache_entry.get("status") == "ALIVE": + if now - cache_entry.get("last_checked", 0) < (21 * 24 * 3600): + self.detailed_stats["skipped_recent"] += 1 + return url, True, None, "Cached (Recent)" domain = url.split("//")[-1].split("/")[0] domain_info = self.learning_data.get("domains", {}).get(domain, {}) @@ -200,13 +196,32 @@ class IntelligentLinkCleaner: """ norm_url = normalize_url(url) if norm_url not in self.inventory: self.inventory[norm_url] = {} - - # If inventory already has a description, we are done - if self.inventory[norm_url].get("ai_summary"): return + + force_full = os.getenv("FORCE_FULL_CHECK", "false").lower() == "true" + + # If inventory already has a description, we are done UNLESS forcing full + if self.inventory[norm_url].get("ai_summary") and not force_full: + # Still fetch GH metadata if missing + if "github.com" in url and ("stars" not in self.inventory[norm_url] or force_full): + gh_meta = await self._fetch_github_metadata(url) + if gh_meta: + self.inventory[norm_url]["stars"] = min(5, gh_meta.get("stars", 0) // 100) # Simple rating + self.inventory[norm_url]["gh_stars"] = gh_meta.get("stars", 0) + self.inventory[norm_url]["last_commit"] = gh_meta.get("pushed_at", "") + return async with self.ai_semaphore: log_event(f" [✨] INVENTORY: Generating summary for {url} (V2 Only)") try: + # 1. Fetch GitHub metadata if applicable + if "github.com" in url: + gh_meta = await self._fetch_github_metadata(url) + if gh_meta: + self.inventory[norm_url]["stars"] = min(5, gh_meta.get("stars", 0) // 100) + self.inventory[norm_url]["gh_stars"] = gh_meta.get("stars", 0) + self.inventory[norm_url]["last_commit"] = gh_meta.get("pushed_at", "") + + # 2. AI Content Analysis from src.agentic_curator import _deep_fetch_content from src.gemini_utils import call_gemini_with_retry web_content = await _deep_fetch_content(url) @@ -222,7 +237,7 @@ class IntelligentLinkCleaner: if ai_data: res_desc = ai_data.get("desc", "").strip() self.inventory[norm_url]["ai_summary"] = res_desc - self.inventory[norm_url]["pub_date"] = ai_data.get("pub_date", "N/A") + self.inventory[norm_url]["pub_date"] = str(ai_data.get("pub_date", "N/A")) self.stats["enriched_descriptions"] += 1 log_event(f" [OK] Cached for V2: {res_desc[:50]}...") except Exception as e: @@ -237,9 +252,7 @@ class IntelligentLinkCleaner: "Connection": "keep-alive" } - # Immediate pass for high-trust domains to avoid over-cleaning - if any(trusted in url.lower() for trusted in ["github.com", "gitlab.com", "microsoft.com", "google.com", "aws.amazon.com"]): - return True, "Trusted Domain" + # NOTE: Trusted domains must also be checked to ensure the link isn't a 404. paywall_indicators = ["sign in", "create free account", "member-only story", "pÑgina de suscripción", "inicia sesión"] @@ -333,11 +346,13 @@ class IntelligentLinkCleaner: # 2. Enrichment Phase (Smart Batching for AI) log_event("[*] Starting Smart AI Enrichment (V2 Descriptions)...", section_break=True) to_enrich = [] + force_full = os.getenv("FORCE_FULL_CHECK", "false").lower() == "true" + for url in unique_urls: if url in self.dead_links: continue norm_url = normalize_url(url) if norm_url not in self.inventory: self.inventory[norm_url] = {} - if not self.inventory[norm_url].get("ai_summary"): + if not self.inventory[norm_url].get("ai_summary") or force_full: to_enrich.append(url) if to_enrich: @@ -427,12 +442,23 @@ class IntelligentLinkCleaner: track("Navigation", "created", "Orphan Audit", "Linked via Curator") self.detailed_stats["operation_types"]["orphans"] = orphans_linked + # 3. Ensure BBDD YAML Persistence: Include database files in the PR payload + self._save_inventory() + self._save_structure_map() + final_payload = {p: "".join([l for l in lines if l is not None]) for p, lines in file_updates.items()} + + # Load fresh YAML content for the PR + import yaml + with open(INVENTORY_PATH, "r") as f: final_payload[INVENTORY_PATH] = f.read() + with open(STRUCTURE_MAP_PATH, "r") as f: final_payload[STRUCTURE_MAP_PATH] = f.read() + if orphans_linked > 0: with open(getattr(self.curator, "index_path", "docs/index.md"), 'r') as f: final_payload[getattr(self.curator, "index_path", "docs/index.md")] = f.read() with open(getattr(self.curator, "mkdocs_path", "mkdocs.yml"), 'r') as f: final_payload[getattr(self.curator, "mkdocs_path", "mkdocs.yml")] = f.read() + if final_payload: self._create_pr(final_payload) def _create_pr(self, updates: Dict[str, str], report_content: str = None): diff --git a/src/main.py b/src/main.py index d359824e..a3522174 100644 --- a/src/main.py +++ b/src/main.py @@ -163,21 +163,18 @@ async def master_orchestrator(): # 2. Resilient Health Check (Identity Rotation) ua = user_agents[idx % len(user_agents)] headers = {"User-Agent": ua, "Referer": "https://www.google.com/"} - - # Trust high-value domains immediately - if any(trusted in expanded_url.lower() for trusted in ["github.com", "gitlab.com", "microsoft.com", "google.com"]): - asset["health"] = "trusted" - else: - try: - async with httpx.AsyncClient(headers=headers, follow_redirects=True, timeout=12, verify=False) as client: - resp = await client.get(expanded_url) - if resp.status_code == 404: - asset["health"] = "dead" # Definitively dead - else: - asset["health"] = "online" - except: - asset["health"] = "uncertain" # Preserving if not 404 - + + # NOTE: All domains must be checked to ensure the link isn't a 404. + try: + async with httpx.AsyncClient(headers=headers, follow_redirects=True, timeout=12, verify=False) as client: + resp = await client.get(expanded_url) + if resp.status_code == 404: + asset["health"] = "dead" # Definitively dead + else: + asset["health"] = "online" + except: + asset["health"] = "timeout" # Assume alive but unreachable for now + # 3. GitHub Metadata Enrichment if "github.com" in expanded_url: match = re.search(r'github\.com/([^/]+)/([^/]+)', expanded_url) @@ -357,6 +354,13 @@ async def master_orchestrator(): "x_audit": x_audit_trail } try: + # --- BBDD Persistence: Include YAML database files in the PR --- + from src.config import INVENTORY_PATH, STRUCTURE_MAP_PATH + if os.path.exists(INVENTORY_PATH): + with open(INVENTORY_PATH, 'r') as f: modified_files_content[INVENTORY_PATH] = f.read() + if os.path.exists(STRUCTURE_MAP_PATH): + with open(STRUCTURE_MAP_PATH, 'r') as f: modified_files_content[STRUCTURE_MAP_PATH] = f.read() + pr_url = git_controller.apply_multi_file_changes(modified_files_content, metrics) if pr_url: print(f"PULL_REQUEST_URL: {pr_url}") diff --git a/src/v2_optimizer.py b/src/v2_optimizer.py index eff44a60..3f527de0 100644 --- a/src/v2_optimizer.py +++ b/src/v2_optimizer.py @@ -47,9 +47,9 @@ class V2VisionEngine: " * 0 stars: Good technical resource (Baseline).\n" " * 1 star (🌟): High-quality technical guide or tool.\n" " * 2 stars (🌟🌟): Exceptional, enterprise-grade resource.\n" - " * 3 stars (🌟🌟🌟): Elite Gem. Recommended for all architects. + " * 3 stars (🌟🌟🌟): Elite Gem. Recommended for all architects.\n" " * 4 stars (🌟🌟🌟🌟): Masterclass content or Essential Industry Tool.\n" - " * 5 stars (🌟🌟🌟🌟🌟): Legendary Resource (e.g., K8s Official Docs, Foundations like Prometheus/Envoy)."\n" + " * 5 stars (🌟🌟🌟🌟🌟): Legendary Resource (e.g., K8s Official Docs, Foundations like Prometheus/Envoy).\n" "- Assign a MATURITY TAG based on content type/status.\n" "PHASE 3: MANDATORY DESCRIPTIONS (V1 PRIORITY)\n" "- If 'Current Desc' is already provided and descriptive, DO NOT CHANGE IT.\n" @@ -193,10 +193,7 @@ class V2VisionEngine: async def _check_single_link_resilient(self, client, link: Dict, ua: str, attempts: int = 3) -> Dict: url = link["url"] - # 1. Immediate Pass for Trusted / Logic-Enriched Domains - if "github.com" in url or "awesome" in link["title"].lower(): - link["health_status"] = "trusted" - return link + # NOTE: All domains must be checked for validity. # 2. Cached Health if url in self.inventory and self.inventory[normalize_url(url)].get("status") == "online": @@ -329,11 +326,12 @@ class V2VisionEngine: await asyncio.sleep(0.3) return refined - def _calculate_tag(self, item: Dict) -> str: + def _calculate_tag(self, item: Dict) -> str: # Dynamic Evolutionary Tagging (Automatic Project Growth Detection) url = item.get("url", "").lower() stars = item.get("gh_stars", 0) - year = int(item.get("year")) if item.get("year", "").isdigit() else 2024 + year_str = str(item.get("year", "2024")) + year = int(year_str) if year_str.isdigit() else 2024 if "github.com" in url or "gitlab.com" in url: if stars > 15000: return "[DE FACTO STANDARD]" @@ -348,23 +346,14 @@ class V2VisionEngine: if "guide" in title or "architecture" in title: return "[ARCHITECTURE-GUIDE]" if "deep dive" in title or "internals" in title: return "[TECHNICAL-DEEP-DIVE]" if "how to" in title or "tutorial" in title: return "[CASE-STUDY]" - return "[EXPERT-ARTICLE]" - if year >= 2025: return "[EMERGING / INNOVATION]" - if year <= 2022: return "[LEGACY / MAINTENANCE]" - return "[TOOLING]" - # Fallback to AI's tag or defaults for articles + # Fallback to AI's tag or defaults tag = item.get("tag", "").upper() valid_tags = ["[DE FACTO STANDARD]", "[ENTERPRISE-STABLE]", "[EMERGING / INNOVATION]", "[LEGACY / MAINTENANCE]", "[ARCHITECTURE-GUIDE]", "[TOOLING]", "[CASE-STUDY]", "[CHEATSHEET]"] if tag in valid_tags: return tag - - # Basic inference for articles - title = item.get("title", "").lower() - if "awesome" in title: return "[FOUNDATIONAL]" - if "guide" in title or "architecture" in title: return "[ARCHITECTURE-GUIDE]" - if "how to" in title or "tutorial" in title: return "[CASE-STUDY]" - return "[ENTERPRISE-STABLE]" + + return "[EXPERT-ARTICLE]" async def _fetch_github_metadata(self, url: str) -> Dict: match = re.search(r'github\.com/([^/]+)/([^/]+)', url)