diff --git a/data/curation_sources.yaml b/data/curation_sources.yaml index 88923ca0..56310579 100644 --- a/data/curation_sources.yaml +++ b/data/curation_sources.yaml @@ -37,9 +37,7 @@ sources: - "google-antigravity" # Google Antigravity Agentic SDK Org - "GoogleDevs" # Google Developers (for SDK and API announcements) feeds: - - "https://www.anthropic.com/news/feed" - - "https://openai.com/news/feed" - - "https://blog.google/technology/developer/rss/" + - "https://openai.com/news/rss.xml" - "https://blog.google/technology/ai/rss/" - topic: "Developer Productivity & AI Agents" @@ -53,13 +51,13 @@ sources: feeds: - "https://github.blog/feed/" - "https://github.blog/category/engineering/feed/" - - "https://cursor.sh/blog/rss" - topic: "Data & Big Data" accounts: - "Databricks" - "ApacheSpark" - "snowflakedb" + - "UberEng" # Uber Engineering - topic: "Infrastructure as Code & GitOps" accounts: @@ -67,11 +65,9 @@ sources: - "PulumiCorp" - "ArgoProj" - "fluxcd" + - "Atlassian" # Atlassian Engineering / Developer Updates feeds: - "https://netflixtechblog.com/feed" - - "https://engineering.atlassian.com/feed" - "https://blog.cloudflare.com/rss/" - - "https://medium.com/feed/uber-engineering" - "https://engineering.fb.com/feed/" - "https://aws.amazon.com/blogs/aws/feed/" - - "https://cloud.google.com/blog/rss/" diff --git a/docs/uncategorized.md b/docs/uncategorized.md new file mode 100644 index 00000000..d20a7301 --- /dev/null +++ b/docs/uncategorized.md @@ -0,0 +1,5 @@ +# Uncategorized + +This section contains highly relevant links and resources that the AI curator has discovered, but which do not yet fit into any of the existing specific categories. + +If you find a link here that deserves a new category or belongs somewhere else, feel free to open a Pull Request! diff --git a/mkdocs.yml b/mkdocs.yml index 155c5ae7..921d930b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -290,6 +290,7 @@ nav: - Freelancing: freelancing.md - Remote Tech Jobs: remote-tech-jobs.md - Clients: customer.md + - Uncategorized: uncategorized.md - About: about.md copyright: 2026 Nubenetes, about. extra: diff --git a/src/agentic_curator.py b/src/agentic_curator.py index 3f482290..1de9dd40 100644 --- a/src/agentic_curator.py +++ b/src/agentic_curator.py @@ -15,12 +15,12 @@ from src.logger import log_event # Configuration V1_DIR = "docs" -def get_best_category_match(suggested: str) -> Optional[str]: - if not suggested: return None +def get_best_category_match(suggested: str) -> str: + if not suggested: return "uncategorized" suggested = suggested.lower().strip() for cat in NUBENETES_CATEGORIES: if suggested in cat or cat in suggested: return cat - return None + return "uncategorized" async def _get_github_activity(url: str) -> Dict: match = re.search(r'github\.com/([^/]+/[^/]+)', url) @@ -104,8 +104,7 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: from src.mandate_ingestor import get_system_mandates dynamic_mandates = get_system_mandates() - for i in range(0, len(to_evaluate), BATCH_SIZE): - batch = to_evaluate[i:i+BATCH_SIZE] + async def process_sub_batch(batch): batch_data = [] for asset in batch: web_content, rich_meta = await _deep_fetch_content(asset["url"]) @@ -130,6 +129,8 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: "- If the community (Reddit, Hacker News) reports the tool as 'unstable', 'abandoned', or 'vaporware', set reputation_penalty: true.\n" "PHASE 2: LINGUISTIC DIVERSITY & CLASSIFICATION\n" "- Calculate 'impact_score' (0-100) based on architectural value, innovation, and technical depth (>= 80 is required for inclusion).\n" + f"- Assign a 'primary_category' strictly from this list: {', '.join(NUBENETES_CATEGORIES)}\n" + "- If none fit well, use 'uncategorized' and propose a better one in 'suggested_new_category'.\n" "- Identify TECHNICAL_HIERARCHY: List (max 10 strings) Area > Topic > Subtopics.\n" "PHASE 3: HIGH-DENSITY TECHNICAL SUMMARIES (Mandate 4)\n" "- Provide an 'en_summary' that is technical, professional and dense.\n" @@ -137,7 +138,7 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: "- Format: Use paragraphs and bullet points if necessary. Aim for 2-5 sentences of depth.\n" "PHASE 4: MULTI-DIMENSIONAL TAGGING\n" "- Assign 1 to 3 tags from: [DE FACTO STANDARD], [ENTERPRISE-STABLE], [EMERGING], [GUIDE], [CASE STUDY], [COMMUNITY-TOOL], [LEGACY].\n" - "Respond ONLY JSON list: [{\"url\": \"...\", \"impact_score\": int, \"reputation_penalty\": bool, \"reputation_summary\": \"...\", \"pub_date\": \"YYYY-MM-DD\", \"primary_category\": \"...\", \"title\": \"...\", \"desc\": \"...\", \"en_summary\": \"High-density summary...\", \"language\": \"...\", \"type\": \"...\", \"level\": \"...\", \"technical_hierarchy\": [...], \"tags\": [...], \"is_microservice\": bool}, ...]\n\n" + "Respond ONLY JSON list: [{\"url\": \"...\", \"impact_score\": int, \"reputation_penalty\": bool, \"reputation_summary\": \"...\", \"pub_date\": \"YYYY-MM-DD\", \"primary_category\": \"...\", \"suggested_new_category\": \"...\", \"title\": \"...\", \"desc\": \"...\", \"en_summary\": \"High-density summary...\", \"language\": \"...\", \"type\": \"...\", \"level\": \"...\", \"technical_hierarchy\": [...], \"tags\": [...], \"is_microservice\": bool}, ...]\n\n" "RESOURCES:\n" + "\n".join([f"- {d['asset']['url']}: (MVQ Penalty: {d['mvq_penalty']}) {d['content']}" for d in batch_data]) ) @@ -167,6 +168,7 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: "reputation_summary": data.get("reputation_summary", ""), "source_provenance": d["asset"].get("source_type", "Social"), "social_preview_url": d["rich_meta"].get("og_image", ""), "category": primary_cat, "status": "online", "last_checked": datetime.now().timestamp(), + "suggested_new_category": data.get("suggested_new_category", ""), "addition_method": "automatic", **d["gh_meta"] } if "youtube.com" in url or "youtu.be" in url: @@ -180,6 +182,9 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: else: evaluations[url] = {"status": "FILTERED"} curator._save_inventory() except Exception as e: log_event(f" [!] Batch AI Error: {e}") + + sub_batches = [to_evaluate[i:i+BATCH_SIZE] for i in range(0, len(to_evaluate), BATCH_SIZE)] + await asyncio.gather(*(process_sub_batch(b) for b in sub_batches)) return evaluations diff --git a/src/config.py b/src/config.py index ca91b684..91c91df2 100644 --- a/src/config.py +++ b/src/config.py @@ -74,5 +74,5 @@ NUBENETES_CATEGORIES = [ 'qa', 'rancher', 'react', 'recruitment', 'registries', 'remote-tech-jobs', 'scaffolding', 'scaleway', 'securityascode', 'serverless', 'servicemesh', 'sonarqube', 'sre', 'stackstorm', 'swagger-code-generator-for-rest-apis', 'tekton', 'terraform', 'test-automation-frameworks', - 'testops', 'visual-studio', 'web-servers', 'web3', 'workfromhome', 'xamarin', 'yaml' + 'testops', 'uncategorized', 'visual-studio', 'web-servers', 'web3', 'workfromhome', 'xamarin', 'yaml' ]