diff --git a/.github/workflows/agentic_cron.yml b/.github/workflows/agentic_cron.yml new file mode 100644 index 00000000..f6ca8452 --- /dev/null +++ b/.github/workflows/agentic_cron.yml @@ -0,0 +1,35 @@ +name: Nubenetes Automated Agentic Curation + +on: + schedule: + - cron: '0 5 * * 0' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + agentic-curation-process: + runs-on: ubuntu-latest + steps: + - name: Sincronización del repositorio + uses: actions/checkout@v4 + - name: Provisión del Entorno Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + - name: Instalación del Árbol de Dependencias + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Ejecución de la Canalización Agéntica Integral + env: + TWITTER_USERNAME: ${{ secrets.TWITTER_USERNAME }} + TWITTER_EMAIL: ${{ secrets.TWITTER_EMAIL }} + TWITTER_PASSWORD: ${{ secrets.TWITTER_PASSWORD }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python src/main.py diff --git a/.gitignore b/.gitignore index dfcfd56f..dbe3436a 100644 --- a/.gitignore +++ b/.gitignore @@ -348,3 +348,8 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ +# Automatización Nubenetes +src/__pycache__/ +*.json +.env +nubenetes_agent_env/ diff --git a/requirements.txt b/requirements.txt index 051e8612..55833426 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ -mkdocs -mkdocs-material -pymdown-extensions -mkdocs-codeinclude-plugin \ No newline at end of file +twikit==1.7.6 +pydantic-ai +google-generativeai +PyGithub +aiohttp +beautifulsoup4 +pytz +python-dotenv diff --git a/src/agentic_curator.py b/src/agentic_curator.py new file mode 100644 index 00000000..5557662d --- /dev/null +++ b/src/agentic_curator.py @@ -0,0 +1,39 @@ +from pydantic import BaseModel, Field +from pydantic_ai import Agent +from typing import List, Optional +from src.config import NUBENETES_CATEGORIES + +class LinkEvaluationResult(BaseModel): + is_exceptional_value: bool = Field(description="¿Es un recurso avanzado o disruptivo?") + category_assignment: Optional[str] = Field(description="Categoría asignada.", enum=NUBENETES_CATEGORIES) + canonical_title: str = Field(description="Título formal.") + technical_description: str = Field(description="Descripción técnica corta.") + evaluation_rationale: str = Field(description="Razonamiento de la decisión.") + +curation_agent = Agent( + 'google-gla:gemini-2.0-flash-exp', + result_type=LinkEvaluationResult, + system_prompt=( + "Actúas como el Ingeniero Curador Principal para 'nubenetes/awesome-kubernetes'. " + "Descarta tutoriales genéricos. Privilegia automatización, GitOps, Service Meshes y operadores. " + "Usa una categoría existente. Redacta descripciones asépticas y técnicas." + ) +) + +async def evaluate_extracted_assets(raw_assets: list[dict]) -> list[dict]: + curated_assets = [] + for asset in raw_assets: + cognitive_prompt = f"Evalúa este candidato:\nURL: {asset['url']}\nContexto: {asset['context']}" + try: + response = await curation_agent.run(cognitive_prompt) + evaluation = response.data + if evaluation.is_exceptional_value and evaluation.category_assignment: + curated_assets.append({ + "url": asset["url"], + "title": evaluation.canonical_title, + "description": evaluation.technical_description, + "category": evaluation.category_assignment + }) + except Exception as e: + print(f"Error evaluando {asset['url']}: {str(e)}") + return curated_assets diff --git a/src/autonomous_discovery.py b/src/autonomous_discovery.py new file mode 100644 index 00000000..a93fbf39 --- /dev/null +++ b/src/autonomous_discovery.py @@ -0,0 +1,42 @@ +from pydantic_ai import Agent +from pydantic import BaseModel, Field +import aiohttp +from src.config import NUBENETES_CATEGORIES + +class DiscoveredResource(BaseModel): + title: str + url: str + description: str + category: str = Field(enum=NUBENETES_CATEGORIES) + +class DiscoveryReport(BaseModel): + new_high_value_resources: list[DiscoveredResource] + +async def fetch_github_trending_k8s() -> str: + url = "https://api.github.com/search/repositories?q=topic:kubernetes+stars:>500&sort=stars&order=desc" + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status == 200: + data = await resp.json() + summaries = [f"{repo['name']} - {repo['html_url']} - {repo['description']}" for repo in data.get('items', [])[:10]] + return "\n".join(summaries) + return "No se pudieron obtener resultados." + +explorer_agent = Agent( + 'google-gla:gemini-2.0-flash-exp', + result_type=DiscoveryReport, + system_prompt=( + "Descubre las 3 herramientas de Kubernetes más populares y recientes. " + "Usa la herramienta 'fetch_github_trending_k8s'. " + "Solo herramientas revolucionarias, categorizadas estrictamente." + ) +) +explorer_agent.tool(fetch_github_trending_k8s) + +async def discover_trending_assets() -> list[dict]: + try: + response = await explorer_agent.run("Busca herramientas revolucionarias en el ecosistema.") + return [dict(res) for res in response.data.new_high_value_resources] + except Exception as e: + print(f"Error en descubrimiento: {str(e)}") + return [] diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..d4bb5f33 --- /dev/null +++ b/src/config.py @@ -0,0 +1,18 @@ +import os +import pytz +from dotenv import load_dotenv + +load_dotenv() + +MADRID_TZ = pytz.timezone("Europe/Madrid") + +TWITTER_USERNAME = os.getenv("TWITTER_USERNAME") +TWITTER_EMAIL = os.getenv("TWITTER_EMAIL") +TWITTER_PASSWORD = os.getenv("TWITTER_PASSWORD") +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY") +GH_TOKEN = os.getenv("GH_TOKEN") + +TARGET_REPO = "nubenetes/awesome-kubernetes" +MAIN_DOC_FILE = "docs/kubernetes.md" + +NUBENETES_CATEGORIES = ['ChromeDevTools', 'GoogleCloudPlatform', 'about', 'ai', 'angular', 'ansible', 'api', 'appointment-scheduling', 'argo', 'aws-architecture', 'aws-backup', 'aws-containers', 'aws-data', 'aws-databases', 'aws-devops', 'aws-iac', 'aws-messaging', 'aws-miscellaneous', 'aws-monitoring', 'aws-networking', 'aws-newfeatures', 'aws-pricing', 'aws-security', 'aws-serverless', 'aws-spain', 'aws-storage', 'aws-tools-scripts', 'aws-training', 'aws', 'azure', 'caching', 'chaos-engineering', 'chatgpt', 'cheatsheets', 'chef', 'cicd-kubernetes-plugins', 'cicd', 'cloud-arch-diagrams', 'cloud-asset-inventory', 'cloudflare', 'container-managers', 'crossplane', 'crunchydata', 'customer', 'databases', 'demos', 'devel-sites', 'developerportals', 'devops-tools', 'devops', 'devsecops', 'digital-money', 'digitalocean', 'docker', 'dom', 'dotnet', 'edge-computing', 'elearning', 'embedded-servlet-containers', 'faq', 'finops', 'flux', 'freelancing', 'git', 'gitops', 'golang', 'grafana', 'helm', 'hr', 'iac', 'ibm_cloud', 'index', 'interview-questions', 'introduction', 'istio', 'java-and-java-performance-optimization', 'java_app_servers', 'java_frameworks', 'javascript', 'jenkins-alternatives', 'jenkins', 'jvm-parameters-matrix-table', 'keptn', 'kubectl-commands', 'kubernetes-alternatives', 'kubernetes-autoscaling', 'kubernetes-backup-migrations', 'kubernetes-based-devel', 'kubernetes-bigdata', 'kubernetes-client-libraries', 'kubernetes-monitoring', 'kubernetes-networking', 'kubernetes-newsletters', 'kubernetes-on-premise', 'kubernetes-operators-controllers', 'kubernetes-releases', 'kubernetes-security', 'kubernetes-storage', 'kubernetes-tools', 'kubernetes-troubleshooting', 'kubernetes-tutorials', 'kubernetes', 'kustomize', 'linux-dev-env', 'linux', 'liquibase', 'lowcode-nocode', 'managed-kubernetes-in-public-cloud', 'matrix-table', 'maven-gradle', 'message-queue', 'mkdocs', 'mlops', 'monitoring', 'networking', 'newsfeeds', 'newsql', 'noops', 'nosql', 'oauth', 'ocp3', 'ocp4', 'openshift-pipelines', 'openshift', 'oraclecloud', 'other-awesome-lists', 'performance-testing-with-jenkins-and-jmeter', 'postman', 'private-cloud-solutions', 'project-management-methodology', 'project-management-tools', 'prometheus', 'public-cloud-solutions', 'pulumi', 'python', 'qa', 'rancher', 'react', 'recruitment', 'registries', 'remote-tech-jobs', 'scaffolding', 'scaleway', 'securityascode', 'serverless', 'servicemesh', 'sonarqube', 'sre', 'stackstorm', 'swagger-code-generator-for-rest-apis', 'tekton', 'terraform', 'test-automation-frameworks', 'testops', 'visual-studio', 'web-servers', 'web3', 'workfromhome', 'xamarin', 'yaml'] diff --git a/src/gitops_manager.py b/src/gitops_manager.py new file mode 100644 index 00000000..ecdf1fe6 --- /dev/null +++ b/src/gitops_manager.py @@ -0,0 +1,29 @@ +from github import Github +from datetime import datetime + +class RepositoryController: + def __init__(self, access_token: str, repository_identifier: str): + self.github_client = Github(access_token) + self.repository = self.github_client.get_repo(repository_identifier) + self.default_branch_name = self.repository.default_branch + + def _create_feature_branch(self, branch_name: str) -> None: + base_branch = self.repository.get_branch(self.default_branch_name) + self.repository.create_git_ref(ref=f"refs/heads/{branch_name}", sha=base_branch.commit.sha) + + def apply_state_changes(self, target_file_path: str, new_document_state: str, metrics: dict) -> None: + timestamp_slug = datetime.now().strftime("%Y%m%d-%H%M") + branch_name = f"bot/agentic-curation-{timestamp_slug}" + self._create_feature_branch(branch_name) + file_meta = self.repository.get_contents(target_file_path, ref=self.default_branch_name) + commit_signature = f"chore(docs): automatización agéntica de curaduría [{timestamp_slug}]" + self.repository.update_file(path=target_file_path, message=commit_signature, content=new_document_state, sha=file_meta.sha, branch=branch_name) + pr_narrative = ( + "## 🤖 Ejecución del Agente Curador de Nubenetes\n\n" + "Este Pull Request ha sido ensamblado de manera autónoma.\n\n" + "### Resumen:\n" + f"- Nuevos Enlaces (Redes): {metrics.get('social_injections', 0)}\n" + f"- Nuevos Enlaces (Descubrimiento): {metrics.get('autonomous_injections', 0)}\n" + "- Purga de enlaces caídos y duplicados completada." + ) + self.repository.create_pull(title=f"Curation: {datetime.now().strftime('%d %b %Y')}", body=pr_narrative, head=branch_name, base=self.default_branch_name) diff --git a/src/ingestion_twikit.py b/src/ingestion_twikit.py new file mode 100644 index 00000000..7eea9cdd --- /dev/null +++ b/src/ingestion_twikit.py @@ -0,0 +1,65 @@ +import asyncio +import os +import re +from datetime import datetime +from twikit import Client +from src.config import MADRID_TZ, TWITTER_USERNAME, TWITTER_EMAIL, TWITTER_PASSWORD + +class SocialDataExtractor: + def __init__(self, target_account: str = "nubenetes"): + self.client = Client('en-US') + self.target_account = target_account + self.cookies_file = 'cookies.json' + + async def _authenticate(self): + if os.path.exists(self.cookies_file): + self.client.load_cookies(self.cookies_file) + else: + await self.client.login( + auth_info_1=TWITTER_USERNAME, + auth_info_2=TWITTER_EMAIL, + password=TWITTER_PASSWORD + ) + self.client.save_cookies(self.cookies_file) + + def _extract_urls_from_text(self, text: str) -> list[str]: + url_pattern = re.compile(r'https?://[^\s<>\"]+|www\.[^\s<>\"]+') + return url_pattern.findall(text) + + async def fetch_links_since(self, since_date: datetime) -> list[dict]: + await self._authenticate() + try: + user = await self.client.get_user_by_screen_name(self.target_account) + except Exception as e: + raise RuntimeError(f"Error resolviendo usuario {self.target_account}: {str(e)}") + + extracted_data = [] + tweets = await self.client.get_user_tweets(user.id, 'Tweets') + + fetching = True + while fetching and tweets: + for tweet in tweets: + tweet_date = tweet.created_at_datetime.astimezone(MADRID_TZ) + if tweet_date < since_date: + fetching = False + break + + full_content = tweet.full_text if hasattr(tweet, 'full_text') else tweet.text + urls = self._extract_urls_from_text(full_content) + + for url in urls: + if "x.com" not in url and "twitter.com" not in url: + extracted_data.append({ + "url": url, + "context": full_content, + "timestamp": tweet_date.isoformat() + }) + + if fetching: + try: + await asyncio.sleep(2) + tweets = await tweets.next() + except Exception: + break + + return extracted_data diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..a7737f43 --- /dev/null +++ b/src/main.py @@ -0,0 +1,32 @@ +import asyncio +from datetime import datetime, timedelta +from src.config import TARGET_REPO, MAIN_DOC_FILE, MADRID_TZ, GH_TOKEN +from src.ingestion_twikit import SocialDataExtractor +from src.markdown_ast import MarkdownSanitizer +from src.agentic_curator import evaluate_extracted_assets +from src.autonomous_discovery import discover_trending_assets +from src.gitops_manager import RepositoryController + +async def master_orchestrator(): + time_horizon = datetime(2026, 4, 25, 11, 0, tzinfo=MADRID_TZ) + twitter_client = SocialDataExtractor() + raw_social_links = await twitter_client.fetch_links_since(time_horizon) + autonomous_links = await discover_trending_assets() + curated_social_links = await evaluate_extracted_assets(raw_social_links) + total_new_assets = curated_social_links + autonomous_links + git_controller = RepositoryController(GH_TOKEN, TARGET_REPO) + markdown_sanitizer = MarkdownSanitizer() + repo_file_data = git_controller.repository.get_contents(MAIN_DOC_FILE) + document_state = repo_file_data.decoded_content.decode("utf-8") + purified_document_state = await markdown_sanitizer.sanitize_document(document_state) + final_document_state = purified_document_state + for asset in total_new_assets: + final_document_state = markdown_sanitizer.inject_curated_link(final_document_state, asset["category"], asset["title"], asset["url"], asset["description"]) + if final_document_state != document_state: + metrics = {"social_injections": len(curated_social_links), "autonomous_injections": len(autonomous_links)} + git_controller.apply_state_changes(MAIN_DOC_FILE, final_document_state, metrics) + else: + print("Sin cambios detectados.") + +if __name__ == "__main__": + asyncio.run(master_orchestrator()) diff --git a/src/markdown_ast.py b/src/markdown_ast.py new file mode 100644 index 00000000..e3e3618b --- /dev/null +++ b/src/markdown_ast.py @@ -0,0 +1,70 @@ +import re +import aiohttp +import asyncio + +class MarkdownSanitizer: + def __init__(self): + self.link_pattern = re.compile(r'\[([^\]]+)\]\((https?://[^\)]+)\)') + + async def _verify_link_health(self, session: aiohttp.ClientSession, url: str) -> bool: + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'} + try: + async with session.head(url, timeout=15, allow_redirects=True, headers=headers) as response: + if response.status < 400: + return True + if response.status in [404, 405]: + async with session.get(url, timeout=15, headers=headers) as get_resp: + return get_resp.status < 400 + except: + return False + return False + + async def sanitize_document(self, markdown_content: str) -> str: + all_links = self.link_pattern.findall(markdown_content) + unique_url_registry = set() + duplicates_flagged = set() + unique_link_pairs = [] + + for text, url in all_links: + clean_url = url.split('#')[0].rstrip('/') + if clean_url in unique_url_registry: + duplicates_flagged.add((text, url)) + else: + unique_url_registry.add(clean_url) + unique_link_pairs.append((text, url)) + + healthy_urls = set() + connector = aiohttp.TCPConnector(limit=50) + async with aiohttp.ClientSession(connector=connector) as session: + tasks = [self._verify_link_health(session, url) for _, url in unique_link_pairs] + health_results = await asyncio.gather(*tasks) + + for (text, url), is_healthy in zip(unique_link_pairs, health_results): + if is_healthy: + healthy_urls.add(url.split('#')[0].rstrip('/')) + + reconstructed_lines = [] + for line in markdown_content.splitlines(): + links_in_line = self.link_pattern.findall(line) + should_retain_line = True + for txt, uri in links_in_line: + clean_uri = uri.split('#')[0].rstrip('/') + if (txt, uri) in duplicates_flagged or clean_uri not in healthy_urls: + should_retain_line = False + if (txt, uri) in duplicates_flagged: + duplicates_flagged.remove((txt, uri)) + break + if should_retain_line: + reconstructed_lines.append(line) + return "\n".join(reconstructed_lines) + + def inject_curated_link(self, markdown_text: str, category: str, title: str, url: str, description: str) -> str: + new_entry = f" - [{title}]({url}) - {description}" + lines = markdown_text.splitlines() + for index, line in enumerate(lines): + if category.lower() in line.lower() and (line.startswith("#") or line.startswith("-")): + lines.insert(index + 1, new_entry) + return "\n".join(lines) + lines.append(f"\n### {category}") + lines.append(new_entry) + return "\n".join(lines)