From 8030dc6f5411cd3475173e2ce6d4798c6cc586ea Mon Sep 17 00:00:00 2001 From: Nubenetes Bot Date: Sun, 10 May 2026 23:13:53 +0200 Subject: [PATCH] feat: implement chronological extraction, ignore pinned posts, and add 'Post Date' column to PR report --- src/gitops_manager.py | 14 ++-- src/ingestion_twikit.py | 175 ++++++++++++++++++++-------------------- src/main.py | 25 +++--- 3 files changed, 108 insertions(+), 106 deletions(-) diff --git a/src/gitops_manager.py b/src/gitops_manager.py index c34b7eb3..0414306c 100644 --- a/src/gitops_manager.py +++ b/src/gitops_manager.py @@ -17,7 +17,6 @@ class RepositoryController: branch_name = f"bot/knowledge-update-{timestamp_slug}" self._create_feature_branch(branch_name) - # Garantizar que siempre haya al menos un commit (técnico) para abrir la PR if not updates: updates["src/memory/last_audit_run.json"] = json.dumps({ "timestamp": metrics.get("end_date"), @@ -46,15 +45,18 @@ class RepositoryController: # --- CONSTRUCCIÓN DEL REPORTE --- full_report = metrics.get('full_report', []) - # 1. Tabla Matricial + # 1. Tabla Matricial con Columna de Fecha matrix_table = "### 📋 Matriz de Auditoría de Enlaces (Full Extraction)\n" - matrix_table += "| Estado | Origen | Motivo | Categoría | URL |\n| :--- | :--- | :--- | :--- | :--- |\n" + matrix_table += "| Estado | Fecha Post | Origen | Motivo | Categoría | URL |\n| :--- | :--- | :--- | :--- | :--- | :--- |\n" counts = {"INCLUDED": 0, "DUPLICATE": 0, "FILTERED": 0} source_counts = {} for item in full_report[:200]: status_emoji = {"INCLUDED": "✅", "DUPLICATE": "👯", "FILTERED": "🛡️"}.get(item['status'], "❓") - matrix_table += f"| {status_emoji} {item['status']} | {item.get('source', 'N/A')} | {item['reason']} | `{item['category']}` | {item['url']} |\n" + # Formatear fecha para legibilidad + date_str = item.get('post_date', 'N/A')[:10] if item.get('post_date') else 'N/A' + + matrix_table += f"| {status_emoji} {item['status']} | {date_str} | {item.get('source', 'N/A')} | {item['reason']} | `{item['category']}` | {item['url']} |\n" counts[item['status']] = counts.get(item['status'], 0) + 1 if item['status'] == "INCLUDED": src = item.get('source', 'Unknown') @@ -81,14 +83,14 @@ class RepositoryController: pr_narrative = ( f"## 💎 Knowledge Update War Room: Kubernetes & Cloud Native\n\n" f"Este reporte detalla el procesamiento de **{metrics.get('total_extracted', 0)}** enlaces detectados.\n\n" - f"**Ventana Temporal:** `{metrics.get('start_date')}` ➔ `{metrics.get('end_date')}`\n\n" + f"**Rango de Posts Analizados:** `{metrics.get('start_date')[:10]}` ➔ `{metrics.get('end_date')[:10]}` (Cronología: Antiguos a Recientes)\n\n" f"{mermaid_pie}\n" f"{mermaid_origin}\n" f"{x_log}\n" f"{matrix_table}\n" f"---\n" f"**Nota de Evaluación:** Este PR incluye {len(metrics.get('added_list', []))} novedades reales. " - f"Ventana temporal automática basada en el histórico de merges." + f"Se ha ignorado el post fijo (Pinned) para evitar duplicidad cíclica." ) self.repository.create_pull( diff --git a/src/ingestion_twikit.py b/src/ingestion_twikit.py index 2a5983d3..94e12a9c 100644 --- a/src/ingestion_twikit.py +++ b/src/ingestion_twikit.py @@ -28,7 +28,6 @@ class SocialDataExtractor: def _extract_urls_from_text(self, text: str) -> list[str]: urls = re.findall(r'https?://[^\s<>\"]+|www\.[^\s<>\"]+', text) - # Filtro de ruido masivo y dominios de marketing/tracking noise_domains = [ "x.com", "twitter.com", "t.co", "abs.twimg", "pbs.twimg", "notoriete-web.com", "google-analytics", "doubleclick", @@ -42,7 +41,6 @@ class SocialDataExtractor: return list(set(valid_urls)) async def _fetch_via_playwright(self, since_date: datetime) -> list[dict]: - """Estrategia de Navegador Real con reintentos y control de tiempo.""" try: from playwright.async_api import async_playwright import playwright_stealth @@ -50,114 +48,119 @@ class SocialDataExtractor: self.log_audit("Playwright", False, "Librerías no disponibles.") return [] - self.log_audit("Playwright Browser", None, f"Buscando posts desde {since_date.date()}...") + self.log_audit("Playwright Browser", None, f"Cronología: Desde {since_date.date()} hasta hoy.") results = [] - for attempt in range(2): # 2 Intentos - try: - async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) - context = await browser.new_context(user_agent=self.user_agents[0], locale="es-ES") - page = await context.new_page() - + try: + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context(user_agent=self.user_agents[0], locale="es-ES") + page = await context.new_page() + + try: + if hasattr(playwright_stealth, 'stealth_async'): await playwright_stealth.stealth_async(page) + elif hasattr(playwright_stealth, 'stealth'): playwright_stealth.stealth(page) + except: pass + + env_cookies = os.getenv("TWITTER_COOKIES") + if env_cookies: try: - if hasattr(playwright_stealth, 'stealth_async'): await playwright_stealth.stealth_async(page) - elif hasattr(playwright_stealth, 'stealth'): playwright_stealth.stealth(page) + cookies = json.loads(env_cookies) + formatted = [] + for c in cookies: + if isinstance(c, dict) and 'name' in c and 'value' in c: + c['domain'] = c.get('domain', '.x.com') + for k in ['sameSite', 'storeId', 'id']: c.pop(k, None) + formatted.append(c) + await context.add_cookies(formatted) except: pass - - env_cookies = os.getenv("TWITTER_COOKIES") - if env_cookies: - try: - cookies = json.loads(env_cookies) - formatted = [] - for c in cookies: - if isinstance(c, dict) and 'name' in c and 'value' in c: - c['domain'] = c.get('domain', '.x.com') - for k in ['sameSite', 'storeId', 'id']: c.pop(k, None) - formatted.append(c) - await context.add_cookies(formatted) - except: pass - # Aumentar timeout y usar estrategia de carga más relajada - await page.goto(f"https://x.com/{self.target_account}", wait_until="domcontentloaded", timeout=90000) - await asyncio.sleep(15) # Tiempo extra para que X renderice el feed + await page.goto(f"https://x.com/{self.target_account}", wait_until="domcontentloaded", timeout=90000) + await asyncio.sleep(15) + + stop_scrolling = False + scroll_count = 0 + max_scrolls = 25 + collected_tweets = {} # URL -> tweet_data para evitar duplicados en scroll + + while not stop_scrolling and scroll_count < max_scrolls: + articles = await page.query_selector_all('article[data-testid="tweet"]') - stop_scrolling = False - scroll_count = 0 - max_scrolls = 20 # Suficiente para llegar a Oct 2024 si no hay miles de tweets - - while not stop_scrolling and scroll_count < max_scrolls: - articles = await page.query_selector_all('article[data-testid="tweet"]') - new_links_found = False + for article in articles: + # 1. Ignorar Pinned Posts (Post Fijo) + social_context = await article.query_selector('[data-testid="socialContext"]') + if social_context: + sc_text = await social_context.inner_text() + if "Fijado" in sc_text or "Pinned" in sc_text: + continue + + # 2. Extraer Fecha + time_el = await article.query_selector('time') + if not time_el: continue - for article in articles: - # Intentar extraer fecha del tweet - time_el = await article.query_selector('time') - if time_el: - datetime_str = await time_el.get_attribute('datetime') - tweet_dt = datetime.fromisoformat(datetime_str.replace('Z', '+00:00')).astimezone(MADRID_TZ) - if tweet_dt < since_date: - stop_scrolling = True - break - - text_el = await article.query_selector('[data-testid="tweetText"]') - tweet_text = await text_el.inner_text() if text_el else "" - - links = await article.query_selector_all('a') - found_in_tweet = [] - for link in links: - href = await link.get_attribute('href') - if href and href.startswith('http') and all(x not in href for x in ["x.com", "twitter.com", "t.co", "abs.twimg", "pbs.twimg"]): - found_in_tweet.append(href) - - found_in_tweet.extend(self._extract_urls_from_text(tweet_text)) - - for u in set(found_in_tweet): - if all(x not in u for x in ["x.com", "twitter.com", "t.co", "archive.org", "help.twitter"]): - results.append({ - "url": u, "context": tweet_text[:200], - "timestamp": tweet_dt.isoformat() if time_el else datetime.now(MADRID_TZ).isoformat(), - "source_type": "X.com (@nubenetes)" - }) - new_links_found = True + datetime_str = await time_el.get_attribute('datetime') + tweet_dt = datetime.fromisoformat(datetime_str.replace('Z', '+00:00')).astimezone(MADRID_TZ) + + if tweet_dt < since_date: + stop_scrolling = True + break - await page.evaluate("window.scrollBy(0, 2000)") - await asyncio.sleep(6) - scroll_count += 1 - if not new_links_found and scroll_count > 5: break # Si no hay nada nuevo en 5 scrolls, parar + # 3. Extraer Enlaces + text_el = await article.query_selector('[data-testid="tweetText"]') + tweet_text = await text_el.inner_text() if text_el else "" + + links = await article.query_selector_all('a') + found_in_tweet = [] + for link in links: + href = await link.get_attribute('href') + if href and href.startswith('http'): + if all(x not in href for x in ["x.com", "twitter.com", "t.co", "abs.twimg", "pbs.twimg"]): + found_in_tweet.append(href) + + found_in_tweet.extend(self._extract_urls_from_text(tweet_text)) + + for u in set(found_in_tweet): + if u not in collected_tweets: + collected_tweets[u] = { + "url": u, "context": tweet_text[:200], + "timestamp": tweet_dt.isoformat(), + "source_type": "X.com (@nubenetes)" + } + + await page.evaluate("window.scrollBy(0, 2500)") + await asyncio.sleep(6) + scroll_count += 1 await browser.close() - unique_res = {r['url']: r for r in results}.values() - return list(unique_res) - except Exception as e: - self.log_audit("Playwright", False, f"Intento {attempt+1} fallido: {str(e)[:50]}") - if attempt == 0: await asyncio.sleep(10) # Pausa antes de reintentar + # Ordenar por fecha: Antiguos a Recientes + results = list(collected_tweets.values()) + results.sort(key=lambda x: x["timestamp"]) + return results + + except Exception as e: + self.log_audit("Playwright", False, str(e)[:60]) return [] async def fetch_links_since(self, since_date: datetime) -> list[dict]: play_links = await self._fetch_via_playwright(since_date) if play_links: - self.log_audit("Estrategia Playwright", True, f"Recuperados {len(play_links)} bookmarks en posts.") + self.log_audit("Estrategia Playwright", True, f"Recuperados {len(play_links)} bookmarks ordenados cronológicamente.") return play_links - # Fallback a RSS con logs mejorados - self.log_audit("RSS Fallback", None, "Consultando puentes RSS por fallo en navegador...") - bridges = ["rssbridge.org", "rss.idoc.pub", "bridge.the-pankratz.de"] + # Fallback a RSS (menos preciso en fechas, pero útil como respaldo) + self.log_audit("RSS Fallback", None, "Intentando vía RSS-Bridge...") + bridges = ["rssbridge.org", "rss.idoc.pub"] for b in bridges: url = f"https://{b}/?action=display&bridge=TwitterBridge&context=By+username&user={self.target_account}&format=Mrss" try: async with aiohttp.ClientSession() as session: - async with session.get(url, timeout=25) as resp: + async with session.get(url, timeout=20) as resp: if resp.status == 200: - xml = await resp.text() - urls = self._extract_urls_from_text(xml) + urls = self._extract_urls_from_text(await resp.text()) valid = [u for u in urls if all(x not in u for x in ["x.com", "twitter.com", "t.co", b])] if valid: self.log_audit(f"RSS-Bridge ({b})", True, f"Encontrados {len(valid)} enlaces.") - return [{"url": u, "context": "RSS Feed Fallback", "timestamp": datetime.now(MADRID_TZ).isoformat(), "source_type": "X.com (RSS)"} for u in valid] - else: - self.log_audit(f"RSS-Bridge ({b})", False, f"HTTP {resp.status}") - except Exception as e: - self.log_audit(f"RSS-Bridge ({b})", False, str(e)[:40]) + return [{"url": u, "context": "RSS Feed", "timestamp": datetime.now(MADRID_TZ).isoformat(), "source_type": "X.com (RSS)"} for u in valid] + except: continue return [] diff --git a/src/main.py b/src/main.py index 72646466..2d1e924f 100644 --- a/src/main.py +++ b/src/main.py @@ -14,26 +14,20 @@ from src.gitops_manager import RepositoryController async def master_orchestrator(): git_controller = RepositoryController(GH_TOKEN, TARGET_REPO) - print("[*] INICIANDO CURADURÍA AGÉNTICA (ESTRATEGIA DE TRANSPARENCIA TOTAL)") + print("[*] INICIANDO CURADURÍA AGÉNTICA (CRONOLOGÍA Y TRANSPARENCIA)") - # 1. Determinar Horizonte Temporal (Oct 2024 por defecto) + # 1. Determinar Horizonte Temporal (Prioridad Oct 2024 si no hay merges) time_horizon = datetime(2024, 10, 1, 0, 0, tzinfo=MADRID_TZ) - - # Solo buscamos PRs mergeadas si el usuario acepta que ya hubo un ciclo exitoso try: pulls = git_controller.repository.get_pulls(state='closed', sort='updated', direction='desc') for pr in pulls: - # Solo consideramos PRs mergeadas por el bot que el usuario ACEPTE como punto de corte if pr.merged and "💎 Knowledge Update" in pr.title: - # Si el usuario dice que no se ha mergeado ninguno 'de este tipo', - # podemos ignorar los antiguos o usar una etiqueta/keyword especial. - # Por ahora, si existe uno de mayo 2026, lo usamos. time_horizon = pr.merged_at.replace(tzinfo=MADRID_TZ) + timedelta(seconds=1) - print(f"[+] Último PR mergeado encontrado ({pr.merged_at}). Retomando desde ahí.") + print(f"[+] Retomando desde último merge exitoso: {pr.merged_at}") break except: pass - print(f"[*] Rango de búsqueda: {time_horizon} ➔ Ahora") + print(f"[*] Buscando posts desde: {time_horizon.date()}") # 2. Ingesta Multi-fuente twitter_client = SocialDataExtractor() @@ -42,7 +36,9 @@ async def master_orchestrator(): print("[*] Buscando novedades en GitHub Trending...") trending = await discover_trending_assets() - for t in trending: t["source_type"] = "GitHub Trending" + for t in trending: + t["source_type"] = "GitHub Trending" + t["timestamp"] = datetime.now(MADRID_TZ).isoformat() all_raw_assets = raw_social + trending @@ -68,7 +64,7 @@ async def master_orchestrator(): clean_url = url.split('#')[0].rstrip('/') status = "INCLUDED" - reason = "Aceptado por relevancia técnica" + reason = "Aceptado" if clean_url in [u.split('#')[0].rstrip('/') for u in existing_urls]: status = "DUPLICATE" @@ -85,7 +81,8 @@ async def master_orchestrator(): "status": status, "reason": reason, "category": curated_urls[url]["category"] if url in curated_urls else "N/A", - "source": asset.get("source_type", "Unknown") + "source": asset.get("source_type", "Unknown"), + "post_date": asset.get("timestamp") }) # 4. Inyección en Markdowns (IA Agentica) @@ -122,7 +119,7 @@ async def master_orchestrator(): } if file_updates or full_extraction_report: - print(f"[+] Finalizado. Generando PR con auditoría completa.") + print(f"[+] Generando PR con auditoría completa.") git_controller.apply_multi_file_changes(file_updates, metrics) if __name__ == "__main__":