github/awesome-kubernetes
1
0
Fork 0
mirror of https://github.com/nubenetes/awesome-kubernetes.git synced 2026-07-28 17:41:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement chronological extraction, ignore pinned posts, and add 'Post Date' column to PR report

Browse Source
This commit is contained in:
Nubenetes Bot
2026-05-10 23:13:53 +02:00
parent 66c5aa3578
commit 8030dc6f54
3 changed files with 108 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/gitops_manager.py
View File

@@ -17,7 +17,6 @@ class RepositoryController:
branch_name = f"bot/knowledge-update-{timestamp_slug}"
self._create_feature_branch(branch_name)
# Garantizar que siempre haya al menos un commit (técnico) para abrir la PR
if not updates:
updates["src/memory/last_audit_run.json"] = json.dumps({
"timestamp": metrics.get("end_date"),
@@ -46,15 +45,18 @@ class RepositoryController:
# --- CONSTRUCCIÓN DEL REPORTE ---
full_report = metrics.get('full_report', [])
# 1. Tabla Matricial
# 1. Tabla Matricial con Columna de Fecha
matrix_table = "### 📋 Matriz de Auditoría de Enlaces (Full Extraction)\n"
matrix_table += "| Estado | Origen | Motivo | Categoría | URL |\n| :--- | :--- | :--- | :--- | :--- |\n"
matrix_table += "| Estado | Fecha Post | Origen | Motivo | Categoría | URL |\n| :--- | :--- | :--- | :--- | :--- | :--- |\n"
counts = {"INCLUDED": 0, "DUPLICATE": 0, "FILTERED": 0}
source_counts = {}
for item in full_report[:200]:
status_emoji = {"INCLUDED": "✅", "DUPLICATE": "👯", "FILTERED": "🛡️"}.get(item['status'], "❓")
matrix_table += f"| {status_emoji} {item['status']} | {item.get('source', 'N/A')} | {item['reason']} | `{item['category']}` | {item['url']} |\n"
# Formatear fecha para legibilidad
date_str = item.get('post_date', 'N/A')[:10] if item.get('post_date') else 'N/A'
matrix_table += f"| {status_emoji} {item['status']} | {date_str} | {item.get('source', 'N/A')} | {item['reason']} | `{item['category']}` | {item['url']} |\n"
counts[item['status']] = counts.get(item['status'], 0) + 1
if item['status'] == "INCLUDED":
src = item.get('source', 'Unknown')
@@ -81,14 +83,14 @@ class RepositoryController:
pr_narrative = (
f"## 💎 Knowledge Update War Room: Kubernetes & Cloud Native\n\n"
f"Este reporte detalla el procesamiento de **{metrics.get('total_extracted', 0)}** enlaces detectados.\n\n"
f"**Ventana Temporal:** `{metrics.get('start_date')}` ➔ `{metrics.get('end_date')}`\n\n"
f"**Rango de Posts Analizados:** `{metrics.get('start_date')[:10]}` ➔ `{metrics.get('end_date')[:10]}` (Cronología: Antiguos a Recientes)\n\n"
f"{mermaid_pie}\n"
f"{mermaid_origin}\n"
f"{x_log}\n"
f"{matrix_table}\n"
f"---\n"
f"**Nota de Evaluación:** Este PR incluye {len(metrics.get('added_list', []))} novedades reales. "
f"Ventana temporal automática basada en el histórico de merges."
f"Se ha ignorado el post fijo (Pinned) para evitar duplicidad cíclica."
)
self.repository.create_pull(

175
src/ingestion_twikit.py
View File

@@ -28,7 +28,6 @@ class SocialDataExtractor:
def _extract_urls_from_text(self, text: str) -> list[str]:
urls = re.findall(r'https?://[^\s<>\"]+|www\.[^\s<>\"]+', text)
# Filtro de ruido masivo y dominios de marketing/tracking
noise_domains = [
"x.com", "twitter.com", "t.co", "abs.twimg", "pbs.twimg",
"notoriete-web.com", "google-analytics", "doubleclick",
@@ -42,7 +41,6 @@ class SocialDataExtractor:
return list(set(valid_urls))
async def _fetch_via_playwright(self, since_date: datetime) -> list[dict]:
"""Estrategia de Navegador Real con reintentos y control de tiempo."""
try:
from playwright.async_api import async_playwright
import playwright_stealth
@@ -50,114 +48,119 @@ class SocialDataExtractor:
self.log_audit("Playwright", False, "Librerías no disponibles.")
return []
self.log_audit("Playwright Browser", None, f"Buscando posts desde {since_date.date()}...")
self.log_audit("Playwright Browser", None, f"Cronología: Desde {since_date.date()} hasta hoy.")
results = []
for attempt in range(2): # 2 Intentos
try:
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(user_agent=self.user_agents[0], locale="es-ES")
page = await context.new_page()
try:
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(user_agent=self.user_agents[0], locale="es-ES")
page = await context.new_page()
try:
if hasattr(playwright_stealth, 'stealth_async'): await playwright_stealth.stealth_async(page)
elif hasattr(playwright_stealth, 'stealth'): playwright_stealth.stealth(page)
except: pass
env_cookies = os.getenv("TWITTER_COOKIES")
if env_cookies:
try:
if hasattr(playwright_stealth, 'stealth_async'): await playwright_stealth.stealth_async(page)
elif hasattr(playwright_stealth, 'stealth'): playwright_stealth.stealth(page)
cookies = json.loads(env_cookies)
formatted = []
for c in cookies:
if isinstance(c, dict) and 'name' in c and 'value' in c:
c['domain'] = c.get('domain', '.x.com')
for k in ['sameSite', 'storeId', 'id']: c.pop(k, None)
formatted.append(c)
await context.add_cookies(formatted)
except: pass
env_cookies = os.getenv("TWITTER_COOKIES")
if env_cookies:
try:
cookies = json.loads(env_cookies)
formatted = []
for c in cookies:
if isinstance(c, dict) and 'name' in c and 'value' in c:
c['domain'] = c.get('domain', '.x.com')
for k in ['sameSite', 'storeId', 'id']: c.pop(k, None)
formatted.append(c)
await context.add_cookies(formatted)
except: pass
# Aumentar timeout y usar estrategia de carga más relajada
await page.goto(f"https://x.com/{self.target_account}", wait_until="domcontentloaded", timeout=90000)
await asyncio.sleep(15) # Tiempo extra para que X renderice el feed
await page.goto(f"https://x.com/{self.target_account}", wait_until="domcontentloaded", timeout=90000)
await asyncio.sleep(15)
stop_scrolling = False
scroll_count = 0
max_scrolls = 25
collected_tweets = {} # URL -> tweet_data para evitar duplicados en scroll
while not stop_scrolling and scroll_count < max_scrolls:
articles = await page.query_selector_all('article[data-testid="tweet"]')
stop_scrolling = False
scroll_count = 0
max_scrolls = 20 # Suficiente para llegar a Oct 2024 si no hay miles de tweets
while not stop_scrolling and scroll_count < max_scrolls:
articles = await page.query_selector_all('article[data-testid="tweet"]')
new_links_found = False
for article in articles:
# 1. Ignorar Pinned Posts (Post Fijo)
social_context = await article.query_selector('[data-testid="socialContext"]')
if social_context:
sc_text = await social_context.inner_text()
if "Fijado" in sc_text or "Pinned" in sc_text:
continue
# 2. Extraer Fecha
time_el = await article.query_selector('time')
if not time_el: continue
for article in articles:
# Intentar extraer fecha del tweet
time_el = await article.query_selector('time')
if time_el:
datetime_str = await time_el.get_attribute('datetime')
tweet_dt = datetime.fromisoformat(datetime_str.replace('Z', '+00:00')).astimezone(MADRID_TZ)
if tweet_dt < since_date:
stop_scrolling = True
break
text_el = await article.query_selector('[data-testid="tweetText"]')
tweet_text = await text_el.inner_text() if text_el else ""
links = await article.query_selector_all('a')
found_in_tweet = []
for link in links:
href = await link.get_attribute('href')
if href and href.startswith('http') and all(x not in href for x in ["x.com", "twitter.com", "t.co", "abs.twimg", "pbs.twimg"]):
found_in_tweet.append(href)
found_in_tweet.extend(self._extract_urls_from_text(tweet_text))
for u in set(found_in_tweet):
if all(x not in u for x in ["x.com", "twitter.com", "t.co", "archive.org", "help.twitter"]):
results.append({
"url": u, "context": tweet_text[:200],
"timestamp": tweet_dt.isoformat() if time_el else datetime.now(MADRID_TZ).isoformat(),
"source_type": "X.com (@nubenetes)"
})
new_links_found = True
datetime_str = await time_el.get_attribute('datetime')
tweet_dt = datetime.fromisoformat(datetime_str.replace('Z', '+00:00')).astimezone(MADRID_TZ)
if tweet_dt < since_date:
stop_scrolling = True
break
await page.evaluate("window.scrollBy(0, 2000)")
await asyncio.sleep(6)
scroll_count += 1
if not new_links_found and scroll_count > 5: break # Si no hay nada nuevo en 5 scrolls, parar
# 3. Extraer Enlaces
text_el = await article.query_selector('[data-testid="tweetText"]')
tweet_text = await text_el.inner_text() if text_el else ""
links = await article.query_selector_all('a')
found_in_tweet = []
for link in links:
href = await link.get_attribute('href')
if href and href.startswith('http'):
if all(x not in href for x in ["x.com", "twitter.com", "t.co", "abs.twimg", "pbs.twimg"]):
found_in_tweet.append(href)
found_in_tweet.extend(self._extract_urls_from_text(tweet_text))
for u in set(found_in_tweet):
if u not in collected_tweets:
collected_tweets[u] = {
"url": u, "context": tweet_text[:200],
"timestamp": tweet_dt.isoformat(),
"source_type": "X.com (@nubenetes)"
}
await page.evaluate("window.scrollBy(0, 2500)")
await asyncio.sleep(6)
scroll_count += 1
await browser.close()
unique_res = {r['url']: r for r in results}.values()
return list(unique_res)
except Exception as e:
self.log_audit("Playwright", False, f"Intento {attempt+1} fallido: {str(e)[:50]}")
if attempt == 0: await asyncio.sleep(10) # Pausa antes de reintentar
# Ordenar por fecha: Antiguos a Recientes
results = list(collected_tweets.values())
results.sort(key=lambda x: x["timestamp"])
return results
except Exception as e:
self.log_audit("Playwright", False, str(e)[:60])
return []
async def fetch_links_since(self, since_date: datetime) -> list[dict]:
play_links = await self._fetch_via_playwright(since_date)
if play_links:
self.log_audit("Estrategia Playwright", True, f"Recuperados {len(play_links)} bookmarks en posts.")
self.log_audit("Estrategia Playwright", True, f"Recuperados {len(play_links)} bookmarks ordenados cronológicamente.")
return play_links
# Fallback a RSS con logs mejorados
self.log_audit("RSS Fallback", None, "Consultando puentes RSS por fallo en navegador...")
bridges = ["rssbridge.org", "rss.idoc.pub", "bridge.the-pankratz.de"]
# Fallback a RSS (menos preciso en fechas, pero útil como respaldo)
self.log_audit("RSS Fallback", None, "Intentando vía RSS-Bridge...")
bridges = ["rssbridge.org", "rss.idoc.pub"]
for b in bridges:
url = f"https://{b}/?action=display&bridge=TwitterBridge&context=By+username&user={self.target_account}&format=Mrss"
try:
async with aiohttp.ClientSession() as session:
async with session.get(url, timeout=25) as resp:
async with session.get(url, timeout=20) as resp:
if resp.status == 200:
xml = await resp.text()
urls = self._extract_urls_from_text(xml)
urls = self._extract_urls_from_text(await resp.text())
valid = [u for u in urls if all(x not in u for x in ["x.com", "twitter.com", "t.co", b])]
if valid:
self.log_audit(f"RSS-Bridge ({b})", True, f"Encontrados {len(valid)} enlaces.")
return [{"url": u, "context": "RSS Feed Fallback", "timestamp": datetime.now(MADRID_TZ).isoformat(), "source_type": "X.com (RSS)"} for u in valid]
else:
self.log_audit(f"RSS-Bridge ({b})", False, f"HTTP {resp.status}")
except Exception as e:
self.log_audit(f"RSS-Bridge ({b})", False, str(e)[:40])
return [{"url": u, "context": "RSS Feed", "timestamp": datetime.now(MADRID_TZ).isoformat(), "source_type": "X.com (RSS)"} for u in valid]
except: continue
return []

25
src/main.py
View File

@@ -14,26 +14,20 @@ from src.gitops_manager import RepositoryController
async def master_orchestrator():
git_controller = RepositoryController(GH_TOKEN, TARGET_REPO)
print("[*] INICIANDO CURADURÍA AGÉNTICA (ESTRATEGIA DE TRANSPARENCIA TOTAL)")
print("[*] INICIANDO CURADURÍA AGÉNTICA (CRONOLOGÍA Y TRANSPARENCIA)")
# 1. Determinar Horizonte Temporal (Oct 2024 por defecto)
# 1. Determinar Horizonte Temporal (Prioridad Oct 2024 si no hay merges)
time_horizon = datetime(2024, 10, 1, 0, 0, tzinfo=MADRID_TZ)
# Solo buscamos PRs mergeadas si el usuario acepta que ya hubo un ciclo exitoso
try:
pulls = git_controller.repository.get_pulls(state='closed', sort='updated', direction='desc')
for pr in pulls:
# Solo consideramos PRs mergeadas por el bot que el usuario ACEPTE como punto de corte
if pr.merged and "💎 Knowledge Update" in pr.title:
# Si el usuario dice que no se ha mergeado ninguno 'de este tipo',
# podemos ignorar los antiguos o usar una etiqueta/keyword especial.
# Por ahora, si existe uno de mayo 2026, lo usamos.
time_horizon = pr.merged_at.replace(tzinfo=MADRID_TZ) + timedelta(seconds=1)
print(f"[+] Último PR mergeado encontrado ({pr.merged_at}). Retomando desde ahí.")
print(f"[+] Retomando desde último merge exitoso: {pr.merged_at}")
break
except: pass
print(f"[*] Rango de búsqueda: {time_horizon} ➔ Ahora")
print(f"[*] Buscando posts desde: {time_horizon.date()}")
# 2. Ingesta Multi-fuente
twitter_client = SocialDataExtractor()
@@ -42,7 +36,9 @@ async def master_orchestrator():
print("[*] Buscando novedades en GitHub Trending...")
trending = await discover_trending_assets()
for t in trending: t["source_type"] = "GitHub Trending"
for t in trending:
t["source_type"] = "GitHub Trending"
t["timestamp"] = datetime.now(MADRID_TZ).isoformat()
all_raw_assets = raw_social + trending
@@ -68,7 +64,7 @@ async def master_orchestrator():
clean_url = url.split('#')[0].rstrip('/')
status = "INCLUDED"
reason = "Aceptado por relevancia técnica"
reason = "Aceptado"
if clean_url in [u.split('#')[0].rstrip('/') for u in existing_urls]:
status = "DUPLICATE"
@@ -85,7 +81,8 @@ async def master_orchestrator():
"status": status,
"reason": reason,
"category": curated_urls[url]["category"] if url in curated_urls else "N/A",
"source": asset.get("source_type", "Unknown")
"source": asset.get("source_type", "Unknown"),
"post_date": asset.get("timestamp")
})
# 4. Inyección en Markdowns (IA Agentica)
@@ -122,7 +119,7 @@ async def master_orchestrator():
}
if file_updates or full_extraction_report:
print(f"[+] Finalizado. Generando PR con auditoría completa.")
print(f"[+] Generando PR con auditoría completa.")
git_controller.apply_multi_file_changes(file_updates, metrics)
if __name__ == "__main__":