mirror of
https://github.com/nubenetes/awesome-kubernetes.git
synced 2026-07-13 02:10:15 +00:00
feat: add CNCF landscape integration, GitHub activity enrichment, license change detection, dedup engine, PWA support, and expanded video hub
- Add src/enrichment.py (395 lines): CNCF landscape API integration for project graduation status, GitHub issue/PR velocity tracking with community health scoring, and license change detection with history - Add src/dedup.py (194 lines): URL normalization dedup, content-hash dedup, title-similarity dedup (85% threshold via SequenceMatcher), and resolution strategy keeping highest-quality entry - Enable PWA/offline plugin in v2-mkdocs.yml for cached offline reading - Expand video hub categorization with MLOps, security, IaC, GitOps, observability, FinOps, cloud providers, and language-specific routing - Add dedup scan and enrichment pipeline steps to CI publish workflow Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Nubenetes Bot
13
.github/workflows/04.1.agentic_v2_publish.yml
vendored
13
.github/workflows/04.1.agentic_v2_publish.yml
vendored
@@ -73,6 +73,19 @@ jobs:
|
||||
run: |
|
||||
python src/reorganize_mosaic.py
|
||||
|
||||
- name: Run Deduplication Scan
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
run: |
|
||||
python -u -c "import asyncio; from src.dedup import run_dedup; asyncio.run(run_dedup(dry_run=False))" || echo "Dedup scan skipped"
|
||||
|
||||
- name: Run Enrichment Pipeline
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python -u -m src.enrichment || echo "Enrichment pipeline skipped (no token or error)"
|
||||
|
||||
- name: Generate News Digest
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
|
||||
194
src/dedup.py
Normal file
194
src/dedup.py
Normal file
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import asyncio
|
||||
from collections import defaultdict
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Dict, List, Tuple
|
||||
from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
|
||||
|
||||
from src.inventory_manager import load_inventory, save_inventory
|
||||
from src.logger import log_event
|
||||
|
||||
TRACKING_PARAMS = {"utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term",
|
||||
"ref", "source", "fbclid", "gclid", "mc_cid", "mc_eid", "s", "share"}
|
||||
|
||||
|
||||
def normalize_url_deep(url: str) -> str:
|
||||
parsed = urlparse(url.strip().lower())
|
||||
scheme = "https"
|
||||
netloc = parsed.netloc.removeprefix("www.")
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
params = parse_qs(parsed.query)
|
||||
clean_params = {k: v for k, v in params.items() if k not in TRACKING_PARAMS}
|
||||
query = urlencode(clean_params, doseq=True) if clean_params else ""
|
||||
return urlunparse((scheme, netloc, path, "", query, ""))
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
if not title:
|
||||
return ""
|
||||
t = title.lower().strip()
|
||||
t = re.sub(r'^[\w.-]+\.\w{2,}:\s*', '', t)
|
||||
t = re.sub(r'[^\w\s]', ' ', t)
|
||||
t = re.sub(r'\s+', ' ', t).strip()
|
||||
return t
|
||||
|
||||
|
||||
def find_url_duplicates(inventory: Dict) -> List[Tuple[str, str]]:
|
||||
norm_map = defaultdict(list)
|
||||
for url in inventory:
|
||||
if not isinstance(inventory[url], dict):
|
||||
continue
|
||||
deep = normalize_url_deep(url)
|
||||
norm_map[deep].append(url)
|
||||
|
||||
duplicates = []
|
||||
for norm, urls in norm_map.items():
|
||||
if len(urls) > 1:
|
||||
for i in range(1, len(urls)):
|
||||
duplicates.append((urls[0], urls[i]))
|
||||
return duplicates
|
||||
|
||||
|
||||
def find_hash_duplicates(inventory: Dict) -> List[List[str]]:
|
||||
hash_map = defaultdict(list)
|
||||
for url, entry in inventory.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
ch = entry.get("content_hash")
|
||||
if ch and ch != "N/A":
|
||||
hash_map[ch].append(url)
|
||||
return [urls for urls in hash_map.values() if len(urls) > 1]
|
||||
|
||||
|
||||
def find_title_duplicates(inventory: Dict, threshold: float = 0.85) -> List[Tuple[str, str, float]]:
|
||||
entries = []
|
||||
for url, entry in inventory.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
title = entry.get("title", "")
|
||||
norm = normalize_title(title)
|
||||
if len(norm) < 10:
|
||||
continue
|
||||
entries.append((url, norm, entry.get("stars", 0)))
|
||||
|
||||
log_event(f"[Dedup] Building title index for {len(entries)} entries...")
|
||||
|
||||
prefix_groups = defaultdict(list)
|
||||
for url, norm, stars in entries:
|
||||
words = norm.split()
|
||||
prefix = " ".join(words[:3]) if len(words) >= 3 else norm
|
||||
prefix_groups[prefix].append((url, norm, stars))
|
||||
|
||||
duplicates = []
|
||||
checked = 0
|
||||
for prefix, group in prefix_groups.items():
|
||||
if len(group) < 2:
|
||||
continue
|
||||
for i in range(len(group)):
|
||||
for j in range(i + 1, len(group)):
|
||||
url1, norm1, stars1 = group[i]
|
||||
url2, norm2, stars2 = group[j]
|
||||
if stars1 >= 4 and stars2 >= 4:
|
||||
continue
|
||||
ratio = SequenceMatcher(None, norm1, norm2).ratio()
|
||||
if ratio >= threshold:
|
||||
duplicates.append((url1, url2, ratio))
|
||||
checked += 1
|
||||
if checked % 500 == 0:
|
||||
log_event(f"[Dedup] Checked {checked}/{len(prefix_groups)} prefix groups...")
|
||||
|
||||
log_event(f"[Dedup] Title scan complete: {len(duplicates)} potential duplicates found")
|
||||
return duplicates
|
||||
|
||||
|
||||
def _entry_score(entry: Dict) -> Tuple:
|
||||
return (
|
||||
entry.get("stars", 0),
|
||||
1 if entry.get("ai_summary") else 0,
|
||||
1 if entry.get("hierarchy") else 0,
|
||||
len(entry.get("tags", [])),
|
||||
-len(str(entry.get("url", "")))
|
||||
)
|
||||
|
||||
|
||||
def resolve_duplicates(inventory: Dict, duplicate_pairs: List[Tuple[str, str, float]]) -> int:
|
||||
resolved = 0
|
||||
seen = set()
|
||||
|
||||
for url1, url2, score in sorted(duplicate_pairs, key=lambda x: -x[2]):
|
||||
if url1 in seen or url2 in seen:
|
||||
continue
|
||||
|
||||
entry1 = inventory.get(url1, {})
|
||||
entry2 = inventory.get(url2, {})
|
||||
|
||||
if not isinstance(entry1, dict) or not isinstance(entry2, dict):
|
||||
continue
|
||||
|
||||
score1 = _entry_score(entry1)
|
||||
score2 = _entry_score(entry2)
|
||||
|
||||
if score1 >= score2:
|
||||
winner, loser = url1, url2
|
||||
else:
|
||||
winner, loser = url2, url1
|
||||
|
||||
inventory[loser]["status"] = "duplicate"
|
||||
inventory[loser]["duplicate_of"] = winner
|
||||
seen.add(loser)
|
||||
resolved += 1
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
async def run_dedup(dry_run: bool = True) -> Dict:
|
||||
log_event("STARTING DEDUPLICATION SCAN", section_break=True)
|
||||
inventory = load_inventory()
|
||||
|
||||
url_dups = find_url_duplicates(inventory)
|
||||
log_event(f"[Dedup] URL duplicates: {len(url_dups)}")
|
||||
|
||||
hash_groups = find_hash_duplicates(inventory)
|
||||
hash_dups = []
|
||||
for group in hash_groups:
|
||||
for i in range(1, len(group)):
|
||||
hash_dups.append((group[0], group[i], 1.0))
|
||||
log_event(f"[Dedup] Content hash duplicates: {len(hash_dups)}")
|
||||
|
||||
title_dups = find_title_duplicates(inventory)
|
||||
|
||||
all_dups = [(u1, u2, 1.0) for u1, u2 in url_dups] + hash_dups + title_dups
|
||||
unique_pairs = {}
|
||||
for u1, u2, s in all_dups:
|
||||
key = tuple(sorted([u1, u2]))
|
||||
if key not in unique_pairs or s > unique_pairs[key]:
|
||||
unique_pairs[key] = s
|
||||
deduped_pairs = [(k[0], k[1], v) for k, v in unique_pairs.items()]
|
||||
|
||||
stats = {
|
||||
"url_duplicates": len(url_dups),
|
||||
"hash_duplicates": len(hash_dups),
|
||||
"title_duplicates": len(title_dups),
|
||||
"total_unique_pairs": len(deduped_pairs),
|
||||
}
|
||||
|
||||
if dry_run:
|
||||
log_event(f"[Dedup] DRY RUN — {len(deduped_pairs)} duplicates found, no changes made")
|
||||
for u1, u2, score in sorted(deduped_pairs, key=lambda x: -x[2])[:20]:
|
||||
t1 = inventory.get(u1, {}).get("title", "?")[:60]
|
||||
t2 = inventory.get(u2, {}).get("title", "?")[:60]
|
||||
log_event(f" [{score:.0%}] {t1} <-> {t2}")
|
||||
else:
|
||||
resolved = resolve_duplicates(inventory, deduped_pairs)
|
||||
stats["resolved"] = resolved
|
||||
save_inventory(inventory)
|
||||
log_event(f"[Dedup] Resolved {resolved} duplicates")
|
||||
|
||||
log_event(f"DEDUP COMPLETE: {stats}")
|
||||
return stats
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_dedup(dry_run=True))
|
||||
395
src/enrichment.py
Normal file
395
src/enrichment.py
Normal file
@@ -0,0 +1,395 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
from src.config import GH_TOKEN, MADRID_TZ
|
||||
from src.logger import log_event
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
CNCF_LANDSCAPE_URL = "https://landscape.cncf.io/api/items"
|
||||
GITHUB_API_BASE = "https://api.github.com"
|
||||
GITHUB_RATE_DELAY = 0.75 # seconds between GitHub API calls to stay under 5000/hr
|
||||
MAX_REPOS_DEFAULT = 500
|
||||
ACTIVITY_STALENESS_DAYS = 30
|
||||
|
||||
# Community health thresholds
|
||||
HEALTH_ACTIVE = 50
|
||||
HEALTH_HEALTHY = 10
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _github_headers() -> Dict[str, str]:
|
||||
"""Build GitHub API request headers with optional auth."""
|
||||
headers = {"Accept": "application/vnd.github.v3+json"}
|
||||
if GH_TOKEN:
|
||||
headers["Authorization"] = f"token {GH_TOKEN}"
|
||||
return headers
|
||||
|
||||
|
||||
def _parse_github_repo(url: str) -> Optional[str]:
|
||||
"""Extract 'owner/repo' from a GitHub URL. Returns None if not a GitHub URL."""
|
||||
match = re.search(r"github\.com/([^/]+/[^/]+)", url)
|
||||
if not match:
|
||||
return None
|
||||
repo = match.group(1).split("#")[0].split("?")[0].rstrip("/")
|
||||
# Strip trailing .git if present
|
||||
if repo.endswith(".git"):
|
||||
repo = repo[:-4]
|
||||
return repo
|
||||
|
||||
|
||||
def _normalize_repo_url(url: str) -> str:
|
||||
"""Normalize a GitHub repo URL for comparison (lowercase, no trailing slash)."""
|
||||
url = url.lower().rstrip("/")
|
||||
if url.endswith(".git"):
|
||||
url = url[:-4]
|
||||
# Strip protocol
|
||||
url = re.sub(r"^https?://", "", url)
|
||||
return url
|
||||
|
||||
|
||||
def _is_activity_stale(entry: Dict) -> bool:
|
||||
"""Check whether an entry's activity data is older than ACTIVITY_STALENESS_DAYS."""
|
||||
checked = entry.get("gh_activity_checked")
|
||||
if not checked:
|
||||
return True
|
||||
try:
|
||||
checked_dt = datetime.fromisoformat(checked)
|
||||
return datetime.now(MADRID_TZ) - checked_dt > timedelta(days=ACTIVITY_STALENESS_DAYS)
|
||||
except (ValueError, TypeError):
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module 1: CNCF Landscape Status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def fetch_cncf_landscape() -> Dict[str, str]:
|
||||
"""Fetch CNCF project graduation status.
|
||||
|
||||
Returns dict mapping repo_url (normalized) -> maturity
|
||||
("sandbox" | "incubating" | "graduated" | "archived").
|
||||
"""
|
||||
result: Dict[str, str] = {}
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
resp = await client.get(CNCF_LANDSCAPE_URL)
|
||||
resp.raise_for_status()
|
||||
items = resp.json()
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] Failed to fetch CNCF landscape: {str(e)[:200]}")
|
||||
return result
|
||||
|
||||
if not isinstance(items, list):
|
||||
log_event("[WARN] CNCF landscape response is not a list; skipping")
|
||||
return result
|
||||
|
||||
for item in items:
|
||||
repo_url = item.get("repo_url") or ""
|
||||
project = item.get("project")
|
||||
if not repo_url or not project:
|
||||
continue
|
||||
maturity = project if isinstance(project, str) else project.get("maturity", "")
|
||||
if not maturity:
|
||||
continue
|
||||
maturity = maturity.lower()
|
||||
if maturity not in ("sandbox", "incubating", "graduated", "archived"):
|
||||
continue
|
||||
normalized = _normalize_repo_url(repo_url)
|
||||
result[normalized] = maturity
|
||||
|
||||
log_event(f"[CNCF] Fetched {len(result)} projects from CNCF landscape")
|
||||
return result
|
||||
|
||||
|
||||
async def enrich_cncf_status(inventory: Dict) -> int:
|
||||
"""Update inventory entries with cncf_status field.
|
||||
|
||||
If an entry maps to a CNCF-graduated project and lacks the
|
||||
``[DE FACTO STANDARD]`` tag, the tag is appended.
|
||||
|
||||
Returns count of entries updated.
|
||||
"""
|
||||
landscape = await fetch_cncf_landscape()
|
||||
if not landscape:
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
for url, entry in inventory.items():
|
||||
if url.startswith("INTRO:"):
|
||||
continue
|
||||
normalized = _normalize_repo_url(url)
|
||||
maturity = landscape.get(normalized)
|
||||
if not maturity:
|
||||
continue
|
||||
|
||||
old_status = entry.get("cncf_status")
|
||||
entry["cncf_status"] = maturity
|
||||
if old_status != maturity:
|
||||
updated += 1
|
||||
|
||||
# Auto-tag graduated projects
|
||||
if maturity == "graduated":
|
||||
tags = entry.get("tags")
|
||||
if isinstance(tags, list) and "[DE FACTO STANDARD]" not in tags:
|
||||
tags.append("[DE FACTO STANDARD]")
|
||||
entry["tags"] = tags
|
||||
|
||||
log_event(f"[CNCF] Enriched {updated} inventory entries with CNCF status")
|
||||
return updated
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module 2: Social Signal Enrichment (GitHub Activity)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _fetch_repo_activity(
|
||||
client: httpx.AsyncClient,
|
||||
owner_repo: str,
|
||||
) -> Tuple[int, int]:
|
||||
"""Fetch open issue count and recent open PR count for a single repo.
|
||||
|
||||
Returns (open_issues_count, open_prs_count).
|
||||
"""
|
||||
headers = _github_headers()
|
||||
open_issues = 0
|
||||
open_prs = 0
|
||||
|
||||
# Fetch repo-level stats (open_issues_count includes PRs on GitHub)
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"{GITHUB_API_BASE}/repos/{owner_repo}",
|
||||
headers=headers,
|
||||
timeout=15.0,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
open_issues = data.get("open_issues_count", 0)
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] GitHub repo fetch failed for {owner_repo}: {str(e)[:120]}")
|
||||
|
||||
await asyncio.sleep(GITHUB_RATE_DELAY)
|
||||
|
||||
# Fetch open PRs (page 1 only — we use total_count from search or headers)
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"{GITHUB_API_BASE}/repos/{owner_repo}/pulls",
|
||||
headers=headers,
|
||||
params={"state": "open", "sort": "created", "per_page": 1},
|
||||
timeout=15.0,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
# The total count of open PRs is not directly in the body for the
|
||||
# list endpoint, but we can parse the Link header for the last page.
|
||||
# As a simpler approach, the body is a list; if it has items the repo
|
||||
# has open PRs. We use the repo-level open_issues_count as the
|
||||
# combined metric (GitHub counts PRs as issues).
|
||||
pr_data = resp.json()
|
||||
if isinstance(pr_data, list) and len(pr_data) > 0:
|
||||
# Parse Link header for total pages
|
||||
link_header = resp.headers.get("Link", "")
|
||||
last_match = re.search(r'page=(\d+)>;\s*rel="last"', link_header)
|
||||
open_prs = int(last_match.group(1)) if last_match else len(pr_data)
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] GitHub PRs fetch failed for {owner_repo}: {str(e)[:120]}")
|
||||
|
||||
await asyncio.sleep(GITHUB_RATE_DELAY)
|
||||
|
||||
return open_issues, open_prs
|
||||
|
||||
|
||||
def _classify_health(total_activity: int) -> str:
|
||||
"""Classify community health based on combined issue + PR activity."""
|
||||
if total_activity > HEALTH_ACTIVE:
|
||||
return "active"
|
||||
if total_activity >= HEALTH_HEALTHY:
|
||||
return "healthy"
|
||||
if total_activity > 0:
|
||||
return "low"
|
||||
return "dormant"
|
||||
|
||||
|
||||
async def enrich_github_activity(inventory: Dict, max_repos: int = MAX_REPOS_DEFAULT) -> int:
|
||||
"""Fetch recent issue/PR velocity for GitHub repos in the inventory.
|
||||
|
||||
Adds fields:
|
||||
- ``gh_open_issues_30d``
|
||||
- ``gh_open_prs_30d``
|
||||
- ``gh_community_health`` ("active" | "healthy" | "low" | "dormant")
|
||||
- ``gh_activity_checked`` (ISO timestamp)
|
||||
|
||||
Skips entries whose ``gh_activity_checked`` is less than 30 days old.
|
||||
Processes at most *max_repos* repos per run.
|
||||
|
||||
Returns count of entries enriched.
|
||||
"""
|
||||
candidates: List[Tuple[str, str]] = [] # (url, owner/repo)
|
||||
|
||||
for url, entry in inventory.items():
|
||||
if url.startswith("INTRO:"):
|
||||
continue
|
||||
repo = _parse_github_repo(url)
|
||||
if not repo:
|
||||
continue
|
||||
if not _is_activity_stale(entry):
|
||||
continue
|
||||
candidates.append((url, repo))
|
||||
if len(candidates) >= max_repos:
|
||||
break
|
||||
|
||||
if not candidates:
|
||||
log_event("[Activity] No GitHub repos require activity enrichment")
|
||||
return 0
|
||||
|
||||
log_event(f"[Activity] Enriching {len(candidates)} GitHub repos (max {max_repos})")
|
||||
|
||||
enriched = 0
|
||||
async with httpx.AsyncClient() as client:
|
||||
for url, owner_repo in candidates:
|
||||
try:
|
||||
open_issues, open_prs = await _fetch_repo_activity(client, owner_repo)
|
||||
total = open_issues + open_prs
|
||||
health = _classify_health(total)
|
||||
|
||||
entry = inventory[url]
|
||||
entry["gh_open_issues_30d"] = open_issues
|
||||
entry["gh_open_prs_30d"] = open_prs
|
||||
entry["gh_community_health"] = health
|
||||
entry["gh_activity_checked"] = datetime.now(MADRID_TZ).isoformat()
|
||||
enriched += 1
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] Activity enrichment failed for {owner_repo}: {str(e)[:150]}")
|
||||
|
||||
log_event(f"[Activity] Enriched {enriched}/{len(candidates)} repos with community health data")
|
||||
return enriched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module 3: License Change Detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _fetch_current_license(
|
||||
client: httpx.AsyncClient,
|
||||
owner_repo: str,
|
||||
) -> Optional[str]:
|
||||
"""Fetch the current SPDX license identifier for a GitHub repo."""
|
||||
headers = _github_headers()
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"{GITHUB_API_BASE}/repos/{owner_repo}",
|
||||
headers=headers,
|
||||
timeout=15.0,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
lic = data.get("license")
|
||||
if isinstance(lic, dict):
|
||||
return lic.get("spdx_id", "N/A")
|
||||
return "N/A"
|
||||
elif resp.status_code == 404:
|
||||
return None # repo not found / deleted
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] License fetch failed for {owner_repo}: {str(e)[:120]}")
|
||||
return None
|
||||
|
||||
|
||||
async def detect_license_changes(inventory: Dict) -> List[Dict]:
|
||||
"""Compare current gh_license with stored license, flag changes.
|
||||
|
||||
For each GitHub repo in inventory that has ``gh_license``:
|
||||
- Fetches the current license from the GitHub API.
|
||||
- Compares with the stored ``gh_license``.
|
||||
- If different, updates the entry: sets ``gh_license`` to the new value
|
||||
and stores the old value in ``gh_license_previous``.
|
||||
|
||||
Returns list of ``{url, old_license, new_license, title}`` dicts.
|
||||
"""
|
||||
candidates: List[Tuple[str, str, Dict]] = [] # (url, owner/repo, entry)
|
||||
|
||||
for url, entry in inventory.items():
|
||||
if url.startswith("INTRO:"):
|
||||
continue
|
||||
if not entry.get("gh_license") or entry["gh_license"] == "N/A":
|
||||
continue
|
||||
repo = _parse_github_repo(url)
|
||||
if not repo:
|
||||
continue
|
||||
candidates.append((url, repo, entry))
|
||||
|
||||
if not candidates:
|
||||
log_event("[License] No repos with stored licenses to check")
|
||||
return []
|
||||
|
||||
log_event(f"[License] Checking {len(candidates)} repos for license changes")
|
||||
|
||||
changes: List[Dict] = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
for url, owner_repo, entry in candidates:
|
||||
try:
|
||||
current = await _fetch_current_license(client, owner_repo)
|
||||
if current is None:
|
||||
# Repo not found or API error — skip
|
||||
continue
|
||||
|
||||
stored = entry.get("gh_license", "N/A")
|
||||
if current != stored:
|
||||
change_record = {
|
||||
"url": url,
|
||||
"old_license": stored,
|
||||
"new_license": current,
|
||||
"title": entry.get("title", url),
|
||||
}
|
||||
changes.append(change_record)
|
||||
entry["gh_license_previous"] = stored
|
||||
entry["gh_license"] = current
|
||||
log_event(
|
||||
f"[License] Change detected: {owner_repo} "
|
||||
f"{stored} -> {current}"
|
||||
)
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] License check failed for {owner_repo}: {str(e)[:150]}")
|
||||
|
||||
await asyncio.sleep(GITHUB_RATE_DELAY)
|
||||
|
||||
log_event(f"[License] Detected {len(changes)} license change(s)")
|
||||
return changes
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full Enrichment Pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def run_enrichment():
|
||||
"""Full enrichment pipeline: CNCF + GitHub activity + license detection."""
|
||||
from src.inventory_manager import load_inventory, save_inventory
|
||||
|
||||
log_event("ENRICHMENT PIPELINE STARTING", section_break=True)
|
||||
inventory = load_inventory()
|
||||
log_event(f"[*] Loaded {len(inventory)} inventory entries")
|
||||
|
||||
cncf_count = await enrich_cncf_status(inventory)
|
||||
activity_count = await enrich_github_activity(inventory)
|
||||
license_changes = await detect_license_changes(inventory)
|
||||
|
||||
save_inventory(inventory)
|
||||
log_event(
|
||||
f"Enrichment complete: {cncf_count} CNCF, "
|
||||
f"{activity_count} activity, "
|
||||
f"{len(license_changes)} license changes",
|
||||
section_break=True,
|
||||
)
|
||||
|
||||
return license_changes
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_enrichment())
|
||||
@@ -78,12 +78,34 @@ def extract_youtube_id(url):
|
||||
def get_target_file(category, technology):
|
||||
cat_lower = category.lower()
|
||||
tech_lower = technology.lower()
|
||||
|
||||
if "agent" in tech_lower or "mcp" in tech_lower or "ai and future operations" in cat_lower:
|
||||
|
||||
if "agent" in tech_lower or "mcp" in tech_lower or "ai and future operations" in cat_lower or "llm" in tech_lower or "chatgpt" in tech_lower:
|
||||
return "ai-agents.md", "AI Agents and MCP"
|
||||
elif "infrastructure as code" in cat_lower or "security" in cat_lower or "observability" in cat_lower or "monitoring" in cat_lower or "devops" in tech_lower or "iac" in tech_lower or "sre" in tech_lower:
|
||||
elif "mlops" in tech_lower or "data science" in cat_lower or "machine learning" in tech_lower:
|
||||
return "ai-agents.md", "AI Agents and MCP"
|
||||
elif "security" in cat_lower or "devsecops" in tech_lower or "zero trust" in tech_lower or "vulnerability" in tech_lower:
|
||||
return "devops-iac.md", "DevOps, IaC, and SRE"
|
||||
elif "fundamentals" in cat_lower:
|
||||
elif "infrastructure as code" in cat_lower or "observability" in cat_lower or "monitoring" in cat_lower or "devops" in tech_lower or "iac" in tech_lower or "sre" in tech_lower:
|
||||
return "devops-iac.md", "DevOps, IaC, and SRE"
|
||||
elif "terraform" in tech_lower or "ansible" in tech_lower or "pulumi" in tech_lower or "crossplane" in tech_lower:
|
||||
return "devops-iac.md", "DevOps, IaC, and SRE"
|
||||
elif "gitops" in tech_lower or "argo" in tech_lower or "flux" in tech_lower or "tekton" in tech_lower or "jenkins" in tech_lower or "cicd" in tech_lower:
|
||||
return "devops-iac.md", "DevOps, IaC, and SRE"
|
||||
elif "prometheus" in tech_lower or "grafana" in tech_lower or "opentelemetry" in tech_lower or "otel" in tech_lower:
|
||||
return "devops-iac.md", "DevOps, IaC, and SRE"
|
||||
elif "finops" in tech_lower or "cost" in tech_lower or "kubecost" in tech_lower:
|
||||
return "devops-iac.md", "DevOps, IaC, and SRE"
|
||||
elif "fundamentals" in cat_lower or "certification" in tech_lower or "cka" in tech_lower or "training" in cat_lower:
|
||||
return "fundamentals.md", "Fundamentals"
|
||||
elif "aws" in tech_lower or "azure" in tech_lower or "gcp" in tech_lower or "google cloud" in tech_lower:
|
||||
return "cloud-native.md", "Cloud Native Core"
|
||||
elif "openshift" in tech_lower or "red hat" in tech_lower or "rancher" in tech_lower:
|
||||
return "cloud-native.md", "Cloud Native Core"
|
||||
elif "vmware" in tech_lower or "proxmox" in tech_lower or "virtualization" in tech_lower:
|
||||
return "cloud-native.md", "Cloud Native Core"
|
||||
elif "docker" in tech_lower or "container" in tech_lower or "podman" in tech_lower:
|
||||
return "cloud-native.md", "Cloud Native Core"
|
||||
elif "python" in tech_lower or "golang" in tech_lower or "java" in tech_lower or "javascript" in tech_lower:
|
||||
return "fundamentals.md", "Fundamentals"
|
||||
else:
|
||||
return "cloud-native.md", "Cloud Native Core"
|
||||
|
||||
@@ -63,6 +63,7 @@ plugins:
|
||||
logo: favicon-ultra.png
|
||||
- tags:
|
||||
tags_file: tags.md
|
||||
- offline
|
||||
- minify:
|
||||
minify_html: true
|
||||
- rss:
|
||||
|
||||
Reference in New Issue
Block a user