mirror of
https://github.com/nubenetes/awesome-kubernetes.git
synced 2026-07-12 09:51:00 +00:00
perf: reduce enrichment from 12min to ~2min (1 API call/repo, 200 limit)
- Consolidate 2 GitHub API calls per repo into 1: open_issues_count already includes PRs on GitHub, second /pulls call was redundant - Reduce MAX_REPOS_DEFAULT 500→200: sufficient for meaningful enrichment, 200 × 0.5s = ~100s vs 500 × 1.5s = ~12.5min in CI - Reduce GITHUB_RATE_DELAY 0.75s→0.5s: still safely under 5000 req/hr - Add 429 rate-limit backoff (5s sleep instead of crashing) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -15,8 +15,8 @@ from src.logger import log_event
|
||||
# ---------------------------------------------------------------------------
|
||||
CNCF_LANDSCAPE_URL = "https://landscape.cncf.io/api/items" # Legacy, now SPA — fallback to GitHub topic search
|
||||
GITHUB_API_BASE = "https://api.github.com"
|
||||
GITHUB_RATE_DELAY = 0.75 # seconds between GitHub API calls to stay under 5000/hr
|
||||
MAX_REPOS_DEFAULT = 500
|
||||
GITHUB_RATE_DELAY = 0.5 # seconds between GitHub API calls (5000/hr limit = ~1.4/s safe)
|
||||
MAX_REPOS_DEFAULT = 200 # cap per run — 200 × 0.5s = ~100s, well within CI timeout
|
||||
ACTIVITY_STALENESS_DAYS = 30
|
||||
|
||||
# Community health thresholds
|
||||
@@ -155,15 +155,15 @@ async def _fetch_repo_activity(
|
||||
client: httpx.AsyncClient,
|
||||
owner_repo: str,
|
||||
) -> Tuple[int, int]:
|
||||
"""Fetch open issue count and recent open PR count for a single repo.
|
||||
"""Fetch open issue + PR count for a single repo in one API call.
|
||||
|
||||
Returns (open_issues_count, open_prs_count).
|
||||
GitHub's open_issues_count includes PRs, so a single /repos endpoint
|
||||
call is sufficient. Returns (open_issues_count, 0) — the caller uses
|
||||
the combined metric for health classification.
|
||||
"""
|
||||
headers = _github_headers()
|
||||
open_issues = 0
|
||||
open_prs = 0
|
||||
|
||||
# Fetch repo-level stats (open_issues_count includes PRs on GitHub)
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"{GITHUB_API_BASE}/repos/{owner_repo}",
|
||||
@@ -173,37 +173,15 @@ async def _fetch_repo_activity(
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
open_issues = data.get("open_issues_count", 0)
|
||||
elif resp.status_code == 429:
|
||||
# Rate limited — back off
|
||||
await asyncio.sleep(5.0)
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] GitHub repo fetch failed for {owner_repo}: {str(e)[:120]}")
|
||||
|
||||
await asyncio.sleep(GITHUB_RATE_DELAY)
|
||||
|
||||
# Fetch open PRs (page 1 only — we use total_count from search or headers)
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"{GITHUB_API_BASE}/repos/{owner_repo}/pulls",
|
||||
headers=headers,
|
||||
params={"state": "open", "sort": "created", "per_page": 1},
|
||||
timeout=15.0,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
# The total count of open PRs is not directly in the body for the
|
||||
# list endpoint, but we can parse the Link header for the last page.
|
||||
# As a simpler approach, the body is a list; if it has items the repo
|
||||
# has open PRs. We use the repo-level open_issues_count as the
|
||||
# combined metric (GitHub counts PRs as issues).
|
||||
pr_data = resp.json()
|
||||
if isinstance(pr_data, list) and len(pr_data) > 0:
|
||||
# Parse Link header for total pages
|
||||
link_header = resp.headers.get("Link", "")
|
||||
last_match = re.search(r'page=(\d+)>;\s*rel="last"', link_header)
|
||||
open_prs = int(last_match.group(1)) if last_match else len(pr_data)
|
||||
except Exception as e:
|
||||
log_event(f"[WARN] GitHub PRs fetch failed for {owner_repo}: {str(e)[:120]}")
|
||||
|
||||
await asyncio.sleep(GITHUB_RATE_DELAY)
|
||||
|
||||
return open_issues, open_prs
|
||||
return open_issues, 0
|
||||
|
||||
|
||||
def _classify_health(total_activity: int) -> str:
|
||||
|
||||
Reference in New Issue
Block a user