diff --git a/src/agentic_curator.py b/src/agentic_curator.py index 6d1f9676..a398588d 100644 --- a/src/agentic_curator.py +++ b/src/agentic_curator.py @@ -91,15 +91,19 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: # MVQ: Check GitHub activity mvq_penalty = False - last_activity = await _get_github_activity(asset['url']) - if last_activity: - years_inactive = (datetime.now(last_activity.tzinfo) - last_activity).days / 365 - if years_inactive > 4: - log_event(f" [⚠️] MVQ Warning: Inactive for {years_inactive:.1f} years.") - mvq_penalty = True + if "github.com" in asset['url']: + log_event(f" [*] Checking GitHub activity...") + last_activity = await _get_github_activity(asset['url']) + if last_activity: + years_inactive = (datetime.now(last_activity.tzinfo) - last_activity).days / 365 + if years_inactive > 4: + log_event(f" [⚠️] MVQ Warning: Inactive for {years_inactive:.1f} years.") + mvq_penalty = True + log_event(f" [*] Fetching web content...") web_content = await _deep_fetch_content(asset['url']) + log_event(f" [*] Calling Gemini for evaluation...") prompt = ( "You act as a Senior Curation Engineer for 'nubenetes/awesome-kubernetes'.\n" "Your mission is to catalog TECHNICAL content about Kubernetes and Cloud Native shared by the user.\n" @@ -152,7 +156,8 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: log_event(f" [!] ERROR EVALUATING {asset['url']}: {e}") evaluations[asset["url"]] = {"status": "FILTERED", "reason": f"Evaluation Failed"} - await asyncio.sleep(1.0) + # Increased safety delay to avoid hitting rate limits too fast (15 RPM default for Gemini Free) + await asyncio.sleep(2.0) try: os.makedirs(os.path.dirname(memory_file), exist_ok=True) diff --git a/src/gemini_utils.py b/src/gemini_utils.py index 6e15e23d..451b0383 100644 --- a/src/gemini_utils.py +++ b/src/gemini_utils.py @@ -103,8 +103,8 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max diagnostics = GeminiDiagnostics() - # Try rotating through all available keys before failing - for key_attempt in range(len(GEMINI_API_KEYS)): + # Try rotating through all available keys + for _ in range(len(GEMINI_API_KEYS)): api_key = GEMINI_API_KEYS[CURRENT_KEY_INDEX] async with httpx.AsyncClient() as client: @@ -112,6 +112,7 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max full_model_name = f"models/{model}" api_url = f"https://generativelanguage.googleapis.com/{GEMINI_API_VERSION}/{full_model_name}:generateContent?key={api_key}" + key_blocked = False for attempt in range(max_retries): try: payload = {"contents": [{"parts": [{"text": prompt}]}]} @@ -127,17 +128,16 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max data = json.loads(match.group(0)) return data[0] if isinstance(data, list) and len(data) > 0 else data diagnostics.add_attempt(model, 200, "JSON not found", text_resp) - break + break # Try next model return text_resp diagnostics.add_attempt(model, 200, "No candidates") - break + break # Try next model elif response.status_code == 429: - # 429: Rotate key immediately to save time log_event(f" [!] API 429 on key {CURRENT_KEY_INDEX+1}. Rotating...") CURRENT_KEY_INDEX = (CURRENT_KEY_INDEX + 1) % len(GEMINI_API_KEYS) - # Break current model loop and move to next key - break + key_blocked = True + break # Break attempt loop elif response.status_code in [500, 503, 504]: await asyncio.sleep(2 * (attempt + 1)) @@ -150,14 +150,16 @@ async def call_gemini_with_retry(prompt: str, response_format: str = "json", max except Exception as e: diagnostics.add_attempt(model, 0, f"Exception: {str(e)}") break + + if key_blocked: + break # Break model loop to try next key - # If we finished all models for a key with 429, skip to next key - if response.status_code == 429: - continue + if key_blocked: + continue # Try next key_attempt - # If we are here and didn't succeed, try next key after a brief pause + # If we are here and didn't succeed with any model, rotate for the next global call CURRENT_KEY_INDEX = (CURRENT_KEY_INDEX + 1) % len(GEMINI_API_KEYS) - await asyncio.sleep(1) + await asyncio.sleep(0.5) raise Exception(f"Critical Gemini failure after key rotation.\n{diagnostics.get_report()}")