From 9d13a6a21aad05920c6b835d2ba55616ffd962ee Mon Sep 17 00:00:00 2001 From: Nubenetes Bot Date: Fri, 19 Jun 2026 15:10:06 +0200 Subject: [PATCH] fix: store gh_pushed as None instead of 'N/A' at enrichment source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The date field gh_pushed was written as the literal string 'N/A' when GitHub data was unavailable (fast_enrich, gemini_utils). That non-date string then reached datetime.fromisoformat() downstream. Now writes None (which all readers already handle); gh_license keeps its 'N/A' string sentinel since it is a displayed value, not a date. Also hardens the twin fromisoformat() call in agentic_curator's MVQ penalty check (guards N/A/None and wraps parsing) so a bad value can't crash a curation batch. Adds scripts/normalize_gh_pushed.py to clean the 281 existing inventory entries (run in CI to avoid SQLite float-serialization churn — see script docstring). Co-Authored-By: Claude Opus 4.8 --- scripts/normalize_gh_pushed.py | 53 ++++++++++++++++++++++++++++++++++ src/agentic_curator.py | 10 +++++-- src/fast_enrich.py | 8 ++--- src/gemini_utils.py | 4 +-- 4 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 scripts/normalize_gh_pushed.py diff --git a/scripts/normalize_gh_pushed.py b/scripts/normalize_gh_pushed.py new file mode 100644 index 00000000..d7f5b7ee --- /dev/null +++ b/scripts/normalize_gh_pushed.py @@ -0,0 +1,53 @@ +"""One-off migration: normalize `gh_pushed` to None. + +The enrichment writers historically stored the string "N/A" in the date +field `gh_pushed` when GitHub data was unavailable. That literal reached +`datetime.fromisoformat()` in the safety audit and curator, producing +"Invalid isoformat string: 'N/A'" warnings (and, combined with other null +fields, an aborted audit). The writers now emit None; this script cleans +the existing inventory so the stored data matches. + +Usage: + python3 -m scripts.normalize_gh_pushed --dry-run # report only + python3 -m scripts.normalize_gh_pushed # apply + save (YAML+SQL) + +NOTE: save_inventory() round-trips through in-memory SQLite, and SQLite's +iterdump renders REAL columns (e.g. epoch timestamps) with version-specific +float precision. Running this on a machine whose SQLite differs from the one +that generated the committed inventory.sql will rewrite ~all rows with no +semantic change. Run it in CI (same SQLite as the pipeline) so the committed +diff stays limited to the actual gh_pushed values. +""" +import sys + +from src.inventory_manager import load_inventory, save_inventory +from src.logger import log_event + +BAD_VALUES = {"N/A", "NONE", ""} + + +def main(dry_run: bool): + inv = load_inventory() + changed = 0 + for url, meta in inv.items(): + if not isinstance(meta, dict): + continue + pushed = meta.get("gh_pushed") + if isinstance(pushed, str) and pushed.strip().upper() in BAD_VALUES: + changed += 1 + if not dry_run: + meta["gh_pushed"] = None + + log_event(f"[Migration] gh_pushed normalize: {changed} entries with non-date value") + if dry_run: + log_event("[Migration] dry-run — no files written") + return + if changed: + save_inventory(inv) + log_event(f"[Migration] saved inventory (YAML+SQL); normalized {changed} entries") + else: + log_event("[Migration] nothing to change") + + +if __name__ == "__main__": + main(dry_run="--dry-run" in sys.argv) diff --git a/src/agentic_curator.py b/src/agentic_curator.py index 031c8a5c..1f44b8ca 100644 --- a/src/agentic_curator.py +++ b/src/agentic_curator.py @@ -115,9 +115,13 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]: gh_meta = await _get_github_activity(asset["url"]) if "github.com" in asset["url"] else {} mvq_penalty = False - if gh_meta.get("gh_pushed"): - ld = datetime.fromisoformat(gh_meta["gh_pushed"].replace("Z", "+00:00")) - if (datetime.now(ld.tzinfo) - ld).days > (365 * 4): mvq_penalty = True + pushed = gh_meta.get("gh_pushed") + if pushed and str(pushed).strip().upper() not in ("N/A", "NONE"): + try: + ld = datetime.fromisoformat(str(pushed).replace("Z", "+00:00")) + if (datetime.now(ld.tzinfo) - ld).days > (365 * 4): mvq_penalty = True + except Exception as e: + log_event(f"[WARN] MVQ penalty date parse for {asset['url']}: {str(e)[:100]}") batch_data.append({ "asset": asset, "content": web_content[:1500], "hash": c_hash, diff --git a/src/fast_enrich.py b/src/fast_enrich.py index 6569cf50..8e17c7de 100644 --- a/src/fast_enrich.py +++ b/src/fast_enrich.py @@ -50,7 +50,7 @@ def get_readable_category(category: str) -> str: async def fetch_github_metadata(client: httpx.AsyncClient, url: str, sem: asyncio.Semaphore) -> tuple[str, dict]: default_meta = { "gh_stars": 0, - "gh_pushed": "N/A", + "gh_pushed": None, # date field: unknown = None, never the string "N/A" "gh_license": "N/A" } match = re.search(r'github\.com/([^/]+/[^/]+)', url) @@ -72,20 +72,20 @@ async def fetch_github_metadata(client: httpx.AsyncClient, url: str, sem: asynci lic_id = lic.get("spdx_id", "N/A") if isinstance(lic, dict) else "N/A" return url, { "gh_stars": data.get("stargazers_count", 0), - "gh_pushed": data.get("pushed_at", "N/A"), + "gh_pushed": data.get("pushed_at"), "gh_license": lic_id } elif resp.status_code == 404: return url, { "gh_stars": 0, - "gh_pushed": "N/A", + "gh_pushed": None, "gh_license": "N/A", "status": "dead" } else: return url, { "gh_stars": 0, - "gh_pushed": "N/A", + "gh_pushed": None, "gh_license": "N/A", "status": "unreachable" } diff --git a/src/gemini_utils.py b/src/gemini_utils.py index 19483d51..a19a1469 100644 --- a/src/gemini_utils.py +++ b/src/gemini_utils.py @@ -211,7 +211,7 @@ async def get_github_activity(url: str) -> Dict: """ default_meta = { "gh_stars": 0, - "gh_pushed": "N/A", + "gh_pushed": None, # date field: unknown = None, never the string "N/A" "gh_license": "N/A" } match = re.search(r'github\.com/([^/]+/[^/]+)', url) @@ -236,7 +236,7 @@ async def get_github_activity(url: str) -> Dict: lic_id = lic.get("spdx_id", "N/A") if isinstance(lic, dict) else "N/A" return { "gh_stars": data.get("stargazers_count", 0), - "gh_pushed": data.get("pushed_at", "N/A"), + "gh_pushed": data.get("pushed_at"), "gh_license": lic_id } except Exception as e: