Merge pull request #378 from nubenetes/fix/normalize-gh-pushed-source

fix: store gh_pushed as None instead of 'N/A' at enrichment source

scripts/normalize_gh_pushed.py

@@ -0,0 +1,53 @@
+"""One-off migration: normalize `gh_pushed` to None.
+
+The enrichment writers historically stored the string "N/A" in the date
+field `gh_pushed` when GitHub data was unavailable. That literal reached
+`datetime.fromisoformat()` in the safety audit and curator, producing
+"Invalid isoformat string: 'N/A'" warnings (and, combined with other null
+fields, an aborted audit). The writers now emit None; this script cleans
+the existing inventory so the stored data matches.
+
+Usage:
+    python3 -m scripts.normalize_gh_pushed --dry-run   # report only
+    python3 -m scripts.normalize_gh_pushed             # apply + save (YAML+SQL)
+
+NOTE: save_inventory() round-trips through in-memory SQLite, and SQLite's
+iterdump renders REAL columns (e.g. epoch timestamps) with version-specific
+float precision. Running this on a machine whose SQLite differs from the one
+that generated the committed inventory.sql will rewrite ~all rows with no
+semantic change. Run it in CI (same SQLite as the pipeline) so the committed
+diff stays limited to the actual gh_pushed values.
+"""
+import sys
+
+from src.inventory_manager import load_inventory, save_inventory
+from src.logger import log_event
+
+BAD_VALUES = {"N/A", "NONE", ""}
+
+
+def main(dry_run: bool):
+    inv = load_inventory()
+    changed = 0
+    for url, meta in inv.items():
+        if not isinstance(meta, dict):
+            continue
+        pushed = meta.get("gh_pushed")
+        if isinstance(pushed, str) and pushed.strip().upper() in BAD_VALUES:
+            changed += 1
+            if not dry_run:
+                meta["gh_pushed"] = None
+
+    log_event(f"[Migration] gh_pushed normalize: {changed} entries with non-date value")
+    if dry_run:
+        log_event("[Migration] dry-run — no files written")
+        return
+    if changed:
+        save_inventory(inv)
+        log_event(f"[Migration] saved inventory (YAML+SQL); normalized {changed} entries")
+    else:
+        log_event("[Migration] nothing to change")
+
+
+if __name__ == "__main__":
+    main(dry_run="--dry-run" in sys.argv)

src/agentic_curator.py

@@ -115,9 +115,13 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]:
             gh_meta = await _get_github_activity(asset["url"]) if "github.com" in asset["url"] else {}
             
             mvq_penalty = False
-            if gh_meta.get("gh_pushed"):
-                ld = datetime.fromisoformat(gh_meta["gh_pushed"].replace("Z", "+00:00"))
-                if (datetime.now(ld.tzinfo) - ld).days > (365 * 4): mvq_penalty = True
+            pushed = gh_meta.get("gh_pushed")
+            if pushed and str(pushed).strip().upper() not in ("N/A", "NONE"):
+                try:
+                    ld = datetime.fromisoformat(str(pushed).replace("Z", "+00:00"))
+                    if (datetime.now(ld.tzinfo) - ld).days > (365 * 4): mvq_penalty = True
+                except Exception as e:
+                    log_event(f"[WARN] MVQ penalty date parse for {asset['url']}: {str(e)[:100]}")
             
             batch_data.append({
                 "asset": asset, "content": web_content[:1500], "hash": c_hash, 

src/fast_enrich.py

@@ -50,7 +50,7 @@ def get_readable_category(category: str) -> str:
 async def fetch_github_metadata(client: httpx.AsyncClient, url: str, sem: asyncio.Semaphore) -> tuple[str, dict]:
     default_meta = {
         "gh_stars": 0,
-        "gh_pushed": "N/A",
+        "gh_pushed": None,  # date field: unknown = None, never the string "N/A"
         "gh_license": "N/A"
     }
     match = re.search(r'github\.com/([^/]+/[^/]+)', url)
@@ -72,20 +72,20 @@ async def fetch_github_metadata(client: httpx.AsyncClient, url: str, sem: asynci
                 lic_id = lic.get("spdx_id", "N/A") if isinstance(lic, dict) else "N/A"
                 return url, {
                     "gh_stars": data.get("stargazers_count", 0),
-                    "gh_pushed": data.get("pushed_at", "N/A"),
+                    "gh_pushed": data.get("pushed_at"),
                     "gh_license": lic_id
                 }
             elif resp.status_code == 404:
                 return url, {
                     "gh_stars": 0,
-                    "gh_pushed": "N/A",
+                    "gh_pushed": None,
                     "gh_license": "N/A",
                     "status": "dead"
                 }
             else:
                 return url, {
                     "gh_stars": 0,
-                    "gh_pushed": "N/A",
+                    "gh_pushed": None,
                     "gh_license": "N/A",
                     "status": "unreachable"
                 }

src/gemini_utils.py

@@ -211,7 +211,7 @@ async def get_github_activity(url: str) -> Dict:
     """
     default_meta = {
         "gh_stars": 0,
-        "gh_pushed": "N/A",
+        "gh_pushed": None,  # date field: unknown = None, never the string "N/A"
         "gh_license": "N/A"
     }
     match = re.search(r'github\.com/([^/]+/[^/]+)', url)
@@ -236,7 +236,7 @@ async def get_github_activity(url: str) -> Dict:
                 lic_id = lic.get("spdx_id", "N/A") if isinstance(lic, dict) else "N/A"
                 return {
                     "gh_stars": data.get("stargazers_count", 0),
-                    "gh_pushed": data.get("pushed_at", "N/A"),
+                    "gh_pushed": data.get("pushed_at"),
                     "gh_license": lic_id
                 }
     except Exception as e: