release: v2.9.4 — normalize gh_pushed to None at enrichment source

This commit is contained in:
Nubenetes Bot
2026-06-19 15:10:39 +02:00
5 changed files with 71 additions and 14 deletions
+5 -5
View File
@@ -142,7 +142,7 @@ Additionally, as of May 2026, Nubenetes has reached the **Platinum Operational T
| :--- | :--- |
| **Total Technical Resources (Links)** | **18647+** |
| **Specialized MD Pages** | **162** |
| **Total Commits** | **6121+** |
| **Total Commits** | **6129+** |
| **Primary AI Engine** | **Google Gemini (Agentic)** |
<!-- HEART_STATS_END -->
@@ -180,7 +180,7 @@ The growth of Nubenetes reflects the acceleration of the Cloud Native ecosystem.
| 6 | 2023 | 30 | 123 | Maintenance & Refinement |
| 7 | 2024 | 53 | 218 | Curation Strategy Pivot |
| 8 | 2025 | 5 | 20 | Stability & Research Phase |
| 9 | 2026 | 2562 | 10,581 | **Agentic AI Surge** (May 2026 Inception) |
| 9 | 2026 | 2570 | 10,614 | **Agentic AI Surge** (May 2026 Inception) |
<!-- ANNUAL_GROWTH_END -->
<!-- ANNUAL_CHART_START -->
@@ -196,8 +196,8 @@ xychart-beta
title "Nubenetes Annual Growth Metrics (20182026)"
x-axis ["2018", "2019", "2020", "2021", "2022", "2023", "2024", "2025", "2026"]
y-axis "Volume (Commits / Estimated New Refs)" 0 --> 11000
bar [1445, 586, 8449, 2193, 1660, 123, 218, 20, 10581]
bar [350, 142, 2046, 531, 402, 30, 53, 5, 2562]
bar [1445, 586, 8449, 2193, 1660, 123, 218, 20, 10614]
bar [350, 142, 2046, 531, 402, 30, 53, 5, 2570]
```
<!-- ANNUAL_CHART_END -->
@@ -207,7 +207,7 @@ xychart-beta
| :--- | :---: | :---: | :--- |
| 2026-04 | 25 | 103 | Active Curation |
| 2026-05 | 2101 | 8,677 | **Agentic Inception (Gemini Era)** |
| 2026-06 | 436 | 1,800 | Active Curation |
| 2026-06 | 444 | 1,833 | Active Curation |
<!-- MONTHLY_SURGE_END -->
### 2.4. Content Distribution and Semantic Clustering
+53
View File
@@ -0,0 +1,53 @@
"""One-off migration: normalize `gh_pushed` to None.
The enrichment writers historically stored the string "N/A" in the date
field `gh_pushed` when GitHub data was unavailable. That literal reached
`datetime.fromisoformat()` in the safety audit and curator, producing
"Invalid isoformat string: 'N/A'" warnings (and, combined with other null
fields, an aborted audit). The writers now emit None; this script cleans
the existing inventory so the stored data matches.
Usage:
python3 -m scripts.normalize_gh_pushed --dry-run # report only
python3 -m scripts.normalize_gh_pushed # apply + save (YAML+SQL)
NOTE: save_inventory() round-trips through in-memory SQLite, and SQLite's
iterdump renders REAL columns (e.g. epoch timestamps) with version-specific
float precision. Running this on a machine whose SQLite differs from the one
that generated the committed inventory.sql will rewrite ~all rows with no
semantic change. Run it in CI (same SQLite as the pipeline) so the committed
diff stays limited to the actual gh_pushed values.
"""
import sys
from src.inventory_manager import load_inventory, save_inventory
from src.logger import log_event
BAD_VALUES = {"N/A", "NONE", ""}
def main(dry_run: bool):
inv = load_inventory()
changed = 0
for url, meta in inv.items():
if not isinstance(meta, dict):
continue
pushed = meta.get("gh_pushed")
if isinstance(pushed, str) and pushed.strip().upper() in BAD_VALUES:
changed += 1
if not dry_run:
meta["gh_pushed"] = None
log_event(f"[Migration] gh_pushed normalize: {changed} entries with non-date value")
if dry_run:
log_event("[Migration] dry-run — no files written")
return
if changed:
save_inventory(inv)
log_event(f"[Migration] saved inventory (YAML+SQL); normalized {changed} entries")
else:
log_event("[Migration] nothing to change")
if __name__ == "__main__":
main(dry_run="--dry-run" in sys.argv)
+7 -3
View File
@@ -115,9 +115,13 @@ async def evaluate_extracted_assets(raw_assets: List[Dict]) -> Dict[str, Dict]:
gh_meta = await _get_github_activity(asset["url"]) if "github.com" in asset["url"] else {}
mvq_penalty = False
if gh_meta.get("gh_pushed"):
ld = datetime.fromisoformat(gh_meta["gh_pushed"].replace("Z", "+00:00"))
if (datetime.now(ld.tzinfo) - ld).days > (365 * 4): mvq_penalty = True
pushed = gh_meta.get("gh_pushed")
if pushed and str(pushed).strip().upper() not in ("N/A", "NONE"):
try:
ld = datetime.fromisoformat(str(pushed).replace("Z", "+00:00"))
if (datetime.now(ld.tzinfo) - ld).days > (365 * 4): mvq_penalty = True
except Exception as e:
log_event(f"[WARN] MVQ penalty date parse for {asset['url']}: {str(e)[:100]}")
batch_data.append({
"asset": asset, "content": web_content[:1500], "hash": c_hash,
+4 -4
View File
@@ -50,7 +50,7 @@ def get_readable_category(category: str) -> str:
async def fetch_github_metadata(client: httpx.AsyncClient, url: str, sem: asyncio.Semaphore) -> tuple[str, dict]:
default_meta = {
"gh_stars": 0,
"gh_pushed": "N/A",
"gh_pushed": None, # date field: unknown = None, never the string "N/A"
"gh_license": "N/A"
}
match = re.search(r'github\.com/([^/]+/[^/]+)', url)
@@ -72,20 +72,20 @@ async def fetch_github_metadata(client: httpx.AsyncClient, url: str, sem: asynci
lic_id = lic.get("spdx_id", "N/A") if isinstance(lic, dict) else "N/A"
return url, {
"gh_stars": data.get("stargazers_count", 0),
"gh_pushed": data.get("pushed_at", "N/A"),
"gh_pushed": data.get("pushed_at"),
"gh_license": lic_id
}
elif resp.status_code == 404:
return url, {
"gh_stars": 0,
"gh_pushed": "N/A",
"gh_pushed": None,
"gh_license": "N/A",
"status": "dead"
}
else:
return url, {
"gh_stars": 0,
"gh_pushed": "N/A",
"gh_pushed": None,
"gh_license": "N/A",
"status": "unreachable"
}
+2 -2
View File
@@ -211,7 +211,7 @@ async def get_github_activity(url: str) -> Dict:
"""
default_meta = {
"gh_stars": 0,
"gh_pushed": "N/A",
"gh_pushed": None, # date field: unknown = None, never the string "N/A"
"gh_license": "N/A"
}
match = re.search(r'github\.com/([^/]+/[^/]+)', url)
@@ -236,7 +236,7 @@ async def get_github_activity(url: str) -> Dict:
lic_id = lic.get("spdx_id", "N/A") if isinstance(lic, dict) else "N/A"
return {
"gh_stars": data.get("stargazers_count", 0),
"gh_pushed": data.get("pushed_at", "N/A"),
"gh_pushed": data.get("pushed_at"),
"gh_license": lic_id
}
except Exception as e: