diff --git a/.github/workflows/agentic_v2_videos.yml b/.github/workflows/agentic_v2_videos.yml index 7ca9dace..4bd599a7 100644 --- a/.github/workflows/agentic_v2_videos.yml +++ b/.github/workflows/agentic_v2_videos.yml @@ -32,6 +32,14 @@ jobs: python-version: '3.11' cache: 'pip' + - name: Restore Incremental Inventory Cache + uses: actions/cache/restore@v4 + with: + path: data/inventory.yaml + key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + inventory-v2- + - name: Dependency Installation run: | python -m pip install --upgrade pip @@ -61,3 +69,10 @@ jobs: git pull --rebase origin develop git push origin develop fi + + - name: Persist Incremental Inventory Cache + if: always() + uses: actions/cache/save@v4 + with: + path: data/inventory.yaml + key: inventory-v2-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/src/enrich_videos.py b/src/enrich_videos.py index 878dcf4a..b90edf90 100644 --- a/src/enrich_videos.py +++ b/src/enrich_videos.py @@ -88,10 +88,14 @@ async def main(): for i in range(0, len(tasks), batch_size): batch = tasks[i:i+batch_size] await asyncio.gather(*batch) - await asyncio.sleep(2) # Safety delay - - with open(INVENTORY_PATH, "w") as f: - yaml.dump(inventory, f, sort_keys=False, allow_unicode=True) + + # Incremental Persistence: Save after each batch + with open(INVENTORY_PATH, "w") as f: + yaml.dump(inventory, f, sort_keys=False, allow_unicode=True) + log_event(f" [💾] Saved progress: {min(i + batch_size, len(tasks))}/{len(tasks)} videos.") + + if i + batch_size < len(tasks): + await asyncio.sleep(2) # Safety delay log_event("✅ Video Hub Enrichment Complete.")