name: 01.1. Automated Agentic Curation

# Last Sync: 2026-05-15 - Triggering Refresh
on:
  schedule:
    - cron: '0 5 1 * *'  # Monthly: 1st day of the month at 05:00 UTC
  workflow_dispatch:
    inputs:
      start_date:
        description: 'Start date (YYYY-MM-DD). Default: 2026-05-15'
        required: true
        default: '2026-05-15'
      days_back:
        description: 'Last N days (e.g. 7). Overrides start date if filled'
        required: false
        default: ''
      exclude_accounts:
        description: 'Accounts to exclude (comma separated, e.g. nubenetes,OpenAI)'
        required: false
        default: ''
      include_k8s:
        description: 'Include Kubernetes & Cloud Native'
        type: boolean
        default: true
      include_cloud:
        description: 'Include Cloud Providers (AWS/Azure/GCP)'
        type: boolean
        default: true
      include_ai:
        description: 'Include AI & Agentic Systems'
        type: boolean
        default: true
      include_dev:
        description: 'Include Developer Productivity & AI Agents'
        type: boolean
        default: true
      include_data:
        description: 'Include Data & Big Data'
        type: boolean
        default: true
      include_iac:
        description: 'Include Infrastructure as Code & GitOps'
        type: boolean
        default: true
      extraction_strategy:
        description: 'Extraction Strategy'
        required: true
        default: 'search'
        type: choice
        options:
          - search
          - scroll
      historical_mode:
        description: 'Activate Historical Mode'
        required: false
        default: 'true'
        type: boolean
      historical_chunked:
        description: 'Chunk execution (multiple PRs)'
        required: false
        default: 'false'
        type: boolean
      historical_until_date:
        description: 'Upper limit date (chunk)'
        required: false
        default: ''
      activate_backup_key:
        description: 'Activate Identity B (Subscription) as backup/rotation'
        type: boolean
        default: false

permissions:
  contents: write
  pull-requests: write
  actions: write
  issues: write

concurrency:
  group: curation-${{ github.ref }}
  cancel-in-progress: true

jobs:
  agentic-curation-process:
    runs-on: ubuntu-latest
    # This job can now be triggered from any branch but will always checkout 'develop'
    steps:
      - name: Repository Synchronization
        uses: actions/checkout@v7
        with:
          ref: develop

      - name: Python 3.11 Environment Provisioning
        uses: actions/setup-python@v6
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Installation of Dependencies
        run: |
          python -m pip install --upgrade pip
          pip install --no-cache-dir pydantic PyGithub aiohttp beautifulsoup4 httpx fake-useragent pytz python-dotenv twikit>=2.1.2 playwright playwright-stealth pyyaml tenacity feedparser

      - name: Cache Playwright Binaries
        uses: actions/cache@v6
        id: playwright-cache
        with:
          path: ~/.cache/ms-playwright
          key: ${{ runner.os }}-playwright-${{ hashFiles('**/requirements.txt') }}

      - name: Install Playwright Browsers
        if: steps.playwright-cache.outputs.cache-hit != 'true'
        run: playwright install chromium --with-deps

      - name: Restore Incremental Inventory Cache
        uses: actions/cache/restore@v6
        with:
          path: data/inventory.yaml
          key: inventory-curation-${{ github.run_id }}-${{ github.run_attempt }}
          restore-keys: |
            inventory-curation-

      - name: Workflow UI Synchronization Check (Mandate 11)
        env:
          PYTHONPATH: .
        run: |
          python src/sync_workflow_ui.py

      - name: Global Agentic Curation Execution
        id: run_curation
        env:
          TWITTER_USERNAME: ${{ secrets.TWITTER_USERNAME }}
          TWITTER_EMAIL: ${{ secrets.TWITTER_EMAIL }}
          TWITTER_PASSWORD: ${{ secrets.TWITTER_PASSWORD }}
          TWITTER_COOKIES: ${{ secrets.TWITTER_COOKIES }}
          GEMINI_API_KEY_1: ${{ secrets.GEMINI_API_KEY_1 }}
          YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

          ACTIVATE_BACKUP_KEY: ${{ github.event.inputs.activate_backup_key || 'false' }}
          EXTRACTION_STRATEGY: ${{ github.event.inputs.extraction_strategy || 'search' }}
          HISTORICAL_MODE: ${{ github.event.inputs.historical_mode || 'false' }}
          HISTORICAL_CHUNKED: ${{ github.event.inputs.historical_chunked || 'false' }}
          HISTORICAL_UNTIL_DATE: ${{ github.event.inputs.historical_until_date || '' }}
          CURATION_START_DATE: ${{ github.event.inputs.start_date || '2026-05-15' }}
          CURATION_DAYS_BACK: ${{ github.event.inputs.days_back || '' }}
          EXCLUDE_ACCOUNTS: ${{ github.event.inputs.exclude_accounts || '' }}
          INCLUDE_K8S: ${{ github.event.inputs.include_k8s || 'true' }}
          INCLUDE_CLOUD: ${{ github.event.inputs.include_cloud || 'true' }}
          INCLUDE_AI: ${{ github.event.inputs.include_ai || 'true' }}
          INCLUDE_DEV: ${{ github.event.inputs.include_dev || 'true' }}
          INCLUDE_DATA: ${{ github.event.inputs.include_data || 'true' }}
          INCLUDE_IAC: ${{ github.event.inputs.include_iac || 'true' }}
          HISTORICAL_CHUNK_DAYS: '180'
          PYTHONPATH: .
          PYTHONUNBUFFERED: "1"
        run: |
          set +e
          python -u src/main.py 2>&1 | tee output.log
          EXIT_CODE=${PIPESTATUS[0]}
          set -e
          
          if [ $EXIT_CODE -eq 42 ]; then
            echo "🚨 CIRCUIT BREAKER TRIPPED: Disabling workflow to prevent quota drain."
            gh workflow disable 01.1.agentic_cron.yml
            exit 1
          elif [ $EXIT_CODE -ne 0 ]; then
            exit $EXIT_CODE
          fi
          
          # Capture PR URL if exists
          PR_URL=$(grep "PULL_REQUEST_URL:" output.log | awk '{print $2}')
          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
          
          # Re-trigger logic for Historical Mode
          if [ "${{ github.event.inputs.historical_chunked }}" == "true" ] && grep -q "NEXT_CHUNK_START:" output.log; then
            NEXT_DATE=$(grep "NEXT_CHUNK_START:" output.log | awk '{print $2}')
            echo "Triggering next historical chunk until: $NEXT_DATE"
            gh workflow run 01.1.agentic_cron.yml -f historical_mode=true -f historical_chunked=true -f historical_until_date=$NEXT_DATE
          fi

      - name: Run Deduplication Scan
        if: success()
        env:
          PYTHONPATH: .
        run: |
          python -u -c "import asyncio; from src.dedup import run_dedup; asyncio.run(run_dedup(dry_run=False))" || echo "Dedup scan skipped"

      - name: Run Enrichment Pipeline
        if: success()
        env:
          PYTHONPATH: .
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python -u -m src.enrichment || echo "Enrichment pipeline skipped"

      - name: Upload Visual Dashboard Artifact
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: curation-visual-report
          path: report.html
          retention-days: 7
          if-no-files-found: ignore

      - name: Create Notification Issue
        if: always()
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          PR_URL="${{ steps.run_curation.outputs.pr_url }}"
          STATUS="${{ job.status }}"
          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          
          if [ -z "$PR_URL" ]; then
            PR_URL="No PR created (no changes or error)"
          fi
          
          TITLE="Curation Report: $STATUS - $(date +'%Y-%m-%d')"
          BODY="### Nubenetes Automated Curation has finished.\n\n**Status:** $STATUS\n**Pull Request:** $PR_URL\n\nCheck the [workflow logs]($RUN_URL) for more details. If a visual report was generated, you can download it from the artifacts section of the workflow run."
          
          gh label create "curation-report" --color "0E8A16" --description "Automated curation run reports" || true
          gh issue create --title "$TITLE" --body "$BODY" --label "curation-report" || echo "Failed to create issue"

      - name: Persist Incremental Inventory to Cache
        if: always()
        uses: actions/cache/save@v6
        with:
          path: data/inventory.yaml
          key: inventory-curation-${{ github.run_id }}-${{ github.run_attempt }}