diff --git a/.github/workflows/00-pr-scanner.yaml b/.github/workflows/00-pr-scanner.yaml index fd84a584..e3d5c737 100644 --- a/.github/workflows/00-pr-scanner.yaml +++ b/.github/workflows/00-pr-scanner.yaml @@ -5,15 +5,15 @@ on: pull_request: types: [opened, reopened, synchronize, ready_for_review] paths-ignore: - - '**.yaml' - - '**.yml' - - '**.md' - - '**.sh' - - 'website/*' - - 'examples/*' - - 'docs/*' - - 'build/*' - - '.github/*' + - "**.yaml" + - "**.yml" + - "**.md" + - "**.sh" + - "website/*" + - "examples/*" + - "docs/*" + - "build/*" + - ".github/*" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -44,3 +44,148 @@ jobs: CGO_ENABLED: 0 GO111MODULE: "" secrets: inherit + + wf-preparation: + name: secret-validator + runs-on: ubuntu-latest + outputs: + TEST_NAMES: ${{ steps.export_tests_to_env.outputs.TEST_NAMES }} + is-secret-set: ${{ steps.check-secret-set.outputs.is-secret-set }} + + steps: + - name: check if the necessary secrets are set in github secrets + id: check-secret-set + env: + CUSTOMER: ${{ secrets.CUSTOMER }} + USERNAME: ${{ secrets.USERNAME }} + PASSWORD: ${{ secrets.PASSWORD }} + CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }} + SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: "echo \"is-secret-set=${{ env.CUSTOMER != '' && env.USERNAME != '' && env.PASSWORD != '' && env.CLIENT_ID != '' && env.SECRET_KEY != '' && env.REGISTRY_USERNAME != '' && env.REGISTRY_PASSWORD != '' }}\" >> $GITHUB_OUTPUT\n" + + - id: export_tests_to_env + name: set test name + run: | + echo "TEST_NAMES=$input" >> $GITHUB_OUTPUT + env: + input: '[ + "scan_nsa", + "scan_mitre", + "scan_with_exceptions", + "scan_repository", + "scan_local_file", + "scan_local_glob_files", + "scan_local_list_of_files", + "scan_nsa_and_submit_to_backend", + "scan_mitre_and_submit_to_backend", + "scan_local_repository_and_submit_to_backend", + "scan_repository_from_url_and_submit_to_backend", + "scan_with_custom_framework", + "scan_customer_configuration", + "scan_compliance_score", + "scan_custom_framework_scanning_file_scope_testing", + "scan_custom_framework_scanning_cluster_scope_testing", + "scan_custom_framework_scanning_cluster_and_file_scope_testing" + ]' + + run-system-tests: + strategy: + fail-fast: false + matrix: + TEST: ${{ fromJson(needs.wf-preparation.outputs.TEST_NAMES) }} + needs: [wf-preparation, pr-scanner] + if: ${{ (needs.wf-preparation.outputs.is-secret-set == 'true') && (always() && (contains(needs.*.result, 'success') || contains(needs.*.result, 'skipped')) && !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled'))) }} + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - uses: actions/setup-go@v4 + name: Installing go + with: + go-version: "1.25" + + - uses: anchore/sbom-action/download-syft@v0 + name: Setup Syft + + - uses: goreleaser/goreleaser-action@v6 + name: Build + with: + distribution: goreleaser + version: latest + args: build --clean --snapshot --single-target + env: + RELEASE: "" + CLIENT: test + CGO_ENABLED: 0 + + - name: chmod +x + run: chmod +x -R ${PWD}/dist/cli_linux_amd64_v1/kubescape + + - name: Checkout systests repo + uses: actions/checkout@v4 + with: + repository: armosec/system-tests + path: system-tests + + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + cache: "pip" + + - name: create env + run: ./create_env.sh + working-directory: system-tests + + - name: Generate uuid + id: uuid + run: | + echo "RANDOM_UUID=$(uuidgen)" >> $GITHUB_OUTPUT + + - name: Create k8s Kind Cluster + id: kind-cluster-install + uses: helm/kind-action@v1.10.0 + with: + cluster_name: ${{ steps.uuid.outputs.RANDOM_UUID }} + + - name: run-tests-on-local-built-kubescape + env: + CUSTOMER: ${{ secrets.CUSTOMER }} + USERNAME: ${{ secrets.USERNAME }} + PASSWORD: ${{ secrets.PASSWORD }} + CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }} + SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + working-directory: system-tests + run: | + echo "Test history:" + echo " ${{ matrix.TEST }} " >/tmp/testhistory + cat /tmp/testhistory + source systests_python_env/bin/activate + + python3 systest-cli.py \ + -t ${{ matrix.TEST }} \ + -b production \ + -c CyberArmorTests \ + --duration 3 \ + --logger DEBUG \ + --kwargs kubescape=${GITHUB_WORKSPACE}/dist/cli_linux_amd64_v1/kubescape + + deactivate + + - name: Test Report + uses: mikepenz/action-junit-report@v5 + if: always() + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + report_paths: "system-tests/**/results_xml_format/**.xml" + commit: ${{github.event.workflow_run.head_sha}} diff --git a/build/goreleaser-post-e2e.sh b/build/goreleaser-post-e2e.sh index dabf3bc2..18881977 100644 --- a/build/goreleaser-post-e2e.sh +++ b/build/goreleaser-post-e2e.sh @@ -4,10 +4,10 @@ # # A small, robust POSIX shell script intended to be called from the goreleaser # `builds[].hooks.post` entry. It is responsible for optionally running the -# repository smoke/e2e tests against the artifact produced in `dist/`. +# repository smoke tests against the artifact produced in `dist/`. # # Usage: -# RUN_E2E=true -> enable running smoke/e2e tests +# RUN_E2E=true -> enable running smoke tests # E2E_FAIL_ON_ERROR=1 -> (default) treat test failures as fatal (exit non-zero) # E2E_FAIL_ON_ERROR=0 -> treat test failures as non-fatal (log, but exit 0) # @@ -59,12 +59,12 @@ log "E2E_FAIL_ON_ERROR=${E2E_FAIL_ON_ERROR}" # Only run on linux/amd64 to avoid running multiple times (once per build) # and to ensure we can run the binary on the current host (assuming host is amd64). if [ -n "${GOARCH:-}" ] && [ "${GOARCH}" != "amd64" ]; then - log "Skipping e2e/smoke tests for non-amd64 build (GOARCH=${GOARCH})." + log "Skipping smoke tests for non-amd64 build (GOARCH=${GOARCH})." exit 0 fi if ! is_true "${RUN_E2E}"; then - log "RUN_E2E is not enabled. Skipping e2e/smoke tests. (RUN_E2E=${RUN_E2E})" + log "RUN_E2E is not enabled. Skipping smoke tests. (RUN_E2E=${RUN_E2E})" exit 0 fi @@ -79,7 +79,7 @@ if [ -d "$REPO_ROOT/dist" ]; then fi if [ -z "$ART_PATH" ] || [ ! -f "$ART_PATH" ]; then - log "No kubescape artifact found in dist/ matching *linux_amd64*/kubescape. Skipping e2e/smoke tests." + log "No kubescape artifact found in dist/ matching *linux_amd64*/kubescape. Skipping smoke tests." # If we are supposed to run E2E, not finding the artifact is probably an error. if is_true "${E2E_FAIL_ON_ERROR}"; then log "E2E_FAIL_ON_ERROR enabled -> failing because artifact was not found." @@ -100,16 +100,6 @@ elif command -v python >/dev/null 2>&1; then PYTHON=python fi -# Prefer Python 3.9 for system-tests (matches historical CI workflow) -SYSTEST_PYTHON_BIN="" -if command -v python3.9 >/dev/null 2>&1; then - SYSTEST_PYTHON_BIN=python3.9 -fi - -# If you want system-tests to fall back instead of failing when 3.9 is missing, -# set SYSTEST_REQUIRE_PY39=0. -: "${SYSTEST_REQUIRE_PY39:=1}" - if [ -z "$PYTHON" ]; then log "python3 (or python) not found in PATH." if is_true "${E2E_FAIL_ON_ERROR}"; then @@ -143,10 +133,10 @@ rc=$? set -e if [ $rc -eq 0 ]; then - log "Smoke/e2e tests passed (exit code 0)." + log "Smoke tests passed (exit code 0)." fi -log "Smoke/e2e tests exited with code: $rc" +log "Smoke tests exited with code: $rc" gha_group_end if [ $rc -ne 0 ]; then @@ -158,201 +148,4 @@ if [ $rc -ne 0 ]; then fi fi -# ----------------------------------------------------------------------------- -# System Tests (replicating b-binary-build-and-e2e-tests.yaml) -# ----------------------------------------------------------------------------- - -log "Starting System Tests (armosec/system-tests)..." - -# Check if we have connectivity to a cluster -if ! command -v kubectl >/dev/null 2>&1; then - log "kubectl not found. Skipping system tests that require a cluster." -elif ! kubectl config current-context >/dev/null 2>&1; then - log "No active kubernetes context found (kubectl config current-context failed). Skipping system tests." -else - log "Kubernetes cluster connection verified." - - # Create a temporary directory for system tests - SYSTEST_DIR=$(mktemp -d) - log "Cloning system-tests into $SYSTEST_DIR" - - if git clone --depth 1 https://github.com/armosec/system-tests.git "$SYSTEST_DIR"; then - - # Save current directory to return later - PUSHED_DIR=$(pwd) - cd "$SYSTEST_DIR" - - # Setup Python Environment - log "Setting up system tests python environment..." - if [ -f "./create_env.sh" ]; then - # The script expects to run inside the dir - chmod +x ./create_env.sh - - # Require Python 3.9 by default (matches b-binary-build-and-e2e-tests.yaml) - if [ -z "${SYSTEST_PYTHON_BIN:-}" ]; then - if is_true "${SYSTEST_REQUIRE_PY39}"; then - log "python3.9 not found in PATH; refusing to run system-tests because other Python versions may fail (deps/tooling mismatch)." - log "Install python3.9 or set SYSTEST_REQUIRE_PY39=0 to allow fallback." - - # Honor E2E_FAIL_ON_ERROR: if enabled, fail the release; otherwise skip system tests. - if is_true "${E2E_FAIL_ON_ERROR}"; then - exit 4 - else - log "E2E_FAIL_ON_ERROR disabled -> skipping system tests due to missing python3.9." - cd "$PUSHED_DIR" - rm -rf "$SYSTEST_DIR" - exit 0 - fi - else - log "python3.9 not found in PATH; continuing with default python (may fail depending on deps)." - fi - else - log "Using ${SYSTEST_PYTHON_BIN} for system-tests environment creation" - export PYTHON="${SYSTEST_PYTHON_BIN}" - export PYTHON_BIN="${SYSTEST_PYTHON_BIN}" - export Python_BIN="${SYSTEST_PYTHON_BIN}" - fi - - ./create_env.sh >/dev/null 2>&1 || log "Warning: create_env.sh returned non-zero" - - # Activate the environment if it exists - if [ -f "systests_python_env/bin/activate" ]; then - # shellcheck disable=SC1091 - . "systests_python_env/bin/activate" - else - log "Warning: systests_python_env/bin/activate not found. Trying global python." - fi - else - log "create_env.sh not found. Attempting to use existing python environment." - fi - - # List of tests to run (from b-binary-build-and-e2e-tests.yaml defaults) - TESTS="scan_nsa \ -scan_mitre \ -scan_with_exceptions \ -scan_repository \ -scan_local_file \ -scan_local_glob_files \ -scan_local_list_of_files \ -scan_nsa_and_submit_to_backend \ -scan_mitre_and_submit_to_backend \ -scan_local_repository_and_submit_to_backend \ -scan_repository_from_url_and_submit_to_backend \ -scan_with_custom_framework \ -scan_customer_configuration \ -scan_compliance_score \ -scan_custom_framework_scanning_file_scope_testing \ -scan_custom_framework_scanning_cluster_scope_testing \ -scan_custom_framework_scanning_cluster_and_file_scope_testing" - - FAILURES=0 - - # Prefer the virtualenv interpreter; otherwise use python3.11 if available - if [ -x "systests_python_env/bin/python" ]; then - SYSTEST_PYTHON="systests_python_env/bin/python" - elif [ -n "${SYSTEST_PYTHON_BIN:-}" ]; then - SYSTEST_PYTHON="${SYSTEST_PYTHON_BIN}" - else - SYSTEST_PYTHON="python3" - fi - - SYSTEST_PY_VER="$($SYSTEST_PYTHON --version 2>/dev/null || true)" - log "System tests will run with: $SYSTEST_PYTHON ($SYSTEST_PY_VER)" - - # Abort if the venv ended up on Python 3.10+ when we're expecting 3.9 (matches historical CI) - if is_true "${SYSTEST_REQUIRE_PY39}"; then - case "$SYSTEST_PY_VER" in - "Python 3.10."*|"Python 3.11."*|"Python 3.12."*|"Python 3.13."*|"Python 3.14."*) - log "System-tests virtualenv was created with $SYSTEST_PY_VER; refusing to run (expected Python 3.9.x)." - log "Ensure python3.9 is available and that create_env.sh uses it." - - # Honor E2E_FAIL_ON_ERROR: if enabled, fail the release; otherwise skip system tests. - if is_true "${E2E_FAIL_ON_ERROR}"; then - exit 5 - else - log "E2E_FAIL_ON_ERROR disabled -> skipping system tests due to unexpected Python version." - cd "$PUSHED_DIR" - rm -rf "$SYSTEST_DIR" - exit 0 - fi - ;; - esac - fi - - # Where to store per-test logs (helps diagnosing failures in CI) - RESULTS_DIR="$SYSTEST_DIR/results" - mkdir -p "$RESULTS_DIR" - - # Run tests - for t in $TESTS; do - gha_group_start "System test: $t" - log "Running system test: $t" - - LOG_FILE="$RESULTS_DIR/${t}.log" - - set +e - # Note: We must pass the absolute path to kubescape binary - # Capture output to file and also print to console for live feedback. - $SYSTEST_PYTHON systest-cli.py \ - -t "$t" \ - -b production \ - -c CyberArmorTests \ - --duration 3 \ - --logger DEBUG \ - --kwargs kubescape="$ART_PATH" 2>&1 | tee "$LOG_FILE" - - t_rc=$? - set -e - - if [ $t_rc -ne 0 ]; then - log "Test $t FAILED. (log: $LOG_FILE)" - FAILURES=$((FAILURES + 1)) - else - log "Test $t PASSED. (log: $LOG_FILE)" - fi - gha_group_end - done - - # Copy JUnit XML results (if any) into a stable workspace path for reporting - # Old workflow used glob '**/results_xml_format/**.xml' - DEST_DIR="$REPO_ROOT/test-results/system-tests" - mkdir -p "$DEST_DIR" - - if find "$SYSTEST_DIR" -type f -path "*/results_xml_format/*.xml" -print -quit 2>/dev/null | grep -q .; then - gha_group_start "Collect system-tests JUnit XML" - log "Copying system-tests JUnit XML results into: $DEST_DIR" - # Preserve directory structure under results_xml_format to avoid filename collisions - find "$SYSTEST_DIR" -type f -path "*/results_xml_format/*.xml" -print0 2>/dev/null | while IFS= read -r -d '' f; do - rel="${f#"$SYSTEST_DIR"/}" - out_dir="$DEST_DIR/$(dirname "$rel")" - mkdir -p "$out_dir" - cp -f "$f" "$out_dir/" - done - gha_group_end - else - log "No system-tests JUnit XML results found under results_xml_format/." - fi - - # Cleanup - # deactivate 2>/dev/null || true - cd "$PUSHED_DIR" - rm -rf "$SYSTEST_DIR" - - if [ $FAILURES -gt 0 ]; then - log "System tests completed with $FAILURES failures." - if is_true "${E2E_FAIL_ON_ERROR}"; then - exit 1 - fi - else - log "All system tests passed." - fi - - else - log "Failed to clone system-tests repo. Skipping." - if is_true "${E2E_FAIL_ON_ERROR}"; then - exit 1 - fi - fi -fi - exit 0