mirror of
https://github.com/kubescape/kubescape.git
synced 2026-02-14 09:59:54 +00:00
Refactor GitHub Actions workflow for system tests and update smoke test logging
Signed-off-by: Matthias Bertschy <matthias.bertschy@gmail.com>
This commit is contained in:
Matthias Bertschy
163
.github/workflows/00-pr-scanner.yaml
vendored
163
.github/workflows/00-pr-scanner.yaml
vendored
@@ -5,15 +5,15 @@ on:
|
||||
pull_request:
|
||||
types: [opened, reopened, synchronize, ready_for_review]
|
||||
paths-ignore:
|
||||
- '**.yaml'
|
||||
- '**.yml'
|
||||
- '**.md'
|
||||
- '**.sh'
|
||||
- 'website/*'
|
||||
- 'examples/*'
|
||||
- 'docs/*'
|
||||
- 'build/*'
|
||||
- '.github/*'
|
||||
- "**.yaml"
|
||||
- "**.yml"
|
||||
- "**.md"
|
||||
- "**.sh"
|
||||
- "website/*"
|
||||
- "examples/*"
|
||||
- "docs/*"
|
||||
- "build/*"
|
||||
- ".github/*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -44,3 +44,148 @@ jobs:
|
||||
CGO_ENABLED: 0
|
||||
GO111MODULE: ""
|
||||
secrets: inherit
|
||||
|
||||
wf-preparation:
|
||||
name: secret-validator
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
TEST_NAMES: ${{ steps.export_tests_to_env.outputs.TEST_NAMES }}
|
||||
is-secret-set: ${{ steps.check-secret-set.outputs.is-secret-set }}
|
||||
|
||||
steps:
|
||||
- name: check if the necessary secrets are set in github secrets
|
||||
id: check-secret-set
|
||||
env:
|
||||
CUSTOMER: ${{ secrets.CUSTOMER }}
|
||||
USERNAME: ${{ secrets.USERNAME }}
|
||||
PASSWORD: ${{ secrets.PASSWORD }}
|
||||
CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: "echo \"is-secret-set=${{ env.CUSTOMER != '' && env.USERNAME != '' && env.PASSWORD != '' && env.CLIENT_ID != '' && env.SECRET_KEY != '' && env.REGISTRY_USERNAME != '' && env.REGISTRY_PASSWORD != '' }}\" >> $GITHUB_OUTPUT\n"
|
||||
|
||||
- id: export_tests_to_env
|
||||
name: set test name
|
||||
run: |
|
||||
echo "TEST_NAMES=$input" >> $GITHUB_OUTPUT
|
||||
env:
|
||||
input: '[
|
||||
"scan_nsa",
|
||||
"scan_mitre",
|
||||
"scan_with_exceptions",
|
||||
"scan_repository",
|
||||
"scan_local_file",
|
||||
"scan_local_glob_files",
|
||||
"scan_local_list_of_files",
|
||||
"scan_nsa_and_submit_to_backend",
|
||||
"scan_mitre_and_submit_to_backend",
|
||||
"scan_local_repository_and_submit_to_backend",
|
||||
"scan_repository_from_url_and_submit_to_backend",
|
||||
"scan_with_custom_framework",
|
||||
"scan_customer_configuration",
|
||||
"scan_compliance_score",
|
||||
"scan_custom_framework_scanning_file_scope_testing",
|
||||
"scan_custom_framework_scanning_cluster_scope_testing",
|
||||
"scan_custom_framework_scanning_cluster_and_file_scope_testing"
|
||||
]'
|
||||
|
||||
run-system-tests:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
TEST: ${{ fromJson(needs.wf-preparation.outputs.TEST_NAMES) }}
|
||||
needs: [wf-preparation, pr-scanner]
|
||||
if: ${{ (needs.wf-preparation.outputs.is-secret-set == 'true') && (always() && (contains(needs.*.result, 'success') || contains(needs.*.result, 'skipped')) && !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled'))) }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: recursive
|
||||
|
||||
- uses: actions/setup-go@v4
|
||||
name: Installing go
|
||||
with:
|
||||
go-version: "1.25"
|
||||
|
||||
- uses: anchore/sbom-action/download-syft@v0
|
||||
name: Setup Syft
|
||||
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
name: Build
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: latest
|
||||
args: build --clean --snapshot --single-target
|
||||
env:
|
||||
RELEASE: ""
|
||||
CLIENT: test
|
||||
CGO_ENABLED: 0
|
||||
|
||||
- name: chmod +x
|
||||
run: chmod +x -R ${PWD}/dist/cli_linux_amd64_v1/kubescape
|
||||
|
||||
- name: Checkout systests repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: armosec/system-tests
|
||||
path: system-tests
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
cache: "pip"
|
||||
|
||||
- name: create env
|
||||
run: ./create_env.sh
|
||||
working-directory: system-tests
|
||||
|
||||
- name: Generate uuid
|
||||
id: uuid
|
||||
run: |
|
||||
echo "RANDOM_UUID=$(uuidgen)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Create k8s Kind Cluster
|
||||
id: kind-cluster-install
|
||||
uses: helm/kind-action@v1.10.0
|
||||
with:
|
||||
cluster_name: ${{ steps.uuid.outputs.RANDOM_UUID }}
|
||||
|
||||
- name: run-tests-on-local-built-kubescape
|
||||
env:
|
||||
CUSTOMER: ${{ secrets.CUSTOMER }}
|
||||
USERNAME: ${{ secrets.USERNAME }}
|
||||
PASSWORD: ${{ secrets.PASSWORD }}
|
||||
CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
working-directory: system-tests
|
||||
run: |
|
||||
echo "Test history:"
|
||||
echo " ${{ matrix.TEST }} " >/tmp/testhistory
|
||||
cat /tmp/testhistory
|
||||
source systests_python_env/bin/activate
|
||||
|
||||
python3 systest-cli.py \
|
||||
-t ${{ matrix.TEST }} \
|
||||
-b production \
|
||||
-c CyberArmorTests \
|
||||
--duration 3 \
|
||||
--logger DEBUG \
|
||||
--kwargs kubescape=${GITHUB_WORKSPACE}/dist/cli_linux_amd64_v1/kubescape
|
||||
|
||||
deactivate
|
||||
|
||||
- name: Test Report
|
||||
uses: mikepenz/action-junit-report@v5
|
||||
if: always()
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
report_paths: "system-tests/**/results_xml_format/**.xml"
|
||||
commit: ${{github.event.workflow_run.head_sha}}
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
#
|
||||
# A small, robust POSIX shell script intended to be called from the goreleaser
|
||||
# `builds[].hooks.post` entry. It is responsible for optionally running the
|
||||
# repository smoke/e2e tests against the artifact produced in `dist/`.
|
||||
# repository smoke tests against the artifact produced in `dist/`.
|
||||
#
|
||||
# Usage:
|
||||
# RUN_E2E=true -> enable running smoke/e2e tests
|
||||
# RUN_E2E=true -> enable running smoke tests
|
||||
# E2E_FAIL_ON_ERROR=1 -> (default) treat test failures as fatal (exit non-zero)
|
||||
# E2E_FAIL_ON_ERROR=0 -> treat test failures as non-fatal (log, but exit 0)
|
||||
#
|
||||
@@ -59,12 +59,12 @@ log "E2E_FAIL_ON_ERROR=${E2E_FAIL_ON_ERROR}"
|
||||
# Only run on linux/amd64 to avoid running multiple times (once per build)
|
||||
# and to ensure we can run the binary on the current host (assuming host is amd64).
|
||||
if [ -n "${GOARCH:-}" ] && [ "${GOARCH}" != "amd64" ]; then
|
||||
log "Skipping e2e/smoke tests for non-amd64 build (GOARCH=${GOARCH})."
|
||||
log "Skipping smoke tests for non-amd64 build (GOARCH=${GOARCH})."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! is_true "${RUN_E2E}"; then
|
||||
log "RUN_E2E is not enabled. Skipping e2e/smoke tests. (RUN_E2E=${RUN_E2E})"
|
||||
log "RUN_E2E is not enabled. Skipping smoke tests. (RUN_E2E=${RUN_E2E})"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -79,7 +79,7 @@ if [ -d "$REPO_ROOT/dist" ]; then
|
||||
fi
|
||||
|
||||
if [ -z "$ART_PATH" ] || [ ! -f "$ART_PATH" ]; then
|
||||
log "No kubescape artifact found in dist/ matching *linux_amd64*/kubescape. Skipping e2e/smoke tests."
|
||||
log "No kubescape artifact found in dist/ matching *linux_amd64*/kubescape. Skipping smoke tests."
|
||||
# If we are supposed to run E2E, not finding the artifact is probably an error.
|
||||
if is_true "${E2E_FAIL_ON_ERROR}"; then
|
||||
log "E2E_FAIL_ON_ERROR enabled -> failing because artifact was not found."
|
||||
@@ -100,16 +100,6 @@ elif command -v python >/dev/null 2>&1; then
|
||||
PYTHON=python
|
||||
fi
|
||||
|
||||
# Prefer Python 3.9 for system-tests (matches historical CI workflow)
|
||||
SYSTEST_PYTHON_BIN=""
|
||||
if command -v python3.9 >/dev/null 2>&1; then
|
||||
SYSTEST_PYTHON_BIN=python3.9
|
||||
fi
|
||||
|
||||
# If you want system-tests to fall back instead of failing when 3.9 is missing,
|
||||
# set SYSTEST_REQUIRE_PY39=0.
|
||||
: "${SYSTEST_REQUIRE_PY39:=1}"
|
||||
|
||||
if [ -z "$PYTHON" ]; then
|
||||
log "python3 (or python) not found in PATH."
|
||||
if is_true "${E2E_FAIL_ON_ERROR}"; then
|
||||
@@ -143,10 +133,10 @@ rc=$?
|
||||
set -e
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
log "Smoke/e2e tests passed (exit code 0)."
|
||||
log "Smoke tests passed (exit code 0)."
|
||||
fi
|
||||
|
||||
log "Smoke/e2e tests exited with code: $rc"
|
||||
log "Smoke tests exited with code: $rc"
|
||||
gha_group_end
|
||||
|
||||
if [ $rc -ne 0 ]; then
|
||||
@@ -158,201 +148,4 @@ if [ $rc -ne 0 ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# System Tests (replicating b-binary-build-and-e2e-tests.yaml)
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
log "Starting System Tests (armosec/system-tests)..."
|
||||
|
||||
# Check if we have connectivity to a cluster
|
||||
if ! command -v kubectl >/dev/null 2>&1; then
|
||||
log "kubectl not found. Skipping system tests that require a cluster."
|
||||
elif ! kubectl config current-context >/dev/null 2>&1; then
|
||||
log "No active kubernetes context found (kubectl config current-context failed). Skipping system tests."
|
||||
else
|
||||
log "Kubernetes cluster connection verified."
|
||||
|
||||
# Create a temporary directory for system tests
|
||||
SYSTEST_DIR=$(mktemp -d)
|
||||
log "Cloning system-tests into $SYSTEST_DIR"
|
||||
|
||||
if git clone --depth 1 https://github.com/armosec/system-tests.git "$SYSTEST_DIR"; then
|
||||
|
||||
# Save current directory to return later
|
||||
PUSHED_DIR=$(pwd)
|
||||
cd "$SYSTEST_DIR"
|
||||
|
||||
# Setup Python Environment
|
||||
log "Setting up system tests python environment..."
|
||||
if [ -f "./create_env.sh" ]; then
|
||||
# The script expects to run inside the dir
|
||||
chmod +x ./create_env.sh
|
||||
|
||||
# Require Python 3.9 by default (matches b-binary-build-and-e2e-tests.yaml)
|
||||
if [ -z "${SYSTEST_PYTHON_BIN:-}" ]; then
|
||||
if is_true "${SYSTEST_REQUIRE_PY39}"; then
|
||||
log "python3.9 not found in PATH; refusing to run system-tests because other Python versions may fail (deps/tooling mismatch)."
|
||||
log "Install python3.9 or set SYSTEST_REQUIRE_PY39=0 to allow fallback."
|
||||
|
||||
# Honor E2E_FAIL_ON_ERROR: if enabled, fail the release; otherwise skip system tests.
|
||||
if is_true "${E2E_FAIL_ON_ERROR}"; then
|
||||
exit 4
|
||||
else
|
||||
log "E2E_FAIL_ON_ERROR disabled -> skipping system tests due to missing python3.9."
|
||||
cd "$PUSHED_DIR"
|
||||
rm -rf "$SYSTEST_DIR"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
log "python3.9 not found in PATH; continuing with default python (may fail depending on deps)."
|
||||
fi
|
||||
else
|
||||
log "Using ${SYSTEST_PYTHON_BIN} for system-tests environment creation"
|
||||
export PYTHON="${SYSTEST_PYTHON_BIN}"
|
||||
export PYTHON_BIN="${SYSTEST_PYTHON_BIN}"
|
||||
export Python_BIN="${SYSTEST_PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
./create_env.sh >/dev/null 2>&1 || log "Warning: create_env.sh returned non-zero"
|
||||
|
||||
# Activate the environment if it exists
|
||||
if [ -f "systests_python_env/bin/activate" ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. "systests_python_env/bin/activate"
|
||||
else
|
||||
log "Warning: systests_python_env/bin/activate not found. Trying global python."
|
||||
fi
|
||||
else
|
||||
log "create_env.sh not found. Attempting to use existing python environment."
|
||||
fi
|
||||
|
||||
# List of tests to run (from b-binary-build-and-e2e-tests.yaml defaults)
|
||||
TESTS="scan_nsa \
|
||||
scan_mitre \
|
||||
scan_with_exceptions \
|
||||
scan_repository \
|
||||
scan_local_file \
|
||||
scan_local_glob_files \
|
||||
scan_local_list_of_files \
|
||||
scan_nsa_and_submit_to_backend \
|
||||
scan_mitre_and_submit_to_backend \
|
||||
scan_local_repository_and_submit_to_backend \
|
||||
scan_repository_from_url_and_submit_to_backend \
|
||||
scan_with_custom_framework \
|
||||
scan_customer_configuration \
|
||||
scan_compliance_score \
|
||||
scan_custom_framework_scanning_file_scope_testing \
|
||||
scan_custom_framework_scanning_cluster_scope_testing \
|
||||
scan_custom_framework_scanning_cluster_and_file_scope_testing"
|
||||
|
||||
FAILURES=0
|
||||
|
||||
# Prefer the virtualenv interpreter; otherwise use python3.11 if available
|
||||
if [ -x "systests_python_env/bin/python" ]; then
|
||||
SYSTEST_PYTHON="systests_python_env/bin/python"
|
||||
elif [ -n "${SYSTEST_PYTHON_BIN:-}" ]; then
|
||||
SYSTEST_PYTHON="${SYSTEST_PYTHON_BIN}"
|
||||
else
|
||||
SYSTEST_PYTHON="python3"
|
||||
fi
|
||||
|
||||
SYSTEST_PY_VER="$($SYSTEST_PYTHON --version 2>/dev/null || true)"
|
||||
log "System tests will run with: $SYSTEST_PYTHON ($SYSTEST_PY_VER)"
|
||||
|
||||
# Abort if the venv ended up on Python 3.10+ when we're expecting 3.9 (matches historical CI)
|
||||
if is_true "${SYSTEST_REQUIRE_PY39}"; then
|
||||
case "$SYSTEST_PY_VER" in
|
||||
"Python 3.10."*|"Python 3.11."*|"Python 3.12."*|"Python 3.13."*|"Python 3.14."*)
|
||||
log "System-tests virtualenv was created with $SYSTEST_PY_VER; refusing to run (expected Python 3.9.x)."
|
||||
log "Ensure python3.9 is available and that create_env.sh uses it."
|
||||
|
||||
# Honor E2E_FAIL_ON_ERROR: if enabled, fail the release; otherwise skip system tests.
|
||||
if is_true "${E2E_FAIL_ON_ERROR}"; then
|
||||
exit 5
|
||||
else
|
||||
log "E2E_FAIL_ON_ERROR disabled -> skipping system tests due to unexpected Python version."
|
||||
cd "$PUSHED_DIR"
|
||||
rm -rf "$SYSTEST_DIR"
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Where to store per-test logs (helps diagnosing failures in CI)
|
||||
RESULTS_DIR="$SYSTEST_DIR/results"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
# Run tests
|
||||
for t in $TESTS; do
|
||||
gha_group_start "System test: $t"
|
||||
log "Running system test: $t"
|
||||
|
||||
LOG_FILE="$RESULTS_DIR/${t}.log"
|
||||
|
||||
set +e
|
||||
# Note: We must pass the absolute path to kubescape binary
|
||||
# Capture output to file and also print to console for live feedback.
|
||||
$SYSTEST_PYTHON systest-cli.py \
|
||||
-t "$t" \
|
||||
-b production \
|
||||
-c CyberArmorTests \
|
||||
--duration 3 \
|
||||
--logger DEBUG \
|
||||
--kwargs kubescape="$ART_PATH" 2>&1 | tee "$LOG_FILE"
|
||||
|
||||
t_rc=$?
|
||||
set -e
|
||||
|
||||
if [ $t_rc -ne 0 ]; then
|
||||
log "Test $t FAILED. (log: $LOG_FILE)"
|
||||
FAILURES=$((FAILURES + 1))
|
||||
else
|
||||
log "Test $t PASSED. (log: $LOG_FILE)"
|
||||
fi
|
||||
gha_group_end
|
||||
done
|
||||
|
||||
# Copy JUnit XML results (if any) into a stable workspace path for reporting
|
||||
# Old workflow used glob '**/results_xml_format/**.xml'
|
||||
DEST_DIR="$REPO_ROOT/test-results/system-tests"
|
||||
mkdir -p "$DEST_DIR"
|
||||
|
||||
if find "$SYSTEST_DIR" -type f -path "*/results_xml_format/*.xml" -print -quit 2>/dev/null | grep -q .; then
|
||||
gha_group_start "Collect system-tests JUnit XML"
|
||||
log "Copying system-tests JUnit XML results into: $DEST_DIR"
|
||||
# Preserve directory structure under results_xml_format to avoid filename collisions
|
||||
find "$SYSTEST_DIR" -type f -path "*/results_xml_format/*.xml" -print0 2>/dev/null | while IFS= read -r -d '' f; do
|
||||
rel="${f#"$SYSTEST_DIR"/}"
|
||||
out_dir="$DEST_DIR/$(dirname "$rel")"
|
||||
mkdir -p "$out_dir"
|
||||
cp -f "$f" "$out_dir/"
|
||||
done
|
||||
gha_group_end
|
||||
else
|
||||
log "No system-tests JUnit XML results found under results_xml_format/."
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
# deactivate 2>/dev/null || true
|
||||
cd "$PUSHED_DIR"
|
||||
rm -rf "$SYSTEST_DIR"
|
||||
|
||||
if [ $FAILURES -gt 0 ]; then
|
||||
log "System tests completed with $FAILURES failures."
|
||||
if is_true "${E2E_FAIL_ON_ERROR}"; then
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
log "All system tests passed."
|
||||
fi
|
||||
|
||||
else
|
||||
log "Failed to clone system-tests repo. Skipping."
|
||||
if is_true "${E2E_FAIL_ON_ERROR}"; then
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user