added test and makes new relevant tests run tests

2026-02-14 10:19:54 +00:00 · 2025-10-11 11:17:26 -07:00
parent 3f069125a2
commit a2a424b201
6 changed files with 338 additions and 29 deletions
--- a/.github/workflows/affected-tests.yml
+++ b/.github/workflows/affected-tests.yml
@@ -35,6 +35,18 @@ jobs:
      - name: Go Mod Download
        run: go mod download

+      # 1) Build and compile tests only (no execution)
+      - name: Build (compile-only)
+        run: go build ./...
+
+      - name: Compile tests (no execution)
+        shell: bash
+        run: |
+          set -euo pipefail
+          while read -r pkg; do
+            go test -c -o /dev/null "$pkg" || exit 1
+          done < <(go list ./...)
+
      - name: Get PR Base SHA
        id: pr-info
        env:
@@ -49,6 +61,7 @@ jobs:
          echo "BASE_SHA=${BASE_SHA}" >> "$GITHUB_OUTPUT"
          echo "Base SHA: ${BASE_SHA}"

+      # 2) Detect relevant unit packages and e2e tests
      - name: Compute affected packages
        id: affected
        env:
@@ -71,6 +84,22 @@ jobs:
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
          fi

+      - name: Compute affected e2e tests
+        id: affected_e2e
+        env:
+          BASE_SHA: ${{ steps.pr-info.outputs.BASE_SHA }}
+        run: |
+          set -euo pipefail
+          go run ./scripts/affected-packages.go -mode=suites -base "${BASE_SHA}" > /tmp/affected-e2e.txt
+          awk -F: '$1=="preflight"{print $2}' /tmp/affected-e2e.txt > /tmp/preflight-tests.txt
+          awk -F: '$1=="support-bundle"{print $2}' /tmp/affected-e2e.txt > /tmp/support-tests.txt
+          if [ -s /tmp/preflight-tests.txt ] || [ -s /tmp/support-tests.txt ]; then
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      # 3) Run filtered tests only
      - name: Run unit tests for affected packages
        if: steps.affected.outputs.has_changes == 'true'
        run: |
@@ -85,6 +114,28 @@ jobs:
            xargs -a /tmp/affected.txt go test -race -count=1 -v
          fi

+      - name: Run preflight e2e (filtered)
+        if: steps.affected_e2e.outputs.has_changes == 'true'
+        run: |
+          set -euo pipefail
+          if [ -s /tmp/preflight-tests.txt ]; then
+            regex="$(tr '\n' '|' < /tmp/preflight-tests.txt | sed 's/|$//')"
+            go test -v -count=1 ./test/e2e/preflight -run "^((${regex}))$"
+          else
+            echo "No preflight e2e changes"
+          fi
+
+      - name: Run support-bundle e2e (filtered)
+        if: steps.affected_e2e.outputs.has_changes == 'true'
+        run: |
+          set -euo pipefail
+          if [ -s /tmp/support-tests.txt ]; then
+            regex="$(tr '\n' '|' < /tmp/support-tests.txt | sed 's/|$//')"
+            go test -v -count=1 ./test/e2e/support-bundle -run "^((${regex}))$"
+          else
+            echo "No support-bundle e2e changes"
+          fi
+
      - name: No affected packages — skip tests
        if: steps.affected.outputs.has_changes != 'true'
        run: echo "No Go packages affected by this PR; skipping tests."
--- a/.github/workflows/build-test-deploy.yaml
+++ b/.github/workflows/build-test-deploy.yaml
@@ -37,7 +37,8 @@ jobs:
      - run: make tidy-diff

  test-integration:
-    runs-on: ubuntu-latest
+  if: github.event_name == 'push'
+  runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v5
      - uses: actions/setup-go@v6
@@ -65,7 +66,8 @@ jobs:
          path: bin/preflight

  validate-preflight-e2e:
-    runs-on: ubuntu-latest
+  if: github.event_name == 'push'
+  runs-on: ubuntu-latest
    needs: compile-preflight
    steps:
      - uses: actions/checkout@v5
@@ -95,7 +97,8 @@ jobs:
          path: bin/support-bundle

  validate-supportbundle-e2e:
-    runs-on: ubuntu-latest
+  if: github.event_name == 'push'
+  runs-on: ubuntu-latest
    needs: compile-supportbundle
    steps:
      - uses: actions/checkout@v5
@@ -113,7 +116,8 @@ jobs:

  # Additional e2e tests for support bundle that run in Go, these create a Kind cluster
  validate-supportbundle-e2e-go:
-     runs-on: ubuntu-latest
+   if: github.event_name == 'push'
+   runs-on: ubuntu-latest
     needs: compile-supportbundle
     steps:
      - uses: actions/checkout@v5
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -66,7 +66,7 @@ jobs:

  # Unit and integration tests
  test:
-    if: needs.changes.outputs.go-files == 'true'
+    if: github.event_name == 'push' && needs.changes.outputs.go-files == 'true'
    needs: [changes, lint]
    runs-on: ubuntu-latest
    timeout-minutes: 20
@@ -100,7 +100,7 @@ jobs:

  # E2E tests
  e2e:
-    if: needs.changes.outputs.go-files == 'true' || github.event_name == 'push'
+    if: github.event_name == 'push'
    needs: [changes, build]
    runs-on: ubuntu-latest
    timeout-minutes: 15
--- a/.github/workflows/regression-test.yaml
+++ b/.github/workflows/regression-test.yaml
@@ -3,8 +3,6 @@ name: Regression Test Suite
 on:
  push:
    branches: [main, v1beta3]
-  pull_request:
-    types: [opened, synchronize, reopened]
  workflow_dispatch:
    inputs:
      update_baselines:
@@ -14,6 +12,7 @@ on:

 jobs:
  regression-test:
+    if: github.event_name != 'pull_request'
    runs-on: ubuntu-22.04
    timeout-minutes: 25

--- a/scripts/affected-packages.go
+++ b/scripts/affected-packages.go
@@ -8,9 +8,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -185,6 +187,44 @@ func computeAffectedPackages(directPkgs map[string]struct{}) (map[string]struct{
 	return affected, nil
 }

+// listTestFunctions scans a directory for Go test files and returns names of functions
+// that match the pattern `func TestXxx(t *testing.T)`.
+func listTestFunctions(dir string) ([]string, error) {
+	var tests []string
+	// Regex to capture test function names. This is a simple heuristic suitable for our codebase.
+	testFuncRe := regexp.MustCompile(`^func\s+(Test[\w\d_]+)\s*\(`)
+
+	walkFn := func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if !strings.HasSuffix(d.Name(), "_test.go") {
+			return nil
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		scanner := bufio.NewScanner(bytes.NewReader(b))
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if m := testFuncRe.FindStringSubmatch(line); m != nil {
+				tests = append(tests, m[1])
+			}
+		}
+		return scanner.Err()
+	}
+
+	if err := filepath.WalkDir(dir, walkFn); err != nil {
+		return nil, err
+	}
+	sort.Strings(tests)
+	return tests, nil
+}
+
 func main() {
 	baseRef := flag.String("base", "origin/main", "Git base ref to diff against (e.g., origin/main)")
 	printAllOnChanges := flag.Bool("all-on-mod-change", true, "Run all tests if go.mod or go.sum changed")
@@ -241,27 +281,22 @@ func main() {
 		}
 	}

-	// If module files changed, be conservative.
+	// Track module change to drive conservative behavior.
+	moduleChanged := false
 	if *printAllOnChanges {
 		for _, f := range files {
 			if f == "go.mod" || f == "go.sum" {
-				if *mode == "packages" {
-					if *verbose {
-						fmt.Fprintln(os.Stderr, "Detected module file change (go.mod/go.sum); selecting all packages ./...")
-					}
-					fmt.Println("./...")
-					return
-				}
-				if *mode == "suites" {
-					if *verbose {
-						fmt.Fprintln(os.Stderr, "Detected module file change (go.mod/go.sum); selecting all e2e suites")
-					}
-					fmt.Println("preflight")
-					fmt.Println("support-bundle")
-					return
-				}
+				moduleChanged = true
+				break
 			}
 		}
+		if moduleChanged && *mode == "packages" {
+			if *verbose {
+				fmt.Fprintln(os.Stderr, "Detected module file change (go.mod/go.sum); selecting all packages ./...")
+			}
+			fmt.Println("./...")
+			return
+		}
 	}

 	directPkgs, err := mapFilesToPackages(files)
@@ -317,7 +352,7 @@ func main() {
 			fmt.Println(p)
 		}
 	case "suites":
-		// Determine if preflight and/or support-bundle regression suites should run
+		// Determine impacted suites by dependency mapping, then print exact test names for those suites.
 		preflightRoot := "github.com/replicatedhq/troubleshoot/cmd/preflight"
 		supportRoot := "github.com/replicatedhq/troubleshoot/cmd/troubleshoot"

@@ -354,11 +389,50 @@ func main() {
 			fmt.Fprintf(os.Stderr, "  preflight: %v\n", preflightHit)
 			fmt.Fprintf(os.Stderr, "  support-bundle: %v\n", supportHit)
 		}
-		if preflightHit {
-			fmt.Println("preflight")
+
+		// If module files changed, conservatively select all tests for both suites.
+		if moduleChanged {
+			preTests, err := listTestFunctions("test/e2e/preflight")
+			if err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(2)
+			}
+			for _, tname := range preTests {
+				fmt.Printf("preflight:%s\n", tname)
+			}
+			sbTests, err := listTestFunctions("test/e2e/support-bundle")
+			if err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(2)
+			}
+			for _, tname := range sbTests {
+				fmt.Printf("support-bundle:%s\n", tname)
+			}
+			return
 		}
-		if supportHit {
-			fmt.Println("support-bundle")
+
+		// Collect tests for impacted suites and print as `<suite>:<TestName>`
+		if preflightHit || supportHit {
+			if preflightHit {
+				preTests, err := listTestFunctions("test/e2e/preflight")
+				if err != nil {
+					fmt.Fprintln(os.Stderr, err)
+					os.Exit(2)
+				}
+				for _, tname := range preTests {
+					fmt.Printf("preflight:%s\n", tname)
+				}
+			}
+			if supportHit {
+				sbTests, err := listTestFunctions("test/e2e/support-bundle")
+				if err != nil {
+					fmt.Fprintln(os.Stderr, err)
+					os.Exit(2)
+				}
+				for _, tname := range sbTests {
+					fmt.Printf("support-bundle:%s\n", tname)
+				}
+			}
 		}
 	default:
 		fmt.Fprintln(os.Stderr, "unknown mode; use 'packages' or 'suites'")
--- a/test-affected-detection.sh
+++ b/test-affected-detection.sh
@@ -0,0 +1,181 @@
+#!/bin/bash
+# Comprehensive test for affected test detection
+# Tests various code change scenarios to ensure correct suite detection
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+TESTS_PASSED=0
+TESTS_FAILED=0
+
+echo "========================================"
+echo "Affected Test Detection Validation"
+echo "========================================"
+echo ""
+
+# Helper function to run test
+run_test() {
+    local test_name="$1"
+    local test_file="$2"
+    local expected_suites="$3"
+
+    echo -e "${BLUE}Test: $test_name${NC}"
+    echo "File: $test_file"
+    echo "Expected: $expected_suites"
+
+    # Get affected tests from explicit changed files (no git required); detector prints <suite>:<TestName>
+    local detector_output=$(go run ./scripts/affected-packages.go -mode=suites -changed-files "$test_file" 2>/dev/null)
+    # Derive suites from prefixes for comparison
+    local actual_suites=$(echo "$detector_output" | cut -d':' -f1 | grep -v '^$' | sort | uniq | tr '\n' ' ' | xargs)
+
+    # Compare results
+    if [ "$actual_suites" = "$expected_suites" ]; then
+        echo -e "${GREEN}✓ PASS${NC} - Got: $actual_suites"
+        if [ -n "$detector_output" ]; then
+            echo "Tests:" && echo "$detector_output" | sed 's/^/  - /'
+        fi
+        TESTS_PASSED=$((TESTS_PASSED + 1))
+    else
+        echo -e "${RED}✗ FAIL${NC} - Got: '$actual_suites', Expected: '$expected_suites'"
+        if [ -n "$detector_output" ]; then
+            echo "Tests:" && echo "$detector_output" | sed 's/^/  - /'
+        fi
+        TESTS_FAILED=$((TESTS_FAILED + 1))
+    fi
+    echo ""
+}
+
+# Test 1: Preflight-only package (should only trigger preflight)
+run_test "Preflight-only package change" \
+    "pkg/preflight/run.go" \
+    "preflight"
+
+# Test 2: Support-bundle-only package
+run_test "Support-bundle-only package change" \
+    "pkg/supportbundle/supportbundle.go" \
+    "support-bundle"
+
+# Test 3: Shared package - collect
+run_test "Shared package (collect) change" \
+    "pkg/collect/run.go" \
+    "preflight support-bundle"
+
+# Test 4: Shared package - analyze
+run_test "Shared package (analyze) change" \
+    "pkg/analyze/analyzer.go" \
+    "preflight support-bundle"
+
+# Test 5: Shared package - k8sutil
+run_test "Shared package (k8sutil) change" \
+    "pkg/k8sutil/config.go" \
+    "preflight support-bundle"
+
+# Test 6: Shared package - convert
+run_test "Shared package (convert) change" \
+    "pkg/convert/output.go" \
+    "preflight support-bundle"
+
+# Test 7: Shared package - redact (another shared one)
+run_test "Shared package (redact) change" \
+    "pkg/redact/redact.go" \
+    "preflight support-bundle"
+
+# Test 8: Preflight command (should only trigger preflight)
+run_test "Preflight command change" \
+    "cmd/preflight/main.go" \
+    "preflight"
+
+# Test 9: Support-bundle types (support-bundle only package)
+run_test "Support-bundle types change" \
+    "pkg/supportbundle/types/types.go" \
+    "support-bundle"
+
+# Test 10: Workflow file (should not trigger e2e)
+echo -e "${BLUE}Test: Workflow file change (should trigger nothing)${NC}"
+echo "File: .github/workflows/affected-tests.yml"
+echo "Expected: (no suites)"
+
+detector_output=$(go run ./scripts/affected-packages.go -mode=suites -changed-files ".github/workflows/affected-tests.yml" 2>/dev/null)
+actual_suites=$(echo "$detector_output" | cut -d':' -f1 | grep -v '^$' | sort | uniq | tr '\n' ' ' | xargs)
+
+if [ -z "$actual_suites" ]; then
+    echo -e "${GREEN}✓ PASS${NC} - No suites affected (as expected)"
+    TESTS_PASSED=$((TESTS_PASSED + 1))
+else
+    echo -e "${RED}✗ FAIL${NC} - Got: '$actual_suites', Expected: (empty)"
+    TESTS_FAILED=$((TESTS_FAILED + 1))
+fi
+echo ""
+
+# Test 11: go.mod change (should trigger all)
+echo -e "${BLUE}Test: go.mod change (should trigger all suites)${NC}"
+echo "File: go.mod"
+echo "Expected: preflight support-bundle"
+
+detector_output=$(go run ./scripts/affected-packages.go -mode=suites -changed-files "go.mod" 2>/dev/null)
+actual_suites=$(echo "$detector_output" | cut -d':' -f1 | grep -v '^$' | sort | uniq | tr '\n' ' ' | xargs)
+
+if [ "$actual_suites" = "preflight support-bundle" ]; then
+    echo -e "${GREEN}✓ PASS${NC} - Got: $actual_suites"
+    TESTS_PASSED=$((TESTS_PASSED + 1))
+else
+    echo -e "${RED}✗ FAIL${NC} - Got: '$actual_suites', Expected: 'preflight support-bundle'"
+    TESTS_FAILED=$((TESTS_FAILED + 1))
+fi
+echo ""
+
+# Test 12: Multiple files across different areas
+echo -e "${BLUE}Test: Multiple file changes (support-bundle + shared)${NC}"
+echo "Files: pkg/supportbundle/supportbundle.go + pkg/collect/run.go"
+echo "Expected: preflight support-bundle"
+
+detector_output=$(go run ./scripts/affected-packages.go -mode=suites -changed-files "pkg/supportbundle/supportbundle.go,pkg/collect/run.go" 2>/dev/null)
+actual_suites=$(echo "$detector_output" | cut -d':' -f1 | grep -v '^$' | sort | uniq | tr '\n' ' ' | xargs)
+
+if [ "$actual_suites" = "preflight support-bundle" ]; then
+    echo -e "${GREEN}✓ PASS${NC} - Got: $actual_suites"
+    TESTS_PASSED=$((TESTS_PASSED + 1))
+else
+    echo -e "${RED}✗ FAIL${NC} - Got: '$actual_suites', Expected: 'preflight support-bundle'"
+    TESTS_FAILED=$((TESTS_FAILED + 1))
+fi
+echo ""
+
+# Test 13: README change (should not trigger e2e)
+echo -e "${BLUE}Test: Documentation change (should trigger nothing)${NC}"
+echo "File: README.md"
+echo "Expected: (no suites)"
+
+detector_output=$(go run ./scripts/affected-packages.go -mode=suites -changed-files "README.md" 2>/dev/null)
+actual_suites=$(echo "$detector_output" | cut -d':' -f1 | grep -v '^$' | sort | uniq | tr '\n' ' ' | xargs)
+
+if [ -z "$actual_suites" ]; then
+    echo -e "${GREEN}✓ PASS${NC} - No suites affected (as expected)"
+    TESTS_PASSED=$((TESTS_PASSED + 1))
+else
+    echo -e "${RED}✗ FAIL${NC} - Got: '$actual_suites', Expected: (empty)"
+    TESTS_FAILED=$((TESTS_FAILED + 1))
+fi
+echo ""
+
+# Summary
+echo "========================================"
+echo -e "${GREEN}Tests Passed: $TESTS_PASSED${NC}"
+echo -e "${RED}Tests Failed: $TESTS_FAILED${NC}"
+echo "========================================"
+
+if [ $TESTS_FAILED -eq 0 ]; then
+    echo -e "${GREEN}✓ All tests passed!${NC}"
+    exit 0
+else
+    echo -e "${RED}✗ Some tests failed${NC}"
+    exit 1
+fi
+
+