github/troubleshoot
1
0
Fork 0
mirror of https://github.com/replicatedhq/troubleshoot.git synced 2026-02-14 10:19:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(collect): cluster resource errors json file has wrong name (#1936)

Browse Source 
* fix(ci): regression test updates binary to latest release
* fix cluster resources collector
This commit is contained in:
Ethan Mosbaugh
2025-11-27 13:17:03 -08:00
committed by GitHub
parent e45e2cadd3
commit 9343b43e77
9 changed files with 326 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

399
.github/workflows/regression-test.yaml vendored
View File

@@ -13,29 +13,17 @@ on:
default: false
jobs:
regression-test:
# Build binaries once (shared by all test jobs)
build-binaries:
if: github.actor != 'dependabot[bot]'
runs-on: ubuntu-22.04
timeout-minutes: 25
timeout-minutes: 5
steps:
# 1. SETUP
- name: Checkout code
uses: actions/checkout@v5
with:
fetch-depth: 0 # Fetch all history for git describe to work
- name: Create output directory
run: mkdir -p test/output
- name: Create k3s cluster
id: k3s
uses: replicatedhq/action-k3s@main
with:
version: v1.31.2-k3s1
- name: Verify cluster access
run: kubectl get nodes -o wide
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v6
@@ -49,84 +37,83 @@ jobs:
./bin/preflight version
./bin/support-bundle version
- name: Upload binaries
uses: actions/upload-artifact@v5
with:
name: binaries-${{ github.run_id }}
path: |
bin/preflight
bin/support-bundle
retention-days: 1
# Preflight v1beta3 test (parallel job 1)
test-preflight-v1beta3:
needs: [build-binaries]
if: github.actor != 'dependabot[bot]'
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Create output directory
run: mkdir -p test/output
- name: Create k3s cluster
uses: replicatedhq/action-k3s@main
with:
version: v1.31.2-k3s1
- name: Verify cluster access
run: kubectl get nodes -o wide
- name: Wait for all pods to be ready
run: |
echo "Waiting for all pods to be running..."
kubectl get pods --all-namespaces
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
kubectl get pods --all-namespaces
- name: Download binaries
uses: actions/download-artifact@v5
with:
name: binaries-${{ github.run_id }}
path: bin/
- name: Make binaries executable
run: |
chmod +x bin/preflight bin/support-bundle
./bin/preflight version
- name: Setup Python for comparison
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install Python dependencies
run: |
pip install pyyaml deepdiff
run: pip install pyyaml deepdiff
# 2. EXECUTE SPECS (in parallel)
- name: Run all specs in parallel
- name: Run preflight v1beta3
continue-on-error: true
run: |
echo "Running all 3 specs in parallel..."
echo "Running preflight v1beta3..."
./bin/preflight \
examples/preflight/complex-v1beta3.yaml \
--values examples/preflight/values-complex-full.yaml \
--interactive=false \
--format=json \
--auto-update=false \
--output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true
# Run v1beta3 in background
(
echo "Starting preflight v1beta3..."
./bin/preflight \
examples/preflight/complex-v1beta3.yaml \
--values examples/preflight/values-complex-full.yaml \
--interactive=false \
--format=json \
--output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
if [ -n "$BUNDLE" ]; then
mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz
echo "✓ v1beta3 bundle saved"
fi
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
if [ -n "$BUNDLE" ]; then
mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz
echo "✓ v1beta3 bundle saved"
fi
) &
PID_V1BETA3=$!
# Run v1beta2 in background
(
echo "Starting preflight v1beta2..."
./bin/preflight \
examples/preflight/all-analyzers-v1beta2.yaml \
--interactive=false \
--format=json \
--output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
if [ -n "$BUNDLE" ]; then
mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz
echo "✓ v1beta2 bundle saved"
fi
) &
PID_V1BETA2=$!
# Run support bundle in background
(
echo "Starting support bundle..."
./bin/support-bundle \
examples/collect/host/all-kubernetes-collectors.yaml \
--interactive=false \
--output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true
if [ -f test/output/supportbundle.tar.gz ]; then
echo "✓ Support bundle saved"
fi
) &
PID_SUPPORTBUNDLE=$!
# Wait for all to complete
echo "Waiting for all specs to complete..."
wait $PID_V1BETA3
wait $PID_V1BETA2
wait $PID_SUPPORTBUNDLE
echo "All specs completed!"
# Verify bundles exist
ls -lh test/output/*.tar.gz || echo "Warning: Some bundles may be missing"
# 3. COMPARE BUNDLES
- name: Compare preflight v1beta3 bundle
id: compare-v1beta3
id: compare
continue-on-error: true
run: |
echo "Comparing v1beta3 preflight bundle against baseline..."
@@ -143,8 +130,93 @@ jobs:
--report test/output/diff-report-v1beta3.json \
--spec-type preflight
- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v5
with:
name: test-results-v1beta3-${{ github.run_id }}-${{ github.run_attempt }}
path: |
test/output/preflight-v1beta3-bundle.tar.gz
test/output/preflight-results-v1beta3.json
test/output/diff-report-v1beta3.json
test/output/v1beta3.log
retention-days: 30
- name: Set job outcome
if: always()
run: |
if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
echo "comparison_failed=true" >> $GITHUB_OUTPUT
exit 1
fi
# Preflight v1beta2 test (parallel job 2)
test-preflight-v1beta2:
needs: [build-binaries]
if: github.actor != 'dependabot[bot]'
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Create output directory
run: mkdir -p test/output
- name: Create k3s cluster
uses: replicatedhq/action-k3s@main
with:
version: v1.31.2-k3s1
- name: Verify cluster access
run: kubectl get nodes -o wide
- name: Wait for all pods to be ready
run: |
echo "Waiting for all pods to be running..."
kubectl get pods --all-namespaces
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
kubectl get pods --all-namespaces
- name: Download binaries
uses: actions/download-artifact@v5
with:
name: binaries-${{ github.run_id }}
path: bin/
- name: Make binaries executable
run: |
chmod +x bin/preflight bin/support-bundle
./bin/preflight version
- name: Setup Python for comparison
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install Python dependencies
run: pip install pyyaml deepdiff
- name: Run preflight v1beta2
continue-on-error: true
run: |
echo "Running preflight v1beta2..."
./bin/preflight \
examples/preflight/all-analyzers-v1beta2.yaml \
--interactive=false \
--format=json \
--auto-update=false \
--output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true
BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
if [ -n "$BUNDLE" ]; then
mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz
echo "✓ v1beta2 bundle saved"
fi
- name: Compare preflight v1beta2 bundle
id: compare-v1beta2
id: compare
continue-on-error: true
run: |
echo "Comparing v1beta2 preflight bundle against baseline..."
@@ -161,8 +233,90 @@ jobs:
--report test/output/diff-report-v1beta2.json \
--spec-type preflight
- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v5
with:
name: test-results-v1beta2-${{ github.run_id }}-${{ github.run_attempt }}
path: |
test/output/preflight-v1beta2-bundle.tar.gz
test/output/preflight-results-v1beta2.json
test/output/diff-report-v1beta2.json
test/output/v1beta2.log
retention-days: 30
- name: Set job outcome
if: always()
run: |
if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
echo "comparison_failed=true" >> $GITHUB_OUTPUT
exit 1
fi
# Support bundle test (parallel job 3)
test-supportbundle:
needs: [build-binaries]
if: github.actor != 'dependabot[bot]'
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Create output directory
run: mkdir -p test/output
- name: Create k3s cluster
uses: replicatedhq/action-k3s@main
with:
version: v1.31.2-k3s1
- name: Verify cluster access
run: kubectl get nodes -o wide
- name: Wait for all pods to be ready
run: |
echo "Waiting for all pods to be running..."
kubectl get pods --all-namespaces
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
kubectl get pods --all-namespaces
- name: Download binaries
uses: actions/download-artifact@v5
with:
name: binaries-${{ github.run_id }}
path: bin/
- name: Make binaries executable
run: |
chmod +x bin/preflight bin/support-bundle
./bin/support-bundle version
- name: Setup Python for comparison
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install Python dependencies
run: pip install pyyaml deepdiff
- name: Run support bundle
continue-on-error: true
run: |
echo "Running support bundle..."
./bin/support-bundle \
examples/collect/host/all-kubernetes-collectors.yaml \
--interactive=false \
--auto-update=false \
--output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true
if [ -f test/output/supportbundle.tar.gz ]; then
echo "✓ Support bundle saved"
fi
- name: Compare support bundle
id: compare-supportbundle
id: compare
continue-on-error: true
run: |
echo "Comparing support bundle against baseline..."
@@ -179,44 +333,81 @@ jobs:
--report test/output/diff-report-supportbundle.json \
--spec-type supportbundle
# 4. REPORT RESULTS
- name: Generate summary report
- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v5
with:
name: test-results-supportbundle-${{ github.run_id }}-${{ github.run_attempt }}
path: |
test/output/supportbundle.tar.gz
test/output/diff-report-supportbundle.json
test/output/supportbundle.log
retention-days: 30
- name: Set job outcome
if: always()
run: |
if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
echo "comparison_failed=true" >> $GITHUB_OUTPUT
exit 1
fi
# Report results (runs after all tests complete)
report-results:
needs: [test-preflight-v1beta3, test-preflight-v1beta2, test-supportbundle]
if: always() && github.actor != 'dependabot[bot]'
runs-on: ubuntu-22.04
timeout-minutes: 5
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install Python dependencies
run: pip install pyyaml deepdiff
- name: Download all test artifacts
uses: actions/download-artifact@v5
with:
path: test/output
pattern: test-results-*
- name: Reorganize artifacts
run: |
# Move artifacts from nested directories to test/output
find test/output/test-results-* -type f -exec mv {} test/output/ \;
# Clean up empty directories
find test/output/test-results-* -type d -empty -delete || true
- name: Generate summary report
run: |
python3 scripts/generate_summary.py \
--reports test/output/diff-report-*.json \
--output-file $GITHUB_STEP_SUMMARY \
--output-console
- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v5
with:
name: regression-test-results-${{ github.run_id }}-${{ github.run_attempt }}
path: |
test/output/*.tar.gz
test/output/*.json
retention-days: 30
- name: Check for regressions
if: always()
run: |
echo "Checking comparison results..."
# Check if any comparisons failed
FAILURES=0
if [ "${{ steps.compare-v1beta3.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta3.outputs.baseline_missing }}" != "true" ]; then
if [ "${{ needs.test-preflight-v1beta3.result }}" == "failure" ]; then
echo "❌ v1beta3 comparison failed"
FAILURES=$((FAILURES + 1))
fi
if [ "${{ steps.compare-v1beta2.outcome }}" == "failure" ] && [ "${{ steps.compare-v1beta2.outputs.baseline_missing }}" != "true" ]; then
if [ "${{ needs.test-preflight-v1beta2.result }}" == "failure" ]; then
echo "❌ v1beta2 comparison failed"
FAILURES=$((FAILURES + 1))
fi
if [ "${{ steps.compare-supportbundle.outcome }}" == "failure" ] && [ "${{ steps.compare-supportbundle.outputs.baseline_missing }}" != "true" ]; then
if [ "${{ needs.test-supportbundle.result }}" == "failure" ]; then
echo "❌ Support bundle comparison failed"
FAILURES=$((FAILURES + 1))
fi
@@ -230,7 +421,6 @@ jobs:
echo "✅ All comparisons passed or skipped (no baseline)"
fi
# 5. UPDATE BASELINES (optional, manual trigger only)
- name: Update baselines
if: github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch'
run: |
@@ -260,7 +450,7 @@ jobs:
{
"updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
"git_sha": "${{ github.sha }}",
"k8s_version": "v1.28.3",
"k8s_version": "v1.31.2-k3s1",
"workflow_run": "${{ github.run_id }}"
}
EOF
@@ -271,6 +461,3 @@ jobs:
git add test/baselines/
git commit -m "chore: update regression test baselines from run ${{ github.run_id }}"
git push
# 6. CLEANUP
# Note: k3s cluster cleanup is handled automatically by the action

6
pkg/collect/cluster_resources.go
View File

@@ -232,7 +232,7 @@ func (c *CollectClusterResources) Collect(progressChan chan<- interface{}) (Coll
// replicasets
replicasets, replicasetsErrors := replicasets(ctx, client, namespaceNames)
for k, v := range replicasets {
output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, fmt.Sprintf("%s-errors.json", constants.CLUSTER_RESOURCES_STATEFULSETS), k), bytes.NewBuffer(v))
output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_REPLICASETS, k), bytes.NewBuffer(v))
}
output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, fmt.Sprintf("%s-errors.json", constants.CLUSTER_RESOURCES_REPLICASETS)), marshalErrors(replicasetsErrors))
@@ -370,9 +370,9 @@ func (c *CollectClusterResources) Collect(progressChan chan<- interface{}) (Coll
// endpointslices
endpointslices, endpointslicesErrors := endpointslices(ctx, client, namespaceNames)
for k, v := range endpointslices {
_ = output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_ENDPOINTSICES, k), bytes.NewBuffer(v))
_ = output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, constants.CLUSTER_RESOURCES_ENDPOINTSLICES, k), bytes.NewBuffer(v))
}
_ = output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, fmt.Sprintf("%s-errors.json", constants.CLUSTER_RESOURCES_ENDPOINTSICES)), marshalErrors(endpointslicesErrors))
_ = output.SaveResult(c.BundlePath, path.Join(constants.CLUSTER_RESOURCES_DIR, fmt.Sprintf("%s-errors.json", constants.CLUSTER_RESOURCES_ENDPOINTSLICES)), marshalErrors(endpointslicesErrors))
// Service Accounts
servicesAccounts, servicesAccountsErrors := serviceAccounts(ctx, client, namespaceNames)

2
pkg/constants/constants.go
View File

@@ -56,7 +56,7 @@ const (
CLUSTER_RESOURCES_CLUSTER_ROLE_BINDINGS = "clusterrolebindings"
CLUSTER_RESOURCES_PRIORITY_CLASS = "priorityclasses"
CLUSTER_RESOURCES_ENDPOINTS = "endpoints"
CLUSTER_RESOURCES_ENDPOINTSICES = "endpointslices"
CLUSTER_RESOURCES_ENDPOINTSLICES = "endpointslices"
CLUSTER_RESOURCES_SERVICE_ACCOUNTS = "serviceaccounts"
CLUSTER_RESOURCES_LEASES = "leases"
CLUSTER_RESOURCES_VOLUME_ATTACHMENTS = "volumeattachments"

42
scripts/update_baselines.sh
View File

@@ -48,14 +48,12 @@ trap "rm -rf $TEMP_DIR" EXIT
echo -e "\n${BLUE}Step 1: Downloading artifacts...${NC}"
# Download artifacts from the run
cd "$TEMP_DIR"
if ! gh run download "$RUN_ID" --name "regression-test-results-${RUN_ID}-1" 2>/dev/null; then
# Try without attempt suffix
if ! gh run download "$RUN_ID" 2>/dev/null; then
echo -e "${RED}Error: Failed to download artifacts from run ${RUN_ID}${NC}"
exit 1
fi
# Download all test-results artifacts from the run
# Try downloading all artifacts (will download test-results-v1beta3-*, test-results-v1beta2-*, test-results-supportbundle-*)
if ! gh run download "$RUN_ID" --dir "$TEMP_DIR" 2>/dev/null; then
echo -e "${RED}Error: Failed to download artifacts from run ${RUN_ID}${NC}"
echo "Make sure the workflow run completed successfully."
exit 1
fi
echo -e "${GREEN}✓ Artifacts downloaded${NC}"
@@ -63,27 +61,29 @@ echo -e "${GREEN}✓ Artifacts downloaded${NC}"
# Check which bundles are present
echo -e "\n${BLUE}Step 2: Checking available bundles...${NC}"
V1BETA3_BUNDLE=""
V1BETA2_BUNDLE=""
SUPPORTBUNDLE=""
# Use find to locate bundles in artifact subdirectories
V1BETA3_BUNDLE=$(find "$TEMP_DIR" -name "preflight-v1beta3-bundle.tar.gz" | head -1)
V1BETA2_BUNDLE=$(find "$TEMP_DIR" -name "preflight-v1beta2-bundle.tar.gz" | head -1)
SUPPORTBUNDLE=$(find "$TEMP_DIR" -name "supportbundle.tar.gz" | head -1)
if [ -f "preflight-v1beta3-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta3-bundle.tar.gz" ]; then
V1BETA3_BUNDLE=$(find . -name "preflight-v1beta3-bundle.tar.gz" | head -1)
if [ -n "$V1BETA3_BUNDLE" ]; then
echo -e "${GREEN}✓${NC} Found v1beta3 preflight bundle"
fi
if [ -f "preflight-v1beta2-bundle.tar.gz" ] || [ -f "test/output/preflight-v1beta2-bundle.tar.gz" ]; then
V1BETA2_BUNDLE=$(find . -name "preflight-v1beta2-bundle.tar.gz" | head -1)
if [ -n "$V1BETA2_BUNDLE" ]; then
echo -e "${GREEN}✓${NC} Found v1beta2 preflight bundle"
fi
if [ -f "supportbundle.tar.gz" ] || [ -f "test/output/supportbundle.tar.gz" ]; then
SUPPORTBUNDLE=$(find . -name "supportbundle.tar.gz" | head -1)
if [ -n "$SUPPORTBUNDLE" ]; then
echo -e "${GREEN}✓${NC} Found support bundle"
fi
if [ -z "$V1BETA3_BUNDLE" ] && [ -z "$V1BETA2_BUNDLE" ] && [ -z "$SUPPORTBUNDLE" ]; then
echo -e "${RED}Error: No bundles found in artifacts${NC}"
echo "Downloaded artifacts:"
ls -la "$TEMP_DIR"
echo -e "\nSearching for bundles:"
find "$TEMP_DIR" -name "*.tar.gz" -o -name "*.json"
exit 1
fi
@@ -110,21 +110,21 @@ echo -e "\n${BLUE}Step 3: Updating baselines...${NC}"
# Update v1beta3 baseline
if [ -n "$V1BETA3_BUNDLE" ]; then
mkdir -p test/baselines/preflight-v1beta3
cp "$TEMP_DIR/$V1BETA3_BUNDLE" test/baselines/preflight-v1beta3/baseline.tar.gz
cp "$V1BETA3_BUNDLE" test/baselines/preflight-v1beta3/baseline.tar.gz
echo -e "${GREEN}✓${NC} Updated preflight-v1beta3 baseline"
fi
# Update v1beta2 baseline
if [ -n "$V1BETA2_BUNDLE" ]; then
mkdir -p test/baselines/preflight-v1beta2
cp "$TEMP_DIR/$V1BETA2_BUNDLE" test/baselines/preflight-v1beta2/baseline.tar.gz
cp "$V1BETA2_BUNDLE" test/baselines/preflight-v1beta2/baseline.tar.gz
echo -e "${GREEN}✓${NC} Updated preflight-v1beta2 baseline"
fi
# Update support bundle baseline
if [ -n "$SUPPORTBUNDLE" ]; then
mkdir -p test/baselines/supportbundle
cp "$TEMP_DIR/$SUPPORTBUNDLE" test/baselines/supportbundle/baseline.tar.gz
cp "$SUPPORTBUNDLE" test/baselines/supportbundle/baseline.tar.gz
echo -e "${GREEN}✓${NC} Updated supportbundle baseline"
fi
@@ -139,7 +139,7 @@ cat > test/baselines/metadata.json <<EOF
"updated_at": "$CURRENT_DATE",
"git_sha": "$GIT_SHA",
"workflow_run_id": "$RUN_ID",
"k8s_version": "v1.28.3",
"k8s_version": "v1.31.2-k3s1",
"updated_by": "$(git config user.name) <$(git config user.email)>"
}
EOF

7
test/baselines/metadata.json Normal file
View File

@@ -0,0 +1,7 @@
{
"updated_at": "2025-11-27T05:43:03Z",
"git_sha": "280a582f4f5d1dcb242dc9e4441ba1797b0209c7",
"workflow_run_id": "19726392590",
"k8s_version": "v1.31.2-k3s1",
"updated_by": "Ethan Mosbaugh <ethan@replicated.com>"
}

BIN
test/baselines/preflight-v1beta2/baseline.tar.gz
View File

Binary file not shown.

BIN
test/baselines/preflight-v1beta3/baseline.tar.gz
View File

Binary file not shown.

BIN
test/baselines/supportbundle/baseline.tar.gz
View File

Binary file not shown.

2
test/e2e/support-bundle/cluster_resources_e2e_test.go
View File

@@ -47,7 +47,7 @@ func TestClusterResources(t *testing.T) {
"roles",
"events",
"rolebindings",
"statefulsets-errors.json",
"replicasets",
"jobs",
"serviceaccounts",
"configmaps",