From fb908401dd0664f677762c2232a4898a19ef10a5 Mon Sep 17 00:00:00 2001 From: Amir Alavi Date: Mon, 10 Aug 2026 16:56:26 -0400 Subject: [PATCH] refactor(test): extract e2e test scripts into reusable lib modules Split run-e2e-tests.sh into test/lib helpers for kind setup, dependency install, shared utilities, and centralized version defaults. --- .github/workflows/manifests.yaml | 2 +- docs/contributor-guide.md | 19 ++++++ hack/kind_config.yaml | 2 + test/kind-config.yaml | 2 + test/lib/e2e-common.sh | 45 ++++++++++++++ test/lib/e2e-versions.env | 6 ++ test/lib/install-kubevirt.sh | 25 ++++++++ test/lib/install-metrics-server.sh | 20 +++++++ test/lib/setup-kind.sh | 39 ++++++++++++ test/run-e2e-tests.sh | 95 +++++++----------------------- 10 files changed, 181 insertions(+), 74 deletions(-) create mode 100644 test/lib/e2e-common.sh create mode 100644 test/lib/e2e-versions.env create mode 100644 test/lib/install-kubevirt.sh create mode 100644 test/lib/install-metrics-server.sh create mode 100644 test/lib/setup-kind.sh diff --git a/.github/workflows/manifests.yaml b/.github/workflows/manifests.yaml index b1acc4a83..ed854ebc5 100644 --- a/.github/workflows/manifests.yaml +++ b/.github/workflows/manifests.yaml @@ -13,7 +13,7 @@ jobs: descheduler-version: ["v0.36.0"] descheduler-api: ["v1alpha2"] manifest: ["deployment"] - kind-version: ["v0.31.0"] # keep in sync with test/run-e2e-tests.sh + kind-version: ["v0.31.0"] # keep in sync with test/lib/e2e-versions.env runs-on: ubuntu-latest steps: - name: Checkout Repo diff --git a/docs/contributor-guide.md b/docs/contributor-guide.md index e8de05663..2e2795169 100644 --- a/docs/contributor-guide.md +++ b/docs/contributor-guide.md @@ -29,6 +29,25 @@ View all CLI options. ``` ## Run Tests + +### All-in-one e2e (kind cluster created by the test script) + +``` +KIND_E2E=1 make test-e2e +``` + +Optional skip flags (set to any non-empty value to skip that step): + +| Variable | Skips | +|----------|-------| +| `SKIP_INSTALL` | kind node-image build and cluster create | +| `SKIP_KUBECTL_INSTALL` | kubectl download | +| `SKIP_KIND_INSTALL` | kind binary download | +| `SKIP_KUBEVIRT_INSTALL` | KubeVirt operator install | +| `SKIP_METRICS_SERVER_INSTALL` | metrics-server install | + +### Manual kind cluster (iterative development) + ``` GOOS=linux make dev-image make kind-multi-node diff --git a/hack/kind_config.yaml b/hack/kind_config.yaml index c4057022b..de76c5ee5 100644 --- a/hack/kind_config.yaml +++ b/hack/kind_config.yaml @@ -1,3 +1,5 @@ +# Used by e2e tests (test/run-e2e-tests.sh) and make kind-multi-node. +# Worker nodes carry topology zone labels required by topology spread constraint e2e tests. kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: diff --git a/test/kind-config.yaml b/test/kind-config.yaml index 752e993cd..640cfbc6c 100644 --- a/test/kind-config.yaml +++ b/test/kind-config.yaml @@ -1,3 +1,5 @@ +# Used by the manifests GitHub Actions workflow (.github/workflows/manifests.yaml). +# Three-node cluster without topology zone labels; not used by e2e tests. kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: diff --git a/test/lib/e2e-common.sh b/test/lib/e2e-common.sh new file mode 100644 index 000000000..ebf3b9a6d --- /dev/null +++ b/test/lib/e2e-common.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +e2e_repo_root() { + local script_dir="$1" + cd "${script_dir}/.." && pwd +} + +collect_logs() { + local namespace pod + echo "Collecting pods and logs" + for namespace in "$@"; do + kubectl get pods -n "${namespace}" + for pod in $(kubectl get pods -n "${namespace}" -o name); do + echo "Logs for ${pod}" + kubectl logs -n "${namespace}" "${pod}" + done + done +} + +kind_load_image() { + local image="$1" + if [ "${CONTAINER_ENGINE}" = "podman" ]; then + local archive + archive="/tmp/$(echo "${image}" | tr '/:' '_').tar" + podman save "${image}" -o "${archive}" + ${KIND_SUDO} kind load image-archive "${archive}" + rm "${archive}" + else + ${KIND_SUDO} kind load docker-image "${image}" + fi +} diff --git a/test/lib/e2e-versions.env b/test/lib/e2e-versions.env new file mode 100644 index 000000000..e5943344c --- /dev/null +++ b/test/lib/e2e-versions.env @@ -0,0 +1,6 @@ +# E2E tool version defaults. Override via environment variables. +# Keep .github/workflows/manifests.yaml matrix versions in sync with these values. + +K8S_VERSION=${KUBERNETES_VERSION:-v1.36.1} +KIND_VERSION=${KIND_VERSION:-v0.31.0} +METRICS_SERVER_VERSION=${METRICS_SERVER_VERSION:-v0.8.1} diff --git a/test/lib/install-kubevirt.sh b/test/lib/install-kubevirt.sh new file mode 100644 index 000000000..6856a498d --- /dev/null +++ b/test/lib/install-kubevirt.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml" +kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml" +# TODO(1.37): drop this patch when the k8s 1.34 e2e lane is removed. ImageVolume needs k8s >= 1.35. +if [[ "${K8S_VERSION}" == v1.34* ]]; then + kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true,"disabledFeatureGates":["ImageVolume"]}}}}' +else + kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' +fi +kubectl wait --timeout=300s --for=condition=Available -n kubevirt kv/kubevirt diff --git a/test/lib/install-metrics-server.sh b/test/lib/install-metrics-server.sh new file mode 100644 index 000000000..6deed63ba --- /dev/null +++ b/test/lib/install-metrics-server.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl apply -f "https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml" +kubectl patch -n kube-system deployment metrics-server --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' +kubectl wait --timeout=180s --for=condition=Available -n kube-system deployment/metrics-server diff --git a/test/lib/setup-kind.sh b/test/lib/setup-kind.sh new file mode 100644 index 000000000..bb4166493 --- /dev/null +++ b/test/lib/setup-kind.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KIND_NODE_IMAGE=${KIND_NODE_IMAGE:-localhost/kindest/node:${K8S_VERSION}} + +if [ -z "${SKIP_KUBECTL_INSTALL}" ]; then + curl -Lo kubectl "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ +fi +if [ -z "${SKIP_KIND_INSTALL}" ]; then + wget "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" + chmod +x kind-linux-amd64 + mv kind-linux-amd64 kind + export PATH=$PATH:$PWD +fi + +if [ -z "${SKIP_INSTALL}" ]; then + ${KIND_SUDO} kind build node-image "${K8S_VERSION}" --image "${KIND_NODE_IMAGE}" + ${KIND_SUDO} kind create cluster --image "${KIND_NODE_IMAGE}" --config="${REPO_ROOT}/hack/kind_config.yaml" +fi +${CONTAINER_ENGINE} pull registry.k8s.io/pause +kind_load_image registry.k8s.io/pause +kind_load_image "${DESCHEDULER_IMAGE}" +${KIND_SUDO} kind get kubeconfig > /tmp/admin.conf + +export KUBECONFIG="/tmp/admin.conf" +mkdir -p ~/gopath/src/sigs.k8s.io/ diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index 3bc0a733b..eefaf5141 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -18,101 +18,50 @@ set -x set -o errexit set -o nounset +BASEDIR=$(dirname "$0") +# shellcheck source=test/lib/e2e-common.sh +source "${BASEDIR}/lib/e2e-common.sh" +# shellcheck source=test/lib/e2e-versions.env +source "${BASEDIR}/lib/e2e-versions.env" +REPO_ROOT="$(e2e_repo_root "${BASEDIR}")" + # Set to empty if unbound/empty SKIP_INSTALL=${SKIP_INSTALL:-} KIND_E2E=${KIND_E2E:-} CONTAINER_ENGINE=${CONTAINER_ENGINE:-docker} KIND_SUDO=${KIND_SUDO:-} -KIND_VERSION=${KIND_VERSION:-v0.31.0} SKIP_KUBECTL_INSTALL=${SKIP_KUBECTL_INSTALL:-} SKIP_KIND_INSTALL=${SKIP_KIND_INSTALL:-} SKIP_KUBEVIRT_INSTALL=${SKIP_KUBEVIRT_INSTALL:-} -K8S_VERSION=${KUBERNETES_VERSION:-} +SKIP_METRICS_SERVER_INSTALL=${SKIP_METRICS_SERVER_INSTALL:-} +KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-$(grep 'kubevirt.io/api ' "${REPO_ROOT}/go.mod" | awk '{print $2}')} # Build a descheduler image IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags) -BASEDIR=$(dirname "$0") -KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-$(grep 'kubevirt.io/api ' "${BASEDIR}/../go.mod" | awk '{print $2}')} -VERSION="${IMAGE_TAG}" make -C ${BASEDIR}/.. image +VERSION="${IMAGE_TAG}" make -C "${REPO_ROOT}" image export DESCHEDULER_IMAGE="docker.io/library/descheduler:${IMAGE_TAG}" echo "DESCHEDULER_IMAGE: ${DESCHEDULER_IMAGE}" -# This just runs e2e tests. -if [ -n "$KIND_E2E" ]; then - K8S_VERSION=${K8S_VERSION:-v1.36.1} - KIND_NODE_IMAGE=${KIND_NODE_IMAGE:-localhost/kindest/node:${K8S_VERSION}} - if [ -z "${SKIP_KUBECTL_INSTALL}" ]; then - curl -Lo kubectl https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ - fi - if [ -z "${SKIP_KIND_INSTALL}" ]; then - wget https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64 - chmod +x kind-linux-amd64 - mv kind-linux-amd64 kind - export PATH=$PATH:$PWD - fi - - # If we did not set SKIP_INSTALL - if [ -z "$SKIP_INSTALL" ]; then - ${KIND_SUDO} kind build node-image ${K8S_VERSION} --image ${KIND_NODE_IMAGE} - ${KIND_SUDO} kind create cluster --image ${KIND_NODE_IMAGE} --config=./hack/kind_config.yaml - fi - ${CONTAINER_ENGINE} pull registry.k8s.io/pause - if [ "${CONTAINER_ENGINE}" == "podman" ]; then - podman save registry.k8s.io/pause -o /tmp/pause.tar - ${KIND_SUDO} kind load image-archive /tmp/pause.tar - rm /tmp/pause.tar - podman save ${DESCHEDULER_IMAGE} -o /tmp/descheduler.tar - ${KIND_SUDO} kind load image-archive /tmp/descheduler.tar - rm /tmp/descheduler.tar - else - ${KIND_SUDO} kind load docker-image registry.k8s.io/pause - ${KIND_SUDO} kind load docker-image ${DESCHEDULER_IMAGE} - fi - ${KIND_SUDO} kind get kubeconfig > /tmp/admin.conf - - export KUBECONFIG="/tmp/admin.conf" - mkdir -p ~/gopath/src/sigs.k8s.io/ +if [ -n "${KIND_E2E}" ]; then + # shellcheck source=test/lib/setup-kind.sh + source "${BASEDIR}/lib/setup-kind.sh" fi # Deploy rbac, sa and binding for a descheduler running through a deployment -kubectl apply -f kubernetes/base/rbac.yaml +kubectl apply -f "${REPO_ROOT}/kubernetes/base/rbac.yaml" -collect_logs() { - echo "Collecting pods and logs" - kubectl get pods -n default - kubectl get pods -n kubevirt - - for pod in $(kubectl get pods -n default -o name); do - echo "Logs for ${pod}" - kubectl logs -n default ${pod} - done - - for pod in $(kubectl get pods -n kubevirt -o name); do - echo "Logs for ${pod}" - kubectl logs -n kubevirt ${pod} - done -} - -trap "collect_logs" ERR +trap 'collect_logs default kubevirt' ERR if [ -z "${SKIP_KUBEVIRT_INSTALL}" ]; then - kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml - kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml - # TODO(1.37): drop this patch when the k8s 1.34 e2e lane is removed. ImageVolume needs k8s >= 1.35. - if [[ "${K8S_VERSION}" == v1.34* ]]; then - kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true,"disabledFeatureGates":["ImageVolume"]}}}}' - else - kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' - fi - kubectl wait --timeout=300s --for=condition=Available -n kubevirt kv/kubevirt + # shellcheck source=test/lib/install-kubevirt.sh + source "${BASEDIR}/lib/install-kubevirt.sh" fi -METRICS_SERVER_VERSION="v0.8.1" -kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml -kubectl patch -n kube-system deployment metrics-server --type=json \ - -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' -kubectl wait --timeout=180s --for=condition=Available -n kube-system deployment/metrics-server +if [ -z "${SKIP_METRICS_SERVER_INSTALL}" ]; then + # shellcheck source=test/lib/install-metrics-server.sh + source "${BASEDIR}/lib/install-metrics-server.sh" +fi PRJ_PREFIX="sigs.k8s.io/descheduler" -go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 --args --descheduler-image ${DESCHEDULER_IMAGE} --kubevirt-version-tag ${KUBEVIRT_VERSION} --pod-run-as-user-id 1000 --pod-run-as-group-id 1000 +go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 --args --descheduler-image "${DESCHEDULER_IMAGE}" --kubevirt-version-tag "${KUBEVIRT_VERSION}" --pod-run-as-user-id 1000 --pod-run-as-group-id 1000