Merge pull request #1914 from a7i/refactor/e2e-dry-helpers

refactor(test): extract e2e test scripts into reusable lib modules

.github/workflows/manifests.yaml

@@ -13,7 +13,7 @@ jobs:
         descheduler-version: ["v0.36.0"]
         descheduler-api: ["v1alpha2"]
         manifest: ["deployment"]
-        kind-version: ["v0.31.0"] # keep in sync with test/run-e2e-tests.sh
+        kind-version: ["v0.31.0"] # keep in sync with test/lib/e2e-versions.env
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repo

docs/contributor-guide.md

@@ -29,6 +29,25 @@ View all CLI options.
 ```
 
 ## Run Tests
+
+### All-in-one e2e (kind cluster created by the test script)
+
+```
+KIND_E2E=1 make test-e2e
+```
+
+Optional skip flags (set to any non-empty value to skip that step):
+
+| Variable | Skips |
+|----------|-------|
+| `SKIP_INSTALL` | kind node-image build and cluster create |
+| `SKIP_KUBECTL_INSTALL` | kubectl download |
+| `SKIP_KIND_INSTALL` | kind binary download |
+| `SKIP_KUBEVIRT_INSTALL` | KubeVirt operator install |
+| `SKIP_METRICS_SERVER_INSTALL` | metrics-server install |
+
+### Manual kind cluster (iterative development)
+
 ```
 GOOS=linux make dev-image
 make kind-multi-node

hack/kind_config.yaml

@@ -1,3 +1,5 @@
+# Used by e2e tests (test/run-e2e-tests.sh) and make kind-multi-node.
+# Worker nodes carry topology zone labels required by topology spread constraint e2e tests.
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:

test/kind-config.yaml

@@ -1,3 +1,5 @@
+# Used by the manifests GitHub Actions workflow (.github/workflows/manifests.yaml).
+# Three-node cluster without topology zone labels; not used by e2e tests.
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:

test/lib/e2e-common.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+e2e_repo_root() {
+  local script_dir="$1"
+  cd "${script_dir}/.." && pwd
+}
+
+collect_logs() {
+  local namespace pod
+  echo "Collecting pods and logs"
+  for namespace in "$@"; do
+    kubectl get pods -n "${namespace}"
+    for pod in $(kubectl get pods -n "${namespace}" -o name); do
+      echo "Logs for ${pod}"
+      kubectl logs -n "${namespace}" "${pod}"
+    done
+  done
+}
+
+kind_load_image() {
+  local image="$1"
+  if [ "${CONTAINER_ENGINE}" = "podman" ]; then
+    local archive
+    archive="/tmp/$(echo "${image}" | tr '/:' '_').tar"
+    podman save "${image}" -o "${archive}"
+    ${KIND_SUDO} kind load image-archive "${archive}"
+    rm "${archive}"
+  else
+    ${KIND_SUDO} kind load docker-image "${image}"
+  fi
+}

test/lib/e2e-versions.env

@@ -0,0 +1,6 @@
+# E2E tool version defaults. Override via environment variables.
+# Keep .github/workflows/manifests.yaml matrix versions in sync with these values.
+
+K8S_VERSION=${KUBERNETES_VERSION:-v1.36.1}
+KIND_VERSION=${KIND_VERSION:-v0.31.0}
+METRICS_SERVER_VERSION=${METRICS_SERVER_VERSION:-v0.8.1}

test/lib/install-kubevirt.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml"
+kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml"
+# TODO(1.37): drop this patch when the k8s 1.34 e2e lane is removed. ImageVolume needs k8s >= 1.35.
+if [[ "${K8S_VERSION}" == v1.34* ]]; then
+  kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true,"disabledFeatureGates":["ImageVolume"]}}}}'
+else
+  kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
+fi
+kubectl wait --timeout=300s --for=condition=Available -n kubevirt kv/kubevirt

test/lib/install-metrics-server.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kubectl apply -f "https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml"
+kubectl patch -n kube-system deployment metrics-server --type=json \
+  -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]'
+kubectl wait --timeout=180s --for=condition=Available -n kube-system deployment/metrics-server

test/lib/setup-kind.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+KIND_NODE_IMAGE=${KIND_NODE_IMAGE:-localhost/kindest/node:${K8S_VERSION}}
+
+if [ -z "${SKIP_KUBECTL_INSTALL}" ]; then
+  curl -Lo kubectl "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/
+fi
+if [ -z "${SKIP_KIND_INSTALL}" ]; then
+  wget "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+  chmod +x kind-linux-amd64
+  mv kind-linux-amd64 kind
+  export PATH=$PATH:$PWD
+fi
+
+if [ -z "${SKIP_INSTALL}" ]; then
+  ${KIND_SUDO} kind build node-image "${K8S_VERSION}" --image "${KIND_NODE_IMAGE}"
+  ${KIND_SUDO} kind create cluster --image "${KIND_NODE_IMAGE}" --config="${REPO_ROOT}/hack/kind_config.yaml"
+fi
+${CONTAINER_ENGINE} pull registry.k8s.io/pause
+kind_load_image registry.k8s.io/pause
+kind_load_image "${DESCHEDULER_IMAGE}"
+${KIND_SUDO} kind get kubeconfig > /tmp/admin.conf
+
+export KUBECONFIG="/tmp/admin.conf"
+mkdir -p ~/gopath/src/sigs.k8s.io/

test/run-e2e-tests.sh

@@ -18,101 +18,50 @@ set -x
 set -o errexit
 set -o nounset
 
+BASEDIR=$(dirname "$0")
+# shellcheck source=test/lib/e2e-common.sh
+source "${BASEDIR}/lib/e2e-common.sh"
+# shellcheck source=test/lib/e2e-versions.env
+source "${BASEDIR}/lib/e2e-versions.env"
+REPO_ROOT="$(e2e_repo_root "${BASEDIR}")"
+
 # Set to empty if unbound/empty
 SKIP_INSTALL=${SKIP_INSTALL:-}
 KIND_E2E=${KIND_E2E:-}
 CONTAINER_ENGINE=${CONTAINER_ENGINE:-docker}
 KIND_SUDO=${KIND_SUDO:-}
-KIND_VERSION=${KIND_VERSION:-v0.31.0}
 SKIP_KUBECTL_INSTALL=${SKIP_KUBECTL_INSTALL:-}
 SKIP_KIND_INSTALL=${SKIP_KIND_INSTALL:-}
 SKIP_KUBEVIRT_INSTALL=${SKIP_KUBEVIRT_INSTALL:-}
-K8S_VERSION=${KUBERNETES_VERSION:-}
+SKIP_METRICS_SERVER_INSTALL=${SKIP_METRICS_SERVER_INSTALL:-}
+KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-$(grep 'kubevirt.io/api ' "${REPO_ROOT}/go.mod" | awk '{print $2}')}
 
 # Build a descheduler image
 IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags)
-BASEDIR=$(dirname "$0")
-KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-$(grep 'kubevirt.io/api ' "${BASEDIR}/../go.mod" | awk '{print $2}')}
-VERSION="${IMAGE_TAG}" make -C ${BASEDIR}/.. image
+VERSION="${IMAGE_TAG}" make -C "${REPO_ROOT}" image
 
 export DESCHEDULER_IMAGE="docker.io/library/descheduler:${IMAGE_TAG}"
 echo "DESCHEDULER_IMAGE: ${DESCHEDULER_IMAGE}"
 
-# This just runs e2e tests.
-if [ -n "$KIND_E2E" ]; then
-    K8S_VERSION=${K8S_VERSION:-v1.36.1}
-    KIND_NODE_IMAGE=${KIND_NODE_IMAGE:-localhost/kindest/node:${K8S_VERSION}}
-    if [ -z "${SKIP_KUBECTL_INSTALL}" ]; then
-        curl -Lo kubectl https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/
-    fi
-    if [ -z "${SKIP_KIND_INSTALL}" ]; then
-        wget https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64
-        chmod +x kind-linux-amd64
-        mv kind-linux-amd64 kind
-        export PATH=$PATH:$PWD
-    fi
-
-    # If we did not set SKIP_INSTALL
-    if [ -z "$SKIP_INSTALL" ]; then
-        ${KIND_SUDO} kind build node-image ${K8S_VERSION} --image ${KIND_NODE_IMAGE}
-        ${KIND_SUDO} kind create cluster --image ${KIND_NODE_IMAGE} --config=./hack/kind_config.yaml
-    fi
-    ${CONTAINER_ENGINE} pull registry.k8s.io/pause
-    if [ "${CONTAINER_ENGINE}" == "podman" ]; then
-      podman save registry.k8s.io/pause -o /tmp/pause.tar
-      ${KIND_SUDO} kind load image-archive /tmp/pause.tar
-      rm /tmp/pause.tar
-      podman save ${DESCHEDULER_IMAGE} -o /tmp/descheduler.tar
-      ${KIND_SUDO} kind load image-archive /tmp/descheduler.tar
-      rm /tmp/descheduler.tar
-    else
-      ${KIND_SUDO} kind load docker-image registry.k8s.io/pause
-      ${KIND_SUDO} kind load docker-image ${DESCHEDULER_IMAGE}
-    fi
-    ${KIND_SUDO} kind get kubeconfig > /tmp/admin.conf
-
-    export KUBECONFIG="/tmp/admin.conf"
-    mkdir -p ~/gopath/src/sigs.k8s.io/
+if [ -n "${KIND_E2E}" ]; then
+  # shellcheck source=test/lib/setup-kind.sh
+  source "${BASEDIR}/lib/setup-kind.sh"
 fi
 
 # Deploy rbac, sa and binding for a descheduler running through a deployment
-kubectl apply -f kubernetes/base/rbac.yaml
+kubectl apply -f "${REPO_ROOT}/kubernetes/base/rbac.yaml"
 
-collect_logs() {
-  echo "Collecting pods and logs"
-  kubectl get pods -n default
-  kubectl get pods -n kubevirt
-
-  for pod in $(kubectl get pods -n default -o name); do
-    echo "Logs for ${pod}"
-    kubectl logs -n default ${pod}
-  done
-
-  for pod in $(kubectl get pods -n kubevirt -o name); do
-    echo "Logs for ${pod}"
-    kubectl logs -n kubevirt ${pod}
-  done
-}
-
-trap "collect_logs" ERR
+trap 'collect_logs default kubevirt' ERR
 
 if [ -z "${SKIP_KUBEVIRT_INSTALL}" ]; then
-  kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml
-  kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml
-  # TODO(1.37): drop this patch when the k8s 1.34 e2e lane is removed. ImageVolume needs k8s >= 1.35.
-  if [[ "${K8S_VERSION}" == v1.34* ]]; then
-    kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true,"disabledFeatureGates":["ImageVolume"]}}}}'
-  else
-    kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
-  fi
-  kubectl wait --timeout=300s --for=condition=Available -n kubevirt kv/kubevirt
+  # shellcheck source=test/lib/install-kubevirt.sh
+  source "${BASEDIR}/lib/install-kubevirt.sh"
 fi
 
-METRICS_SERVER_VERSION="v0.8.1"
-kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml
-kubectl patch -n kube-system deployment metrics-server --type=json \
-  -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]'
-kubectl wait --timeout=180s --for=condition=Available -n kube-system deployment/metrics-server
+if [ -z "${SKIP_METRICS_SERVER_INSTALL}" ]; then
+  # shellcheck source=test/lib/install-metrics-server.sh
+  source "${BASEDIR}/lib/install-metrics-server.sh"
+fi
 
 PRJ_PREFIX="sigs.k8s.io/descheduler"
-go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 --args --descheduler-image ${DESCHEDULER_IMAGE} --kubevirt-version-tag ${KUBEVIRT_VERSION} --pod-run-as-user-id 1000 --pod-run-as-group-id 1000
+go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 --args --descheduler-image "${DESCHEDULER_IMAGE}" --kubevirt-version-tag "${KUBEVIRT_VERSION}" --pod-run-as-user-id 1000 --pod-run-as-group-id 1000