Add acceptance test (#222)

* Add acceptance test * Update README.md * Update README.md * Added acceptance-test-diagram.svg * Update README.md * Add files via upload * Added acceptance-test-diagram.svg * Added acceptance-test-diagram.svg * Update README.md * Update README.md
2026-02-14 16:39:51 +00:00 · 2020-01-31 21:45:23 +09:00
parent 0c6ca03eb9
commit f8cca818af
15 changed files with 562 additions and 2 deletions
--- a/acceptance_test/Dockerfile
+++ b/acceptance_test/Dockerfile
@@ -0,0 +1,29 @@
+FROM alpine:3.11
+
+RUN apk add --update ttf-freefont chromium
+RUN apk add nss-tools
+# https://kubernetes.io/docs/tasks/tools/install-kubectl/
+RUN wget -O /usr/local/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v1.17.0/bin/linux/amd64/kubectl" && \
+    chmod +x /usr/local/bin/kubectl && \
+    kubectl version --client
+
+# run as a dedicated user
+RUN adduser -D -h /runner runner
+
+COPY output/kubelogin_linux_amd64 /usr/local/bin/kubectl-oidc_login
+
+COPY kubeconfig_oidc.yaml /runner/
+COPY output/kubeconfig.yaml /runner/output/
+COPY output/ca.crt /runner/output/
+
+COPY output/kubelogin_acceptance_test_linux_amd64 /usr/local/bin/kubelogin_acceptance_test_linux_amd64
+
+RUN chown -R runner:runner /runner
+USER runner
+WORKDIR /runner
+
+# import the CA certificate for chromium
+RUN mkdir -p .pki/nssdb && \
+    certutil -A -d sql:.pki/nssdb -n kubernetes -i output/ca.crt -t "TC,,"
+
+CMD ["kubelogin_acceptance_test_linux_amd64", "-test.v"]
--- a/acceptance_test/Makefile
+++ b/acceptance_test/Makefile
@@ -0,0 +1,70 @@
+CLUSTER_NAME := kubelogin-acceptance-test
+OUTPUT_DIR := $(CURDIR)/output
+
+KUBECONFIG := $(OUTPUT_DIR)/kubeconfig.yaml
+export KUBECONFIG
+
+.PHONY: all
+all: build setup test
+
+.PHONY: clean
+clean:
+	-rm -r $(OUTPUT_DIR)
+
+# build binaries for the container
+.PHONY: build
+build: $(OUTPUT_DIR)/kubelogin_linux_amd64 $(OUTPUT_DIR)/kubelogin_acceptance_test_linux_amd64
+$(OUTPUT_DIR)/kubelogin_linux_amd64:
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o $@ ..
+$(OUTPUT_DIR)/kubelogin_acceptance_test_linux_amd64: $(wildcard *.go)
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go test -c -o $@ .
+
+# create a Kubernetes cluster with Dex
+.PHONY:
+setup: create-cluster deploy
+
+.PHONY: create-cluster
+create-cluster:
+	kind create cluster --name $(CLUSTER_NAME) --config cluster.yaml
+delete-cluster:
+	kind delete cluster --name $(CLUSTER_NAME)
+
+.PHONY: deploy
+deploy: $(OUTPUT_DIR)/server.crt $(OUTPUT_DIR)/server.key
+	kubectl apply -f dex.yaml
+	kubectl -n dex create secret tls dex.tls --cert=$(OUTPUT_DIR)/server.crt --key=$(OUTPUT_DIR)/server.key
+	kubectl -n dex rollout status deployment server
+	kubectl apply -f role.yaml
+
+$(OUTPUT_DIR)/ca.key:
+	docker cp $(CLUSTER_NAME)-control-plane:/etc/kubernetes/pki/ca.key $@
+$(OUTPUT_DIR)/ca.crt:
+	docker cp $(CLUSTER_NAME)-control-plane:/etc/kubernetes/pki/ca.crt $@
+$(OUTPUT_DIR)/server.key:
+	openssl genrsa -out $@ 2048
+$(OUTPUT_DIR)/server.csr: openssl.cnf $(OUTPUT_DIR)/server.key
+	openssl req -new -key $(OUTPUT_DIR)/server.key -out $@ -subj "/CN=kubelogin-acceptance-test-control-plane" -config openssl.cnf
+	#openssl req -noout -text -in $@
+$(OUTPUT_DIR)/server.crt: openssl.cnf $(OUTPUT_DIR)/server.csr $(OUTPUT_DIR)/ca.crt $(OUTPUT_DIR)/ca.key
+	openssl x509 -req -in $(OUTPUT_DIR)/server.csr -CA $(OUTPUT_DIR)/ca.crt -CAkey $(OUTPUT_DIR)/ca.key -CAcreateserial -out $@ -sha256 -days 10 -extensions v3_req -extfile openssl.cnf
+	#openssl x509 -text -in $@
+
+# run the test on a container:
+.PHONY: test
+test: docker-run
+
+# - enable host network to access Dex via the node port
+# - add /etc/hosts to access Dex
+DOCKER_RUN_FLAGS := \
+	--net host \
+	--add-host kubelogin-acceptance-test-control-plane:127.0.0.1
+
+.PHONY: docker-build
+docker-build: build
+	docker build -t kubelogin-acceptance-test .
+.PHONY: docker-run
+docker-run: docker-build
+	docker run --rm $(DOCKER_RUN_FLAGS) kubelogin-acceptance-test
+.PHONY: docker-shell
+docker-shell: docker-build
+	docker run -it --rm $(DOCKER_RUN_FLAGS) kubelogin-acceptance-test /bin/sh
--- a/acceptance_test/README.md
+++ b/acceptance_test/README.md
@@ -0,0 +1,93 @@
+# kubelogin/acceptance_test
+
+This is an acceptance test to verify behavior of kubelogin using a real Kubernetes cluster and OpenID Connect provider.
+It runs on [GitHub Actions](https://github.com/int128/kubelogin/actions?query=workflow%3Aacceptance-test).
+
+Let's look at the diagram.
+
+![diagram](../docs/acceptance-test-diagram.svg)
+
+It performs the test by the following steps:
+
+1. Create a Kubernetes cluster using Kind.
+1. Generate a TLS server certificate for Dex.
+1. Deploy Dex and the cluster role to the cluster.
+1. Run the test in a container.
+1. Run kubectl.
+1. kubectl automatically runs kubelogin.
+1. Open the browser and navigate to `http://localhost:8000`.
+1. Enter the username and password on the browser.
+1. kubelogin gets an authorization code from the browser.
+1. kubelogin gets a token.
+1. kubectl accesses an API with the token.
+1. kube-apiserver verifies the token by Dex.
+1. Check if kubectl exited with code 0.
+
+
+## Technical consideration
+
+### Network and DNS
+
+Consider the following issues:
+
+- kube-apiserver runs on the host network.
+- kube-apiserver cannot resolve a service name by kube-dns, e.g. `server.dex.svc.cluster.local`.
+- kube-apiserver cannot access a cluster IP.
+- Chromium requires exactly match of domain name between Dex URL and a server certificate.
+
+kube-apiserver accesses Dex via the following route:
+
+```
+kube-apiserver
+↓
+kind-control-plane:30443 (host port)
+↓
+dex-service:30443 (node port)
+↓
+dex-pod-container:30443 (pod container port)
+```
+
+### TLS server certificate
+
+Consider the following issues:
+
+- kube-apiserver requires `--oidc-issuer` is HTTPS URL.
+- kube-apiserver requires a CA certificate at startup, if `--oidc-ca-file` is given.
+- kube-apiserver has a CA key pair at `/etc/kubernetes/pki`.
+- It is not possible to put a file into kube-apiserver at startup.
+- It is not possible to issue a certificate using Let's Encrypt in runtime.
+- Chromium requires a valid certificate in the NSS database.
+
+In the test, it gets the CA certificate from the kind container, generates a server certificate and puts it into the test container.
+As a result,
+
+- Dex uses the server certificate for serving TLS connection.
+- kube-apiserver uses the CA certificate for verifying TLS connection.
+- kubelogin uses the CA certificate for verifying TLS connection.
+- Chromium uses the CA certificate for verifying TLS connection.
+
+### Test environment
+
+- Set the issuer URL to kubectl (see [`kubeconfig.yaml`](kubeconfig.yaml)) and kube-apiserver (see [`cluster.yaml`](cluster.yaml)).
+- Create a cluster role which has read-only access (see [`role.yaml`](role.yaml)).
+- Install Chromium in the test container.
+- Change `/etc/hosts` to access Dex from the test container, by using `--add-host` flag of Docker.
+
+### Test scenario
+
+- Run `kubectl` and open the browser concurrently.
+- It need to wait until `http://localhost:8000` is available. It prevents the browser error.
+- It need to kill sub-processes finally, i.e. kubectl and kubelogin.
+
+
+## Run locally
+
+You need to set up Docker and Kind.
+
+```shell script
+# run the test
+make
+
+# clean up
+make delete-cluster
+```
--- a/acceptance_test/acceptance_test.go
+++ b/acceptance_test/acceptance_test.go
@@ -0,0 +1,140 @@
+package acceptance_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/chromedp/chromedp"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	tokenCacheDir = "output/token-cache"
+	kubeconfigEnv = "KUBECONFIG=output/kubeconfig.yaml:kubeconfig_oidc.yaml"
+)
+
+func init() {
+	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
+}
+
+func Test(t *testing.T) {
+	if _, err := os.Stat("output/kubeconfig.yaml"); err != nil {
+		t.Skipf("skip the test: %s", err)
+	}
+	if err := os.RemoveAll(tokenCacheDir); err != nil {
+		t.Fatalf("could not remove the token cache: %s", err)
+	}
+	ctx := context.TODO()
+	eg, ctx := errgroup.WithContext(ctx)
+	eg.Go(func() error { return runKubectl(ctx, t, eg) })
+	eg.Go(func() error { return runBrowser(ctx) })
+	if err := eg.Wait(); err != nil {
+		t.Errorf("error: %s", err)
+	}
+}
+
+func runKubectl(ctx context.Context, t *testing.T, eg *errgroup.Group) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	cmd := exec.Command("kubectl", "--user=oidc", "--namespace=dex", "get", "deploy")
+	cmd.Env = append(os.Environ(), kubeconfigEnv)
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	eg.Go(func() error {
+		<-ctx.Done()
+		if cmd.Process == nil {
+			log.Printf("process not started")
+			return nil
+		}
+		if cmd.ProcessState != nil && cmd.ProcessState.Exited() {
+			log.Printf("process terminated with exit code %d", cmd.ProcessState.ExitCode())
+			return nil
+		}
+		log.Printf("sending SIGTERM to pid %d", cmd.Process.Pid)
+		// kill the child processes
+		// https://medium.com/@felixge/killing-a-child-process-and-all-of-its-children-in-go-54079af94773
+		if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM); err != nil {
+			t.Errorf("could not send a signal: %s", err)
+		}
+		return nil
+	})
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("could not run a command: %w", err)
+	}
+	return nil
+}
+
+func runBrowser(ctx context.Context) error {
+	execOpts := chromedp.DefaultExecAllocatorOptions[:]
+	execOpts = append(execOpts, chromedp.NoSandbox)
+	ctx, cancel := chromedp.NewExecAllocator(ctx, execOpts...)
+	defer cancel()
+	ctx, cancel = chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf))
+	defer cancel()
+	ctx, cancel = context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	if err := openKubeloginAndLogInToDex(ctx); err != nil {
+		return fmt.Errorf("could not run the browser: %w", err)
+	}
+	return nil
+}
+
+func openKubeloginAndLogInToDex(ctx context.Context) error {
+	for {
+		var location string
+		err := chromedp.Run(ctx,
+			chromedp.Navigate(`http://localhost:8000`),
+			chromedp.Location(&location),
+		)
+		if err != nil {
+			return err
+		}
+		log.Printf("location: %s", location)
+		if strings.HasPrefix(location, `http://`) || strings.HasPrefix(location, `https://`) {
+			break
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+	err := chromedp.Run(ctx,
+		// https://kubelogin-acceptance-test-control-plane:30443/dex/auth/local
+		chromedp.WaitVisible(`#login`),
+		logPageMetadata(),
+		chromedp.SendKeys(`#login`, `admin@example.com`),
+		chromedp.SendKeys(`#password`, `password`),
+		chromedp.Submit(`#submit-login`),
+		// https://kubelogin-acceptance-test-control-plane:30443/dex/approval
+		chromedp.WaitVisible(`.dex-btn.theme-btn--success`),
+		logPageMetadata(),
+		chromedp.Submit(`.dex-btn.theme-btn--success`),
+		// http://localhost:8000
+		chromedp.WaitReady(`body`),
+		logPageMetadata(),
+	)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func logPageMetadata() chromedp.Action {
+	var location string
+	var title string
+	return chromedp.Tasks{
+		chromedp.Location(&location),
+		chromedp.Title(&title),
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			log.Printf("location: %s [%s]", location, title)
+			return nil
+		}),
+	}
+}
--- a/acceptance_test/cluster.yaml
+++ b/acceptance_test/cluster.yaml
@@ -0,0 +1,20 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+# https://github.com/dexidp/dex/blob/master/Documentation/kubernetes.md
+kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: ClusterConfiguration
+    metadata:
+      name: config
+    apiServer:
+      extraArgs:
+        oidc-issuer-url: https://kubelogin-acceptance-test-control-plane:30443/dex
+        oidc-client-id: YOUR_CLIENT_ID
+        oidc-username-claim: email
+        oidc-ca-file: /etc/kubernetes/pki/ca.crt
+nodes:
+  - role: control-plane
+    extraPortMappings:
+      - containerPort: 30443
+        hostPort: 30443
--- a/acceptance_test/dex.yaml
+++ b/acceptance_test/dex.yaml
@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dex
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: server
+  namespace: dex
+  labels:
+    app: server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: server
+  template:
+    metadata:
+      labels:
+        app: server
+    spec:
+      containers:
+        - name: server
+          image: quay.io/dexidp/dex:v2.21.0
+          command:
+            - /usr/local/bin/dex
+            - serve
+            - /etc/dex/cfg/config.yaml
+          ports:
+            - containerPort: 30443
+              name: https
+          volumeMounts:
+            - name: config
+              mountPath: /etc/dex/cfg
+            - name: tls
+              mountPath: /etc/dex/tls
+      volumes:
+        - name: config
+          configMap:
+            name: server
+            items:
+              - key: config.yaml
+                path: config.yaml
+        - name: tls
+          secret:
+            secretName: dex.tls
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: server
+  namespace: dex
+  labels:
+    app: server
+spec:
+  type: NodePort
+  ports:
+    - name: https
+      protocol: TCP
+      nodePort: 30443
+      port: 30443
+      targetPort: https
+  selector:
+    app: server
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: server
+  namespace: dex
+data:
+  config.yaml: |
+    issuer: https://kubelogin-acceptance-test-control-plane:30443/dex
+    web:
+      https: 0.0.0.0:30443
+      tlsCert: /etc/dex/tls/tls.crt
+      tlsKey: /etc/dex/tls/tls.key
+    storage:
+      type: sqlite3
+      config:
+        file: /tmp/dex.db
+    staticClients:
+      - id: YOUR_CLIENT_ID
+        redirectURIs:
+          - http://localhost:8000
+        name: kubelogin
+        secret: YOUR_CLIENT_SECRET
+    staticPasswords:
+      - email: "admin@example.com"
+        # bcrypt hash of the string "password"
+        hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W"
+        username: "admin"
+        userID: "08a8684b-db88-4b73-90a9-3cd1661f5466"
+    # required for staticPasswords
+    enablePasswordDB: true
--- a/acceptance_test/kubeconfig_oidc.yaml
+++ b/acceptance_test/kubeconfig_oidc.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=https://kubelogin-acceptance-test-control-plane:30443/dex
+      - --oidc-client-id=YOUR_CLIENT_ID
+      - --oidc-client-secret=YOUR_CLIENT_SECRET
+      - --oidc-extra-scope=email
+      - --certificate-authority=output/ca.crt
+      - --token-cache-dir=output/token-cache
+      - --listen-address=127.0.0.1:8000
+      - --skip-open-browser
+      - -v1
+      command: kubectl
+contexts:
+- context:
+    cluster: kind-kubelogin-acceptance-test
+    user: oidc
+  name: oidc
+current-context: oidc
--- a/acceptance_test/openssl.cnf
+++ b/acceptance_test/openssl.cnf
@@ -0,0 +1,15 @@
+[ req ]
+distinguished_name	= req_distinguished_name
+req_extensions = v3_req
+
+[ req_distinguished_name ]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+extendedKeyUsage = serverAuth
+
+[alt_names]
+DNS.1 = kubelogin-acceptance-test-control-plane
+DNS.2 = kubelogin-acceptance-test-control-plane:30443
--- a/acceptance_test/role.yaml
+++ b/acceptance_test/role.yaml
@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: readonly-all-resources
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: readonly-all-resources
+subjects:
+  - kind: User
+    name: admin@example.com
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: readonly-all-resources
+  apiGroup: rbac.authorization.k8s.io