Refactor: system test (#386)

2026-02-14 16:39:51 +00:00 · 2020-09-26 16:36:05 +09:00
parent ed0a5318ec
commit 7bc76a5e79
13 changed files with 169 additions and 113 deletions
--- a/.github/workflows/system-test.yaml
+++ b/.github/workflows/system-test.yaml
@@ -12,7 +12,7 @@ jobs:
    steps:
      - uses: actions/setup-go@v2
        with:
-          go-version: 1.14
+          go-version: 1.15
        id: go
      - uses: actions/checkout@v2
      - uses: actions/cache@v2
@@ -24,6 +24,6 @@ jobs:
      # https://packages.ubuntu.com/xenial/libnss3-tools
      - run: sudo apt update
      - run: sudo apt install -y libnss3-tools
+      - run: mkdir -p ~/.pki/nssdb
      - run: echo '127.0.0.1 dex-server' | sudo tee -a /etc/hosts
-      - run: make -C system_test -j3 setup
-      - run: make -C system_test test
+      - run: make -C system_test -j3
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,5 @@
 /.idea

-/system_test/output/
 /acceptance_test/output/

 /dist/output
--- a/system_test/.gitignore
+++ b/system_test/.gitignore
@@ -0,0 +1,6 @@
+/bin/
+
+/cert/ca.*
+/cert/server.*
+
+/cluster/kubeconfig.yaml
--- a/system_test/Makefile
+++ b/system_test/Makefile
@@ -1,109 +1,37 @@
-CLUSTER_NAME := kubelogin-system-test
-OUTPUT_DIR := $(CURDIR)/output
+CERT_DIR := cert

-PATH := $(PATH):$(OUTPUT_DIR)/bin
-export PATH
-KUBECONFIG := $(OUTPUT_DIR)/kubeconfig.yaml
-export KUBECONFIG
-
-# run the login script instead of opening chrome
-BROWSER := $(OUTPUT_DIR)/bin/chromelogin
-export BROWSER
-
-.PHONY: test
-test: build
-	# see the setup instruction
-	kubectl oidc-login setup \
-		--oidc-issuer-url=https://dex-server:10443/dex \
-		--oidc-client-id=YOUR_CLIENT_ID \
-		--oidc-client-secret=YOUR_CLIENT_SECRET \
-		--oidc-extra-scope=email \
-		--certificate-authority=$(OUTPUT_DIR)/ca.crt
-	# set up the kubeconfig
-	kubectl config set-credentials oidc \
-		--exec-api-version=client.authentication.k8s.io/v1beta1 \
-		--exec-command=kubectl \
-		--exec-arg=oidc-login \
-		--exec-arg=get-token \
-		--exec-arg=--oidc-issuer-url=https://dex-server:10443/dex \
-		--exec-arg=--oidc-client-id=YOUR_CLIENT_ID \
-		--exec-arg=--oidc-client-secret=YOUR_CLIENT_SECRET \
-		--exec-arg=--oidc-extra-scope=email \
-		--exec-arg=--certificate-authority=$(OUTPUT_DIR)/ca.crt
-	# make sure we can access the cluster
-	kubectl --user=oidc cluster-info
-	# switch the current context
-	kubectl config set-context --current --user=oidc
-	# make sure we can access the cluster
-	kubectl cluster-info
+.PHONY: login
+login: setup
+	$(MAKE) -C login

 .PHONY: setup
-setup: build dex cluster setup-chrome
+setup: dex cluster setup-chrome
+
+.PHONY: dex
+dex: cert
+	$(MAKE) -C dex
+
+.PHONY: cluster
+cluster: cert
+	$(MAKE) -C cluster

 .PHONY: setup-chrome
-setup-chrome: $(OUTPUT_DIR)/ca.crt
+setup-chrome: cert
 	# add the dex server certificate to the trust store
-	mkdir -p ~/.pki/nssdb
-	cd ~/.pki/nssdb && certutil -A -d sql:. -n dex -i $(OUTPUT_DIR)/ca.crt -t "TC,,"
+	if [ -d ~/.pki/nssdb ]; then certutil -A -d sql:$(HOME)/.pki/nssdb -n dex -i $(CERT_DIR)/ca.crt -t "TC,,"; fi

-# build binaries
-.PHONY: build
-build: $(OUTPUT_DIR)/bin/kubectl-oidc_login $(OUTPUT_DIR)/bin/chromelogin
-$(OUTPUT_DIR)/bin/kubectl-oidc_login:
-	go build -o $@ ..
-$(OUTPUT_DIR)/bin/chromelogin: chromelogin/main.go
-	go build -o $@ ./chromelogin
+.PHONY: cert
+cert:
+	$(MAKE) -C cert

-# create a Dex server
-.PHONY: dex
-dex: $(OUTPUT_DIR)/server.crt $(OUTPUT_DIR)/server.key
-	docker create --name dex-server -p 10443:10443 --network kind quay.io/dexidp/dex:v2.21.0 serve /dex.yaml
-	docker cp $(OUTPUT_DIR)/server.crt dex-server:/
-	docker cp $(OUTPUT_DIR)/server.key dex-server:/
-	docker cp dex.yaml dex-server:/
-	docker start dex-server
-	docker logs dex-server
+.PHONY: terminate
+terminate:
+	$(MAKE) -C cluster terminate
+	$(MAKE) -C dex terminate

-$(OUTPUT_DIR)/ca.key:
-	mkdir -p $(OUTPUT_DIR)
-	openssl genrsa -out $@ 2048
-$(OUTPUT_DIR)/ca.csr: $(OUTPUT_DIR)/ca.key
-	openssl req -new -key $(OUTPUT_DIR)/ca.key -out $@ -subj "/CN=dex-ca" -config openssl.cnf
-$(OUTPUT_DIR)/ca.crt: $(OUTPUT_DIR)/ca.key $(OUTPUT_DIR)/ca.csr
-	openssl x509 -req -in $(OUTPUT_DIR)/ca.csr -signkey $(OUTPUT_DIR)/ca.key -out $@ -days 10
-$(OUTPUT_DIR)/server.key:
-	mkdir -p $(OUTPUT_DIR)
-	openssl genrsa -out $@ 2048
-$(OUTPUT_DIR)/server.csr: openssl.cnf $(OUTPUT_DIR)/server.key
-	openssl req -new -key $(OUTPUT_DIR)/server.key -out $@ -subj "/CN=dex-server" -config openssl.cnf
-$(OUTPUT_DIR)/server.crt: openssl.cnf $(OUTPUT_DIR)/server.csr $(OUTPUT_DIR)/ca.crt $(OUTPUT_DIR)/ca.key
-	openssl x509 -req -in $(OUTPUT_DIR)/server.csr -CA $(OUTPUT_DIR)/ca.crt -CAkey $(OUTPUT_DIR)/ca.key -CAcreateserial -out $@ -sha256 -days 10 -extensions v3_req -extfile openssl.cnf
-
-# create a Kubernetes cluster
-.PHONY: cluster
-cluster: dex create-cluster
-	# add the Dex container IP to /etc/hosts
-	docker inspect -f '{{.NetworkSettings.Networks.kind.IPAddress}}' dex-server | sed -e 's,$$, dex-server,' | \
-		docker exec -i $(CLUSTER_NAME)-control-plane tee -a /etc/hosts
-	# wait for kube-apiserver oidc initialization
-	# (oidc authenticator will retry oidc discovery every 10s)
-	sleep 10
-
-.PHONY: create-cluster
-create-cluster: $(OUTPUT_DIR)/ca.crt
-	cp $(OUTPUT_DIR)/ca.crt /tmp/kubelogin-system-test-dex-ca.crt
-	kind create cluster --name $(CLUSTER_NAME) --config cluster.yaml
-	kubectl create clusterrole cluster-readonly --verb=get,watch,list --resource='*.*'
-	kubectl create clusterrolebinding cluster-readonly --clusterrole=cluster-readonly --user=admin@example.com
-
-# clean up the resources
 .PHONY: clean
 clean:
-	-rm -r $(OUTPUT_DIR)
-.PHONY: delete-cluster
-delete-cluster:
-	kind delete cluster --name $(CLUSTER_NAME)
-.PHONY: delete-dex
-delete-dex:
-	docker stop dex-server
-	docker rm dex-server
+	$(MAKE) -C cert clean
+	$(MAKE) -C cluster clean
+	$(MAKE) -C dex clean
+	$(MAKE) -C login clean
--- a/system_test/README.md
+++ b/system_test/README.md
@@ -30,14 +30,14 @@ It prepares the following resources:
 1. Generate a pair of CA certificate and TLS server certificate for Dex.
 1. Run Dex on a container.
 1. Create a Kubernetes cluster using Kind.
-1. Mutate `/etc/hosts` of the CI machine to access Dex.
-1. Mutate `/etc/hosts` of the kube-apiserver pod to access Dex.
+1. Mutate `/etc/hosts` of the machine so that the browser access Dex.
+1. Mutate `/etc/hosts` of the kind container so that kube-apiserver access Dex.

 It performs the test by the following steps:

 1. Run kubectl.
 1. kubectl automatically runs kubelogin.
-1. kubelogin automatically runs [chromelogin](chromelogin).
+1. kubelogin automatically runs [chromelogin](login/chromelogin).
 1. chromelogin opens the browser, navigates to `http://localhost:8000` and enter the username and password.
 1. kubelogin gets an authorization code from the browser.
 1. kubelogin gets a token.
@@ -54,21 +54,30 @@ You need to set up the following components:
 - Kind
 - Chrome or Chromium

-You need to add the following line to `/etc/hosts` so that the browser can access the Dex.
+Add the following line to `/etc/hosts` so that the browser can access the Dex.

 ```
 127.0.0.1 dex-server
 ```

+Generate CA certificate and add `cert/ca.crt` into your trust store.
+For macOS, you can add it by Keychain.
+
+```shell script
+make -C cert
+```
+
 Run the test.

 ```shell script
-# run the test
 make
+```

-# clean up
-make delete-cluster
-make delete-dex
+Clean up.
+
+```shell script
+make terminate
+make clean
 ```


@@ -102,11 +111,11 @@ Consider the following issues:

 As a result,

- kube-apiserver uses the CA certificate of `/usr/local/share/ca-certificates/dex-ca.crt`. See the `extraMounts` section of [`cluster.yaml`](cluster.yaml).
+- kube-apiserver uses the CA certificate of `/usr/local/share/ca-certificates/dex-ca.crt`. See the `extraMounts` section of [`cluster.yaml`](cluster/cluster.yaml).
 - kubelogin uses the CA certificate in `output/ca.crt`.
 - Chrome uses the CA certificate in `~/.pki/nssdb`.

 ### Test environment

- Set the issuer URL to kube-apiserver. See [`cluster.yaml`](cluster.yaml).
- Set `BROWSER` environment variable to run [`chromelogin`](chromelogin) by `xdg-open`.
+- Set the issuer URL to kube-apiserver. See [`cluster.yaml`](cluster/cluster.yaml).
+- Set `BROWSER` environment variable to run [`chromelogin`](login/chromelogin) by `xdg-open`.
--- a/system_test/cert/Makefile
+++ b/system_test/cert/Makefile
@@ -0,0 +1,19 @@
+.PHONY: all
+all: ca.key ca.crt server.key server.crt
+
+ca.key:
+	openssl genrsa -out $@ 2048
+ca.csr: ca.key
+	openssl req -new -key ca.key -out $@ -subj "/CN=dex-ca" -config openssl.cnf
+ca.crt: ca.key ca.csr
+	openssl x509 -req -in ca.csr -signkey ca.key -out $@ -days 10
+server.key:
+	openssl genrsa -out $@ 2048
+server.csr: openssl.cnf server.key
+	openssl req -new -key server.key -out $@ -subj "/CN=dex-server" -config openssl.cnf
+server.crt: openssl.cnf server.csr ca.crt ca.key
+	openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out $@ -sha256 -days 10 -extensions v3_req -extfile openssl.cnf
+
+.PHONY: clean
+clean:
+	-rm ca.* server.*
--- a/system_test/cert/openssl.cnf
+++ b/system_test/cert/openssl.cnf
--- a/system_test/cluster/Makefile
+++ b/system_test/cluster/Makefile
@@ -0,0 +1,23 @@
+CLUSTER_NAME := kubelogin-system-test
+CERT_DIR := ../cert
+
+KUBECONFIG := kubeconfig.yaml
+export KUBECONFIG
+
+.PHONY: cluster
+cluster:
+	cp $(CERT_DIR)/ca.crt /tmp/kubelogin-system-test-dex-ca.crt
+	kind create cluster --name $(CLUSTER_NAME) --config cluster.yaml
+	# add the Dex container IP to /etc/hosts
+	docker inspect -f '{{.NetworkSettings.Networks.kind.IPAddress}}' dex-server | sed -e 's,$$, dex-server,' | \
+		docker exec -i $(CLUSTER_NAME)-control-plane tee -a /etc/hosts
+	# wait for kube-apiserver oidc initialization
+	# (oidc authenticator will retry oidc discovery every 10s)
+	sleep 10
+	# add the cluster role
+	kubectl create clusterrole cluster-readonly --verb=get,watch,list --resource='*.*'
+	kubectl create clusterrolebinding cluster-readonly --clusterrole=cluster-readonly --user=admin@example.com
+
+.PHONY: terminate
+terminate:
+	kind delete cluster --name $(CLUSTER_NAME)
--- a/system_test/cluster/cluster.yaml
+++ b/system_test/cluster/cluster.yaml
--- a/system_test/dex/Makefile
+++ b/system_test/dex/Makefile
@@ -0,0 +1,20 @@
+CERT_DIR := ../cert
+
+.PHONY: dex
+dex: dex.yaml
+	# wait for kind network
+	while true; do if docker network inspect kind; then break; fi; sleep 1; done
+	# create a container
+	docker create --name dex-server -p 10443:10443 --network kind quay.io/dexidp/dex:v2.21.0 serve /dex.yaml
+	# deploy the config
+	docker cp $(CERT_DIR)/server.crt dex-server:/
+	docker cp $(CERT_DIR)/server.key dex-server:/
+	docker cp dex.yaml dex-server:/
+	# start the container
+	docker start dex-server
+	docker logs dex-server
+
+.PHONY: terminate
+terminate:
+	docker stop dex-server
+	docker rm dex-server
--- a/system_test/dex/dex.yaml
+++ b/system_test/dex/dex.yaml
--- a/system_test/login/Makefile
+++ b/system_test/login/Makefile
@@ -0,0 +1,52 @@
+CERT_DIR := ../cert
+
+BIN_DIR := $(PWD)/bin
+PATH := $(PATH):$(BIN_DIR)
+export PATH
+
+KUBECONFIG := ../cluster/kubeconfig.yaml
+export KUBECONFIG
+
+# run the login script instead of opening chrome
+BROWSER := $(BIN_DIR)/chromelogin
+export BROWSER
+
+.PHONY: test
+test: build
+	# see the setup instruction
+	kubectl oidc-login setup \
+		--oidc-issuer-url=https://dex-server:10443/dex \
+		--oidc-client-id=YOUR_CLIENT_ID \
+		--oidc-client-secret=YOUR_CLIENT_SECRET \
+		--oidc-extra-scope=email \
+		--certificate-authority=$(CERT_DIR)/ca.crt
+	# set up the kubeconfig
+	kubectl config set-credentials oidc \
+		--exec-api-version=client.authentication.k8s.io/v1beta1 \
+		--exec-command=kubectl \
+		--exec-arg=oidc-login \
+		--exec-arg=get-token \
+		--exec-arg=--oidc-issuer-url=https://dex-server:10443/dex \
+		--exec-arg=--oidc-client-id=YOUR_CLIENT_ID \
+		--exec-arg=--oidc-client-secret=YOUR_CLIENT_SECRET \
+		--exec-arg=--oidc-extra-scope=email \
+		--exec-arg=--certificate-authority=$(CERT_DIR)/ca.crt
+	# make sure we can access the cluster
+	kubectl --user=oidc cluster-info
+	# switch the current context
+	kubectl config set-context --current --user=oidc
+	# make sure we can access the cluster
+	kubectl cluster-info
+
+.PHONY: build
+build: $(BIN_DIR)/kubectl-oidc_login $(BIN_DIR)/chromelogin
+
+$(BIN_DIR)/kubectl-oidc_login:
+	go build -o $@ ../../
+
+$(BIN_DIR)/chromelogin: $(wildcard chromelogin/*.go)
+	go build -o $@ ./chromelogin
+
+.PHONY: clean
+clean:
+	-rm -r $(BIN_DIR)
--- a/system_test/login/chromelogin/main.go
+++ b/system_test/login/chromelogin/main.go