diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0f04682 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..2f40efd --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: [ "*" ] + pull_request: + branches: [ "*" ] + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2.3.0 + with: + version: v1.45.2 + only-new-issues: false + args: --timeout 5m --config .golangci.yml + diff: + name: diff + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-go@v2 + with: + go-version: '1.17' + - run: make yaml-installation-file + - name: Checking if YAML installer file is not aligned + run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi + - name: Checking if YAML installer generated untracked files + run: test -z "$(git ls-files --others --exclude-standard 2> /dev/null)" + - name: Checking if source code is not formatted + run: test -z "$(git diff 2> /dev/null)" diff --git a/.gitignore b/.gitignore index 8537fd2..56fa963 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,6 @@ bin **/*.kubeconfig **/*.crt **/*.key +**/*.pem +**/*.csr .DS_Store - diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..698490b --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,35 @@ +linters-settings: + gci: + sections: + - standard + - default + - prefix(github.com/clastix/kamaji) + +linters: + disable: + - wrapcheck + - gomnd + - scopelint + - golint + - interfacer + - maligned + - varnamelen + - testpackage + - tagliatelle + - paralleltest + - ireturn + - goerr113 + - gochecknoglobals + - exhaustivestruct + - wsl + - exhaustive + - lll + - gosec + - gomoddirectives + - godox + - gochecknoinits + - funlen + - dupl + - maintidx + - cyclop + enable-all: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..896df5c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Build the manager binary +FROM golang:1.17 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api/ api/ +COPY controllers/ controllers/ +COPY internal/ internal/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +COPY ./kamaji.yaml . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a6631bd --- /dev/null +++ b/Makefile @@ -0,0 +1,215 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.0.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# clastix.io/operator-bundle:$VERSION and clastix.io/operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= clastix.io/operator + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# Image URL to use all building/pushing image targets +IMG ?= controller:latest +# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) +CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.21 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +fmt: ## Run go fmt against code. + go fmt ./... + +vet: ## Run go vet against code. + go vet ./... + +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + +##@ Build + +build: generate fmt vet ## Build manager binary. + go build -o bin/manager main.go + +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +docker-build: test ## Build docker image with the manager. + docker build -t ${IMG} . + +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +##@ Deployment + +dev: generate manifests uninstall install rbac ## Full installation for development purposes + go fmt ./... + +load: dev docker-build + kind load docker-image --name kamaji ${IMG} + +rbac: manifests kustomize ## Install RBAC into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/rbac | kubectl apply -f - + +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/default | kubectl delete -f - + +yaml-installation-file: manifests kustomize ## Create yaml installation file + $(KUSTOMIZE) build config/default > config/install.yaml + + +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +controller-gen: ## Download controller-gen locally if necessary. + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1) + +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: ## Download kustomize locally if necessary. + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) + +ENVTEST = $(shell pwd)/bin/setup-envtest +envtest: ## Download envtest-setup locally if necessary. + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(1) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +go mod init tmp ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ +rm -rf $$TMP_DIR ;\ +} +endef + +.PHONY: bundle +bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files. + operator-sdk generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + operator-sdk bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$${OS}-$${ARCH}-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..3f1116c --- /dev/null +++ b/PROJECT @@ -0,0 +1,19 @@ +domain: clastix.io +layout: +- go.kubebuilder.io/v3 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: operator +repo: github.com/clastix/kamaji +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: clastix.io + group: kamaji + kind: TenantControlPlane + path: github.com/clastix/kamaji/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index 81a76eb..baa460c 100644 --- a/README.md +++ b/README.md @@ -119,4 +119,4 @@ A. Lighter Multi-Tenancy solutions, like Capsule shares the Kubernetes control p Q. So I need a costly cloud infrastructure to try Kamaji? -A. No, it is possible to try Kamaji on your laptop with [KinD](./deploy/kind/README.md). We're porting it also on Canonical MicroK8s as an add-on, _tbd_. \ No newline at end of file +A. No, it is possible to try Kamaji on your laptop with [KinD](./deploy/kind/README.md). We're porting it also on Canonical MicroK8s as an add-on, _tbd_. diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..a20981d --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,23 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +// Package v1alpha1 contains API Schema definitions for the kamaji v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=kamaji.clastix.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "kamaji.clastix.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha1/tenantcontrolplane_funcs.go b/api/v1alpha1/tenantcontrolplane_funcs.go new file mode 100644 index 0000000..a7c5aca --- /dev/null +++ b/api/v1alpha1/tenantcontrolplane_funcs.go @@ -0,0 +1,43 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Client) (string, error) { + var loadBalancerStatus corev1.LoadBalancerStatus + + svc := &corev1.Service{} + err := client.Get(ctx, types.NamespacedName{Namespace: in.GetNamespace(), Name: in.GetName()}, svc) + if err != nil { + return "", errors.Wrap(err, "cannot retrieve Service for the TenantControlPlane") + } + + switch { + case len(in.Spec.NetworkProfile.Address) > 0: + // Returning the hard-coded value in the specification in case of non LoadBalanced resources + return in.Spec.NetworkProfile.Address, nil + case svc.Spec.Type == corev1.ServiceTypeLoadBalancer: + loadBalancerStatus = svc.Status.LoadBalancer + if len(loadBalancerStatus.Ingress) == 0 { + return "", fmt.Errorf("cannot retrieve the TenantControlPlane address, Service resource is not yet exposed as LoadBalancer") + } + + for _, lb := range loadBalancerStatus.Ingress { + if ip := lb.IP; len(ip) > 0 { + return ip, nil + } + } + } + + return "", fmt.Errorf("the actual resource doesn't have yet a valid IP address") +} diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go new file mode 100644 index 0000000..b1826d7 --- /dev/null +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -0,0 +1,296 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/clastix/kamaji/internal/etcd" +) + +// NetworkProfileSpec defines the desired state of NetworkProfile. +type NetworkProfileSpec struct { + // Address where API server of will be exposed. + // In case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager. + Address string `json:"address,omitempty"` + // AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of + // ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort) + AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"` + // Port where API server of will be exposed + // +kubebuilder:default=6443 + Port int32 `json:"port"` + + // Domain of the tenant control plane + Domain string `json:"domain"` + // Kubernetes Service + ServiceCIDR string `json:"serviceCidr"` + // CIDR for Kubernetes Pods + PodCIDR string `json:"podCidr"` + DNSServiceIPs []string `json:"dnsServiceIPs"` +} + +type KubeletSpec struct { + // CGroupFS defines the cgroup driver for Kubelet + // https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/ + CGroupFS CGroupDriver `json:"cgroupfs,omitempty"` +} + +// KubernetesSpec defines the desired state of Kubernetes. +type KubernetesSpec struct { + // Kubernetes Version for the tenant control plane + Version string `json:"version"` + Kubelet KubeletSpec `json:"kubelet"` + + // List of enabled Admission Controllers for the Tenant cluster. + // Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers + // +kubebuilder:default=CertificateApproval;CertificateSigning;CertificateSubjectRestriction;DefaultIngressClass;DefaultStorageClass;DefaultTolerationSeconds;LimitRanger;MutatingAdmissionWebhook;NamespaceLifecycle;PersistentVolumeClaimResize;Priority;ResourceQuota;RuntimeClass;ServiceAccount;StorageObjectInUseProtection;TaintNodesByCondition;ValidatingAdmissionWebhook + AdmissionControllers AdmissionControllers `json:"admissionControllers,omitempty"` +} + +// AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. +type AdditionalMetadata struct { + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, +// such as the number of Pod replicas, the Service resource, or the Ingress. +type ControlPlane struct { + // Defining the options for the deployed Tenant Control Plane as Deployment resource. + Deployment DeploymentSpec `json:"deployment,omitempty"` + // Defining the options for the Tenant Control Plane Service resource. + Service ServiceSpec `json:"service"` + // Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane + Ingress IngressSpec `json:"ingress,omitempty"` +} + +// IngressSpec defines the options for the ingress which will expose API Server of the Tenant Control Plane. +type IngressSpec struct { + AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"` + Enabled bool `json:"enabled"` + IngressClassName string `json:"ingressClassName,omitempty"` + // Hostname is an optional field which will be used as Ingress's Host. If it is not defined, + // Ingress's host will be "..", where domain is specified under NetworkProfileSpec + Hostname string `json:"hostname,omitempty"` +} + +type DeploymentSpec struct { + // +kubebuilder:default=2 + Replicas int32 `json:"replicas,omitempty"` + AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"` +} + +type ServiceSpec struct { + AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"` + // ServiceType allows specifying how to expose the Tenant Control Plane. + ServiceType ServiceType `json:"serviceType"` +} + +// TenantControlPlaneSpec defines the desired state of TenantControlPlane. +type TenantControlPlaneSpec struct { + ControlPlane ControlPlane `json:"controlPlane"` + + // Kubernetes specification for tenant control plane + Kubernetes KubernetesSpec `json:"kubernetes"` + + // NetworkProfile specifies how the network is + NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"` +} + +// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. +type APIServerCertificatesStatus struct { + SecretName string `json:"secretName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. +type ETCDCertificateStatus struct { + SecretName string `json:"secretName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. +type ETCDCertificatesStatus struct { + APIServer ETCDCertificateStatus `json:"apiServer,omitempty"` + CA ETCDCertificateStatus `json:"ca,omitempty"` +} + +// CertificatePrivateKeyPair defines the status. +type CertificatePrivateKeyPairStatus struct { + SecretName string `json:"secretName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// CertificatePrivateKeyPair defines the status. +type PublicKeyPrivateKeyPairStatus struct { + SecretName string `json:"secretName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// ETCDCertificates defines the observed state of ETCD Certificates. +type CertificatesStatus struct { + CA CertificatePrivateKeyPairStatus `json:"ca,omitempty"` + APIServer CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"` + APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"` + FrontProxyCA CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"` + FrontProxyClient CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"` + SA PublicKeyPrivateKeyPairStatus `json:"sa,omitempty"` + ETCD *ETCDCertificatesStatus `json:"etcd,omitempty"` +} + +// ETCDStatus defines the observed state of ETCDStatus. +type ETCDStatus struct { + Role etcd.Role `json:"role,omitempty"` + User etcd.User `json:"user,omitempty"` +} + +// StorageStatus defines the observed state of StorageStatus. +type StorageStatus struct { + ETCD *ETCDStatus `json:"etcd,omitempty"` +} + +// TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig. +type KubeconfigStatus struct { + SecretName string `json:"secretName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// TenantControlPlaneKubeconfigsStatus stores information about all the generated kubeconfigs. +type KubeconfigsStatus struct { + Admin KubeconfigStatus `json:"admin,omitempty"` + ControllerManager KubeconfigStatus `json:"controlerManager,omitempty"` + Scheduler KubeconfigStatus `json:"scheduler,omitempty"` +} + +// KubeadmConfigStatus contains the status of the configuration required by kubeadm. +type KubeadmConfigStatus struct { + ConfigmapName string `json:"configmapName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` + ResourceVersion string `json:"resourceVersion"` +} + +// KubeadmPhasesStatus contains the status of of a kubeadm phase action. +type KubeadmPhaseStatus struct { + KubeadmConfigResourceVersion string `json:"kubeadmConfigResourceVersion,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// KubeadmPhasesStatus contains the status of the different kubeadm phases action. +type KubeadmPhasesStatus struct { + UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"` + UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"` + AddonCoreDNS KubeadmPhaseStatus `json:"addonCoreDNS"` + AddonKubeProxy KubeadmPhaseStatus `json:"addonKubeProxy"` + BootstrapToken KubeadmPhaseStatus `json:"bootstrapToken"` +} + +// TenantControlPlaneStatus defines the observed state of TenantControlPlane. +type TenantControlPlaneStatus struct { + // Storage Status contains information about Kubernetes storage system + Storage StorageStatus `json:"storage,omitempty"` + // Certificates contains information about the different certificates + // that are necessary to run a kubernetes control plane + Certificates CertificatesStatus `json:"certificates,omitempty"` + // KubeConfig contains information about the kubenconfigs that control plane pieces need + KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"` + // Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster + Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"` + // KubeadmConfig contains the status of the configuration required by kubeadm + KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"` + // KubeadmPhase contains the status of the kubeadm phases action + KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"` + // ControlPlaneEndpoint contains the status of the kubernetes control plane + ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"` +} + +// KubernetesStatus defines the status of the resources deployed in the management cluster, +// such as Deployment and Service. +type KubernetesStatus struct { + // KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status. + Version KubernetesVersion `json:"version,omitempty"` + Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"` + Service KubernetesServiceStatus `json:"service,omitempty"` + Ingress KubernetesIngressStatus `json:"ingress,omitempty"` +} + +// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady +type KubernetesVersionStatus string + +var ( + VersionProvisioning KubernetesVersionStatus = "Provisioning" + VersionUpgrading KubernetesVersionStatus = "Upgrading" + VersionReady KubernetesVersionStatus = "Ready" + VersionNotReady KubernetesVersionStatus = "NotReady" +) + +type KubernetesVersion struct { + // Version is the running Kubernetes version of the Tenant Control Plane. + Version string `json:"version,omitempty"` + // +kubebuilder:default=Provisioning + // Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade. + Status *KubernetesVersionStatus `json:"status"` +} + +// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster. +type KubernetesDeploymentStatus struct { + appv1.DeploymentStatus `json:",inline"` + // The name of the Deployment for the given cluster. + Name string `json:"name"` + // The namespace which the Deployment for the given cluster is deployed. + Namespace string `json:"namespace"` +} + +// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster. +type KubernetesServiceStatus struct { + corev1.ServiceStatus `json:",inline"` + // The name of the Service for the given cluster. + Name string `json:"name"` + // The namespace which the Service for the given cluster is deployed. + Namespace string `json:"namespace"` + // The port where the service is running + Port int32 `json:"port"` +} + +// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster. +type KubernetesIngressStatus struct { + networkingv1.IngressStatus `json:",inline"` + // The name of the Ingress for the given cluster. + Name string `json:"name"` + // The namespace which the Ingress for the given cluster is deployed. + Namespace string `json:"namespace"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=tcp +// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Kubernetes version" +// +kubebuilder:printcolumn:name="Control-Plane-Endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)" +// +kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age" + +// TenantControlPlane is the Schema for the tenantcontrolplanes API. +type TenantControlPlane struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TenantControlPlaneSpec `json:"spec,omitempty"` + Status TenantControlPlaneStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// TenantControlPlaneList contains a list of TenantControlPlane. +type TenantControlPlaneList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TenantControlPlane `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TenantControlPlane{}, &TenantControlPlaneList{}) +} diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go new file mode 100644 index 0000000..8cb98d1 --- /dev/null +++ b/api/v1alpha1/types.go @@ -0,0 +1,37 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import corev1 "k8s.io/api/core/v1" + +// +kubebuilder:validation:Enum=AlwaysAdmit;AlwaysDeny;AlwaysPullImages;CertificateApproval;CertificateSigning;CertificateSubjectRestriction;DefaultIngressClass;DefaultStorageClass;DefaultTolerationSeconds;DenyEscalatingExec;DenyExecOnPrivileged;DenyServiceExternalIPs;EventRateLimit;ExtendedResourceToleration;ImagePolicyWebhook;LimitPodHardAntiAffinityTopology;LimitRanger;MutatingAdmissionWebhook;NamespaceAutoProvision;NamespaceExists;NamespaceLifecycle;NodeRestriction;OwnerReferencesPermissionEnforcement;PersistentVolumeClaimResize;PersistentVolumeLabel;PodNodeSelector;PodSecurity;PodSecurityPolicy;PodTolerationRestriction;Priority;ResourceQuota;RuntimeClass;SecurityContextDeny;ServiceAccount;StorageObjectInUseProtection;TaintNodesByCondition;ValidatingAdmissionWebhook +type AdmissionController string + +type AdmissionControllers []AdmissionController + +func (a AdmissionControllers) ToSlice() []string { + out := make([]string, len(a)) + + for i, v := range a { + out[i] = string(v) + } + + return out +} + +// +kubebuilder:validation:Enum=systemd;cgroupfs +type CGroupDriver string + +func (c CGroupDriver) String() string { + return (string)(c) +} + +const ( + ServiceTypeLoadBalancer = (ServiceType)(corev1.ServiceTypeLoadBalancer) + ServiceTypeClusterIP = (ServiceType)(corev1.ServiceTypeClusterIP) + ServiceTypeNodePort = (ServiceType)(corev1.ServiceTypeNodePort) +) + +// +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer +type ServiceType corev1.ServiceType diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..469a9ed --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,598 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIServerCertificatesStatus) DeepCopyInto(out *APIServerCertificatesStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerCertificatesStatus. +func (in *APIServerCertificatesStatus) DeepCopy() *APIServerCertificatesStatus { + if in == nil { + return nil + } + out := new(APIServerCertificatesStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdditionalMetadata) DeepCopyInto(out *AdditionalMetadata) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalMetadata. +func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata { + if in == nil { + return nil + } + out := new(AdditionalMetadata) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in AdmissionControllers) DeepCopyInto(out *AdmissionControllers) { + { + in := &in + *out = make(AdmissionControllers, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdmissionControllers. +func (in AdmissionControllers) DeepCopy() AdmissionControllers { + if in == nil { + return nil + } + out := new(AdmissionControllers) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificatePrivateKeyPairStatus) DeepCopyInto(out *CertificatePrivateKeyPairStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificatePrivateKeyPairStatus. +func (in *CertificatePrivateKeyPairStatus) DeepCopy() *CertificatePrivateKeyPairStatus { + if in == nil { + return nil + } + out := new(CertificatePrivateKeyPairStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificatesStatus) DeepCopyInto(out *CertificatesStatus) { + *out = *in + in.CA.DeepCopyInto(&out.CA) + in.APIServer.DeepCopyInto(&out.APIServer) + in.APIServerKubeletClient.DeepCopyInto(&out.APIServerKubeletClient) + in.FrontProxyCA.DeepCopyInto(&out.FrontProxyCA) + in.FrontProxyClient.DeepCopyInto(&out.FrontProxyClient) + in.SA.DeepCopyInto(&out.SA) + if in.ETCD != nil { + in, out := &in.ETCD, &out.ETCD + *out = new(ETCDCertificatesStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificatesStatus. +func (in *CertificatesStatus) DeepCopy() *CertificatesStatus { + if in == nil { + return nil + } + out := new(CertificatesStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlPlane) DeepCopyInto(out *ControlPlane) { + *out = *in + in.Deployment.DeepCopyInto(&out.Deployment) + in.Service.DeepCopyInto(&out.Service) + in.Ingress.DeepCopyInto(&out.Ingress) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlane. +func (in *ControlPlane) DeepCopy() *ControlPlane { + if in == nil { + return nil + } + out := new(ControlPlane) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { + *out = *in + in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. +func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { + if in == nil { + return nil + } + out := new(DeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDCertificateStatus) DeepCopyInto(out *ETCDCertificateStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDCertificateStatus. +func (in *ETCDCertificateStatus) DeepCopy() *ETCDCertificateStatus { + if in == nil { + return nil + } + out := new(ETCDCertificateStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDCertificatesStatus) DeepCopyInto(out *ETCDCertificatesStatus) { + *out = *in + in.APIServer.DeepCopyInto(&out.APIServer) + in.CA.DeepCopyInto(&out.CA) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDCertificatesStatus. +func (in *ETCDCertificatesStatus) DeepCopy() *ETCDCertificatesStatus { + if in == nil { + return nil + } + out := new(ETCDCertificatesStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETCDStatus) DeepCopyInto(out *ETCDStatus) { + *out = *in + in.Role.DeepCopyInto(&out.Role) + in.User.DeepCopyInto(&out.User) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStatus. +func (in *ETCDStatus) DeepCopy() *ETCDStatus { + if in == nil { + return nil + } + out := new(ETCDStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressSpec) DeepCopyInto(out *IngressSpec) { + *out = *in + in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec. +func (in *IngressSpec) DeepCopy() *IngressSpec { + if in == nil { + return nil + } + out := new(IngressSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmConfigStatus. +func (in *KubeadmConfigStatus) DeepCopy() *KubeadmConfigStatus { + if in == nil { + return nil + } + out := new(KubeadmConfigStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeadmPhaseStatus) DeepCopyInto(out *KubeadmPhaseStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmPhaseStatus. +func (in *KubeadmPhaseStatus) DeepCopy() *KubeadmPhaseStatus { + if in == nil { + return nil + } + out := new(KubeadmPhaseStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeadmPhasesStatus) DeepCopyInto(out *KubeadmPhasesStatus) { + *out = *in + in.UploadConfigKubeadm.DeepCopyInto(&out.UploadConfigKubeadm) + in.UploadConfigKubelet.DeepCopyInto(&out.UploadConfigKubelet) + in.AddonCoreDNS.DeepCopyInto(&out.AddonCoreDNS) + in.AddonKubeProxy.DeepCopyInto(&out.AddonKubeProxy) + in.BootstrapToken.DeepCopyInto(&out.BootstrapToken) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmPhasesStatus. +func (in *KubeadmPhasesStatus) DeepCopy() *KubeadmPhasesStatus { + if in == nil { + return nil + } + out := new(KubeadmPhasesStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeconfigStatus) DeepCopyInto(out *KubeconfigStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigStatus. +func (in *KubeconfigStatus) DeepCopy() *KubeconfigStatus { + if in == nil { + return nil + } + out := new(KubeconfigStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeconfigsStatus) DeepCopyInto(out *KubeconfigsStatus) { + *out = *in + in.Admin.DeepCopyInto(&out.Admin) + in.ControllerManager.DeepCopyInto(&out.ControllerManager) + in.Scheduler.DeepCopyInto(&out.Scheduler) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigsStatus. +func (in *KubeconfigsStatus) DeepCopy() *KubeconfigsStatus { + if in == nil { + return nil + } + out := new(KubeconfigsStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletSpec) DeepCopyInto(out *KubeletSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletSpec. +func (in *KubeletSpec) DeepCopy() *KubeletSpec { + if in == nil { + return nil + } + out := new(KubeletSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesDeploymentStatus) DeepCopyInto(out *KubernetesDeploymentStatus) { + *out = *in + in.DeploymentStatus.DeepCopyInto(&out.DeploymentStatus) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDeploymentStatus. +func (in *KubernetesDeploymentStatus) DeepCopy() *KubernetesDeploymentStatus { + if in == nil { + return nil + } + out := new(KubernetesDeploymentStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesIngressStatus) DeepCopyInto(out *KubernetesIngressStatus) { + *out = *in + in.IngressStatus.DeepCopyInto(&out.IngressStatus) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesIngressStatus. +func (in *KubernetesIngressStatus) DeepCopy() *KubernetesIngressStatus { + if in == nil { + return nil + } + out := new(KubernetesIngressStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesServiceStatus) DeepCopyInto(out *KubernetesServiceStatus) { + *out = *in + in.ServiceStatus.DeepCopyInto(&out.ServiceStatus) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesServiceStatus. +func (in *KubernetesServiceStatus) DeepCopy() *KubernetesServiceStatus { + if in == nil { + return nil + } + out := new(KubernetesServiceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) { + *out = *in + out.Kubelet = in.Kubelet + if in.AdmissionControllers != nil { + in, out := &in.AdmissionControllers, &out.AdmissionControllers + *out = make(AdmissionControllers, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec. +func (in *KubernetesSpec) DeepCopy() *KubernetesSpec { + if in == nil { + return nil + } + out := new(KubernetesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) { + *out = *in + in.Version.DeepCopyInto(&out.Version) + in.Deployment.DeepCopyInto(&out.Deployment) + in.Service.DeepCopyInto(&out.Service) + in.Ingress.DeepCopyInto(&out.Ingress) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus. +func (in *KubernetesStatus) DeepCopy() *KubernetesStatus { + if in == nil { + return nil + } + out := new(KubernetesStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesVersion) DeepCopyInto(out *KubernetesVersion) { + *out = *in + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(KubernetesVersionStatus) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesVersion. +func (in *KubernetesVersion) DeepCopy() *KubernetesVersion { + if in == nil { + return nil + } + out := new(KubernetesVersion) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) { + *out = *in + if in.DNSServiceIPs != nil { + in, out := &in.DNSServiceIPs, &out.DNSServiceIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkProfileSpec. +func (in *NetworkProfileSpec) DeepCopy() *NetworkProfileSpec { + if in == nil { + return nil + } + out := new(NetworkProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PublicKeyPrivateKeyPairStatus) DeepCopyInto(out *PublicKeyPrivateKeyPairStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublicKeyPrivateKeyPairStatus. +func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStatus { + if in == nil { + return nil + } + out := new(PublicKeyPrivateKeyPairStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { + *out = *in + in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec. +func (in *ServiceSpec) DeepCopy() *ServiceSpec { + if in == nil { + return nil + } + out := new(ServiceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageStatus) DeepCopyInto(out *StorageStatus) { + *out = *in + if in.ETCD != nil { + in, out := &in.ETCD, &out.ETCD + *out = new(ETCDStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus. +func (in *StorageStatus) DeepCopy() *StorageStatus { + if in == nil { + return nil + } + out := new(StorageStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantControlPlane) DeepCopyInto(out *TenantControlPlane) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlane. +func (in *TenantControlPlane) DeepCopy() *TenantControlPlane { + if in == nil { + return nil + } + out := new(TenantControlPlane) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TenantControlPlane) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantControlPlaneList) DeepCopyInto(out *TenantControlPlaneList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TenantControlPlane, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneList. +func (in *TenantControlPlaneList) DeepCopy() *TenantControlPlaneList { + if in == nil { + return nil + } + out := new(TenantControlPlaneList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TenantControlPlaneList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) { + *out = *in + in.ControlPlane.DeepCopyInto(&out.ControlPlane) + in.Kubernetes.DeepCopyInto(&out.Kubernetes) + in.NetworkProfile.DeepCopyInto(&out.NetworkProfile) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneSpec. +func (in *TenantControlPlaneSpec) DeepCopy() *TenantControlPlaneSpec { + if in == nil { + return nil + } + out := new(TenantControlPlaneSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantControlPlaneStatus) DeepCopyInto(out *TenantControlPlaneStatus) { + *out = *in + in.Storage.DeepCopyInto(&out.Storage) + in.Certificates.DeepCopyInto(&out.Certificates) + in.KubeConfig.DeepCopyInto(&out.KubeConfig) + in.Kubernetes.DeepCopyInto(&out.Kubernetes) + in.KubeadmConfig.DeepCopyInto(&out.KubeadmConfig) + in.KubeadmPhase.DeepCopyInto(&out.KubeadmPhase) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatus. +func (in *TenantControlPlaneStatus) DeepCopy() *TenantControlPlaneStatus { + if in == nil { + return nil + } + out := new(TenantControlPlaneStatus) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml new file mode 100644 index 0000000..a82633b --- /dev/null +++ b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -0,0 +1,889 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: tenantcontrolplanes.kamaji.clastix.io +spec: + group: kamaji.clastix.io + names: + kind: TenantControlPlane + listKind: TenantControlPlaneList + plural: tenantcontrolplanes + shortNames: + - tcp + singular: tenantcontrolplane + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .spec.kubernetes.version + name: Version + type: string + - description: Kubernetes version + jsonPath: .status.kubernetesResources.version.status + name: Status + type: string + - description: Tenant Control Plane Endpoint (API server) + jsonPath: .status.controlPlaneEndpoint + name: Control-Plane-Endpoint + type: string + - description: Secret which contains admin kubeconfig + jsonPath: .status.kubeconfig.admin.secretName + name: Kubeconfig + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TenantControlPlane is the Schema for the tenantcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. + properties: + controlPlane: + description: ControlPlane defines how the Tenant Control Plane Kubernetes + resources must be created in the Admin Cluster, such as the number + of Pod replicas, the Service resource, or the Ingress. + properties: + deployment: + description: Defining the options for the deployed Tenant Control + Plane as Deployment resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, + such as labels and annotations, must be attached to the + created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + replicas: + default: 2 + format: int32 + type: integer + type: object + ingress: + description: Defining the options for an Optional Ingress which + will expose API Server of the Tenant Control Plane + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, + such as labels and annotations, must be attached to the + created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + enabled: + type: boolean + hostname: + description: Hostname is an optional field which will be used + as Ingress's Host. If it is not defined, Ingress's host + will be "..", where domain is + specified under NetworkProfileSpec + type: string + ingressClassName: + type: string + required: + - enabled + type: object + service: + description: Defining the options for the Tenant Control Plane + Service resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, + such as labels and annotations, must be attached to the + created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + serviceType: + description: ServiceType allows specifying how to expose the + Tenant Control Plane. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + required: + - serviceType + type: object + required: + - service + type: object + kubernetes: + description: Kubernetes specification for tenant control plane + properties: + admissionControllers: + default: + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - PersistentVolumeClaimResize + - Priority + - ResourceQuota + - RuntimeClass + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + description: 'List of enabled Admission Controllers for the Tenant + cluster. Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers' + items: + enum: + - AlwaysAdmit + - AlwaysDeny + - AlwaysPullImages + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - DenyEscalatingExec + - DenyExecOnPrivileged + - DenyServiceExternalIPs + - EventRateLimit + - ExtendedResourceToleration + - ImagePolicyWebhook + - LimitPodHardAntiAffinityTopology + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceAutoProvision + - NamespaceExists + - NamespaceLifecycle + - NodeRestriction + - OwnerReferencesPermissionEnforcement + - PersistentVolumeClaimResize + - PersistentVolumeLabel + - PodNodeSelector + - PodSecurity + - PodSecurityPolicy + - PodTolerationRestriction + - Priority + - ResourceQuota + - RuntimeClass + - SecurityContextDeny + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + type: string + type: array + kubelet: + properties: + cgroupfs: + description: CGroupFS defines the cgroup driver for Kubelet + https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/ + enum: + - systemd + - cgroupfs + type: string + type: object + version: + description: Kubernetes Version for the tenant control plane + type: string + required: + - kubelet + - version + type: object + networkProfile: + description: NetworkProfile specifies how the network is + properties: + address: + description: Address where API server of will be exposed. In case + of LoadBalancer Service, this can be empty in order to use the + exposed IP provided by the cloud controller manager. + type: string + allowAddressAsExternalIP: + description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address + in the section of ExternalIPs of the Kubernetes Service (only + ClusterIP or NodePort) + type: boolean + dnsServiceIPs: + items: + type: string + type: array + domain: + description: Domain of the tenant control plane + type: string + podCidr: + description: CIDR for Kubernetes Pods + type: string + port: + default: 6443 + description: Port where API server of will be exposed + format: int32 + type: integer + serviceCidr: + description: Kubernetes Service + type: string + required: + - dnsServiceIPs + - domain + - podCidr + - port + - serviceCidr + type: object + required: + - controlPlane + - kubernetes + type: object + status: + description: TenantControlPlaneStatus defines the observed state of TenantControlPlane. + properties: + certificates: + description: Certificates contains information about the different + certificates that are necessary to run a kubernetes control plane + properties: + apiServer: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + apiServerKubeletClient: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + etcd: + description: ETCDAPIServerCertificate defines the observed state + of ETCD Certificate for API server. + properties: + apiServer: + description: ETCDAPIServerCertificate defines the observed + state of ETCD Certificate for API server. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: ETCDAPIServerCertificate defines the observed + state of ETCD Certificate for API server. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + frontProxyCA: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + frontProxyClient: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + sa: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint contains the status of the kubernetes + control plane + type: string + kubeadmPhase: + description: KubeadmPhase contains the status of the kubeadm phases + action + properties: + addonCoreDNS: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + addonKubeProxy: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + bootstrapToken: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubeadm: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubelet: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + required: + - addonCoreDNS + - addonKubeProxy + - bootstrapToken + - uploadConfigKubeadm + - uploadConfigKubelet + type: object + kubeadmconfig: + description: KubeadmConfig contains the status of the configuration + required by kubeadm + properties: + configmapName: + type: string + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + required: + - resourceVersion + type: object + kubeconfig: + description: KubeConfig contains information about the kubenconfigs + that control plane pieces need + properties: + admin: + description: TenantControlPlaneKubeconfigsStatus contains information + about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + controlerManager: + description: TenantControlPlaneKubeconfigsStatus contains information + about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + scheduler: + description: TenantControlPlaneKubeconfigsStatus contains information + about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + kubernetesResources: + description: Kubernetes contains information about the reconciliation + of the required Kubernetes resources deployed in the admin cluster + properties: + deployment: + description: KubernetesDeploymentStatus defines the status for + the Tenant Control Plane Deployment in the management cluster. + properties: + availableReplicas: + description: Total number of available pods (ready for at + least minReadySeconds) targeted by this deployment. + format: int32 + type: integer + collisionCount: + description: Count of hash collisions for the Deployment. + The Deployment controller uses this field as a collision + avoidance mechanism when it needs to create the name for + the newest ReplicaSet. + format: int32 + type: integer + conditions: + description: Represents the latest available observations + of a deployment's current state. + items: + description: DeploymentCondition describes the state of + a deployment at a certain point. + properties: + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details + about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of deployment condition. + type: string + required: + - status + - type + type: object + type: array + name: + description: The name of the Deployment for the given cluster. + type: string + namespace: + description: The namespace which the Deployment for the given + cluster is deployed. + type: string + observedGeneration: + description: The generation observed by the deployment controller. + format: int64 + type: integer + readyReplicas: + description: readyReplicas is the number of pods targeted + by this Deployment with a Ready Condition. + format: int32 + type: integer + replicas: + description: Total number of non-terminated pods targeted + by this deployment (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by + this deployment. This is the total number of pods that are + still required for the deployment to have 100% available + capacity. They may either be pods that are running but not + yet available or pods that still have not been created. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted + by this deployment that have the desired template spec. + format: int32 + type: integer + required: + - name + - namespace + type: object + ingress: + description: KubernetesIngressStatus defines the status for the + Tenant Control Plane Ingress in the management cluster. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the + load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points + for the load-balancer. Traffic intended for the service + should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status + of a load-balancer ingress point: traffic intended + for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress + points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or OpenStack + load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem + with the service port The format of the + error shall comply with the following rules: + - built-in error values shall be specified + in this file and those shall use CamelCase + names - cloud provider specific error values + must have names that comply with the format + foo.example.com/CamelCase. --- The regex + it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the + service port of which status is recorded + here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of + the service port of which status is recorded + here The supported values are: "TCP", "UDP", + "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Ingress for the given cluster. + type: string + namespace: + description: The namespace which the Ingress for the given + cluster is deployed. + type: string + required: + - name + - namespace + type: object + service: + description: KubernetesServiceStatus defines the status for the + Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect + of the current state of this API Resource. --- This struct + is intended for direct use as an array at the field path + .status.conditions. For example, type FooStatus struct{ + \ // Represents the observations of a foo's current + state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // + +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the + condition transitioned from one status to another. + This should be when the underlying condition changed. If + that is not known, then using the time when the API + field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty + string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to + the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value + should be a CamelCase string. This field may not be + empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status of the + load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress points + for the load-balancer. Traffic intended for the service + should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status + of a load-balancer ingress point: traffic intended + for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress + points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or OpenStack + load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem + with the service port The format of the + error shall comply with the following rules: + - built-in error values shall be specified + in this file and those shall use CamelCase + names - cloud provider specific error values + must have names that comply with the format + foo.example.com/CamelCase. --- The regex + it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the + service port of which status is recorded + here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of + the service port of which status is recorded + here The supported values are: "TCP", "UDP", + "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given + cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: + - name + - namespace + - port + type: object + version: + description: KubernetesVersion contains the information regarding + the running Kubernetes version, and its upgrade status. + properties: + status: + default: Provisioning + description: Status returns the current status of the Kubernetes + version, such as its provisioning state, or completed upgrade. + enum: + - Provisioning + - Upgrading + - Ready + - NotReady + type: string + version: + description: Version is the running Kubernetes version of + the Tenant Control Plane. + type: string + required: + - status + type: object + type: object + storage: + description: Storage Status contains information about Kubernetes + storage system + properties: + etcd: + description: ETCDStatus defines the observed state of ETCDStatus. + properties: + role: + properties: + exists: + type: boolean + name: + type: string + permissions: + items: + properties: + key: + type: string + rangeEnd: + type: string + type: + type: integer + type: object + type: array + required: + - exists + - name + type: object + user: + properties: + exists: + type: boolean + name: + type: string + roles: + items: + type: string + type: array + required: + - exists + - name + type: object + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..14670fe --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,21 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/kamaji.clastix.io_tenantcontrolplanes.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_clusters.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_clusters.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..ec5c150 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_clusters.yaml b/config/crd/patches/cainjection_in_clusters.yaml new file mode 100644 index 0000000..12ade19 --- /dev/null +++ b/config/crd/patches/cainjection_in_clusters.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: tenantcontrolplanes.kamaji.clastix.io diff --git a/config/crd/patches/webhook_in_clusters.yaml b/config/crd/patches/webhook_in_clusters.yaml new file mode 100644 index 0000000..da00cf3 --- /dev/null +++ b/config/crd/patches/webhook_in_clusters.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: tenantcontrolplanes.kamaji.clastix.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..9504706 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: kamaji-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kamaji- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 0000000..4e2232f --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,27 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + protocol: TCP + name: https + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml new file mode 100644 index 0000000..6c40015 --- /dev/null +++ b/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/config/install.yaml b/config/install.yaml new file mode 100644 index 0000000..43af0be --- /dev/null +++ b/config/install.yaml @@ -0,0 +1,1051 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: kamaji-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: tenantcontrolplanes.kamaji.clastix.io +spec: + group: kamaji.clastix.io + names: + kind: TenantControlPlane + listKind: TenantControlPlaneList + plural: tenantcontrolplanes + shortNames: + - tcp + singular: tenantcontrolplane + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .spec.kubernetes.version + name: Version + type: string + - description: Kubernetes version + jsonPath: .status.kubernetesResources.version.status + name: Status + type: string + - description: Tenant Control Plane Endpoint (API server) + jsonPath: .status.controlPlaneEndpoint + name: Control-Plane-Endpoint + type: string + - description: Secret which contains admin kubeconfig + jsonPath: .status.kubeconfig.admin.secretName + name: Kubeconfig + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TenantControlPlane is the Schema for the tenantcontrolplanes API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. + properties: + controlPlane: + description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress. + properties: + deployment: + description: Defining the options for the deployed Tenant Control Plane as Deployment resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + replicas: + default: 2 + format: int32 + type: integer + type: object + ingress: + description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + enabled: + type: boolean + hostname: + description: Hostname is an optional field which will be used as Ingress's Host. If it is not defined, Ingress's host will be "..", where domain is specified under NetworkProfileSpec + type: string + ingressClassName: + type: string + required: + - enabled + type: object + service: + description: Defining the options for the Tenant Control Plane Service resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + serviceType: + description: ServiceType allows specifying how to expose the Tenant Control Plane. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + required: + - serviceType + type: object + required: + - service + type: object + kubernetes: + description: Kubernetes specification for tenant control plane + properties: + admissionControllers: + default: + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - PersistentVolumeClaimResize + - Priority + - ResourceQuota + - RuntimeClass + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + description: 'List of enabled Admission Controllers for the Tenant cluster. Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers' + items: + enum: + - AlwaysAdmit + - AlwaysDeny + - AlwaysPullImages + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - DenyEscalatingExec + - DenyExecOnPrivileged + - DenyServiceExternalIPs + - EventRateLimit + - ExtendedResourceToleration + - ImagePolicyWebhook + - LimitPodHardAntiAffinityTopology + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceAutoProvision + - NamespaceExists + - NamespaceLifecycle + - NodeRestriction + - OwnerReferencesPermissionEnforcement + - PersistentVolumeClaimResize + - PersistentVolumeLabel + - PodNodeSelector + - PodSecurity + - PodSecurityPolicy + - PodTolerationRestriction + - Priority + - ResourceQuota + - RuntimeClass + - SecurityContextDeny + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + type: string + type: array + kubelet: + properties: + cgroupfs: + description: CGroupFS defines the cgroup driver for Kubelet https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/ + enum: + - systemd + - cgroupfs + type: string + type: object + version: + description: Kubernetes Version for the tenant control plane + type: string + required: + - kubelet + - version + type: object + networkProfile: + description: NetworkProfile specifies how the network is + properties: + address: + description: Address where API server of will be exposed. In case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager. + type: string + allowAddressAsExternalIP: + description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort) + type: boolean + dnsServiceIPs: + items: + type: string + type: array + domain: + description: Domain of the tenant control plane + type: string + podCidr: + description: CIDR for Kubernetes Pods + type: string + port: + default: 6443 + description: Port where API server of will be exposed + format: int32 + type: integer + serviceCidr: + description: Kubernetes Service + type: string + required: + - dnsServiceIPs + - domain + - podCidr + - port + - serviceCidr + type: object + required: + - controlPlane + - kubernetes + type: object + status: + description: TenantControlPlaneStatus defines the observed state of TenantControlPlane. + properties: + certificates: + description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane + properties: + apiServer: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + apiServerKubeletClient: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + etcd: + description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. + properties: + apiServer: + description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + frontProxyCA: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + frontProxyClient: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + sa: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint contains the status of the kubernetes control plane + type: string + kubeadmPhase: + description: KubeadmPhase contains the status of the kubeadm phases action + properties: + addonCoreDNS: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + addonKubeProxy: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + bootstrapToken: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubeadm: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubelet: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + required: + - addonCoreDNS + - addonKubeProxy + - bootstrapToken + - uploadConfigKubeadm + - uploadConfigKubelet + type: object + kubeadmconfig: + description: KubeadmConfig contains the status of the configuration required by kubeadm + properties: + configmapName: + type: string + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + required: + - resourceVersion + type: object + kubeconfig: + description: KubeConfig contains information about the kubenconfigs that control plane pieces need + properties: + admin: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + controlerManager: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + scheduler: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + kubernetesResources: + description: Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster + properties: + deployment: + description: KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster. + properties: + availableReplicas: + description: Total number of available pods (ready for at least minReadySeconds) targeted by this deployment. + format: int32 + type: integer + collisionCount: + description: Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet. + format: int32 + type: integer + conditions: + description: Represents the latest available observations of a deployment's current state. + items: + description: DeploymentCondition describes the state of a deployment at a certain point. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of deployment condition. + type: string + required: + - status + - type + type: object + type: array + name: + description: The name of the Deployment for the given cluster. + type: string + namespace: + description: The namespace which the Deployment for the given cluster is deployed. + type: string + observedGeneration: + description: The generation observed by the deployment controller. + format: int64 + type: integer + readyReplicas: + description: readyReplicas is the number of pods targeted by this Deployment with a Ready Condition. + format: int32 + type: integer + replicas: + description: Total number of non-terminated pods targeted by this deployment (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by this deployment. This is the total number of pods that are still required for the deployment to have 100% available capacity. They may either be pods that are running but not yet available or pods that still have not been created. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted by this deployment that have the desired template spec. + format: int32 + type: integer + required: + - name + - namespace + type: object + ingress: + description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Ingress for the given cluster. + type: string + namespace: + description: The namespace which the Ingress for the given cluster is deployed. + type: string + required: + - name + - namespace + type: object + service: + description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: + - name + - namespace + - port + type: object + version: + description: KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status. + properties: + status: + default: Provisioning + description: Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade. + enum: + - Provisioning + - Upgrading + - Ready + - NotReady + type: string + version: + description: Version is the running Kubernetes version of the Tenant Control Plane. + type: string + required: + - status + type: object + type: object + storage: + description: Storage Status contains information about Kubernetes storage system + properties: + etcd: + description: ETCDStatus defines the observed state of ETCDStatus. + properties: + role: + properties: + exists: + type: boolean + name: + type: string + permissions: + items: + properties: + key: + type: string + rangeEnd: + type: string + type: + type: integer + type: object + type: array + required: + - exists + - name + type: object + user: + properties: + exists: + type: boolean + name: + type: string + roles: + items: + type: string + type: array + required: + - exists + - name + type: object + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kamaji-controller-manager + namespace: kamaji-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kamaji-leader-election-role + namespace: kamaji-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kamaji-manager-role +rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/finalizers + verbs: + - update +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kamaji-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kamaji-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kamaji-leader-election-rolebinding + namespace: kamaji-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kamaji-leader-election-role +subjects: +- kind: ServiceAccount + name: kamaji-controller-manager + namespace: kamaji-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kamaji-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kamaji-manager-role +subjects: +- kind: ServiceAccount + name: kamaji-controller-manager + namespace: kamaji-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kamaji-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kamaji-proxy-role +subjects: +- kind: ServiceAccount + name: kamaji-controller-manager + namespace: kamaji-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 + kind: ControllerManagerConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: 799b98bc.clastix.io +kind: ConfigMap +metadata: + name: kamaji-manager-config + namespace: kamaji-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: kamaji-controller-manager-metrics-service + namespace: kamaji-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: controller-manager + name: kamaji-controller-manager + namespace: kamaji-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + command: + - /manager + image: quay.io/clastix/kamaji:latest + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + securityContext: + runAsNonRoot: true + serviceAccountName: kamaji-controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml new file mode 100644 index 0000000..918b03d --- /dev/null +++ b/config/manager/controller_manager_config.yaml @@ -0,0 +1,11 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: 799b98bc.clastix.io diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..ccf51fa --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/clastix/kamaji + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..db09628 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + securityContext: + runAsNonRoot: true + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + imagePullPolicy: IfNotPresent + name: manager + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml new file mode 100644 index 0000000..12c4ee4 --- /dev/null +++ b/config/manifests/kustomization.yaml @@ -0,0 +1,27 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/operator.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard + +# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. +# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. +# These patches remove the unnecessary "cert" volume and its manager container volumeMount. +#patchesJson6902: +#- target: +# group: apps +# version: v1 +# kind: Deployment +# name: controller-manager +# namespace: system +# patch: |- +# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/containers/1/volumeMounts/0 +# # Remove the "cert" volume, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing volumes in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/volumes/0 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..ed13716 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..d19136a --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 0000000..80e1857 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 0000000..ec7acc0 --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 0000000..71f1797 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..731832a --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..4190ec8 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..1d1321e --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..8148604 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,94 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/finalizers + verbs: + - update +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..2070ede --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 0000000..7cd6025 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/config/rbac/tenantcluster_editor_role.yaml b/config/rbac/tenantcluster_editor_role.yaml new file mode 100644 index 0000000..450c0da --- /dev/null +++ b/config/rbac/tenantcluster_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit clusters. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tenantcontrolplane-editor-role +rules: +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/status + verbs: + - get diff --git a/config/rbac/tenantcluster_viewer_role.yaml b/config/rbac/tenantcluster_viewer_role.yaml new file mode 100644 index 0000000..41f950e --- /dev/null +++ b/config/rbac/tenantcluster_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view clusters. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tenantcontrolplane-viewer-role +rules: +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes + verbs: + - get + - list + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/status + verbs: + - get diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml new file mode 100644 index 0000000..b3b3923 --- /dev/null +++ b/config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml @@ -0,0 +1,48 @@ +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: test +spec: + controlPlane: + deployment: + replicas: 2 + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: deployment + service: + additionalMetadata: + annotations: + environment.clastix.io: test + tier.clastix.io: "0" + labels: + tenant.clastix.io: test + kind.clastix.io: service + serviceType: ClusterIP + ingress: + enabled: true + hostname: kamaji.local + ingressClassName: nginx + additionalMetadata: + annotations: + kubernetes.io/ingress.allow-http: "false" + nginx.ingress.kubernetes.io/secure-backends: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + kubernetes: + version: "v1.23.1" + kubelet: + cgroupfs: systemd + admissionControllers: + - ResourceQuota + - LimitRanger + networkProfile: + address: "127.0.0.1" + port: 6443 + domain: "clastix.labs" + serviceCidr: "10.96.0.0/16" + podCidr: "10.244.0.0/16" + dnsServiceIPs: + - "10.96.0.10" diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 0000000..6f2e031 --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- kamaji_v1alpha1_tenantcontrolplane.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml new file mode 100644 index 0000000..c770478 --- /dev/null +++ b/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml new file mode 100644 index 0000000..50cd2d0 --- /dev/null +++ b/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml new file mode 100644 index 0000000..ebd7145 --- /dev/null +++ b/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.13.1 + labels: + suite: basic + test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml new file mode 100644 index 0000000..79b4a63 --- /dev/null +++ b/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.13.1 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.13.1 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.13.1 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.13.1 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.13.1 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/controllers/suite_test.go b/controllers/suite_test.go new file mode 100644 index 0000000..d9dda6b --- /dev/null +++ b/controllers/suite_test.go @@ -0,0 +1,65 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config // nolint +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Controller Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = kamajiv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go new file mode 100644 index 0000000..ca2c5b9 --- /dev/null +++ b/controllers/tenantcontrolplane_controller.go @@ -0,0 +1,350 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "context" + "fmt" + "strings" + + "github.com/google/uuid" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" +) + +const ( + separator = "," + finalizer = "finalizer.kamaji.clastix.io" +) + +// TenantControlPlaneReconciler reconciles a TenantControlPlane object. +type TenantControlPlaneReconciler struct { + client.Client + Scheme *runtime.Scheme + Config TenantControlPlaneReconcilerConfig +} + +// TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler. +type TenantControlPlaneReconcilerConfig struct { + ETCDCASecretName string + ETCDCASecretNamespace string + ETCDClientSecretName string + ETCDClientSecretNamespace string + ETCDEndpoints string + ETCDCompactionInterval string + TmpBaseDirectory string +} + +//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/finalizers,verbs=update +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete + +func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + + tenantControlPlane := &kamajiv1alpha1.TenantControlPlane{} + isTenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName, tenantControlPlane) + if err != nil { + return ctrl.Result{}, err + } + if !isTenantControlPlane { + return ctrl.Result{}, nil + } + + markedToBeDeleted := tenantControlPlane.GetDeletionTimestamp() != nil + hasFinalizer := hasFinalizer(*tenantControlPlane) + + if markedToBeDeleted && !hasFinalizer { + return ctrl.Result{}, nil + } + + if markedToBeDeleted { + registeredDeleteableResources := []resources.DeleteableResource{ + &resources.ETCDSetupResource{ + Name: "etcd-setup", + Client: r.Client, + Scheme: r.Scheme, + Log: log, + ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName), + ETCDCACertsSecret: getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName), + Endpoints: getArrayFromString(r.Config.ETCDEndpoints), + }, + } + + for _, resource := range registeredDeleteableResources { + if err := resource.Delete(ctx, tenantControlPlane); err != nil { + return ctrl.Result{}, err + } + } + + if hasFinalizer { + controllerutil.RemoveFinalizer(tenantControlPlane, finalizer) + if err := r.Update(ctx, tenantControlPlane); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil + } + + if !hasFinalizer { + controllerutil.AddFinalizer(tenantControlPlane, finalizer) + if err := r.Update(ctx, tenantControlPlane); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + registeredResources := []resources.Resource{ + &resources.KubernetesServiceResource{ + Client: r.Client, + }, + &resources.KubeadmConfigResource{ + Name: "kubeadmconfig", + Port: tenantControlPlane.Spec.NetworkProfile.Port, + KubernetesVersion: tenantControlPlane.Spec.Kubernetes.Version, + PodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, + ServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR, + Domain: tenantControlPlane.Spec.NetworkProfile.Domain, + ETCDs: getArrayFromString(r.Config.ETCDEndpoints), + ETCDCompactionInterval: r.Config.ETCDCompactionInterval, + Client: r.Client, + Scheme: r.Scheme, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.CACertificate{ + Name: "ca", + Client: r.Client, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.FrontProxyCACertificate{ + Name: "front-proxy-ca-certificate", + Client: r.Client, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.SACertificate{ + Name: "sa-certificate", + Client: r.Client, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.APIServerCertificate{ + Name: "api-server-certificate", + Client: r.Client, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.APIServerKubeletClientCertificate{ + Name: "api-server-kubelet-client-certificate", + Client: r.Client, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.FrontProxyClientCertificate{ + Name: "front-proxy-client-certificate", + Client: r.Client, + Log: log, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.KubeconfigResource{ + Name: "admin-kubeconfig", + Client: r.Client, + Scheme: r.Scheme, + Log: log, + KubeConfigFileName: resources.AdminKubeConfigFileName, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.KubeconfigResource{ + Name: "controller-manager-kubeconfig", + Client: r.Client, + Scheme: r.Scheme, + Log: log, + KubeConfigFileName: resources.ControllerManagerKubeConfigFileName, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.KubeconfigResource{ + Name: "scheduler-kubeconfig", + Client: r.Client, + Scheme: r.Scheme, + Log: log, + KubeConfigFileName: resources.SchedulerKubeConfigFileName, + TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane), + }, + &resources.ETCDCACertificatesResource{ + Name: "etcd-ca-certificates", + Client: r.Client, + Log: log, + ETCDCASecretName: r.Config.ETCDCASecretName, + ETCDCASecretNamespace: r.Config.ETCDCASecretNamespace, + }, + &resources.ETCDCertificatesResource{ + Name: "etcd-certificates", + Client: r.Client, + Log: log, + }, + &resources.ETCDSetupResource{ + Name: "etcd-setup", + Client: r.Client, + Scheme: r.Scheme, + Log: log, + ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName), + ETCDCACertsSecret: getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName), + Endpoints: getArrayFromString(r.Config.ETCDEndpoints), + }, + &resources.KubernetesUpgrade{ + Name: "upgrade", + Client: r.Client, + }, + &resources.KubernetesDeploymentResource{ + Client: r.Client, + ETCDEndpoints: getArrayFromString(r.Config.ETCDEndpoints), + ETCDCompactionInterval: r.Config.ETCDCompactionInterval, + }, + &resources.KubernetesIngressResource{ + Client: r.Client, + }, + &resources.KubeadmPhaseResource{ + Name: "upload-config-kubeadm", + Client: r.Client, + Log: log, + KubeadmPhase: resources.PhaseUploadConfigKubeadm, + }, + &resources.KubeadmPhaseResource{ + Name: "upload-config-kubelet", + Client: r.Client, + Log: log, + KubeadmPhase: resources.PhaseUploadConfigKubelet, + }, + &resources.KubeadmPhaseResource{ + Name: "addon-coredns", + Client: r.Client, + Log: log, + KubeadmPhase: resources.PhaseAddonCoreDNS, + }, + &resources.KubeadmPhaseResource{ + Name: "addon-kubeproxy", + Client: r.Client, + Log: log, + KubeadmPhase: resources.PhaseAddonKubeProxy, + }, + &resources.KubeadmPhaseResource{ + Name: "bootstrap-token", + Client: r.Client, + Log: log, + KubeadmPhase: resources.PhaseBootstrapToken, + }, + } + + for _, resource := range registeredResources { + result, err := resources.Handle(ctx, resource, tenantControlPlane) + if err != nil { + return ctrl.Result{}, err + } + + if result == controllerutil.OperationResultNone { + continue + } + + if err := r.updateStatus(ctx, req.NamespacedName, resource); err != nil { + return ctrl.Result{}, err + } + + log.Info(fmt.Sprintf("%s has been configured", resource.GetName())) + + return ctrl.Result{}, nil + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&kamajiv1alpha1.TenantControlPlane{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Service{}). + Owns(&networkingv1.Ingress{}). + Complete(r) +} + +func (r *TenantControlPlaneReconciler) getTenantControlPlane(ctx context.Context, namespacedName k8stypes.NamespacedName, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { + if err := r.Client.Get(ctx, namespacedName, tenantControlPlane); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespacedName k8stypes.NamespacedName, resource resources.Resource) error { + tenantControlPlane := &kamajiv1alpha1.TenantControlPlane{} + isTenantControlPlane, err := r.getTenantControlPlane(ctx, namespacedName, tenantControlPlane) + if err != nil { + return err + } + + if !isTenantControlPlane { + return fmt.Errorf("error updating tenantControlPlane %s: not found", namespacedName.Name) + } + + if err := resource.UpdateTenantControlPlaneStatus(ctx, tenantControlPlane); err != nil { + return err + } + + if err := r.Status().Update(ctx, tenantControlPlane); err != nil { + return fmt.Errorf("error updating tenantControlPlane status: %w", err) + } + + return nil +} + +func getArrayFromString(s string) []string { + var a []string + a = append(a, strings.Split(s, separator)...) + + return a +} + +func getNamespacedName(namespace string, name string) k8stypes.NamespacedName { + return k8stypes.NamespacedName{Namespace: namespace, Name: name} +} + +func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New()) +} + +func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool { + for _, f := range tenantControlPlane.GetFinalizers() { + if f == finalizer { + return true + } + } + + return false +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d54d9e3 --- /dev/null +++ b/go.mod @@ -0,0 +1,142 @@ +module github.com/clastix/kamaji + +go 1.17 + +require ( + github.com/go-logr/logr v1.2.0 + github.com/google/uuid v1.1.2 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.17.0 + github.com/pkg/errors v0.9.1 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.10.1 + go.etcd.io/etcd/api/v3 v3.5.1 + go.etcd.io/etcd/client/v3 v3.5.1 + google.golang.org/grpc v1.43.0 + k8s.io/api v0.23.5 + k8s.io/apimachinery v0.23.5 + k8s.io/client-go v0.23.5 + k8s.io/cluster-bootstrap v0.0.0 + k8s.io/component-base v0.23.5 + k8s.io/kube-proxy v0.0.0 + k8s.io/kubelet v0.0.0 + k8s.io/kubernetes v1.23.5 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 + sigs.k8s.io/controller-runtime v0.11.0 +) + +require ( + cloud.google.com/go v0.99.0 // indirect + cloud.google.com/go/storage v1.18.2 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/Microsoft/go-winio v0.4.17 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect + github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 // indirect + github.com/coredns/caddy v1.1.0 // indirect + github.com/coredns/corefile-migration v1.0.14 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/envoyproxy/go-control-plane v0.10.1 // indirect + github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/zapr v1.2.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/googleapis/gax-go/v2 v2.1.1 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect + github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/spf13/afero v1.7.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.4.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect + go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect + golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/api v0.63.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/apiextensions-apiserver v0.23.5 // indirect + k8s.io/apiserver v0.23.5 // indirect + k8s.io/klog/v2 v2.60.1 // indirect + k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect + k8s.io/system-validators v1.6.0 // indirect + sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) + +replace ( + k8s.io/api => k8s.io/api v0.23.5 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.5 + k8s.io/apimachinery => k8s.io/apimachinery v0.23.5 + k8s.io/apiserver => k8s.io/apiserver v0.23.5 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.5 + k8s.io/client-go => k8s.io/client-go v0.23.5 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.5 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.5 + k8s.io/code-generator => k8s.io/code-generator v0.23.5 + k8s.io/component-base => k8s.io/component-base v0.23.5 + k8s.io/component-helpers => k8s.io/component-helpers v0.23.5 + k8s.io/controller-manager => k8s.io/controller-manager v0.23.5 + k8s.io/cri-api => k8s.io/cri-api v0.23.5 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.5 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.5 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.5 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.5 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.5 + k8s.io/kubectl => k8s.io/kubectl v0.23.5 + k8s.io/kubelet => k8s.io/kubelet v0.23.5 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.5 + k8s.io/metrics => k8s.io/metrics v0.23.5 + k8s.io/mount-utils => k8s.io/mount-utils v0.23.5 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.5 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.5 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b6649b8 --- /dev/null +++ b/go.sum @@ -0,0 +1,1710 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 h1:SC+c6A1qTFstO9qmB86mPV2IpYme/2ZoEQ0hrP+wo+Q= +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690 h1:N9r8OBSXAgEUfho3SQtZLY8zo6E1OdOMvelvP22aVFc= +bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.1 h1:8rBq3zRjnHx8UtBvaOWqBB1xq9jH6/wltfQLlTMh2Fw= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.18.2 h1:5NQw6tOn3eMm0oE8vTkfjau18kjL79FlMjy/CHTpmoY= +cloud.google.com/go/storage v1.18.2/go.mod h1:AiIj7BWXyhO5gGVmYJ+S8tbkCx3yb0IMjua8Aw4naVM= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037 h1:+PdD6GLKejR9DizMAKT5DpSAkKswvZrurk1/eEt9+pw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v55.0.0+incompatible h1:L4/vUGbg1Xkw5L20LZD+hJI5I+ibWSytqQ68lTCfLwY= +github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A= +github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/GoogleCloudPlatform/k8s-cloud-provider v1.16.1-0.20210702024009-ea6160c1d0e3 h1:FCalqNmQYSMCCHoCtAxZN/ZgLc8ufgeo5Z3wrIoJZvs= +github.com/GoogleCloudPlatform/k8s-cloud-provider v1.16.1-0.20210702024009-ea6160c1d0e3/go.mod h1:8XasY4ymP2V/tn2OOV9ZadmiTE1FIB/h3W+yNlPttKw= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.22 h1:CulZ3GW8sNJExknToo+RWD+U+6ZM5kkNfuxywSDPd08= +github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e h1:GCzyKMDDjSGnlpl3clrdAK7I1AaVoaiKDOYkUzChZzg= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/auth0/go-jwt-middleware v1.0.1 h1:/fsQ4vRr4zod1wKReUH+0A3ySRjGiT9G34kypO/EKwI= +github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= +github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/aws/aws-sdk-go v1.38.49 h1:E31vxjCe6a5I+mJLmUGaZobiWmg9KdWaud9IfceYeYQ= +github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4 h1:w/jqZtC9YD4DS/Vp9GhWfWcCpuAL58oTnLoI8vE9YHU= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0 h1:TW8f/UvntYoVDMN1K2HlT82qH1rb0sOjpGw3m6Ym+i4= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313 h1:eIHD9GNM3Hp7kcRW5mvcz7WTR3ETeoYYKwpgA04kaXE= +github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 h1:F6fR7MjvOIk+FLQOeBCAbbKItVgbdj0l9VWPiHeBEiY= +github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo= +github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.11 h1:QCGOUN+i70jEEL/A6JVIbhy4f4fanzAzSR4kNG7SlcE= +github.com/containerd/containerd v1.4.11/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0= +github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= +github.com/coredns/corefile-migration v1.0.14 h1:Tz3WZhoj2NdP8drrQH86NgnCng+VrPjNeg2Oe1ALKag= +github.com/coredns/corefile-migration v1.0.14/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd h1:uVsMphB1eRx7xB1njzL3fuMdWRN8HtVzoUOItHMwv5c= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1 h1:cgDRLG7bs59Zd+apAWuzLQL95obVYAymNJek76W3mgw= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2 h1:JiO+kJTpmYGjEodY7O1Zk8oZcNz1+f30UtwtXoFUPzE= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= +github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 h1:WXb3TSNmHp2vHoCroCIB1foO/yQ36swABL8aOVeDpgg= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= +github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM= +github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZPbFy2P7jH5LKJ3La+0ZeknkkmrSgqb0= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cadvisor v0.43.0 h1:z0ULgYPKZ7L/c7Zjq+ZD6ltklWwYdCSvBMgSjNC/hGo= +github.com/google/cadvisor v0.43.0/go.mod h1:+RdMSbc3FVr5NYCD2dOEJy/LI0jYJ/0xJXkzWXEyiFQ= +github.com/google/cel-go v0.9.0 h1:u1hg7lcZ/XWw2d3aV1jFS30ijQQ6q0/h1C2ZBeBD1gY= +github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-spec v0.6.0 h1:xuthJSiJGoSzq+lVEBIW1MTpaaZXknMCYC4WzVAWOsE= +github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 h1:zC34cGQu69FG7qzJ3WiKW244WfhDC3xxYMeNOX2gtUQ= +github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.12.0 h1:k3y1FYv6nuKyNTqj6w9gXOx5r5CfLj/k/euUeBXj1OY= +github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= +github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/heketi/heketi v10.3.0+incompatible h1:X4DBFPzcyWZWhia32d94UhDECQJHH0M5kpRb1gxxUHk= +github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= +github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6 h1:oJ/NLadJn5HoxvonA6VxG31lg0d6XOURNA09BTtM4fY= +github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5 h1:qPmlgoeRS18y2dT+iAH5vEKZgIqgiPi2Y8UCu/b7Aq8= +github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= +github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/lpabon/godbc v0.1.1 h1:ilqjArN1UOENJJdM34I2YHKmF/B0gGq4VLoSGy9iAao= +github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= +github.com/lyft/protoc-gen-star v0.5.3 h1:zSGLzsUew8RT+ZKPHc3jnf8XLaVyHzTcAFBzHtCNR20= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 h1:PS1dLCGtD8bb9RPKJrc8bS7qHL6JnW1CZvwzH9dPoUs= +github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/ipvs v1.0.1 h1:aoZ7fhLTXgDbzVrAnvV+XbKOU8kOET7B3+xULDF/1o0= +github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb h1:e+l77LJOEqXTIQihQJVkA6ZxPOUmfPM5e4H7rcpgtSk= +github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww= +github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/quobyte/api v0.1.8 h1:+sOX1gIlC/OaLipqVZWrHgly9Kh9Qo8OygeS0mWAg30= +github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= +github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.4.0 h1:Rqcx6Sf/bWQUmmfGQhcFx3wQQEfb2UZWhAKvGRairm0= +github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= +github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.7.0 h1:xc1yh8vgcNB8yQ+UqY4cpD56Ogo573e+CJ/C4YmMFTg= +github.com/spf13/afero v1.7.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= +github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/storageos/go-api v2.2.0+incompatible h1:U0SablXoZIg06gvSlg8BCdzq1C/SkHVygOVX95Z2MU0= +github.com/storageos/go-api v2.2.0+incompatible/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vmware/govmomi v0.20.3 h1:gpw/0Ku+6RgF3jsi7fnCLmlcikBHfKBCUcu1qgc16OU= +github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0 h1:OtISOGfH6sOWa1/qXqqAiOIAO6Z5J3AEAE18WAq6BiQ= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1 h1:v28cktvBq+7vGyJXF8G+rWJmj+1XUmMtqcLnH8hDocM= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1 h1:XIQcHCFSG53bJETYeRJtIxdLv2EWRGxcfzR8lSnTH4E= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v2 v2.305.1 h1:vtxYCKWA9x31w0WJj7DdqsHFNjhkigdAnziDtkZb/l4= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.1 h1:oImGuV5LGKjCqXdjkMHCyWa5OO1gYKCnC/1sgdfj1Uk= +go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= +go.etcd.io/etcd/pkg/v3 v3.5.0 h1:ntrg6vvKRW26JRmHTE0iNlDgYK6JX3hg/4cD62X0ixk= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20210220032938-85be41e4509f h1:GrkO5AtFUU9U/1f5ctbIBXtBGeSJbWwIYfIsTcFMaX4= +golang.org/x/exp v0.0.0-20210220032938-85be41e4509f/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f h1:kgfVkAEEQXXQ0qc6dH7n6y37NAYmTFmz0YRwrRjgxKw= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff h1:VX/uD7MK0AHXGiScH3fsieUQUcpmRERPDYtqZdJnA+Q= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q= +gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b h1:Qh4dB5D/WpoUUp3lSod7qgoyEHbDGPUWjIbnqdqqe1k= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0 h1:n2bqqK895ygnBpdPDYetfy23K7fJ22wsrZKCyfuRkkA= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0= +gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.1 h1:XM28wIgFzaBmeZ5dNHIpWLQpt/9DGKxk+rCg/22nnYE= +gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= +k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= +k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= +k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= +k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= +k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE= +k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= +k8s.io/cli-runtime v0.23.5 h1:Z7XUpGoJZYZB2uNjQfJjMbyDKyVkoBGye62Ap0sWQHY= +k8s.io/cli-runtime v0.23.5/go.mod h1:oY6QDF2qo9xndSq32tqcmRp2UyXssdGrLfjAVymgbx4= +k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= +k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= +k8s.io/cloud-provider v0.23.5 h1:cf5Il2oV++RtlqgNesHd+tDFtOp85dG0t9KN/pmb71s= +k8s.io/cloud-provider v0.23.5/go.mod h1:xMZFA6pIYKweqTkWCYVgRSVMAjqOvxVr3u/kmfyxvkU= +k8s.io/cluster-bootstrap v0.23.5 h1:BhGnjfd2J0W/XqissyT+fX8bZwKVnvaTmjsDmtda3DY= +k8s.io/cluster-bootstrap v0.23.5/go.mod h1:8/Gz6VTOMmEDDhn8U/nx0McnQR4YETAqiYXIlqR8hdQ= +k8s.io/code-generator v0.23.5 h1:xn3a6J5pUL49AoH6SPrOFtnB5cvdMl76f/bEY176R3c= +k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= +k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= +k8s.io/component-helpers v0.23.5 h1:6uTMNP6xxJrSzYTC7BCcH2S/PbSZGxSUZG0PG+nT4tM= +k8s.io/component-helpers v0.23.5/go.mod h1:5riXJgjTIs+ZB8xnf5M2anZ8iQuq37a0B/0BgoPQuSM= +k8s.io/controller-manager v0.23.5 h1:49YQZ+XEVCe2Xi6ESgKYPouXCQ6OXemhwGDrQZOJOtg= +k8s.io/controller-manager v0.23.5/go.mod h1:n/KRlUzAtkFcZodZ/w0GlQdmErVKh7lS/wS0bbo7W4I= +k8s.io/cri-api v0.23.5 h1:841+VfuaykmA/htPp7vePxvQrDVstVwGVbB2S6UITbo= +k8s.io/cri-api v0.23.5/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= +k8s.io/csi-translation-lib v0.23.5 h1:QtbG9dSku/Dh0RvVsdKT+j2HtyOBu+ug6+tioB8MBNE= +k8s.io/csi-translation-lib v0.23.5/go.mod h1:8RyFkoHAJrFU7c7MN1ZUjctm3ZhHclKm1FIHNSyGcuw= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c h1:GohjlNKauSai7gN4wsJkeZ3WAJx4Sh+oT/b5IYn5suA= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-aggregator v0.23.5 h1:UZ+qE3hGo6DcgKySf27Jg7d3X9/6JQkVLUiHZAoAfCY= +k8s.io/kube-aggregator v0.23.5/go.mod h1:3ynYx07Co6dzjpKPgipM+1/Mt2Jcm7dY++cRlKLr5s8= +k8s.io/kube-controller-manager v0.23.5 h1:fLMZXu8vIaPKQz/iVA8a5MdcNtibfUK9wTmEy3DP1Bc= +k8s.io/kube-controller-manager v0.23.5/go.mod h1:Pkg5lIk9YG9Qjj4F7Dn0gi6/k8cEYP63oLdgrlrrtu4= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-proxy v0.23.5 h1:WdLf577zVdOGuZNqERdyXk+WjBwbd3Neiu0dMhY21gE= +k8s.io/kube-proxy v0.23.5/go.mod h1:/yCbRrOHgPCb1g1k4XmMJPmNesfdPhZTGrvwNlNgwo8= +k8s.io/kube-scheduler v0.23.5 h1:3Srw/iJTvsU7nLWl6+OqPO8x/i4SfESna/SB6DslBz8= +k8s.io/kube-scheduler v0.23.5/go.mod h1:IJGf4WngeoAHLj4ms4n3Poa29ttmaxCXxIqpgU0ky7E= +k8s.io/kubectl v0.23.5 h1:DmDULqCaF4qstj0Im143XmncvqWtJxHzK8IrW2BzlU0= +k8s.io/kubectl v0.23.5/go.mod h1:lLgw7cVY8xbd7o637vOXPca/w6HC205KsPCRDYRCxwE= +k8s.io/kubelet v0.23.5 h1:eCGJ7olStiyF7TYHlUTjpXg2ltw7Bs9OPZcch8HP2Go= +k8s.io/kubelet v0.23.5/go.mod h1:M0aj0gaX+rOaGfCfqkV6P7QbwtMwqbL6RdwviHmnehU= +k8s.io/kubernetes v1.23.5 h1:bxpSv2BKc2MqYRfyqQqLVdodLZ2r+NZ/rEdZXyUAvug= +k8s.io/kubernetes v1.23.5/go.mod h1:avI3LUTUYZugxwh52KMVM7v9ZjB5gYJ6D3FIoZ1SHUo= +k8s.io/legacy-cloud-providers v0.23.5 h1:Oq0Kb/YY6qaXwdRXvlPWWtz5i6mJ62X98nuoKHfSHR8= +k8s.io/legacy-cloud-providers v0.23.5/go.mod h1:IENlwY686f1fbakotgNf7gAQuIyCvOUIAXkPPPE/7KU= +k8s.io/metrics v0.23.5 h1:u+3oGwdUWr30LT2v6MHdSJQ4Pbht9s0cVIWyuY5nf9c= +k8s.io/metrics v0.23.5/go.mod h1:WNAtV2a5BYbmDS8+7jSqYYV6E3efuGTpIwJ8PTD1wgs= +k8s.io/mount-utils v0.23.5 h1:MOhJKZTpfC21r5OamKYWMdVNtTMDD9wZfTkLOhI5nuE= +k8s.io/mount-utils v0.23.5/go.mod h1:OTN3LQPiOGMfx/SmVlsnySwsAmh4gYrDYLchlMHtf98= +k8s.io/pod-security-admission v0.23.5 h1:60MTMOK+/hPDgYzZ2C6zFLGaNzYDiMyy6TuZ/rv9Db8= +k8s.io/pod-security-admission v0.23.5/go.mod h1:aSyWfjev8Zil5DaZBZ+ICAObZmZlRqhnAZHxA9r71UI= +k8s.io/sample-apiserver v0.23.5 h1:awmp5/3cY4Fiu3729MG3FL8I+Tkh6AAf+FQAx6qJZt0= +k8s.io/sample-apiserver v0.23.5/go.mod h1:m4cnT3HgRY5Dt2AjMVKGnb31D6rGY0B+xpKtRJUUC8w= +k8s.io/system-validators v1.6.0 h1:21qaPNdZ+mQrm4qc5shU0T5Eh49t/miFqZsn4sW8Hr0= +k8s.io/system-validators v1.6.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 h1:dUk62HQ3ZFhD48Qr8MIXCiKA8wInBQCtuE4QGfFW7yA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= +sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ= +sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/kustomize/api v0.10.1 h1:KgU7hfYoscuqag84kxtzKdEC3mKMb99DPI3a0eaV1d0= +sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= +sigs.k8s.io/kustomize/cmd/config v0.10.2 h1:2GD3+knDaqZo6rSibkc4kKGp8auNBJrGPZQCTWN4Rtc= +sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ= +sigs.k8s.io/kustomize/kustomize/v4 v4.4.1 h1:6hgMEo3Gt0XmhDt4vo0FJ0LRDMc4i8JC6SUW24D4hQM= +sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io= +sigs.k8s.io/kustomize/kyaml v0.13.0 h1:9c+ETyNfSrVhxvphs+K2dzT3dh5oVPPEqPOE/cUpScY= +sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..de2c605 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,2 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 0000000..98d423b --- /dev/null +++ b/helm/README.md @@ -0,0 +1,15 @@ +# Deploy with Helm + +## Pre-requisites + +1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) + +``` +make -C ../deploy/etcd +``` + +## Install + +``` +helm upgrade --install --namespace kamaji-system --create-namespace kamaji ./kamaji +``` diff --git a/helm/kamaji/.helmignore b/helm/kamaji/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/kamaji/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/kamaji/Chart.yaml b/helm/kamaji/Chart.yaml new file mode 100644 index 0000000..238d870 --- /dev/null +++ b/helm/kamaji/Chart.yaml @@ -0,0 +1,35 @@ +apiVersion: v2 +name: kamaji +description: A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde. + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: 0.1.0 + +home: https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji +sources: ["https://github.com/clastix/kamaji-internal"] +kubeVersion: ">=1.18" +maintainers: +- email: iam@mendrugory.com + name: Gonzalo Gabriel Jiménez Fuentes +- email: dario@tranchitella.eu + name: Dario Tranchitella +- email: me@maxgio.it + name: Massimiliano Giovagnoli diff --git a/helm/kamaji/Makefile b/helm/kamaji/Makefile new file mode 100644 index 0000000..a191fe4 --- /dev/null +++ b/helm/kamaji/Makefile @@ -0,0 +1,9 @@ +docs: HELMDOCS_VERSION := v1.8.1 +docs: docker + @docker run --rm -v "$$(pwd):/helm-docs" -u $$(id -u) jnorwood/helm-docs:$(HELMDOCS_VERSION) + +docker: + @hash docker 2>/dev/null || {\ + echo "You need docker" &&\ + exit 1;\ + } diff --git a/helm/kamaji/README.md b/helm/kamaji/README.md new file mode 100644 index 0000000..64e4fea --- /dev/null +++ b/helm/kamaji/README.md @@ -0,0 +1,100 @@ +# kamaji + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square) + +A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde. + +**Homepage:** + +## Installing the Chart + +To install the chart with the release name `kamaji`: + +### Pre-requisites + +1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) +2. Create the `Secret` containing the Etcd CA cert keypair: + +``` +kubectl -n kamaji-system create secret generic etcd-certs \ + --from-file=/path/to/etcd/ca.crt \ + --from-file=/path/to/etcd/ca.key +``` + +3. Create a `Secret` containing the Etcd root user client cert keypair: + +``` +kubectl -n kamaji-system create secret tls root-client-certs \ + --cert=/path/to/etcd/root.pem \ + --key=/path/to/etcd/root-key.pem +``` + +### Install Kamaji + +```console +helm upgrade --install --namespace kamaji-system --create-namespace kamaji . +``` + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Gonzalo Gabriel Jiménez Fuentes | | | +| Dario Tranchitella | | | +| Massimiliano Giovagnoli | | | + +## Source Code + +* + +## Requirements + +Kubernetes: `>=1.18` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods | +| configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") | +| etcd.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") | +| etcd.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji") | +| etcd.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") | +| etcd.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") | +| etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) | +| etcd.endpoints | string | `"etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. | +| extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones | +| fullnameOverride | string | `""` | | +| healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"quay.io/clastix/kamaji"` | The container image of the Kamaji controller. | +| image.tag | string | `"latest"` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | Name of the ingress class to route through this controller. | +| ingress.enabled | bool | `false` | Whether to expose the Kamaji controller through an Ingress. | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container | +| loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) | +| metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kamaji controller | +| podAnnotations | object | `{}` | The annotations to apply to the Kamaji controller pods. | +| podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kamaji controller pods. | +| readinessProbe | object | `{"httpGet":{"path":"/readyz","port":"healthcheck"},"initialDelaySeconds":5,"periodSeconds":10}` | The readinessProbe for the controller container | +| replicaCount | int | `1` | The number of the pod replicas for the Kamaji controller. | +| resources.limits.cpu | string | `"200m"` | | +| resources.limits.memory | string | `"100Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"20Mi"` | | +| securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. | +| service.port | int | `8443` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `"kamaji-controller-manager"` | | +| temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") | +| tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate | diff --git a/helm/kamaji/README.md.gotmpl b/helm/kamaji/README.md.gotmpl new file mode 100644 index 0000000..6955117 --- /dev/null +++ b/helm/kamaji/README.md.gotmpl @@ -0,0 +1,45 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +## Installing the Chart + +To install the chart with the release name `kamaji`: + +### Pre-requisites + +1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) +2. Create the `Secret` containing the Etcd CA cert keypair: + +``` +kubectl -n kamaji-system create secret generic etcd-certs \ + --from-file=/path/to/etcd/ca.crt \ + --from-file=/path/to/etcd/ca.key +``` + +3. Create a `Secret` containing the Etcd root user client cert keypair: + +``` +kubectl -n kamaji-system create secret tls root-client-certs \ + --cert=/path/to/etcd/root.pem \ + --key=/path/to/etcd/root-key.pem +``` + +### Install Kamaji + +```console +helm upgrade --install --namespace kamaji-system --create-namespace kamaji . +``` + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/helm/kamaji/crds/tenantcontrolplane.yaml b/helm/kamaji/crds/tenantcontrolplane.yaml new file mode 100644 index 0000000..dbd0a87 --- /dev/null +++ b/helm/kamaji/crds/tenantcontrolplane.yaml @@ -0,0 +1,716 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: tenantcontrolplanes.kamaji.clastix.io +spec: + group: kamaji.clastix.io + names: + kind: TenantControlPlane + listKind: TenantControlPlaneList + plural: tenantcontrolplanes + shortNames: + - tcp + singular: tenantcontrolplane + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .spec.kubernetes.version + name: Version + type: string + - description: Tenant Control Plane Endpoint (API server) + jsonPath: .status.controlPlaneEndpoint + name: Control-Plane-Endpoint + type: string + - description: Secret which contains admin kubeconfig + jsonPath: .status.kubeconfig.admin.secretName + name: Kubeconfig + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TenantControlPlane is the Schema for the tenantcontrolplanes API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TenantControlPlaneSpec defines the desired state of TenantControlPlane + properties: + controlPlane: + description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress. + properties: + deployment: + description: Defining the options for the deployed Tenant Control Plane as Deployment resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + replicas: + default: 2 + format: int32 + type: integer + type: object + ingress: + description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + enabled: + type: boolean + hostname: + description: Hostname is an optional field which will be used as Ingress's Host. If it is not defined, Ingress's host will be "..", where domain is specified under NetworkProfileSpec + type: string + ingressClassName: + type: string + required: + - enabled + type: object + service: + description: Defining the options for the Tenant Control Plane Service resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + serviceType: + description: ServiceType allows specifying how to expose the Tenant Control Plane. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + required: + - serviceType + type: object + required: + - service + type: object + kubernetes: + description: Kubernetes specification for tenant control plane + properties: + admissionControllers: + default: + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - PersistentVolumeClaimResize + - Priority + - ResourceQuota + - RuntimeClass + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + description: 'List of enabled Admission Controllers for the Tenant cluster. Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers' + items: + enum: + - AlwaysAdmit + - AlwaysDeny + - AlwaysPullImages + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - DenyEscalatingExec + - DenyExecOnPrivileged + - DenyServiceExternalIPs + - EventRateLimit + - ExtendedResourceToleration + - ImagePolicyWebhook + - LimitPodHardAntiAffinityTopology + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceAutoProvision + - NamespaceExists + - NamespaceLifecycle + - NodeRestriction + - OwnerReferencesPermissionEnforcement + - PersistentVolumeClaimResize + - PersistentVolumeLabel + - PodNodeSelector + - PodSecurity + - PodSecurityPolicy + - PodTolerationRestriction + - Priority + - ResourceQuota + - RuntimeClass + - SecurityContextDeny + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + type: string + type: array + kubelet: + properties: + cgroupfs: + description: CGroupFS defines the cgroup driver for Kubelet https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/ + enum: + - systemd + - cgroupfs + type: string + type: object + version: + description: Kubernetes Version for the tenant control plane + type: string + required: + - kubelet + - version + type: object + networkProfile: + description: NetworkProfile specifies how the network is + properties: + address: + description: Address where API server of will be exposed + type: string + dnsServiceIPs: + items: + type: string + type: array + domain: + description: Domain of the tenant control plane + type: string + podCidr: + description: CIDR for Kubernetes Pods + type: string + port: + default: 6443 + description: Port where API server of will be exposed + format: int32 + type: integer + serviceCidr: + description: Kubernetes Service + type: string + required: + - address + - dnsServiceIPs + - domain + - podCidr + - port + - serviceCidr + type: object + required: + - controlPlane + - kubernetes + type: object + status: + description: TenantControlPlaneStatus defines the observed state of TenantControlPlane + properties: + certificates: + description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane + properties: + apiServer: + description: CertificatePrivateKeyPair defines the status + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + apiServerKubeletClient: + description: CertificatePrivateKeyPair defines the status + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: CertificatePrivateKeyPair defines the status + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + etcd: + description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server + properties: + apiServer: + description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + frontProxyCA: + description: CertificatePrivateKeyPair defines the status + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + frontProxyClient: + description: CertificatePrivateKeyPair defines the status + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + sa: + description: CertificatePrivateKeyPair defines the status + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint contains the status of the kubernetes control plane + type: string + kubeadmPhase: + description: KubeadmPhase contains the status of the kubeadm phases action + properties: + addonCoreDNS: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + addonKubeProxy: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + bootstrapToken: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubeadm: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubelet: + description: KubeadmPhasesStatus contains the status of of a kubeadm phase action + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + required: + - addonCoreDNS + - addonKubeProxy + - bootstrapToken + - uploadConfigKubeadm + - uploadConfigKubelet + type: object + kubeadmconfig: + description: KubeadmConfig contains the status of the configuration required by kubeadm + properties: + configmapName: + type: string + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + required: + - resourceVersion + type: object + kubeconfig: + description: KubeConfig contains information about the kubenconfigs that control plane pieces need + properties: + admin: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + controlerManager: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + scheduler: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + kubernetesResources: + description: KubernetesResources contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster + properties: + deployment: + description: KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster. + properties: + availableReplicas: + description: Total number of available pods (ready for at least minReadySeconds) targeted by this deployment. + format: int32 + type: integer + collisionCount: + description: Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet. + format: int32 + type: integer + conditions: + description: Represents the latest available observations of a deployment's current state. + items: + description: DeploymentCondition describes the state of a deployment at a certain point. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of deployment condition. + type: string + required: + - status + - type + type: object + type: array + name: + description: The name of the Deployment for the given cluster. + type: string + namespace: + description: The namespace which the Deployment for the given cluster is deployed. + type: string + observedGeneration: + description: The generation observed by the deployment controller. + format: int64 + type: integer + readyReplicas: + description: readyReplicas is the number of pods targeted by this Deployment with a Ready Condition. + format: int32 + type: integer + replicas: + description: Total number of non-terminated pods targeted by this deployment (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by this deployment. This is the total number of pods that are still required for the deployment to have 100% available capacity. They may either be pods that are running but not yet available or pods that still have not been created. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted by this deployment that have the desired template spec. + format: int32 + type: integer + required: + - name + - namespace + type: object + ingress: + description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Ingress for the given cluster. + type: string + namespace: + description: The namespace which the Ingress for the given cluster is deployed. + type: string + required: + - name + - namespace + type: object + service: + description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: + - name + - namespace + - port + type: object + type: object + storage: + description: Storage Status contains information about Kubernetes storage system + properties: + etcd: + description: ETCDStatus defines the observed state of ETCDStatus + properties: + role: + properties: + exists: + type: boolean + name: + type: string + permissions: + items: + properties: + key: + type: string + rangeEnd: + type: string + type: + type: integer + type: object + type: array + required: + - exists + - name + type: object + user: + properties: + exists: + type: boolean + name: + type: string + roles: + items: + type: string + type: array + required: + - exists + - name + type: object + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/helm/kamaji/templates/NOTES.txt b/helm/kamaji/templates/NOTES.txt new file mode 100644 index 0000000..dd09de3 --- /dev/null +++ b/helm/kamaji/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kamaji.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "kamaji.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "kamaji.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "kamaji.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/helm/kamaji/templates/_helpers.tpl b/helm/kamaji/templates/_helpers.tpl new file mode 100644 index 0000000..fdcc465 --- /dev/null +++ b/helm/kamaji/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kamaji.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kamaji.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kamaji.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kamaji.labels" -}} +helm.sh/chart: {{ include "kamaji.chart" . }} +{{ include "kamaji.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kamaji.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kamaji.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: controller-manager +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kamaji.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "kamaji.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/kamaji/templates/controller.yaml b/helm/kamaji/templates/controller.yaml new file mode 100644 index 0000000..fba49ff --- /dev/null +++ b/helm/kamaji/templates/controller.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kamaji.fullname" . }} + labels: + {{- include "kamaji.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "kamaji.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "kamaji.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + serviceAccountName: {{ include "kamaji.serviceAccountName" . }} + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + - args: + - --config-file={{ .Values.configPath }} + - --etcd-ca-secret-name={{ .Values.etcd.caSecret.name }} + - --etcd-ca-secret-namespace={{ .Values.etcd.caSecret.namespace }} + - --etcd-client-secret-name={{ .Values.etcd.clientSecret.name }} + - --etcd-client-secret-namespace={{ .Values.etcd.clientSecret.namespace }} + - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }} + - --etcd-endpoints={{ .Values.etcd.endpoints }} + - --health-probe-bind-address={{ .Values.healthProbeBindAddress }} + - --leader-elect + - --metrics-bind-address={{ .Values.metricsBindAddress }} + - --tmp-directory={{ .Values.temporaryDirectoryPath }} + {{- if .Values.loggingDevel.enable }} + - --zap-devel + {{- end }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + command: + - /manager + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + name: manager + ports: + - containerPort: 8081 + name: healthcheck + protocol: TCP + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + terminationGracePeriodSeconds: 10 + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/kamaji/templates/ingress.yaml b/helm/kamaji/templates/ingress.yaml new file mode 100644 index 0000000..0ac6e4d --- /dev/null +++ b/helm/kamaji/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "kamaji.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "kamaji.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/kamaji/templates/rbac.yaml b/helm/kamaji/templates/rbac.yaml new file mode 100644 index 0000000..ed1f69e --- /dev/null +++ b/helm/kamaji/templates/rbac.yaml @@ -0,0 +1,210 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kamaji.serviceAccountName" . }} + labels: + {{- include "kamaji.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kamaji-leader-election-role + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kamaji-manager-role +rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/finalizers + verbs: + - update +- apiGroups: + - kamaji.clastix.io + resources: + - tenantcontrolplanes/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kamaji-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kamaji-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kamaji-leader-election-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kamaji-leader-election-role +subjects: +- kind: ServiceAccount + name: {{ include "kamaji.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kamaji-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kamaji-manager-role +subjects: +- kind: ServiceAccount + name: {{ include "kamaji.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kamaji-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kamaji-proxy-role +subjects: +- kind: ServiceAccount + name: {{ include "kamaji.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/helm/kamaji/templates/service.yaml b/helm/kamaji/templates/service.yaml new file mode 100644 index 0000000..2509a0d --- /dev/null +++ b/helm/kamaji/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kamaji.fullname" . }} + labels: + {{- include "kamaji.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.service.type }} + ports: + - name: https + port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "kamaji.selectorLabels" . | nindent 4 }} diff --git a/helm/kamaji/values.sample.yaml b/helm/kamaji/values.sample.yaml new file mode 100644 index 0000000..aa04ad4 --- /dev/null +++ b/helm/kamaji/values.sample.yaml @@ -0,0 +1,2 @@ +etcd: + endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379" diff --git a/helm/kamaji/values.yaml b/helm/kamaji/values.yaml new file mode 100644 index 0000000..ccf86ba --- /dev/null +++ b/helm/kamaji/values.yaml @@ -0,0 +1,136 @@ +# Default values for kamaji. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- The number of the pod replicas for the Kamaji controller. +replicaCount: 1 + +image: + # -- The container image of the Kamaji controller. + repository: quay.io/clastix/kamaji + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: latest + +# -- A list of extra arguments to add to the kamaji controller default ones +extraArgs: [] + +# -- Configuration file path alternative. (default "./kamaji.yaml") +configPath: "./kamaji.yaml" + +etcd: + caSecret: + # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") + name: etcd-certs + # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji") + namespace: kamaji-system + + clientSecret: + # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs") + name: root-client-certs + # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") + namespace: kamaji-system + + # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) + compactionInterval: 0 + + # -- (string) Comma-separated list of the endpoints of the etcd cluster's members. + endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379" + +# -- The address the probe endpoint binds to. (default ":8081") +healthProbeBindAddress: ":8081" + +# -- The livenessProbe for the controller container +livenessProbe: + httpGet: + path: /healthz + port: healthcheck + initialDelaySeconds: 15 + periodSeconds: 20 + +# -- The readinessProbe for the controller container +readinessProbe: + httpGet: + path: /readyz + port: healthcheck + initialDelaySeconds: 5 + periodSeconds: 10 + +# -- (string) The address the metric endpoint binds to. (default ":8080") +metricsBindAddress: ":8080" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: kamaji-controller-manager + +# -- The annotations to apply to the Kamaji controller pods. +podAnnotations: {} + +# -- The securityContext to apply to the Kamaji controller pods. +podSecurityContext: + runAsNonRoot: true + +# -- The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. +securityContext: + allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8443 + +ingress: + # -- Whether to expose the Kamaji controller through an Ingress. + enabled: false + # -- Name of the ingress class to route through this controller. + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + +# -- Kubernetes node selector rules to schedule Kamaji controller +nodeSelector: {} + +# -- Kubernetes node taints that the Kamaji controller pods would tolerate +tolerations: [] + +# -- Kubernetes affinity rules to apply to Kamaji controller pods +affinity: {} + +# -- Directory which will be used to work with temporary files. (default "/tmp/kamaji") +temporaryDirectoryPath: "/tmp/kamaji" + +loggingDevel: + # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) + enable: false diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..3c9b10c --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,123 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "flag" + "fmt" + "log" + + "github.com/pkg/errors" + "github.com/spf13/pflag" + "github.com/spf13/viper" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + config *viper.Viper + cfgFile string +) + +const ( + envPrefix = "KAMAJI" + defaultETCDCASecretName = "etcd-certs" + defaultETCDCASecretNamespace = "kamaji-system" + defaultETCDEndpoints = "etcd-server:2379" + defaultETCDCompactionInterval = "0" + defaultETCDClientSecretName = "root-client-certs" + defaultETCDClientSecretNamespace = "kamaji-system" + defaultTmpDirectory = "/tmp/kamaji" +) + +func InitConfig() (*viper.Viper, error) { + config = viper.New() + + // Setup flags with standard "flag" module + flag.StringVar(&cfgFile, "config-file", "kamaji.yaml", "Configuration file alternative.") + flag.String("metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.Bool("leader-elect", false, "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.String("etcd-ca-secret-name", defaultETCDCASecretName, "Name of the secret which contains CA's certificate and private key.") + flag.String("etcd-ca-secret-namespace", defaultETCDCASecretNamespace, "Namespace of the secret which contains CA's certificate and private key.") + flag.String("etcd-client-secret-name", defaultETCDClientSecretName, "Name of the secret which contains ETCD client certificates") + flag.String("etcd-client-secret-namespace", defaultETCDClientSecretNamespace, "Name of the namespace where the secret which contains ETCD client certificates is") + flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji-system.svc.cluster.local,etcd-1.etcd.kamaji-system.svc.cluster.local,etcd-2.etcd.kamaji-system.svc.cluster.local)") + flag.String("etcd-compaction-interval", defaultETCDCompactionInterval, "ETCD Compaction interval (i.e. \"5m0s\"). (default: \"0\" (disabled))") + flag.String("tmp-directory", defaultTmpDirectory, "Directory which will be used to work with temporary files.") + + // Setup zap configuration + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // Add flag set to pflag in order to parse with viper + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + if err := config.BindPFlags(pflag.CommandLine); err != nil { + log.Printf("error binding flags: %s", err) + + return nil, err + } + + // Setup environment variables + if err := config.BindEnv("metrics-bind-address", fmt.Sprintf("%s_METRICS_BIND_ADDRESS", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("health-probe-bind-address", fmt.Sprintf("%s_HEALTH_PROBE_BIND_ADDRESS", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("leader-elect", fmt.Sprintf("%s_LEADER_ELECTION", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("etcd-ca-secret-name", fmt.Sprintf("%s_ETCD_CA_SECRET_NAME", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("etcd-ca-secret-namespace", fmt.Sprintf("%s_ETCD_CA_SECRET_NAMESPACE", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("etcd-client-secret-name", fmt.Sprintf("%s_ETCD_CLIENT_SECRET_NAME", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("etcd-client-secret-namespace", fmt.Sprintf("%s_ETCD_CLIENT_SECRET_NAMESPACE", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("etcd-endpoints", fmt.Sprintf("%s_ETCD_ENDPOINTS", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("etcd-compaction-interval", fmt.Sprintf("%s_ETCD_COMPACTION_INTERVAL", envPrefix)); err != nil { + return nil, err + } + if err := config.BindEnv("tmp-directory", fmt.Sprintf("%s_TMP_DIRECTORY", envPrefix)); err != nil { + return nil, err + } + + // Setup config file + if cfgFile != "" { + // Using flag-passed config file + config.SetConfigFile(cfgFile) + } else { + // Using default config file + config.AddConfigPath(".") + config.SetConfigName("kamaji") + } + config.SetConfigType("yaml") + if err := config.ReadInConfig(); err != nil { + if errors.Is(err, &viper.ConfigParseError{}) { + log.Printf("error parsing config file: %v", err) + + return nil, err + } + log.Println("No config file used") + + return nil, err + } + + log.Printf("Using config file: %v", viper.ConfigFileUsed()) + + return config, nil +} diff --git a/internal/constants/labels.go b/internal/constants/labels.go new file mode 100644 index 0000000..9521ca3 --- /dev/null +++ b/internal/constants/labels.go @@ -0,0 +1,9 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package constants + +const ( + ProjectNameLabelKey = "kamaji.clastix.io/project" + ProjectNameLabelValue = "kamaji" +) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go new file mode 100644 index 0000000..d9f349c --- /dev/null +++ b/internal/crypto/crypto.go @@ -0,0 +1,125 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package crypto + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "time" +) + +func GetCertificate(cert []byte) (*x509.Certificate, error) { + pemContent, _ := pem.Decode(cert) + if pemContent == nil { + return nil, fmt.Errorf("no right PEM block") + } + + return x509.ParseCertificate(pemContent.Bytes) +} + +func GetPrivateKey(privKey []byte) (*rsa.PrivateKey, error) { + pemContent, _ := pem.Decode(privKey) + if pemContent == nil { + return nil, fmt.Errorf("no right PEM block") + } + + return x509.ParsePKCS1PrivateKey(pemContent.Bytes) +} + +func GetPublickKey(pubKey []byte) (*rsa.PublicKey, error) { + pemContent, _ := pem.Decode(pubKey) + if pemContent == nil { + return nil, fmt.Errorf("no right PEM block") + } + + pub, err := x509.ParsePKIXPublicKey(pemContent.Bytes) + if err != nil { + return nil, err + } + + return pub.(*rsa.PublicKey), nil // nolint:forcetypeassert +} + +func GenerateCertificateKeyPairBytes(template *x509.Certificate, bitSize int, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*bytes.Buffer, *bytes.Buffer, error) { + certPrivKey, err := rsa.GenerateKey(rand.Reader, bitSize) + if err != nil { + return nil, nil, err + } + + certBytes, err := x509.CreateCertificate(rand.Reader, template, caCert, &certPrivKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + + certPEM := &bytes.Buffer{} + if err := pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }); err != nil { + return nil, nil, err + } + + certPrivKeyPEM := &bytes.Buffer{} + if err := pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }); err != nil { + return nil, nil, err + } + + return certPEM, certPrivKeyPEM, nil +} + +func IsValidKeyPairBytes(pubKeyBytes []byte, privKeyBytes []byte) (bool, error) { + privKey, err := GetPrivateKey(privKeyBytes) + if err != nil { + return false, err + } + + pubKey, err := GetPublickKey(pubKeyBytes) + if err != nil { + return false, err + } + + return checkPublicKeys(privKey.PublicKey, *pubKey), nil +} + +func IsValidCertificateKeyPairBytes(certBytes []byte, privKeyBytes []byte) (bool, error) { + cert, err := GetCertificate(certBytes) + if err != nil { + return false, err + } + + privKey, err := GetPrivateKey(privKeyBytes) + if err != nil { + return false, err + } + + return isValidCertificateKeyPairBytes(*cert, *privKey), nil +} + +func isValidCertificateKeyPairBytes(cert x509.Certificate, privKey rsa.PrivateKey) bool { + return checkCertificateValidity(cert) && checkCertificateKeyPair(cert, privKey) +} + +func checkCertificateValidity(cert x509.Certificate) bool { + now := time.Now() + + return now.Before(cert.NotAfter) && now.After(cert.NotBefore) +} + +func checkCertificateKeyPair(cert x509.Certificate, privKey rsa.PrivateKey) bool { + return checkPublicKeys(*cert.PublicKey.(*rsa.PublicKey), privKey.PublicKey) // nolint:forcetypeassert +} + +func checkPublicKeys(a rsa.PublicKey, b rsa.PublicKey) bool { + isN := a.N.Cmp(b.N) == 0 + isE := a.E == b.E + + return isN && isE +} diff --git a/internal/etcd/api.go b/internal/etcd/api.go new file mode 100644 index 0000000..a423f79 --- /dev/null +++ b/internal/etcd/api.go @@ -0,0 +1,177 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package etcd + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "time" + + "go.etcd.io/etcd/api/v3/authpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + etcdclient "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc/codes" +) + +const ( + etcdTimeout = 10 // seconds + + // rangeEnd is the key following the last key of the range. + // If rangeEnd is ‘\0’, the range is all keys greater than or equal to the key argument + // source: https://etcd.io/docs/v3.5/learning/api/ + rangeEnd = "\\0" +) + +func NewClient(config Config) (*etcdclient.Client, error) { + cert, err := tls.X509KeyPair(config.ETCDCertificate, config.ETCDPrivateKey) + if err != nil { + return nil, err + } + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(config.ETCDCA) + + cfg := etcdclient.Config{ + Endpoints: config.Endpoints, + TLS: &tls.Config{ // nolint:gosec + Certificates: []tls.Certificate{cert}, + RootCAs: pool, + }, + } + + return etcdclient.New(cfg) +} + +func GetUser(ctx context.Context, client *etcdclient.Client, user *User) error { + ctxWithTimeout, cancel := context.WithTimeout(ctx, etcdTimeout*time.Second) + defer cancel() + + response, err := client.UserGet(ctxWithTimeout, user.Name) + if err != nil { + var etcdError rpctypes.EtcdError + if errors.As(err, &etcdError) && etcdError.Code() == codes.FailedPrecondition { + return nil + } + + return err + } + + user.Roles = response.Roles + user.Exists = true + + return nil +} + +func AddUser(ctx context.Context, client *etcdclient.Client, username string) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + opts := etcdclient.UserAddOptions{ + NoPassword: true, + } + _, err := client.Auth.UserAddWithOptions(ctxWithTimeout, username, "", &opts) + + return err +} + +func RemoveUser(ctx context.Context, client *etcdclient.Client, username string) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + _, err := client.Auth.UserDelete(ctxWithTimeout, username) + + return err +} + +func GetRole(ctx context.Context, client *etcdclient.Client, role *Role) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + response, err := client.RoleGet(ctxWithTimeout, role.Name) + if err != nil { + var etcdError rpctypes.EtcdError + if errors.As(err, &etcdError) && etcdError.Code() == codes.FailedPrecondition { + return nil + } + + return err + } + + role.Exists = true + for _, perm := range response.Perm { + permission := Permission{ + Type: int(perm.PermType), + Key: string(perm.Key), + RangeEnd: string(perm.RangeEnd), + } + role.Permissions = append(role.Permissions, permission) + } + + return nil +} + +func AddRole(ctx context.Context, client *etcdclient.Client, roleName string) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + _, err := client.Auth.RoleAdd(ctxWithTimeout, roleName) + + return err +} + +func RemoveRole(ctx context.Context, client *etcdclient.Client, roleName string) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + _, err := client.Auth.RoleDelete(ctxWithTimeout, roleName) + + return err +} + +func GrantUserRole(ctx context.Context, client *etcdclient.Client, user User, role Role) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + _, err := client.UserGrantRole(ctxWithTimeout, user.Name, role.Name) + if err != nil { + return err + } + + return nil +} + +func GrantRolePermission(ctx context.Context, client *etcdclient.Client, role Role) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + permission := etcdclient.PermissionType(authpb.READWRITE) + key := BuildKey(role.Name) + _, err := client.RoleGrantPermission(ctxWithTimeout, role.Name, key, rangeEnd, permission) + if err != nil { + return err + } + + return nil +} + +func CleanUpPrefix(ctx context.Context, client *etcdclient.Client, name string) error { + ctxWithTimeout, cancel := getContextWithTimeout(ctx) + defer cancel() + + withRange := etcdclient.WithRange(rangeEnd) + prefix := BuildKey(name) + _, err := client.Delete(ctxWithTimeout, prefix, withRange) + + return err +} + +func BuildKey(roleName string) string { + return fmt.Sprintf("/%s/", roleName) +} + +func getContextWithTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, etcdTimeout*time.Second) +} diff --git a/internal/etcd/certs.go b/internal/etcd/certs.go new file mode 100644 index 0000000..5d91fc6 --- /dev/null +++ b/internal/etcd/certs.go @@ -0,0 +1,57 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package etcd + +import ( + "bytes" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "math/rand" + "time" + + "github.com/clastix/kamaji/internal/crypto" +) + +func GetETCDCACertificateAndKeyPair(tenant string, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) { + template := getCertTemplate(tenant) + + caCertBytes, err := crypto.GetCertificate(caCert) + if err != nil { + return nil, nil, err + } + + caPrivKeyBytes, err := crypto.GetPrivateKey(caPrivKey) + if err != nil { + return nil, nil, err + } + + return crypto.GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes) +} + +func IsETCDCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) { + return crypto.IsValidCertificateKeyPairBytes(cert, privKey) +} + +func getCertTemplate(tenant string) *x509.Certificate { + serialNumber := big.NewInt(rand.Int63()) + + return &x509.Certificate{ + PublicKeyAlgorithm: x509.RSA, + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: tenant, + Organization: []string{certOrganization}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(certExpirationDelayYears, 0, 0), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + x509.ExtKeyUsageCodeSigning, + }, + KeyUsage: x509.KeyUsageDigitalSignature, + } +} diff --git a/internal/etcd/constant.go b/internal/etcd/constant.go new file mode 100644 index 0000000..86a672d --- /dev/null +++ b/internal/etcd/constant.go @@ -0,0 +1,10 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package etcd + +const ( + certExpirationDelayYears = 10 + certOrganization = "system:masters" + certBitSize = 2048 +) diff --git a/internal/etcd/types.go b/internal/etcd/types.go new file mode 100644 index 0000000..3207f7f --- /dev/null +++ b/internal/etcd/types.go @@ -0,0 +1,71 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package etcd + +type Config struct { + ETCDCertificate []byte + ETCDPrivateKey []byte + ETCDCA []byte + Endpoints []string +} + +type Permission struct { + Type int `json:"type,omitempty"` + Key string `json:"key,omitempty"` + RangeEnd string `json:"rangeEnd,omitempty"` +} + +func (in *Permission) DeepCopyInto(out *Permission) { + *out = *in +} + +func (in *Permission) DeepCopy() *Permission { + if in == nil { + return nil + } + out := new(Permission) + in.DeepCopyInto(out) + + return out +} + +type Role struct { + Name string `json:"name"` + Permissions []Permission `json:"permissions,omitempty"` + Exists bool `json:"exists"` +} + +func (in *Role) DeepCopyInto(out *Role) { + *out = *in +} + +func (in *Role) DeepCopy() *Role { + if in == nil { + return nil + } + out := new(Role) + in.DeepCopyInto(out) + + return out +} + +type User struct { + Name string `json:"name"` + Roles []string `json:"roles,omitempty"` + Exists bool `json:"exists"` +} + +func (in *User) DeepCopyInto(out *User) { + *out = *in +} + +func (in *User) DeepCopy() *User { + if in == nil { + return nil + } + out := new(User) + in.DeepCopyInto(out) + + return out +} diff --git a/internal/kubeadm/addon.go b/internal/kubeadm/addon.go new file mode 100644 index 0000000..16a6b81 --- /dev/null +++ b/internal/kubeadm/addon.go @@ -0,0 +1,287 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api/v1" + "k8s.io/component-base/config/v1alpha1" + kubeproxyconfig "k8s.io/kube-proxy/config/v1alpha1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/utils/pointer" +) + +func CoreDNSAddon(client kubernetes.Interface, config *Configuration) error { + return dns.EnsureDNSAddon(&config.InitConfiguration.ClusterConfiguration, client) +} + +func KubeProxyAddon(client kubernetes.Interface, config *Configuration) error { + if err := proxy.CreateServiceAccount(client); err != nil { + return errors.Wrap(err, "error when creating kube-proxy service account") + } + + if err := createKubeProxyConfigMap(client, config); err != nil { + return err + } + + if err := createKubeProxyAddon(client); err != nil { + return err + } + + if err := proxy.CreateRBACRules(client); err != nil { + return errors.Wrap(err, "error when creating kube-proxy RBAC rules") + } + + return nil +} + +func createKubeProxyConfigMap(client kubernetes.Interface, config *Configuration) error { + configConf, err := getKubeproxyConfigmapContent(config) + if err != nil { + return err + } + + kubeconfigConf, err := getKubeproxyKubeconfigContent(config) + if err != nil { + return err + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeProxyConfigMap, + Namespace: "kube-system", + Labels: map[string]string{ + "app": "kube-proxy", + }, + }, + Data: map[string]string{ + kubeadmconstants.KubeProxyConfigMapKey: string(configConf), + "kubeconfig.conf": string(kubeconfigConf), + }, + } + + return apiclient.CreateOrUpdateConfigMap(client, configMap) +} + +func createKubeProxyAddon(client kubernetes.Interface) error { + daemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-proxy", + Namespace: "kube-system", + Labels: map[string]string{ + "k8s-app": "kube-proxy", + }, + }, + Spec: appsv1.DaemonSetSpec{ + RevisionHistoryLimit: pointer.Int32(10), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": "kube-proxy", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "k8s-app": "kube-proxy", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Command: []string{ + "/usr/local/bin/kube-proxy", + "--config=/var/lib/kube-proxy/config.conf", + "--hostname-override=$(NODE_NAME)", + }, + Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + Image: "k8s.gcr.io/kube-proxy:v1.21.2", + ImagePullPolicy: corev1.PullIfNotPresent, + Name: "kube-proxy", + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.Bool(true), + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/var/lib/kube-proxy", + Name: "kube-proxy", + }, + { + MountPath: "/run/xtables.lock", + Name: "xtables-lock", + }, + { + MountPath: "/lib/modules", + Name: "lib-modules", + ReadOnly: true, + }, + }, + }, + }, + DNSPolicy: corev1.DNSClusterFirst, + HostNetwork: true, + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + PriorityClassName: "system-node-critical", + RestartPolicy: corev1.RestartPolicyAlways, + SchedulerName: "default-scheduler", + ServiceAccountName: "kube-proxy", + TerminationGracePeriodSeconds: pointer.Int64(30), + Volumes: []corev1.Volume{ + { + Name: "kube-proxy", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: pointer.Int32(420), + LocalObjectReference: corev1.LocalObjectReference{ + Name: "kube-proxy", + }, + }, + }, + }, + { + Name: "xtables-lock", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/xtables.lock", + Type: (*corev1.HostPathType)(pointer.String(string(corev1.HostPathFileOrCreate))), + }, + }, + }, + { + Name: "lib-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + Type: (*corev1.HostPathType)(pointer.String(string(corev1.HostPathUnset))), + }, + }, + }, + }, + }, + }, + }, + } + + return apiclient.CreateOrUpdateDaemonSet(client, daemonSet) +} + +func getKubeproxyConfigmapContent(config *Configuration) ([]byte, error) { + zeroDuration := metav1.Duration{Duration: 0} + oneSecondDuration := metav1.Duration{Duration: time.Second} + kubeProxyConfiguration := kubeproxyconfig.KubeProxyConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeProxyConfiguration", + APIVersion: "kubeproxy.config.k8s.io/v1alpha1", + }, + BindAddress: "0.0.0.0", + BindAddressHardFail: false, + ClientConnection: v1alpha1.ClientConnectionConfiguration{ + AcceptContentTypes: "", + Burst: 0, + ContentType: "", + Kubeconfig: "/var/lib/kube-proxy/kubeconfig.conf", + QPS: 0, + }, + ClusterCIDR: config.Parameters.TenantControlPlanePodCIDR, + ConfigSyncPeriod: zeroDuration, + Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ + MaxPerCore: pointer.Int32(0), + Min: nil, + TCPCloseWaitTimeout: nil, + TCPEstablishedTimeout: nil, + }, + DetectLocalMode: "", + EnableProfiling: false, + HealthzBindAddress: "", + HostnameOverride: "", + IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: false, + MasqueradeBit: nil, + MinSyncPeriod: oneSecondDuration, + SyncPeriod: zeroDuration, + }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + ExcludeCIDRs: nil, + MinSyncPeriod: zeroDuration, + Scheduler: "", + StrictARP: false, + SyncPeriod: zeroDuration, + TCPTimeout: zeroDuration, + TCPFinTimeout: zeroDuration, + UDPTimeout: zeroDuration, + }, + MetricsBindAddress: "", + Mode: "iptables", + NodePortAddresses: nil, + OOMScoreAdj: nil, + PortRange: "", + ShowHiddenMetricsForVersion: "", + UDPIdleTimeout: zeroDuration, + Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{ + EnableDSR: false, + NetworkName: "", + SourceVip: "", + }, + } + + return EncondeToYaml(&kubeProxyConfiguration) +} + +func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) { + kubeconfig := clientcmdapi.Config{ + APIVersion: "v1", + Kind: "Config", + Clusters: []clientcmdapi.NamedCluster{ + { + Name: "default", + Cluster: clientcmdapi.Cluster{ + CertificateAuthority: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + Server: fmt.Sprintf("https://%s:%d", config.Parameters.TenantControlPlaneAddress, config.Parameters.TenantControlPlanePort), + }, + }, + }, + Contexts: []clientcmdapi.NamedContext{ + { + Context: clientcmdapi.Context{ + Cluster: "default", + Namespace: "default", + AuthInfo: "default", + }, + }, + }, + AuthInfos: []clientcmdapi.NamedAuthInfo{ + { + Name: "default", + AuthInfo: clientcmdapi.AuthInfo{ + TokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + }, + }, + } + + return EncondeToYaml(&kubeconfig) +} diff --git a/internal/kubeadm/bootstraptoken.go b/internal/kubeadm/bootstraptoken.go new file mode 100644 index 0000000..14aa08d --- /dev/null +++ b/internal/kubeadm/bootstraptoken.go @@ -0,0 +1,73 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" +) + +func BootstrapToken(client kubernetes.Interface, config *Configuration) error { + initConfiguration := config.InitConfiguration + + if err := node.UpdateOrCreateTokens(client, false, initConfiguration.BootstrapTokens); err != nil { + return errors.Wrap(err, "error updating or creating token") + } + + if err := node.AllowBoostrapTokensToGetNodes(client); err != nil { + return errors.Wrap(err, "error allowing bootstrap tokens to get Nodes") + } + + if err := node.AllowBootstrapTokensToPostCSRs(client); err != nil { + return errors.Wrap(err, "error allowing bootstrap tokens to post CSRs") + } + + if err := node.AutoApproveNodeBootstrapTokens(client); err != nil { + return errors.Wrap(err, "error auto-approving node bootstrap tokens") + } + + if err := node.AutoApproveNodeCertificateRotation(client); err != nil { + return err + } + + bootstrapConfig := &clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + "": { + Server: config.Kubeconfig.Clusters[0].Cluster.Server, + CertificateAuthorityData: config.Kubeconfig.Clusters[0].Cluster.CertificateAuthorityData, + }, + }, + } + bootstrapBytes, err := clientcmd.Write(*bootstrapConfig) + if err != nil { + return err + } + + err = apiclient.CreateOrUpdateConfigMap(client, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: bootstrapapi.ConfigMapClusterInfo, + Namespace: metav1.NamespacePublic, + }, + Data: map[string]string{ + bootstrapapi.KubeConfigKey: string(bootstrapBytes), + }, + }) + if err != nil { + return err + } + + if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil { + return errors.Wrap(err, "error creating clusterinfo RBAC rules") + } + + return nil +} diff --git a/internal/kubeadm/certificates.go b/internal/kubeadm/certificates.go new file mode 100644 index 0000000..c38a557 --- /dev/null +++ b/internal/kubeadm/certificates.go @@ -0,0 +1,165 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "crypto" + "crypto/x509" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + + cryptoKamaji "github.com/clastix/kamaji/internal/crypto" +) + +func GenerateCACertificatePrivateKeyPair(baseName string, config *Configuration) (*CertificatePrivateKeyPair, error) { + defer deleteCertificateDirectory(config.InitConfiguration.CertificatesDir) + + kubeadmCert, err := getKubeadmCert(baseName) + if err != nil { + return nil, err + } + + if _, _, err = initPhaseAsCA(kubeadmCert, config); err != nil { + return nil, err + } + + contents, err := readCertificateFiles(baseName, config.InitConfiguration.CertificatesDir, "crt", "key") + if err != nil { + return nil, err + } + + certificatePrivateKeyPair := &CertificatePrivateKeyPair{ + Certificate: contents[0], + PrivateKey: contents[1], + } + + return certificatePrivateKeyPair, err +} + +func GenerateCertificatePrivateKeyPair(baseName string, config *Configuration, ca CertificatePrivateKeyPair) (*CertificatePrivateKeyPair, error) { + defer deleteCertificateDirectory(config.InitConfiguration.CertificatesDir) + + certificate, _ := cryptoKamaji.GetCertificate(ca.Certificate) + signer, _ := cryptoKamaji.GetPrivateKey(ca.PrivateKey) + + kubeadmCert, err := getKubeadmCert(baseName) + if err != nil { + return nil, err + } + + if err = initPhaseFromCA(kubeadmCert, config, certificate, signer); err != nil { + return nil, err + } + + contents, err := readCertificateFiles(baseName, config.InitConfiguration.CertificatesDir, "crt", "key") + if err != nil { + return nil, err + } + + certificatePrivateKeyPair := &CertificatePrivateKeyPair{ + Certificate: contents[0], + PrivateKey: contents[1], + } + + return certificatePrivateKeyPair, err +} + +func getKubeadmCert(baseName string) (*certs.KubeadmCert, error) { + switch baseName { + case kubeadmconstants.CACertAndKeyBaseName: + return certs.KubeadmCertRootCA(), nil + case kubeadmconstants.APIServerCertAndKeyBaseName: + return certs.KubeadmCertAPIServer(), nil + case kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName: + return certs.KubeadmCertKubeletClient(), nil + case kubeadmconstants.FrontProxyCACertAndKeyBaseName: + return certs.KubeadmCertFrontProxyCA(), nil + case kubeadmconstants.FrontProxyClientCertAndKeyBaseName: + return certs.KubeadmCertFrontProxyClient(), nil + default: + return nil, fmt.Errorf("wrong ca file name %s", baseName) + } +} + +func GeneratePublicKeyPrivateKeyPair(baseName string, config *Configuration) (*PublicKeyPrivateKeyPair, error) { + defer deleteCertificateDirectory(config.InitConfiguration.CertificatesDir) + + if err := initPhaseCertsSA(config); err != nil { + return nil, err + } + + contents, err := readCertificateFiles(baseName, config.InitConfiguration.CertificatesDir, "pub", "key") + if err != nil { + return nil, err + } + + publicKeyPrivateKeyPair := &PublicKeyPrivateKeyPair{ + PublicKey: contents[0], + PrivateKey: contents[1], + } + + return publicKeyPrivateKeyPair, err +} + +func IsCertificatePrivateKeyPairValid(certificate []byte, privKey []byte) (bool, error) { + if len(certificate) == 0 { + return false, nil + } + if len(privKey) == 0 { + return false, nil + } + + return cryptoKamaji.IsValidCertificateKeyPairBytes(certificate, privKey) +} + +func IsPublicKeyPrivateKeyPairValid(pubKey []byte, privKey []byte) (bool, error) { + if len(pubKey) == 0 { + return false, nil + } + if len(privKey) == 0 { + return false, nil + } + + return cryptoKamaji.IsValidKeyPairBytes(pubKey, privKey) +} + +func initPhaseCertsSA(config *Configuration) error { + return certs.CreateServiceAccountKeyAndPublicKeyFiles(config.InitConfiguration.CertificatesDir, config.InitConfiguration.PublicKeyAlgorithm()) +} + +func initPhaseFromCA(kubeadmCert *certs.KubeadmCert, config *Configuration, certificate *x509.Certificate, signer crypto.Signer) error { + return kubeadmCert.CreateFromCA(&config.InitConfiguration, certificate, signer) +} + +func initPhaseAsCA(kubeadmCert *certs.KubeadmCert, config *Configuration) (*x509.Certificate, crypto.Signer, error) { + return kubeadmCert.CreateAsCA(&config.InitConfiguration) +} + +func readCertificateFiles(name string, directory string, extensions ...string) ([][]byte, error) { + result := make([][]byte, 0, len(extensions)) + + for _, extension := range extensions { + fileName := fmt.Sprintf("%s.%s", name, extension) + path := filepath.Join(directory, fileName) + content, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + result = append(result, content) + } + + return result, nil +} + +func deleteCertificateDirectory(certificateDirectory string) { + if err := os.RemoveAll(certificateDirectory); err != nil { + // TODO(prometherion): we should log rather than printing to stdout + fmt.Printf("Error removing %s: %s", certificateDirectory, err.Error()) // nolint:forbidigo + } +} diff --git a/internal/kubeadm/configuration.go b/internal/kubeadm/configuration.go new file mode 100644 index 0000000..01105fd --- /dev/null +++ b/internal/kubeadm/configuration.go @@ -0,0 +1,142 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "encoding/json" + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +func CreateKubeadmInitConfiguration(params Parameters) Configuration { + config := kubeadmapi.InitConfiguration{ + ClusterConfiguration: getKubeadmClusterConfiguration(params), + BootstrapTokens: []bootstraptokenv1.BootstrapToken{ + { + Groups: []string{"system:bootstrappers:kubeadm:default-node-token"}, + TTL: &metav1.Duration{Duration: 48 * time.Hour}, + Usages: []string{ + "signing", + "authentication", + }, + }, + }, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ + AdvertiseAddress: params.TenantControlPlaneAddress, + BindPort: params.TenantControlPlanePort, + }, + NodeRegistration: kubeadmapi.NodeRegistrationOptions{ + CRISocket: "unix:///run/containerd/containerd.sock", + Name: params.TenantControlPlaneName, + }, + } + + return Configuration{InitConfiguration: config} +} + +func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfiguration { + return kubeadmapi.ClusterConfiguration{ + KubernetesVersion: params.TenantControlPlaneVersion, + ClusterName: params.TenantControlPlaneName, + CertificatesDir: "/etc/kubernetes/pki", + ImageRepository: "k8s.gcr.io", + Networking: kubeadmapi.Networking{ + DNSDomain: "cluster.local", + PodSubnet: params.TenantControlPlanePodCIDR, + ServiceSubnet: params.TenantControlPlaneServiceCIDR, + }, + DNS: kubeadmapi.DNS{ + Type: "CoreDNS", + }, + ControlPlaneEndpoint: params.TenantControlPlaneEndpoint, + Etcd: kubeadmapi.Etcd{ + External: &kubeadmapi.ExternalEtcd{ + Endpoints: formatETCDEndpoints(params.ETCDs), + CAFile: "/etc/kubernetes/pki/etcd/ca.crt", + CertFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt", + KeyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key", + }, + }, + APIServer: kubeadmapi.APIServer{ + CertSANs: []string{ + "127.0.0.1", + "localhost", + fmt.Sprintf("%s.%s", params.TenantControlPlaneName, params.TenantControlPlaneDomain), + params.TenantControlPlaneName, + fmt.Sprintf("%s.%s.svc", params.TenantControlPlaneName, params.TenantControlPlaneNamespace), + fmt.Sprintf("%s.%s.svc.cluster.local", params.TenantControlPlaneName, params.TenantControlPlaneNamespace), + params.TenantControlPlaneAddress, + }, + ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + "etcd-compaction-interval": params.ETCDCompactionInterval, + "etcd-prefix": fmt.Sprintf("/%s", params.TenantControlPlaneName), + }, + }, + }, + } +} + +func GetKubeadmInitConfigurationMap(config Configuration) (map[string]string, error) { + initConfigurationString, err := getJSONStringFromStruct(config.InitConfiguration) + if err != nil { + return map[string]string{}, err + } + + clusterConfigurationString, err := getJSONStringFromStruct(config.InitConfiguration.ClusterConfiguration) + if err != nil { + return map[string]string{}, err + } + + return map[string]string{ + kubeadmconstants.InitConfigurationKind: initConfigurationString, + kubeadmconstants.ClusterConfigurationKind: clusterConfigurationString, + }, nil +} + +func GetKubeadmInitConfigurationFromMap(config map[string]string) (*Configuration, error) { + initConfigurationString, ok := config[kubeadmconstants.InitConfigurationKind] + if !ok { + return nil, fmt.Errorf("%s is not in the map", kubeadmconstants.InitConfigurationKind) + } + + clusterConfigurationString, ok := config[kubeadmconstants.ClusterConfigurationKind] + if !ok { + return nil, fmt.Errorf("%s is not in the map", kubeadmconstants.ClusterConfigurationKind) + } + + initConfiguration := kubeadmapi.InitConfiguration{} + if err := json.Unmarshal([]byte(initConfigurationString), &initConfiguration); err != nil { + return nil, err + } + + if err := json.Unmarshal([]byte(clusterConfigurationString), &initConfiguration.ClusterConfiguration); err != nil { + return nil, err + } + + return &Configuration{InitConfiguration: initConfiguration}, nil +} + +func getJSONStringFromStruct(i interface{}) (string, error) { + b, err := json.Marshal(i) + if err != nil { + return "", err + } + + return string(b), nil +} + +func formatETCDEndpoints(etcds []string) []string { + formatedETCDs := make([]string, 0, len(etcds)) + for _, etcd := range etcds { + formatedETCDs = append(formatedETCDs, fmt.Sprintf("https://%s/", etcd)) + } + + return formatedETCDs +} diff --git a/internal/kubeadm/kubeconfig.go b/internal/kubeadm/kubeconfig.go new file mode 100644 index 0000000..8d7fe81 --- /dev/null +++ b/internal/kubeadm/kubeconfig.go @@ -0,0 +1,52 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "io/ioutil" + "os" + "path" + "path/filepath" + + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" +) + +func buildCertificateDirectoryWithCA(ca CertificatePrivateKeyPair, directory string) error { + if err := os.MkdirAll(directory, os.FileMode(0o755)); err != nil { + return err + } + + certPath := path.Join(directory, kubeadmconstants.CACertName) + if err := ioutil.WriteFile(certPath, ca.Certificate, os.FileMode(0o600)); err != nil { + return err + } + + keyPath := path.Join(directory, kubeadmconstants.CAKeyName) + if err := ioutil.WriteFile(keyPath, ca.PrivateKey, os.FileMode(0o600)); err != nil { + return err + } + + return nil +} + +func CreateKubeconfig(kubeconfigName string, ca CertificatePrivateKeyPair, config *Configuration) ([]byte, error) { + if err := buildCertificateDirectoryWithCA(ca, config.InitConfiguration.CertificatesDir); err != nil { + return nil, err + } + + defer deleteCertificateDirectory(config.InitConfiguration.CertificatesDir) + + if err := kubeconfig.CreateKubeConfigFile(kubeconfigName, config.InitConfiguration.CertificatesDir, &config.InitConfiguration); err != nil { + return nil, err + } + + path := filepath.Join(config.InitConfiguration.CertificatesDir, kubeconfigName) + + return ioutil.ReadFile(path) +} + +func IsKubeconfigValid(kubeconfigBytes []byte) bool { + return len(kubeconfigBytes) > 0 +} diff --git a/internal/kubeadm/types.go b/internal/kubeadm/types.go new file mode 100644 index 0000000..6515fe3 --- /dev/null +++ b/internal/kubeadm/types.go @@ -0,0 +1,52 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + + kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig" +) + +type Configuration struct { + InitConfiguration kubeadmapi.InitConfiguration + Kubeconfig kubeconfigutil.Kubeconfig + Parameters Parameters +} + +type Parameters struct { + TenantControlPlaneName string + TenantControlPlaneNamespace string + TenantControlPlaneEndpoint string + TenantControlPlaneAddress string + TenantControlPlanePort int32 + TenantControlPlaneDomain string + TenantControlPlanePodCIDR string + TenantControlPlaneServiceCIDR string + TenantDNSServiceIPs []string + TenantControlPlaneVersion string + TenantControlPlaneCGroupDriver string + ETCDs []string + ETCDCompactionInterval string + CertificatesDir string + KubeconfigDir string +} + +type KubeletConfiguration struct { + TenantControlPlaneDomain string + TenantControlPlaneDNSServiceIPs []string + TenantControlPlaneCgroupDriver string +} + +type CertificatePrivateKeyPair struct { + Name string + Certificate []byte + PrivateKey []byte +} + +type PublicKeyPrivateKeyPair struct { + Name string + PublicKey []byte + PrivateKey []byte +} diff --git a/internal/kubeadm/uploadconfig.go b/internal/kubeadm/uploadconfig.go new file mode 100644 index 0000000..8291598 --- /dev/null +++ b/internal/kubeadm/uploadconfig.go @@ -0,0 +1,188 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/client-go/kubernetes" + kubelettypes "k8s.io/kubelet/config/v1beta1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/pkg/apis/rbac" + "k8s.io/utils/pointer" +) + +func UploadKubeadmConfig(client kubernetes.Interface, config *Configuration) error { + return uploadconfig.UploadConfiguration(&config.InitConfiguration, client) +} + +func UploadKubeletConfig(client kubernetes.Interface, config *Configuration) error { + kubeletConfiguration := KubeletConfiguration{ + TenantControlPlaneDomain: config.InitConfiguration.Networking.DNSDomain, + TenantControlPlaneDNSServiceIPs: config.Parameters.TenantDNSServiceIPs, + TenantControlPlaneCgroupDriver: config.Parameters.TenantControlPlaneCGroupDriver, + } + content, err := getKubeletConfigmapContent(kubeletConfiguration) + if err != nil { + return err + } + + configMapName, err := configMapName(config.Parameters.TenantControlPlaneVersion) + if err != nil { + return err + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(content), + }, + } + + if err := apiclient.CreateOrUpdateConfigMap(client, configMap); err != nil { + return err + } + + if err := createConfigMapRBACRules(client, config.Parameters.TenantControlPlaneVersion); err != nil { + return errors.Wrap(err, "error creating kubelet configuration configmap RBAC rules") + } + + return nil +} + +func getKubeletConfigmapContent(kubeletConfiguration KubeletConfiguration) ([]byte, error) { + zeroDuration := metav1.Duration{Duration: 0} + + kc := kubelettypes.KubeletConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeletConfiguration", + APIVersion: "kubelet.config.k8s.io/v1beta1", + }, + Authentication: kubelettypes.KubeletAuthentication{ + Anonymous: kubelettypes.KubeletAnonymousAuthentication{ + Enabled: pointer.Bool(false), + }, + Webhook: kubelettypes.KubeletWebhookAuthentication{ + Enabled: pointer.Bool(true), + CacheTTL: zeroDuration, + }, + X509: kubelettypes.KubeletX509Authentication{ + ClientCAFile: "/etc/kubernetes/pki/ca.crt", + }, + }, + Authorization: kubelettypes.KubeletAuthorization{ + Mode: kubelettypes.KubeletAuthorizationModeWebhook, + Webhook: kubelettypes.KubeletWebhookAuthorization{ + CacheAuthorizedTTL: zeroDuration, + CacheUnauthorizedTTL: zeroDuration, + }, + }, + CgroupDriver: kubeletConfiguration.TenantControlPlaneCgroupDriver, + ClusterDNS: kubeletConfiguration.TenantControlPlaneDNSServiceIPs, + ClusterDomain: kubeletConfiguration.TenantControlPlaneDomain, + CPUManagerReconcilePeriod: zeroDuration, + EvictionHard: map[string]string{ + "imagefs.available": "0%", + "nodefs.available": "0%", + "nodefs.inodesFree": "0%", + }, + EvictionPressureTransitionPeriod: zeroDuration, + FileCheckFrequency: zeroDuration, + HealthzBindAddress: "127.0.0.1", + HealthzPort: pointer.Int32(10248), + HTTPCheckFrequency: zeroDuration, + ImageGCHighThresholdPercent: pointer.Int32(100), + NodeStatusUpdateFrequency: zeroDuration, + NodeStatusReportFrequency: zeroDuration, + RotateCertificates: true, + RuntimeRequestTimeout: zeroDuration, + ShutdownGracePeriod: zeroDuration, + ShutdownGracePeriodCriticalPods: zeroDuration, + StaticPodPath: "/etc/kubernetes/manifests", + StreamingConnectionIdleTimeout: zeroDuration, + SyncFrequency: zeroDuration, + VolumeStatsAggPeriod: zeroDuration, + } + + return EncondeToYaml(&kc) +} + +func createConfigMapRBACRules(client kubernetes.Interface, kubernetesVersion string) error { + configMapName, err := configMapName(kubernetesVersion) + if err != nil { + return err + } + + configMapRBACName, err := configMapRBACName(kubernetesVersion) + if err != nil { + return err + } + + if err := apiclient.CreateOrUpdateRole(client, &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapRBACName, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get"}, + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{configMapName}, + }, + }, + }); err != nil { + return err + } + + return apiclient.CreateOrUpdateRoleBinding(client, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapRBACName, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: configMapRBACName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodesGroup, + }, + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, + }, + }, + }) +} + +func configMapName(kubernetesVersion string) (string, error) { + version, err := k8sversion.ParseSemantic(kubernetesVersion) + if err != nil { + return "", err + } + + return kubeadmconstants.GetKubeletConfigMapName(version, true), nil +} + +func configMapRBACName(kubernetesVersion string) (string, error) { + version, err := k8sversion.ParseSemantic(kubernetesVersion) + if err != nil { + return "", err + } + + return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, version.Major(), version.Minor()), nil +} diff --git a/internal/kubeadm/utils.go b/internal/kubeadm/utils.go new file mode 100644 index 0000000..05919de --- /dev/null +++ b/internal/kubeadm/utils.go @@ -0,0 +1,20 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeadm + +import ( + "bytes" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" +) + +func EncondeToYaml(o runtime.Object) ([]byte, error) { + scheme := runtime.NewScheme() + encoder := json.NewYAMLSerializer(json.SimpleMetaFactory{}, scheme, scheme) + buf := bytes.NewBuffer([]byte{}) + err := encoder.Encode(o, buf) + + return buf.Bytes(), err +} diff --git a/internal/kubeconfig/kubeconfig.go b/internal/kubeconfig/kubeconfig.go new file mode 100644 index 0000000..64dd23c --- /dev/null +++ b/internal/kubeconfig/kubeconfig.go @@ -0,0 +1,28 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package kubeconfig + +import ( + "bytes" + + "k8s.io/apimachinery/pkg/util/yaml" + v1 "k8s.io/client-go/tools/clientcmd/api/v1" +) + +type Kubeconfig v1.Config + +func GetKubeconfigFromBytesBuffer(buffer *bytes.Buffer) (*Kubeconfig, error) { + kubeconfig := &Kubeconfig{} + if err := yaml.NewYAMLOrJSONDecoder(buffer, buffer.Len()).Decode(kubeconfig); err != nil { + return nil, err + } + + return kubeconfig, nil +} + +func GetKubeconfigFromBytes(b []byte) (*Kubeconfig, error) { + buffer := bytes.NewBuffer(b) + + return GetKubeconfigFromBytesBuffer(buffer) +} diff --git a/internal/resources/api_server_certificate.go b/internal/resources/api_server_certificate.go new file mode 100644 index 0000000..afd46f6 --- /dev/null +++ b/internal/resources/api_server_certificate.go @@ -0,0 +1,136 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type APIServerCertificate struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + TmpDirectory string +} + +func (r *APIServerCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Certificates.APIServer.SecretName != r.resource.GetName() +} + +func (r *APIServerCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *APIServerCertificate) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *APIServerCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *APIServerCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *APIServerCertificate) GetClient() client.Client { + return r.Client +} + +func (r *APIServerCertificate) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *APIServerCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *APIServerCertificate) GetName() string { + return r.Name +} + +func (r *APIServerCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Certificates.APIServer.LastUpdate = metav1.Now() + tenantControlPlane.Status.Certificates.APIServer.SecretName = r.resource.GetName() + + return nil +} + +func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestConfigRV := getLatestConfigRV(*tenantControlPlane) + actualConfigRV := r.resource.GetLabels()["latest-config-rv"] + if latestConfigRV == actualConfigRV { + isValid, err := kubeadm.IsCertificatePrivateKeyPairValid( + r.resource.Data[kubeadmconstants.APIServerCertName], + r.resource.Data[kubeadmconstants.APIServerKeyName], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.APIServerCertAndKeyBaseName, err.Error())) + } + if isValid { + return nil + } + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName} + secretCA := &corev1.Secret{} + if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil { + return err + } + + ca := kubeadm.CertificatePrivateKeyPair{ + Name: kubeadmconstants.CACertAndKeyBaseName, + Certificate: secretCA.Data[kubeadmconstants.CACertName], + PrivateKey: secretCA.Data[kubeadmconstants.CAKeyName], + } + certificateKeyPair, err := kubeadm.GenerateCertificatePrivateKeyPair(kubeadmconstants.APIServerCertAndKeyBaseName, config, ca) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.APIServerCertName: certificateKeyPair.Certificate, + kubeadmconstants.APIServerKeyName: certificateKeyPair.PrivateKey, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-config-rv": latestConfigRV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} diff --git a/internal/resources/api_server_kubelet_client_certificate.go b/internal/resources/api_server_kubelet_client_certificate.go new file mode 100644 index 0000000..bc8fc32 --- /dev/null +++ b/internal/resources/api_server_kubelet_client_certificate.go @@ -0,0 +1,136 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type APIServerKubeletClientCertificate struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + TmpDirectory string +} + +func (r *APIServerKubeletClientCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName != r.resource.GetName() +} + +func (r *APIServerKubeletClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *APIServerKubeletClientCertificate) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *APIServerKubeletClientCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *APIServerKubeletClientCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *APIServerKubeletClientCertificate) GetClient() client.Client { + return r.Client +} + +func (r *APIServerKubeletClientCertificate) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *APIServerKubeletClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *APIServerKubeletClientCertificate) GetName() string { + return r.Name +} + +func (r *APIServerKubeletClientCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Certificates.APIServerKubeletClient.LastUpdate = metav1.Now() + tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName = r.resource.GetName() + + return nil +} + +func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestConfigRV := getLatestConfigRV(*tenantControlPlane) + actualConfigRV := r.resource.GetLabels()["latest-config-rv"] + if latestConfigRV == actualConfigRV { + isValid, err := kubeadm.IsCertificatePrivateKeyPairValid( + r.resource.Data[kubeadmconstants.APIServerKubeletClientCertName], + r.resource.Data[kubeadmconstants.APIServerKubeletClientKeyName], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, err.Error())) + } + if isValid { + return nil + } + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName} + secretCA := &corev1.Secret{} + if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil { + return err + } + + ca := kubeadm.CertificatePrivateKeyPair{ + Name: kubeadmconstants.CACertAndKeyBaseName, + Certificate: secretCA.Data[kubeadmconstants.CACertName], + PrivateKey: secretCA.Data[kubeadmconstants.CAKeyName], + } + certificateKeyPair, err := kubeadm.GenerateCertificatePrivateKeyPair(kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, config, ca) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.APIServerKubeletClientCertName: certificateKeyPair.Certificate, + kubeadmconstants.APIServerKubeletClientKeyName: certificateKeyPair.PrivateKey, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-config-rv": latestConfigRV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} diff --git a/internal/resources/ca_certificate.go b/internal/resources/ca_certificate.go new file mode 100644 index 0000000..0b8bf53 --- /dev/null +++ b/internal/resources/ca_certificate.go @@ -0,0 +1,119 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type CACertificate struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + TmpDirectory string +} + +func (r *CACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() +} + +func (r *CACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *CACertificate) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *CACertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *CACertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *CACertificate) GetClient() client.Client { + return r.Client +} + +func (r *CACertificate) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *CACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *CACertificate) GetName() string { + return r.Name +} + +func (r *CACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Certificates.CA.LastUpdate = metav1.Now() + tenantControlPlane.Status.Certificates.CA.SecretName = r.resource.GetName() + + return nil +} + +func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + isValid, err := kubeadm.IsCertificatePrivateKeyPairValid( + r.resource.Data[kubeadmconstants.CACertName], + r.resource.Data[kubeadmconstants.CAKeyName], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.CACertAndKeyBaseName, err.Error())) + } + if isValid { + return nil + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + ca, err := kubeadm.GenerateCACertificatePrivateKeyPair(kubeadmconstants.CACertAndKeyBaseName, config) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.CACertName: ca.Certificate, + kubeadmconstants.CAKeyName: ca.PrivateKey, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} diff --git a/internal/resources/constants.go b/internal/resources/constants.go new file mode 100644 index 0000000..7b6c6f5 --- /dev/null +++ b/internal/resources/constants.go @@ -0,0 +1,8 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +const ( + defaultIngressPort = 443 +) diff --git a/internal/resources/etcd_ca_certificates.go b/internal/resources/etcd_ca_certificates.go new file mode 100644 index 0000000..f206db4 --- /dev/null +++ b/internal/resources/etcd_ca_certificates.go @@ -0,0 +1,119 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + "reflect" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/etcd" + "github.com/clastix/kamaji/internal/utilities" +) + +type ETCDCACertificatesResource struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + ETCDCASecretName string + ETCDCASecretNamespace string +} + +func (r *ETCDCACertificatesResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + if tenantControlPlane.Status.Certificates.ETCD == nil { + return true + } + + return tenantControlPlane.Status.Certificates.ETCD.CA.SecretName != r.resource.GetName() +} + +func (r *ETCDCACertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *ETCDCACertificatesResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *ETCDCACertificatesResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *ETCDCACertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *ETCDCACertificatesResource) GetName() string { + return r.Name +} + +func (r *ETCDCACertificatesResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Status.Certificates.ETCD == nil { + tenantControlPlane.Status.Certificates.ETCD = &kamajiv1alpha1.ETCDCertificatesStatus{} + } + + tenantControlPlane.Status.Certificates.ETCD.CA.SecretName = r.resource.GetName() + tenantControlPlane.Status.Certificates.ETCD.CA.LastUpdate = metav1.Now() + + return nil +} + +func (r *ETCDCACertificatesResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *ETCDCACertificatesResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + r.resource.SetLabels(utilities.KamajiLabels()) + + etcdCASecretNamespacedName := k8stypes.NamespacedName{Namespace: r.ETCDCASecretNamespace, Name: r.ETCDCASecretName} + etcdCASecret := &corev1.Secret{} + if err := r.Client.Get(ctx, etcdCASecretNamespacedName, etcdCASecret); err != nil { + return err + } + + isValid, err := etcd.IsETCDCertificateAndKeyPairValid(r.resource.Data[kubeadmconstants.CACertName], r.resource.Data[kubeadmconstants.CAKeyName]) + if err != nil { + r.Log.Info(fmt.Sprintf("etcd certificates are not valid: %s", err.Error())) + } + + if reflect.DeepEqual(etcdCASecret.Data[kubeadmconstants.CACertName], r.resource.Data[kubeadmconstants.CACertName]) && + reflect.DeepEqual(etcdCASecret.Data[kubeadmconstants.CAKeyName], r.resource.Data[kubeadmconstants.CAKeyName]) { + if isValid { + return nil + } + + return fmt.Errorf("CA certificates provided into secrets %s/%s are not valid", r.ETCDCASecretNamespace, r.ETCDCASecretName) + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.CACertName: etcdCASecret.Data[kubeadmconstants.CACertName], + kubeadmconstants.CAKeyName: etcdCASecret.Data[kubeadmconstants.CAKeyName], + } + + if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { + return err + } + + return nil + } +} diff --git a/internal/resources/etcd_certificates.go b/internal/resources/etcd_certificates.go new file mode 100644 index 0000000..ac20ff8 --- /dev/null +++ b/internal/resources/etcd_certificates.go @@ -0,0 +1,124 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/etcd" + "github.com/clastix/kamaji/internal/utilities" +) + +type ETCDCertificatesResource struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string +} + +func (r *ETCDCertificatesResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + if tenantControlPlane.Status.Certificates.ETCD == nil { + return true + } + + return tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName != r.resource.GetName() +} + +func (r *ETCDCertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *ETCDCertificatesResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *ETCDCertificatesResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *ETCDCertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *ETCDCertificatesResource) GetName() string { + return r.Name +} + +func (r *ETCDCertificatesResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Status.Certificates.ETCD == nil { + tenantControlPlane.Status.Certificates.ETCD = &kamajiv1alpha1.ETCDCertificatesStatus{} + } + + tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName = r.resource.GetName() + tenantControlPlane.Status.Certificates.ETCD.APIServer.LastUpdate = metav1.Now() + + return nil +} + +func (r *ETCDCertificatesResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *ETCDCertificatesResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + r.resource.SetLabels(utilities.KamajiLabels()) + + if tenantControlPlane.Status.Certificates.ETCD == nil { + return fmt.Errorf("etcd is still synchronizing latest changes") + } + + etcdCASecretNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName} + etcdCASecret := &corev1.Secret{} + if err := r.Client.Get(ctx, etcdCASecretNamespacedName, etcdCASecret); err != nil { + return err + } + + isValid, err := etcd.IsETCDCertificateAndKeyPairValid(r.resource.Data[kubeadmconstants.APIServerEtcdClientCertName], r.resource.Data[kubeadmconstants.APIServerEtcdClientKeyName]) + if err != nil { + r.Log.Info(fmt.Sprintf("etcd certificates are not valid: %s", err.Error())) + } + + if isValid { + return nil + } + + cert, privKey, err := etcd.GetETCDCACertificateAndKeyPair( + tenantControlPlane.GetName(), + etcdCASecret.Data[kubeadmconstants.CACertName], + etcdCASecret.Data[kubeadmconstants.CAKeyName], + ) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.APIServerEtcdClientCertName: cert.Bytes(), + kubeadmconstants.APIServerEtcdClientKeyName: privKey.Bytes(), + } + + if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { + return err + } + + return nil + } +} diff --git a/internal/resources/etcd_setup.go b/internal/resources/etcd_setup.go new file mode 100644 index 0000000..358851b --- /dev/null +++ b/internal/resources/etcd_setup.go @@ -0,0 +1,279 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + + "github.com/go-logr/logr" + etcdclient "go.etcd.io/etcd/client/v3" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/etcd" +) + +const ( + caKeyName = kubeadmconstants.CACertName +) + +type resource struct { + role etcd.Role + user etcd.User +} + +type ETCDSetupResource struct { + resource *resource + Client client.Client + Scheme *runtime.Scheme + Log logr.Logger + Name string + Endpoints []string + ETCDClientCertsSecret k8stypes.NamespacedName + ETCDCACertsSecret k8stypes.NamespacedName +} + +func (r *ETCDSetupResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + if tenantControlPlane.Status.Storage.ETCD == nil { + return true + } + + return tenantControlPlane.Status.Storage.ETCD.Role.Name != r.resource.role.Name || + tenantControlPlane.Status.Storage.ETCD.User.Name != r.resource.user.Name +} + +func (r *ETCDSetupResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *ETCDSetupResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *ETCDSetupResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &resource{ + role: etcd.Role{Name: tenantControlPlane.Name, Exists: false}, + user: etcd.User{Name: tenantControlPlane.Name, Exists: false}, + } + + return nil +} + +func (r *ETCDSetupResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return r.reconcile(ctx) +} + +func (r *ETCDSetupResource) Delete(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if err := r.Define(ctx, tenantControlPlane); err != nil { + return err + } + + client, err := r.getETCDClient(ctx) + if err != nil { + return err + } + + if err = r.deleteData(ctx, client, tenantControlPlane); err != nil { + return err + } + + if err = r.deleteUser(ctx, client, tenantControlPlane); err != nil { + return err + } + + if err = r.deleteRole(ctx, client, tenantControlPlane); err != nil { + return err + } + + return nil +} + +func (r *ETCDSetupResource) GetName() string { + return r.Name +} + +func (r *ETCDSetupResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Status.Storage.ETCD == nil { + tenantControlPlane.Status.Storage.ETCD = &kamajiv1alpha1.ETCDStatus{} + } + tenantControlPlane.Status.Storage.ETCD.Role = r.resource.role + tenantControlPlane.Status.Storage.ETCD.User = r.resource.user + + return nil +} + +func (r *ETCDSetupResource) reconcile(ctx context.Context) (controllerutil.OperationResult, error) { + reconcilationResult := controllerutil.OperationResultNone + var operationResult controllerutil.OperationResult + + client, err := r.getETCDClient(ctx) + if err != nil { + return reconcilationResult, err + } + + operationResult, err = r.reconcileUser(ctx, client) + if err != nil { + return reconcilationResult, err + } + reconcilationResult = updateOperationResult(reconcilationResult, operationResult) + + operationResult, err = r.reconcileRole(ctx, client) + if err != nil { + return controllerutil.OperationResultNone, err + } + reconcilationResult = updateOperationResult(reconcilationResult, operationResult) + + operationResult, err = r.grantUserRole(ctx, client) + if err != nil { + return controllerutil.OperationResultNone, err + } + reconcilationResult = updateOperationResult(reconcilationResult, operationResult) + + operationResult, err = r.grantRolePermissions(ctx, client) + if err != nil { + return controllerutil.OperationResultNone, err + } + reconcilationResult = updateOperationResult(reconcilationResult, operationResult) + + return reconcilationResult, nil +} + +func (r *ETCDSetupResource) getETCDClient(ctx context.Context) (*etcdclient.Client, error) { + var certsClientSecret corev1.Secret + if err := r.Client.Get(ctx, r.ETCDClientCertsSecret, &certsClientSecret); err != nil { + return nil, err + } + + var certsCASecret corev1.Secret + if err := r.Client.Get(ctx, r.ETCDCACertsSecret, &certsCASecret); err != nil { + return nil, err + } + + config := etcd.Config{ + ETCDCertificate: certsClientSecret.Data[corev1.TLSCertKey], + ETCDPrivateKey: certsClientSecret.Data[corev1.TLSPrivateKeyKey], + ETCDCA: certsCASecret.Data[caKeyName], + Endpoints: r.Endpoints, + } + + return etcd.NewClient(config) +} + +func (r *ETCDSetupResource) reconcileUser(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) { + if err := etcd.GetUser(ctx, client, &r.resource.user); err != nil { + return controllerutil.OperationResultNone, err + } + + if !r.resource.user.Exists { + if err := etcd.AddUser(ctx, client, r.resource.user.Name); err != nil { + return controllerutil.OperationResultNone, err + } + + return controllerutil.OperationResultCreated, nil + } + + return controllerutil.OperationResultNone, nil +} + +func (r *ETCDSetupResource) reconcileRole(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) { + if err := etcd.GetRole(ctx, client, &r.resource.role); err != nil { + return controllerutil.OperationResultNone, err + } + + if !r.resource.role.Exists { + if err := etcd.AddRole(ctx, client, r.resource.role.Name); err != nil { + return controllerutil.OperationResultNone, err + } + + return controllerutil.OperationResultCreated, nil + } + + return controllerutil.OperationResultNone, nil +} + +func (r *ETCDSetupResource) grantUserRole(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) { + if err := etcd.GetUser(ctx, client, &r.resource.user); err != nil { + return controllerutil.OperationResultNone, err + } + + if len(r.resource.user.Roles) > 0 && isRole(r.resource.user.Roles, r.resource.role.Name) { + return controllerutil.OperationResultNone, nil + } + + if err := etcd.GrantUserRole(ctx, client, r.resource.user, r.resource.role); err != nil { + return controllerutil.OperationResultNone, err + } + + return controllerutil.OperationResultUpdated, nil +} + +func (r *ETCDSetupResource) grantRolePermissions(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) { + if err := etcd.GetRole(ctx, client, &r.resource.role); err != nil { + return controllerutil.OperationResultNone, err + } + + if len(r.resource.role.Permissions) > 0 && isPermission(r.resource.role.Permissions, r.resource.role.Name) { + return controllerutil.OperationResultNone, nil + } + + if err := etcd.GrantRolePermission(ctx, client, r.resource.role); err != nil { + return controllerutil.OperationResultNone, err + } + + return controllerutil.OperationResultUpdated, nil +} + +func (r *ETCDSetupResource) deleteData(ctx context.Context, client *etcdclient.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + return etcd.CleanUpPrefix(ctx, client, tenantControlPlane.GetName()) +} + +func (r *ETCDSetupResource) deleteUser(ctx context.Context, client *etcdclient.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if err := etcd.GetUser(ctx, client, &r.resource.user); err != nil { + return err + } + + if !r.resource.user.Exists { + return nil + } + + return etcd.RemoveUser(ctx, client, tenantControlPlane.GetName()) +} + +func (r *ETCDSetupResource) deleteRole(ctx context.Context, client *etcdclient.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if err := etcd.GetRole(ctx, client, &r.resource.role); err != nil { + return err + } + + if !r.resource.role.Exists { + return nil + } + + return etcd.RemoveRole(ctx, client, tenantControlPlane.GetName()) +} + +func isRole(s []string, x string) bool { + for _, o := range s { + if o == x { + return true + } + } + + return false +} + +func isPermission(s []etcd.Permission, role string) bool { + key := etcd.BuildKey(role) + for _, o := range s { + if o.Key == key { + return true + } + } + + return false +} diff --git a/internal/resources/front-proxy-client-certificate.go b/internal/resources/front-proxy-client-certificate.go new file mode 100644 index 0000000..0c5b462 --- /dev/null +++ b/internal/resources/front-proxy-client-certificate.go @@ -0,0 +1,136 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type FrontProxyClientCertificate struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + TmpDirectory string +} + +func (r *FrontProxyClientCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName != r.resource.GetName() +} + +func (r *FrontProxyClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *FrontProxyClientCertificate) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *FrontProxyClientCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *FrontProxyClientCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *FrontProxyClientCertificate) GetClient() client.Client { + return r.Client +} + +func (r *FrontProxyClientCertificate) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *FrontProxyClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *FrontProxyClientCertificate) GetName() string { + return r.Name +} + +func (r *FrontProxyClientCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Certificates.FrontProxyClient.LastUpdate = metav1.Now() + tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName = r.resource.GetName() + + return nil +} + +func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestConfigRV := getLatestConfigRV(*tenantControlPlane) + actualConfigRV := r.resource.GetLabels()["latest-config-rv"] + if latestConfigRV == actualConfigRV { + isValid, err := kubeadm.IsCertificatePrivateKeyPairValid( + r.resource.Data[kubeadmconstants.FrontProxyClientCertName], + r.resource.Data[kubeadmconstants.FrontProxyClientKeyName], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.FrontProxyClientCertAndKeyBaseName, err.Error())) + } + if isValid { + return nil + } + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName} + secretCA := &corev1.Secret{} + if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil { + return err + } + + ca := kubeadm.CertificatePrivateKeyPair{ + Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, + Certificate: secretCA.Data[kubeadmconstants.FrontProxyCACertName], + PrivateKey: secretCA.Data[kubeadmconstants.FrontProxyCAKeyName], + } + certificateKeyPair, err := kubeadm.GenerateCertificatePrivateKeyPair(kubeadmconstants.FrontProxyClientCertAndKeyBaseName, config, ca) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.FrontProxyClientCertName: certificateKeyPair.Certificate, + kubeadmconstants.FrontProxyClientKeyName: certificateKeyPair.PrivateKey, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-config-rv": latestConfigRV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} diff --git a/internal/resources/front_proxy_ca_certificate.go b/internal/resources/front_proxy_ca_certificate.go new file mode 100644 index 0000000..2fc0d7f --- /dev/null +++ b/internal/resources/front_proxy_ca_certificate.go @@ -0,0 +1,121 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type FrontProxyCACertificate struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + TmpDirectory string +} + +func (r *FrontProxyCACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName != r.resource.GetName() +} + +func (r *FrontProxyCACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *FrontProxyCACertificate) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *FrontProxyCACertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + r.Name = "front-proxy-ca-certificate" + + return nil +} + +func (r *FrontProxyCACertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *FrontProxyCACertificate) GetClient() client.Client { + return r.Client +} + +func (r *FrontProxyCACertificate) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *FrontProxyCACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *FrontProxyCACertificate) GetName() string { + return r.Name +} + +func (r *FrontProxyCACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Certificates.FrontProxyCA.LastUpdate = metav1.Now() + tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName = r.resource.GetName() + + return nil +} + +func (r *FrontProxyCACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + isValid, err := kubeadm.IsCertificatePrivateKeyPairValid( + r.resource.Data[kubeadmconstants.FrontProxyCACertName], + r.resource.Data[kubeadmconstants.FrontProxyCAKeyName], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.FrontProxyCACertAndKeyBaseName, err.Error())) + } + if isValid { + return nil + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + ca, err := kubeadm.GenerateCACertificatePrivateKeyPair(kubeadmconstants.FrontProxyCACertAndKeyBaseName, config) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.FrontProxyCACertName: ca.Certificate, + kubeadmconstants.FrontProxyCAKeyName: ca.PrivateKey, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go new file mode 100644 index 0000000..8dcafe6 --- /dev/null +++ b/internal/resources/k8s_deployment_resource.go @@ -0,0 +1,605 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "crypto/md5" + "fmt" + "path" + "sort" + "strings" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + quantity "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type KubernetesDeploymentResource struct { + resource *appsv1.Deployment + Client client.Client + ETCDEndpoints []string + ETCDCompactionInterval string + Name string +} + +func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String() +} + +func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return !r.isStatusEqual(tenantControlPlane) || tenantControlPlane.Spec.Kubernetes.Version != tenantControlPlane.Status.Kubernetes.Version.Version +} + +func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantControlPlane.GetName(), + Namespace: tenantControlPlane.GetNamespace(), + Labels: utilities.CommonLabels(tenantControlPlane.GetName()), + }, + } + + r.Name = "deployment" + + return nil +} + +// secretHashValue function returns the md5 value for the given secret: +// this will trigger a new rollout in case of value change. +func (r *KubernetesDeploymentResource) secretHashValue(ctx context.Context, namespace, name string) (string, error) { + secret := &corev1.Secret{} + + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + return "", errors.Wrap(err, "cannot retrieve *corev1.Secret for resource version retrieval") + } + // Go access map values in random way, it means we have to sort them + keys := make([]string, 0, len(secret.Data)) + + for k := range secret.Data { + keys = append(keys, k) + } + + sort.Strings(keys) + // Generating MD5 of Secret values, sorted by key + h := md5.New() + + for _, key := range keys { + h.Write(secret.Data[key]) + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + maxSurge := intstr.FromString("100%") + + maxUnavailable := intstr.FromInt(0) + + etcdEndpoints := make([]string, len(r.ETCDEndpoints)) + for i, v := range r.ETCDEndpoints { + etcdEndpoints[i] = fmt.Sprintf("https://%s", v) + } + + address, err := tenantControlPlane.GetAddress(ctx, r.Client) + if err != nil { + return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create TenantControlPlane Deployment") + } + + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error { + labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels) + r.resource.SetLabels(labels) + + annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations) + r.resource.SetAnnotations(annotations) + + r.resource.Spec.Replicas = pointer.Int32(tenantControlPlane.Spec.ControlPlane.Deployment.Replicas) + r.resource.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kamaji.clastix.io/soot": tenantControlPlane.GetName(), + }, + } + r.resource.Spec.Template.ObjectMeta = metav1.ObjectMeta{ + Labels: map[string]string{ + "kamaji.clastix.io/soot": tenantControlPlane.GetName(), + "component.kamaji.clastix.io/api-server-certificate": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName) + + return + }(), + "component.kamaji.clastix.io/api-server-kubelet-client-certificate": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName) + + return + }(), + "component.kamaji.clastix.io/ca": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName) + + return + }(), + "component.kamaji.clastix.io/controller-manager-kubeconfig": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName) + + return + }(), + "component.kamaji.clastix.io/etcd-ca-certificates": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName) + + return + }(), + "component.kamaji.clastix.io/etcd-certificates": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName) + + return + }(), + "component.kamaji.clastix.io/front-proxy-ca-certificate": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName) + + return + }(), + "component.kamaji.clastix.io/front-proxy-client-certificate": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName) + + return + }(), + "component.kamaji.clastix.io/service-account": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName) + + return + }(), + "component.kamaji.clastix.io/scheduler-kubeconfig": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName) + + return + }(), + }, + } + r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{ + { + Name: "etc-kubernetes-pki", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName), + }, + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName), + }, + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName), + }, + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName), + }, + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName), + }, + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName), + }, + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName), + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName, + }, + Items: []corev1.KeyToPath{ + { + Key: constants.CACertName, + Path: constants.EtcdCACertName, + }, + }, + }, + }, + }, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "etc-ca-certificates", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "etc-ssl-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "usr-share-ca-certificates", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "usr-local-share-ca-certificates", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "scheduler-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.KubeConfig.Scheduler.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + { + Name: "controller-manager-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName, + DefaultMode: pointer.Int32Ptr(420), + }, + }, + }, + } + + r.resource.Spec.Template.Spec.Containers = []corev1.Container{ + { + Name: "kube-apiserver", + Image: fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version), + Command: []string{ + "kube-apiserver", + "--allow-privileged=true", + "--authorization-mode=Node,RBAC", + fmt.Sprintf("--advertise-address=%s", address), + fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)), + fmt.Sprintf("--enable-admission-plugins=%s", strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ",")), + "--enable-bootstrap-token-auth=true", + fmt.Sprintf("--etcd-compaction-interval=%s", r.ETCDCompactionInterval), + fmt.Sprintf("--etcd-cafile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)), + fmt.Sprintf("--etcd-certfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)), + fmt.Sprintf("--etcd-keyfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)), + fmt.Sprintf("--etcd-servers=%s", strings.Join(etcdEndpoints, ",")), + fmt.Sprintf("--etcd-prefix=/%s", tenantControlPlane.GetName()), + fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR), + fmt.Sprintf("--kubelet-client-certificate=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName)), + fmt.Sprintf("--kubelet-client-key=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName)), + "--kubelet-preferred-address-types=Hostname,InternalIP,ExternalIP", + fmt.Sprintf("--proxy-client-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName)), + fmt.Sprintf("--proxy-client-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName)), + "--requestheader-allowed-names=front-proxy-client", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-username-headers=X-Remote-User", + fmt.Sprintf("--secure-port=%d", tenantControlPlane.Spec.NetworkProfile.Port), + fmt.Sprintf("--service-account-issuer=https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port), + fmt.Sprintf("--service-account-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName)), + fmt.Sprintf("--service-account-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)), + fmt.Sprintf("--tls-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName)), + fmt.Sprintf("--tls-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName)), + }, + Resources: corev1.ResourceRequirements{ + Limits: nil, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: quantity.MustParse("250m"), + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/livez", + Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/livez", + Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: corev1.PullAlways, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "etc-kubernetes-pki", + ReadOnly: true, + MountPath: v1beta3.DefaultCertificatesDir, + }, + { + Name: "etc-ca-certificates", + ReadOnly: true, + MountPath: "/etc/ca-certificates", + }, + { + Name: "etc-ssl-certs", + ReadOnly: true, + MountPath: "/etc/ssl/certs", + }, + { + Name: "usr-share-ca-certificates", + ReadOnly: true, + MountPath: "/usr/share/ca-certificates", + }, + { + Name: "usr-local-share-ca-certificates", + ReadOnly: true, + MountPath: "/usr/local/share/ca-certificates", + }, + }, + }, + { + Name: "kube-scheduler", + Image: fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version), + Command: []string{ + "kube-scheduler", + "--authentication-kubeconfig=/etc/kubernetes/scheduler.conf", + "--authorization-kubeconfig=/etc/kubernetes/scheduler.conf", + "--bind-address=0.0.0.0", + "--kubeconfig=/etc/kubernetes/scheduler.conf", + "--leader-elect=true", + }, + Resources: corev1.ResourceRequirements{ + Limits: nil, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: quantity.MustParse("100m"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "scheduler-kubeconfig", + ReadOnly: true, + MountPath: "/etc/kubernetes", + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(10259), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(10259), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: corev1.PullIfNotPresent, + }, + { + Name: "kube-controller-manager", + Image: fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version), + Command: []string{ + "kube-controller-manager", + "--allocate-node-cidrs=true", + "--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf", + "--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf", + "--bind-address=0.0.0.0", + fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)), + fmt.Sprintf("--cluster-name=%s", tenantControlPlane.GetName()), + fmt.Sprintf("--cluster-signing-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)), + fmt.Sprintf("--cluster-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)), + "--controllers=*,bootstrapsigner,tokencleaner", + "--kubeconfig=/etc/kubernetes/controller-manager.conf", + "--leader-elect=true", + fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR), + fmt.Sprintf("--cluster-cidr=%s", tenantControlPlane.Spec.NetworkProfile.PodCIDR), + fmt.Sprintf("--requestheader-client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)), + fmt.Sprintf("--root-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)), + fmt.Sprintf("--service-account-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)), + "--use-service-account-credentials=true", + }, + Resources: corev1.ResourceRequirements{ + Limits: nil, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: quantity.MustParse("200m"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "controller-manager-kubeconfig", + ReadOnly: true, + MountPath: "/etc/kubernetes", + }, + { + Name: "etc-kubernetes-pki", + ReadOnly: true, + MountPath: v1beta3.DefaultCertificatesDir, + }, + { + Name: "etc-ca-certificates", + ReadOnly: true, + MountPath: "/etc/ca-certificates", + }, + { + Name: "etc-ssl-certs", + ReadOnly: true, + MountPath: "/etc/ssl/certs", + }, + { + Name: "usr-share-ca-certificates", + ReadOnly: true, + MountPath: "/usr/share/ca-certificates", + }, + { + Name: "usr-local-share-ca-certificates", + ReadOnly: true, + MountPath: "/usr/local/share/ca-certificates", + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(10257), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(10257), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 0, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + ImagePullPolicy: corev1.PullIfNotPresent, + }, + } + r.resource.Spec.Strategy = appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &maxUnavailable, + MaxSurge: &maxSurge, + }, + } + + return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + }) +} + +func (r *KubernetesDeploymentResource) GetName() string { + return r.Name +} + +func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + switch { + case !r.isProgressingUpgrade(): + tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionReady + tenantControlPlane.Status.Kubernetes.Version.Version = tenantControlPlane.Spec.Kubernetes.Version + case r.isUpgrading(tenantControlPlane): + tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionUpgrading + case r.isProvisioning(tenantControlPlane): + tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionProvisioning + case r.isNotReady(): + tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionNotReady + } + + tenantControlPlane.Status.Kubernetes.Deployment = kamajiv1alpha1.KubernetesDeploymentStatus{ + DeploymentStatus: r.resource.Status, + Name: r.resource.GetName(), + Namespace: r.resource.GetNamespace(), + } + + return nil +} + +func (r *KubernetesDeploymentResource) isProgressingUpgrade() bool { + if r.resource.ObjectMeta.GetGeneration() != r.resource.Status.ObservedGeneration { + return true + } + + if r.resource.Status.UnavailableReplicas > 0 { + return true + } + + return false +} + +func (r *KubernetesDeploymentResource) isUpgrading(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return len(tenantControlPlane.Status.Kubernetes.Version.Version) > 0 && + tenantControlPlane.Spec.Kubernetes.Version != tenantControlPlane.Status.Kubernetes.Version.Version && + r.isProgressingUpgrade() +} + +func (r *KubernetesDeploymentResource) isProvisioning(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return len(tenantControlPlane.Status.Kubernetes.Version.Version) == 0 +} + +func (r *KubernetesDeploymentResource) isNotReady() bool { + return r.resource.Status.ReadyReplicas == 0 +} diff --git a/internal/resources/k8s_ingress_resource.go b/internal/resources/k8s_ingress_resource.go new file mode 100644 index 0000000..ca7c61d --- /dev/null +++ b/internal/resources/k8s_ingress_resource.go @@ -0,0 +1,137 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + networkingv1 "k8s.io/api/networking/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type KubernetesIngressResource struct { + resource *networkingv1.Ingress + Client client.Client + Name string +} + +func (r *KubernetesIngressResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return !(tenantControlPlane.Status.Kubernetes.Ingress.Name == r.resource.GetName() && + tenantControlPlane.Status.Kubernetes.Ingress.Namespace == r.resource.GetNamespace()) +} + +func (r *KubernetesIngressResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return !tenantControlPlane.Spec.ControlPlane.Ingress.Enabled +} + +func (r *KubernetesIngressResource) CleanUp(ctx context.Context) (bool, error) { + if err := r.Client.Delete(ctx, r.resource); err != nil { + if !k8serrors.IsNotFound(err) { + return false, err + } + + return false, nil + } + + return true, nil +} + +func (r *KubernetesIngressResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if tenantControlPlane.Spec.ControlPlane.Ingress.Enabled { + tenantControlPlane.Status.Kubernetes.Ingress.IngressStatus = r.resource.Status + tenantControlPlane.Status.Kubernetes.Ingress.Name = r.resource.GetName() + tenantControlPlane.Status.Kubernetes.Ingress.Namespace = r.resource.GetNamespace() + + return nil + } + + tenantControlPlane.Status.Kubernetes.Ingress = kamajiv1alpha1.KubernetesIngressStatus{} + + return nil +} + +func (r *KubernetesIngressResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantControlPlane.GetName(), + Namespace: tenantControlPlane.GetNamespace(), + Labels: utilities.CommonLabels(tenantControlPlane.GetName()), + }, + } + + r.Name = "ingress" + + return nil +} + +func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error { + labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Ingress.AdditionalMetadata.Labels) + r.resource.SetLabels(labels) + + annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Ingress.AdditionalMetadata.Annotations) + r.resource.SetAnnotations(annotations) + + if tenantControlPlane.Spec.ControlPlane.Ingress.IngressClassName != "" { + r.resource.Spec.IngressClassName = &tenantControlPlane.Spec.ControlPlane.Ingress.IngressClassName + } + + var rule networkingv1.IngressRule + if len(r.resource.Spec.Rules) > 0 { + rule = r.resource.Spec.Rules[0] + } + + var path networkingv1.HTTPIngressPath + if rule.HTTP != nil && len(rule.HTTP.Paths) > 0 { + path = rule.HTTP.Paths[0] + } + + path.Path = "/" + path.PathType = (*networkingv1.PathType)(pointer.StringPtr(string(networkingv1.PathTypePrefix))) + + if path.Backend.Service == nil { + path.Backend.Service = &networkingv1.IngressServiceBackend{} + } + + if tenantControlPlane.Status.Kubernetes.Service.Name == "" || + tenantControlPlane.Status.Kubernetes.Service.Port == 0 { + return fmt.Errorf("ingress cannot be configured yet") + } + + path.Backend.Service.Name = tenantControlPlane.Status.Kubernetes.Service.Name + path.Backend.Service.Port.Number = tenantControlPlane.Status.Kubernetes.Service.Port + + if rule.HTTP == nil { + rule.HTTP = &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + {}, + }, + } + } + + rule.HTTP.Paths[0] = path + rule.Host = tenantControlPlane.Spec.ControlPlane.Ingress.Hostname + if rule.Host == "" { + rule.Host = fmt.Sprintf("%s.%s.%s", tenantControlPlane.GetName(), tenantControlPlane.GetNamespace(), tenantControlPlane.Spec.NetworkProfile.Domain) + } + + r.resource.Spec.Rules = []networkingv1.IngressRule{ + rule, + } + + return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + }) +} + +func (r *KubernetesIngressResource) GetName() string { + return r.Name +} diff --git a/internal/resources/k8s_service_resource.go b/internal/resources/k8s_service_resource.go new file mode 100644 index 0000000..7c40dc4 --- /dev/null +++ b/internal/resources/k8s_service_resource.go @@ -0,0 +1,117 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +// KubernetesServiceResource must be the first Resource processed by the TenantControlPlane: +// when a TenantControlPlan is expecting a dynamic IP address, the Service will get it from the controller-manager. +type KubernetesServiceResource struct { + resource *corev1.Service + Client client.Client + Name string +} + +func (r *KubernetesServiceResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Kubernetes.Service.Name != r.resource.GetName() || + tenantControlPlane.Status.Kubernetes.Service.Namespace != r.resource.GetNamespace() || + tenantControlPlane.Status.Kubernetes.Service.Port != r.resource.Spec.Ports[0].Port +} + +func (r *KubernetesServiceResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *KubernetesServiceResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *KubernetesServiceResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Kubernetes.Service.ServiceStatus = r.resource.Status + tenantControlPlane.Status.Kubernetes.Service.Name = r.resource.GetName() + tenantControlPlane.Status.Kubernetes.Service.Namespace = r.resource.GetNamespace() + tenantControlPlane.Status.Kubernetes.Service.Port = r.resource.Spec.Ports[0].Port + + return nil +} + +func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantControlPlane.GetName(), + Namespace: tenantControlPlane.GetNamespace(), + Labels: utilities.CommonLabels(tenantControlPlane.GetName()), + }, + } + + r.Name = "service" + + return nil +} + +func (r *KubernetesServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + // We don't need to check error here: in case of dynamic external IP, the Service must be created in advance. + // After that, the specific cloud controller-manager will provide an IP that will be then used. + address, _ := tenantControlPlane.GetAddress(ctx, r.Client) + + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error { + var servicePort corev1.ServicePort + if len(r.resource.Spec.Ports) > 0 { + servicePort = r.resource.Spec.Ports[0] + } + servicePort.Protocol = corev1.ProtocolTCP + servicePort.Port = tenantControlPlane.Spec.NetworkProfile.Port + servicePort.TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)) + + r.resource.Spec.Ports = []corev1.ServicePort{servicePort} + r.resource.Spec.Selector = map[string]string{ + "kamaji.clastix.io/soot": tenantControlPlane.GetName(), + } + + labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Labels) + r.resource.SetLabels(labels) + + annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Annotations) + r.resource.SetAnnotations(annotations) + + switch tenantControlPlane.Spec.ControlPlane.Service.ServiceType { + case kamajiv1alpha1.ServiceTypeLoadBalancer: + r.resource.Spec.Type = corev1.ServiceTypeLoadBalancer + + if len(address) > 0 { + r.resource.Spec.LoadBalancerIP = address + } + case kamajiv1alpha1.ServiceTypeNodePort: + r.resource.Spec.Type = corev1.ServiceTypeNodePort + r.resource.Spec.Ports[0].NodePort = tenantControlPlane.Spec.NetworkProfile.Port + + if tenantControlPlane.Spec.NetworkProfile.AllowAddressAsExternalIP && len(address) > 0 { + r.resource.Spec.ExternalIPs = []string{address} + } + default: + r.resource.Spec.Type = corev1.ServiceTypeClusterIP + + if tenantControlPlane.Spec.NetworkProfile.AllowAddressAsExternalIP && len(address) > 0 { + r.resource.Spec.ExternalIPs = []string{address} + } + } + + return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + }) +} + +func (r *KubernetesServiceResource) GetName() string { + return r.Name +} diff --git a/internal/resources/kubeadm_config.go b/internal/resources/kubeadm_config.go new file mode 100644 index 0000000..6f993eb --- /dev/null +++ b/internal/resources/kubeadm_config.go @@ -0,0 +1,142 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type KubeadmConfigResource struct { + resource *corev1.ConfigMap + Client client.Client + Scheme *runtime.Scheme + Log logr.Logger + Name string + Port int32 + Domain string + PodCIDR string + ServiceCIDR string + KubernetesVersion string + ETCDs []string + ETCDCompactionInterval string + TmpDirectory string +} + +func (r *KubeadmConfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + address, err := tenantControlPlane.GetAddress(ctx, r.Client) + if err != nil { + return true + } + + return !(tenantControlPlane.Status.KubeadmConfig.ResourceVersion == r.resource.ObjectMeta.ResourceVersion && + tenantControlPlane.Status.KubeadmConfig.ConfigmapName == r.resource.GetName() && + tenantControlPlane.Status.ControlPlaneEndpoint == r.getControlPlaneEndpoint(tenantControlPlane, address)) +} + +func (r *KubeadmConfigResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *KubeadmConfigResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *KubeadmConfigResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *KubeadmConfigResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *KubeadmConfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + address, err := tenantControlPlane.GetAddress(ctx, r.Client) + if err != nil { + return controllerutil.OperationResultNone, err + } + + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(tenantControlPlane, address)) +} + +func (r *KubeadmConfigResource) GetName() string { + return r.Name +} + +func (r *KubeadmConfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + address, _ := tenantControlPlane.GetAddress(ctx, r.Client) + + tenantControlPlane.Status.KubeadmConfig.LastUpdate = metav1.Now() + tenantControlPlane.Status.KubeadmConfig.ResourceVersion = r.resource.ObjectMeta.ResourceVersion + tenantControlPlane.Status.KubeadmConfig.ConfigmapName = r.resource.GetName() + tenantControlPlane.Status.ControlPlaneEndpoint = r.getControlPlaneEndpoint(tenantControlPlane, address) + + return nil +} + +func (r *KubeadmConfigResource) getControlPlaneEndpoint(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) string { + if !tenantControlPlane.Spec.ControlPlane.Ingress.Enabled { + return fmt.Sprintf("%s:%d", address, tenantControlPlane.Spec.NetworkProfile.Port) + } + + if tenantControlPlane.Spec.ControlPlane.Ingress.Hostname != "" { + return tenantControlPlane.Spec.ControlPlane.Ingress.Hostname + } + + return getTenantControllerExternalFQDN(*tenantControlPlane) +} + +func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) controllerutil.MutateFn { + return func() error { + r.resource.SetLabels(utilities.KamajiLabels()) + + params := kubeadm.Parameters{ + TenantControlPlaneName: tenantControlPlane.GetName(), + TenantControlPlaneNamespace: tenantControlPlane.GetNamespace(), + TenantControlPlaneEndpoint: r.getControlPlaneEndpoint(tenantControlPlane, address), + TenantControlPlaneAddress: address, + TenantControlPlanePort: r.Port, + TenantControlPlaneDomain: r.Domain, + TenantControlPlanePodCIDR: r.PodCIDR, + TenantControlPlaneServiceCIDR: r.ServiceCIDR, + TenantControlPlaneVersion: r.KubernetesVersion, + ETCDs: r.ETCDs, + ETCDCompactionInterval: r.ETCDCompactionInterval, + CertificatesDir: r.TmpDirectory, + } + + config := kubeadm.CreateKubeadmInitConfiguration(params) + data, err := kubeadm.GetKubeadmInitConfigurationMap(config) + if err != nil { + return err + } + + r.resource.Data = data + + if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { + return err + } + + return nil + } +} diff --git a/internal/resources/kubeadm_phases.go b/internal/resources/kubeadm_phases.go new file mode 100644 index 0000000..f50a4be --- /dev/null +++ b/internal/resources/kubeadm_phases.go @@ -0,0 +1,271 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig" +) + +const kubeadmPhaseTimeout = 10 // seconds + +type KubeadmPhase int + +const ( + PhaseUploadConfigKubeadm KubeadmPhase = iota + PhaseUploadConfigKubelet + PhaseAddonCoreDNS + PhaseAddonKubeProxy + PhaseBootstrapToken +) + +func (d KubeadmPhase) String() string { + return [...]string{"PhaseUploadConfigKubeadm", "PhaseUploadConfigKubelet", "PhaseAddonCoreDNS", "PhaseAddonKubeProxy", "PhaseBootstrapToken"}[d] +} + +const ( + kubeconfigAdminKeyName = "admin.conf" +) + +type KubeadmPhaseResource struct { + Client client.Client + Log logr.Logger + Name string + KubeadmPhase KubeadmPhase + kubeadmConfigResourceVersion string +} + +func (r *KubeadmPhaseResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + status, err := r.getStatus(tenantControlPlane) + if err != nil { + return true + } + + return status.KubeadmConfigResourceVersion == r.kubeadmConfigResourceVersion +} + +func (r *KubeadmPhaseResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return !r.isStatusEqual(tenantControlPlane) +} + +func (r *KubeadmPhaseResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *KubeadmPhaseResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *KubeadmPhaseResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + return nil +} + +func (r *KubeadmPhaseResource) getKubeadmPhaseFunction() (func(clientset.Interface, *kubeadm.Configuration) error, error) { + switch r.KubeadmPhase { + case PhaseUploadConfigKubeadm: + return kubeadm.UploadKubeadmConfig, nil + case PhaseUploadConfigKubelet: + return kubeadm.UploadKubeletConfig, nil + case PhaseAddonCoreDNS: + return kubeadm.CoreDNSAddon, nil + case PhaseAddonKubeProxy: + return kubeadm.KubeProxyAddon, nil + case PhaseBootstrapToken: + return func(client clientset.Interface, config *kubeadm.Configuration) error { + bootstrapTokensEnrichment(config.InitConfiguration.BootstrapTokens) + + return kubeadm.BootstrapToken(client, config) + }, nil + default: + return nil, fmt.Errorf("no available functionality for phase %s", r.KubeadmPhase) + } +} + +func bootstrapTokensEnrichment(bootstrapTokens []bootstraptokenv1.BootstrapToken) { + var bootstrapToken bootstraptokenv1.BootstrapToken + if len(bootstrapTokens) > 0 { + bootstrapToken = bootstrapTokens[0] + } + + enrichBootstrapToken(&bootstrapToken) + bootstrapTokens[0] = bootstrapToken +} + +func enrichBootstrapToken(bootstrapToken *bootstraptokenv1.BootstrapToken) { + if bootstrapToken.Token == nil { + bootstrapToken.Token = &bootstraptokenv1.BootstrapTokenString{} + } + + if bootstrapToken.Token.ID == "" { + bootstrapToken.Token.ID = fmt.Sprintf("%s.%s", randomString(6), randomString(16)) + } +} + +func (r *KubeadmPhaseResource) GetClient() client.Client { + return r.Client +} + +func (r *KubeadmPhaseResource) GetTmpDirectory() string { + return "" +} + +func (r *KubeadmPhaseResource) GetName() string { + return r.Name +} + +func (r *KubeadmPhaseResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + status, err := r.getStatus(tenantControlPlane) + if err != nil { + return err + } + + status.LastUpdate = metav1.Now() + status.KubeadmConfigResourceVersion = r.kubeadmConfigResourceVersion + + return nil +} + +func (r *KubeadmPhaseResource) getStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.KubeadmPhaseStatus, error) { + switch r.KubeadmPhase { + case PhaseUploadConfigKubeadm: + return &tenantControlPlane.Status.KubeadmPhase.UploadConfigKubeadm, nil + case PhaseUploadConfigKubelet: + return &tenantControlPlane.Status.KubeadmPhase.UploadConfigKubelet, nil + case PhaseAddonCoreDNS: + return &tenantControlPlane.Status.KubeadmPhase.AddonCoreDNS, nil + case PhaseAddonKubeProxy: + return &tenantControlPlane.Status.KubeadmPhase.AddonKubeProxy, nil + case PhaseBootstrapToken: + return &tenantControlPlane.Status.KubeadmPhase.BootstrapToken, nil + default: + return nil, fmt.Errorf("%s is not a right kubeadm phase", r.KubeadmPhase) + } +} + +func (r *KubeadmPhaseResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return r.reconcile(ctx, tenantControlPlane) +} + +func (r *KubeadmPhaseResource) reconcile(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + config, resourceVersion, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + kubeconfig, err := r.getKubeconfig(ctx, tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + config.Kubeconfig = *kubeconfig + config.Parameters = kubeadm.Parameters{ + TenantControlPlaneName: tenantControlPlane.GetName(), + TenantDNSServiceIPs: tenantControlPlane.Spec.NetworkProfile.DNSServiceIPs, + TenantControlPlaneVersion: tenantControlPlane.Spec.Kubernetes.Version, + TenantControlPlanePodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, + TenantControlPlaneAddress: tenantControlPlane.Spec.NetworkProfile.Address, + TenantControlPlanePort: tenantControlPlane.Spec.NetworkProfile.Port, + TenantControlPlaneCGroupDriver: tenantControlPlane.Spec.Kubernetes.Kubelet.CGroupFS.String(), + } + + status, err := r.getStatus(tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + if resourceVersion == status.KubeadmConfigResourceVersion { + r.kubeadmConfigResourceVersion = resourceVersion + + return controllerutil.OperationResultNone, nil + } + + client, err := r.getRESTClient(ctx, tenantControlPlane) + if err != nil { + return controllerutil.OperationResultNone, err + } + + fun, err := r.getKubeadmPhaseFunction() + if err != nil { + return controllerutil.OperationResultNone, err + } + if err = fun(client, config); err != nil { + return controllerutil.OperationResultNone, err + } + + r.kubeadmConfigResourceVersion = resourceVersion + + if status.LastUpdate.IsZero() { + return controllerutil.OperationResultCreated, nil + } + + return controllerutil.OperationResultUpdated, nil +} + +func (r *KubeadmPhaseResource) getKubeconfigSecret(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*corev1.Secret, error) { + kubeconfigSecretName := tenantControlPlane.Status.KubeConfig.Admin.SecretName + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: kubeconfigSecretName} + secret := &corev1.Secret{} + if err := r.Client.Get(ctx, namespacedName, secret); err != nil { + return nil, err + } + + return secret, nil +} + +func (r *KubeadmPhaseResource) getKubeconfig(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kubeconfigutil.Kubeconfig, error) { + secretKubeconfig, err := r.getKubeconfigSecret(ctx, tenantControlPlane) + if err != nil { + return nil, err + } + + bytes, ok := secretKubeconfig.Data[kubeconfigAdminKeyName] + if !ok { + return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeconfigAdminKeyName) + } + + return kubeconfigutil.GetKubeconfigFromBytes(bytes) +} + +func (r *KubeadmPhaseResource) getRESTClient(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*clientset.Clientset, error) { + config, err := r.getRESTClientConfig(ctx, tenantControlPlane) + if err != nil { + return nil, err + } + + return clientset.NewForConfig(config) +} + +func (r *KubeadmPhaseResource) getRESTClientConfig(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) { + kubeconfig, err := r.getKubeconfig(ctx, tenantControlPlane) + if err != nil { + return nil, err + } + + config := &restclient.Config{ + Host: fmt.Sprintf("https://%s:%d", getTenantControllerInternalFQDN(*tenantControlPlane), tenantControlPlane.Spec.NetworkProfile.Port), + TLSClientConfig: restclient.TLSClientConfig{ + CAData: kubeconfig.Clusters[0].Cluster.CertificateAuthorityData, + CertData: kubeconfig.AuthInfos[0].AuthInfo.ClientCertificateData, + KeyData: kubeconfig.AuthInfos[0].AuthInfo.ClientKeyData, + }, + Timeout: time.Second * kubeadmPhaseTimeout, + } + + return config, nil +} diff --git a/internal/resources/kubeadm_upgrade.go b/internal/resources/kubeadm_upgrade.go new file mode 100644 index 0000000..f5b70eb --- /dev/null +++ b/internal/resources/kubeadm_upgrade.go @@ -0,0 +1,190 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/version" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig" + kamajiupgrade "github.com/clastix/kamaji/internal/upgrade" +) + +type KubernetesUpgrade struct { + Name string + Client client.Client + upgrade upgrade.Upgrade + + inProgress bool +} + +func (k *KubernetesUpgrade) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + k.upgrade = upgrade.Upgrade{ + Before: upgrade.ClusterState{ + KubeVersion: tenantControlPlane.Status.Kubernetes.Version.Version, + }, + After: upgrade.ClusterState{ + KubeVersion: tenantControlPlane.Spec.Kubernetes.Version, + }, + } + + return nil +} + +func (k *KubernetesUpgrade) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (k *KubernetesUpgrade) CleanUp(context.Context) (bool, error) { + return false, nil +} + +func (k *KubernetesUpgrade) CreateOrUpdate(ctx context.Context, plane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + // A new installation, no need to upgrade + if len(plane.Status.Kubernetes.Version.Version) == 0 { + k.inProgress = false + + return controllerutil.OperationResultNone, nil + } + // No version change, no need to upgrade + if plane.Status.Kubernetes.Version.Version == plane.Spec.Kubernetes.Version { + k.inProgress = false + + return controllerutil.OperationResultNone, nil + } + // An upgrade is in progress, let it go + if status := plane.Status.Kubernetes.Version.Status; status != nil && *status == kamajiv1alpha1.VersionUpgrading { + return controllerutil.OperationResultNone, nil + } + // Checking if the upgrade is allowed, or not + restClient, err := k.getRESTClient(ctx, plane) + if err != nil { + return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create REST client required for Kubernetes upgrade plan") + } + + versionGetter := kamajiupgrade.NewKamajiKubeVersionGetter(restClient) + + if _, err = upgrade.GetAvailableUpgrades(versionGetter, false, false, true, restClient, ""); err != nil { + return controllerutil.OperationResultNone, errors.Wrap(err, "cannot retrieve available Upgrades for Kubernetes upgrade plan") + } + + if err = k.isUpgradable(); err != nil { + return controllerutil.OperationResultNone, fmt.Errorf("the required upgrade plan is not available") + } + + k.inProgress = true + + return controllerutil.OperationResultNone, nil +} + +func (k *KubernetesUpgrade) GetName() string { + return k.Name +} + +func (k *KubernetesUpgrade) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool { + return k.inProgress +} + +func (k *KubernetesUpgrade) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if k.inProgress { + tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionUpgrading + } + + if tenantControlPlane.Spec.Kubernetes.Version == tenantControlPlane.Status.Kubernetes.Version.Version { + tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionReady + } + + return nil +} + +func (k *KubernetesUpgrade) getKubeconfigSecret(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*corev1.Secret, error) { + kubeconfigSecretName := tenantControlPlane.Status.KubeConfig.Admin.SecretName + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: kubeconfigSecretName} + secret := &corev1.Secret{} + if err := k.Client.Get(ctx, namespacedName, secret); err != nil { + return nil, err + } + + return secret, nil +} + +func (k *KubernetesUpgrade) getKubeconfig(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kubeconfigutil.Kubeconfig, error) { + secretKubeconfig, err := k.getKubeconfigSecret(ctx, tenantControlPlane) + if err != nil { + return nil, err + } + + bytes, ok := secretKubeconfig.Data[kubeconfigAdminKeyName] + if !ok { + return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeconfigAdminKeyName) + } + + return kubeconfigutil.GetKubeconfigFromBytes(bytes) +} + +func (k *KubernetesUpgrade) getRESTClient(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*clientset.Clientset, error) { + config, err := k.getRESTClientConfig(ctx, tenantControlPlane) + if err != nil { + return nil, err + } + + return clientset.NewForConfig(config) +} + +func (k *KubernetesUpgrade) getRESTClientConfig(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) { + kubeconfig, err := k.getKubeconfig(ctx, tenantControlPlane) + if err != nil { + return nil, err + } + + config := &restclient.Config{ + Host: fmt.Sprintf("https://%s:%d", getTenantControllerInternalFQDN(*tenantControlPlane), tenantControlPlane.Spec.NetworkProfile.Port), + TLSClientConfig: restclient.TLSClientConfig{ + CAData: kubeconfig.Clusters[0].Cluster.CertificateAuthorityData, + CertData: kubeconfig.AuthInfos[0].AuthInfo.ClientCertificateData, + KeyData: kubeconfig.AuthInfos[0].AuthInfo.ClientKeyData, + }, + Timeout: time.Second * kubeadmPhaseTimeout, + } + + return config, nil +} + +func (k *KubernetesUpgrade) isUpgradable() error { + newK8sVersion, err := version.ParseSemantic(k.upgrade.After.KubeVersion) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("unable to parse normalized version %q as a semantic version", k.upgrade.After.KubeVersion)) + } + + oldK8sVersion, err := version.ParseSemantic(k.upgrade.Before.KubeVersion) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("unable to parse normalized version %q as a semantic version", k.upgrade.After.KubeVersion)) + } + + if newK8sVersion.Minor() < oldK8sVersion.Minor() { + return fmt.Errorf("cannot downgrade to a previous minor version of Kubernetes") + } + // Patch upgrades are allowed + if newK8sVersion.Minor() == oldK8sVersion.Minor() { + return nil + } + // Following minor release upgrades are allowed + if newK8sVersion.Minor() > oldK8sVersion.WithMinor(oldK8sVersion.Minor()+1).Minor() { + return nil + } + + return fmt.Errorf("an upgrade to a non consecutive Kubernetes minor release is forbidden") +} diff --git a/internal/resources/kubeconfig.go b/internal/resources/kubeconfig.go new file mode 100644 index 0000000..446e45f --- /dev/null +++ b/internal/resources/kubeconfig.go @@ -0,0 +1,186 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +const ( + AdminKubeConfigFileName = kubeadmconstants.AdminKubeConfigFileName + ControllerManagerKubeConfigFileName = kubeadmconstants.ControllerManagerKubeConfigFileName + SchedulerKubeConfigFileName = kubeadmconstants.SchedulerKubeConfigFileName + localhost = "127.0.0.1" +) + +type KubeconfigResource struct { + resource *corev1.Secret + Client client.Client + Scheme *runtime.Scheme + Log logr.Logger + Name string + KubeConfigFileName string + TmpDirectory string +} + +func (r *KubeconfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *KubeconfigResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *KubeconfigResource) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *KubeconfigResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *KubeconfigResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *KubeconfigResource) GetClient() client.Client { + return r.Client +} + +func (r *KubeconfigResource) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *KubeconfigResource) GetName() string { + return r.Name +} + +func (r *KubeconfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + status, err := r.getKubeconfigStatus(tenantControlPlane) + if err != nil { + return err + } + + status.LastUpdate = metav1.Now() + status.SecretName = r.resource.GetName() + + return nil +} + +func (r *KubeconfigResource) getKubeconfigStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.KubeconfigStatus, error) { + switch r.KubeConfigFileName { + case kubeadmconstants.AdminKubeConfigFileName: + return &tenantControlPlane.Status.KubeConfig.Admin, nil + case kubeadmconstants.ControllerManagerKubeConfigFileName: + return &tenantControlPlane.Status.KubeConfig.ControllerManager, nil + case kubeadmconstants.SchedulerKubeConfigFileName: + return &tenantControlPlane.Status.KubeConfig.Scheduler, nil + default: + return nil, fmt.Errorf("kubeconfigfilename %s is not a right name", r.KubeConfigFileName) + } +} + +func (r *KubeconfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestConfigRV := getLatestConfigRV(*tenantControlPlane) + actualConfigRV := r.resource.GetLabels()["latest-config-rv"] + if latestConfigRV == actualConfigRV { + if kubeadm.IsKubeconfigValid(r.resource.Data[r.KubeConfigFileName]) { + return nil + } + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + if err := r.customizeConfig(config); err != nil { + return err + } + + apiServerCertificatesSecretNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName} + apiServerCertificatesSecret := &corev1.Secret{} + if err := r.Client.Get(ctx, apiServerCertificatesSecretNamespacedName, apiServerCertificatesSecret); err != nil { + return err + } + + kubeconfig, err := kubeadm.CreateKubeconfig( + r.KubeConfigFileName, + kubeadm.CertificatePrivateKeyPair{ + Certificate: apiServerCertificatesSecret.Data[kubeadmconstants.CACertName], + PrivateKey: apiServerCertificatesSecret.Data[kubeadmconstants.CAKeyName], + }, + config, + ) + if err != nil { + return err + } + r.resource.Data = map[string][]byte{ + r.KubeConfigFileName: kubeconfig, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-config-rv": latestConfigRV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} + +func (r *KubeconfigResource) customizeConfig(config *kubeadm.Configuration) error { + switch r.KubeConfigFileName { + case kubeadmconstants.AdminKubeConfigFileName: + return r.ingressAsAdvertiseAddress(config) + case kubeadmconstants.ControllerManagerKubeConfigFileName: + return r.localhostAsAdvertiseAddress(config) + case kubeadmconstants.SchedulerKubeConfigFileName: + return r.localhostAsAdvertiseAddress(config) + default: + return nil + } +} + +func (r *KubeconfigResource) ingressAsAdvertiseAddress(config *kubeadm.Configuration) error { + config.InitConfiguration.LocalAPIEndpoint.BindPort = defaultIngressPort + + return nil +} + +func (r *KubeconfigResource) localhostAsAdvertiseAddress(config *kubeadm.Configuration) error { + config.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = localhost + + return nil +} diff --git a/internal/resources/resource.go b/internal/resources/resource.go new file mode 100644 index 0000000..359c5c9 --- /dev/null +++ b/internal/resources/resource.go @@ -0,0 +1,96 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" +) + +type Resource interface { + Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error + ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool + CleanUp(context.Context) (bool, error) + CreateOrUpdate(context.Context, *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) + GetName() string + ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool + UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error +} + +type DeleteableResource interface { + Delete(context.Context, *kamajiv1alpha1.TenantControlPlane) error +} + +type KubeadmResource interface { + Resource + GetClient() client.Client + GetTmpDirectory() string +} + +type HandlerConfig struct { + Resource Resource + TenantControlPlane *kamajiv1alpha1.TenantControlPlane +} + +// Handle handles the given resource and returns a boolean to say if the tenantControlPlane has been modified. +func Handle(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + if err := resource.Define(ctx, tenantControlPlane); err != nil { + return "", err + } + + if !resource.ShouldCleanup(tenantControlPlane) { + return createOrUpdate(ctx, resource, tenantControlPlane) + } + + cleanUp, err := resource.CleanUp(ctx) + if err != nil { + return controllerutil.OperationResultNone, err + } + + if cleanUp { + return controllerutil.OperationResultUpdated, nil + } + + return controllerutil.OperationResultNone, err +} + +func createOrUpdate(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + result, err := resource.CreateOrUpdate(ctx, tenantControlPlane) + if err != nil { + return "", err + } + + if result == controllerutil.OperationResultNone && resource.ShouldStatusBeUpdated(ctx, tenantControlPlane) { + return controllerutil.OperationResultUpdatedStatusOnly, nil + } + + return result, nil +} + +func getKubeadmConfiguration(ctx context.Context, r KubeadmResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kubeadm.Configuration, string, error) { + var configmap corev1.ConfigMap + namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.KubeadmConfig.ConfigmapName} + if err := r.GetClient().Get(ctx, namespacedName, &configmap); err != nil { + return nil, "", err + } + + config, err := kubeadm.GetKubeadmInitConfigurationFromMap(configmap.Data) + if err != nil { + return nil, "", err + } + + tmpDirectory := r.GetTmpDirectory() + if tmpDirectory != "" { + config.InitConfiguration.ClusterConfiguration.CertificatesDir = tmpDirectory + } + + return config, configmap.ObjectMeta.ResourceVersion, nil +} diff --git a/internal/resources/sa_certificate.go b/internal/resources/sa_certificate.go new file mode 100644 index 0000000..c38f540 --- /dev/null +++ b/internal/resources/sa_certificate.go @@ -0,0 +1,124 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/utilities" +) + +type SACertificate struct { + resource *corev1.Secret + Client client.Client + Log logr.Logger + Name string + TmpDirectory string +} + +func (r *SACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + return tenantControlPlane.Status.Certificates.SA.SecretName != r.resource.GetName() +} + +func (r *SACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (r *SACertificate) CleanUp(ctx context.Context) (bool, error) { + return false, nil +} + +func (r *SACertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.getPrefixedName(tenantControlPlane), + Namespace: tenantControlPlane.GetNamespace(), + }, + } + + return nil +} + +func (r *SACertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return utilities.AddTenantPrefix(r.Name, tenantControlPlane) +} + +func (r *SACertificate) GetClient() client.Client { + return r.Client +} + +func (r *SACertificate) GetTmpDirectory() string { + return r.TmpDirectory +} + +func (r *SACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane)) +} + +func (r *SACertificate) GetName() string { + return r.Name +} + +func (r *SACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + tenantControlPlane.Status.Certificates.SA.LastUpdate = metav1.Now() + tenantControlPlane.Status.Certificates.SA.SecretName = r.resource.GetName() + + return nil +} + +func (r *SACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + latestConfigRV := getLatestConfigRV(*tenantControlPlane) + actualConfigRV := r.resource.GetLabels()["latest-config-rv"] + if latestConfigRV == actualConfigRV { + isValid, err := kubeadm.IsPublicKeyPrivateKeyPairValid( + r.resource.Data[kubeadmconstants.ServiceAccountPublicKeyName], + r.resource.Data[kubeadmconstants.ServiceAccountPrivateKeyName], + ) + if err != nil { + r.Log.Info(fmt.Sprintf("%s public_key-private_key pair is not valid: %s", kubeadmconstants.ServiceAccountKeyBaseName, err.Error())) + } + if isValid { + return nil + } + } + + config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane) + if err != nil { + return err + } + + sa, err := kubeadm.GeneratePublicKeyPrivateKeyPair(kubeadmconstants.ServiceAccountKeyBaseName, config) + if err != nil { + return err + } + + r.resource.Data = map[string][]byte{ + kubeadmconstants.ServiceAccountPublicKeyName: sa.PublicKey, + kubeadmconstants.ServiceAccountPrivateKeyName: sa.PrivateKey, + } + + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(), + map[string]string{ + "latest-config-rv": latestConfigRV, + "kamaji.clastix.io/name": tenantControlPlane.GetName(), + "kamaji.clastix.io/component": r.GetName(), + }, + )) + + return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + } +} diff --git a/internal/resources/utils.go b/internal/resources/utils.go new file mode 100644 index 0000000..419b2bc --- /dev/null +++ b/internal/resources/utils.go @@ -0,0 +1,77 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "fmt" + "math/rand" + "time" + + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var letters = []byte("abcdefghijklmnopqrstuvwxyz") + +func updateOperationResult(current controllerutil.OperationResult, op controllerutil.OperationResult) controllerutil.OperationResult { + if current == controllerutil.OperationResultCreated || op == controllerutil.OperationResultCreated { + return controllerutil.OperationResultCreated + } + + if current == controllerutil.OperationResultUpdated || op == controllerutil.OperationResultUpdated { + return controllerutil.OperationResultUpdated + } + + if current == controllerutil.OperationResultUpdatedStatus || op == controllerutil.OperationResultUpdatedStatus { + return controllerutil.OperationResultUpdatedStatus + } + + if current == controllerutil.OperationResultUpdatedStatusOnly || op == controllerutil.OperationResultUpdatedStatusOnly { + return controllerutil.OperationResultUpdatedStatusOnly + } + + return controllerutil.OperationResultNone +} + +func secretProjection(secretName, certKeyName, keyName string) *v1.SecretProjection { + return &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: secretName, + }, + Items: []v1.KeyToPath{ + { + Key: certKeyName, + Path: certKeyName, + }, + { + Key: keyName, + Path: keyName, + }, + }, + } +} + +func randomString(n int) string { + rand.Seed(time.Now().UnixNano()) + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + + return string(b) +} + +func getTenantControllerExternalFQDN(tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("%s.%s.%s", tenantControlPlane.GetName(), tenantControlPlane.GetNamespace(), tenantControlPlane.Spec.NetworkProfile.Domain) +} + +func getTenantControllerInternalFQDN(tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("%s.%s.svc.cluster.local", tenantControlPlane.GetName(), tenantControlPlane.GetNamespace()) +} + +func getLatestConfigRV(tenantControlPlane kamajiv1alpha1.TenantControlPlane) string { + return tenantControlPlane.Status.KubeadmConfig.ResourceVersion +} diff --git a/internal/upgrade/kube_version_getter.go b/internal/upgrade/kube_version_getter.go new file mode 100644 index 0000000..66c66f5 --- /dev/null +++ b/internal/upgrade/kube_version_getter.go @@ -0,0 +1,53 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package upgrade + +import ( + "fmt" + "runtime" + + "github.com/pkg/errors" + versionutil "k8s.io/apimachinery/pkg/util/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" +) + +type kamajiKubeVersionGetter struct { + upgrade.VersionGetter +} + +func NewKamajiKubeVersionGetter(restClient *kubernetes.Clientset) upgrade.VersionGetter { + kubeVersionGetter := upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(restClient), KubeadmVersion) + + return &kamajiKubeVersionGetter{VersionGetter: kubeVersionGetter} +} + +func (k kamajiKubeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) { + return k.VersionGetter.ClusterVersion() +} + +func (k kamajiKubeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) { + kubeadmVersionInfo := apimachineryversion.Info{ + GitVersion: KubeadmVersion, + GoVersion: runtime.Version(), + Compiler: runtime.Compiler, + Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), + } + + kubeadmVersion, err := versionutil.ParseSemantic(kubeadmVersionInfo.String()) + if err != nil { + return "", nil, errors.Wrap(err, "Couldn't parse kubeadm version") + } + + return kubeadmVersionInfo.String(), kubeadmVersion, nil +} + +func (k kamajiKubeVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) { + return k.VersionGetter.VersionFromCILabel(ciVersionLabel, description) +} + +func (k kamajiKubeVersionGetter) KubeletVersions() (map[string]uint16, error) { + return k.VersionGetter.KubeletVersions() +} diff --git a/internal/upgrade/kubeadm_version.go b/internal/upgrade/kubeadm_version.go new file mode 100644 index 0000000..c953d92 --- /dev/null +++ b/internal/upgrade/kubeadm_version.go @@ -0,0 +1,8 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package upgrade + +const ( + KubeadmVersion = "v1.23.5" +) diff --git a/internal/utilities/utilities.go b/internal/utilities/utilities.go new file mode 100644 index 0000000..2a2c1a6 --- /dev/null +++ b/internal/utilities/utilities.go @@ -0,0 +1,44 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package utilities + +import ( + "fmt" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/constants" +) + +const ( + separator = "-" +) + +func KamajiLabels() map[string]string { + return map[string]string{ + constants.ProjectNameLabelKey: constants.ProjectNameLabelValue, + } +} + +func CommonLabels(clusterName string) map[string]string { + return map[string]string{ + "kamaji.clastix.io/type": "cluster", + "kamaji.clastix.io/cluster": clusterName, + } +} + +func MergeMaps(maps ...map[string]string) map[string]string { + result := map[string]string{} + + for _, m := range maps { + for k, v := range m { + result[k] = v + } + } + + return result +} + +func AddTenantPrefix(name string, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string { + return fmt.Sprintf("%s%s%s", tenantControlPlane.GetName(), separator, name) +} diff --git a/kamaji.yaml b/kamaji.yaml new file mode 100755 index 0000000..5fd8faf --- /dev/null +++ b/kamaji.yaml @@ -0,0 +1,10 @@ +etcd-ca-secret-name: "etcd-certs" +etcd-ca-secret-namespace: kamaji-system +etcd-endpoints: etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379 +etcd-client-secret-name: root-client-certs +etcd-client-secret-namespaced: kamaji-system +etcd-compaction: "0" +metrics-bind-address: :8080 +health-probe-bind-address: :8081 +leader-elect: false +tmp-directory: /tmp/kamaji diff --git a/main.go b/main.go new file mode 100644 index 0000000..32c6c77 --- /dev/null +++ b/main.go @@ -0,0 +1,101 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + "os" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/controllers" + "github.com/clastix/kamaji/internal/config" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + conf, err := config.InitConfig() + if err != nil { + log.Fatalf("Error reading configuration.") + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: conf.GetString("metrics-bind-address"), + Port: 9443, + HealthProbeBindAddress: conf.GetString("health-probe-bind-address"), + LeaderElection: conf.GetBool("leader-elect"), + LeaderElectionID: "799b98bc.clastix.io", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + reconciler := &controllers.TenantControlPlaneReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: controllers.TenantControlPlaneReconcilerConfig{ + ETCDCASecretName: conf.GetString("etcd-ca-secret-name"), + ETCDCASecretNamespace: conf.GetString("etcd-ca-secret-namespace"), + ETCDClientSecretName: conf.GetString("etcd-client-secret-name"), + ETCDClientSecretNamespace: conf.GetString("etcd-client-secret-namespace"), + ETCDEndpoints: conf.GetString("etcd-endpoints"), + ETCDCompactionInterval: conf.GetString("etcd-compaction-interval"), + TmpBaseDirectory: conf.GetString("tmp-directory"), + }, + } + + if err := reconciler.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Namespace") + os.Exit(1) + } + + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +}