diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7c2d815e..a2cc1183 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,18 +2,6 @@ name: e2e permissions: {} on: - push: - branches: [ "*" ] - paths: - - '.github/workflows/e2e.yml' - - 'api/**' - - 'controllers/**' - - 'pkg/**' - - 'e2e/*' - - 'Dockerfile' - - 'go.*' - - 'main.go' - - 'Makefile' pull_request: branches: [ "*" ] paths: diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index 76c72de8..ae7c126a 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -31,6 +31,7 @@ jobs: fi - name: Run chart-testing (lint) run: ct lint --debug --config ./.github/configs/ct.yaml --lint-conf ./.github/configs/lintconf.yaml + - name: Run docs-testing (helm-docs) id: helm-docs run: | @@ -44,5 +45,5 @@ jobs: fi - name: Run chart-testing (install) - run: make helm-test + run: HELM_KIND_CONFIG="./hack/kind-cluster.yml" make helm-test if: steps.list-changed.outputs.changed == 'true' \ No newline at end of file diff --git a/.github/workflows/releaser.yml b/.github/workflows/releaser.yml index 90a00557..980a35d0 100644 --- a/.github/workflows/releaser.yml +++ b/.github/workflows/releaser.yml @@ -11,7 +11,40 @@ concurrency: cancel-in-progress: true jobs: + seccomp-generation: + name: Seccomp Generation + strategy: + fail-fast: false + matrix: + # differently from the e2e workflow + # we don't need all the versions of kubernetes + # to generate the seccomp profile. + k8s-version: [ 'v1.30.0' ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + with: + go-version-file: 'go.mod' + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4 + with: + version: v3.14.2 + - name: unit tracing + run: sudo make trace-unit + - name: e2e tracing + run: sudo KIND_K8S_VERSION=${{ matrix.k8s-version }} make trace-e2e + - name: build seccomp profile + run: make seccomp + - name: upload artifact + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: capsule-seccomp + path: capsule-seccomp.json + create-release: + needs: seccomp-generation runs-on: ubuntu-latest permissions: contents: write @@ -33,6 +66,11 @@ jobs: - uses: anchore/sbom-action/download-syft@79202aee38a39bd2039be442e58d731b63baf740 - name: Install Cosign uses: sigstore/cosign-installer@c56c2d3e59e4281cc41dea2217323ba5694b171e # v3.8.0 + - name: download artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: capsule-seccomp + path: ./capsule-seccomp.json - name: Run GoReleaser uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0 with: diff --git a/.github/workflows/seccomp.yaml b/.github/workflows/seccomp.yaml new file mode 100644 index 00000000..c4435583 --- /dev/null +++ b/.github/workflows/seccomp.yaml @@ -0,0 +1,54 @@ +name: seccomp +permissions: {} + +on: + pull_request: + branches: [ "*" ] + paths: + - '.github/workflows/e2e.yml' + - 'api/**' + - 'controllers/**' + - 'pkg/**' + - 'e2e/*' + - 'Dockerfile' + - 'go.*' + - 'main.go' + - 'Makefile' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + seccomp-generation: + name: Seccomp Generation + strategy: + fail-fast: false + matrix: + # differently from the e2e workflow + # we don't need all the versions of kubernetes + # to generate the seccomp profile. + k8s-version: [ 'v1.30.0' ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + with: + go-version-file: 'go.mod' + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4 + with: + version: v3.14.2 + - name: unit tracing + run: sudo make trace-unit + - name: e2e tracing + run: sudo KIND_K8S_VERSION=${{ matrix.k8s-version }} make trace-e2e + - name: build seccomp profile + run: make seccomp + - name: upload artifact + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: capsule-seccomp + path: capsule-seccomp.json + diff --git a/.goreleaser.yml b/.goreleaser.yml index 0f575573..1b566914 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -45,6 +45,8 @@ release: - `ghcr.io/projectcapsule/charts/{{ .ProjectName }}:{{ .Version }}` [Review the Major Changes section first before upgrading to a new version](https://artifacthub.io/packages/helm/projectcapsule/capsule/{{ .Version }}#major-changes) + extra_files: + - glob: ./capsule-seccomp.json checksum: name_template: 'checksums.txt' changelog: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index cd399541..00000000 --- a/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# Build the manager binary -FROM golang:1.23.6 as builder - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -ARG TARGETARCH -ARG GIT_HEAD_COMMIT -ARG GIT_TAG_COMMIT -ARG GIT_LAST_TAG -ARG GIT_MODIFIED -ARG GIT_REPO -ARG BUILD_DATE - -# Copy the go source -COPY main.go main.go -COPY version.go version.go -COPY api/ api/ -COPY controllers/ controllers/ -COPY pkg/ pkg/ - -# Build -RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH GO111MODULE=on go build \ - -gcflags "-N -l" \ - -ldflags "-X main.GitRepo=$GIT_REPO -X main.GitTag=$GIT_LAST_TAG -X main.GitCommit=$GIT_HEAD_COMMIT -X main.GitDirty=$GIT_MODIFIED -X main.BuildTime=$BUILD_DATE" \ - -o manager - -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=builder /workspace/manager . -USER nonroot:nonroot - -ENTRYPOINT ["/manager"] diff --git a/Dockerfile.tracing b/Dockerfile.tracing new file mode 100644 index 00000000..ea91197a --- /dev/null +++ b/Dockerfile.tracing @@ -0,0 +1,17 @@ +# Target Binary +ARG TARGET_IMAGE +FROM ${TARGET_IMAGE} AS target + +# Inject Harpoon Image +FROM alegrey91/harpoon:v0.9.4 +WORKDIR / +COPY --from=target /ko-app/capsule ./manager + +ENTRYPOINT ["/harpoon", \ + "capture", \ + "-f", "main.main", \ + "-E", "NAMESPACE=capsule-system", \ + "-i", "2", \ + "-c", "-e", \ + "-S", "-D", "/tmp/results/", \ + "--", "/manager"] diff --git a/Makefile b/Makefile index 1951c529..3ad94f3b 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ BUILD_DATE ?= $(shell git log -1 --format="%at" | xargs -I{} sh -c 'if [ "$ IMG_BASE ?= $(REPOSITORY) IMG ?= $(IMG_BASE):$(VERSION) CAPSULE_IMG ?= $(REGISTRY)/$(IMG_BASE) +CLUSTER_NAME ?= capsule ## Tool Binaries KUBECTL ?= kubectl @@ -77,17 +78,21 @@ helm-lint: docker helm-schema: helm-plugin-schema cd charts/capsule && $(HELM) schema +helm-test: HELM_KIND_CONFIG ?= "" helm-test: kind ct ko-build-all - @$(KIND) create cluster --wait=60s --name capsule-charts --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} + @mkdir -p /tmp/results || true + @$(KIND) create cluster --wait=60s --name capsule-charts --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} --config $(HELM_KIND_CONFIG) @make helm-test-exec @$(KIND) delete cluster --name capsule-charts helm-test-exec: kind - @$(KIND) load docker-image --name capsule-charts $(CAPSULE_IMG):$(VERSION) + $(MAKE) docker-build-capsule-trace + $(MAKE) e2e-load-image CLUSTER_NAME=capsule-charts IMAGE=$(CAPSULE_IMG) VERSION=latest + $(MAKE) e2e-load-image CLUSTER_NAME=capsule-charts IMAGE=$(CAPSULE_IMG) VERSION=tracing @kubectl create ns capsule-system || true @kubectl apply --server-side=true -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.crds.yaml @kubectl apply --server-side=true -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml - @ct install --config $(SRC_ROOT)/.github/configs/ct.yaml --namespace=capsule-system --all --debug + @$(CT) install --config $(SRC_ROOT)/.github/configs/ct.yaml --namespace=capsule-system --all --debug docker: @hash docker 2>/dev/null || {\ @@ -178,6 +183,14 @@ ko-build-capsule: ko .PHONY: ko-build-all ko-build-all: ko-build-capsule +.PHONY: docker-build-capsule-trace +docker-build-capsule-trace: ko-build-capsule + @docker build \ + --no-cache \ + --build-arg TARGET_IMAGE=$(CAPSULE_IMG):$(VERSION) \ + -t $(CAPSULE_IMG):tracing \ + -f Dockerfile.tracing . + # Docker Image Publish # ------------------ @@ -238,6 +251,13 @@ KO_VERSION = v0.14.1 ko: $(call go-install-tool,$(KO),github.com/google/ko@$(KO_VERSION)) +HARPOON := $(shell pwd)/bin/harpoon +HARPOON_VERSION := v0.9.4 +harpoon: ## Download harpoon locally if necessary. + @mkdir $(shell pwd)/bin + @curl -s https://raw.githubusercontent.com/alegrey91/harpoon/main/install | \ + sudo bash -s -- --install-version $(HARPOON_VERSION) --install-dir $(shell pwd)/bin + #################### # -- Helpers #################### @@ -264,12 +284,6 @@ GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ } endef -# Generate bundle manifests and metadata, then validate generated files. -bundle: manifests - operator-sdk generate kustomize manifests -q - kustomize build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - operator-sdk bundle validate ./bundle - # Sorting imports .PHONY: goimports goimports: @@ -291,11 +305,12 @@ e2e: ginkgo $(MAKE) e2e-build && $(MAKE) e2e-exec && $(MAKE) e2e-destroy e2e-build: kind - $(KIND) create cluster --wait=60s --name capsule --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} + $(KIND) create cluster --wait=60s --name $(CLUSTER_NAME) --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} + $(MAKE) e2e-load-image CLUSTER_NAME=$(CLUSTER_NAME) IMAGE=$(CAPSULE_IMG) VERSION=$(VERSION) $(MAKE) e2e-install .PHONY: e2e-install -e2e-install: e2e-load-image +e2e-install: helm upgrade \ --dependency-update \ --debug \ @@ -310,9 +325,43 @@ e2e-install: e2e-load-image capsule \ ./charts/capsule +.PHONY: trace-install +trace-install: + helm upgrade \ + --dependency-update \ + --debug \ + --install \ + --namespace capsule-system \ + --create-namespace \ + --set 'manager.resources=null'\ + --set 'manager.livenessProbe.failureThreshold=10' \ + --set 'manager.readinessProbe.failureThreshold=10' \ + --values charts/capsule/ci/tracing-values.yaml \ + capsule \ + ./charts/capsule + +.PHONY: trace-e2e +trace-e2e: kind + $(MAKE) docker-build-capsule-trace + $(KIND) create cluster --wait=60s --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} --config hack/kind-cluster.yml + $(MAKE) e2e-load-image CLUSTER_NAME=capsule-tracing IMAGE=$(CAPSULE_IMG) VERSION=tracing + $(MAKE) trace-install + $(MAKE) e2e-exec + $(KIND) delete cluster --name capsule-tracing + +.PHONY: trace-unit +trace-unit: harpoon + $(HARPOON) analyze -e .git/ -e assets/ -e charts/ -e config/ -e docs/ -e e2e/ -e hack/ --directory /tmp/artifacts/ --save + $(HARPOON) hunt -D /tmp/results -F harpoon-report.yml --include-cmd-stdout --save + +.PHONY: seccomp +seccomp: + $(HARPOON) build --add-syscall-sets=dynamic,docker -D /tmp/results --name capsule-seccomp.json --save + .PHONY: e2e-load-image +e2e-load-image: LOAD_IMAGE ?= $(IMAGE):$(VERSION) e2e-load-image: kind ko-build-all - $(KIND) load docker-image --nodes capsule-control-plane --name capsule $(CAPSULE_IMG):$(VERSION) + $(KIND) load docker-image $(IMAGE):$(VERSION) --name $(CLUSTER_NAME) .PHONY: e2e-exec e2e-exec: ginkgo diff --git a/charts/capsule/README.md b/charts/capsule/README.md index dc2f525f..50f743fa 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -129,6 +129,7 @@ Here the values you can override: | nodeSelector | object | `{}` | Set the node selector for the Capsule pod | | podAnnotations | object | `{}` | Annotations to add to the capsule pod. | | podSecurityContext | object | `{"runAsGroup":1002,"runAsNonRoot":true,"runAsUser":1002,"seccompProfile":{"type":"RuntimeDefault"}}` | Set the securityContext for the Capsule pod | +| ports | list | `[]` | Set additional ports for the deployment | | priorityClassName | string | `""` | Set the priority class name of the Capsule pod | | proxy.enabled | bool | `false` | Enable Installation of Capsule Proxy | | replicaCount | int | `1` | Set the replica count for capsule pod | @@ -147,6 +148,7 @@ Here the values you can override: | Key | Type | Default | Description | |-----|------|---------|-------------| | manager.hostNetwork | bool | `false` | Specifies if the container should be started in hostNetwork mode. Required for use in some managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working | +| manager.hostPID | bool | `false` | Specifies if the container should be started in hostPID mode. | | manager.image.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy. | | manager.image.registry | string | `"ghcr.io"` | Set the image registry of capsule. | | manager.image.repository | string | `"projectcapsule/capsule"` | Set the image repository of capsule. | @@ -165,6 +167,9 @@ Here the values you can override: | manager.rbac.existingRoles | list | `[]` | Specifies further cluster roles to be added to the Capsule manager service account. | | manager.readinessProbe | object | `{"httpGet":{"path":"/readyz","port":10080}}` | Configure the readiness probe using Deployment probe spec | | manager.resources | object | `{}` | Set the resource requests/limits for the Capsule manager container | +| manager.securityContext | object | `{}` | Set the securityContext for the Capsule container | +| manager.volumeMounts | list | `[]` | Set the additional volumeMounts needed for the Capsule manager container | +| manager.volumes | list | `[]` | Set the additional volumes needed for the Capsule manager container | | manager.webhookPort | int | `9443` | Set an alternative to the default container port. Useful for use in some kubernetes clusters (such as GKE Private) with aggregator routing turned on, because pod ports have to be opened manually on the firewall side | ### ServiceMonitor Parameters diff --git a/charts/capsule/ci/tracing-values.yaml b/charts/capsule/ci/tracing-values.yaml new file mode 100644 index 00000000..f0d86a08 --- /dev/null +++ b/charts/capsule/ci/tracing-values.yaml @@ -0,0 +1,38 @@ +# Custome values for capsule tracing. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +manager: + image: + registry: ghcr.io + repository: projectcapsule/capsule + pullPolicy: Never + tag: tracing + hostNetwork: true + hostPID: true + volumes: + - name: debugfs + hostPath: + path: /sys/kernel/debug + type: Directory + - name: data + hostPath: + path: /tmp/results + type: Directory + volumeMounts: + - name: debugfs + mountPath: /sys/kernel/debug + - mountPath: /tmp/results + name: data + securityContext: + capabilities: + add: + - SYS_ADMIN + - NET_ADMIN + - PERFOM + privileged: true +podSecurityContext: + seccompProfile: + type: "Unconfined" + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 diff --git a/charts/capsule/templates/deployment.yaml b/charts/capsule/templates/deployment.yaml index 60a2e4c1..d219ef7c 100644 --- a/charts/capsule/templates/deployment.yaml +++ b/charts/capsule/templates/deployment.yaml @@ -37,6 +37,11 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet {{- end }} + {{- if .Values.manager.hostPID }} + hostPID: {{ .Values.manager.hostPID }} + {{- else }} + hostPID: false + {{- end }} priorityClassName: {{ .Values.priorityClassName }} {{- with .Values.nodeSelector }} nodeSelector: @@ -59,13 +64,16 @@ spec: secret: defaultMode: 420 secretName: {{ include "capsule.secretTlsName" . }} + {{- if .Values.manager.volumes }} + {{- toYaml .Values.manager.volumes | nindent 8 }} + {{- end }} containers: - name: manager args: - - --webhook-port={{ .Values.manager.webhookPort }} - - --enable-leader-election - - --zap-log-level={{ default 4 .Values.manager.options.logLevel }} - - --configuration-name={{ .Values.manager.options.capsuleConfiguration }} + - --webhook-port={{ .Values.manager.webhookPort }} + - --enable-leader-election + - --zap-log-level={{ default 4 .Values.manager.options.logLevel }} + - --configuration-name={{ .Values.manager.options.capsuleConfiguration }} image: {{ include "capsule.managerFullyQualifiedDockerImage" . }} imagePullPolicy: {{ .Values.manager.image.pullPolicy }} env: @@ -74,23 +82,35 @@ spec: fieldRef: fieldPath: metadata.namespace ports: + {{- if not (.Values.manager.hostNetwork) }} - name: webhook-server containerPort: {{ .Values.manager.webhookPort }} protocol: TCP - name: metrics containerPort: 8080 protocol: TCP + {{- end }} + {{- with .Values.manager.ports }} + {{- . | nindent 12 }} + {{- end }} livenessProbe: {{- toYaml .Values.manager.livenessProbe | nindent 12}} readinessProbe: {{- toYaml .Values.manager.readinessProbe | nindent 12}} volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- if .Values.manager.volumeMounts }} + {{- toYaml .Values.manager.volumeMounts | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.manager.resources | nindent 12 }} securityContext: + {{- if .Values.manager.securityContext }} + {{- toYaml .Values.manager.securityContext | nindent 12 }} + {{- else }} {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index 677a1d12..c1d70eaa 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -110,6 +110,9 @@ manager: # with pods' IP CIDR and admission webhooks are not working hostNetwork: false + # -- Specifies if the container should be started in hostPID mode. + hostPID: false + # -- Set an alternative to the default container port. # # Useful for use in some kubernetes clusters (such as GKE Private) with @@ -155,6 +158,15 @@ manager: # -- Set the resource requests/limits for the Capsule manager container resources: {} + # -- Set the additional volumes needed for the Capsule manager container + volumes: [] + + # -- Set the additional volumeMounts needed for the Capsule manager container + volumeMounts: [] + + # -- Set the securityContext for the Capsule container + securityContext: {} + # -- Configuration for `imagePullSecrets` so that you can use a private images registry. imagePullSecrets: [] @@ -175,7 +187,6 @@ podSecurityContext: runAsNonRoot: true runAsUser: 1002 - # -- Set the securityContext for the Capsule container securityContext: capabilities: @@ -198,6 +209,9 @@ tolerations: [] # -- Set the replica count for capsule pod replicaCount: 1 +# -- Set additional ports for the deployment +ports: [] + # -- Set affinity rules for the Capsule pod affinity: {} diff --git a/hack/kind-cluster.yml b/hack/kind-cluster.yml new file mode 100644 index 00000000..88bd511a --- /dev/null +++ b/hack/kind-cluster.yml @@ -0,0 +1,13 @@ +# With Kind configuration is used to +# share a folder between the outside sistem +# and the internal container (capsule-controller-manager), +# In this way we will be able to get the metadata +# generated by harpoon at the end of the e2e tests execution. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: capsule-tracing +nodes: +- role: control-plane + extraMounts: + - hostPath: /tmp/results + containerPath: /tmp/results