Add gcs cloudstorage configuration docs (#1862)

* add gcs docs

* add explicit gcs keys

* gcs helm tests

* add iam permissions docs for gcs

* Update gcs docs with exact setup steps for workload identity
This commit is contained in:
Volodymyr Stoiko
2026-03-09 16:48:17 +02:00
committed by GitHub
parent 5f2f34e826
commit 33f5310e8e
11 changed files with 372 additions and 9 deletions

View File

@@ -50,6 +50,9 @@ jobs:
- name: Helm lint (Azure Blob values)
run: helm lint ./helm-chart -f ./helm-chart/tests/fixtures/values-azblob.yaml
- name: Helm lint (GCS values)
run: helm lint ./helm-chart -f ./helm-chart/tests/fixtures/values-gcs.yaml
- name: Helm lint (cloud refs values)
run: helm lint ./helm-chart -f ./helm-chart/tests/fixtures/values-cloud-refs.yaml
@@ -72,3 +75,6 @@ jobs:
- name: Validate Azure Blob template
run: helm template kubeshark ./helm-chart -f ./helm-chart/tests/fixtures/values-azblob.yaml | kubeconform -strict -kubernetes-version 1.35.0 -summary
- name: Validate GCS template
run: helm template kubeshark ./helm-chart -f ./helm-chart/tests/fixtures/values-gcs.yaml | kubeconform -strict -kubernetes-version 1.35.0 -summary

View File

@@ -145,6 +145,7 @@ helm-test-full: helm-test ## Run Helm tests with kubeconform schema validation.
helm template kubeshark ./helm-chart | kubeconform -strict -kubernetes-version 1.35.0 -summary
helm template kubeshark ./helm-chart -f ./helm-chart/tests/fixtures/values-s3.yaml | kubeconform -strict -kubernetes-version 1.35.0 -summary
helm template kubeshark ./helm-chart -f ./helm-chart/tests/fixtures/values-azblob.yaml | kubeconform -strict -kubernetes-version 1.35.0 -summary
helm template kubeshark ./helm-chart -f ./helm-chart/tests/fixtures/values-gcs.yaml | kubeconform -strict -kubernetes-version 1.35.0 -summary
lint: ## Lint the source code.
golangci-lint run

View File

@@ -330,6 +330,12 @@ type SnapshotsCloudAzblobConfig struct {
StorageKey string `yaml:"storageKey" json:"storageKey" default:""`
}
type SnapshotsCloudGCSConfig struct {
Bucket string `yaml:"bucket" json:"bucket" default:""`
Project string `yaml:"project" json:"project" default:""`
CredentialsJson string `yaml:"credentialsJson" json:"credentialsJson" default:""`
}
type SnapshotsCloudConfig struct {
Provider string `yaml:"provider" json:"provider" default:""`
Prefix string `yaml:"prefix" json:"prefix" default:""`
@@ -337,6 +343,7 @@ type SnapshotsCloudConfig struct {
Secrets []string `yaml:"secrets" json:"secrets" default:"[]"`
S3 SnapshotsCloudS3Config `yaml:"s3" json:"s3"`
Azblob SnapshotsCloudAzblobConfig `yaml:"azblob" json:"azblob"`
GCS SnapshotsCloudGCSConfig `yaml:"gcs" json:"gcs"`
}
type SnapshotsConfig struct {

View File

@@ -146,10 +146,10 @@ Example for overriding image names:
| `tap.capture.dbMaxSize` | Maximum size for capture database (e.g., `4Gi`, `2000Mi`). | `500Mi` |
| `tap.snapshots.local.storageClass` | Storage class for local snapshots volume. When empty, uses `emptyDir`. When set, creates a PVC with this storage class | `""` |
| `tap.snapshots.local.storageSize` | Storage size for local snapshots volume (supports K8s quantity format: `1Gi`, `500Mi`, etc.) | `20Gi` |
| `tap.snapshots.cloud.provider` | Cloud storage provider for snapshots: `s3` or `azblob`. Empty string disables cloud storage. See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `""` |
| `tap.snapshots.cloud.provider` | Cloud storage provider for snapshots: `s3`, `azblob`, or `gcs`. Empty string disables cloud storage. See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `""` |
| `tap.snapshots.cloud.prefix` | Key prefix in the bucket/container (e.g. `snapshots/`). See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `""` |
| `tap.snapshots.cloud.configMaps` | Names of pre-existing ConfigMaps with cloud storage env vars. Alternative to inline `s3`/`azblob` values below. See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `[]` |
| `tap.snapshots.cloud.secrets` | Names of pre-existing Secrets with cloud storage credentials. Alternative to inline `s3`/`azblob` values below. See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `[]` |
| `tap.snapshots.cloud.configMaps` | Names of pre-existing ConfigMaps with cloud storage env vars. Alternative to inline `s3`/`azblob`/`gcs` values below. See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `[]` |
| `tap.snapshots.cloud.secrets` | Names of pre-existing Secrets with cloud storage credentials. Alternative to inline `s3`/`azblob`/`gcs` values below. See [Cloud Storage docs](docs/snapshots_cloud_storage.md). | `[]` |
| `tap.snapshots.cloud.s3.bucket` | S3 bucket name. When set, the chart auto-creates a ConfigMap with `SNAPSHOT_AWS_BUCKET`. | `""` |
| `tap.snapshots.cloud.s3.region` | AWS region for the S3 bucket. | `""` |
| `tap.snapshots.cloud.s3.accessKey` | AWS access key ID. When set, the chart auto-creates a Secret with `SNAPSHOT_AWS_ACCESS_KEY`. | `""` |
@@ -159,6 +159,9 @@ Example for overriding image names:
| `tap.snapshots.cloud.azblob.storageAccount` | Azure storage account name. When set, the chart auto-creates a ConfigMap with `SNAPSHOT_AZBLOB_STORAGE_ACCOUNT`. | `""` |
| `tap.snapshots.cloud.azblob.container` | Azure blob container name. | `""` |
| `tap.snapshots.cloud.azblob.storageKey` | Azure storage account access key. When set, the chart auto-creates a Secret with `SNAPSHOT_AZBLOB_STORAGE_KEY`. | `""` |
| `tap.snapshots.cloud.gcs.bucket` | GCS bucket name. When set, the chart auto-creates a ConfigMap with `SNAPSHOT_GCS_BUCKET`. | `""` |
| `tap.snapshots.cloud.gcs.project` | GCP project ID. | `""` |
| `tap.snapshots.cloud.gcs.credentialsJson` | Service account JSON key. When set, the chart auto-creates a Secret with `SNAPSHOT_GCS_CREDENTIALS_JSON`. | `""` |
| `tap.delayedDissection.cpu` | CPU allocation for delayed dissection jobs | `1` |
| `tap.delayedDissection.memory` | Memory allocation for delayed dissection jobs | `4Gi` |
| `tap.release.repo` | URL of the Helm chart repository | `https://helm.kubeshark.com` |

View File

@@ -2,7 +2,7 @@
Kubeshark can upload and download snapshots to cloud object storage, enabling cross-cluster sharing, backup/restore, and long-term retention.
Supported providers: **Amazon S3** (`s3`) and **Azure Blob Storage** (`azblob`).
Supported providers: **Amazon S3** (`s3`), **Azure Blob Storage** (`azblob`), and **Google Cloud Storage** (`gcs`).
## Helm Values
@@ -10,7 +10,7 @@ Supported providers: **Amazon S3** (`s3`) and **Azure Blob Storage** (`azblob`).
tap:
snapshots:
cloud:
provider: "" # "s3" or "azblob" (empty = disabled)
provider: "" # "s3", "azblob", or "gcs" (empty = disabled)
prefix: "" # key prefix in the bucket/container (e.g. "snapshots/")
configMaps: [] # names of pre-existing ConfigMaps with cloud config env vars
secrets: [] # names of pre-existing Secrets with cloud credentials
@@ -25,6 +25,10 @@ tap:
storageAccount: ""
container: ""
storageKey: ""
gcs:
bucket: ""
project: ""
credentialsJson: ""
```
- `provider` selects which cloud backend to use. Leave empty to disable cloud storage.
@@ -278,3 +282,212 @@ tap:
secrets:
- kubeshark-azblob-creds
```
---
## Google Cloud Storage
### Environment Variables
| Variable | Required | Description |
|----------|----------|-------------|
| `SNAPSHOT_GCS_BUCKET` | Yes | GCS bucket name |
| `SNAPSHOT_GCS_PROJECT` | No | GCP project ID |
| `SNAPSHOT_GCS_CREDENTIALS_JSON` | No | Service account JSON key (empty = use Application Default Credentials) |
| `SNAPSHOT_CLOUD_PREFIX` | No | Key prefix in the bucket (e.g. `snapshots/`) |
### Authentication Methods
Credentials are resolved in this order:
1. **Service Account JSON Key** -- If `SNAPSHOT_GCS_CREDENTIALS_JSON` is set, the provided JSON key is used directly.
2. **Application Default Credentials** -- When no JSON key is provided, the GCP SDK default credential chain is used:
- **Workload Identity** (GKE pod identity) -- recommended for production on GKE
- GCE instance metadata (Compute Engine default service account)
- Standard GCP environment variables (`GOOGLE_APPLICATION_CREDENTIALS`)
- `gcloud` CLI credentials
The provider validates bucket access on startup via `Bucket.Attrs()`. If the bucket is inaccessible, the hub will fail to start.
### Required IAM Permissions
The service account needs different IAM roles depending on the access level:
**Read-only** (download, list, and sync snapshots from cloud):
| Role | Permissions provided | Purpose |
|------|---------------------|---------|
| `roles/storage.legacyBucketReader` | `storage.buckets.get`, `storage.objects.list` | Hub startup (bucket validation) + listing snapshots |
| `roles/storage.objectViewer` | `storage.objects.get`, `storage.objects.list` | Downloading snapshots, checking existence, reading metadata |
```bash
gcloud storage buckets add-iam-policy-binding gs://BUCKET_NAME \
--member="serviceAccount:SA_EMAIL" \
--role="roles/storage.legacyBucketReader"
gcloud storage buckets add-iam-policy-binding gs://BUCKET_NAME \
--member="serviceAccount:SA_EMAIL" \
--role="roles/storage.objectViewer"
```
**Read-write** (upload and delete snapshots in addition to read):
Add `roles/storage.objectAdmin` instead of `roles/storage.objectViewer` to also grant `storage.objects.create` and `storage.objects.delete`:
| Role | Permissions provided | Purpose |
|------|---------------------|---------|
| `roles/storage.legacyBucketReader` | `storage.buckets.get`, `storage.objects.list` | Hub startup (bucket validation) + listing snapshots |
| `roles/storage.objectAdmin` | `storage.objects.*` | Full object CRUD (upload, download, delete, list, metadata) |
```bash
gcloud storage buckets add-iam-policy-binding gs://BUCKET_NAME \
--member="serviceAccount:SA_EMAIL" \
--role="roles/storage.legacyBucketReader"
gcloud storage buckets add-iam-policy-binding gs://BUCKET_NAME \
--member="serviceAccount:SA_EMAIL" \
--role="roles/storage.objectAdmin"
```
### Example: Inline Values (simplest approach)
```yaml
tap:
snapshots:
cloud:
provider: "gcs"
gcs:
bucket: my-kubeshark-snapshots
project: my-gcp-project
```
Or with a service account key via `--set`:
```bash
helm install kubeshark kubeshark/kubeshark \
--set tap.snapshots.cloud.provider=gcs \
--set tap.snapshots.cloud.gcs.bucket=my-kubeshark-snapshots \
--set tap.snapshots.cloud.gcs.project=my-gcp-project \
--set-file tap.snapshots.cloud.gcs.credentialsJson=service-account.json
```
### Example: Workload Identity (recommended for GKE)
Create a ConfigMap with bucket configuration:
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: kubeshark-gcs-config
data:
SNAPSHOT_GCS_BUCKET: my-kubeshark-snapshots
SNAPSHOT_GCS_PROJECT: my-gcp-project
```
Set Helm values:
```yaml
tap:
snapshots:
cloud:
provider: "gcs"
configMaps:
- kubeshark-gcs-config
```
Configure GKE Workload Identity to allow the Kubernetes service account to impersonate the GCP service account:
```bash
# Ensure the GKE cluster has Workload Identity enabled
# (--workload-pool=PROJECT_ID.svc.id.goog at cluster creation)
# Create a GCP service account (if not already created)
gcloud iam service-accounts create kubeshark-gcs \
--display-name="Kubeshark GCS Snapshots"
# Grant bucket access (read-write — see Required IAM Permissions above)
gcloud storage buckets add-iam-policy-binding gs://BUCKET_NAME \
--member="serviceAccount:kubeshark-gcs@PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/storage.legacyBucketReader"
gcloud storage buckets add-iam-policy-binding gs://BUCKET_NAME \
--member="serviceAccount:kubeshark-gcs@PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/storage.objectAdmin"
# Allow the K8s service account to impersonate the GCP service account
# Note: the K8s SA name is "<release-name>-service-account" (default: "kubeshark-service-account")
gcloud iam service-accounts add-iam-policy-binding \
kubeshark-gcs@PROJECT_ID.iam.gserviceaccount.com \
--role="roles/iam.workloadIdentityUser" \
--member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/kubeshark-service-account]"
```
Set Helm values — the `tap.annotations` field adds the Workload Identity annotation to the service account:
```yaml
tap:
annotations:
iam.gke.io/gcp-service-account: kubeshark-gcs@PROJECT_ID.iam.gserviceaccount.com
snapshots:
cloud:
provider: "gcs"
configMaps:
- kubeshark-gcs-config
```
Or via `--set`:
```bash
helm install kubeshark kubeshark/kubeshark \
--set tap.snapshots.cloud.provider=gcs \
--set tap.snapshots.cloud.gcs.bucket=BUCKET_NAME \
--set tap.snapshots.cloud.gcs.project=PROJECT_ID \
--set tap.annotations."iam\.gke\.io/gcp-service-account"=kubeshark-gcs@PROJECT_ID.iam.gserviceaccount.com
```
No `credentialsJson` secret is needed — GKE injects credentials automatically via the Workload Identity metadata server.
### Example: Service Account Key
Create a Secret with the service account JSON key:
```yaml
apiVersion: v1
kind: Secret
metadata:
name: kubeshark-gcs-creds
type: Opaque
stringData:
SNAPSHOT_GCS_CREDENTIALS_JSON: |
{
"type": "service_account",
"project_id": "my-gcp-project",
"private_key_id": "...",
"private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n",
"client_email": "kubeshark@my-gcp-project.iam.gserviceaccount.com",
...
}
```
Create a ConfigMap with bucket configuration:
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: kubeshark-gcs-config
data:
SNAPSHOT_GCS_BUCKET: my-kubeshark-snapshots
SNAPSHOT_GCS_PROJECT: my-gcp-project
```
Set Helm values:
```yaml
tap:
snapshots:
cloud:
provider: "gcs"
configMaps:
- kubeshark-gcs-config
secrets:
- kubeshark-gcs-creds
```

View File

@@ -65,8 +65,8 @@ spec:
- -cloud-storage-provider
- '{{ .Values.tap.snapshots.cloud.provider }}'
{{- end }}
{{- $hasInlineConfig := or .Values.tap.snapshots.cloud.prefix .Values.tap.snapshots.cloud.s3.bucket .Values.tap.snapshots.cloud.s3.region .Values.tap.snapshots.cloud.s3.roleArn .Values.tap.snapshots.cloud.s3.externalId .Values.tap.snapshots.cloud.azblob.storageAccount .Values.tap.snapshots.cloud.azblob.container }}
{{- $hasInlineSecrets := or .Values.tap.snapshots.cloud.s3.accessKey .Values.tap.snapshots.cloud.s3.secretKey .Values.tap.snapshots.cloud.azblob.storageKey }}
{{- $hasInlineConfig := or .Values.tap.snapshots.cloud.prefix .Values.tap.snapshots.cloud.s3.bucket .Values.tap.snapshots.cloud.s3.region .Values.tap.snapshots.cloud.s3.roleArn .Values.tap.snapshots.cloud.s3.externalId .Values.tap.snapshots.cloud.azblob.storageAccount .Values.tap.snapshots.cloud.azblob.container .Values.tap.snapshots.cloud.gcs.bucket .Values.tap.snapshots.cloud.gcs.project }}
{{- $hasInlineSecrets := or .Values.tap.snapshots.cloud.s3.accessKey .Values.tap.snapshots.cloud.s3.secretKey .Values.tap.snapshots.cloud.azblob.storageKey .Values.tap.snapshots.cloud.gcs.credentialsJson }}
{{- if or .Values.tap.secrets .Values.tap.snapshots.cloud.configMaps .Values.tap.snapshots.cloud.secrets $hasInlineConfig $hasInlineSecrets }}
envFrom:
{{- range .Values.tap.secrets }}

View File

@@ -1,5 +1,5 @@
{{- $hasConfigValues := or .Values.tap.snapshots.cloud.prefix .Values.tap.snapshots.cloud.s3.bucket .Values.tap.snapshots.cloud.s3.region .Values.tap.snapshots.cloud.s3.roleArn .Values.tap.snapshots.cloud.s3.externalId .Values.tap.snapshots.cloud.azblob.storageAccount .Values.tap.snapshots.cloud.azblob.container -}}
{{- $hasSecretValues := or .Values.tap.snapshots.cloud.s3.accessKey .Values.tap.snapshots.cloud.s3.secretKey .Values.tap.snapshots.cloud.azblob.storageKey -}}
{{- $hasConfigValues := or .Values.tap.snapshots.cloud.prefix .Values.tap.snapshots.cloud.s3.bucket .Values.tap.snapshots.cloud.s3.region .Values.tap.snapshots.cloud.s3.roleArn .Values.tap.snapshots.cloud.s3.externalId .Values.tap.snapshots.cloud.azblob.storageAccount .Values.tap.snapshots.cloud.azblob.container .Values.tap.snapshots.cloud.gcs.bucket .Values.tap.snapshots.cloud.gcs.project -}}
{{- $hasSecretValues := or .Values.tap.snapshots.cloud.s3.accessKey .Values.tap.snapshots.cloud.s3.secretKey .Values.tap.snapshots.cloud.azblob.storageKey .Values.tap.snapshots.cloud.gcs.credentialsJson -}}
{{- if $hasConfigValues }}
---
apiVersion: v1
@@ -31,6 +31,12 @@ data:
{{- if .Values.tap.snapshots.cloud.azblob.container }}
SNAPSHOT_AZBLOB_CONTAINER: {{ .Values.tap.snapshots.cloud.azblob.container | quote }}
{{- end }}
{{- if .Values.tap.snapshots.cloud.gcs.bucket }}
SNAPSHOT_GCS_BUCKET: {{ .Values.tap.snapshots.cloud.gcs.bucket | quote }}
{{- end }}
{{- if .Values.tap.snapshots.cloud.gcs.project }}
SNAPSHOT_GCS_PROJECT: {{ .Values.tap.snapshots.cloud.gcs.project | quote }}
{{- end }}
{{- end }}
{{- if $hasSecretValues }}
---
@@ -52,4 +58,7 @@ stringData:
{{- if .Values.tap.snapshots.cloud.azblob.storageKey }}
SNAPSHOT_AZBLOB_STORAGE_KEY: {{ .Values.tap.snapshots.cloud.azblob.storageKey | quote }}
{{- end }}
{{- if .Values.tap.snapshots.cloud.gcs.credentialsJson }}
SNAPSHOT_GCS_CREDENTIALS_JSON: {{ .Values.tap.snapshots.cloud.gcs.credentialsJson | quote }}
{{- end }}
{{- end }}

View File

@@ -111,6 +111,76 @@ tests:
value: "c29tZWtleQ=="
documentIndex: 1
- it: should render ConfigMap with GCS config only
set:
tap.snapshots.cloud.gcs.bucket: my-gcs-bucket
tap.snapshots.cloud.gcs.project: my-gcp-project
asserts:
- hasDocuments:
count: 1
- isKind:
of: ConfigMap
documentIndex: 0
- equal:
path: data.SNAPSHOT_GCS_BUCKET
value: "my-gcs-bucket"
documentIndex: 0
- equal:
path: data.SNAPSHOT_GCS_PROJECT
value: "my-gcp-project"
documentIndex: 0
- notExists:
path: data.SNAPSHOT_GCS_CREDENTIALS_JSON
documentIndex: 0
- it: should render ConfigMap and Secret with GCS config and credentials
set:
tap.snapshots.cloud.gcs.bucket: my-gcs-bucket
tap.snapshots.cloud.gcs.project: my-gcp-project
tap.snapshots.cloud.gcs.credentialsJson: '{"type":"service_account"}'
asserts:
- hasDocuments:
count: 2
- isKind:
of: ConfigMap
documentIndex: 0
- equal:
path: data.SNAPSHOT_GCS_BUCKET
value: "my-gcs-bucket"
documentIndex: 0
- equal:
path: data.SNAPSHOT_GCS_PROJECT
value: "my-gcp-project"
documentIndex: 0
- isKind:
of: Secret
documentIndex: 1
- equal:
path: metadata.name
value: RELEASE-NAME-cloud-secret
documentIndex: 1
- equal:
path: stringData.SNAPSHOT_GCS_CREDENTIALS_JSON
value: '{"type":"service_account"}'
documentIndex: 1
- it: should render ConfigMap with GCS bucket only (no project)
set:
tap.snapshots.cloud.gcs.bucket: my-gcs-bucket
asserts:
- hasDocuments:
count: 1
- isKind:
of: ConfigMap
documentIndex: 0
- equal:
path: data.SNAPSHOT_GCS_BUCKET
value: "my-gcs-bucket"
documentIndex: 0
- notExists:
path: data.SNAPSHOT_GCS_PROJECT
documentIndex: 0
- it: should render ConfigMap with only prefix
set:
tap.snapshots.cloud.prefix: snapshots/prod
@@ -130,6 +200,9 @@ tests:
- notExists:
path: data.SNAPSHOT_AZBLOB_STORAGE_ACCOUNT
documentIndex: 0
- notExists:
path: data.SNAPSHOT_GCS_BUCKET
documentIndex: 0
- it: should render ConfigMap with role ARN without credentials (IAM auth)
set:

View File

@@ -0,0 +1,9 @@
tap:
snapshots:
cloud:
provider: gcs
prefix: snapshots/
gcs:
bucket: kubeshark-snapshots
project: my-gcp-project
credentialsJson: '{"type":"service_account","project_id":"my-gcp-project"}'

View File

@@ -41,6 +41,44 @@ tests:
secretRef:
name: RELEASE-NAME-cloud-secret
- it: should render envFrom with inline GCS config
set:
tap.snapshots.cloud.gcs.bucket: my-gcs-bucket
tap.snapshots.cloud.gcs.project: my-gcp-project
asserts:
- contains:
path: spec.template.spec.containers[0].envFrom
content:
configMapRef:
name: RELEASE-NAME-cloud-config
- it: should render envFrom secret ref with inline GCS credentials
set:
tap.snapshots.cloud.gcs.bucket: my-gcs-bucket
tap.snapshots.cloud.gcs.credentialsJson: '{"type":"service_account"}'
asserts:
- contains:
path: spec.template.spec.containers[0].envFrom
content:
configMapRef:
name: RELEASE-NAME-cloud-config
- contains:
path: spec.template.spec.containers[0].envFrom
content:
secretRef:
name: RELEASE-NAME-cloud-secret
- it: should render cloud-storage-provider arg when provider is gcs
set:
tap.snapshots.cloud.provider: gcs
asserts:
- contains:
path: spec.template.spec.containers[0].command
content: -cloud-storage-provider
- contains:
path: spec.template.spec.containers[0].command
content: gcs
- it: should render envFrom with external configMaps
set:
tap.snapshots.cloud.configMaps:

View File

@@ -57,6 +57,10 @@ tap:
storageAccount: ""
container: ""
storageKey: ""
gcs:
bucket: ""
project: ""
credentialsJson: ""
release:
repo: https://helm.kubeshark.com
name: kubeshark