Compare commits

..

16 Commits

Author SHA1 Message Date
Enrico Candino
93025d301b Bump Charts to 1.0.1-rc2 (#586) 2025-12-03 11:44:49 +01:00
Enrico Candino
e385ceb66f Fixed missing Kubernetes host version when specified (#585)
* fix for missing host version

* added test

* fix test

* fix test
2025-12-03 09:21:27 +01:00
Enrico Candino
5c49c3d6b7 Fix create events rbac (#575)
* cleanup logs in kubelet provider

* added events create rbac to kubelet

* fix lint, moved fetch pod logic in separate func
2025-11-25 13:48:04 +01:00
Enrico Candino
521ff17ef6 Added test for SubPathExpr (#569)
* small fixes

* added test for subpathexpr

* removed old comment
2025-11-21 16:28:59 +01:00
Enrico Candino
5b4f31ef73 bump version to 1.0.1-rc1 in Chart.yaml (#567) 2025-11-17 18:24:57 +01:00
Enrico Candino
8856419e70 added check for failing tests (#566) 2025-11-17 12:58:59 +01:00
Enrico Candino
8760afd5bc Added --namespace flag to k3kcli policy create (#564)
* added --namespace flag to policy create to actually bind the new policy to existing namespaces

* fix lint

* fix tests

* added overwrite flag

* updated cli docs

* fix tests 2

* moved double quotes to single quote

* fix test
2025-11-14 21:45:28 +01:00
Enrico Candino
27730305c2 Added labels and annotations flags to cluster and policy create (#565)
* added labels and annotations flags to cluster create

* added labels and annotations flag to create policy command
2025-11-14 16:54:01 +01:00
Enrico Candino
d0e50a580d Added cluster details in cli during creation (#562)
* added cluster details in cli during cluster creation, silenced usage, removed static persistence type from help

* fix docs
2025-11-14 12:53:15 +01:00
Enrico Candino
7dc4726bbd Fixed panic during kubeconfig generate (#554)
* fix panic during kubeconfig generate

* moved check
2025-11-11 17:18:26 +01:00
Enrico Candino
7144cf9e66 Moved CRDs to Helm templates folder (#552)
* moved CRDs of Cluster and VirtualClusterPolicy

Updated the generate script to output CRDs to the correct directory and include the keep resource policy annotation.

* fix crd directory in tests
2025-11-11 16:22:56 +01:00
Enrico Candino
de0d2a0019 Add Job Summary reports to Conformance tests (#553)
* simplify shared conformance tests

* summary

* added failed test to summary

* space

* fix failed tests file

* removed sigs test
2025-11-11 13:01:23 +01:00
Enrico Candino
a84c49f9b6 Update Go version and some deps (#551)
* bump to Go 1.24.10

* bump k8s libs to v0.31.13 and v1.31.13

* bump cli deps (cobra, viper, pflag)
2025-11-07 12:34:34 +01:00
Enrico Candino
e79e6dbfc4 add upload permissions (#550) 2025-11-06 16:53:51 +01:00
Enrico Candino
2b6441e54e Added trivy vulns check (#549)
* image check

* added k3kcli
2025-11-06 12:46:15 +01:00
Enrico Candino
49a8d2a0ba Bump Charts to 1.0.0 (#543) 2025-11-03 16:44:11 +01:00
41 changed files with 1081 additions and 577 deletions

View File

@@ -13,6 +13,10 @@ jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -34,4 +38,51 @@ jobs:
env:
REPO: ${{ github.repository }}
REGISTRY: ""
- name: Run Trivy vulnerability scanner (k3kcli)
uses: aquasecurity/trivy-action@0.28.0
with:
ignore-unfixed: true
severity: 'MEDIUM,HIGH,CRITICAL'
scan-type: 'fs'
scan-ref: 'dist/k3kcli_linux_amd64_v1/k3kcli'
format: 'sarif'
output: 'trivy-results-k3kcli.sarif'
- name: Upload Trivy scan results to GitHub Security tab (k3kcli)
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: trivy-results-k3kcli.sarif
category: k3kcli
- name: Run Trivy vulnerability scanner (k3k)
uses: aquasecurity/trivy-action@0.28.0
with:
ignore-unfixed: true
severity: 'MEDIUM,HIGH,CRITICAL'
scan-type: 'image'
scan-ref: '${{ github.repository }}:v0.0.0-amd64'
format: 'sarif'
output: 'trivy-results-k3k.sarif'
- name: Upload Trivy scan results to GitHub Security tab (k3k)
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: trivy-results-k3k.sarif
category: k3k
- name: Run Trivy vulnerability scanner (k3k-kubelet)
uses: aquasecurity/trivy-action@0.28.0
with:
ignore-unfixed: true
severity: 'MEDIUM,HIGH,CRITICAL'
scan-type: 'image'
scan-ref: '${{ github.repository }}-kubelet:v0.0.0-amd64'
format: 'sarif'
output: 'trivy-results-k3k-kubelet.sarif'
- name: Upload Trivy scan results to GitHub Security tab (k3k-kubelet)
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: trivy-results-k3k-kubelet.sarif
category: k3k-kubelet

View File

@@ -0,0 +1,158 @@
name: Conformance Tests - Shared Mode
on:
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
permissions:
contents: read
jobs:
conformance:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
type:
- parallel
- serial
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install helm
uses: azure/setup-helm@v4.3.0
- name: Install hydrophone
run: go install sigs.k8s.io/hydrophone@latest
- name: Install k3d and kubectl
run: |
wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
k3d version
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
- name: Setup Kubernetes (k3d)
env:
REPO_NAME: k3k-registry
REPO_PORT: 12345
run: |
echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
k3d registry create ${REPO_NAME} --port ${REPO_PORT}
k3d cluster create k3k --servers 2 \
-p "30000-30010:30000-30010@server:0" \
--registry-use k3d-${REPO_NAME}:${REPO_PORT}
kubectl cluster-info
kubectl get nodes
- name: Setup K3k
env:
REPO: k3k-registry:12345
run: |
echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
make build
make package
make push
# add k3kcli to $PATH
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
VERSION=$(make version)
k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
make install
echo "Wait for K3k controller to be available"
kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
- name: Check k3kcli
run: k3kcli -v
- name: Create virtual cluster
run: |
kubectl create namespace k3k-mycluster
cat <<EOF | kubectl apply -f -
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: mycluster
namespace: k3k-mycluster
spec:
mirrorHostNodes: true
tlsSANs:
- "127.0.0.1"
expose:
nodePort:
serverPort: 30001
EOF
echo "Wait for bootstrap secret to be available"
kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
k3kcli kubeconfig generate --name mycluster
export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
kubectl cluster-info
kubectl get nodes
kubectl get pods -A
- name: Run conformance tests (parallel)
if: matrix.type == 'parallel'
run: |
# Run conformance tests in parallel mode (skipping serial)
hydrophone --conformance --parallel 4 --skip='\[Serial\]' \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Run conformance tests (serial)
if: matrix.type == 'serial'
run: |
# Run serial conformance tests
hydrophone --focus='\[Serial\].*\[Conformance\]' \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Archive conformance logs
uses: actions/upload-artifact@v4
if: always()
with:
name: conformance-${{ matrix.type }}-logs
path: /tmp/e2e.log
- name: Job Summary
if: always()
run: |
echo '## 📊 Conformance Tests Results (${{ matrix.type }})' >> $GITHUB_STEP_SUMMARY
echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY
echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY
RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3)
RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g')
echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY
# only include failed tests section if there are any
if grep -q '\[FAIL\]' /tmp/e2e.log; then
echo '' >> $GITHUB_STEP_SUMMARY
echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi

View File

@@ -123,3 +123,23 @@ jobs:
with:
name: conformance-${{ matrix.type }}-logs
path: /tmp/e2e.log
- name: Job Summary
if: always()
run: |
echo '## 📊 Conformance Tests Results (${{ matrix.type }})' >> $GITHUB_STEP_SUMMARY
echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY
echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY
RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3)
RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g')
echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY
# only include failed tests section if there are any
if grep -q '\[FAIL\]' /tmp/e2e.log; then
echo '' >> $GITHUB_STEP_SUMMARY
echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi

View File

@@ -1,302 +0,0 @@
name: Conformance Tests
on:
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
test:
description: "Run specific test"
type: choice
options:
- conformance
- sig-api-machinery
- sig-apps
- sig-architecture
- sig-auth
- sig-cli
- sig-instrumentation
- sig-network
- sig-node
- sig-scheduling
- sig-storage
permissions:
contents: read
jobs:
conformance:
runs-on: ubuntu-latest
if: inputs.test == '' || inputs.test == 'conformance'
strategy:
fail-fast: false
matrix:
type:
- parallel
- serial
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install helm
uses: azure/setup-helm@v4.3.0
- name: Install hydrophone
run: go install sigs.k8s.io/hydrophone@latest
- name: Install k3d and kubectl
run: |
wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
k3d version
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
- name: Setup Kubernetes (k3d)
env:
REPO_NAME: k3k-registry
REPO_PORT: 12345
run: |
echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
k3d registry create ${REPO_NAME} --port ${REPO_PORT}
k3d cluster create k3k --servers 3 \
-p "30000-30010:30000-30010@server:0" \
--registry-use k3d-${REPO_NAME}:${REPO_PORT}
kubectl cluster-info
kubectl get nodes
- name: Setup K3k
env:
REPO: k3k-registry:12345
run: |
echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
make build
make package
make push
# add k3kcli to $PATH
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
VERSION=$(make version)
k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
make install
echo "Wait for K3k controller to be available"
kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
- name: Check k3kcli
run: k3kcli -v
- name: Create virtual cluster
run: |
kubectl create namespace k3k-mycluster
cat <<EOF | kubectl apply -f -
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: mycluster
namespace: k3k-mycluster
spec:
servers: 2
mirrorHostNodes: true
tlsSANs:
- "127.0.0.1"
expose:
nodePort:
serverPort: 30001
EOF
echo "Wait for bootstrap secret to be available"
kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
k3kcli kubeconfig generate --name mycluster
export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
kubectl cluster-info
kubectl get nodes
kubectl get pods -A
- name: Run conformance tests (parallel)
if: matrix.type == 'parallel'
run: |
# Run conformance tests in parallel mode (skipping serial)
hydrophone --conformance --parallel 4 --skip='\[Serial\]' \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Run conformance tests (serial)
if: matrix.type == 'serial'
run: |
# Run serial conformance tests
hydrophone --focus='\[Serial\].*\[Conformance\]' \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Archive conformance logs
uses: actions/upload-artifact@v4
if: always()
with:
name: conformance-${{ matrix.type }}-logs
path: /tmp/e2e.log
sigs:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
tests:
- name: sig-api-machinery
focus: '\[sig-api-machinery\].*\[Conformance\]'
- name: sig-apps
focus: '\[sig-apps\].*\[Conformance\]'
- name: sig-architecture
focus: '\[sig-architecture\].*\[Conformance\]'
- name: sig-auth
focus: '\[sig-auth\].*\[Conformance\]'
- name: sig-cli
focus: '\[sig-cli\].*\[Conformance\]'
- name: sig-instrumentation
focus: '\[sig-instrumentation\].*\[Conformance\]'
- name: sig-network
focus: '\[sig-network\].*\[Conformance\]'
- name: sig-node
focus: '\[sig-node\].*\[Conformance\]'
- name: sig-scheduling
focus: '\[sig-scheduling\].*\[Conformance\]'
- name: sig-storage
focus: '\[sig-storage\].*\[Conformance\]'
steps:
- name: Validate input and fail fast
if: inputs.test != '' && inputs.test != matrix.tests.name
run: |
echo "Failing this job as it's not the intended target."
exit 1
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install helm
uses: azure/setup-helm@v4.3.0
- name: Install hydrophone
run: go install sigs.k8s.io/hydrophone@latest
- name: Install k3d and kubectl
run: |
wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
k3d version
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
- name: Setup Kubernetes (k3d)
env:
REPO_NAME: k3k-registry
REPO_PORT: 12345
run: |
echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
k3d registry create ${REPO_NAME} --port ${REPO_PORT}
k3d cluster create k3k --servers 3 \
-p "30000-30010:30000-30010@server:0" \
--registry-use k3d-${REPO_NAME}:${REPO_PORT}
kubectl cluster-info
kubectl get nodes
- name: Setup K3k
env:
REPO: k3k-registry:12345
run: |
echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
make build
make package
make push
# add k3kcli to $PATH
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
VERSION=$(make version)
k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
make install
echo "Wait for K3k controller to be available"
kubectl wait -n k3k-system pod --for condition=Ready -l "app.kubernetes.io/name=k3k" --timeout=5m
- name: Check k3kcli
run: k3kcli -v
- name: Create virtual cluster
run: |
kubectl create namespace k3k-mycluster
cat <<EOF | kubectl apply -f -
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: mycluster
namespace: k3k-mycluster
spec:
servers: 2
mirrorHostNodes: true
tlsSANs:
- "127.0.0.1"
expose:
nodePort:
serverPort: 30001
EOF
echo "Wait for bootstrap secret to be available"
kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
k3kcli kubeconfig generate --name mycluster
export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
kubectl cluster-info
kubectl get nodes
kubectl get pods -A
- name: Run sigs tests
run: |
FOCUS="${{ matrix.tests.focus }}"
echo "Running with --focus=${FOCUS}"
hydrophone --focus "${FOCUS}" \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Archive conformance logs
uses: actions/upload-artifact@v4
if: always()
with:
name: ${{ matrix.tests.name }}-logs
path: /tmp/e2e.log

View File

@@ -19,7 +19,7 @@ CRD_REF_DOCS := go run github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VER)
ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
ENVTEST_DIR ?= $(shell pwd)/.envtest
E2E_LABEL_FILTER ?= "e2e"
E2E_LABEL_FILTER ?= e2e
export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path)

View File

@@ -1,6 +1,5 @@
# K3k: Kubernetes in Kubernetes
[![Experimental](https://img.shields.io/badge/status-experimental-orange.svg)](https://shields.io/)
[![Go Report Card](https://goreportcard.com/badge/github.com/rancher/k3k)](https://goreportcard.com/report/github.com/rancher/k3k)
![Tests](https://github.com/rancher/k3k/actions/workflows/test.yaml/badge.svg)
![Build](https://github.com/rancher/k3k/actions/workflows/build.yml/badge.svg)
@@ -12,10 +11,6 @@ K3k, Kubernetes in Kubernetes, is a tool that empowers you to create and manage
K3k integrates seamlessly with Rancher for simplified management of your embedded clusters.
**Experimental Tool**
This project is still under development and is considered experimental. It may have limitations, bugs, or changes. Please use with caution and report any issues you encounter. We appreciate your feedback as we continue to refine and improve this tool.
## Features and Benefits
@@ -60,7 +55,7 @@ This section provides instructions on how to install K3k and the `k3kcli`.
helm install --namespace k3k-system --create-namespace k3k k3k/k3k
```
**NOTE:** K3k is currently under development. We recommend using the latest released version when possible.
We recommend using the latest released version when possible.
### Install the `k3kcli`
@@ -72,7 +67,7 @@ To install it, simply download the latest available version for your architectur
For example, you can download the Linux amd64 version with:
```
wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v0.3.5/k3kcli-linux-amd64 && \
wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v1.0.0/k3kcli-linux-amd64 && \
chmod +x k3kcli && \
sudo mv k3kcli /usr/local/bin
```
@@ -80,7 +75,7 @@ wget -qO k3kcli https://github.com/rancher/k3k/releases/download/v0.3.5/k3kcli-l
You should now be able to run:
```bash
-> % k3kcli --version
k3kcli version v0.3.5
k3kcli version v1.0.0
```

View File

@@ -2,5 +2,5 @@ apiVersion: v2
name: k3k
description: A Helm chart for K3K
type: application
version: 1.0.0-rc3
appVersion: v1.0.0-rc3
version: 1.0.1-rc2
appVersion: v1.0.1-rc2

View File

@@ -3,6 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
helm.sh/resource-policy: keep
controller-gen.kubebuilder.io/version: v0.16.0
name: clusters.k3k.io
spec:

View File

@@ -3,6 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
helm.sh/resource-policy: keep
controller-gen.kubebuilder.io/version: v0.16.0
name: virtualclusterpolicies.k3k.io
spec:

View File

@@ -1,12 +1,14 @@
package cmds
import (
"bytes"
"context"
"errors"
"fmt"
"net/url"
"os"
"strings"
"text/template"
"time"
"github.com/sirupsen/logrus"
@@ -37,6 +39,8 @@ type CreateConfig struct {
agentArgs []string
serverEnvs []string
agentEnvs []string
labels []string
annotations []string
persistenceType string
storageClassName string
storageRequestSize string
@@ -111,7 +115,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
}
}
logrus.Infof("Creating cluster [%s] in namespace [%s]", name, namespace)
logrus.Infof("Creating cluster '%s' in namespace '%s'", name, namespace)
cluster := newCluster(name, namespace, config)
@@ -134,19 +138,30 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
if err := client.Create(ctx, cluster); err != nil {
if apierrors.IsAlreadyExists(err) {
logrus.Infof("Cluster [%s] already exists", name)
logrus.Infof("Cluster '%s' already exists", name)
} else {
return err
}
}
if err := waitForClusterReconciled(ctx, client, cluster, config.timeout); err != nil {
return fmt.Errorf("failed to wait for cluster to be reconciled: %w", err)
}
clusterDetails, err := printClusterDetails(cluster)
if err != nil {
return fmt.Errorf("failed to print cluster details: %w", err)
}
logrus.Info(clusterDetails)
logrus.Infof("Waiting for cluster to be available..")
if err := waitForCluster(ctx, client, cluster, config.timeout); err != nil {
if err := waitForClusterReady(ctx, client, cluster, config.timeout); err != nil {
return fmt.Errorf("failed to wait for cluster to become ready (status: %s): %w", cluster.Status.Phase, err)
}
logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
logrus.Infof("Extracting Kubeconfig for '%s' cluster", name)
// retry every 5s for at most 2m, or 25 times
availableBackoff := wait.Backoff{
@@ -173,8 +188,10 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
func newCluster(name, namespace string, config *CreateConfig) *v1beta1.Cluster {
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Name: name,
Namespace: namespace,
Labels: parseKeyValuePairs(config.labels, "label"),
Annotations: parseKeyValuePairs(config.annotations, "annotation"),
},
TypeMeta: metav1.TypeMeta{
Kind: "Cluster",
@@ -257,7 +274,18 @@ func env(envSlice []string) []v1.EnvVar {
return envVars
}
func waitForCluster(ctx context.Context, k8sClient client.Client, cluster *v1beta1.Cluster, timeout time.Duration) error {
func waitForClusterReconciled(ctx context.Context, k8sClient client.Client, cluster *v1beta1.Cluster, timeout time.Duration) error {
return wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, func(ctx context.Context) (bool, error) {
key := client.ObjectKeyFromObject(cluster)
if err := k8sClient.Get(ctx, key, cluster); err != nil {
return false, fmt.Errorf("failed to get resource: %w", err)
}
return cluster.Status.HostVersion != "", nil
})
}
func waitForClusterReady(ctx context.Context, k8sClient client.Client, cluster *v1beta1.Cluster, timeout time.Duration) error {
interval := 5 * time.Second
return wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
@@ -341,3 +369,73 @@ func caCertSecret(certName, clusterName, clusterNamespace string, cert, key []by
},
}
}
func parseKeyValuePairs(pairs []string, pairType string) map[string]string {
resultMap := make(map[string]string)
for _, p := range pairs {
var k, v string
keyValue := strings.SplitN(p, "=", 2)
k = keyValue[0]
if len(keyValue) == 2 {
v = keyValue[1]
}
resultMap[k] = v
logrus.Debugf("Adding '%s=%s' %s to Cluster", k, v, pairType)
}
return resultMap
}
const clusterDetailsTemplate = `Cluster details:
Mode: {{ .Mode }}
Servers: {{ .Servers }}{{ if .Agents }}
Agents: {{ .Agents }}{{ end }}
Version: {{ if .Version }}{{ .Version }}{{ else }}{{ .HostVersion }}{{ end }} (Host: {{ .HostVersion }})
Persistence:
Type: {{.Persistence.Type}}{{ if .Persistence.StorageClassName }}
StorageClass: {{ .Persistence.StorageClassName }}{{ end }}{{ if .Persistence.StorageRequestSize }}
Size: {{ .Persistence.StorageRequestSize }}{{ end }}`
func printClusterDetails(cluster *v1beta1.Cluster) (string, error) {
type templateData struct {
Mode v1beta1.ClusterMode
Servers int32
Agents int32
Version string
HostVersion string
Persistence struct {
Type v1beta1.PersistenceMode
StorageClassName string
StorageRequestSize string
}
}
data := templateData{
Mode: cluster.Spec.Mode,
Servers: ptr.Deref(cluster.Spec.Servers, 0),
Agents: ptr.Deref(cluster.Spec.Agents, 0),
Version: cluster.Spec.Version,
HostVersion: cluster.Status.HostVersion,
}
data.Persistence.Type = cluster.Spec.Persistence.Type
data.Persistence.StorageClassName = ptr.Deref(cluster.Spec.Persistence.StorageClassName, "")
data.Persistence.StorageRequestSize = cluster.Spec.Persistence.StorageRequestSize
tmpl, err := template.New("clusterDetails").Parse(clusterDetailsTemplate)
if err != nil {
return "", err
}
var buf bytes.Buffer
if err = tmpl.Execute(&buf, data); err != nil {
return "", err
}
return buf.String(), nil
}

View File

@@ -17,13 +17,15 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) {
cmd.Flags().StringVar(&cfg.clusterCIDR, "cluster-cidr", "", "cluster CIDR")
cmd.Flags().StringVar(&cfg.serviceCIDR, "service-cidr", "", "service CIDR")
cmd.Flags().BoolVar(&cfg.mirrorHostNodes, "mirror-host-nodes", false, "Mirror Host Cluster Nodes")
cmd.Flags().StringVar(&cfg.persistenceType, "persistence-type", string(v1beta1.DynamicPersistenceMode), "persistence mode for the nodes (dynamic, ephemeral, static)")
cmd.Flags().StringVar(&cfg.persistenceType, "persistence-type", string(v1beta1.DynamicPersistenceMode), "persistence mode for the nodes (dynamic, ephemeral)")
cmd.Flags().StringVar(&cfg.storageClassName, "storage-class-name", "", "storage class name for dynamic persistence type")
cmd.Flags().StringVar(&cfg.storageRequestSize, "storage-request-size", "", "storage size for dynamic persistence type")
cmd.Flags().StringSliceVar(&cfg.serverArgs, "server-args", []string{}, "servers extra arguments")
cmd.Flags().StringSliceVar(&cfg.agentArgs, "agent-args", []string{}, "agents extra arguments")
cmd.Flags().StringSliceVar(&cfg.serverEnvs, "server-envs", []string{}, "servers extra Envs")
cmd.Flags().StringSliceVar(&cfg.agentEnvs, "agent-envs", []string{}, "agents extra Envs")
cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)")
cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)")
cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version")
cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)")
cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host")
@@ -42,7 +44,7 @@ func validateCreateConfig(cfg *CreateConfig) error {
case v1beta1.EphemeralPersistenceMode, v1beta1.DynamicPersistenceMode:
return nil
default:
return errors.New(`persistence-type should be one of "dynamic", "ephemeral" or "static"`)
return errors.New(`persistence-type should be one of "dynamic" or "ephemeral"`)
}
}

View File

@@ -0,0 +1,95 @@
package cmds
import (
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/utils/ptr"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
func Test_printClusterDetails(t *testing.T) {
tests := []struct {
name string
cluster *v1beta1.Cluster
want string
wantErr bool
}{
{
name: "simple cluster",
cluster: &v1beta1.Cluster{
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
Version: "123",
Persistence: v1beta1.PersistenceConfig{
Type: v1beta1.DynamicPersistenceMode,
},
},
Status: v1beta1.ClusterStatus{
HostVersion: "456",
},
},
want: `Cluster details:
Mode: shared
Servers: 0
Version: 123 (Host: 456)
Persistence:
Type: dynamic`,
},
{
name: "simple cluster with no version",
cluster: &v1beta1.Cluster{
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
Persistence: v1beta1.PersistenceConfig{
Type: v1beta1.DynamicPersistenceMode,
},
},
Status: v1beta1.ClusterStatus{
HostVersion: "456",
},
},
want: `Cluster details:
Mode: shared
Servers: 0
Version: 456 (Host: 456)
Persistence:
Type: dynamic`,
},
{
name: "cluster with agents",
cluster: &v1beta1.Cluster{
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
Agents: ptr.To[int32](3),
Persistence: v1beta1.PersistenceConfig{
Type: v1beta1.DynamicPersistenceMode,
StorageClassName: ptr.To("local-path"),
StorageRequestSize: "3gb",
},
},
Status: v1beta1.ClusterStatus{
HostVersion: "456",
},
},
want: `Cluster details:
Mode: shared
Servers: 0
Agents: 3
Version: 456 (Host: 456)
Persistence:
Type: dynamic
StorageClass: local-path
Size: 3gb`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clusterDetails, err := printClusterDetails(tt.cluster)
assert.NoError(t, err)
assert.Equal(t, tt.want, clusterDetails)
})
}
}

View File

@@ -48,7 +48,7 @@ func delete(appCtx *AppContext) func(cmd *cobra.Command, args []string) error {
namespace := appCtx.Namespace(name)
logrus.Infof("Deleting [%s] cluster in namespace [%s]", name, namespace)
logrus.Infof("Deleting '%s' cluster in namespace '%s'", name, namespace)
cluster := v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{

View File

@@ -18,7 +18,11 @@ import (
)
type VirtualClusterPolicyCreateConfig struct {
mode string
mode string
labels []string
annotations []string
namespaces []string
overwrite bool
}
func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
@@ -41,6 +45,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
}
cmd.Flags().StringVar(&config.mode, "mode", "shared", "The allowed mode type of the policy")
cmd.Flags().StringArrayVar(&config.labels, "labels", []string{}, "Labels to add to the policy object (e.g. key=value)")
cmd.Flags().StringArrayVar(&config.annotations, "annotations", []string{}, "Annotations to add to the policy object (e.g. key=value)")
cmd.Flags().StringSliceVar(&config.namespaces, "namespace", []string{}, "The namespaces where to bind the policy")
cmd.Flags().BoolVar(&config.overwrite, "overwrite", false, "Overwrite namespace binding of existing policy")
return cmd
}
@@ -51,9 +59,12 @@ func policyCreateAction(appCtx *AppContext, config *VirtualClusterPolicyCreateCo
client := appCtx.Client
policyName := args[0]
_, err := createPolicy(ctx, client, v1beta1.ClusterMode(config.mode), policyName)
_, err := createPolicy(ctx, client, config, policyName)
if err != nil {
return err
}
return err
return bindPolicyToNamespaces(ctx, client, config, policyName)
}
}
@@ -71,7 +82,7 @@ func createNamespace(ctx context.Context, client client.Client, name, policyName
return err
}
logrus.Infof(`Creating namespace [%s]`, name)
logrus.Infof(`Creating namespace '%s'`, name)
if err := client.Create(ctx, ns); err != nil {
return err
@@ -81,19 +92,21 @@ func createNamespace(ctx context.Context, client client.Client, name, policyName
return nil
}
func createPolicy(ctx context.Context, client client.Client, mode v1beta1.ClusterMode, policyName string) (*v1beta1.VirtualClusterPolicy, error) {
logrus.Infof("Creating policy [%s]", policyName)
func createPolicy(ctx context.Context, client client.Client, config *VirtualClusterPolicyCreateConfig, policyName string) (*v1beta1.VirtualClusterPolicy, error) {
logrus.Infof("Creating policy '%s'", policyName)
policy := &v1beta1.VirtualClusterPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: policyName,
Name: policyName,
Labels: parseKeyValuePairs(config.labels, "label"),
Annotations: parseKeyValuePairs(config.annotations, "annotation"),
},
TypeMeta: metav1.TypeMeta{
Kind: "VirtualClusterPolicy",
APIVersion: "k3k.io/v1beta1",
},
Spec: v1beta1.VirtualClusterPolicySpec{
AllowedMode: mode,
AllowedMode: v1beta1.ClusterMode(config.mode),
},
}
@@ -102,8 +115,67 @@ func createPolicy(ctx context.Context, client client.Client, mode v1beta1.Cluste
return nil, err
}
logrus.Infof("Policy [%s] already exists", policyName)
logrus.Infof("Policy '%s' already exists", policyName)
}
return policy, nil
}
func bindPolicyToNamespaces(ctx context.Context, client client.Client, config *VirtualClusterPolicyCreateConfig, policyName string) error {
var errs []error
for _, namespace := range config.namespaces {
var ns v1.Namespace
if err := client.Get(ctx, types.NamespacedName{Name: namespace}, &ns); err != nil {
if apierrors.IsNotFound(err) {
logrus.Warnf(`Namespace '%s' not found, skipping`, namespace)
} else {
errs = append(errs, err)
}
continue
}
if ns.Labels == nil {
ns.Labels = map[string]string{}
}
oldPolicy := ns.Labels[policy.PolicyNameLabelKey]
// same policy found, no need to update
if oldPolicy == policyName {
logrus.Debugf(`Policy '%s' already bound to namespace '%s'`, policyName, namespace)
continue
}
// no old policy, safe to update
if oldPolicy == "" {
if err := client.Update(ctx, &ns); err != nil {
errs = append(errs, err)
} else {
logrus.Infof(`Added policy '%s' to namespace '%s'`, policyName, namespace)
}
continue
}
// different policy, warn or check for overwrite flag
if oldPolicy != policyName {
if config.overwrite {
logrus.Infof(`Found policy '%s' bound to namespace '%s'. Overwriting it with '%s'`, oldPolicy, namespace, policyName)
ns.Labels[policy.PolicyNameLabelKey] = policyName
if err := client.Update(ctx, &ns); err != nil {
errs = append(errs, err)
} else {
logrus.Infof(`Added policy '%s' to namespace '%s'`, policyName, namespace)
}
} else {
logrus.Warnf(`Found policy '%s' bound to namespace '%s'. Skipping. To overwrite it use the --overwrite flag`, oldPolicy, namespace)
}
}
}
return errors.Join(errs...)
}

View File

@@ -31,13 +31,17 @@ func policyDeleteAction(appCtx *AppContext) func(cmd *cobra.Command, args []stri
policy.Name = name
if err := client.Delete(ctx, policy); err != nil {
if apierrors.IsNotFound(err) {
logrus.Warnf("Policy not found")
} else {
if !apierrors.IsNotFound(err) {
return err
}
logrus.Warnf("Policy '%s' not found", name)
return nil
}
logrus.Infof("Policy '%s' deleted", name)
return nil
}
}

View File

@@ -34,9 +34,10 @@ func NewRootCmd() *cobra.Command {
appCtx := &AppContext{}
rootCmd := &cobra.Command{
Use: "k3kcli",
Short: "CLI for K3K",
Version: buildinfo.Version,
SilenceUsage: true,
Use: "k3kcli",
Short: "CLI for K3K",
Version: buildinfo.Version,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
InitializeConfig(cmd)

View File

@@ -18,14 +18,16 @@ k3kcli cluster create [command options] NAME
--agent-args strings agents extra arguments
--agent-envs strings agents extra Envs
--agents int number of agents
--annotations stringArray Annotations to add to the cluster object (e.g. key=value)
--cluster-cidr string cluster CIDR
--custom-certs string The path for custom certificate directory
-h, --help help for create
--kubeconfig-server string override the kubeconfig server host
--labels stringArray Labels to add to the cluster object (e.g. key=value)
--mirror-host-nodes Mirror Host Cluster Nodes
--mode string k3k mode type (shared, virtual) (default "shared")
-n, --namespace string namespace of the k3k cluster
--persistence-type string persistence mode for the nodes (dynamic, ephemeral, static) (default "dynamic")
--persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic")
--policy string The policy to create the cluster in
--server-args strings servers extra arguments
--server-envs strings servers extra Envs

View File

@@ -15,8 +15,12 @@ k3kcli policy create [command options] NAME
### Options
```
-h, --help help for create
--mode string The allowed mode type of the policy (default "shared")
--annotations stringArray Annotations to add to the policy object (e.g. key=value)
-h, --help help for create
--labels stringArray Labels to add to the policy object (e.g. key=value)
--mode string The allowed mode type of the policy (default "shared")
--namespace strings The namespaces where to bind the policy
--overwrite Overwrite namespace binding of existing policy
```
### Options inherited from parent commands

63
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/rancher/k3k
go 1.24.2
go 1.24.10
replace (
github.com/google/cel-go => github.com/google/cel-go v0.20.1
@@ -18,8 +18,10 @@ require (
github.com/onsi/gomega v1.36.0
github.com/rancher/dynamiclistener v1.27.5
github.com/sirupsen/logrus v1.9.3
github.com/spf13/viper v1.20.1
github.com/stretchr/testify v1.10.0
github.com/spf13/cobra v1.10.1
github.com/spf13/pflag v1.0.10
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/testcontainers/testcontainers-go v0.35.0
github.com/testcontainers/testcontainers-go/modules/k3s v0.35.0
github.com/virtual-kubelet/virtual-kubelet v1.11.1-0.20250530103808-c9f64e872803
@@ -28,17 +30,17 @@ require (
go.uber.org/zap v1.27.0
gopkg.in/yaml.v2 v2.4.0
helm.sh/helm/v3 v3.14.4
k8s.io/api v0.31.4
k8s.io/apiextensions-apiserver v0.31.4
k8s.io/apimachinery v0.31.4
k8s.io/apiserver v0.31.4
k8s.io/cli-runtime v0.31.4
k8s.io/client-go v0.31.4
k8s.io/component-base v0.31.4
k8s.io/component-helpers v0.31.4
k8s.io/kubectl v0.31.4
k8s.io/kubelet v0.31.4
k8s.io/kubernetes v1.31.4
k8s.io/api v0.31.13
k8s.io/apiextensions-apiserver v0.31.13
k8s.io/apimachinery v0.31.13
k8s.io/apiserver v0.31.13
k8s.io/cli-runtime v0.31.13
k8s.io/client-go v0.31.13
k8s.io/component-base v0.31.13
k8s.io/component-helpers v0.31.13
k8s.io/kubectl v0.31.13
k8s.io/kubelet v0.31.13
k8s.io/kubernetes v1.31.13
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
sigs.k8s.io/controller-runtime v0.19.4
)
@@ -86,7 +88,7 @@ require (
github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
@@ -96,7 +98,7 @@ require (
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -153,7 +155,7 @@ require (
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -164,15 +166,13 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rubenv/sql-migrate v1.7.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.7.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.12.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/cobra v1.9.1
github.com/spf13/pflag v1.0.6
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -196,16 +196,17 @@ require (
go.opentelemetry.io/otel/trace v1.33.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.40.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/net v0.42.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.26.0 // indirect
golang.org/x/tools v0.35.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 // indirect
@@ -216,7 +217,7 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/klog/v2 v2.130.1
k8s.io/kms v0.31.4 // indirect
k8s.io/kms v0.31.13 // indirect
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
oras.land/oras-go v1.2.5 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect

117
go.sum
View File

@@ -137,8 +137,8 @@ github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6
github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -166,8 +166,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -353,8 +353,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
@@ -388,8 +388,8 @@ github.com/rubenv/sql-migrate v1.7.1 h1:f/o0WgfO/GqNuVg+6801K/KW3WdDSupzSjDYODmi
github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2AS+ezLDFb4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
@@ -404,18 +404,19 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -433,8 +434,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo=
@@ -520,6 +522,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -527,8 +531,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
@@ -560,8 +565,9 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
@@ -578,8 +584,9 @@ golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -603,8 +610,9 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
@@ -617,8 +625,9 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -633,8 +642,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -647,8 +656,8 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk=
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc=
google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg=
google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic=
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q=
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 h1:TqExAhdPaB60Ux47Cn0oLV07rGnxZzIsaRhQaqS666A=
@@ -700,34 +709,34 @@ helm.sh/helm/v3 v3.14.4 h1:6FSpEfqyDalHq3kUr4gOMThhgY55kXUEjdQoyODYnrM=
helm.sh/helm/v3 v3.14.4/go.mod h1:Tje7LL4gprZpuBNTbG34d1Xn5NmRT3OWfBRwpOSer9I=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM=
k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw=
k8s.io/apiextensions-apiserver v0.31.4 h1:FxbqzSvy92Ca9DIs5jqot883G0Ln/PGXfm/07t39LS0=
k8s.io/apiextensions-apiserver v0.31.4/go.mod h1:hIW9YU8UsqZqIWGG99/gsdIU0Ar45Qd3A12QOe/rvpg=
k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM=
k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
k8s.io/apiserver v0.31.4 h1:JbtnTaXVYEAYIHJil6Wd74Wif9sd8jVcBw84kwEmp7o=
k8s.io/apiserver v0.31.4/go.mod h1:JJjoTjZ9PTMLdIFq7mmcJy2B9xLN3HeAUebW6xZyIP0=
k8s.io/cli-runtime v0.31.4 h1:iczCWiyXaotW+hyF5cWP8RnEYBCzZfJUF6otJ2m9mw0=
k8s.io/cli-runtime v0.31.4/go.mod h1:0/pRzAH7qc0hWx40ut1R4jLqiy2w/KnbqdaAI2eFG8U=
k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ=
k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg=
k8s.io/component-base v0.31.4 h1:wCquJh4ul9O8nNBSB8N/o8+gbfu3BVQkVw9jAUY/Qtw=
k8s.io/component-base v0.31.4/go.mod h1:G4dgtf5BccwiDT9DdejK0qM6zTK0jwDGEKnCmb9+u/s=
k8s.io/component-helpers v0.31.4 h1:pqokuXozyWVrVBMmx0AMcKqNWqXhR00OZvpAE5hG5NM=
k8s.io/component-helpers v0.31.4/go.mod h1:Ddq5GYRK/1uNoPNgJh9N5osPutvBweQEcIG6b8kcvgQ=
k8s.io/api v0.31.13 h1:sco9Cq2pY4Ysv9qZiWzcR97MmA/35nwYQ/VCTzOcWmc=
k8s.io/api v0.31.13/go.mod h1:4D8Ry8RqqLDemNLwGYC6v5wOy51N7hitr4WQ6oSWfLY=
k8s.io/apiextensions-apiserver v0.31.13 h1:8xtWKVpV/YbYX0UX2k6w+cgxfxKhX0UNGuo/VXAdg8g=
k8s.io/apiextensions-apiserver v0.31.13/go.mod h1:zxpMLWXBxnJqKUIruJ+ulP+Xlfe5lPZPxq1z0cLwA2U=
k8s.io/apimachinery v0.31.13 h1:rkG0EiBkBkEzURo/8dKGx/oBF202Z2LqHuSD8Cm3bG4=
k8s.io/apimachinery v0.31.13/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
k8s.io/apiserver v0.31.13 h1:Ke9/X2m3vHSgsminpAbUxULDNMbvAfjrRX73Gqx6CZc=
k8s.io/apiserver v0.31.13/go.mod h1:5nBPhL2g7am/CS+/OI5A6+olEbo0C7tQ8QNDODLd+WY=
k8s.io/cli-runtime v0.31.13 h1:oz37PuIe4JyUDfTf8JKcZye1obyYAwF146gRpcj+AR8=
k8s.io/cli-runtime v0.31.13/go.mod h1:x6QI7U97fvrplKgd3JEvCpoZKR9AorjvDjBEr1GZG+g=
k8s.io/client-go v0.31.13 h1:Q0LG51uFbzNd9fzIj5ilA0Sm1wUholHvDaNwVKzqdCA=
k8s.io/client-go v0.31.13/go.mod h1:UB4yTzQeRAv+vULOKp2jdqA5LSwV55bvc3RQ5tM48LM=
k8s.io/component-base v0.31.13 h1:/uVLq7yHk9azReqeCFAZSr/8NXydzpz7yDZ6p/yiwBQ=
k8s.io/component-base v0.31.13/go.mod h1:uMXtKNyDqeNdZYL6SRCr9wB6FutL9pOlQmkK2dRVAKQ=
k8s.io/component-helpers v0.31.13 h1:Yy7j+Va7u6v0DXaKqMEOfIcq5pFnvzFcSGM58/lskeA=
k8s.io/component-helpers v0.31.13/go.mod h1:nXTLwkwCjXcrPG62D0IYiKuKi6JkFM2mBe2myrOUeug=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kms v0.31.4 h1:DVk9T1PHxG7IUMfWs1sDhBTbzGnM7lhMJO8lOzOzTIs=
k8s.io/kms v0.31.4/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
k8s.io/kms v0.31.13 h1:pJCG79BqdCmGetUsETwKfq+OE/D3M1DdqH14EKQl0lI=
k8s.io/kms v0.31.13/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4=
k8s.io/kubectl v0.31.4 h1:c8Af8xd1VjyoKyWMW0xHv2+tYxEjne8s6OOziMmaD10=
k8s.io/kubectl v0.31.4/go.mod h1:0E0rpXg40Q57wRE6LB9su+4tmwx1IzZrmIEvhQPk0i4=
k8s.io/kubelet v0.31.4 h1:6TokbMv+HnFG7Oe9tVS/J0VPGdC4GnsQZXuZoo7Ixi8=
k8s.io/kubelet v0.31.4/go.mod h1:8ZM5LZyANoVxUtmayUxD/nsl+6GjREo7kSanv8AoL4U=
k8s.io/kubernetes v1.31.4 h1:VQDX52gTQnq8C/jCo48AQuDsWbWMh9XXxhQRDYjgakw=
k8s.io/kubernetes v1.31.4/go.mod h1:9xmT2buyTYj8TRKwRae7FcuY8k5+xlxv7VivvO0KKfs=
k8s.io/kubectl v0.31.13 h1:VcSyzFsZ7Fi991FzK80hy+9clUIhChbnQg2L6eZRQzA=
k8s.io/kubectl v0.31.13/go.mod h1:IxUKvsKrvqEL7NcaBCQCVDLzcYghu8b9yMiYKx8nYho=
k8s.io/kubelet v0.31.13 h1:wN9NXmj9DRFTMph1EhAtdQ6+UfEHKV3B7XMKcJr122c=
k8s.io/kubelet v0.31.13/go.mod h1:DxEqJViO7GE5dZXvEJGsP5HORNTSj9MhMQi1JDirCQs=
k8s.io/kubernetes v1.31.13 h1:c/YugS3TqG6YQMNRclrcWVabgIuqyap++lM5AuCtD5M=
k8s.io/kubernetes v1.31.13/go.mod h1:9xmT2buyTYj8TRKwRae7FcuY8k5+xlxv7VivvO0KKfs=
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=

View File

@@ -92,7 +92,7 @@ func NewTestEnv() *TestEnv {
By("bootstrapping test environment")
testEnv := &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
ErrorIfCRDPathMissing: true,
BinaryAssetsDirectory: tempDir,
Scheme: buildScheme(),

View File

@@ -86,7 +86,7 @@ func New(hostConfig rest.Config, hostMgr, virtualMgr manager.Manager, logger log
CoreClient: coreClient,
ClusterNamespace: namespace,
ClusterName: name,
logger: logger,
logger: logger.WithValues("cluster", name),
serverIP: serverIP,
dnsIP: dnsIP,
}
@@ -95,8 +95,12 @@ func New(hostConfig rest.Config, hostMgr, virtualMgr manager.Manager, logger log
}
// GetContainerLogs retrieves the logs of a container by name from the provider.
func (p *Provider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
hostPodName := p.Translator.TranslateName(namespace, podName)
func (p *Provider) GetContainerLogs(ctx context.Context, namespace, name, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
hostPodName := p.Translator.TranslateName(namespace, name)
logger := p.logger.WithValues("namespace", namespace, "name", name, "pod", hostPodName, "container", containerName)
logger.V(1).Info("GetContainerLogs")
options := corev1.PodLogOptions{
Container: containerName,
Timestamps: opts.Timestamps,
@@ -125,20 +129,27 @@ func (p *Provider) GetContainerLogs(ctx context.Context, namespace, podName, con
}
closer, err := p.CoreClient.Pods(p.ClusterNamespace).GetLogs(hostPodName, &options).Stream(ctx)
p.logger.Error(err, fmt.Sprintf("got error when getting logs for %s in %s", hostPodName, p.ClusterNamespace))
if err != nil {
logger.Error(err, "Error getting logs from container")
}
return closer, err
}
// RunInContainer executes a command in a container in the pod, copying data
// between in/out/err and the container's stdin/stdout/stderr.
func (p *Provider) RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error {
hostPodName := p.Translator.TranslateName(namespace, podName)
func (p *Provider) RunInContainer(ctx context.Context, namespace, name, containerName string, cmd []string, attach api.AttachIO) error {
hostPodName := p.Translator.TranslateName(namespace, name)
logger := p.logger.WithValues("namespace", namespace, "name", name, "pod", hostPodName, "container", containerName)
logger.V(1).Info("RunInContainer")
req := p.CoreClient.RESTClient().Post().
Resource("pods").
Name(hostPodName).
Namespace(p.ClusterNamespace).
SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
Container: containerName,
Command: cmd,
@@ -150,10 +161,11 @@ func (p *Provider) RunInContainer(ctx context.Context, namespace, podName, conta
exec, err := remotecommand.NewSPDYExecutor(&p.ClientConfig, http.MethodPost, req.URL())
if err != nil {
logger.Error(err, "Error creating SPDY executor")
return err
}
return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
if err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
Stdin: attach.Stdin(),
Stdout: attach.Stdout(),
Stderr: attach.Stderr(),
@@ -161,18 +173,28 @@ func (p *Provider) RunInContainer(ctx context.Context, namespace, podName, conta
TerminalSizeQueue: &translatorSizeQueue{
resizeChan: attach.Resize(),
},
})
}); err != nil {
logger.Error(err, "Error while executing command in container")
return err
}
return nil
}
// AttachToContainer attaches to the executing process of a container in the pod, copying data
// between in/out/err and the container's stdin/stdout/stderr.
func (p *Provider) AttachToContainer(ctx context.Context, namespace, podName, containerName string, attach api.AttachIO) error {
hostPodName := p.Translator.TranslateName(namespace, podName)
func (p *Provider) AttachToContainer(ctx context.Context, namespace, name, containerName string, attach api.AttachIO) error {
hostPodName := p.Translator.TranslateName(namespace, name)
logger := p.logger.WithValues("namespace", namespace, "name", name, "pod", hostPodName, "container", containerName)
logger.V(1).Info("AttachToContainer")
req := p.CoreClient.RESTClient().Post().
Resource("pods").
Name(hostPodName).
Namespace(p.ClusterNamespace).
SubResource("attach")
req.VersionedParams(&corev1.PodAttachOptions{
Container: containerName,
TTY: attach.TTY(),
@@ -183,10 +205,11 @@ func (p *Provider) AttachToContainer(ctx context.Context, namespace, podName, co
exec, err := remotecommand.NewSPDYExecutor(&p.ClientConfig, http.MethodPost, req.URL())
if err != nil {
logger.Error(err, "Error creating SPDY executor")
return err
}
return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
if err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
Stdin: attach.Stdin(),
Stdout: attach.Stdout(),
Stderr: attach.Stderr(),
@@ -194,7 +217,12 @@ func (p *Provider) AttachToContainer(ctx context.Context, namespace, podName, co
TerminalSizeQueue: &translatorSizeQueue{
resizeChan: attach.Resize(),
},
})
}); err != nil {
logger.Error(err, "Error while attaching to container")
return err
}
return nil
}
// GetStatsSummary gets the stats for the node, including running pods
@@ -203,7 +231,8 @@ func (p *Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
nodeList := &corev1.NodeList{}
if err := p.CoreClient.RESTClient().Get().Resource("nodes").Do(ctx).Into(nodeList); err != nil {
return nil, fmt.Errorf("unable to get nodes of cluster %s in namespace %s: %w", p.ClusterName, p.ClusterNamespace, err)
p.logger.Error(err, "Unable to get nodes of cluster")
return nil, err
}
// fetch the stats from all the nodes
@@ -221,14 +250,13 @@ func (p *Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
Suffix("stats/summary").
DoRaw(ctx)
if err != nil {
return nil, fmt.Errorf(
"unable to get stats of node '%s', from cluster %s in namespace %s: %w",
n.Name, p.ClusterName, p.ClusterNamespace, err,
)
p.logger.Error(err, "Unable to get stats/summary from cluster node", "node", n.Name)
return nil, err
}
stats := &stats.Summary{}
if err := json.Unmarshal(res, stats); err != nil {
p.logger.Error(err, "Error unmarshaling stats/summary from cluster node", "node", n.Name)
return nil, err
}
@@ -241,6 +269,7 @@ func (p *Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
pods, err := p.GetPods(ctx)
if err != nil {
p.logger.Error(err, "Error getting pods from cluster for stats")
return nil, err
}
@@ -278,9 +307,12 @@ func (p *Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
// GetMetricsResource gets the metrics for the node, including running pods
func (p *Provider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
p.logger.V(1).Info("GetMetricsResource")
statsSummary, err := p.GetStatsSummary(ctx)
if err != nil {
return nil, errors.Join(err, errors.New("error fetching MetricsResource"))
p.logger.Error(err, "Error getting stats summary from cluster for metrics")
return nil, err
}
registry := compbasemetrics.NewKubeRegistry()
@@ -288,15 +320,20 @@ func (p *Provider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily,
metricFamily, err := registry.Gather()
if err != nil {
return nil, errors.Join(err, errors.New("error gathering metrics from collector"))
p.logger.Error(err, "Error gathering metrics from collector")
return nil, err
}
return metricFamily, nil
}
// PortForward forwards a local port to a port on the pod
func (p *Provider) PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error {
hostPodName := p.Translator.TranslateName(namespace, pod)
func (p *Provider) PortForward(ctx context.Context, namespace, name string, port int32, stream io.ReadWriteCloser) error {
hostPodName := p.Translator.TranslateName(namespace, name)
logger := p.logger.WithValues("namespace", namespace, "name", name, "pod", hostPodName, "port", port)
logger.V(1).Info("PortForward")
req := p.CoreClient.RESTClient().Post().
Resource("pods").
Name(hostPodName).
@@ -305,6 +342,7 @@ func (p *Provider) PortForward(ctx context.Context, namespace, pod string, port
transport, upgrader, err := spdy.RoundTripperFor(&p.ClientConfig)
if err != nil {
logger.Error(err, "Error creating RoundTripper for PortForward")
return err
}
@@ -318,10 +356,16 @@ func (p *Provider) PortForward(ctx context.Context, namespace, pod string, port
// so more work is needed to detect a close and handle that appropriately.
fw, err := portforward.New(dialer, []string{portAsString}, stopChannel, readyChannel, stream, stream)
if err != nil {
logger.Error(err, "Error creating new PortForward")
return err
}
return fw.ForwardPorts()
if err := fw.ForwardPorts(); err != nil {
logger.Error(err, "Error forwarding ports")
return err
}
return nil
}
// CreatePod executes createPod with retry
@@ -331,16 +375,22 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
// createPod takes a Kubernetes Pod and deploys it within the provider.
func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
logger := p.logger.WithValues("namespace", pod.Namespace, "name", pod.Name)
logger.V(1).Info("CreatePod")
// fieldPath envs are not being translated correctly using the virtual kubelet pod controller
// as a workaround we will try to fetch the pod from the virtual cluster and copy over the envSource
var sourcePod corev1.Pod
if err := p.VirtualClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, &sourcePod); err != nil {
logger.Error(err, "Error getting Pod from Virtual Cluster")
return err
}
tPod := sourcePod.DeepCopy()
p.Translator.TranslateTo(tPod)
logger = p.logger.WithValues("pod", tPod.Name)
// get Cluster definition
clusterKey := types.NamespacedName{
Namespace: p.ClusterNamespace,
@@ -350,7 +400,8 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
var cluster v1beta1.Cluster
if err := p.HostClient.Get(ctx, clusterKey, &cluster); err != nil {
return fmt.Errorf("unable to get cluster %s in namespace %s: %w", p.ClusterName, p.ClusterNamespace, err)
logger.Error(err, "Error getting Virtual Cluster definition")
return err
}
// these values shouldn't be set on create
@@ -384,16 +435,18 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
// fieldpath annotations
if err := p.configureFieldPathEnv(&sourcePod, tPod); err != nil {
return fmt.Errorf("unable to fetch fieldpath annotations for pod %s/%s: %w", pod.Namespace, pod.Name, err)
}
// volumes will often refer to resources in the virtual cluster, but instead need to refer to the sync'd
// host cluster version
if err := p.transformVolumes(pod.Namespace, tPod.Spec.Volumes); err != nil {
return fmt.Errorf("unable to sync volumes for pod %s/%s: %w", pod.Namespace, pod.Name, err)
logger.Error(err, "Unable to fetch fieldpath annotations for pod")
return err
}
// volumes will often refer to resources in the virtual cluster
// but instead need to refer to the synced host cluster version
p.transformVolumes(pod.Namespace, tPod.Spec.Volumes)
// sync serviceaccount token to a the host cluster
if err := p.transformTokens(ctx, pod, tPod); err != nil {
return fmt.Errorf("unable to transform tokens for pod %s/%s: %w", pod.Namespace, pod.Name, err)
logger.Error(err, "Unable to transform tokens for pod")
return err
}
for i, imagePullSecret := range tPod.Spec.ImagePullSecrets {
@@ -403,17 +456,20 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
// inject networking information to the pod including the virtual cluster controlplane endpoint
configureNetworking(tPod, pod.Name, pod.Namespace, p.serverIP, p.dnsIP)
p.logger.Info("creating pod",
"host_namespace", tPod.Namespace, "host_name", tPod.Name,
"virtual_namespace", pod.Namespace, "virtual_name", pod.Name,
)
// set ownerReference to the cluster object
if err := controllerutil.SetControllerReference(&cluster, tPod, p.HostClient.Scheme()); err != nil {
logger.Error(err, "Unable to set owner reference for pod")
return err
}
return p.HostClient.Create(ctx, tPod)
if err := p.HostClient.Create(ctx, tPod); err != nil {
logger.Error(err, "Error creating pod on host cluster")
return err
}
logger.Info("Pod created on host cluster")
return nil
}
// withRetry retries passed function with interval and timeout
@@ -445,7 +501,7 @@ func (p *Provider) withRetry(ctx context.Context, f func(context.Context, *corev
// transformVolumes changes the volumes to the representation in the host cluster. Will return an error
// if one/more volumes couldn't be transformed
func (p *Provider) transformVolumes(podNamespace string, volumes []corev1.Volume) error {
func (p *Provider) transformVolumes(podNamespace string, volumes []corev1.Volume) {
for _, volume := range volumes {
if strings.HasPrefix(volume.Name, kubeAPIAccessPrefix) {
continue
@@ -479,8 +535,6 @@ func (p *Provider) transformVolumes(podNamespace string, volumes []corev1.Volume
}
}
}
return nil
}
// UpdatePod executes updatePod with retry
@@ -489,7 +543,10 @@ func (p *Provider) UpdatePod(ctx context.Context, pod *corev1.Pod) error {
}
func (p *Provider) updatePod(ctx context.Context, pod *corev1.Pod) error {
p.logger.V(1).Info("got a request for update pod")
hostPodName := p.Translator.TranslateName(pod.Namespace, pod.Name)
logger := p.logger.WithValues("namespace", pod.Namespace, "name", pod.Name, "pod", hostPodName)
logger.V(1).Info("UpdatePod")
// Once scheduled a Pod cannot update other fields than the image of the containers, initcontainers and a few others
// See: https://kubernetes.io/docs/concepts/workloads/pods/#pod-update-and-replacement
@@ -498,28 +555,31 @@ func (p *Provider) updatePod(ctx context.Context, pod *corev1.Pod) error {
var currentVirtualPod corev1.Pod
if err := p.VirtualClient.Get(ctx, client.ObjectKeyFromObject(pod), &currentVirtualPod); err != nil {
return fmt.Errorf("unable to get pod to update from virtual cluster: %w", err)
logger.Error(err, "Unable to get pod to update from virtual cluster")
return err
}
hostNamespaceName := types.NamespacedName{
Namespace: p.ClusterNamespace,
Name: p.Translator.TranslateName(pod.Namespace, pod.Name),
Name: hostPodName,
}
var currentHostPod corev1.Pod
logger = logger.WithValues()
var currentHostPod corev1.Pod
if err := p.HostClient.Get(ctx, hostNamespaceName, &currentHostPod); err != nil {
return fmt.Errorf("unable to get pod to update from host cluster: %w", err)
logger.Error(err, "Unable to get Pod to update from host cluster")
return err
}
// Handle ephemeral containers
if !cmp.Equal(currentHostPod.Spec.EphemeralContainers, pod.Spec.EphemeralContainers) {
p.logger.Info("Updating ephemeral containers")
logger.V(1).Info("Updating ephemeral containers")
currentHostPod.Spec.EphemeralContainers = pod.Spec.EphemeralContainers
if _, err := p.CoreClient.Pods(p.ClusterNamespace).UpdateEphemeralContainers(ctx, currentHostPod.Name, &currentHostPod, metav1.UpdateOptions{}); err != nil {
p.logger.Error(err, "error when updating ephemeral containers")
logger.Error(err, "error when updating ephemeral containers")
return err
}
@@ -528,7 +588,8 @@ func (p *Provider) updatePod(ctx context.Context, pod *corev1.Pod) error {
// fieldpath annotations
if err := p.configureFieldPathEnv(&currentVirtualPod, &currentHostPod); err != nil {
return fmt.Errorf("unable to fetch fieldpath annotations for pod %s/%s: %w", pod.Namespace, pod.Name, err)
logger.Error(err, "Unable to fetch fieldpath annotations for Pod")
return err
}
currentVirtualPod.Spec.Containers = updateContainerImages(currentVirtualPod.Spec.Containers, pod.Spec.Containers)
@@ -542,7 +603,8 @@ func (p *Provider) updatePod(ctx context.Context, pod *corev1.Pod) error {
currentVirtualPod.Labels = pod.Labels
if err := p.VirtualClient.Update(ctx, &currentVirtualPod); err != nil {
return fmt.Errorf("unable to update pod in the virtual cluster: %w", err)
logger.Error(err, "Unable to update Pod in virtual cluster")
return err
}
// Update Pod in the host cluster
@@ -558,9 +620,12 @@ func (p *Provider) updatePod(ctx context.Context, pod *corev1.Pod) error {
maps.Copy(currentHostPod.Labels, pod.Labels)
if err := p.HostClient.Update(ctx, &currentHostPod); err != nil {
return fmt.Errorf("unable to update pod in the host cluster: %w", err)
logger.Error(err, "Unable to update Pod in host cluster")
return err
}
logger.Info("Pod updated in virtual and host cluster")
return nil
}
@@ -590,20 +655,24 @@ func (p *Provider) DeletePod(ctx context.Context, pod *corev1.Pod) error {
// expected to call the NotifyPods callback with a terminal pod status where all the containers are in a terminal
// state, as well as the pod. DeletePod may be called multiple times for the same pod.
func (p *Provider) deletePod(ctx context.Context, pod *corev1.Pod) error {
p.logger.Info(fmt.Sprintf("got request to delete pod %s/%s", pod.Namespace, pod.Name))
hostName := p.Translator.TranslateName(pod.Namespace, pod.Name)
hostPodName := p.Translator.TranslateName(pod.Namespace, pod.Name)
err := p.CoreClient.Pods(p.ClusterNamespace).Delete(ctx, hostName, metav1.DeleteOptions{})
logger := p.logger.WithValues("namespace", pod.Namespace, "name", pod.Name, "pod", hostPodName)
logger.V(1).Info("DeletePod")
err := p.CoreClient.Pods(p.ClusterNamespace).Delete(ctx, hostPodName, metav1.DeleteOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
p.logger.Info(fmt.Sprintf("pod %s/%s already deleted from host cluster", p.ClusterNamespace, hostName))
logger.Info("Pod to delete not found in host cluster")
return nil
}
return fmt.Errorf("unable to delete pod %s/%s: %w", pod.Namespace, pod.Name, err)
logger.Error(err, "Error trying to delete pod from host cluster")
return err
}
p.logger.Info(fmt.Sprintf("pod %s/%s deleted from host cluster", p.ClusterNamespace, hostName))
logger.Info("Pod deleted from host cluster")
return nil
}
@@ -613,21 +682,18 @@ func (p *Provider) deletePod(ctx context.Context, pod *corev1.Pod) error {
// concurrently outside of the calling goroutine. Therefore it is recommended
// to return a version after DeepCopy.
func (p *Provider) GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error) {
p.logger.V(1).Info("got a request for get pod", "namespace", namespace, "name", name)
hostNamespaceName := types.NamespacedName{
Namespace: p.ClusterNamespace,
Name: p.Translator.TranslateName(namespace, name),
hostPodName := p.Translator.TranslateName(namespace, name)
logger := p.logger.WithValues("namespace", namespace, "name", name, "pod", hostPodName)
logger.V(1).Info("GetPod")
pod, err := p.getPodFromHostCluster(ctx, hostPodName)
if err != nil {
logger.Error(err, "Error getting pod from host cluster for GetPod")
return nil, err
}
var pod corev1.Pod
if err := p.HostClient.Get(ctx, hostNamespaceName, &pod); err != nil {
return nil, fmt.Errorf("error when retrieving pod: %w", err)
}
p.Translator.TranslateFrom(&pod)
return &pod, nil
return pod, nil
}
// GetPodStatus retrieves the status of a pod by name from the provider.
@@ -635,28 +701,49 @@ func (p *Provider) GetPod(ctx context.Context, namespace, name string) (*corev1.
// concurrently outside of the calling goroutine. Therefore it is recommended
// to return a version after DeepCopy.
func (p *Provider) GetPodStatus(ctx context.Context, namespace, name string) (*corev1.PodStatus, error) {
p.logger.V(1).Info("got a request for pod status", "namespace", namespace, "name", name)
hostPodName := p.Translator.TranslateName(namespace, name)
pod, err := p.GetPod(ctx, namespace, name)
logger := p.logger.WithValues("namespace", namespace, "name", name, "pod", hostPodName)
logger.V(1).Info("GetPodStatus")
pod, err := p.getPodFromHostCluster(ctx, hostPodName)
if err != nil {
return nil, fmt.Errorf("unable to get pod for status: %w", err)
logger.Error(err, "Error getting pod from host cluster for PodStatus")
return nil, err
}
p.logger.V(1).Info("got pod status", "namespace", namespace, "name", name, "status", pod.Status)
return pod.Status.DeepCopy(), nil
}
func (p *Provider) getPodFromHostCluster(ctx context.Context, hostPodName string) (*corev1.Pod, error) {
key := types.NamespacedName{
Namespace: p.ClusterNamespace,
Name: hostPodName,
}
var pod corev1.Pod
if err := p.HostClient.Get(ctx, key, &pod); err != nil {
return nil, err
}
p.Translator.TranslateFrom(&pod)
return &pod, nil
}
// GetPods retrieves a list of all pods running on the provider (can be cached).
// The Pods returned are expected to be immutable, and may be accessed
// concurrently outside of the calling goroutine. Therefore it is recommended
// to return a version after DeepCopy.
func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) {
p.logger.V(1).Info("GetPods")
selector := labels.NewSelector()
requirement, err := labels.NewRequirement(translate.ClusterNameLabel, selection.Equals, []string{p.ClusterName})
if err != nil {
return nil, fmt.Errorf("unable to create label selector: %w", err)
p.logger.Error(err, "Error creating label selector for GetPods")
return nil, err
}
selector = selector.Add(*requirement)
@@ -665,7 +752,8 @@ func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) {
err = p.HostClient.List(ctx, &podList, &client.ListOptions{LabelSelector: selector})
if err != nil {
return nil, fmt.Errorf("unable to list pods: %w", err)
p.logger.Error(err, "Error listing pods from host cluster")
return nil, err
}
retPods := []*corev1.Pod{}

View File

@@ -381,6 +381,11 @@ func (s *SharedAgent) role(ctx context.Context) error {
Resources: []string{"persistentvolumeclaims", "pods", "pods/log", "pods/attach", "pods/exec", "pods/ephemeralcontainers", "secrets", "configmaps", "services"},
Verbs: []string{"*"},
},
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{"create"},
},
{
APIGroups: []string{"networking.k8s.io"},
Resources: []string{"ingresses"},

View File

@@ -269,8 +269,8 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
// if the Version is not specified we will try to use the same Kubernetes version of the host.
// This version is stored in the Status object, and it will not be updated if already set.
if cluster.Spec.Version == "" && cluster.Status.HostVersion == "" {
log.V(1).Info("Cluster version not set. Using host version.")
if cluster.Status.HostVersion == "" {
log.V(1).Info("Cluster host version not set.")
hostVersion, err := c.DiscoveryClient.ServerVersion()
if err != nil {
@@ -278,8 +278,9 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
}
// update Status HostVersion
k8sVersion := strings.Split(hostVersion.GitVersion, "+")[0]
cluster.Status.HostVersion = k8sVersion + "-k3s1"
k8sVersion, _, _ := strings.Cut(hostVersion.GitVersion, "+")
k8sVersion, _, _ = strings.Cut(k8sVersion, "-")
cluster.Status.HostVersion = k8sVersion
}
token, err := c.token(ctx, cluster)

View File

@@ -41,7 +41,7 @@ var (
var _ = BeforeSuite(func() {
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
ErrorIfCRDPathMissing: true,
}

View File

@@ -2,7 +2,6 @@ package cluster_test
import (
"context"
"fmt"
"time"
"k8s.io/utils/ptr"
@@ -73,7 +72,6 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
serverVersion, err := k8s.ServerVersion()
Expect(err).To(Not(HaveOccurred()))
expectedHostVersion := fmt.Sprintf("%s-k3s1", serverVersion.GitVersion)
Eventually(func() string {
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
@@ -82,7 +80,7 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
}).
WithTimeout(time.Second * 30).
WithPolling(time.Second).
Should(Equal(expectedHostVersion))
Should(Equal(serverVersion.GitVersion))
// check NetworkPolicy
expectedNetworkPolicy := &networkingv1.NetworkPolicy{

View File

@@ -25,21 +25,24 @@ var Backoff = wait.Backoff{
Jitter: 0.1,
}
// Image returns the rancher/k3s image tagged with the specified Version.
// If Version is empty it will use with the same k8s version of the host cluster,
// stored in the Status object. It will return the latest version as last fallback.
// K3SImage returns the rancher/k3s image tagged with the found K3SVersion.
func K3SImage(cluster *v1beta1.Cluster, k3SImage string) string {
image := k3SImage
imageVersion := "latest"
return k3SImage + ":" + K3SVersion(cluster)
}
// K3SVersion returns the rancher/k3s specified version.
// If empty it will return the k3s version of the Kubernetes version of the host cluster, stored in the Status object.
// Returns the latest version as fallback.
func K3SVersion(cluster *v1beta1.Cluster) string {
if cluster.Spec.Version != "" {
imageVersion = cluster.Spec.Version
} else if cluster.Status.HostVersion != "" {
imageVersion = cluster.Status.HostVersion
return cluster.Spec.Version
}
return image + ":" + imageVersion
if cluster.Status.HostVersion != "" {
return cluster.Status.HostVersion + "-k3s1"
}
return "latest"
}
// SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix.

View File

@@ -51,7 +51,7 @@ func Test_K3S_Image(t *testing.T) {
},
},
},
expectedData: "rancher/k3s:v4.5.6",
expectedData: "rancher/k3s:v4.5.6-k3s1",
},
{
name: "cluster with empty version spec and empty hostVersion status",

View File

@@ -108,13 +108,27 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1bet
ip := k3kService.Spec.ClusterIP
port := int32(443)
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case v1.ServiceTypeNodePort:
ip = hostServerIP
port = k3kService.Spec.Ports[0].NodePort
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case v1.ServiceTypeLoadBalancer:
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
port = k3kService.Spec.Ports[0].Port
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
} else {
logrus.Warn("No ingress found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
@@ -122,7 +136,7 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1bet
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("ip %s not in tlsSANs", ip)
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
@@ -133,7 +147,7 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1bet
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("ip not found in tlsSANs. This could cause issue with the certificate validation.")
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}

View File

@@ -38,7 +38,7 @@ var (
var _ = BeforeSuite(func() {
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
ErrorIfCRDPathMissing: true,
}
cfg, err := testEnv.Start()

View File

@@ -10,4 +10,11 @@ CONTROLLER_TOOLS_VERSION=v0.16.0
go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION} \
crd:generateEmbeddedObjectMeta=true,allowDangerousTypes=false \
object paths=./pkg/apis/... \
output:crd:dir=./charts/k3k/crds
output:crd:dir=./charts/k3k/templates/crds
# add the 'helm.sh/resource-policy: keep' annotation to the CRDs
for f in ./charts/k3k/templates/crds/*.yaml; do
sed -i '0,/^[[:space:]]*annotations:/s/^[[:space:]]*annotations:/&\n helm.sh\/resource-policy: keep/' "$f"
echo "Validating $f"
yq . "$f" > /dev/null
done

View File

@@ -7,10 +7,6 @@ import (
"time"
"k8s.io/apimachinery/pkg/util/rand"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -47,12 +43,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterNamespace := "k3k-" + clusterName
DeferCleanup(func() {
err := k8sClient.Delete(context.Background(), &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: clusterNamespace,
},
})
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
DeleteNamespaces(clusterNamespace)
})
_, stderr, err = K3kcli("cluster", "create", clusterName)
@@ -66,7 +57,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
_, stderr, err = K3kcli("cluster", "delete", clusterName)
Expect(err).To(Not(HaveOccurred()), string(stderr))
Expect(stderr).To(ContainSubstring("Deleting [%s] cluster in namespace [%s]", clusterName, clusterNamespace))
Expect(stderr).To(ContainSubstring(`Deleting '%s' cluster in namespace '%s'`, clusterName, clusterNamespace))
// The deletion could take a bit
Eventually(func() string {
@@ -78,6 +69,24 @@ var _ = When("using the k3kcli", Label("cli"), func() {
WithPolling(time.Second).
Should(BeEmpty())
})
It("can create a cluster with the specified kubernetes version", func() {
var (
stderr string
err error
)
clusterName := "cluster-" + rand.String(5)
clusterNamespace := "k3k-" + clusterName
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
})
_, stderr, err = K3kcli("cluster", "create", "--version", "v1.33.6-k3s1", clusterName)
Expect(err).To(Not(HaveOccurred()), string(stderr))
Expect(stderr).To(ContainSubstring("You can start using the cluster"))
})
})
When("trying the policy commands", func() {
@@ -92,7 +101,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
_, stderr, err = K3kcli("policy", "create", policyName)
Expect(err).To(Not(HaveOccurred()), string(stderr))
Expect(stderr).To(ContainSubstring("Creating policy [%s]", policyName))
Expect(stderr).To(ContainSubstring(`Creating policy '%s'`, policyName))
stdout, stderr, err = K3kcli("policy", "list")
Expect(err).To(Not(HaveOccurred()), string(stderr))
@@ -102,7 +111,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
stdout, stderr, err = K3kcli("policy", "delete", policyName)
Expect(err).To(Not(HaveOccurred()), string(stderr))
Expect(stdout).To(BeEmpty())
Expect(stderr).To(BeEmpty())
Expect(stderr).To(ContainSubstring(`Policy '%s' deleted`, policyName))
stdout, stderr, err = K3kcli("policy", "list")
Expect(err).To(Not(HaveOccurred()), string(stderr))
@@ -122,12 +131,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterNamespace := "k3k-" + clusterName
DeferCleanup(func() {
err := k8sClient.Delete(context.Background(), &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: clusterNamespace,
},
})
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
DeleteNamespaces(clusterNamespace)
})
_, stderr, err = K3kcli("cluster", "create", clusterName)
@@ -140,7 +144,7 @@ var _ = When("using the k3kcli", Label("cli"), func() {
_, stderr, err = K3kcli("cluster", "delete", clusterName)
Expect(err).To(Not(HaveOccurred()), string(stderr))
Expect(stderr).To(ContainSubstring("Deleting [%s] cluster in namespace [%s]", clusterName, clusterNamespace))
Expect(stderr).To(ContainSubstring(`Deleting '%s' cluster in namespace '%s'`, clusterName, clusterNamespace))
})
})
})

View File

@@ -13,7 +13,7 @@ import (
. "github.com/onsi/gomega"
)
var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), Label(certificatesTestsLabel), func() {
var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label(e2eTestLabel), Label(certificatesTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {

View File

@@ -5,7 +5,7 @@ import (
. "github.com/onsi/gomega"
)
var _ = When("two virtual clusters are installed", Label("e2e"), Label(networkingTestsLabel), func() {
var _ = When("two virtual clusters are installed", Label(e2eTestLabel), Label(networkingTestsLabel), func() {
var (
cluster1 *VirtualCluster
cluster2 *VirtualCluster

View File

@@ -19,7 +19,7 @@ import (
. "github.com/onsi/gomega"
)
var _ = When("an ephemeral cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() {
var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(persistenceTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {
@@ -110,7 +110,7 @@ var _ = When("an ephemeral cluster is installed", Label("e2e"), Label(persistenc
})
})
var _ = When("a dynamic cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() {
var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persistenceTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {

176
tests/cluster_pod_test.go Normal file
View File

@@ -0,0 +1,176 @@
package k3k_test
import (
"context"
"time"
"k8s.io/kubernetes/pkg/api/v1/pod"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/k3k-kubelet/translate"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = When("a cluster creates a pod with an invalid configuration", Label(e2eTestLabel), func() {
var (
virtualCluster *VirtualCluster
virtualPod *v1.Pod
)
BeforeEach(func() {
virtualCluster = NewVirtualCluster()
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
})
p := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "nginx-",
Namespace: "default",
Labels: map[string]string{
"name": "var-expansion-test",
},
Annotations: map[string]string{
"notmysubpath": "mypath",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "nginx",
Image: "nginx",
Env: []v1.EnvVar{
{
Name: "POD_NAME",
Value: "nginx",
},
{
Name: "ANNOTATION",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "metadata.annotations['mysubpath']",
},
},
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir",
MountPath: "/volume_mount",
},
{
Name: "workdir",
MountPath: "/subpath_mount",
SubPathExpr: "$(ANNOTATION)/$(POD_NAME)",
},
},
},
},
Volumes: []v1.Volume{{
Name: "workdir",
VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
}},
},
}
ctx := context.Background()
var err error
virtualPod, err = virtualCluster.Client.CoreV1().Pods(p.Namespace).Create(ctx, p, metav1.CreateOptions{})
Expect(err).To(Not(HaveOccurred()))
})
It("should be in Pending status with the CreateContainerConfigError until we fix the annotation", func() {
ctx := context.Background()
By("Checking the container status of the Pod in the Virtual Cluster")
Eventually(func(g Gomega) {
pod, err := virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred())
g.Expect(pod.Status.Phase).To(Equal(v1.PodPending))
containerStatuses := pod.Status.ContainerStatuses
g.Expect(containerStatuses).To(HaveLen(1))
waitingState := containerStatuses[0].State.Waiting
g.Expect(waitingState).NotTo(BeNil())
g.Expect(waitingState.Reason).To(Equal("CreateContainerConfigError"))
}).
WithPolling(time.Second).
WithTimeout(time.Minute).
Should(Succeed())
By("Checking the container status of the Pod in the Host Cluster")
Eventually(func(g Gomega) {
translator := translate.NewHostTranslator(virtualCluster.Cluster)
hostPodName := translator.NamespacedName(virtualPod)
pod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred())
g.Expect(pod.Status.Phase).To(Equal(v1.PodPending))
containerStatuses := pod.Status.ContainerStatuses
g.Expect(containerStatuses).To(HaveLen(1))
waitingState := containerStatuses[0].State.Waiting
g.Expect(waitingState).NotTo(BeNil())
g.Expect(waitingState.Reason).To(Equal("CreateContainerConfigError"))
}).
WithPolling(time.Second).
WithTimeout(time.Minute).
Should(Succeed())
By("Fixing the annotation")
var err error
virtualPod, err = virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
virtualPod.Annotations["mysubpath"] = virtualPod.Annotations["notmysubpath"]
delete(virtualPod.Annotations, "notmysubpath")
virtualPod, err = virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Update(ctx, virtualPod, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Checking the status of the Pod in the Virtual Cluster")
Eventually(func(g Gomega) {
vPod, err := virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred())
_, cond := pod.GetPodCondition(&vPod.Status, v1.PodReady)
g.Expect(cond).NotTo(BeNil())
g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
}).
WithPolling(time.Second).
WithTimeout(time.Minute).
Should(Succeed())
By("Checking the status of the Pod in the Host Cluster")
Eventually(func(g Gomega) {
translator := translate.NewHostTranslator(virtualCluster.Cluster)
hostPodName := translator.NamespacedName(virtualPod)
hPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred())
_, cond := pod.GetPodCondition(&hPod.Status, v1.PodReady)
g.Expect(cond).NotTo(BeNil())
g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
}).
WithPolling(time.Second).
WithTimeout(time.Minute).
Should(Succeed())
})
})

View File

@@ -18,7 +18,7 @@ import (
. "github.com/onsi/gomega"
)
var _ = When("a cluster's status is tracked", Label("e2e"), Label(statusTestsLabel), func() {
var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusTestsLabel), func() {
var (
namespace *corev1.Namespace
vcp *v1beta1.VirtualClusterPolicy

View File

@@ -14,7 +14,7 @@ import (
. "github.com/onsi/gomega"
)
var _ = When("a shared mode cluster is created", Ordered, Label("e2e"), func() {
var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), func() {
var (
virtualCluster *VirtualCluster
virtualConfigMap *corev1.ConfigMap

View File

@@ -18,7 +18,7 @@ import (
. "github.com/onsi/gomega"
)
var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a shared mode cluster update its envs", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var virtualCluster *VirtualCluster
ctx := context.Background()
BeforeEach(func() {
@@ -162,7 +162,7 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(update
})
})
var _ = When("a shared mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a shared mode cluster update its server args", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var virtualCluster *VirtualCluster
ctx := context.Background()
BeforeEach(func() {
@@ -221,7 +221,7 @@ var _ = When("a shared mode cluster update its server args", Label("e2e"), Label
})
})
var _ = When("a virtual mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a virtual mode cluster update its envs", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var virtualCluster *VirtualCluster
ctx := context.Background()
BeforeEach(func() {
@@ -362,7 +362,7 @@ var _ = When("a virtual mode cluster update its envs", Label("e2e"), Label(updat
})
})
var _ = When("a virtual mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a virtual mode cluster update its server args", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var virtualCluster *VirtualCluster
ctx := context.Background()
BeforeEach(func() {
@@ -424,7 +424,7 @@ var _ = When("a virtual mode cluster update its server args", Label("e2e"), Labe
})
})
var _ = When("a shared mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var (
virtualCluster *VirtualCluster
nginxPod *v1.Pod
@@ -510,7 +510,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), Label(upd
})
})
var _ = When("a virtual mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a virtual mode cluster update its version", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var (
virtualCluster *VirtualCluster
nginxPod *v1.Pod
@@ -611,7 +611,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), Label(up
})
})
var _ = When("a shared mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a shared mode cluster scales up servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var (
virtualCluster *VirtualCluster
nginxPod *v1.Pod
@@ -696,7 +696,7 @@ var _ = When("a shared mode cluster scales up servers", Label("e2e"), Label(upda
})
})
var _ = When("a shared mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a shared mode cluster scales down servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var (
virtualCluster *VirtualCluster
nginxPod *v1.Pod
@@ -783,7 +783,7 @@ var _ = When("a shared mode cluster scales down servers", Label("e2e"), Label(up
})
})
var _ = When("a virtual mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a virtual mode cluster scales up servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var (
virtualCluster *VirtualCluster
nginxPod *v1.Pod
@@ -868,7 +868,7 @@ var _ = When("a virtual mode cluster scales up servers", Label("e2e"), Label(upd
})
})
var _ = When("a virtual mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var _ = When("a virtual mode cluster scales down servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
var (
virtualCluster *VirtualCluster
nginxPod *v1.Pod

View File

@@ -17,6 +17,7 @@ import (
"k8s.io/kubectl/pkg/scheme"
"k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -113,24 +114,18 @@ func DeleteNamespaces(names ...string) {
defer wg.Done()
defer GinkgoRecover()
deleteNamespace(name)
By(fmt.Sprintf("Deleting namespace %s", name))
err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
GracePeriodSeconds: ptr.To[int64](0),
})
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
}()
}
wg.Wait()
}
func deleteNamespace(name string) {
GinkgoHelper()
By(fmt.Sprintf("Deleting namespace %s", name))
err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
GracePeriodSeconds: ptr.To[int64](0),
})
Expect(err).To(Not(HaveOccurred()))
}
func NewCluster(namespace string) *v1beta1.Cluster {
return &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{

View File

@@ -42,6 +42,7 @@ import (
const (
k3kNamespace = "k3k-system"
e2eTestLabel = "e2e"
slowTestsLabel = "slow"
updateTestsLabel = "update"
persistenceTestsLabel = "persistence"