diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 35de1c3e..4804f0ae 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -1413,6 +1413,15 @@ spec: MountPath is the path within server and agent pods where the secret contents will be mounted. type: string + name: + description: |- + Name is the name of the secret mount volume that will be used + as the name of volume and volume mount for the server or agent pod + if empty then the secret name will be used instead. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string optional: description: optional field specify whether the Secret or its keys must be defined type: boolean diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index 6d01db49..dde99c7e 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -612,6 +612,12 @@ Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + | | | *`optional`* __boolean__ | optional field specify whether the Secret or its keys must be defined + | | +| *`name`* __string__ | Name is the name of the secret mount volume that will be used + +as the name of volume and volume mount for the server or agent pod + +if empty then the secret name will be used instead. + | | MaxLength: 63 + +MinLength: 1 + +Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + + | *`mountPath`* __string__ | MountPath is the path within server and agent pods where the + secret contents will be mounted. + | | | *`subPath`* __string__ | SubPath is an optional path within the secret to mount instead of the root. + diff --git a/docs/crds/crds.md b/docs/crds/crds.md index 35bd7a27..385c9c83 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -452,6 +452,7 @@ _Appears in:_ | `items` _[KeyToPath](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#keytopath-v1-core) array_ | items If unspecified, each key-value pair in the Data field of the referenced
Secret will be projected into the volume as a file whose name is the
key and content is the value. If specified, the listed keys will be
projected into the specified paths, and unlisted keys will not be
present. If a key is specified which is not present in the Secret,
the volume setup will error unless it is marked optional. Paths must be
relative and may not contain the '..' path or start with '..'. | | | | `defaultMode` _integer_ | defaultMode is Optional: mode bits used to set permissions on created files by default.
Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.
YAML accepts both octal and decimal values, JSON requires decimal values
for mode bits. Defaults to 0644.
Directories within the path are not affected by this setting.
This might be in conflict with other options that affect the file
mode, like fsGroup, and the result can be other mode bits set. | | | | `optional` _boolean_ | optional field specify whether the Secret or its keys must be defined | | | +| `name` _string_ | Name is the name of the secret mount volume that will be used
as the name of volume and volume mount for the server or agent pod
if empty then the secret name will be used instead. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
| | `mountPath` _string_ | MountPath is the path within server and agent pods where the
secret contents will be mounted. | | | | `subPath` _string_ | SubPath is an optional path within the secret to mount instead of the root.
When specified, only the specified key from the secret will be mounted as a file
at MountPath, keeping the parent directory writable. | | | | `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.
This can be 'server', 'agent', or 'all' (for both). | | Enum: [server agent all]
| diff --git a/k3k-kubelet/node.go b/k3k-kubelet/node.go index 26f9a438..c40b6c0f 100644 --- a/k3k-kubelet/node.go +++ b/k3k-kubelet/node.go @@ -78,7 +78,7 @@ func loadTLSConfig(cfg config, token, agentIP, podIP string) (*tls.Config, error }, func() error { var err error - tlsCrt, err = k3s.GetServingKubeletCrt(client) + tlsCrt, err = client.GetServingKubeletCrt() return err }); err != nil { diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 17738d42..43690751 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -241,6 +241,15 @@ type ClusterSpec struct { type SecretMount struct { // Embeds SecretName, Items, DefaultMode, and Optional corev1.SecretVolumeSource `json:",inline"` + // Name is the name of the secret mount volume that will be used + // as the name of volume and volume mount for the server or agent pod + // if empty then the secret name will be used instead. + // + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +optional + Name string `json:"name,omitempty"` // MountPath is the path within server and agent pods where the // secret contents will be mounted. // diff --git a/pkg/controller/cluster/agent/agent.go b/pkg/controller/cluster/agent/agent.go index a17986c2..66f4124b 100644 --- a/pkg/controller/cluster/agent/agent.go +++ b/pkg/controller/cluster/agent/agent.go @@ -29,11 +29,11 @@ type Config struct { scheme *runtime.Scheme } -func NewConfig(cluster *v1beta1.Cluster, client ctrlruntimeclient.Client, scheme *runtime.Scheme) *Config { +func NewConfig(cluster *v1beta1.Cluster, client ctrlruntimeclient.Client) *Config { return &Config{ cluster: cluster, client: client, - scheme: scheme, + scheme: client.Scheme(), } } diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 14ee17a4..e392b9a2 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -12,9 +12,9 @@ import ( "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -87,7 +87,7 @@ type Config struct { type ClusterReconciler struct { DiscoveryClient *discovery.DiscoveryClient Client client.Client - Scheme *runtime.Scheme + RestCfg *rest.Config PortAllocator *agent.PortAllocator record.EventRecorder @@ -113,7 +113,7 @@ func Add(ctx context.Context, mgr manager.Manager, config *Config, maxConcurrent reconciler := ClusterReconciler{ DiscoveryClient: discoveryClient, Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + RestCfg: mgr.GetConfig(), EventRecorder: eventRecorder, PortAllocator: portAllocator, Config: Config{ @@ -462,12 +462,17 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster * log := ctrl.LoggerFrom(ctx) log.V(1).Info("Ensuring bootstrap secret") - data, err := bootstrap.Fetch(ctx, serviceIP, token) + k3sClient := k3s.New(k3s.ClientConfig{ + ServerIP: serviceIP, + Token: token, + }) + + data, err := bootstrap.Fetch(ctx, k3sClient, cluster, c.RestCfg) if err != nil { return err } - return bootstrap.SaveToSecret(ctx, c.Client, c.Scheme, cluster, data) + return bootstrap.SaveToSecret(ctx, c.Client, cluster, data) } // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server @@ -495,7 +500,7 @@ func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster } _, err = controllerutil.CreateOrUpdate(ctx, c.Client, kubeconfigSecret, func() error { - if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Client.Scheme()); err != nil { return err } @@ -516,7 +521,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v return err } - if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Client.Scheme()); err != nil { return err } @@ -532,7 +537,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v return err } - if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Client.Scheme()); err != nil { return err } @@ -625,7 +630,7 @@ func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1 currentNetworkPolicy := expectedNetworkPolicy.DeepCopy() result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentNetworkPolicy, func() error { - if err := controllerutil.SetControllerReference(cluster, currentNetworkPolicy, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, currentNetworkPolicy, c.Client.Scheme()); err != nil { return err } @@ -653,7 +658,7 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v currentService := expectedService.DeepCopy() result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error { - if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, currentService, c.Client.Scheme()); err != nil { return err } @@ -688,7 +693,7 @@ func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1beta1. currentServerIngress := expectedServerIngress.DeepCopy() result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerIngress, func() error { - if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Client.Scheme()); err != nil { return err } @@ -828,7 +833,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster // create headless service for the statefulset serverStatefulService := server.StatefulServerService() - if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Client.Scheme()); err != nil { return err } @@ -848,7 +853,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster currentServerStatefulSet := expectedServerStatefulSet.DeepCopy() result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerStatefulSet, func() error { - if err := controllerutil.SetControllerReference(cluster, currentServerStatefulSet, c.Scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, currentServerStatefulSet, c.Client.Scheme()); err != nil { return err } @@ -896,7 +901,7 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet } func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error { - config := agent.NewConfig(cluster, c.Client, c.Scheme) + config := agent.NewConfig(cluster, c.Client) var agentEnsurer agent.ResourceEnsurer if cluster.Spec.Mode == agent.VirtualNodeMode { diff --git a/pkg/controller/cluster/mounts/mounts.go b/pkg/controller/cluster/mounts/mounts.go index 0144e4d2..c04bf3f9 100644 --- a/pkg/controller/cluster/mounts/mounts.go +++ b/pkg/controller/cluster/mounts/mounts.go @@ -41,8 +41,13 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor }, } + volName := secretMount.Name + if volName == "" { + volName = secretMount.SecretName + } + vol := corev1.Volume{ - Name: secretMount.SecretName, + Name: volName, VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ Sources: projectedVolSources, @@ -51,7 +56,7 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor } volMount := corev1.VolumeMount{ - Name: secretMount.SecretName, + Name: volName, MountPath: secretMount.MountPath, SubPath: secretMount.SubPath, } diff --git a/pkg/controller/cluster/server/bootstrap/bootstrap.go b/pkg/controller/cluster/server/bootstrap/bootstrap.go index 19aa84e8..a6d89d1f 100644 --- a/pkg/controller/cluster/server/bootstrap/bootstrap.go +++ b/pkg/controller/cluster/server/bootstrap/bootstrap.go @@ -5,8 +5,8 @@ import ( "encoding/json" "errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -19,22 +19,29 @@ import ( "github.com/rancher/k3k/pkg/k3s" ) -const ( - TLSDir = "/var/lib/rancher/k3s/server/tls/" -) - // Fetch requests bootstrap data from k3s using the token and decodes it, // to avoid double encoding when stored as secret. -func Fetch(ctx context.Context, ip, token string) (*k3s.BootstrapData, error) { +func Fetch(ctx context.Context, k3sClient *k3s.Client, cluster *v1beta1.Cluster, restConfig *rest.Config) (*k3s.BootstrapData, error) { log := ctrl.LoggerFrom(ctx) - log.V(1).Info("Fetching bootstrap data from K3s API") - return fetchFromK3sServer(ip, token) + config, err := k3sClient.GetServerConfig() + if err != nil { + return nil, err + } + + if config.ClusterInit { + log.V(1).Info("Fetching bootstrap data from K3s server API") + return k3sClient.GetServerBootstrap() + } + + log.V(1).Info("Fetching bootstrap data from K3s server Pod") + + return k3s.ReadBootstrapFromK3sPod(ctx, restConfig, cluster.Name, cluster.Namespace) } // SaveToSecret marshals the bootstrap data and stores it in a Secret owned by the cluster, // creating the Secret if it does not exist or updating it otherwise. -func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error { +func SaveToSecret(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error { bootstrapData, err := json.Marshal(data) if err != nil { return err @@ -48,7 +55,7 @@ func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, } _, err = controllerutil.CreateOrUpdate(ctx, c, secret, func() error { - if err := controllerutil.SetControllerReference(cluster, secret, scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, secret, c.Scheme()); err != nil { return err } @@ -85,12 +92,3 @@ func LoadFromSecret(ctx context.Context, client client.Client, cluster *v1beta1. return &bootstrap, err } - -func fetchFromK3sServer(serviceIP, token string) (*k3s.BootstrapData, error) { - client := k3s.New(k3s.ClientConfig{ - ServerIP: serviceIP, - Token: token, - }) - - return k3s.GetServerBootstrap(client) -} diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index daf83699..334f946f 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -131,7 +131,7 @@ configure_cgroups() { EXTRA_ARGS="$EXTRA_ARGS --kubelet-arg=kubelet-cgroups=$root_cgroup_parent/k3s --kubelet-arg=cgroup-root=$root_cgroup_parent" } -EXTRA_ARGS={{.EXTRA_ARGS}} +EXTRA_ARGS="{{.EXTRA_ARGS}}" configure_cgroups case "{{.CLUSTER_MODE}}" in diff --git a/pkg/controller/cluster/statefulset.go b/pkg/controller/cluster/statefulset.go index 5f3df307..c4aa121e 100644 --- a/pkg/controller/cluster/statefulset.go +++ b/pkg/controller/cluster/statefulset.go @@ -72,12 +72,6 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err) } - // If the StatefulSet is being deleted, we need to remove the finalizers from its pods - // and remove the finalizer from the StatefulSet itself. - if !sts.DeletionTimestamp.IsZero() { - return p.handleDeletion(ctx, &sts) - } - // get cluster name from the object clusterKey := clusterNamespacedName(&sts) @@ -88,6 +82,12 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req } } + // If the StatefulSet is being deleted, we need to remove the finalizers from its pods + // and remove the finalizer from the StatefulSet itself. + if !sts.DeletionTimestamp.IsZero() { + return p.handleDeletion(ctx, &sts) + } + podList, err := p.listPods(ctx, &sts) if err != nil { return reconcile.Result{}, err @@ -106,6 +106,26 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req } } + // skip adding finalizer to server pods if etcd is not used + clusterToken, err := getClusterToken(ctx, p.Client, &cluster) + if err != nil { + return reconcile.Result{}, err + } + + k3sClient := k3s.New(k3s.ClientConfig{ + ServerIP: fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace), + Token: clusterToken, + }) + + config, err := k3sClient.GetServerConfig() + if err != nil { + return reconcile.Result{}, err + } + + if !config.ClusterInit { + return reconcile.Result{}, nil + } + for _, pod := range podList.Items { if err := p.handleServerPod(ctx, cluster, &pod); err != nil { return reconcile.Result{}, err diff --git a/pkg/controller/cluster/token.go b/pkg/controller/cluster/token.go index 1444a722..134c1063 100644 --- a/pkg/controller/cluster/token.go +++ b/pkg/controller/cluster/token.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "errors" "fmt" "k8s.io/apimachinery/pkg/types" @@ -73,7 +74,7 @@ func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1be key = client.ObjectKeyFromObject(&tokenSecret) result, err := controllerutil.CreateOrUpdate(ctx, c.Client, &tokenSecret, func() error { - return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Scheme) + return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Client.Scheme()) }) if result != controllerutil.OperationResultNone { @@ -113,3 +114,21 @@ func TokenSecretObj(token, name, namespace string) corev1.Secret { func TokenSecretName(clusterName string) string { return controller.SafeConcatNameWithPrefix(clusterName, "token") } + +func getClusterToken(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (string, error) { + tokenSecretName := TokenSecretName(cluster.Name) + if cluster.Spec.TokenSecretRef != nil && cluster.Spec.TokenSecretRef.Name != "" { + tokenSecretName = cluster.Spec.TokenSecretRef.Name + } + + var tokenSecret corev1.Secret + if err := client.Get(ctx, types.NamespacedName{Name: tokenSecretName, Namespace: cluster.Namespace}, &tokenSecret); err != nil { + return "", err + } + + if tokenSecret.Data != nil { + return string(tokenSecret.Data["token"]), nil + } + + return "", errors.New("token secret is empty") +} diff --git a/pkg/k3s/bootstrap.go b/pkg/k3s/bootstrap.go index 10399d49..e050945a 100644 --- a/pkg/k3s/bootstrap.go +++ b/pkg/k3s/bootstrap.go @@ -1,9 +1,25 @@ package k3s import ( + "bytes" + "context" "encoding/base64" "fmt" "net/http" + "path/filepath" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + corev1 "k8s.io/api/core/v1" + + "github.com/rancher/k3k/pkg/controller" +) + +const ( + TLSDir = "/var/lib/rancher/k3s/server/tls/" ) type BootstrapData struct { @@ -20,20 +36,29 @@ type cert struct { Content string } -func GetServerBootstrap(c *Client) (*BootstrapData, error) { +type K3SConfig struct { + ClusterInit bool `json:"ClusterInit"` +} + +func (c *Client) GetServerConfig() (*K3SConfig, error) { + endpoint := "/v1-k3s/config" + return do[*K3SConfig](c, endpoint, "node", http.MethodGet) +} + +func (c *Client) GetServerBootstrap() (*BootstrapData, error) { endpoint := "/v1-k3s/server-bootstrap" - bootstrap, err := do[BootstrapData](c, endpoint, "server", http.MethodGet) + bootstrap, err := do[*BootstrapData](c, endpoint, "server", http.MethodGet) if err != nil { return nil, err } // we still need to decode each certs since the bootstrap data endpoint base64 encode each cert - if err := decode(&bootstrap); err != nil { + if err := decode(bootstrap); err != nil { return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err) } - return &bootstrap, nil + return bootstrap, nil } func decode(data *BootstrapData) error { @@ -87,3 +112,76 @@ func decode(data *BootstrapData) error { return nil } + +func ReadBootstrapFromK3sPod(ctx context.Context, restConfig *rest.Config, clusterName, clusterNamespace string) (*BootstrapData, error) { + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, err + } + + // using the first server in the statefulset to get the bootstrap data + serverPodName := controller.SafeConcatNameWithPrefix(clusterName, "server-0") + + // skipping etcd since reading from pod only required with external datastore + bootstrapCerts := map[string]string{ + "server-ca.crt": "", + "server-ca.key": "", + "client-ca.crt": "", + "client-ca.key": "", + } + for certName := range bootstrapCerts { + command := []string{"cat", filepath.Join(TLSDir, certName)} + + certData, err := podExec(ctx, clientset, restConfig, clusterNamespace, serverPodName, command) + if err != nil { + return nil, err + } + + bootstrapCerts[certName] = string(certData) + } + + bootstrap := &BootstrapData{ + ServerCA: cert{Content: bootstrapCerts["server-ca.crt"]}, + ServerCAKey: cert{Content: bootstrapCerts["server-ca.key"]}, + ClientCA: cert{Content: bootstrapCerts["client-ca.crt"]}, + ClientCAKey: cert{Content: bootstrapCerts["client-ca.key"]}, + } + + return bootstrap, nil +} + +func podExec(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, namespace, name string, command []string) ([]byte, error) { + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(name). + Namespace(namespace). + SubResource("exec") + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("error adding to scheme: %v", err) + } + + parameterCodec := runtime.NewParameterCodec(scheme) + + req.VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdout: true, + }, parameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, req.URL()) + if err != nil { + return nil, fmt.Errorf("error while creating Executor: %v", err) + } + + var stdout bytes.Buffer + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + }) + if err != nil { + return nil, fmt.Errorf("error in Stream: %v", err) + } + + return stdout.Bytes(), nil +} diff --git a/pkg/k3s/bootstrap_test.go b/pkg/k3s/bootstrap_test.go new file mode 100644 index 00000000..90b79a90 --- /dev/null +++ b/pkg/k3s/bootstrap_test.go @@ -0,0 +1,152 @@ +package k3s + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_GetServerConfig(t *testing.T) { + tests := []struct { + name string + serverResponse string + clientConfig ClientConfig + expectedConfig *K3SConfig + expectedErr error + isServerRunning bool + }{ + { + name: "server not ready", + isServerRunning: false, + serverResponse: "", + expectedConfig: nil, + expectedErr: ErrServerNotReady, + clientConfig: ClientConfig{ + ServerIP: "127.0.0.1:33333", + }, + }, + { + name: "cluster init is true", + isServerRunning: true, + serverResponse: `{"ClusterInit": true}`, + expectedConfig: &K3SConfig{ClusterInit: true}, + expectedErr: ErrServerNotReady, + clientConfig: ClientConfig{}, + }, + { + name: "cluster init is false", + isServerRunning: true, + serverResponse: `{"ClusterInit": false}`, + expectedConfig: &K3SConfig{ClusterInit: false}, + expectedErr: ErrServerNotReady, + clientConfig: ClientConfig{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + + mockServer := httptest.NewUnstartedServer(mux) + if tt.isServerRunning { + mockServer.StartTLS() + defer mockServer.Close() + } + + mux.Handle("/v1-k3s/config", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(tt.serverResponse)) + require.NoError(t, err) + })) + + if tt.clientConfig.ServerIP == "" { + u, err := url.Parse(mockServer.URL) + require.NoError(t, err) + + tt.clientConfig.ServerIP = u.Host + } + + k3sClient := New(tt.clientConfig) + + k3sConfig, err := k3sClient.GetServerConfig() + if err != nil && tt.expectedErr != nil { + assert.EqualError(t, err, tt.expectedErr.Error()) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.expectedConfig, k3sConfig) + }) + } +} + +func Test_GetServerBootstrap(t *testing.T) { + fakeBootstrap := &BootstrapData{ + ClientCA: cert{ + Content: "dGVzdA==", + }, + ClientCAKey: cert{ + Content: "dGVzdA==", + }, + ServerCA: cert{ + Content: "dGVzdA==", + }, + ServerCAKey: cert{ + Content: "dGVzdA==", + }, + ETCDServerCA: cert{ + Content: "dGVzdA==", + }, + ETCDServerCAKey: cert{ + Content: "dGVzdA==", + }, + } + + expectedBootstrap := &BootstrapData{ + ClientCA: cert{ + Content: "test", + }, + ClientCAKey: cert{ + Content: "test", + }, + ServerCA: cert{ + Content: "test", + }, + ServerCAKey: cert{ + Content: "test", + }, + ETCDServerCA: cert{ + Content: "test", + }, + ETCDServerCAKey: cert{ + Content: "test", + }, + } + + mux := http.NewServeMux() + + mockServer := httptest.NewTLSServer(mux) + defer mockServer.Close() + + serverResponse, err := json.Marshal(fakeBootstrap) + require.NoError(t, err) + + mux.Handle("/v1-k3s/server-bootstrap", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write(serverResponse) + require.NoError(t, err) + })) + + u, err := url.Parse(mockServer.URL) + require.NoError(t, err) + + k3sClient := New(ClientConfig{ServerIP: u.Host}) + + bootstrap, err := k3sClient.GetServerBootstrap() + require.NoError(t, err) + + assert.Equal(t, expectedBootstrap, bootstrap) +} diff --git a/pkg/k3s/certs.go b/pkg/k3s/certs.go index 072b43aa..4db63a41 100644 --- a/pkg/k3s/certs.go +++ b/pkg/k3s/certs.go @@ -5,7 +5,7 @@ import ( "net/http" ) -func GetServingKubeletCrt(c *Client) (*tls.Certificate, error) { +func (c *Client) GetServingKubeletCrt() (*tls.Certificate, error) { endpoint := "/v1-k3s/serving-kubelet.crt" tlsCrtData, err := c.do(endpoint, "node", http.MethodGet) diff --git a/pkg/k3s/certs_test.go b/pkg/k3s/certs_test.go new file mode 100644 index 00000000..ec397967 --- /dev/null +++ b/pkg/k3s/certs_test.go @@ -0,0 +1,58 @@ +package k3s + +import ( + "crypto/tls" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + fakeCertPEM = `-----BEGIN CERTIFICATE----- +MIIBgTCCASegAwIBAgIUDZWXjBG6lXVaJ7cv3NAaaCX4QxkwCgYIKoZIzj0EAwIw +FjEUMBIGA1UEAwwLazNrLWt1YmVsZXQwHhcNMjYwNjExMTEwMzQ3WhcNMzYwNjA4 +MTEwMzQ3WjAWMRQwEgYDVQQDDAtrM2sta3ViZWxldDBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABFrEAV6qpR7m8VUXL1mL9/bmuLa1QvXkiUXhWvuJ+dg7G3p1kSNC +35d4w3IAN626oyMpMD1FL9kw5U6Gd17bcoSjUzBRMB0GA1UdDgQWBBS9KJuhT7sK +3nwpq1W6KkJ34PRsrDAfBgNVHSMEGDAWgBS9KJuhT7sK3nwpq1W6KkJ34PRsrDAP +BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIQCEg8kpyTuhsbvj9+p+ +B+pbwZb+fgGO3iYuawrvVYZwHAIgIu6PYkP0ZGsUjhMkZUUNfQgZ42Lwq1CmKnqv +rWa+ZUI= +-----END CERTIFICATE----- +` + fakeKeyPEM = `-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg/030b66JWbB6xK5m +1UuPpRbu2OLo7NyZQ1f/u1mxfq+hRANCAARaxAFeqqUe5vFVFy9Zi/f25ri2tUL1 +5IlF4Vr7ifnYOxt6dZEjQt+XeMNyADetuqMjKTA9RS/ZMOVOhnde23KE +-----END PRIVATE KEY----- +` +) + +func Test_GetServingKubeletCert(t *testing.T) { + mux := http.NewServeMux() + + mockServer := httptest.NewTLSServer(mux) + defer mockServer.Close() + + mux.Handle("/v1-k3s/serving-kubelet.crt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(fakeCertPEM + fakeKeyPEM)) + require.NoError(t, err) + })) + + u, err := url.Parse(mockServer.URL) + require.NoError(t, err) + + k3sClient := New(ClientConfig{ServerIP: u.Host}) + + expectedCert, err := tls.X509KeyPair([]byte(fakeCertPEM), []byte(fakeKeyPEM)) + require.NoError(t, err) + + cert, err := k3sClient.GetServingKubeletCrt() + require.NoError(t, err) + + assert.Equal(t, expectedCert, *cert) +} diff --git a/pkg/k3s/client.go b/pkg/k3s/client.go index ae706ad4..6be06353 100644 --- a/pkg/k3s/client.go +++ b/pkg/k3s/client.go @@ -27,6 +27,12 @@ type Client struct { var ErrServerNotReady = errors.New("server not ready") +const ( + k3sNodePasswordHeader = "k3s-Node-Password" + k3sNodeIPHeader = "k3s-Node-IP" + k3sNodeNameHeader = "k3s-Node-Name" +) + func New(config ClientConfig) *Client { httpClient := &http.Client{ Transport: http.DefaultTransport, @@ -44,11 +50,11 @@ func New(config ClientConfig) *Client { headers := http.Header{} if config.Token != "" { - headers.Set("k3s-Node-Password", config.Token) + headers.Set(k3sNodePasswordHeader, config.Token) } if config.NodeName != "" { - headers.Set("k3s-Node-Name", config.NodeName) + headers.Set(k3sNodeNameHeader, config.NodeName) } var nodeIPs []string diff --git a/pkg/k3s/client_test.go b/pkg/k3s/client_test.go new file mode 100644 index 00000000..dcc74b71 --- /dev/null +++ b/pkg/k3s/client_test.go @@ -0,0 +1,50 @@ +package k3s + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_NewClient(t *testing.T) { + tests := []struct { + name string + clientConfig ClientConfig + expectedHeaders http.Header + }{ + { + name: "empty config", + clientConfig: ClientConfig{}, + expectedHeaders: nil, + }, + { + name: "client config passed", + clientConfig: ClientConfig{ + Token: "test_token", + ServerIP: "0.0.0.0:1234", + NodeName: "test_node", + AgentIP: "1.1.1.1", + PodIP: "2.2.2.2", + }, + expectedHeaders: map[string][]string{ + k3sNodePasswordHeader: {"test_token"}, + k3sNodeNameHeader: {"test_node"}, + k3sNodeIPHeader: {"1.1.1.1,2.2.2.2"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := New(tt.clientConfig) + assert.Len(t, client.staticHeaders, len(tt.expectedHeaders)) + + for expectedHeader, expectedValue := range tt.expectedHeaders { + value, ok := client.staticHeaders[http.CanonicalHeaderKey(expectedHeader)] + assert.True(t, ok, "expected header %s is not found", expectedHeader) + assert.Equal(t, expectedValue, value) + } + }) + } +}