Make the Kubernetes cluster domain configurable (#6957)

Co-authored-by: Romain Rossi <romain.rossi@byzaneo.io>
Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com>
Co-authored-by: Robert Kaussow <xoxys@rknet.org>

docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md

@@ -517,6 +517,19 @@ Enables namespace isolation per Woodpecker organization. When enabled, each orga
 
 With this feature enabled, Woodpecker creates separate Kubernetes namespaces for each organization using the format `{WOODPECKER_BACKEND_K8S_NAMESPACE}-{organization-id}`. Namespaces are created automatically when needed, but they are not automatically deleted when organizations are removed from Woodpecker.
 
+---
+
+### BACKEND_K8S_CLUSTER_DOMAIN
+
+- Name: `WOODPECKER_BACKEND_K8S_CLUSTER_DOMAIN`
+- Default: `cluster.local`
+
+The DNS domain of the Kubernetes cluster. It is used to build the DNS search entry that lets a step reach a service by its hostname.
+
+Set this value if your cluster was installed with a custom cluster domain.
+
+---
+
 ### BACKEND_K8S_VOLUME_SIZE
 
 - Name: `WOODPECKER_BACKEND_K8S_VOLUME_SIZE`

pipeline/backend/kubernetes/flags.go

@@ -31,6 +31,12 @@ var Flags = []cli.Flag{
 		Usage:   "Whether to enable namespace segregation per organization feature. When enabled, Woodpecker will create the Kubernetes resources to separated Kubernetes namespaces per Woodpecker organization.",
 		Value:   false,
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_CLUSTER_DOMAIN"),
+		Name:    "backend-k8s-cluster-domain",
+		Usage:   "backend k8s cluster domain, used to build the DNS search entry",
+		Value:   defaultClusterDomain,
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_VOLUME_SIZE"),
 		Name:    "backend-k8s-volume-size",

pipeline/backend/kubernetes/kubernetes.go

@@ -61,6 +61,7 @@ type kube struct {
 type config struct {
 	Namespace                       string
 	EnableNamespacePerOrg           bool
+	ClusterDomain                   string
 	StorageClass                    string
 	VolumeSize                      string
 	StorageRwx                      bool
@@ -110,6 +111,7 @@ func configFromCliContext(ctx context.Context) (*config, error) {
 			config := config{
 				Namespace:                    c.String("backend-k8s-namespace"),
 				EnableNamespacePerOrg:        c.Bool("backend-k8s-namespace-per-org"),
+				ClusterDomain:                c.String("backend-k8s-cluster-domain"),
 				StorageClass:                 c.String("backend-k8s-storage-class"),
 				VolumeSize:                   c.String("backend-k8s-volume-size"),
 				StorageRwx:                   c.Bool("backend-k8s-storage-rwx"),

pipeline/backend/kubernetes/pod.go

@@ -34,11 +34,12 @@ import (
 const (
 	// StepLabelLegacy is the legacy label name from before the introduction of the woodpecker-ci.org namespace.
 	// This will be removed in the future.
-	StepLabelLegacy       = "step"
-	StepLabel             = "woodpecker-ci.org/step"
-	TaskUUIDLabel         = "woodpecker-ci.org/task-uuid"
-	podPrefix             = "wp-"
-	defaultFSGroup  int64 = 1000
+	StepLabelLegacy            = "step"
+	StepLabel                  = "woodpecker-ci.org/step"
+	TaskUUIDLabel              = "woodpecker-ci.org/task-uuid"
+	podPrefix                  = "wp-"
+	defaultFSGroup       int64 = 1000
+	defaultClusterDomain       = "cluster.local"
 	// Because of https://docs.redhat.com/en/documentation/openshift_container_platform/4.10/html/nodes/working-with-clusters
 	initContainerMemLimit = "12Mi"
 )
@@ -188,7 +189,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ
 		HostAliases:       hostAliases(step.ExtraHosts),
 		Hostname:          getHostnameOrEmpty(step.Name),
 		Subdomain:         subdomain,
-		DNSConfig:         dnsConfig(config.GetNamespace(step.OrgID), subdomain),
+		DNSConfig:         dnsConfig(config.GetNamespace(step.OrgID), subdomain, config.ClusterDomain),
 		NodeSelector:      nodeSelector(options.NodeSelector, config.PodNodeSelector, config.PodNodeSelectorAllowFromStep, step.Environment["CI_SYSTEM_PLATFORM"]),
 		Tolerations:       tolerations(options.Tolerations),
 		Affinity:          affinity(options.Affinity, config.PodAffinity, config.PodAffinityAllowFromStep),
@@ -743,9 +744,13 @@ func mapToEnvVars(m map[string]string) []kube_core_v1.EnvVar {
 	return ev
 }
 
-func dnsConfig(namespace, subdomain string) *kube_core_v1.PodDNSConfig {
+func dnsConfig(namespace, subdomain, clusterDomain string) *kube_core_v1.PodDNSConfig {
+	if clusterDomain == "" {
+		clusterDomain = defaultClusterDomain
+	}
+
 	return &kube_core_v1.PodDNSConfig{
-		Searches: []string{fmt.Sprintf("%s.%s.svc.cluster.local", subdomain, namespace)},
+		Searches: []string{fmt.Sprintf("%s.%s.svc.%s", subdomain, namespace, clusterDomain)},
 	}
 }
 

pipeline/backend/kubernetes/pod_test.go

@@ -202,6 +202,26 @@ func TestPodHostnameSanitized(t *testing.T) {
 	assert.Equal(t, "update-repos", pod.Spec.Hostname)
 }
 
+func TestPodClusterDomain(t *testing.T) {
+	step := &types.Step{
+		Name:  "build",
+		Image: "alpine:latest",
+		UUID:  "01he8bebctabr3kgk0qj36d2me-0",
+	}
+
+	pod, err := mkPod(step, &config{
+		Namespace:     "woodpecker",
+		ClusterDomain: "k8s.example.com",
+	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}, taskUUID)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"wp-hsvc-11301.woodpecker.svc.k8s.example.com"}, pod.Spec.DNSConfig.Searches)
+
+	// an unset cluster domain falls back to the Kubernetes default
+	pod, err = mkPod(step, &config{Namespace: "woodpecker"}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}, taskUUID)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"wp-hsvc-11301.woodpecker.svc.cluster.local"}, pod.Spec.DNSConfig.Searches)
+}
+
 func TestTinyPod(t *testing.T) {
 	const expected = `
 	{