container.training/prepare-tf/source/stage2.tmpl

terraform {
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "2.7.1"
    }
  }
}

%{ for index, cluster in clusters ~}

provider "kubernetes" {
  alias = "cluster_${index}"
  config_path = "./kubeconfig.${index}"
}

resource "kubernetes_namespace" "shpod_${index}" {
  provider = kubernetes.cluster_${index}
  metadata {
    name = "shpod"
  }
}

resource "kubernetes_deployment" "shpod_${index}" {
  provider = kubernetes.cluster_${index}
  metadata {
    name = "shpod"
    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
  }
  spec {
    selector {
      match_labels = {
        app = "shpod"
      }
    }
    template {
      metadata {
        labels = {
          app = "shpod"
        }
      }
      spec {
        service_account_name = "shpod"
        container {
          image = "jpetazzo/shpod"
          name = "shpod"
          env {
            name = "PASSWORD"
            value = random_string.shpod_${index}.result
          }
          lifecycle {
            post_start {
              exec {
                command = [ "sh", "-c", "curl http://myip.enix.org/REMOTE_ADDR > /etc/HOSTIP || true" ]
              }
            }
          }
          resources {
            limits = {
              cpu    = "2"
              memory = "500M"
            }
            requests = {
              cpu    = "100m"
              memory = "250M"
            }
          }
        }
      }
    }
  }
}

resource "kubernetes_service" "shpod_${index}" {
  provider = kubernetes.cluster_${index}
  lifecycle {
    # Folks might alter their shpod Service to expose extra ports.
    # Don't reset their changes.
    ignore_changes = [ spec ]
  }
  metadata {
    name = "shpod"
    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
  }
  spec {
    selector = {
      app = "shpod"
    }
    port {
      name = "ssh"
      port = 22
      target_port = 22
      node_port = 32222
    }
    type = "NodePort"
  }
}

resource "kubernetes_service_account" "shpod_${index}" {
  provider = kubernetes.cluster_${index}
  metadata {
    name = "shpod"
    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
  }
}

resource "kubernetes_cluster_role_binding" "shpod_${index}" {
  provider = kubernetes.cluster_${index}
  metadata {
    name = "shpod"
  }
  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = "cluster-admin"
  }
  subject {
    kind      = "ServiceAccount"
    name      = "shpod"
    namespace = "shpod"
  }
  subject {
    api_group = "rbac.authorization.k8s.io"
    kind      = "Group"
    name      = "shpod-cluster-admins"
  }
}

resource "random_string" "shpod_${index}" {
  length  = 6
  special = false
  upper   = false
}

provider "helm" {
  alias = "cluster_${index}"
  kubernetes {
    config_path = "./kubeconfig.${index}"
  }
}

resource "helm_release" "metrics_server_${index}" {
  # Some providers pre-install metrics-server.
  # Some don't. Let's install metrics-server,
  # but only if it's not already installed.
  count = yamldecode(file("./flags.${index}"))["has_metrics_server"] ? 0 : 1
  provider = helm.cluster_${index}
  repository = "https://kubernetes-sigs.github.io/metrics-server/"
  chart = "metrics-server"
  version = "3.8.2"
  name = "metrics-server"
  namespace = "metrics-server"
  create_namespace = true
  set {
    name = "args"
    value = "{--kubelet-insecure-tls}"
  }
}

resource "kubernetes_config_map" "kubeconfig_${index}" {
  provider = kubernetes.cluster_${index}
  metadata {
    name = "kubeconfig"
    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
  }
  data = {
    kubeconfig_from_provider = file("./kubeconfig.${index}")
    kubeconfig_cluster_admin = <<-EOT
      kind: Config
      apiVersion: v1
      current-context: cluster-admin@k8s-${index}
      clusters:
      - name: k8s-${index}
        cluster:
          certificate-authority-data: $${yamldecode(file("./kubeconfig.${index}")).clusters.0.cluster.certificate-authority-data}
          server: $${yamldecode(file("./kubeconfig.${index}")).clusters.0.cluster.server}
      contexts:
      - name: cluster-admin@k8s-${index}
        context:
          cluster: k8s-${index}
          user: cluster-admin
      users:
      - name: cluster-admin
        user:
          client-key-data: $${base64encode(tls_private_key.cluster_admin_${index}.private_key_pem)}
          client-certificate-data: $${base64encode(kubernetes_certificate_signing_request_v1.cluster_admin_${index}.certificate)}
    EOT
  }
}

resource "tls_private_key" "cluster_admin_${index}" {
  algorithm = "RSA"
}

resource "tls_cert_request" "cluster_admin_${index}" {
  key_algorithm = tls_private_key.cluster_admin_${index}.algorithm
  private_key_pem = tls_private_key.cluster_admin_${index}.private_key_pem
  subject {
    common_name = "cluster-admin"
    # Note: CSR API v1 doesn't allow issuing certs with "system:masters" anymore.
    #organization = "system:masters"
    # We'll use this custom group name instead.cluster-admin user.
    organization = "shpod-cluster-admins"
  }
}

resource "kubernetes_certificate_signing_request_v1" "cluster_admin_${index}" {
  provider = kubernetes.cluster_${index}
  metadata {
    name = "cluster-admin"
  }
  spec {
    usages = ["client auth"]
    request = tls_cert_request.cluster_admin_${index}.cert_request_pem
    signer_name = "kubernetes.io/kube-apiserver-client"
  }
  auto_approve = true
}

%{ endfor ~}

output "ip_addresses_of_nodes" {
  value = join("\n", [
  %{ for index, cluster in clusters ~}
    join("\t", concat(
      [ random_string.shpod_${index}.result, "ssh -l k8s -p 32222" ],
      split(" ", file("./externalips.${index}"))
    )),
  %{ endfor ~}
  ])
}