🔧 Improve GKE config to spread across multiple locations

GCP quotas are fairly limited (on my account, I can only
use 8 public IP addresses per zone, which means that I cannot
deploy many public clusters in a single zone). I tried to
use private clusters, but that causes other problems.
This refactoring makes it possible to spread clusters
across multiple zones. Since I have access to 20+ zones
in Europe and 20+ zones in the US, this lets me create a
lot of public clusters and simplifies the module quite a bit.

prepare-tf/README.md

@@ -57,6 +57,22 @@ The value of the `location` variable is provider-specific. Examples:
 | Linode        | `eu-central`      | `linode-cli regions list`
 | Oracle Cloud  | `eu-stockholm-1`  | `oci iam region list`
 
+You can also specify multiple locations, and then they will be
+used in round-robin fashion.
+
+For example, with Google Cloud, since the default quotas are very
+low (my account is limited to 8 public IP addresses per zone, and
+my requests to increase that quota were denied) you can do the
+following:
+
+```bash
+export TF_VAR_location=$(gcloud compute zones list --format=json | jq -r .[].name | grep ^europe)
+```
+
+Then when you apply, clusters will be created across all available
+zones in Europe. (When I write this, there are 20+ zones in Europe,
+so even with my quota, I can create 40 clusters.)
+
 3. Run!
 
 ```bash

prepare-tf/source/locals.tf

@@ -9,7 +9,7 @@ resource "time_static" "_" {}
 
 locals {
   timestamp = formatdate("YYYY-MM-DD-hh-mm", time_static._.rfc3339)
-  tag = random_string._.result
+  tag       = random_string._.result
   # Common tags to be assigned to all resources
   common_tags = [
     "created-by-terraform",

prepare-tf/source/main.tf

@@ -7,7 +7,7 @@ module "clusters" {
   enable_arm_pool    = var.enable_arm_pool
   node_size          = var.node_size
   common_tags        = local.common_tags
-  location           = var.location
+  location           = each.value.location
 }
 
 locals {
@@ -18,12 +18,13 @@ locals {
       kubeconfig_path  = format("./stage2/kubeconfig.%03d", i)
       externalips_path = format("./stage2/externalips.%03d", i)
       flags_path       = format("./stage2/flags.%03d", i)
+      location         = local.locations[i % length(local.locations)]
     }
   }
 }
 
 resource "local_file" "stage2" {
-  filename = "./stage2/main.tf"
+  filename        = "./stage2/main.tf"
   file_permission = "0644"
   content = templatefile(
     "./stage2.tmpl",
@@ -69,8 +70,8 @@ resource "null_resource" "wait_for_nodes" {
 }
 
 data "external" "externalips" {
-  for_each = local.clusters
-  depends_on = [ null_resource.wait_for_nodes ]
+  for_each   = local.clusters
+  depends_on = [null_resource.wait_for_nodes]
   program = [
     "sh",
     "-c",

prepare-tf/source/modules/digitalocean/main.tf

@@ -1,8 +1,8 @@
 resource "digitalocean_kubernetes_cluster" "_" {
-  name    = var.cluster_name
-  tags    = var.common_tags
+  name = var.cluster_name
+  tags = var.common_tags
   # Region is mandatory, so let's provide a default value.
-  region  = var.location!=null ? var.location : "nyc1"
+  region  = var.location != null ? var.location : "nyc1"
   version = var.k8s_version
 
   node_pool {

prepare-tf/source/modules/googlecloud/main.tf

@@ -1,10 +1,44 @@
 resource "google_container_cluster" "_" {
   name               = var.cluster_name
-  project            = "prepare-tf"
-  # "location" is mandatory, so let's provide a default value if none was given.
-  location           = var.location!=null ? var.location : "europe-north1-a"
+  project            = local.project
+  location           = local.location
   min_master_version = var.k8s_version
 
+  # To deploy private clusters, uncomment the section below,
+  # and uncomment the block in network.tf.
+  # Private clusters require extra resources (Cloud NAT,
+  # router, network, subnet) and the quota for some of these
+  # resources is fairly low on GCP; so if you want to deploy
+  # a lot of private clusters (more than 10), you can use these
+  # blocks as a base but you will probably have to refactor
+  # things quite a bit (you will at least need to define a single
+  # shared router and use it across all the clusters).
+  /*
+  network    = google_compute_network._.name
+  subnetwork = google_compute_subnetwork._.name
+
+  private_cluster_config {
+    enable_private_nodes = true
+    # This must be set to "false".
+    # (Otherwise, access to the public endpoint is disabled.)
+    enable_private_endpoint = false
+    # This must be set to a /28.
+    # I think it shouldn't collide with the pod network subnet.
+    master_ipv4_cidr_block = "10.255.255.0/28"
+  }
+  # Private clusters require "VPC_NATIVE" networking mode
+  # (as opposed to the legacy "ROUTES").
+  networking_mode = "VPC_NATIVE"
+  # ip_allocation_policy is required for VPC_NATIVE clusters.
+  ip_allocation_policy {
+    # This is the block that will be used for pods.
+    cluster_ipv4_cidr_block = "10.0.0.0/12"
+    # The services block is optional
+    # (GKE will pick one automatically).
+    #services_ipv4_cidr_block = ""
+  }
+  */
+
   node_pool {
     name = "x86"
     node_config {

prepare-tf/source/modules/googlecloud/network.tf

@@ -0,0 +1,38 @@
+/*
+resource "google_compute_network" "_" {
+  name    = var.cluster_name
+  project = local.project
+  # The default is to create subnets automatically.
+  # However, this creates one subnet per zone in all regions,
+  # which causes a quick exhaustion of the subnet quota.
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "_" {
+  name          = var.cluster_name
+  ip_cidr_range = "10.254.0.0/16"
+  region        = local.region
+  network       = google_compute_network._.id
+  project       = local.project
+}
+
+resource "google_compute_router" "_" {
+  name    = var.cluster_name
+  region  = local.region
+  network = google_compute_network._.name
+  project = local.project
+}
+
+resource "google_compute_router_nat" "_" {
+  name    = var.cluster_name
+  router  = google_compute_router._.name
+  region  = local.region
+  project = local.project
+  # Everyone in the network is allowed to NAT out.
+  # (We would change this if we only wanted to allow specific subnets to NAT out.)
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+  # Pick NAT addresses automatically.
+  # (We would change this if we wanted to use specific addresses to NAT out.)
+  nat_ip_allocate_option = "AUTO_ONLY"
+}
+*/

prepare-tf/source/modules/googlecloud/variables.tf

@@ -56,3 +56,13 @@ variable "k8s_version" {
   type    = string
   default = "1.21"
 }
+
+locals {
+  location = var.location != null ? var.location : "europe-north1-a"
+  region   = replace(local.location, "/-[a-z]$/", "")
+  # Unfortunately, the following line doesn't work
+  # (that attribute just returns an empty string)
+  # so we have to hard-code the project name.
+  #project = data.google_client_config._.project
+  project = "prepare-tf"
+}

prepare-tf/source/modules/linode/main.tf

@@ -1,8 +1,8 @@
 resource "linode_lke_cluster" "_" {
-  label       = var.cluster_name
-  tags        = var.common_tags
+  label = var.cluster_name
+  tags  = var.common_tags
   # "region" is mandatory, so let's provide a default value if none was given.
-  region      = var.location!=null ? var.location : "eu-central"
+  region      = var.location != null ? var.location : "eu-central"
   k8s_version = var.k8s_version
 
   pool {

prepare-tf/source/variables.tf

@@ -34,3 +34,7 @@ variable "location" {
   default = null
 }
 
+# TODO: perhaps handle if it's space-separated instead of newline?
+locals {
+  locations = var.location == null ? [null] : split("\n", var.location)
+}