diff --git a/prepare-tf/README.md b/prepare-tf/README.md index 1463a2ab..d4281e2d 100644 --- a/prepare-tf/README.md +++ b/prepare-tf/README.md @@ -57,6 +57,22 @@ The value of the `location` variable is provider-specific. Examples: | Linode | `eu-central` | `linode-cli regions list` | Oracle Cloud | `eu-stockholm-1` | `oci iam region list` +You can also specify multiple locations, and then they will be +used in round-robin fashion. + +For example, with Google Cloud, since the default quotas are very +low (my account is limited to 8 public IP addresses per zone, and +my requests to increase that quota were denied) you can do the +following: + +```bash +export TF_VAR_location=$(gcloud compute zones list --format=json | jq -r .[].name | grep ^europe) +``` + +Then when you apply, clusters will be created across all available +zones in Europe. (When I write this, there are 20+ zones in Europe, +so even with my quota, I can create 40 clusters.) + 3. Run! ```bash diff --git a/prepare-tf/source/locals.tf b/prepare-tf/source/locals.tf index 3fe1798a..4c99c286 100644 --- a/prepare-tf/source/locals.tf +++ b/prepare-tf/source/locals.tf @@ -9,7 +9,7 @@ resource "time_static" "_" {} locals { timestamp = formatdate("YYYY-MM-DD-hh-mm", time_static._.rfc3339) - tag = random_string._.result + tag = random_string._.result # Common tags to be assigned to all resources common_tags = [ "created-by-terraform", diff --git a/prepare-tf/source/main.tf b/prepare-tf/source/main.tf index 08d0e57f..53f60696 100644 --- a/prepare-tf/source/main.tf +++ b/prepare-tf/source/main.tf @@ -7,7 +7,7 @@ module "clusters" { enable_arm_pool = var.enable_arm_pool node_size = var.node_size common_tags = local.common_tags - location = var.location + location = each.value.location } locals { @@ -18,12 +18,13 @@ locals { kubeconfig_path = format("./stage2/kubeconfig.%03d", i) externalips_path = format("./stage2/externalips.%03d", i) flags_path = format("./stage2/flags.%03d", i) + location = local.locations[i % length(local.locations)] } } } resource "local_file" "stage2" { - filename = "./stage2/main.tf" + filename = "./stage2/main.tf" file_permission = "0644" content = templatefile( "./stage2.tmpl", @@ -69,8 +70,8 @@ resource "null_resource" "wait_for_nodes" { } data "external" "externalips" { - for_each = local.clusters - depends_on = [ null_resource.wait_for_nodes ] + for_each = local.clusters + depends_on = [null_resource.wait_for_nodes] program = [ "sh", "-c", diff --git a/prepare-tf/source/modules/digitalocean/main.tf b/prepare-tf/source/modules/digitalocean/main.tf index c315acf9..29e886cf 100644 --- a/prepare-tf/source/modules/digitalocean/main.tf +++ b/prepare-tf/source/modules/digitalocean/main.tf @@ -1,8 +1,8 @@ resource "digitalocean_kubernetes_cluster" "_" { - name = var.cluster_name - tags = var.common_tags + name = var.cluster_name + tags = var.common_tags # Region is mandatory, so let's provide a default value. - region = var.location!=null ? var.location : "nyc1" + region = var.location != null ? var.location : "nyc1" version = var.k8s_version node_pool { diff --git a/prepare-tf/source/modules/googlecloud/main.tf b/prepare-tf/source/modules/googlecloud/main.tf index 3a3a562f..5718dfbb 100644 --- a/prepare-tf/source/modules/googlecloud/main.tf +++ b/prepare-tf/source/modules/googlecloud/main.tf @@ -1,10 +1,44 @@ resource "google_container_cluster" "_" { name = var.cluster_name - project = "prepare-tf" - # "location" is mandatory, so let's provide a default value if none was given. - location = var.location!=null ? var.location : "europe-north1-a" + project = local.project + location = local.location min_master_version = var.k8s_version + # To deploy private clusters, uncomment the section below, + # and uncomment the block in network.tf. + # Private clusters require extra resources (Cloud NAT, + # router, network, subnet) and the quota for some of these + # resources is fairly low on GCP; so if you want to deploy + # a lot of private clusters (more than 10), you can use these + # blocks as a base but you will probably have to refactor + # things quite a bit (you will at least need to define a single + # shared router and use it across all the clusters). + /* + network = google_compute_network._.name + subnetwork = google_compute_subnetwork._.name + + private_cluster_config { + enable_private_nodes = true + # This must be set to "false". + # (Otherwise, access to the public endpoint is disabled.) + enable_private_endpoint = false + # This must be set to a /28. + # I think it shouldn't collide with the pod network subnet. + master_ipv4_cidr_block = "10.255.255.0/28" + } + # Private clusters require "VPC_NATIVE" networking mode + # (as opposed to the legacy "ROUTES"). + networking_mode = "VPC_NATIVE" + # ip_allocation_policy is required for VPC_NATIVE clusters. + ip_allocation_policy { + # This is the block that will be used for pods. + cluster_ipv4_cidr_block = "10.0.0.0/12" + # The services block is optional + # (GKE will pick one automatically). + #services_ipv4_cidr_block = "" + } + */ + node_pool { name = "x86" node_config { diff --git a/prepare-tf/source/modules/googlecloud/network.tf b/prepare-tf/source/modules/googlecloud/network.tf new file mode 100644 index 00000000..5828413e --- /dev/null +++ b/prepare-tf/source/modules/googlecloud/network.tf @@ -0,0 +1,38 @@ +/* +resource "google_compute_network" "_" { + name = var.cluster_name + project = local.project + # The default is to create subnets automatically. + # However, this creates one subnet per zone in all regions, + # which causes a quick exhaustion of the subnet quota. + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "_" { + name = var.cluster_name + ip_cidr_range = "10.254.0.0/16" + region = local.region + network = google_compute_network._.id + project = local.project +} + +resource "google_compute_router" "_" { + name = var.cluster_name + region = local.region + network = google_compute_network._.name + project = local.project +} + +resource "google_compute_router_nat" "_" { + name = var.cluster_name + router = google_compute_router._.name + region = local.region + project = local.project + # Everyone in the network is allowed to NAT out. + # (We would change this if we only wanted to allow specific subnets to NAT out.) + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + # Pick NAT addresses automatically. + # (We would change this if we wanted to use specific addresses to NAT out.) + nat_ip_allocate_option = "AUTO_ONLY" +} +*/ \ No newline at end of file diff --git a/prepare-tf/source/modules/googlecloud/variables.tf b/prepare-tf/source/modules/googlecloud/variables.tf index fcd27cad..a7232919 100644 --- a/prepare-tf/source/modules/googlecloud/variables.tf +++ b/prepare-tf/source/modules/googlecloud/variables.tf @@ -56,3 +56,13 @@ variable "k8s_version" { type = string default = "1.21" } + +locals { + location = var.location != null ? var.location : "europe-north1-a" + region = replace(local.location, "/-[a-z]$/", "") + # Unfortunately, the following line doesn't work + # (that attribute just returns an empty string) + # so we have to hard-code the project name. + #project = data.google_client_config._.project + project = "prepare-tf" +} diff --git a/prepare-tf/source/modules/linode/main.tf b/prepare-tf/source/modules/linode/main.tf index b273a556..2f163245 100644 --- a/prepare-tf/source/modules/linode/main.tf +++ b/prepare-tf/source/modules/linode/main.tf @@ -1,8 +1,8 @@ resource "linode_lke_cluster" "_" { - label = var.cluster_name - tags = var.common_tags + label = var.cluster_name + tags = var.common_tags # "region" is mandatory, so let's provide a default value if none was given. - region = var.location!=null ? var.location : "eu-central" + region = var.location != null ? var.location : "eu-central" k8s_version = var.k8s_version pool { diff --git a/prepare-tf/source/variables.tf b/prepare-tf/source/variables.tf index da7b559c..cfabf7ae 100644 --- a/prepare-tf/source/variables.tf +++ b/prepare-tf/source/variables.tf @@ -34,3 +34,7 @@ variable "location" { default = null } +# TODO: perhaps handle if it's space-separated instead of newline? +locals { + locations = var.location == null ? [null] : split("\n", var.location) +}