From 48963fa7a5a4508d1a54faf91c6cb36f702d4ec4 Mon Sep 17 00:00:00 2001 From: Alex Monk Date: Mon, 7 Oct 2019 11:43:57 +0100 Subject: [PATCH 1/2] Tidy config around per-AZ worker counts Allow maximum to be specified on a cluster level, don't use "extra" in the keys --- modules/gsp-cluster/main.tf | 23 ++++++++++++----------- modules/gsp-cluster/variables.tf | 9 +++++++-- modules/k8s-cluster/main.tf | 6 +++--- modules/k8s-cluster/variables.tf | 9 +++++++-- pipelines/deployer/deployer.defaults.yaml | 3 ++- pipelines/deployer/deployer.tf | 19 ++++++++++++------- pipelines/deployer/deployer.yaml | 3 ++- 7 files changed, 45 insertions(+), 27 deletions(-) diff --git a/modules/gsp-cluster/main.tf b/modules/gsp-cluster/main.tf index f89dcee09..cc06e64d0 100644 --- a/modules/gsp-cluster/main.tf +++ b/modules/gsp-cluster/main.tf @@ -1,15 +1,16 @@ module "k8s-cluster" { - source = "../k8s-cluster" - vpc_id = "${var.vpc_id}" - private_subnet_ids = ["${var.private_subnet_ids}"] - public_subnet_ids = ["${var.public_subnet_ids}"] - cluster_name = "${var.cluster_name}" - worker_count = "${var.worker_count}" - worker_instance_type = "${var.worker_instance_type}" - extra_workers_per_az_count = "${var.extra_workers_per_az_count}" - ci_worker_count = "${var.ci_worker_count}" - ci_worker_instance_type = "${var.ci_worker_instance_type}" - eks_version = "${var.eks_version}" + source = "../k8s-cluster" + vpc_id = "${var.vpc_id}" + private_subnet_ids = ["${var.private_subnet_ids}"] + public_subnet_ids = ["${var.public_subnet_ids}"] + cluster_name = "${var.cluster_name}" + worker_count = "${var.worker_count}" + worker_instance_type = "${var.worker_instance_type}" + minimum_workers_per_az_count = "${var.minimum_workers_per_az_count}" + maximum_workers_per_az_count = "${var.maximum_workers_per_az_count}" + ci_worker_count = "${var.ci_worker_count}" + ci_worker_instance_type = "${var.ci_worker_instance_type}" + eks_version = "${var.eks_version}" apiserver_allowed_cidrs = ["${concat( formatlist("%s/32", var.egress_ips), diff --git a/modules/gsp-cluster/variables.tf b/modules/gsp-cluster/variables.tf index 088af8278..031999dc3 100644 --- a/modules/gsp-cluster/variables.tf +++ b/modules/gsp-cluster/variables.tf @@ -60,9 +60,14 @@ variable "worker_instance_type" { default = "t3.medium" } -variable "extra_workers_per_az_count" { +variable "minimum_workers_per_az_count" { type = "string" - default = "0" + default = "1" +} + +variable "maximum_workers_per_az_count" { + type = "string" + default = "5" } variable "ci_worker_count" { diff --git a/modules/k8s-cluster/main.tf b/modules/k8s-cluster/main.tf index 37ec00f1e..3e6181bbf 100644 --- a/modules/k8s-cluster/main.tf +++ b/modules/k8s-cluster/main.tf @@ -93,9 +93,9 @@ resource "aws_cloudformation_stack" "worker-nodes-per-az" { ClusterControlPlaneSecurityGroup = "${aws_security_group.controller.id}" NodeGroupName = "worker-${element(data.aws_subnet.private_subnets.*.availability_zone, count.index)}" - NodeAutoScalingGroupMinSize = "${var.extra_workers_per_az_count}" # "${var.worker_count / 3}" - NodeAutoScalingGroupDesiredCapacity = "${var.extra_workers_per_az_count}" # "${var.worker_count / 3}" - NodeAutoScalingGroupMaxSize = "${var.extra_workers_per_az_count + 2}" # "${var.worker_count / 3 + 2}" + NodeAutoScalingGroupMinSize = "${var.minimum_workers_per_az_count}" + NodeAutoScalingGroupDesiredCapacity = "${var.minimum_workers_per_az_count}" + NodeAutoScalingGroupMaxSize = "${var.maximum_workers_per_az_count}" NodeInstanceType = "${var.worker_instance_type}" NodeInstanceProfile = "${aws_cloudformation_stack.worker-nodes.outputs["NodeInstanceProfile"]}" diff --git a/modules/k8s-cluster/variables.tf b/modules/k8s-cluster/variables.tf index b792e94b1..a43994e07 100644 --- a/modules/k8s-cluster/variables.tf +++ b/modules/k8s-cluster/variables.tf @@ -32,9 +32,14 @@ variable "worker_count" { default = "3" } -variable "extra_workers_per_az_count" { +variable "minimum_workers_per_az_count" { type = "string" - default = "0" + default = "1" +} + +variable "maximum_workers_per_az_count" { + type = "string" + default = "5" } variable "ci_worker_instance_type" { diff --git a/pipelines/deployer/deployer.defaults.yaml b/pipelines/deployer/deployer.defaults.yaml index 639b08fc4..9e0a5a3c8 100644 --- a/pipelines/deployer/deployer.defaults.yaml +++ b/pipelines/deployer/deployer.defaults.yaml @@ -20,7 +20,8 @@ disable-destroy: true config-approvers: [] config-approval-count: 2 -extra-workers-per-az-count: 0 +minimum-workers-per-az-count: 1 +maximum-workers-per-az-count: 5 task-toolbox-image: govsvc/task-toolbox task-toolbox-tag: latest diff --git a/pipelines/deployer/deployer.tf b/pipelines/deployer/deployer.tf index 4645e2be4..81098a9bf 100644 --- a/pipelines/deployer/deployer.tf +++ b/pipelines/deployer/deployer.tf @@ -58,7 +58,11 @@ variable "worker_count" { default = "3" } -variable "extra_workers_per_az_count" { +variable "minimum_workers_per_az_count" { + type = "string" +} + +variable "maximum_workers_per_az_count" { type = "string" } @@ -134,12 +138,13 @@ module "gsp-cluster" { "3.8.110.67/32", # autom8 concourse ] - eks_version = "${var.eks_version}" - worker_instance_type = "${var.worker_instance_type}" - worker_count = "${var.worker_count}" - extra_workers_per_az_count = "${var.extra_workers_per_az_count}" - ci_worker_instance_type = "${var.ci_worker_instance_type}" - ci_worker_count = "${var.ci_worker_count}" + eks_version = "${var.eks_version}" + worker_instance_type = "${var.worker_instance_type}" + worker_count = "${var.worker_count}" + minimum_workers_per_az_count = "${var.minimum_workers_per_az_count}" + maximum_workers_per_az_count = "${var.maximum_workers_per_az_count}" + ci_worker_instance_type = "${var.ci_worker_instance_type}" + ci_worker_count = "${var.ci_worker_count}" vpc_id = "${module.gsp-network.vpc_id}" private_subnet_ids = "${module.gsp-network.private_subnet_ids}" diff --git a/pipelines/deployer/deployer.yaml b/pipelines/deployer/deployer.yaml index 8b4080fcc..ba912bed3 100644 --- a/pipelines/deployer/deployer.yaml +++ b/pipelines/deployer/deployer.yaml @@ -537,7 +537,8 @@ resources: eks_version: ((eks-version)) worker_instance_type: ((worker-instance-type)) worker_count: ((worker-count)) - extra_workers_per_az_count: ((extra-workers-per-az-count)) + minimum_workers_per_az_count: ((minimum-workers-per-az-count)) + maximum_workers_per_az_count: ((maximum-workers-per-az-count)) enable_nlb: ((enable-nlb)) ci_worker_instance_type: ((ci-worker-instance-type)) ci_worker_count: ((ci-worker-count)) From 08893472ec9120e0d8f394902676113ba2cc1c3b Mon Sep 17 00:00:00 2001 From: Alex Monk Date: Mon, 7 Oct 2019 12:16:22 +0100 Subject: [PATCH 2/2] Add new keys to lint-terraform-values-output.sh --- hack/lint-terraform-values-output.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/lint-terraform-values-output.sh b/hack/lint-terraform-values-output.sh index f4261e3dc..e5659b131 100755 --- a/hack/lint-terraform-values-output.sh +++ b/hack/lint-terraform-values-output.sh @@ -19,6 +19,8 @@ set -eu --var cluster_domain=x \ --var aws_account_role_arn=x \ --var eks_version=x \ + --var minimum_workers_per_az_count=1 \ + --var maximum_workers_per_az_count=3 \ )