[tf][testnet] k8s metrics server install for autoscale

Closes: aptos-labs#495

terraform/helm/k8s-metrics/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

terraform/helm/k8s-metrics/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: metrics-server
+  repository: https://kubernetes-sigs.github.io/metrics-server/
+  version: 3.8.2
+digest: sha256:fa1a19fa0f1ff4bae7f9e397277af3a832718ba50351e6ddf3b72a398d17fd0a
+generated: "2022-04-12T17:19:04.312907-07:00"

terraform/helm/k8s-metrics/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: aptos-k8s-metrics
+version: 0.1.0
+
+dependencies:
+  - name: metrics-server
+    version: 3.8.2
+    repository: "https://kubernetes-sigs.github.io/metrics-server/"

terraform/helm/k8s-metrics/README.md

@@ -0,0 +1,4 @@
+Aptos K8s Metrics Addon
+================================
+
+Includes addons for CoreDNS autoscaling and Metrics Server

terraform/helm/k8s-metrics/templates/dns.yaml

@@ -0,0 +1,25 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: hpa-coredns
+  namespace: kube-system
+spec:
+  maxReplicas: {{ .Values.coredns.maxReplicas }}
+  minReplicas: 2
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: coredns
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: AverageValue
+          averageUtilization: 70

terraform/helm/k8s-metrics/values.yaml

@@ -0,0 +1,2 @@
+coredns:
+  maxReplicas: 2

terraform/testnet/addons.tf

@@ -0,0 +1,16 @@
+resource "helm_release" "metrics-server" {
+  count       = var.enable_k8s_metrics_server ? 1 : 0
+  name        = "metrics-server"
+  namespace   = "kube-system"
+  chart       = "${path.module}/../helm/k8s-metrics"
+  max_history = 10
+  wait        = false
+
+  values = [
+    jsonencode({
+      coredns = {
+        maxReplicas = var.num_validators
+      }
+    })
+  ]
+}

terraform/testnet/forge/files/cleanup_forge.sh

@@ -3,6 +3,4 @@
 # Script to be run in a Forge pod after the test runs as cleanup
 # This separates the pod lifecycle from aptos/aptos
 
-# CoreDNS in EKS needs to scale down to avoid cluster resource contention
-# after the cluster has been scaled down by forge test cleanup
-kubectl scale deployment/coredns --namespace kube-system --replicas=2
+echo "Forge clean"

terraform/testnet/forge/files/init_forge.sh

@@ -3,9 +3,6 @@
 # Init script to be run in a Forge pod
 # This separates the pod lifecycle from aptos/aptos
 
-# CoreDNS in EKS needs to scale with the size of Forge clusters
-kubectl scale deployment/coredns --namespace kube-system --replicas={{ .Values.forge.numValidators }}
-
 # set up internal helm repo, naming it testnet-internal
 helm plugin install https://github.com/hypnoglow/helm-s3.git
 helm repo add testnet-internal s3://{{ .Values.forge.helmBucket }}/charts

terraform/testnet/variables.tf

@@ -164,3 +164,8 @@ variable "enable_dev_vault" {
   description = "TEST ONLY: Enables Vault in Dev Mode for all validators"
   default     = false
 }
+
+variable "enable_k8s_metrics_server" {
+  description = "Installs kubernetes metrics server: https://github.com/kubernetes-sigs/metrics-server"
+  default     = false
+}