[helm] expose fullnode real network address for discovery (aptos-labs#75

)
gramanje · Mar 4, 2022 · 43e00af · 43e00af
1 parent 444cb09
commit 43e00af
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 35 deletions.
diff --git a/terraform/testnet/testnet/templates/clustertest.yaml b/terraform/testnet/testnet/templates/clustertest.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.cluster_test.enabled }}
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: {{ include "testnet.fullname" . }}-cluster-test

diff --git a/terraform/testnet/testnet/templates/external-dns.yaml b/terraform/testnet/testnet/templates/external-dns.yaml
@@ -10,7 +10,7 @@ metadata:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: external-dns
@@ -27,7 +27,7 @@ rules:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: external-dns-viewer

diff --git a/terraform/testnet/testnet/templates/genesis.yaml b/terraform/testnet/testnet/templates/genesis.yaml
@@ -13,7 +13,7 @@ data:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: {{ include "testnet.fullname" . }}-genesis
@@ -26,7 +26,7 @@ rules:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: {{ include "testnet.fullname" . }}-genesis
@@ -99,7 +99,11 @@ spec:
             vault write -f "/transit/keys/val${N}__validator_network/rotate"
             diem-genesis-tool owner-key --validator-backend "$VAULT_BACKEND;namespace=val$N" --shared-backend "$FILE_BACKEND;namespace=val$N"
             diem-genesis-tool operator-key --validator-backend "$VAULT_BACKEND;namespace=val$N" --shared-backend "$FILE_BACKEND;namespace=val$N"
+            {{- if .Values.service.fullnode.enableOnchainDiscovery }}
+            diem-genesis-tool validator-config --validator-backend "$VAULT_BACKEND;namespace=val$N" --shared-backend "$FILE_BACKEND;namespace=val$N" --validator-address "/dns4/val$N-diem-validator-validator-lb/tcp/6180" --fullnode-address "/dns4/fn$N.{{.Values.service.domain}}/tcp/6182" --owner-name "val$N" --chain-id "{{ .Values.genesis.chain_id | default .Values.genesis.era }}"
+            {{- else }}
             diem-genesis-tool validator-config --validator-backend "$VAULT_BACKEND;namespace=val$N" --shared-backend "$FILE_BACKEND;namespace=val$N" --validator-address "/dns4/val$N-diem-validator-validator-lb/tcp/6180" --fullnode-address "/dns4/val$N-diem-validator-fullnode-lb/tcp/6182" --owner-name "val$N" --chain-id "{{ .Values.genesis.chain_id | default .Values.genesis.era }}"
+            {{- end }}
             diem-genesis-tool set-operator --shared-backend "$FILE_BACKEND;namespace=val$N" --operator-name "val$N"
           done
 

diff --git a/terraform/testnet/testnet/templates/ingress.yaml b/terraform/testnet/testnet/templates/ingress.yaml
@@ -1,4 +1,4 @@
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ include "testnet.fullname" . }}
@@ -24,43 +24,67 @@ spec:
     http:
       paths:
       - path: /*
+        pathType: Prefix
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-faucet
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-faucet
+            port:
+              number: 80
   {{- if .Values.service.pfn.exposeApi }}
   - host: pfn.{{ .Values.service.domain }}
     http:
       paths:
       - path: /*
+        pathType: Prefix
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-pfn
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-pfn
+            port:
+              number: 80
   {{- end }}
   - http:
       paths:
       {{- if .Values.faucet.enabled }}
       - path: /mint
+        pathType: Exact
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-faucet
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-faucet
+            port:
+              number: 80
       {{- end }}
       - path: /waypoint.txt
+        pathType: Exact
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-waypoint
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-waypoint
+            port:
+              number: 80
       - path: /chainid.txt
+        pathType: Exact
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-waypoint
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-waypoint
+            port:
+              number: 80
       - path: /era.txt
+        pathType: Exact
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-waypoint
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-waypoint
+            port:
+              number: 80
       - path: /genesis.blob
+        pathType: Exact
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-waypoint
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-waypoint
+            port:
+              number: 80
       - path: /*
+        pathType: Prefix
         backend:
-          serviceName: {{ include "testnet.fullname" . }}-api
-          servicePort: 80
+          service:
+            name: {{ include "testnet.fullname" . }}-api
+            port:
+              number: 80
diff --git a/terraform/testnet/testnet/templates/monitoring.yaml b/terraform/testnet/testnet/templates/monitoring.yaml
@@ -86,7 +86,7 @@ spec:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: {{ include "testnet.fullname" . }}-prometheus
@@ -111,7 +111,7 @@ rules:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: {{ include "testnet.fullname" . }}-prometheus

diff --git a/terraform/testnet/testnet/templates/service.yaml b/terraform/testnet/testnet/templates/service.yaml
@@ -1,30 +1,39 @@
+{{- $fullName := include "testnet.fullname" . -}}
+{{- $labels := include "testnet.labels" . -}}
+{{- $domain := .Values.service.domain -}}
+{{- $loadBalancerSourceRanges := .Values.service.fullnode.loadBalancerSourceRanges -}}
+
+{{- if .Values.service.fullnode.enableOnchainDiscovery }}
+# create internet facing network address for fullnode 
+# to allow onchain discovery
+{{- range $i, $e := .Values.genesis.numValidators | int | until }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "testnet.fullname" . }}-fullnode
+  name: {{ printf "%s-fullnode-%d" $fullName $i | quote }}
   labels:
-    {{- include "testnet.labels" . | nindent 4 }}
+    {{- $labels | nindent 4 }}
   annotations:
     service.beta.kubernetes.io/aws-load-balancer-type: nlb
-    service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: {{ .Values.service.aws_tags | quote }}
-    {{- if .Values.service.domain }}
-    external-dns.alpha.kubernetes.io/hostname: fn.{{ .Values.service.domain }}
+    {{- if $domain }}
+    external-dns.alpha.kubernetes.io/hostname: {{ printf "fn%d.%s" $i $domain | quote }}
     {{- end }}
     alb.ingress.kubernetes.io/healthcheck-path: /-/healthy
 spec:
   selector:
-    app.kubernetes.io/instance: val0
+    app.kubernetes.io/instance: {{ printf "val%d" $i | quote }}
     app.kubernetes.io/part-of: diem-validator
     app.kubernetes.io/name: haproxy
   ports:
   - name: diemnet
     port: 6182
   type: LoadBalancer
   externalTrafficPolicy: Local
-  {{- with .Values.service.fullnode.loadBalancerSourceRanges }}
   loadBalancerSourceRanges:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
+    {{- toYaml $loadBalancerSourceRanges | nindent 4 }}
+---
+{{- end }}
+{{- end }}
 
 ---
 

diff --git a/terraform/testnet/testnet/templates/vault.yaml b/terraform/testnet/testnet/templates/vault.yaml
@@ -13,7 +13,7 @@ data:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: {{ include "testnet.fullname" . }}-tokenreview
@@ -26,7 +26,7 @@ rules:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: {{ include "testnet.fullname" . }}-tokenreview

diff --git a/terraform/testnet/testnet/values.yaml b/terraform/testnet/testnet/values.yaml
@@ -71,6 +71,7 @@ faucet:
 
 service:
   fullnode:
+    enableOnchainDiscovery: false
     loadBalancerSourceRanges:
   monitoring:
     loadBalancerSourceRanges: