diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000000..378eac25d3 --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1 @@ +build diff --git a/deploy/00-node-grpc-ingress.yaml b/deploy/00-node-grpc-ingress.yaml new file mode 100644 index 0000000000..be9fa26350 --- /dev/null +++ b/deploy/00-node-grpc-ingress.yaml @@ -0,0 +1,207 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: node-grpc-NETWORKNAME + namespace: NETWORKNAME + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "GRPCS" + certmanager.k8s.io/cluster-issuer: letsencrypt-production +spec: + tls: + - hosts: + - NETWORKNAME.mobilecoin.com + - '*.NETWORKNAME.mobilecoin.com' + secretName: NETWORKNAME-mobilecoin-wildcard-tls + rules: + - host: node1.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /attest + backend: + serviceName: node1 + servicePort: 3223 + - path: /blockchain + backend: + serviceName: node1 + servicePort: 8443 + - path: /consensus_client + backend: + serviceName: node1 + servicePort: 3223 + - path: /consensus_peer + backend: + serviceName: node1 + servicePort: 8443 + - path: /node_user + backend: + serviceName: node1 + servicePort: 3223 + - host: peer1.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /consensus_peer + backend: + serviceName: node1 + servicePort: 8443 + - path: /attest + backend: + serviceName: node1 + servicePort: 8443 + - path: /blockchain + backend: + serviceName: node1 + servicePort: 8443 + - host: node2.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /attest + backend: + serviceName: node2 + servicePort: 3223 + - path: /blockchain + backend: + serviceName: node2 + servicePort: 8443 + - path: /consensus_client + backend: + serviceName: node2 + servicePort: 3223 + - path: /consensus_peer + backend: + serviceName: node2 + servicePort: 8443 + - path: /node_user + backend: + serviceName: node2 + servicePort: 3223 + - host: peer2.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /consensus_peer + backend: + serviceName: node2 + servicePort: 8443 + - path: /attest + backend: + serviceName: node2 + servicePort: 8443 + - path: /blockchain + backend: + serviceName: node2 + servicePort: 8443 + - host: node3.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /attest + backend: + serviceName: node3 + servicePort: 3223 + - path: /blockchain + backend: + serviceName: node3 + servicePort: 8443 + - path: /consensus_client + backend: + serviceName: node3 + servicePort: 3223 + - path: /consensus_peer + backend: + serviceName: node3 + servicePort: 8443 + - path: /node_user + backend: + serviceName: node3 + servicePort: 3223 + - host: peer3.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /consensus_peer + backend: + serviceName: node3 + servicePort: 8443 + - path: /attest + backend: + serviceName: node3 + servicePort: 8443 + - path: /blockchain + backend: + serviceName: node3 + servicePort: 8443 + - host: node4.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /attest + backend: + serviceName: node4 + servicePort: 3223 + - path: /blockchain + backend: + serviceName: node4 + servicePort: 8443 + - path: /consensus_client + backend: + serviceName: node4 + servicePort: 3223 + - path: /consensus_peer + backend: + serviceName: node4 + servicePort: 8443 + - path: /node_user + backend: + serviceName: node4 + servicePort: 3223 + - host: peer4.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /consensus_peer + backend: + serviceName: node4 + servicePort: 8443 + - path: /attest + backend: + serviceName: node4 + servicePort: 8443 + - path: /blockchain + backend: + serviceName: node4 + servicePort: 8443 + - host: node5.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /attest + backend: + serviceName: node5 + servicePort: 3223 + - path: /blockchain + backend: + serviceName: node5 + servicePort: 8443 + - path: /consensus_client + backend: + serviceName: node5 + servicePort: 3223 + - path: /consensus_peer + backend: + serviceName: node5 + servicePort: 8443 + - path: /node_user + backend: + serviceName: node5 + servicePort: 3223 + - host: peer5.NETWORKNAME.mobilecoin.com + http: + paths: + - path: /consensus_peer + backend: + serviceName: node5 + servicePort: 8443 + - path: /attest + backend: + serviceName: node5 + servicePort: 8443 + - path: /blockchain + backend: + serviceName: node5 + servicePort: 8443 diff --git a/deploy/01-node-certificates.yaml b/deploy/01-node-certificates.yaml new file mode 100644 index 0000000000..cf71c58ffa --- /dev/null +++ b/deploy/01-node-certificates.yaml @@ -0,0 +1,119 @@ +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Certificate +metadata: + name: node1-peer-tls + namespace: NETWORKNAME +spec: + # Secret names are always required. + secretName: node1-peer-tls + duration: 8760h # 365d + renewBefore: 360h # 15d + organization: + - mobilecoin + keySize: 2048 + keyAlgorithm: rsa + keyEncoding: pkcs1 + usages: + - server auth + - client auth + dnsNames: + - node1.NETWORKNAME.mobilecoin.com + issuerRef: + name: internal-ca-issuer + kind: ClusterIssuer +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Certificate +metadata: + name: node2-peer-tls + namespace: NETWORKNAME +spec: + # Secret names are always required. + secretName: node2-peer-tls + duration: 8760h # 365d + renewBefore: 360h # 15d + organization: + - mobilecoin + keySize: 2048 + keyAlgorithm: rsa + keyEncoding: pkcs1 + usages: + - server auth + - client auth + dnsNames: + - node2.NETWORKNAME.mobilecoin.com + issuerRef: + name: internal-ca-issuer + kind: ClusterIssuer +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Certificate +metadata: + name: node3-peer-tls + namespace: NETWORKNAME +spec: + # Secret names are always required. + secretName: node3-peer-tls + duration: 8760h # 365d + renewBefore: 360h # 15d + organization: + - mobilecoin + keySize: 2048 + keyAlgorithm: rsa + keyEncoding: pkcs1 + usages: + - server auth + - client auth + dnsNames: + - node3.NETWORKNAME.mobilecoin.com + issuerRef: + name: internal-ca-issuer + kind: ClusterIssuer +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Certificate +metadata: + name: node4-peer-tls + namespace: NETWORKNAME +spec: + # Secret names are always required. + secretName: node4-peer-tls + duration: 8760h # 365d + renewBefore: 360h # 15d + organization: + - mobilecoin + keySize: 2048 + keyAlgorithm: rsa + keyEncoding: pkcs1 + usages: + - server auth + - client auth + dnsNames: + - node4.NETWORKNAME.mobilecoin.com + issuerRef: + name: internal-ca-issuer + kind: ClusterIssuer +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Certificate +metadata: + name: node5-peer-tls + namespace: NETWORKNAME +spec: + # Secret names are always required. + secretName: node5-peer-tls + duration: 8760h # 365d + renewBefore: 360h # 15d + organization: + - mobilecoin + keySize: 2048 + keyAlgorithm: rsa + keyEncoding: pkcs1 + usages: + - server auth + - client auth + dnsNames: + - node5.NETWORKNAME.mobilecoin.com + issuerRef: + name: internal-ca-issuer + kind: ClusterIssuer diff --git a/deploy/01-node-services.yaml b/deploy/01-node-services.yaml new file mode 100644 index 0000000000..481855eb03 --- /dev/null +++ b/deploy/01-node-services.yaml @@ -0,0 +1,104 @@ +apiVersion: v1 +kind: Service +metadata: + name: node1 + namespace: NETWORKNAME +spec: + type: ClusterIP + selector: + app: mobilenode-NETWORKNAME + nodename: node1 + ports: + - name: cns-port + port: 8443 + targetPort: cns-port + - name: cns-mgmt + port: 8000 + targetPort: cns-mgmt + - name: cns-client + port: 3223 + targetPort: cns-client +--- +apiVersion: v1 +kind: Service +metadata: + name: node2 + namespace: NETWORKNAME +spec: + type: ClusterIP + selector: + app: mobilenode-NETWORKNAME + nodename: node2 + ports: + - name: cns-port + port: 8443 + targetPort: cns-port + - name: cns-mgmt + port: 8000 + targetPort: cns-mgmt + - name: cns-client + port: 3223 + targetPort: cns-client +--- +apiVersion: v1 +kind: Service +metadata: + name: node3 + namespace: NETWORKNAME +spec: + type: ClusterIP + selector: + app: mobilenode-NETWORKNAME + nodename: node3 + ports: + - name: cns-port + port: 8443 + targetPort: cns-port + - name: cns-mgmt + port: 8000 + targetPort: cns-mgmt + - name: cns-client + port: 3223 + targetPort: cns-client +--- +apiVersion: v1 +kind: Service +metadata: + name: node4 + namespace: NETWORKNAME +spec: + type: ClusterIP + selector: + app: mobilenode-NETWORKNAME + nodename: node4 + ports: + - name: cns-port + port: 8443 + targetPort: cns-port + - name: cns-mgmt + port: 8000 + targetPort: cns-mgmt + - name: cns-client + port: 3223 + targetPort: cns-client +--- +apiVersion: v1 +kind: Service +metadata: + name: node5 + namespace: NETWORKNAME +spec: + type: ClusterIP + selector: + app: mobilenode-NETWORKNAME + nodename: node5 + ports: + - name: cns-port + port: 8443 + targetPort: cns-port + - name: cns-mgmt + port: 8000 + targetPort: cns-mgmt + - name: cns-client + port: 3223 + targetPort: cns-client \ No newline at end of file diff --git a/deploy/01-supervisor-daemon.yaml b/deploy/01-supervisor-daemon.yaml new file mode 100644 index 0000000000..f063173474 --- /dev/null +++ b/deploy/01-supervisor-daemon.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-daemon-config + namespace: NETWORKNAME +data: + supervisor.conf: | + [supervisord] + nodaemon=true \ No newline at end of file diff --git a/deploy/01-supervisor-mobilecoind.yaml b/deploy/01-supervisor-mobilecoind.yaml new file mode 100644 index 0000000000..ef139ea7ba --- /dev/null +++ b/deploy/01-supervisor-mobilecoind.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-mobilecoind + namespace: NETWORKNAME +data: + mobilecoind.conf: | + [program:mobilecoind] + command=/usr/bin/mobilecoind + --peer mc://node1.NETWORKNAME.mobilecoin.com:443/ + --peer mc://node2.NETWORKNAME.mobilecoin.com:443/ + --peer mc://node3.NETWORKNAME.mobilecoin.com:443/ + --peer mc://node4.NETWORKNAME.mobilecoin.com:443/ + --peer mc://node5.NETWORKNAME.mobilecoin.com:443/ + --tx-source-url https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com/ + --tx-source-url https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com/ + --ledger-db /ledger + --poll-interval 1 + --mobilecoind-db /mobilecoind + --service-port 3229 + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true diff --git a/deploy/01-supervisor-sgx-config.yaml b/deploy/01-supervisor-sgx-config.yaml new file mode 100644 index 0000000000..8ebe97e0d2 --- /dev/null +++ b/deploy/01-supervisor-sgx-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-sgx-config + namespace: NETWORKNAME +data: + sgx.conf: | + [program:aesm-service] + command=/opt/intel/sgx-aesm-service/aesm/aesm_service --no-daemon + environment=AESM_PATH="/opt/intel/sgx-aesm-service/aesm",LD_LIBRARY_PATH="/opt/intel/sgx-aesm-service/aesm" + stdout_logfile=/dev/null + stderr_logfile=/dev/null + autorestart=true diff --git a/deploy/03-node1.yaml b/deploy/03-node1.yaml new file mode 100644 index 0000000000..75a23aac9f --- /dev/null +++ b/deploy/03-node1.yaml @@ -0,0 +1,267 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-ledger-dist-node1 + namespace: NETWORKNAME +data: + ledger_dist.conf: | + [program:ledger-distribution] + command=/usr/bin/ledger-distribution + --dest s3://mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com?region=us-west-1 + --ledger-path /ledger + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-consensus-node1 + namespace: NETWORKNAME +data: + consensus-service-node.conf: | + [program:consensus-service] + command=/usr/bin/consensus-service + --client-responder-id node1.NETWORKNAME.mobilecoin.com:443 + --peer-responder-id peer1.NETWORKNAME.mobilecoin.com:443 + --peer-listen-uri=mcp://0.0.0.0:8443/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --client-listen-uri=mc://0.0.0.0:3223/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --management-listen-addr=0.0.0.0:8000 + --network /config/network.toml + --ledger-path /ledger + --ias-spid %(ENV_PROD_IAS_SPID)s + --ias-api-key %(ENV_PROD_IAS_API_KEY)s + --msg-signer-key %(ENV_NODE1_SIGNER_KEY)s + --sealed-block-signing-key %(ENV_SEALED_BLOCK_SIGNING_KEY)s + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-consensus-node1 + namespace: NETWORKNAME +data: + network.toml: | + broadcast_peers = [ + "mcp://peer2.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA7i8HyecUp-0lcw5c8yP3BNP_R6JRlpXoxjJD-gWcqUE=", + "mcp://peer3.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAHzaqHvf66h03cd3-Ny4VOYoa0zhcnak8tKTb3P0gMHE=", + "mcp://peer4.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAYYmxZMX2B4kkhfPap8NrV4JMp_s6j06oxwlMD0DAiC8=", + "mcp://peer5.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA0XQ0Hy_nKTlPyxD7LYBEWVoayf_JfUsIaYy7olOZhNM=", + ] + + tx_source_urls = [ + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node3.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node4.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node5.NETWORKNAME.mobilecoin.com/", + ] + + quorum_set = { threshold = 3, members = [ + { type = "Node", args = "peer2.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer3.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer4.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer5.NETWORKNAME.mobilecoin.com:443" }, + ] } +--- +# Node1 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node1 + namespace: NETWORKNAME + labels: + app: mobilenode-NETWORKNAME + nodename: node1 +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mobilenode-NETWORKNAME + nodename: node1 + template: + metadata: + labels: + app: mobilenode-NETWORKNAME + nodename: node1 + spec: + # Need better node-selector semantics. This depends on agentpool naming + nodeSelector: + builder-node: "false" + sgx-enabled-node: "true" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mobilenode-NETWORKNAME + topologyKey: "kubernetes.io/hostname" + imagePullSecrets: + - name: docker-credentials + initContainers: + - name: install-ledger-data + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/bin/bash" ] + args: + - -c + - | + set -x + rm -rf /ledger/* + cp -r /var/lib/mobilecoin/origin_data/data.mdb /ledger + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + + containers: + + - name: node + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + # Override entrypoint for this container + command: [ "/usr/bin/supervisord" ] + ports: + - name: cns-client + containerPort: 3223 + - name: cns-port + containerPort: 8443 + - name: cns-mgmt + containerPort: 8000 + livenessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node1.NETWORKNAME.mobilecoin.com + - -addr=:8443 + failureThreshold: 3 + periodSeconds: 30 + readinessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node1.NETWORKNAME.mobilecoin.com + - -addr=:8443 + # This will need to be updated as initial sync and startup times increase. + failureThreshold: 30 + periodSeconds: 30 + env: + # Contents of /opt/intel/sgxsdk/environment and AESM requirements + - name: "PATH" + value: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/intel/sgxsdk/bin:/opt/intel/sgxsdk/bin/x64" + # Rust env settings + - name: "RUST_BACKTRACE" + value: "1" + - name: "RUST_LOG" + value: "debug,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,reqwest=warn,rusoto_core=error,rusto_signature=error,h2=error" + - name: "SGX_MODE" + value: "HW" + - name: "IAS_MODE" + value: "DEV" + - name: LOCAL_NODE_ID + value: "peer1.NETWORKNAME.mobilecoin.com:443" + - name: MC_LOG_UDP_JSON + value: "127.0.0.1:16666" + - name: MC_LOG_EXTRA_CONTEXT + value: "mc.local_node_id=$(LOCAL_NODE_ID)" + - name: MC_BRANCH + value: "NETWORKNAME" + - name: AWS_PATH + value: "AWS_PATH=s3://mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com?region=us-west-1" + - name: PROD_IAS_SPID + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: spid + - name: PROD_IAS_API_KEY + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: primary-key + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-secret-access-key + - name: NODE1_SIGNER_KEY + valueFrom: + secretKeyRef: + name: node-signer-keys + key: node1 + - name: "SEALED_BLOCK_SIGNING_KEY" + value: "/keys/sealed-block-signing-key" + volumeMounts: + - name: dev-isgx + mountPath: /dev/isgx + - name: aesm-socket-dir + mountPath: /var/run/aesmd + - name: config-dir + mountPath: /config + - name: ledger-db-dir + mountPath: /ledger + - name: keys-dir + mountPath: /keys + - name: node-cert + mountPath: /certs + readOnly: true + - name: supervisor-conf + mountPath: /etc/supervisor/conf.d + readOnly: true + securityContext: + privileged: true + + volumes: + # Volume-mapped SGX device + - name: dev-isgx + hostPath: + path: /dev/isgx + type: CharDevice + - name: ledger-db-dir + emptyDir: {} + - name: config-dir + configMap: + name: config-consensus-node1 + - name: keys-dir + emptyDir: {} + - name: aesm-socket-dir + emptyDir: {} + - name: node-cert + secret: + secretName: node1-peer-tls + - name: supervisor-conf + projected: + sources: + - configMap: + name: supervisor-daemon-config + - configMap: + name: supervisor-sgx-config + - configMap: + name: supervisor-consensus-node1 + - configMap: + name: supervisor-ledger-dist-node1 diff --git a/deploy/03-node2.yaml b/deploy/03-node2.yaml new file mode 100644 index 0000000000..d590d4b4bc --- /dev/null +++ b/deploy/03-node2.yaml @@ -0,0 +1,267 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-ledger-dist-node2 + namespace: NETWORKNAME +data: + ledger_dist.conf: | + [program:ledger-distribution] + command=/usr/bin/ledger-distribution + --dest s3://mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com?region=us-west-1 + --ledger-path /ledger + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-consensus-node2 + namespace: NETWORKNAME +data: + consensus-service-node.conf: | + [program:consensus-service] + command=/usr/bin/consensus-service + --client-responder-id node2.NETWORKNAME.mobilecoin.com:443 + --peer-responder-id peer2.NETWORKNAME.mobilecoin.com:443 + --peer-listen-uri=mcp://0.0.0.0:8443/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --client-listen-uri=mc://0.0.0.0:3223/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --management-listen-addr=0.0.0.0:8000 + --network /config/network.toml + --ledger-path /ledger + --ias-spid %(ENV_PROD_IAS_SPID)s + --ias-api-key %(ENV_PROD_IAS_API_KEY)s + --msg-signer-key %(ENV_NODE2_SIGNER_KEY)s + --sealed-block-signing-key %(ENV_SEALED_BLOCK_SIGNING_KEY)s + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-consensus-node2 + namespace: NETWORKNAME +data: + network.toml: | + broadcast_peers = [ + "mcp://peer1.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAg1iGuFv37uEFrn71kn31TE38TC-ma_Fb4wX239H0c8M=", + "mcp://peer3.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAHzaqHvf66h03cd3-Ny4VOYoa0zhcnak8tKTb3P0gMHE=", + "mcp://peer4.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAYYmxZMX2B4kkhfPap8NrV4JMp_s6j06oxwlMD0DAiC8=", + "mcp://peer5.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA0XQ0Hy_nKTlPyxD7LYBEWVoayf_JfUsIaYy7olOZhNM=", + ] + + tx_source_urls = [ + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node3.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node4.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node5.NETWORKNAME.mobilecoin.com/", + ] + + quorum_set = { threshold = 3, members = [ + { type = "Node", args = "peer1.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer3.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer4.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer5.NETWORKNAME.mobilecoin.com:443" }, + ] } +--- +# node2 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node2 + namespace: NETWORKNAME + labels: + app: mobilenode-NETWORKNAME + nodename: node2 +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mobilenode-NETWORKNAME + nodename: node2 + template: + metadata: + labels: + app: mobilenode-NETWORKNAME + nodename: node2 + spec: + # Need better node-selector semantics. This depends on agentpool naming + nodeSelector: + builder-node: "false" + sgx-enabled-node: "true" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mobilenode-NETWORKNAME + topologyKey: "kubernetes.io/hostname" + imagePullSecrets: + - name: docker-credentials + initContainers: + - name: install-ledger-data + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/bin/bash" ] + args: + - -c + - | + set -x + rm -rf /ledger/* + cp -r /var/lib/mobilecoin/origin_data/data.mdb /ledger + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + + containers: + + - name: node + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + # Override entrypoint for this container + command: [ "/usr/bin/supervisord" ] + ports: + - name: cns-client + containerPort: 3223 + - name: cns-port + containerPort: 8443 + - name: cns-mgmt + containerPort: 8000 + livenessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node2.NETWORKNAME.mobilecoin.com + - -addr=:8443 + failureThreshold: 3 + periodSeconds: 30 + readinessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node2.NETWORKNAME.mobilecoin.com + - -addr=:8443 + # This will need to be updated as initial sync and startup times increase. + failureThreshold: 30 + periodSeconds: 30 + env: + # Contents of /opt/intel/sgxsdk/environment and AESM requirements + - name: "PATH" + value: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/intel/sgxsdk/bin:/opt/intel/sgxsdk/bin/x64" + # Rust env settings + - name: "RUST_BACKTRACE" + value: "1" + - name: "RUST_LOG" + value: "debug,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,reqwest=warn,rusoto_core=error,rusto_signature=error,h2=error" + - name: "SGX_MODE" + value: "HW" + - name: "IAS_MODE" + value: "DEV" + - name: LOCAL_NODE_ID + value: "peer2.NETWORKNAME.mobilecoin.com:443" + - name: MC_LOG_UDP_JSON + value: "127.0.0.1:16666" + - name: MC_LOG_EXTRA_CONTEXT + value: "mc.local_node_id=$(LOCAL_NODE_ID)" + - name: MC_BRANCH + value: "NETWORKNAME" + - name: AWS_PATH + value: "AWS_PATH=s3://mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com?region=us-west-1" + - name: PROD_IAS_SPID + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: spid + - name: PROD_IAS_API_KEY + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: primary-key + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-secret-access-key + - name: NODE2_SIGNER_KEY + valueFrom: + secretKeyRef: + name: node-signer-keys + key: node2 + - name: "SEALED_BLOCK_SIGNING_KEY" + value: "/keys/sealed-block-signing-key" + volumeMounts: + - name: dev-isgx + mountPath: /dev/isgx + - name: aesm-socket-dir + mountPath: /var/run/aesmd + - name: config-dir + mountPath: /config + - name: ledger-db-dir + mountPath: /ledger + - name: keys-dir + mountPath: /keys + - name: node-cert + mountPath: /certs + readOnly: true + - name: supervisor-conf + mountPath: /etc/supervisor/conf.d + readOnly: true + securityContext: + privileged: true + + volumes: + # Volume-mapped SGX device + - name: dev-isgx + hostPath: + path: /dev/isgx + type: CharDevice + - name: ledger-db-dir + emptyDir: {} + - name: config-dir + configMap: + name: config-consensus-node2 + - name: keys-dir + emptyDir: {} + - name: aesm-socket-dir + emptyDir: {} + - name: node-cert + secret: + secretName: node2-peer-tls + - name: supervisor-conf + projected: + sources: + - configMap: + name: supervisor-daemon-config + - configMap: + name: supervisor-sgx-config + - configMap: + name: supervisor-consensus-node2 + - configMap: + name: supervisor-ledger-dist-node2 diff --git a/deploy/03-node3.yaml b/deploy/03-node3.yaml new file mode 100644 index 0000000000..7da0b480c8 --- /dev/null +++ b/deploy/03-node3.yaml @@ -0,0 +1,267 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-ledger-dist-node3 + namespace: NETWORKNAME +data: + ledger_dist.conf: | + [program:ledger-distribution] + command=/usr/bin/ledger-distribution + --dest s3://mobilecoin.chain/node3.NETWORKNAME.mobilecoin.com?region=us-west-1 + --ledger-path /ledger + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-consensus-node3 + namespace: NETWORKNAME +data: + consensus-service-node.conf: | + [program:consensus-service] + command=/usr/bin/consensus-service + --client-responder-id node3.NETWORKNAME.mobilecoin.com:443 + --peer-responder-id peer3.NETWORKNAME.mobilecoin.com:443 + --peer-listen-uri=mcp://0.0.0.0:8443/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --client-listen-uri=mc://0.0.0.0:3223/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --management-listen-addr=0.0.0.0:8000 + --network /config/network.toml + --ledger-path /ledger + --ias-spid %(ENV_PROD_IAS_SPID)s + --ias-api-key %(ENV_PROD_IAS_API_KEY)s + --msg-signer-key %(ENV_NODE3_SIGNER_KEY)s + --sealed-block-signing-key %(ENV_SEALED_BLOCK_SIGNING_KEY)s + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-consensus-node3 + namespace: NETWORKNAME +data: + network.toml: | + broadcast_peers = [ + "mcp://peer1.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAg1iGuFv37uEFrn71kn31TE38TC-ma_Fb4wX239H0c8M=", + "mcp://peer2.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA7i8HyecUp-0lcw5c8yP3BNP_R6JRlpXoxjJD-gWcqUE=", + "mcp://peer4.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAYYmxZMX2B4kkhfPap8NrV4JMp_s6j06oxwlMD0DAiC8=", + "mcp://peer5.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA0XQ0Hy_nKTlPyxD7LYBEWVoayf_JfUsIaYy7olOZhNM=", + ] + + tx_source_urls = [ + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node4.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node5.NETWORKNAME.mobilecoin.com/", + ] + + quorum_set = { threshold = 3, members = [ + { type = "Node", args = "peer1.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer2.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer4.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer5.NETWORKNAME.mobilecoin.com:443" }, + ] } +--- +# node3 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node3 + namespace: NETWORKNAME + labels: + app: mobilenode-NETWORKNAME + nodename: node3 +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mobilenode-NETWORKNAME + nodename: node3 + template: + metadata: + labels: + app: mobilenode-NETWORKNAME + nodename: node3 + spec: + # Need better node-selector semantics. This depends on agentpool naming + nodeSelector: + builder-node: "false" + sgx-enabled-node: "true" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mobilenode-NETWORKNAME + topologyKey: "kubernetes.io/hostname" + imagePullSecrets: + - name: docker-credentials + initContainers: + - name: install-ledger-data + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/bin/bash" ] + args: + - -c + - | + set -x + rm -rf /ledger/* + cp -r /var/lib/mobilecoin/origin_data/data.mdb /ledger + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + + containers: + + - name: node + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + # Override entrypoint for this container + command: [ "/usr/bin/supervisord" ] + ports: + - name: cns-client + containerPort: 3223 + - name: cns-port + containerPort: 8443 + - name: cns-mgmt + containerPort: 8000 + livenessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node3.NETWORKNAME.mobilecoin.com + - -addr=:8443 + failureThreshold: 3 + periodSeconds: 30 + readinessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node3.NETWORKNAME.mobilecoin.com + - -addr=:8443 + # This will need to be updated as initial sync and startup times increase. + failureThreshold: 30 + periodSeconds: 30 + env: + # Contents of /opt/intel/sgxsdk/environment and AESM requirements + - name: "PATH" + value: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/intel/sgxsdk/bin:/opt/intel/sgxsdk/bin/x64" + # Rust env settings + - name: "RUST_BACKTRACE" + value: "1" + - name: "RUST_LOG" + value: "debug,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,reqwest=warn,rusoto_core=error,rusto_signature=error,h2=error" + - name: "SGX_MODE" + value: "HW" + - name: "IAS_MODE" + value: "DEV" + - name: LOCAL_NODE_ID + value: "peer3.NETWORKNAME.mobilecoin.com:443" + - name: MC_LOG_UDP_JSON + value: "127.0.0.1:16666" + - name: MC_LOG_EXTRA_CONTEXT + value: "mc.local_node_id=$(LOCAL_NODE_ID)" + - name: MC_BRANCH + value: "NETWORKNAME" + - name: AWS_PATH + value: "AWS_PATH=s3://mobilecoin.chain/node3.NETWORKNAME.mobilecoin.com?region=us-west-1" + - name: PROD_IAS_SPID + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: spid + - name: PROD_IAS_API_KEY + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: primary-key + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-secret-access-key + - name: NODE3_SIGNER_KEY + valueFrom: + secretKeyRef: + name: node-signer-keys + key: node3 + - name: "SEALED_BLOCK_SIGNING_KEY" + value: "/keys/sealed-block-signing-key" + volumeMounts: + - name: dev-isgx + mountPath: /dev/isgx + - name: aesm-socket-dir + mountPath: /var/run/aesmd + - name: config-dir + mountPath: /config + - name: ledger-db-dir + mountPath: /ledger + - name: keys-dir + mountPath: /keys + - name: node-cert + mountPath: /certs + readOnly: true + - name: supervisor-conf + mountPath: /etc/supervisor/conf.d + readOnly: true + securityContext: + privileged: true + + volumes: + # Volume-mapped SGX device + - name: dev-isgx + hostPath: + path: /dev/isgx + type: CharDevice + - name: ledger-db-dir + emptyDir: {} + - name: config-dir + configMap: + name: config-consensus-node3 + - name: keys-dir + emptyDir: {} + - name: aesm-socket-dir + emptyDir: {} + - name: node-cert + secret: + secretName: node3-peer-tls + - name: supervisor-conf + projected: + sources: + - configMap: + name: supervisor-daemon-config + - configMap: + name: supervisor-sgx-config + - configMap: + name: supervisor-consensus-node3 + - configMap: + name: supervisor-ledger-dist-node3 diff --git a/deploy/03-node4.yaml b/deploy/03-node4.yaml new file mode 100644 index 0000000000..390f73d169 --- /dev/null +++ b/deploy/03-node4.yaml @@ -0,0 +1,269 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-ledger-dist-node4 + namespace: NETWORKNAME +data: + ledger_dist.conf: | + [program:ledger-distribution] + command=/usr/bin/ledger-distribution + --dest s3://mobilecoin.chain/node4.NETWORKNAME.mobilecoin.com?region=us-west-1 + --ledger-path /ledger + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-consensus-node4 + namespace: NETWORKNAME +data: + consensus-service-node.conf: | + [program:consensus-service] + command=/usr/bin/consensus-service + --client-responder-id node4.NETWORKNAME.mobilecoin.com:443 + --peer-responder-id peer4.NETWORKNAME.mobilecoin.com:443 + --peer-listen-uri=mcp://0.0.0.0:8443/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --client-listen-uri=mc://0.0.0.0:3223/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --management-listen-addr=0.0.0.0:8000 + --network /config/network.toml + --ledger-path /ledger + --ias-spid %(ENV_PROD_IAS_SPID)s + --ias-api-key %(ENV_PROD_IAS_API_KEY)s + --msg-signer-key %(ENV_NODE4_SIGNER_KEY)s + --sealed-block-signing-key %(ENV_SEALED_BLOCK_SIGNING_KEY)s + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-consensus-node4 + namespace: NETWORKNAME +data: + network.toml: | + broadcast_peers = [ + "mcp://peer1.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAg1iGuFv37uEFrn71kn31TE38TC-ma_Fb4wX239H0c8M=", + "mcp://peer2.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA7i8HyecUp-0lcw5c8yP3BNP_R6JRlpXoxjJD-gWcqUE=", + "mcp://peer3.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAHzaqHvf66h03cd3-Ny4VOYoa0zhcnak8tKTb3P0gMHE=", + "mcp://peer5.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA0XQ0Hy_nKTlPyxD7LYBEWVoayf_JfUsIaYy7olOZhNM=", + ] + + tx_source_urls = [ + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node3.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node5.NETWORKNAME.mobilecoin.com/", + ] + + quorum_set = { threshold = 3, members = [ + { type = "Node", args = "peer1.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer2.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer3.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer5.NETWORKNAME.mobilecoin.com:443" }, + ] } +--- +# node4 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node4 + namespace: NETWORKNAME + labels: + app: mobilenode-NETWORKNAME + nodename: node4 +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mobilenode-NETWORKNAME + nodename: node4 + template: + metadata: + labels: + app: mobilenode-NETWORKNAME + nodename: node4 + spec: + # Need better node-selector semantics. This depends on agentpool naming + nodeSelector: + builder-node: "false" + sgx-enabled-node: "true" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mobilenode-NETWORKNAME + topologyKey: "kubernetes.io/hostname" + imagePullSecrets: + - name: docker-credentials + initContainers: + - name: install-ledger-data + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/bin/bash" ] + args: + - -c + - | + set -x + rm -rf /ledger/* + cp -r /var/lib/mobilecoin/origin_data/data.mdb /ledger + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + + containers: + + - name: node + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + # Override entrypoint for this container + command: [ "/usr/bin/supervisord" ] + ports: + - name: cns-client + containerPort: 3223 + - name: cns-port + containerPort: 8443 + - name: cns-mgmt + containerPort: 8000 + livenessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node4.NETWORKNAME.mobilecoin.com + - -addr=:8443 + failureThreshold: 3 + periodSeconds: 30 + readinessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node4.NETWORKNAME.mobilecoin.com + - -addr=:8443 + # This will need to be updated as initial sync and startup times increase. + failureThreshold: 30 + periodSeconds: 30 + env: + # Contents of /opt/intel/sgxsdk/environment and AESM requirements + - name: "PATH" + value: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/intel/sgxsdk/bin:/opt/intel/sgxsdk/bin/x64" + # Rust env settings + - name: "RUST_BACKTRACE" + value: "1" + - name: "RUST_LOG" + value: "debug,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,reqwest=warn,rusoto_core=error,rusto_signature=error,h2=error" + - name: "SGX_MODE" + value: "HW" + - name: "IAS_MODE" + value: "DEV" + - name: LOCAL_NODE_ID + value: "peer4.NETWORKNAME.mobilecoin.com:443" + - name: MC_LOG_UDP_JSON + value: "127.0.0.1:16666" + - name: MC_LOG_EXTRA_CONTEXT + value: "mc.local_node_id=$(LOCAL_NODE_ID)" + - name: MC_BRANCH + value: "NETWORKNAME" + - name: AWS_PATH + value: "AWS_PATH=s3://mobilecoin.chain/node4.NETWORKNAME.mobilecoin.com?region=us-west-1" + - name: PROD_IAS_SPID + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: spid + - name: PROD_IAS_API_KEY + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: primary-key + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-secret-access-key + - name: NODE4_SIGNER_KEY + valueFrom: + secretKeyRef: + name: node-signer-keys + key: node4 + - name: "SEALED_BLOCK_SIGNING_KEY" + value: "/keys/sealed-block-signing-key" + volumeMounts: + - name: dev-isgx + mountPath: /dev/isgx + - name: aesm-socket-dir + mountPath: /var/run/aesmd + - name: config-dir + mountPath: /config + - name: ledger-db-dir + mountPath: /ledger + - name: keys-dir + mountPath: /keys + - name: node-cert + mountPath: /certs + readOnly: true + - name: supervisor-conf + mountPath: /etc/supervisor/conf.d + readOnly: true + securityContext: + privileged: true + + volumes: + # Volume-mapped SGX device + - name: dev-isgx + hostPath: + path: /dev/isgx + type: CharDevice + - name: ledger-db-dir + emptyDir: {} + - name: account-db-dir + emptyDir: {} + - name: config-dir + configMap: + name: config-consensus-node4 + - name: keys-dir + emptyDir: {} + - name: aesm-socket-dir + emptyDir: {} + - name: node-cert + secret: + secretName: node4-peer-tls + - name: supervisor-conf + projected: + sources: + - configMap: + name: supervisor-daemon-config + - configMap: + name: supervisor-sgx-config + - configMap: + name: supervisor-consensus-node4 + - configMap: + name: supervisor-ledger-dist-node4 diff --git a/deploy/03-node5.yaml b/deploy/03-node5.yaml new file mode 100644 index 0000000000..d9a5a44c0c --- /dev/null +++ b/deploy/03-node5.yaml @@ -0,0 +1,267 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-ledger-dist-node5 + namespace: NETWORKNAME +data: + ledger_dist.conf: | + [program:ledger-distribution] + command=/usr/bin/ledger-distribution + --dest s3://mobilecoin.chain/node5.NETWORKNAME.mobilecoin.com?region=us-west-1 + --ledger-path /ledger + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: supervisor-consensus-node5 + namespace: NETWORKNAME +data: + consensus-service-node.conf: | + [program:consensus-service] + command=/usr/bin/consensus-service + --client-responder-id node5.NETWORKNAME.mobilecoin.com:443 + --peer-responder-id peer5.NETWORKNAME.mobilecoin.com:443 + --peer-listen-uri=mcp://0.0.0.0:8443/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --client-listen-uri=mc://0.0.0.0:3223/?tls-chain=/certs/tls.crt&tls-key=/certs/tls.key + --management-listen-addr=0.0.0.0:8000 + --network /config/network.toml + --ledger-path /ledger + --ias-spid %(ENV_PROD_IAS_SPID)s + --ias-api-key %(ENV_PROD_IAS_API_KEY)s + --msg-signer-key %(ENV_NODE5_SIGNER_KEY)s + --sealed-block-signing-key %(ENV_SEALED_BLOCK_SIGNING_KEY)s + + stdout_logfile=/dev/fd/1 + stdout_logfile_maxbytes=0 + stderr_logfile=/dev/fd/2 + stderr_logfile_maxbytes=0 + autorestart=true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-consensus-node5 + namespace: NETWORKNAME +data: + network.toml: | + broadcast_peers = [ + "mcp://peer1.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAg1iGuFv37uEFrn71kn31TE38TC-ma_Fb4wX239H0c8M=", + "mcp://peer2.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEA7i8HyecUp-0lcw5c8yP3BNP_R6JRlpXoxjJD-gWcqUE=", + "mcp://peer3.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAHzaqHvf66h03cd3-Ny4VOYoa0zhcnak8tKTb3P0gMHE=", + "mcp://peer4.NETWORKNAME.mobilecoin.com:443/?consensus-msg-key=MCowBQYDK2VwAyEAYYmxZMX2B4kkhfPap8NrV4JMp_s6j06oxwlMD0DAiC8=", + ] + + tx_source_urls = [ + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node1.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node2.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node3.NETWORKNAME.mobilecoin.com/", + "https://s3-us-west-1.amazonaws.com/mobilecoin.chain/node4.NETWORKNAME.mobilecoin.com/", + ] + + quorum_set = { threshold = 3, members = [ + { type = "Node", args = "peer1.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer2.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer3.NETWORKNAME.mobilecoin.com:443" }, + { type = "Node", args = "peer4.NETWORKNAME.mobilecoin.com:443" }, + ] } +--- +# node5 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node5 + namespace: NETWORKNAME + labels: + app: mobilenode-NETWORKNAME + nodename: node5 +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mobilenode-NETWORKNAME + nodename: node5 + template: + metadata: + labels: + app: mobilenode-NETWORKNAME + nodename: node5 + spec: + # Need better node-selector semantics. This depends on agentpool naming + nodeSelector: + builder-node: "false" + sgx-enabled-node: "true" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mobilenode-NETWORKNAME + topologyKey: "kubernetes.io/hostname" + imagePullSecrets: + - name: docker-credentials + initContainers: + - name: install-ledger-data + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/bin/bash" ] + args: + - -c + - | + set -x + rm -rf /ledger/* + cp -r /var/lib/mobilecoin/origin_data/data.mdb /ledger + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + + containers: + + - name: node + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + # Override entrypoint for this container + command: [ "/usr/bin/supervisord" ] + ports: + - name: cns-client + containerPort: 3223 + - name: cns-port + containerPort: 8443 + - name: cns-mgmt + containerPort: 8000 + livenessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node5.NETWORKNAME.mobilecoin.com + - -addr=:8443 + failureThreshold: 3 + periodSeconds: 30 + readinessProbe: + exec: + command: + - /bin/grpc_health_probe + - -tls + - -tls-ca-cert + - /certs/ca.crt + - -tls-server-name + - node5.NETWORKNAME.mobilecoin.com + - -addr=:8443 + # This will need to be updated as initial sync and startup times increase. + failureThreshold: 30 + periodSeconds: 30 + env: + # Contents of /opt/intel/sgxsdk/environment and AESM requirements + - name: "PATH" + value: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/intel/sgxsdk/bin:/opt/intel/sgxsdk/bin/x64" + # Rust env settings + - name: "RUST_BACKTRACE" + value: "1" + - name: "RUST_LOG" + value: "debug,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,reqwest=warn,rusoto_core=error,rusto_signature=error,h2=error" + - name: "SGX_MODE" + value: "HW" + - name: "IAS_MODE" + value: "DEV" + - name: LOCAL_NODE_ID + value: "peer5.NETWORKNAME.mobilecoin.com:443" + - name: MC_LOG_UDP_JSON + value: "127.0.0.1:16666" + - name: MC_LOG_EXTRA_CONTEXT + value: "mc.local_node_id=$(LOCAL_NODE_ID)" + - name: MC_BRANCH + value: "NETWORKNAME" + - name: AWS_PATH + value: "AWS_PATH=s3://mobilecoin.chain/node5.NETWORKNAME.mobilecoin.com?region=us-west-1" + - name: PROD_IAS_SPID + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: spid + - name: PROD_IAS_API_KEY + valueFrom: + secretKeyRef: + name: prod-ias-keys + key: primary-key + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds-env + key: aws-secret-access-key + - name: NODE5_SIGNER_KEY + valueFrom: + secretKeyRef: + name: node-signer-keys + key: node5 + - name: "SEALED_BLOCK_SIGNING_KEY" + value: "/keys/sealed-block-signing-key" + volumeMounts: + - name: dev-isgx + mountPath: /dev/isgx + - name: aesm-socket-dir + mountPath: /var/run/aesmd + - name: config-dir + mountPath: /config + - name: ledger-db-dir + mountPath: /ledger + - name: keys-dir + mountPath: /keys + - name: node-cert + mountPath: /certs + readOnly: true + - name: supervisor-conf + mountPath: /etc/supervisor/conf.d + readOnly: true + securityContext: + privileged: true + + volumes: + # Volume-mapped SGX device + - name: dev-isgx + hostPath: + path: /dev/isgx + type: CharDevice + - name: ledger-db-dir + emptyDir: {} + - name: config-dir + configMap: + name: config-consensus-node5 + - name: keys-dir + emptyDir: {} + - name: aesm-socket-dir + emptyDir: {} + - name: node-cert + secret: + secretName: node5-peer-tls + - name: supervisor-conf + projected: + sources: + - configMap: + name: supervisor-daemon-config + - configMap: + name: supervisor-sgx-config + - configMap: + name: supervisor-consensus-node5 + - configMap: + name: supervisor-ledger-dist-node5 diff --git a/deploy/04-mobilecoind.yaml b/deploy/04-mobilecoind.yaml new file mode 100644 index 0000000000..6de49cac6e --- /dev/null +++ b/deploy/04-mobilecoind.yaml @@ -0,0 +1,104 @@ +apiVersion: v1 +kind: Service +metadata: + name: mobilecoind + namespace: NETWORKNAME +spec: + type: ClusterIP + selector: + app: mobilecoind + ports: + - name: mobilecoind-rpc + port: 3229 + targetPort: mobilecoind-rpc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mobilecoind + namespace: NETWORKNAME + labels: + app: mobilecoind +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: mobilecoind + template: + metadata: + labels: + app: mobilecoind + spec: + nodeSelector: + builder-node: "false" + sgx-enabled-node: "false" + imagePullSecrets: + - name: docker-credentials + + initContainers: + - name: install-ledger-data + image: mobilecoin/node_hw:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/bin/bash" ] + args: + - -c + - | + set -x + rm -rf /ledger/* + cp -r /var/lib/mobilecoin/origin_data/data.mdb /ledger + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + + containers: + + - name: mobilecoind + image: mobilecoin/mobilecoind:DOCKER_TAG + imagePullPolicy: IfNotPresent + command: [ "/usr/bin/supervisord" ] + ports: + - name: mobilecoind-rpc + containerPort: 3229 + livenessProbe: + exec: + command: + - "/bin/grpc_health_probe" + - "-addr=:3229" + failureThreshold: 3 + periodSeconds: 30 + readinessProbe: + exec: + command: + - "/bin/grpc_health_probe" + - "-addr=:3229" + failureThreshold: 4 + periodSeconds: 30 + env: + - name: "RUST_BACKTRACE" + value: "full" + - name: "RUST_LOG" + value: "info,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,reqwest=warn,rusoto_core=error,rusto_signature=error,h2=error" + volumeMounts: + - name: ledger-db-dir + mountPath: /ledger + - name: mobilecoind-db-dir + mountPath: /mobilecoind + - name: supervisor-conf + mountPath: /etc/supervisor/conf.d + readOnly: true + + volumes: + - name: ledger-db-dir + emptyDir: {} + - name: mobilecoind-db-dir + emptyDir: {} + - name: supervisor-conf + projected: + sources: + - configMap: + name: supervisor-daemon-config + - configMap: + name: supervisor-mobilecoind +--- diff --git a/deploy/generate-manifests b/deploy/generate-manifests new file mode 100755 index 0000000000..33a915e4cf --- /dev/null +++ b/deploy/generate-manifests @@ -0,0 +1,59 @@ +#!/bin/bash +set -e + +usage() { + cat << EOF + usage: generate-manifests [-h]|[ARGS] + Generage Kubernetes manifests from templates in this directory + Required arguments: + -n, --namespace Kubernetes namespace to deploy into. Also sets networkname + and subdomain + Optional arguments: + -h, --help Show this message and exit + -t, --tag Docker tag to use. Default is to use -latest. +EOF +} + +while (( "$#" )); do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -k|--ias-api-key) + IAS_KEY="${2}" + shift 2 + ;; + -n|--namespace) + NAMESPACE="${2}" + shift 2 + ;; + -t|--tag) + DOCKER_TAG="${2}" + shift 2 + ;; + -*|--*=) + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + esac +done + +if [ -z $NAMESPACE ]; then + echo + echo "Missing parameter: namespace" + echo + usage + exit 1 +fi + +if [ -z $DOCKER_TAG ]; then + DOCKER_TAG="${NAMESPACE}-latest" +fi + +mkdir -p build + +for i in *.yaml; do + echo "Generating build/${i}" + sed "s/NETWORKNAME/${NAMESPACE}/g; s/DOCKER_TAG/${DOCKER_TAG}/g" ${i} > build/${i} +done diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile index 213d980d99..5af629c491 100644 --- a/jenkins/Jenkinsfile +++ b/jenkins/Jenkinsfile @@ -74,7 +74,7 @@ pipeline { // Generate sample data dir('ops/sample_data') { - sh '../bin/sample-keys --num 1000' + sh '../bin/sample-keys --num 6' sh '../bin/generate_sample_ledger -t 100' sh 'rm -f ./ledger/lock.mdb' } @@ -86,31 +86,6 @@ pipeline { } stage('Build Images') { parallel { - stage('Build bootstrap-tools container') { - options { - skipDefaultCheckout true - } - agent { - kubernetes { - slaveConnectTimeout 240000 - activeDeadlineSeconds 43200 - yamlFile 'jenkins/packaging-pod.yaml' - } - } - steps { - container('jnlp'){ - unstash name: 'sources' - unstash name: 'ops_data' - } - container('kaniko'){ - sh ''' - /kaniko/executor -f $WORKSPACE/ops/Dockerfile-bootstrap-tools -c $WORKSPACE/ops \ - --destination=$DOCKER_OWNER/$BOOTSTRAP_TOOLS_DOCKER_REPO:${NETWORK}-${TAG_VERSION} \ - --destination=$DOCKER_OWNER/$BOOTSTRAP_TOOLS_DOCKER_REPO:${NETWORK}-latest - ''' - } - } - } stage('Build Consensus Service container') { options { skipDefaultCheckout true @@ -169,5 +144,72 @@ pipeline { } } } + stage('Deploy to k8s') { + options { + skipDefaultCheckout true + } + agent { + kubernetes { + slaveConnectTimeout 240000 + activeDeadlineSeconds 43200 + yamlFile 'jenkins/kubectl-pod.yaml' + } + } + steps { + container('jnlp') { + unstash name: 'sources' + dir('deploy') { + sh './generate-manifests -n ${NETWORK} -t ${NETWORK}-${TAG_VERSION}' + } + } + container('kubectl') { + dir('deploy/build') { + script { + sh(script: 'kubectl get po -n ${NETWORK}') + + // Delete the consensus nodes, and {discovery, mobilecoind, ledger} nodes if any + sh(script: 'for i in 03* 04*; do kubectl delete --ignore-not-found -n ${NETWORK} -f ${i}; done') + + // Launch the consensus node deployments + sh(script: 'for i in 03*yaml; do kubectl apply -n ${NETWORK} -f ${i}; done') + + // Wait until the deployments have achieved an Available state. + sh(script: 'for i in 1 2 3 4 5; do kubectl wait --for=condition=Available deploy/node${i} -n ${NETWORK} --timeout=2500s; done') + + // Deploy an internal mobilecoind to test with + sh(script: 'kubectl apply -n ${NETWORK} -f 04-mobilecoind.yaml') + + // Wait for mobilecoind to achieve and Available state + sh(script: 'kubectl wait --for=condition=Available deploy/mobilecoind -n ${NETWORK} --timeout=480s') + + sh(script: 'kubectl get po -n ${NETWORK}') + } + } + } + } + } + stage('Wallet Integration Test') { + steps { + container('rust-builder-default') { + dir('mobilecoind/strategies') { + sh 'pip3 install -r requirements.txt' + sh ''' + python3 -m grpc_tools.protoc -I../../consensus/api/proto \ + --python_out=. ../../consensus/api/proto/external.proto + ''' + sh ''' + python3 -m grpc_tools.protoc -I../api/proto -I../../consensus/api/proto \ + --python_out=. --grpc_python_out=. ../api/proto/mobilecoind_api.proto + ''' + sh ''' + python3 test_client.py \ + --key-dir $WORKSPACE/ops/sample_data/keys \ + --mobilecoind-host mobilecoind.${NETWORK}.svc.cluster.local\ + --mobilecoind-port 3229 + ''' + } + } + } + } } } diff --git a/jenkins/kubectl-pod.yaml b/jenkins/kubectl-pod.yaml new file mode 100644 index 0000000000..a22776b521 --- /dev/null +++ b/jenkins/kubectl-pod.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + build-pipeline-name: mobilecoin-build +spec: + nodeSelector: + builder-node: "true" + serviceAccountName: jenkins-kubernetes-deploy + # bitnami kubectl container has USER 1001 hardcoded + securityContext: + runAsUser: 1001 + allowPrivilegeEscalation: false + containers: + - name: kubectl + image: bitnami/kubectl:1.17-debian-10 + command: + - /bin/cat + tty: true + resources: + limits: + cpu: "0.75" + memory: "512Mi" + requests: + cpu: "0.25" + memory: "256Mi" diff --git a/jenkins/packaging-pod.yaml b/jenkins/packaging-pod.yaml index 4381cafd55..a671c0a15b 100644 --- a/jenkins/packaging-pod.yaml +++ b/jenkins/packaging-pod.yaml @@ -6,7 +6,6 @@ metadata: spec: nodeSelector: builder-node: "true" - # sgx-enabled-node: "false" affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/ops/Dockerfile-bootstrap-tools b/ops/Dockerfile-bootstrap-tools deleted file mode 100644 index d838a28f38..0000000000 --- a/ops/Dockerfile-bootstrap-tools +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu:18.04 -# -# This builds a slim runtime container based on Ubuntu 18.04 LTS -# for bootstrapping ledger and account data -# - -SHELL ["/bin/bash", "-c"] - -# Install any updates -# -RUN apt-get update -q -q && \ - apt-get upgrade --yes --force-yes && \ - apt-get install --yes --force-yes \ - ca-certificates \ - gettext \ - libssl1.1 \ - wget && \ - rm -rf /var/cache/apt && \ - rm -rf /var/lib/apt/lists/* - -COPY bin/generate_sample_ledger /usr/local/bin/ -COPY bin/sample-keys /usr/local/bin/