[besu] Adding a new validator organization with multiple nodes to exi…

…sting network (hyperledger-bevel#1530)

* Implemented new functionality for adding a new validator org with multiple nodes

Signed-off-by: alvaropicazo <[email protected]>

platforms/hyperledger-besu/configuration/add-validator.yaml

@@ -3,7 +3,8 @@
 #  clusters be updated in the network.yaml file that is used as an input to this playbook
 ###########################################################################################
 # To Run this playbook from this directory, use the following command (network.yaml also in this directory)
-#  ansible-playbook add-validator.yaml -e "@./network.yaml"
+#  ansible-playbook add-validator.yaml -e "@./network.yaml" #  "add_new_org='true'" (for adding a new validator organization)
+#  ansible-playbook add-validator.yaml -e "@./network.yaml" #  (for adding just a validator node)
 ############################################################################################
 # Please ensure that the ../../shared/configuration playbooks have been run and a DLT network exists.
 ---
@@ -17,6 +18,51 @@
       path: "./build"
       state: absent
 
+  # Create namespaces and service accounts
+  - name: "Create namespace and service account"
+    include_role: 
+      name: create/namespace_serviceaccount
+    vars:
+      component_ns: "{{ organizationItem.name | lower }}-bes"
+      organisation: "{{ organizationItem.name | lower }}"
+      kubernetes: "{{ organizationItem.k8s }}"
+      gitops: "{{ organizationItem.gitops }}"
+    loop: "{{ network['organizations'] }}"
+    loop_control:
+      loop_var: organizationItem
+    when: add_new_org == 'true'
+
+  # Create Storageclass
+  - name: Create Storage Class
+    include_role:
+      name: create/storageclass
+    vars:
+      storageclass_name: "{{ organizationItem.cloud_provider }}storageclass"
+      git_dir: "{{ organizationItem.gitops.release_dir }}"
+      org: "{{ organizationItem }}"
+      kubernetes: "{{ organizationItem.k8s }}"
+    loop: "{{ network['organizations'] }}"
+    loop_control:
+      loop_var: organizationItem
+    when: add_new_org == 'true'
+
+  # Setup Vault-Kubernetes accesses and Regcred for docker registry
+  - name: "Setup vault"   
+    include_role: 
+      name: "setup/vault_kubernetes"
+    vars:
+      component_ns: "{{ organizationItem.name | lower }}-bes"
+      kubernetes: "{{ organizationItem.k8s }}"
+      vault: "{{ organizationItem.vault }}"
+      component_name: "{{ organizationItem.name | lower}}"
+      component_path: "{{ organizationItem.name | lower }}/"
+      component_auth: "besu{{ organizationItem.name | lower }}"
+      component_type: "besu"
+    loop: "{{ network['organizations'] }}"
+    loop_control:
+      loop_var: organizationItem
+    when: add_new_org == 'true'
+
   # Generate Ambassador certificate for nodes. These certificates are used for Orion TLS as well
   - name: "Create ambassador certificates for Nodes" 
     include_role: 
@@ -41,47 +87,10 @@
       name: create/validator_node  
     vars:
       build_path: "./build"
-      kubernetes: "{{ item.k8s }}"
-      consensus: "{{ network.config.consensus }}"
-      component_ns: "{{ item.name | lower }}-bes"
-      name: "{{ item.name | lower }}"
-      peers: "{{ item.services.validators}}"
-      tm_node: "{{ network.config.tm_nodes | first }}"
-      storageclass_name: "{{ item.cloud_provider }}storageclass"
-      external_url: "{{ item.external_url_suffix }}"
-      vault: "{{ item.vault }}"
-      git_url: "{{ item.gitops.git_url }}"
-      git_branch: "{{ item.gitops.branch }}"
-      docker_url: "{{ network.docker.url }}"
-      charts_dir: "{{ item.gitops.chart_source }}"
-      values_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}"
-    loop: "{{ network['organizations'] }}"
-    when: network.config.consensus == 'ibft' and item.type == 'validator'       
-
-  - name: "Deploy validator nodes"
-    include_role:
-      name: create/validator
-    vars:
-      build_path: "./build"
-      kubernetes: "{{ item.k8s }}"
-      consensus: "{{ network.config.consensus }}"
-      component_ns: "{{ item.name | lower }}-bes"
-      name: "{{ item.name | lower }}"
-      peers: "{{ item.services.validators}}"
-      storageclass_name: "{{ item.cloud_provider }}storageclass"
-      external_url: "{{ item.external_url_suffix }}"
-      vault: "{{ item.vault }}"
-      git_url: "{{ item.gitops.git_url }}"
-      git_branch: "{{ item.gitops.branch }}"
-      docker_url: "{{ network.docker.url }}"
-      charts_dir: "{{ item.gitops.chart_source }}"
-      values_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}"
-    loop: "{{ network['organizations'] }}"
-    when: network.config.consensus == 'ibft' and item.type == 'validator'
-
+    when: network.config.consensus == 'ibft'
 
   vars: #These variables can be overriden from the command line
     install_os: "linux"                 #Default to linux OS
     install_arch:  "amd64"              #Default to amd64 architecture
-    bin_install_dir:  "~/bin"            #Default to /bin install directory for binaries
-    add_new_org: false
+    bin_install_dir:  "~/bin"           #Default to /bin install directory for binaries
+    add_new_org: 'false'                #Default for this playbook is false

platforms/hyperledger-besu/configuration/roles/create/helm_component/templates/validator.tpl

@@ -20,7 +20,9 @@ spec:
 
     staticnodes:
 {% for enode in enode_data_list %}
+{% if enode.enodeval != '' %}
       - "enode://{{ enode.enodeval }}@{{ enode.peer_name }}.{{ enode.external_url }}:{{ enode.p2p_ambassador }}"
+{% endif %}      
 {% endfor %}
 
     metadata:

platforms/hyperledger-besu/configuration/roles/create/validator_node/Readme.md

@@ -4,8 +4,8 @@ This role creates the new enode data for the new validator nodes for an existing
 ### Tasks
 (Variables with * are fetched from the playbook which is calling this role)    
 
-#### 1. Set enode_data_list, enode_data and node_list to an empty list
-This task initializes the enode_data_list, enode_data and node_list variables to an empty string
+#### 1. Set enode_validator_list, new_validator_nodes and node_list to an empty list
+This task initializes the enode_validator_list, new_validator_nodes and node_list variables to an empty array of string
 
 **set_fact**: This module sets the variable assignment as globally accessible variable
 
@@ -18,16 +18,13 @@ This task tries to get enode from the vault for the new validator just to check
 
     loop_var: loop variable used for iterating the loop.
 
-#### 3. Get enode data
+#### 3. Get validator nodes data
 This task gets the enode data in the format of
+org_name
 peer_name
-enodeval
-p2p_ambassador
-raft_ambassador
-node_ambassador
-peer_publicIP
+org_type
 
-**include_task**: enode_data.yaml
+**include_task**: node_data.yaml
 **loop**: loop over the organizations
 **loop_control**: Specify conditions for controlling the loop.
 
@@ -93,9 +90,14 @@ This task extracts the besu binary dependencies and place it at appropriate path
 **when**: *add_new_org* is true.
 
 #### 12. Generate the nodeAddress for and key.pub for new validator
- This task creates the node address for each peer 
+ This task creates the node address and pub key for each new peer 
+
+**include_task**: enode_data.yaml
 **shell**: This task generates the nodeAddress and public key for the new validator node placed in node directories
-**when**: Condition specified here, It runs only when, besu binary is not found or crypto material are not found in vault.
+**loop**: loop over the new peers of validator organizations
+**loop_control**: validator new needs to have the variable status as new.
+
+    loop_var: loop variable used for iterating the loop.
 
 #### 13. Touch file to store information for validators
  This file stores the address of the validator nodes
@@ -112,22 +114,10 @@ This task extracts the besu binary dependencies and place it at appropriate path
 **shell**: This task generates the validator address array 
 **when**: *generate_crypto* count is *True*.
 
-#### 16. Copy the crypto material to Vault
-This task adds the crypto material to Vault
-**include_tasks**: add_to_vault
-**loop**: loops over all the node in a validator organisation
-**loop_control**: Specifies the condition for controlling the loop.
-    loop_var: loop variable used for iterating over the loop.
-##### Input Variables
-     *component_ns: Organization namespace   
-     *vault: Vault uri and token read from network.yaml
-     *peers: Peers in the organization
-**when**: Condition specified here, It runs only when, besu binary is not found or crypto material are not found in vault.
-
-#### 17. Voting for the new validator node to be added
-This task does the voting from every existing validator node for adding the new one.
-**include_tasks**: validator_vote
-**loop**: loops over all the node in a validator organisation
+#### 16. Voting for the new validator node to be added
+This task does the voting from every existing validator node for adding the new one (or new ones).
+**include_tasks**: validator_vote.yaml
+**loop**: loops over all the new nodes that will be added
 **loop_control**: Specifies the condition for controlling the loop.
     loop_var: loop variable used for iterating over the loop.
 ##### Input Variables
@@ -188,3 +178,76 @@ This task creates the build directory if it does not exist
 
 **set_fact**: This modules sets the enode_data_list variable globally to Get information about each validator node present in network.yaml and store it as a list of org, node.
 
+----------------
+
+### validator_vote.yaml
+### Tasks
+
+#### 1. Copy the crypto material to Vault
+This task adds the crypto material to Vault
+**include_tasks**: add_to_vault
+**loop**: loops over all the node in an organisation
+**loop_control**: Specifies the condition for controlling the loop.
+    loop_var: loop variable used for iterating over the loop.
+##### Input Variables
+     *component_ns: Organization namespace   
+     *vault: Vault uri and token read from network.yaml
+     *peers: Peers in the organization
+**when**: Condition specified here, It runs only when, besu binary is not found or crypto material are not found in vault.
+
+#### 2. Set enode_data_list to an empty list
+This task initializes the enode_data_list to an empty array of string
+
+#### 3. Get enode data
+This task gets the enode data in the format of
+peer_name
+enodeval
+p2p_ambassador
+raft_ambassador
+node_ambassador
+peer_publicIP
+
+**include_task**: enode_data.yaml
+**loop**: loop over the organizations
+**loop_control**: Specify conditions for controlling the loop.
+
+    loop_var: loop variable used for iterating the loop.
+
+#### 4. creates nodelist data
+**include_task**: nested_nodelist.yaml
+**loop**: loop over peers in the organization
+**loop_control**: Specify the conditions for controlling the loop.
+
+    loop_var: loop variable used for iterating the loop.
+
+### nested_nodelist.yaml
+### Tasks
+
+#### Check if enode is present in the build directory or not
+ This task checks if enode is present in the build directory or not
+
+**stat**: This module checks the file exist or not.
+##### Input variables
+    *path: path to the enode.
+##### Output variables
+    *file_status: output of the enode exists or not task
+
+#### 5. Voting proposal
+**include_task**: nested_validator_vote.yaml
+**loop**: loop over peers in the organization
+**loop_control**: Specify the conditions for controlling the loop.
+
+    loop_var: loop variable used for iterating the loop.
+
+### nested_validator_vote.yaml
+### Tasks
+
+#### 1. Voting by the validator nodes that were already in the network
+ This task does the voting part by the existing validator nodes
+
+#### 2. Voting by the new validator nodes that have been previously accepted in the same deployment
+ This task does the voting part by the new validator nodes that have been already accepted
+
+#### 3. Adding the new validator address to the new_validator_nodes array for future voting participation
+ This task adds the url of the new added validator node to be used in Task 2 if new validator nodes are pending to be accepted
+

platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/add_to_vault.yaml

@@ -1,11 +1,7 @@
 # This tasks copy the crypto material to the vault
 - name: Copy the crypto material to Vault
   shell: |
-    vault kv put {{ vault.secret_path | default('secret') }}/{{ component_ns }}/crypto/{{ peer.name }}/data key="$(cat {{ build_path }}/crypto/{{ component_name }}/{{ peer.name }}/data/key)" key_pub="$(cat {{ build_path }}/crypto/{{ component_name }}/{{ peer.name }}/data/key.pub)" nodeAddress="$(cat {{ build_path }}/crypto/{{ component_name }}/{{ peer.name }}/data/nodeAddress)"
+    vault kv put {{ vault.secret_path | default('secret') }}/{{ component_ns }}/crypto/{{ peer.0.name }}/data key="$(cat {{ build_path }}/crypto/{{ component_name }}/{{ peer.0.name }}/data/key)" key_pub="$(cat {{ build_path }}/crypto/{{ component_name }}/{{ peer.0.name }}/data/key.pub)" nodeAddress="$(cat {{ build_path }}/crypto/{{ component_name }}/{{ peer.0.name }}/data/nodeAddress)"
   environment:
     VAULT_ADDR: "{{ vault.url }}"
     VAULT_TOKEN: "{{ vault.root_token }}"
-  loop: "{{ org.services.validators }}"
-  when: peer.status == 'new'
-  loop_control:
-    loop_var: peer

platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/main.yaml

@@ -1,12 +1,12 @@
 # Set enode_data_list to empty
 - name: Set enode_validator_list to []
   set_fact:
-    enode_validator_list: [] 
+    enode_validator_list: []
 
 # Set enode_data_list to empty
-- name: Set enode_data_list to []
+- name: Set new_validator_nodes to []
   set_fact:
-    enode_data_list: []
+    new_validator_nodes: [] 
 
 # Set node_list to empty
 - name: Set node_list to []
@@ -101,7 +101,7 @@
   with_indexed_items: "{{ enode_validator_list }}"
   when:  
     - generate_crypto is defined and generate_crypto == True
-    
+
 # This task generates the enode for the new validator
 - name: "Creating the enode and pub key for the new validator"
   include_tasks: create_enode.yaml
@@ -131,50 +131,39 @@
   when:  
     - item[1].type == "validator"
     - generate_crypto is defined and generate_crypto == True
-    - add_new_org is not defined or add_new_org|bool == False
+
+# This file used by besu binary to generate the extra data information
+- name: Touch toEncode.json file
+  file:
+    path: "{{ build_path }}/toEncode.json"
+    state: touch
+  when:  
+    - generate_crypto is defined and generate_crypto == True
+
+# This task displays the JSON file content
+- name: Convert validatorInfo to json format
+  shell: |
+    cat {{ build_path }}/validatorinfo | paste -sd "," -| awk '{print "["$0"]"}'> {{ build_path }}/toEncode.json
 
 # This task displays the JSON file content
-- name: Display the validatorinfo
+- name: Convert validatorInfo to json format
   shell: |
-    cat {{ build_path }}/validatorinfo
+    cat {{ build_path }}/toEncode.json
   register: result
-
+    
 # This task saves the JSON data to a variable for future use
 - name: Save the data to a Variable as a Fact
   set_fact:
     validator_address: "{{ result.stdout }}"    
 
-# This task copy the crypto material to the vault
-- name: Copy the crypto material to Vault
-  include_tasks: add_to_vault.yaml
-  vars:    
-    vault: "{{ org.vault }}"
-    peers: "{{ org.services.validators }}"
-    component_name: "{{ org.name | lower }}"
-  loop: "{{ network['organizations'] }}"
-  loop_control:
-    loop_var: org
-  when: generate_crypto is defined and generate_crypto == True and org.type == 'validator'
-
-# Get enode data for all orgs
-- name: Get enode data for each node of all organization
-  include_tasks: enode_data.yaml
-  loop: "{{ network['organizations'] }}"
-  loop_control:
-    loop_var: org
-
-# Get nodelist for all peers
-- name: Get enode data for each node of all organization
-  include_tasks: nodelist.yaml
-  loop: "{{ network['organizations'] }}"
-  loop_control:
-    loop_var: org
-  when: org.type == 'validator'
-
 # This task contains the voting from the existing validators for the new validator to be added
 - name: Voting for the addition of newest validator
   include_tasks: validator_vote.yaml
-  loop: "{{ network.config.besu_nodes }}"
-  loop_control:
-    loop_var: node
+  vars:
+    besu_nodes: "{{ network.config.besu_nodes }}"
+    validator_addr: "{{ item.0 }}"
+    validator_name: "{{ item.1 }}"
+  with_together:
+    - "{{ validator_address }}"
+    - "{{ enode_validator_list }}"
   when: generate_crypto is defined and generate_crypto == True

platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/nested_enode_data.yaml

@@ -21,7 +21,6 @@
   environment:
     VAULT_ADDR: "{{ vault.url }}"
     VAULT_TOKEN: "{{ vault.root_token }}"
-  when: file_status.stat.exists == False
 
 # Get enode data
 - name: Get enode_data