New command upgrade has been created

installer/core/providers/aws/install.py

@@ -174,7 +174,8 @@ def terraform_apply(self, resources, terraform_with_targets, dry_run):
         self.current_install_status = self.install_statuses.get('tf_init_complete')
 
         self.current_install_status = self.install_statuses.get('tf_plan_start')
-        py_terraform.terraform_plan(apply_resources)
+        response = py_terraform.terraform_plan(apply_resources)
+        self._set_resource_creation_count(response)
         self.current_install_status = self.install_statuses.get('tf_plan_complete')
 
         for resource in resources:
@@ -188,6 +189,27 @@ def terraform_apply(self, resources, terraform_with_targets, dry_run):
             for resource in resources:
                 resource.post_terraform_apply()
 
+    def _set_resource_creation_count(self, plan_response):
+        """
+        Set resources craetion count from terraform plan
+
+        Args:
+            resources (list): Resources created
+        """
+        to_add = to_change = 0
+        try:
+            lines = plan_response[1].split("\n")
+            for line in lines:
+                if "Plan:" in line and "to add" in line and "to change" in line:  # This needs to be changed with reqular expression
+                    req_str = line.split("Plan:")[1].strip()
+                    to_add = int(req_str.split("to add,")[0].strip())
+                    to_change = int(req_str.split("to add,")[1].strip().split("to change,")[0].strip())
+                    break
+        except Exception as e:
+            return
+
+        self.total_resources_count = to_add + to_change
+
     def render_resource_outputs(self, resources):
         """
         After installation is completed list down all the outputs to be rendered by calling render_output hook
@@ -278,7 +300,7 @@ def render_terraform_apply_progress(self, resources, terraform_with_targets):
                 if counter:
                     try:
                         # output_count = len(py_terraform.load_terraform_output())  # This uses terraform output command
-                        output_count = self.files_count_in_output_status_dir()
+                        output_count = self.files_count_in_output_status_dir() - 1
                         prev_output_count = output_count
                     except:
                         output_count = prev_output_count

installer/core/terraform/resources/aws/rds.py

@@ -30,7 +30,8 @@ class RDSResource(TerraformResource):
         'parameter_group_name': {'required': False},
         'vpc_security_group_ids': {'required': False},
         'final_snapshot_identifier': {'required': False},
-        'tags': {'required': False}
+        'tags': {'required': False},
+        'apply_immediately': {'required': False}
     }
 
     def check_exists_before(self, input, tf_outputs):

installer/custom/commands/upgrade.py

@@ -0,0 +1,78 @@
+from core.commands import BaseCommand
+from core.config import Settings
+from core import constants as K
+from core.terraform import PyTerraform
+from resources.iam.all_read_role import AllReadRole
+import importlib
+import os
+
+
+class Upgrade(BaseCommand):
+    """
+    This calss is defined to create new command to upgrade PacBot RDS, ES and roles
+
+    Attributes:
+        validation_class (class): This validate the input and resources
+        input_class (class): Main class to read input from user
+        install_class (class): Provider based install class
+
+    """
+    def __init__(self, args):
+        # args.append((K.CATEGORY_FIELD_NAME, "datastore"))
+        # tf_outputs = PyTerraform.load_terraform_output_from_json_file()
+        # role_file = os.path.join(Settings.TERRAFORM_DIR, "iam_all_read_role_AllReadRole.tf")
+        # if not tf_outputs.get(AllReadRole.get_resource_id(), False):
+        #     args.append((K.CATEGORY_FIELD_NAME, "all_read_role"))
+        #     args.append((K.CATEGORY_FIELD_NAME, "ecs_role"))
+
+        Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True)
+        super().__init__(args)
+
+    def execute(self, provider):
+        """
+        Command execution starting point
+
+        Args:
+            provider (string): Provider name like AWS or Azure etc
+        """
+        self.initialize_install_classes(provider)
+
+        if self.check_pre_requisites() is False:
+            self.exit_system_with_pre_requisites_fail()
+
+        input_instance = self.read_input()
+        self.upgrade_pacbot(input_instance)
+
+    def initialize_install_classes(self, provider):
+        """
+        Initialise classes based on the provider
+
+        Args:
+            provider (string): Provider name like AWS or Azure etc
+        """
+        self.validation_class = getattr(importlib.import_module(
+            provider.provider_module + '.validate'), 'SystemInstallValidation')
+        self.input_class = getattr(importlib.import_module(
+            provider.provider_module + '.input'), 'SystemInstallInput')
+        self.install_class = getattr(importlib.import_module(
+            provider.provider_module + '.install'), 'Install')
+
+    def upgrade_pacbot(self, input_instance):
+        """
+        Upgrade RDS, ES and roles if any by running terraform apply for those resources
+
+        Args:
+            input_instance (Input object): User input values
+        """
+        terraform_with_targets = False
+        resources_to_process = self.get_complete_resources(input_instance)
+
+        self.install_class(
+            self.args,
+            input_instance,
+            check_dependent_resources=False
+        ).execute(
+            resources_to_process,
+            terraform_with_targets,
+            self.dry_run
+        )

installer/resources/batch/job.py

@@ -52,13 +52,13 @@ def pre_terraform_destroy(self):
 
 class RuleEngineJobQueue(BatchJobQueueResource):
     name = "rule-engine"
-    state = "ENABLED"
+    state = Settings.get('JOB_QUEUE_STATUS', "ENABLED")
     priority = 6
     compute_environments = [RuleEngineBatchJobEnv.get_output_attr('arn')]
 
 
 class BatchJobsQueue(BatchJobQueueResource):
     name = "data"
-    state = "ENABLED"
+    state = Settings.get('JOB_QUEUE_STATUS', "ENABLED")
     priority = 6
     compute_environments = [RuleEngineBatchJobEnv.get_output_attr('arn')]

installer/resources/datastore/db.py

@@ -37,6 +37,7 @@ class MySQLDatabase(RDSResource):
     db_subnet_group_name = DBSubnetGroup.get_input_attr('name')
     vpc_security_group_ids = [InfraSecurityGroupResource.get_output_attr('id')]
     skip_final_snapshot = True
+    apply_immediately = True
 
     DEPENDS_ON = [DBOptionGroup, DBParameterGroup, DBSubnetGroup]
 

installer/resources/iam/ecs_role.py

@@ -59,11 +59,6 @@ class ECSContainerServiceForEC2PolicyAttach(iam.IAMRolePolicyAttachmentResource)
     policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
 
 
-class ECSGuardDutyReadOnlyPolicyAttach(iam.IAMRolePolicyAttachmentResource):
-    role = ECSRole.get_output_attr('name')
-    policy_arn = "arn:aws:iam::aws:policy/AmazonGuardDutyReadOnlyAccess"
-
-
 class ECSCloudWatchLogsFullAccessPolicyAttach(iam.IAMRolePolicyAttachmentResource):
     role = ECSRole.get_output_attr('name')
     policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"

installer/settings/common.py

@@ -34,13 +34,13 @@
     'data.aws_info': {'tags': ["roles"]},  # This should not be removed
     'iam.base_role': {'tags': ["roles"]},
     'iam.batch_role': {'tags': ["roles"]},
-    'iam.ecs_role': {'tags': ["roles"]},
+    'iam.ecs_role': {'tags': ["roles", "ecs_role"]},
     'iam.lambda_role': {'tags': ["roles"]},
     'iam.base_role_policy': {'tags': ["roles"]},
-    'iam.all_read_role': {'tags': ["roles"]},
+    'iam.all_read_role': {'tags': ["roles", "all_read_role"]},
     'vpc.security_group': {'tags': ["security"]},
-    'datastore.db': {'tags': ["rds"]},
-    'datastore.es': {'tags': ["es"]},
+    'datastore.db': {'tags': ["rds", "datastore"]},
+    'datastore.es': {'tags': ["es", "datastore"]},
     'pacbot_app.alb': {'tags': ["infra"]},
     'pacbot_app.alb_target_groups': {'tags': ["infra"]},
     'pacbot_app.alb_listener_rules': {'tags': ["infra"]},