Format beta integration (#1700)

* skiped docker validation for beta integration * added changelog * edit after testing * added support to format beta integration * changed the changelog description * edits after build failed * edits after pre-commit failed * edits after unittests failed * edits after unittests failed * Update demisto_sdk/commands/format/update_integration.py Co-authored-by: Shai Yaakovi <[email protected]> * Update demisto_sdk/commands/format/update_description.py Co-authored-by: Shai Yaakovi <[email protected]> Co-authored-by: Shai Yaakovi <[email protected]>
demisto · Jan 3, 2022 · 369d466 · 369d466
1 parent 22f49e3
commit 369d466
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 37 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,8 @@
 * Enhanced the **update-release-notes** command to automatically commit release notes config file upon creation.
 * The **validate** command will validate that an indicator field of type html has fromVersion of 6.1.0 and above.
 * The **format** command will now add fromVersion 6.1.0 to indicator field of type html.
+* Added support for beta integrations in the **format** command.
+
 
 # 1.5.5
 * Fixed an issue in the **update-release-notes** command, which did not work when changes were made in multiple packs.

diff --git a/demisto_sdk/commands/format/format_module.py b/demisto_sdk/commands/format/format_module.py
@@ -83,7 +83,6 @@
                      'image',
                      'javascriptfile',
                      'powershellfile',
-                     'betaintegration',
                      'doc_image',
                      'author_image'
                      ]
@@ -266,6 +265,9 @@ def run_format_on_file(input: str, file_type: str, from_version: str, **kwargs)
     Returns:
         List of Success , List of Error.
     """
+
+    if file_type == 'betaintegration':
+        file_type = 'integration'
     schema_path = os.path.normpath(
         os.path.join(__file__, "..", "..", "common", SCHEMAS_PATH, '{}.yml'.format(file_type)))
     if file_type not in ('integration', 'script') and 'update_docker' in kwargs:

diff --git a/demisto_sdk/commands/format/tests/test_formatting_md_test.py b/demisto_sdk/commands/format/tests/test_formatting_md_test.py
@@ -1,32 +1,13 @@
-import filecmp
-import os
-import shutil
-
-import pytest
-
 from demisto_sdk.commands.format.update_description import DescriptionFormat
 from demisto_sdk.tests.constants_test import (
-    DESCRIPTION_PATH, DESTINATION_FORMAT_DESCRIPTION_COPY,
     SOURCE_DESCRIPTION_FORMATTED_CONTRIB_DETAILS,
-    SOURCE_DESCRIPTION_WITH_CONTRIB_DETAILS)
+    SOURCE_DESCRIPTION_FORMATTED_WITH_BETA_DESCRIPTION,
+    SOURCE_DESCRIPTION_WITH_CONTRIB_DETAILS,
+    SOURCE_DESCRIPTION_WITHOUT_BETA_DESCRIPTION)
 
 
 class TestDescriptionFormat:
-    @pytest.fixture(autouse=True)
-    def description_copy(self):
-        os.makedirs(DESCRIPTION_PATH, exist_ok=True)
-        yield shutil.copyfile(SOURCE_DESCRIPTION_WITH_CONTRIB_DETAILS, DESTINATION_FORMAT_DESCRIPTION_COPY)
-        if os.path.exists(DESTINATION_FORMAT_DESCRIPTION_COPY):
-            os.remove(DESTINATION_FORMAT_DESCRIPTION_COPY)
-        shutil.rmtree(DESCRIPTION_PATH, ignore_errors=True)
-
-    @pytest.fixture(autouse=True)
-    def description_formatter(self, description_copy):
-        description_formatter = DescriptionFormat(
-            input=description_copy)
-        yield description_formatter
-
-    def test_format(self, description_formatter):
+    def test_remove_community_partner_details(self):
         """
         Given
             - A description file that might contain community/partner details
@@ -35,5 +16,23 @@ def test_format(self, description_formatter):
         Then
             - Ensure the details are deleted from the description file
         """
-        description_formatter.remove_community_partner_details()
-        assert filecmp.cmp(description_formatter.source_file, SOURCE_DESCRIPTION_FORMATTED_CONTRIB_DETAILS) is True
+        with open(SOURCE_DESCRIPTION_FORMATTED_CONTRIB_DETAILS, 'r') as f:
+            expected = f.read()
+        formatter = DescriptionFormat(input=SOURCE_DESCRIPTION_WITH_CONTRIB_DETAILS)
+        formatter.remove_community_partner_details()
+        assert formatter.description_content == expected
+
+    def test_format_beta_description(self):
+        """
+        Given
+            - A description file for beta integration with no beta description
+        When
+            - Run format on it
+        Then
+            - Ensure the beta description is added to the description file
+        """
+        with open(SOURCE_DESCRIPTION_FORMATTED_WITH_BETA_DESCRIPTION, 'r') as f:
+            expected = f.read()
+        formatter = DescriptionFormat(input=SOURCE_DESCRIPTION_WITHOUT_BETA_DESCRIPTION)
+        formatter.add_betaintegration_description()
+        assert formatter.description_content == expected
diff --git a/demisto_sdk/commands/format/tests/test_formatting_yml_test.py b/demisto_sdk/commands/format/tests/test_formatting_yml_test.py
@@ -32,10 +32,11 @@
     DESTINATION_FORMAT_SCRIPT_COPY, DESTINATION_FORMAT_TEST_PLAYBOOK,
     FEED_INTEGRATION_EMPTY_VALID, FEED_INTEGRATION_INVALID,
     FEED_INTEGRATION_VALID, GIT_ROOT, INTEGRATION_PATH, PLAYBOOK_PATH,
-    PLAYBOOK_WITH_INCIDENT_INDICATOR_SCRIPTS, SOURCE_FORMAT_INTEGRATION_COPY,
-    SOURCE_FORMAT_INTEGRATION_INVALID, SOURCE_FORMAT_INTEGRATION_VALID,
-    SOURCE_FORMAT_PLAYBOOK, SOURCE_FORMAT_PLAYBOOK_COPY,
-    SOURCE_FORMAT_SCRIPT_COPY, SOURCE_FORMAT_TEST_PLAYBOOK, TEST_PLAYBOOK_PATH)
+    PLAYBOOK_WITH_INCIDENT_INDICATOR_SCRIPTS, SOURCE_BETA_INTEGRATION_FILE,
+    SOURCE_FORMAT_INTEGRATION_COPY, SOURCE_FORMAT_INTEGRATION_INVALID,
+    SOURCE_FORMAT_INTEGRATION_VALID, SOURCE_FORMAT_PLAYBOOK,
+    SOURCE_FORMAT_PLAYBOOK_COPY, SOURCE_FORMAT_SCRIPT_COPY,
+    SOURCE_FORMAT_TEST_PLAYBOOK, TEST_PLAYBOOK_PATH)
 from TestSuite.test_tools import ChangeCWD
 
 ryaml = YAML()
@@ -654,6 +655,21 @@ def test_update_tests_on_integration_with_test_playbook(self):
         assert res is None
         assert formatter.data.get('tests') == ['VMWare Test']
 
+    def test_format_beta_integration(self):
+        """
+        Given
+            - A beta integration yml file
+        When
+            - Run format on it
+        Then
+            - Ensure the display name contains the word (Beta), the param beta is True
+        """
+
+        formatter = IntegrationYMLFormat(input=SOURCE_BETA_INTEGRATION_FILE)
+        formatter.update_beta_integration()
+        assert '(Beta)' in formatter.data['display']
+        assert formatter.data['beta'] is True
+
     def test_update_docker_format(self, tmpdir, mocker, monkeypatch):
         """Test that script and integration formatter update docker image tag
         """

diff --git a/demisto_sdk/commands/format/update_description.py b/demisto_sdk/commands/format/update_description.py
@@ -3,6 +3,8 @@
 
 import click
 
+from demisto_sdk.commands.common.constants import BETA_INTEGRATION_DISCLAIMER
+from demisto_sdk.commands.common.tools import find_type
 from demisto_sdk.commands.format.format_constants import (ERROR_RETURN_CODE,
                                                           SKIP_RETURN_CODE,
                                                           SUCCESS_RETURN_CODE)
@@ -28,23 +30,40 @@ def __init__(self,
                  update_docker: bool = False,
                  **kwargs):
         super().__init__(input, output, path, from_version, no_validate, verbose=verbose, **kwargs)
+        description_type = input.replace('_description.md', '.yml')
+        self.is_beta = False
+        file_type = find_type(description_type)
+        if file_type:
+            self.is_beta = find_type(description_type).value == 'betaintegration'
+        with open(self.source_file, 'r') as f:
+            self.description_content = f.read()
 
     def remove_community_partner_details(self):
         """update description file to not contain community/partner details"""
 
-        with open(self.source_file, 'r') as f:
-            description_content = f.read()
-        formatted_description = re.sub('###.*Contributed Integration[\\S\n ]+?[*]{3}[\n]*', "", description_content)
-        formatted_description = formatted_description.rstrip("\n")
+        formatted_description = re.sub('###.*Contributed Integration[\\S\n ]+?[*]{3}[\n]*', "", self.description_content)
+        self.description_content = formatted_description.rstrip("\n")
+
+    def add_betaintegration_description(self):
+        """update description file of a beta integration to contain beta integration's description"""
+        if BETA_INTEGRATION_DISCLAIMER not in self.description_content:
+            self.description_content = BETA_INTEGRATION_DISCLAIMER + '\n' + self.description_content
 
-        with open(self.source_file, 'w') as f:
-            f.write(formatted_description)
+    def save_md_to_destination_file(self):
+        """Safely saves formatted YML data to destination file."""
+        if self.source_file != self.output_file and self.verbose:
+            click.secho(f'Saving output description file to {self.output_file} \n', fg='white')
+        with open(self.output_file, 'w') as f:
+            f.write(self.description_content)
         f.close()
 
     def run_format(self) -> int:
         try:
             click.secho(f'\n================= Updating file {self.source_file} ================= ', fg='bright_blue')
             self.remove_community_partner_details()
+            if self.is_beta:
+                self.add_betaintegration_description()
+            self.save_md_to_destination_file()
             return SUCCESS_RETURN_CODE
         except Exception as err:
             if self.verbose:

diff --git a/demisto_sdk/commands/format/update_integration.py b/demisto_sdk/commands/format/update_integration.py
@@ -4,9 +4,11 @@
 import click
 
 from demisto_sdk.commands.common.constants import (BANG_COMMAND_NAMES,
+                                                   BETA_INTEGRATION,
                                                    FEED_REQUIRED_PARAMS,
                                                    FETCH_REQUIRED_PARAMS,
                                                    INTEGRATION, TYPE_PWSH)
+from demisto_sdk.commands.common.tools import find_type
 from demisto_sdk.commands.format.format_constants import (ERROR_RETURN_CODE,
                                                           SKIP_RETURN_CODE,
                                                           SUCCESS_RETURN_CODE)
@@ -41,6 +43,10 @@ def __init__(self,
         self.update_docker = update_docker
         if not from_version and self.data.get("script", {}).get("type") == TYPE_PWSH:
             self.from_version = '5.5.0'
+        self.is_beta = False
+        integration_type = find_type(input)
+        if integration_type:
+            self.is_beta = find_type(input).value == 'betaintegration'
 
     def update_proxy_insecure_param_to_default(self):
         """Updates important integration arguments names and description."""
@@ -59,6 +65,7 @@ def update_proxy_insecure_param_to_default(self):
     def set_params_default_additional_info(self):
         from demisto_sdk.commands.common.default_additional_info_loader import \
             load_default_additional_info_dict
+
         default_additional_info = load_default_additional_info_dict()
 
         if self.verbose:
@@ -147,10 +154,14 @@ def update_docker_image(self):
             ScriptYMLFormat.update_docker_image_in_script(self.data['script'], self.source_file,
                                                           self.data.get(self.from_version_key))
 
+    def update_beta_integration(self):
+        self.data['display'] = self.data['name'] + ' (Beta)'
+        self.data['beta'] = True
+
     def run_format(self) -> int:
         try:
             click.secho(f'\n================= Updating file {self.source_file} =================', fg='bright_blue')
-            super().update_yml(file_type=INTEGRATION)
+            super().update_yml(file_type=BETA_INTEGRATION if self.is_beta else INTEGRATION)
             self.update_tests()
             self.update_conf_json('integration')
             self.update_proxy_insecure_param_to_default()
@@ -159,7 +170,12 @@ def run_format(self) -> int:
             self.set_fetch_params_in_config()
             self.set_feed_params_in_config()
             self.update_docker_image()
+
+            if self.is_beta:
+                self.update_beta_integration()
+
             self.save_yml_to_destination_file()
+
             return SUCCESS_RETURN_CODE
         except Exception as err:
             if self.verbose:

diff --git a/demisto_sdk/tests/constants_test.py b/demisto_sdk/tests/constants_test.py
@@ -97,6 +97,7 @@
 SCRIPT_RELEASE_NOTES_TARGET = f"{PACK_TARGET}/Scripts/script-test_CHANGELOG.md"
 INTEGRATION_RELEASE_NOTES_TARGET = f"{PACK_TARGET}/Integrations/integration-test_CHANGELOG.md"
 SOURCE_FORMAT_INTEGRATION_COPY = f"{GIT_ROOT}/demisto_sdk/tests/test_files/format_New_Integration_copy.yml"
+SOURCE_BETA_INTEGRATION_FILE = f"{GIT_ROOT}/demisto_sdk/tests/test_files/source_beta_integration.yml"
 DESTINATION_FORMAT_INTEGRATION_COPY = "new_format_New_Integration_copy.yml"
 SOURCE_FORMAT_SCRIPT_COPY = f"{GIT_ROOT}/demisto_sdk/tests/test_files/format_New_script_copy.yml"
 DESTINATION_FORMAT_SCRIPT_COPY = "new_format_New_script_copy.yml"
@@ -129,6 +130,10 @@
 SOURCE_DESCRIPTION_WITH_CONTRIB_DETAILS = f"{GIT_ROOT}/demisto_sdk/tests/test_files/description_with_contrib_details.md"
 SOURCE_DESCRIPTION_FORMATTED_CONTRIB_DETAILS = f"{GIT_ROOT}/demisto_sdk/tests/test_files/" \
                                                f"description_formatted_contrib_details.md"
+SOURCE_DESCRIPTION_WITHOUT_BETA_DESCRIPTION = f"{GIT_ROOT}/demisto_sdk/tests/test_files/" \
+                                              f"description_without_beta_description-test.md"
+SOURCE_DESCRIPTION_FORMATTED_WITH_BETA_DESCRIPTION = f"{GIT_ROOT}/demisto_sdk/tests/test_files/" \
+    f"description_formatted_with_beta_description-test.md"
 DESTINATION_FORMAT_DESCRIPTION_COPY = "Description/formatted_description-test.md"
 DESCRIPTION_PATH = "Description"
 DESTINATION_FORMAT_INCIDENTTYPE_COPY = "IncidentTypes/incidenttype-copy.json"

diff --git a/demisto_sdk/tests/test_files/description_formatted_with_beta_description-test.md b/demisto_sdk/tests/test_files/description_formatted_with_beta_description-test.md
@@ -0,0 +1,6 @@
+Note: This is a beta Integration, which lets you implement and test pre-release software. Since the integration is beta, it might contain bugs. Updates to the integration during the beta phase might include non-backward compatible features. We appreciate your feedback on the quality and usability of the integration to help us identify issues, fix them, and continually improve.
+## Integration
+- data for the test
+
+### Other section
+more data
diff --git a/demisto_sdk/tests/test_files/description_without_beta_description-test.md b/demisto_sdk/tests/test_files/description_without_beta_description-test.md
@@ -0,0 +1,5 @@
+## Integration
+- data for the test
+
+### Other section
+more data
diff --git a/demisto_sdk/tests/test_files/source_beta_integration.yml b/demisto_sdk/tests/test_files/source_beta_integration.yml
@@ -0,0 +1,35 @@
+category: IT Services
+commonfields:
+  id: test integration
+  version: -1
+configuration:
+- display: Application ID
+  hidden: false
+  name: app_id
+  required: true
+  defaultvalue: 40191
+  type: 0
+description: description
+  service.
+display: test integration
+name: test integration
+script:
+  commands:
+  - deprecated: false
+    description: test command.
+    execution: false
+    name: command-test
+  dockerimage: demisto/pto:1.0.0.24047
+  feed: false
+  isfetch: true
+  longRunning: false
+  longRunningPort: false
+  runonce: false
+  script: '-'
+  subtype: python3
+  type: python
+fromversion: 6.0.0
+defaultmapperin: testing Incoming Mapper
+defaultclassifier: testing
+tests:
+- betaTest