From a32f58e56c0e2fdb905e9de54a03b2703b88d9c0 Mon Sep 17 00:00:00 2001
From: Yishi Wang <yishiwang@microsoft.com>
Date: Tue, 15 Aug 2023 13:02:58 +0800
Subject: [PATCH] {CI} Enhance extension regression test pipeline (#25525)

* Enhance extension regression test pipeline

* fix syntax

* checkout cli custom repo

* rerun setup repo path

* revert repo path

* fix NoneType error

* test fix
---
 scripts/ci/automation_full_test.py            | 20 ++++++++++++++++++-
 .../extension_regression_test.yml             |  8 +++++++-
 scripts/regression_test/regression_test.yml   |  2 +-
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/scripts/ci/automation_full_test.py b/scripts/ci/automation_full_test.py
index 35159cc31fd..52bf2da7f04 100644
--- a/scripts/ci/automation_full_test.py
+++ b/scripts/ci/automation_full_test.py
@@ -36,7 +36,7 @@
 pull_request_number = os.environ.get('PULL_REQUEST_NUMBER', None)
 enable_pipeline_result = bool(job_name and python_version)
 unique_job_name = ' '.join([job_name, python_version, profile, str(instance_idx)]) if enable_pipeline_result else None
-enable_traceback = True if os.environ.get('ENABLE_TRACEBACK').lower() == 'true' else False
+enable_traceback = True if (os.environ.get('ENABLE_TRACEBACK') is not None and os.environ.get('ENABLE_TRACEBACK').lower() == 'true') else False
 cli_jobs = {
             'acr': 45,
             'acs': 62,
@@ -271,6 +271,19 @@ def remove_extension(extension_module):
     return error_flag
 
 
+def rerun_setup(cli_repo_path, extension_repo_path):
+    try:
+        if extension_repo_path:
+            cmd = ['azdev', 'setup', '-c', cli_repo_path, '-r', extension_repo_path, '--debug']
+        else:
+            cmd = ['azdev', 'setup', '-c', cli_repo_path, '--debug']
+        error_flag = run_command(cmd, check_return_code=True)
+    except Exception:
+        error_flag = True
+
+    return error_flag
+
+
 def git_restore(file_path):
     if not file_path:
         return
@@ -599,12 +612,17 @@ def run_extension_instance_modules(self, instance_modules):
         for module, path in instance_modules.items():
             run_command(["git", "checkout", f"regression_test_{os.getenv('BUILD_BUILDID')}"], check_return_code=True)
             error_flag = install_extension(module)
+            logger.info(f"Finish installing extension {module}, error_flag: {error_flag}")
             if not error_flag:
                 azdev_test_result_fp = os.path.join(azdev_test_result_dir, f"test_results_{module}.xml")
                 cmd = ['azdev', 'test', module, '--discover', '--no-exitfirst', '--verbose',
                        '--xml-path', azdev_test_result_fp, '--pytest-args', '"--durations=10"']
                 error_flag = process_test(cmd, azdev_test_result_fp, live_rerun=fix_failure_tests, modules=[module])
+                logger.info(f"Finish testing extension {module}, error_flag: {error_flag}")
             remove_extension(module)
+            logger.info(f"Finish removing extension {module}, error_flag: {error_flag}")
+            if error_flag:
+                rerun_setup(cli_repo_path=os.getenv('BUILD_SOURCESDIRECTORY'), extension_repo_path=working_directory)
             global_error_flag = global_error_flag or error_flag
         return global_error_flag
 
diff --git a/scripts/regression_test/extension_regression_test.yml b/scripts/regression_test/extension_regression_test.yml
index 5cf1a9a2bab..38bd8e8aa16 100644
--- a/scripts/regression_test/extension_regression_test.yml
+++ b/scripts/regression_test/extension_regression_test.yml
@@ -86,6 +86,12 @@ jobs:
       inlineScript: |
         set -ev
         pwd
+        if [[ -n "$(CUSTOM_CLI_REPO)" && -n "$(CUSTOM_CLI_BRANCH)" ]]; then
+          git remote add ${CUSTOM_CLI_REPO} https://github.com/${CUSTOM_CLI_REPO}/azure-cli.git
+          git fetch ${CUSTOM_CLI_REPO} $(CUSTOM_CLI_BRANCH)
+          git checkout -b ${CUSTOM_CLI_BRANCH} ${CUSTOM_CLI_REPO}/${CUSTOM_CLI_BRANCH}
+        fi
+
         git clone https://github.com/Azure/azure-cli-extensions.git
         cd azure-cli-extensions
 
@@ -146,7 +152,7 @@ jobs:
 - job: ResultSummary
   displayName: CLI Extension Regression Test Result Summary
   dependsOn: CLIExtensionRegressionTest
-  condition: succeededOrFailed()
+  condition: and(succeededOrFailed(), eq(variables['CUSTOM_CLI_REPO'], ''))
   pool:
     name: ${{ variables.ubuntu_pool }}
   steps:
diff --git a/scripts/regression_test/regression_test.yml b/scripts/regression_test/regression_test.yml
index 7ed4252a486..fa3dd15e5a7 100644
--- a/scripts/regression_test/regression_test.yml
+++ b/scripts/regression_test/regression_test.yml
@@ -110,7 +110,7 @@ jobs:
           git config --global user.email "AzPyCLI@microsoft.com"
           git config --global user.name "Azure CLI Team"
           git remote add ${GITHUB_REPO} https://${GITHUB_REPO}:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}/azure-cli.git
-          git fetch ${GITHUB_REPO}
+          git fetch ${GITHUB_REPO} ${GITHUB_BRANCH}
 
           git checkout -b ${GITHUB_BRANCH} ${GITHUB_REPO}/${GITHUB_BRANCH}
     - template: ../../.azure-pipelines/templates/azdev_setup.yml