From a0349a911ceeba68ac1be15addfc200ed3d32c92 Mon Sep 17 00:00:00 2001 From: stealthycoin Date: Mon, 12 Nov 2018 21:44:01 -0800 Subject: [PATCH] Rewrite cli skeleton integration tests to be faster The generate cli skeleton tests no longer make a subprocess call per operation in all of our models. This was very time consuming, and doesn't give any benefit over just using the clidriver directly. --- .../test_generatecliskeleton.py | 113 +++++++++++++----- 1 file changed, 80 insertions(+), 33 deletions(-) diff --git a/tests/integration/customizations/test_generatecliskeleton.py b/tests/integration/customizations/test_generatecliskeleton.py index d5736bf0d0dc..b64ca34c90bd 100644 --- a/tests/integration/customizations/test_generatecliskeleton.py +++ b/tests/integration/customizations/test_generatecliskeleton.py @@ -10,47 +10,18 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os import json +import logging +import mock from nose.tools import assert_equal +from awscli.testutils import unittest, aws, capture_output from awscli.clidriver import create_clidriver -from awscli.testutils import unittest, aws from awscli.customizations.preview import PREVIEW_SERVICES -def test_can_generate_skeletons_for_all_service_comands(): - driver = create_clidriver() - help_command = driver.create_help_command() - for command_name, command_obj in help_command.command_table.items(): - if command_name in PREVIEW_SERVICES: - # Skip over any preview services for now. - continue - sub_help = command_obj.create_help_command() - # This avoids command objects like ``PreviewModeCommand`` that - # do not exhibit any visible functionality (i.e. provides a command - # for the CLI). - if hasattr(sub_help, 'command_table'): - for sub_name, sub_command in sub_help.command_table.items(): - op_help = sub_command.create_help_command() - arg_table = op_help.arg_table - if 'generate-cli-skeleton' in arg_table: - yield _test_gen_skeleton, command_name, sub_name - - -def _test_gen_skeleton(command_name, operation_name): - p = aws('%s %s --generate-cli-skeleton' % (command_name, operation_name)) - assert_equal(p.rc, 0, 'Received non zero RC (%s) for command: %s %s' - % (p.rc, command_name, operation_name)) - try: - parsed = json.loads(p.stdout) - except ValueError as e: - raise AssertionError( - "Could not generate CLI skeleton for command: %s %s\n" - "stdout:\n%s\n" - "stderr:\n%s\n" % (command_name, operation_name)) - - class TestIntegGenerateCliSkeleton(unittest.TestCase): """This tests various services to see if the generated skeleton is correct @@ -90,3 +61,79 @@ def test_generate_cli_skeleton_iam(self): json.loads(p.stdout), {'Path': '', 'GroupName': ''} ) + + +def test_can_generate_skeletons_for_all_service_comands(): + environ = { + 'AWS_DATA_PATH': os.environ['AWS_DATA_PATH'], + 'AWS_DEFAULT_REGION': 'us-east-1', + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'AWS_CONFIG_FILE': '', + 'AWS_SHARED_CREDENTIALS_FILE': '', + } + with mock.patch('os.environ', environ): + driver = create_clidriver() + help_command = driver.create_help_command() + for command_name, command_obj in help_command.command_table.items(): + if command_name in PREVIEW_SERVICES: + # Skip over any preview services for now. + continue + sub_help = command_obj.create_help_command() + # This avoids command objects like ``PreviewModeCommand`` that + # do not exhibit any visible functionality (i.e. provides a command + # for the CLI). + if hasattr(sub_help, 'command_table'): + for sub_name, sub_command in sub_help.command_table.items(): + op_help = sub_command.create_help_command() + arg_table = op_help.arg_table + if 'generate-cli-skeleton' in arg_table: + yield _test_gen_skeleton, command_name, sub_name, + + +def _test_gen_skeleton(command_name, operation_name): + command = '%s %s --generate-cli-skeleton' % (command_name, + operation_name) + stdout, stderr, _ = _run_cmd(command) + # Test that a valid JSON blob is emitted to stdout is valid. + try: + json.loads(stdout) + except ValueError as e: + raise AssertionError( + "Could not generate CLI skeleton for command: %s %s\n" + "stdout:\n%s\n" + "stderr:\n%s\n" % (command_name, operation_name, stdout, + stderr)) + + +def _run_cmd(cmd, expected_rc=0): + logging.debug("Calling cmd: %s", cmd) + # Drivers do not seem to be reusable since the formatters seem to not clear + # themselves between runs. This is fine in practice since a driver is only + # called once but for tests it means we need to create a new driver for + # each test, which is far more heavyweight than it needs to be. Might be + # worth seeing if we can make drivers reusable to speed these up generated + # tests. + driver = create_clidriver() + if not isinstance(cmd, list): + cmdlist = cmd.split() + else: + cmdlist = cmd + + with capture_output() as captured: + try: + rc = driver.main(cmdlist) + except SystemExit as e: + # We need to catch SystemExit so that we + # can get a proper rc and still present the + # stdout/stderr to the test runner so we can + # figure out what went wrong. + rc = e.code + stderr = captured.stderr.getvalue() + stdout = captured.stdout.getvalue() + assert_equal( + rc, expected_rc, + "Unexpected rc (expected: %s, actual: %s) for command: %s\n" + "stdout:\n%sstderr:\n%s" % ( + expected_rc, rc, cmd, stdout, stderr)) + return stdout, stderr, rc