EC2_group module refactor (formerly pr/37255) (ansible#38678)

* Refactor ec2_group

Replace nested for loops with list comprehensions

Purge rules before adding new ones in case sg has maximum permitted rules

* Add check mode tests for ec2_group

* add tests

* Remove dead code

* Fix integration test assertions for old boto versions

* Add waiter for security group that is autocreated

* Add support for in-account group rules

* Add common util to get AWS account ID

Fixes ansible#31383

* Fix protocol number and add separate tests for egress rule handling

* Return egress rule treatment to be backwards compatible

* Remove functions that were obsoleted by `Rule` namedtuple

* IP tests

* Move description updates to a function

* Fix string formatting missing index

* Add tests for auto-creation of the same group in quick succession

* Resolve use of brand-new group in a rule without a description

* Clean up duplicated get-security-group function

* Add reverse cleanup in case of dependency issues

* Add crossaccount ELB group support

* Deal with non-STS calls to account API

* Add filtering of owner IDs that match the current account

lib/ansible/module_utils/aws/iam.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+import traceback
+
+try:
+    from botocore.exceptions import ClientError, NoCredentialsError
+except ImportError:
+    pass  # caught by HAS_BOTO3
+
+from ansible.module_utils._text import to_native
+
+
+def get_aws_account_id(module):
+    """ Given AnsibleAWSModule instance, get the active AWS account ID
+
+    get_account_id tries too find out the account that we are working
+    on.  It's not guaranteed that this will be easy so we try in
+    several different ways.  Giving either IAM or STS privilages to
+    the account should be enough to permit this.
+    """
+    account_id = None
+    try:
+        sts_client = module.client('sts')
+        account_id = sts_client.get_caller_identity().get('Account')
+    # non-STS sessions may also get NoCredentialsError from this STS call, so
+    # we must catch that too and try the IAM version
+    except (ClientError, NoCredentialsError):
+        try:
+            iam_client = module.client('iam')
+            account_id = iam_client.get_user()['User']['Arn'].split(':')[4]
+        except ClientError as e:
+            if (e.response['Error']['Code'] == 'AccessDenied'):
+                except_msg = to_native(e)
+                # don't match on `arn:aws` because of China region `arn:aws-cn` and similar
+                account_id = except_msg.search(r"arn:\w+:iam::([0-9]{12,32}):\w+/").group(1)
+            if account_id is None:
+                module.fail_json_aws(e, msg="Could not get AWS account information")
+        except Exception as e:
+            module.fail_json(
+                msg="Failed to get AWS account information, Try allowing sts:GetCallerIdentity or iam:GetUser permissions.",
+                exception=traceback.format_exc()
+            )
+    if not account_id:
+        module.fail_json(msg="Failed while determining AWS account ID. Try allowing sts:GetCallerIdentity or iam:GetUser permissions.")
+    return to_native(account_id)

lib/ansible/module_utils/aws/waiters.py

@@ -27,6 +27,24 @@
                 },
             ]
         },
+        "SecurityGroupExists": {
+            "delay": 5,
+            "maxAttempts": 40,
+            "operation": "DescribeSecurityGroups",
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "expected": True,
+                    "argument": "length(SecurityGroups[]) > `0`",
+                    "state": "success"
+                },
+                {
+                    "matcher": "error",
+                    "expected": "InvalidGroup.NotFound",
+                    "state": "retry"
+                },
+            ]
+        },
         "SubnetExists": {
             "delay": 5,
             "maxAttempts": 40,
@@ -179,6 +197,12 @@ def waf_model(name):
         core_waiter.NormalizedOperationMethod(
             ec2.describe_route_tables
         )),
+    ('EC2', 'security_group_exists'): lambda ec2: core_waiter.Waiter(
+        'security_group_exists',
+        ec2_model('SecurityGroupExists'),
+        core_waiter.NormalizedOperationMethod(
+            ec2.describe_security_groups
+        )),
     ('EC2', 'subnet_exists'): lambda ec2: core_waiter.Waiter(
         'subnet_exists',
         ec2_model('SubnetExists'),