Merge branch 'release-1.11.180'

* release-1.11.180:
  Bumping version to 1.11.180
  Update changelog based on model updates
  codepipeline -> datapipeline
  Move max_items to module level
  Fix list-runs pagination bug

.changes/1.11.180.json

@@ -0,0 +1,12 @@
+[
+  {
+    "category": "``apigateway``", 
+    "description": "Update apigateway command to latest version", 
+    "type": "api-change"
+  }, 
+  {
+    "category": "datapipeline", 
+    "description": "Fixed a bug in datapipeline where list-runs could only handle 100 runs.", 
+    "type": "bugfix"
+  }
+]

CHANGELOG.rst

@@ -2,6 +2,13 @@
 CHANGELOG
 =========
 
+1.11.180
+========
+
+* api-change:``apigateway``: Update apigateway command to latest version
+* bugfix:datapipeline: Fixed a bug in datapipeline where list-runs could only handle 100 runs.
+
+
 1.11.179
 ========
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.11.179'
+__version__ = '1.11.180'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/datapipeline/__init__.py

@@ -50,6 +50,7 @@
 you can use the same key name and specify each value as
 a key value pair. e.g. arrayValue=value1 arrayValue=value2
 """
+MAX_ITEMS_PER_DESCRIBE = 100
 
 
 class DocSectionNotFoundError(Exception):
@@ -390,8 +391,7 @@ def _validate_status_choices(self, statuses):
     def _list_runs(self, parsed_args, parsed_globals):
         query = QueryArgBuilder().build_query(parsed_args)
         object_ids = self._query_objects(parsed_args.pipeline_id, query)
-        objects = self._describe_objects(parsed_args.pipeline_id, object_ids)[
-            'pipelineObjects']
+        objects = self._describe_objects(parsed_args.pipeline_id, object_ids)
         converted = convert_described_objects(
             objects,
             sort_key_func=lambda x: (x.get('@scheduledStartTime'),
@@ -400,9 +400,17 @@ def _list_runs(self, parsed_args, parsed_globals):
         formatter(self.NAME, converted)
 
     def _describe_objects(self, pipeline_id, object_ids):
-        parsed = self.client.describe_objects(
-            pipelineId=pipeline_id, objectIds=object_ids)
-        return parsed
+        # DescribeObjects will only accept 100 objectIds at a time,
+        # so we need to break up the list passed in into chunks that are at
+        # most that size. We then aggregate the results to return.
+        objects = []
+        for i in range(0, len(object_ids), MAX_ITEMS_PER_DESCRIBE):
+            current_object_ids = object_ids[i:i + MAX_ITEMS_PER_DESCRIBE]
+            result = self.client.describe_objects(
+                pipelineId=pipeline_id, objectIds=current_object_ids)
+            objects.extend(result['pipelineObjects'])
+
+        return objects
 
     def _query_objects(self, pipeline_id, query):
         paginator = self.client.get_paginator('query_objects').paginate(

doc/source/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.11.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.11.179'
+release = '1.11.180'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.cfg

@@ -4,7 +4,7 @@ universal = 1
 
 [metadata]
 requires-dist =
-        botocore==1.7.37
+        botocore==1.7.38
         colorama>=0.2.5,<=0.3.7
         docutils>=0.10
         rsa>=3.1.2,<=3.5.0

setup.py

@@ -23,7 +23,7 @@ def find_version(*file_paths):
     raise RuntimeError("Unable to find version string.")
 
 
-requires = ['botocore==1.7.37',
+requires = ['botocore==1.7.38',
             'colorama>=0.2.5,<=0.3.7',
             'docutils>=0.10',
             'rsa>=3.1.2,<=3.5.0',

tests/functional/datapipeline/test_list_runs.py

@@ -0,0 +1,90 @@
+# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from awscli.testutils import BaseAWSCommandParamsTest
+
+
+class TestDataPipelineQueryObjects(BaseAWSCommandParamsTest):
+    maxDiff = None
+
+    prefix = 'datapipeline list-runs '
+
+    def _generate_pipeline_objects(self, object_ids):
+        objects = []
+        for object_id in object_ids:
+            objects.append({
+                'id': object_id,
+                'name': object_id,
+                'fields': [
+                    {'key': '@componentParent', 'stringValue': object_id}
+                ]
+            })
+        return objects
+
+    def test_list_more_than_one_hundred_runs(self):
+        start_date = '2017-10-22T00:37:21'
+        end_date = '2017-10-26T00:37:21'
+        pipeline_id = 'pipeline-id'
+        args = '--pipeline-id %s --start-interval %s,%s' % (
+            pipeline_id, start_date, end_date
+        )
+        command = self.prefix + args
+        object_ids = ['object-id-%s' % i for i in range(150)]
+        objects = self._generate_pipeline_objects(object_ids)
+
+        self.parsed_responses = [
+            {
+                'ids': object_ids[:100],
+                'hasMoreResults': True,
+                'marker': 'marker'
+            },
+            {
+                'ids': object_ids[100:],
+                'hasMoreResults': False
+            },
+            {'pipelineObjects': objects[:100]},
+            {'pipelineObjects': objects[100:]}
+        ]
+
+        self.run_cmd(command, expected_rc=None)
+
+        query = {
+            'selectors': [{
+                'fieldName': '@actualStartTime',
+                'operator': {
+                    'type': 'BETWEEN',
+                    'values': [start_date, end_date]
+                }
+            }]
+        }
+
+        expected_operations_called = [
+            ('QueryObjects', {
+                'pipelineId': pipeline_id,
+                'query': query, 'sphere': 'INSTANCE'
+            }),
+            ('QueryObjects', {
+                'pipelineId': pipeline_id,
+                'marker': 'marker', 'query': query, 'sphere': 'INSTANCE'
+            }),
+            ('DescribeObjects', {
+                'objectIds': object_ids[:100],
+                'pipelineId': pipeline_id
+            }),
+            ('DescribeObjects', {
+                'objectIds': object_ids[100:],
+                'pipelineId': pipeline_id
+            })
+        ]
+        operations_called = [(op.name, params)
+                             for op, params in self.operations_called]
+        self.assertEqual(expected_operations_called, operations_called)