[APIImpact] Quota usage API

This adds a /v2/info/usage API endpoint which exposes to the user
their current limits and usage.

The discovery API does not (appear to) have existing tests, so this
adds a module for that, although only usage tests are added currently.

Implements: blueprint quota-api
Change-Id: I50c98bac50f815bdb9baae024e77afd388f74554

api-ref/source/v2/discovery.inc

@@ -103,3 +103,26 @@ Response Example
 
 .. literalinclude:: samples/stores-list-response.json
    :language: json
+
+Quota usage
+~~~~~~~~~~~
+
+.. rest_method:: GET /v2/info/usage
+
+The user's quota and current usage are displayed, if enabled by
+server-side configuration.
+
+Normal response codes: 200
+
+Request
+-------
+
+There are no request parameters.
+
+This call does not allow a request body.
+
+Response Example
+----------------
+
+.. literalinclude:: samples/usage-response.json
+   :language: json

api-ref/source/v2/samples/usage-response.json

@@ -0,0 +1,20 @@
+{
+    "usage": {
+        "image_size_total": {
+            "limit": 1024,
+            "usage": 256
+        },
+        "image_count_total": {
+            "limit": 10,
+            "usage": 2
+        },
+        "image_stage_total": {
+            "limit": 512,
+            "usage": 0
+        },
+        "image_count_uploading": {
+            "limit": 2,
+            "usage": 0
+        }
+    }
+}

glance/api/middleware/version_negotiation.py

@@ -82,6 +82,7 @@ def _get_allowed_versions(self):
         allowed_versions['v2.6'] = 2
         allowed_versions['v2.7'] = 2
         allowed_versions['v2.9'] = 2
+        allowed_versions['v2.13'] = 2
         if CONF.enabled_backends:
             allowed_versions['v2.8'] = 2
             allowed_versions['v2.10'] = 2

glance/api/v2/discovery.py

@@ -13,12 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
+
 from oslo_config import cfg
+import oslo_serialization.jsonutils as json
 import webob.exc
 
 from glance.common import wsgi
+import glance.db
 from glance.i18n import _
-
+from glance.quota import keystone as ks_quota
 
 CONF = cfg.CONF
 
@@ -63,6 +67,49 @@ def get_stores(self, req):
 
         return {'stores': backends}
 
+    def get_usage(self, req):
+        project_usage = ks_quota.get_usage(req.context)
+        return {'usage':
+                {name: {'usage': usage.usage,
+                        'limit': usage.limit}
+                 for name, usage in project_usage.items()}}
+
+
+class ResponseSerializer(wsgi.JSONResponseSerializer):
+    def __init__(self, usage_schema=None):
+        super(ResponseSerializer, self).__init__()
+        self.schema = usage_schema or get_usage_schema()
+
+    def get_usage(self, response, usage):
+        body = json.dumps(self.schema.filter(usage), ensure_ascii=False)
+        response.unicode_body = str(body)
+        response.content_type = 'application/json'
+
+
+_USAGE_SCHEMA = {
+    'usage': {
+        'type': 'array',
+        'items': {
+            'type': 'object',
+            'additionalProperties': True,
+            'validation_data': {
+                'type': 'object',
+                'additonalProperties': False,
+                'properties': {
+                    'usage': {'type': 'integer'},
+                    'limit': {'type': 'integer'},
+                },
+            },
+        },
+    },
+}
+
+
+def get_usage_schema():
+    return glance.schema.Schema('usage', copy.deepcopy(_USAGE_SCHEMA))
+
 
 def create_resource():
-    return wsgi.Resource(InfoController())
+    usage_schema = get_usage_schema()
+    serializer = ResponseSerializer(usage_schema)
+    return wsgi.Resource(InfoController(), None, serializer)

glance/api/v2/router.py

@@ -588,5 +588,9 @@ def __init__(self, mapper):
                        controller=reject_method_resource,
                        action='reject',
                        allowed_methods='GET')
+        mapper.connect('/info/usage',
+                       controller=info_resource,
+                       action='get_usage',
+                       conditions={'method': ['GET']})
 
         super(API, self).__init__(mapper)

glance/api/versions.py

@@ -78,17 +78,18 @@ def build_version_object(version, path, status):
         version_objs = []
         if CONF.enabled_backends:
             version_objs.extend([
-                build_version_object(2.12, 'v2', 'CURRENT'),
+                build_version_object(2.12, 'v2', 'SUPPORTED'),
                 build_version_object(2.11, 'v2', 'SUPPORTED'),
                 build_version_object('2.10', 'v2', 'SUPPORTED'),
                 build_version_object(2.9, 'v2', 'SUPPORTED'),
                 build_version_object(2.8, 'v2', 'SUPPORTED'),
             ])
         else:
             version_objs.extend([
-                build_version_object(2.9, 'v2', 'CURRENT'),
+                build_version_object(2.9, 'v2', 'SUPPORTED'),
             ])
         version_objs.extend([
+            build_version_object(2.13, 'v2', 'CURRENT'),
             build_version_object(2.7, 'v2', 'SUPPORTED'),
             build_version_object(2.6, 'v2', 'SUPPORTED'),
             build_version_object(2.5, 'v2', 'SUPPORTED'),

glance/quota/keystone.py

@@ -142,3 +142,29 @@ def enforce_image_count_uploading(context, project_id):
         context, project_id, QUOTA_IMAGE_COUNT_UPLOADING,
         lambda: db.user_get_uploading_count(context, project_id),
         delta=0)
+
+
+def get_usage(context, project_id=None):
+    if not CONF.use_keystone_limits:
+        return {}
+
+    if not project_id:
+        project_id = context.project_id
+
+    usages = {
+        QUOTA_IMAGE_SIZE_TOTAL: lambda: db.user_get_storage_usage(
+            context, project_id) // units.Mi,
+        QUOTA_IMAGE_STAGING_TOTAL: lambda: db.user_get_staging_usage(
+            context, project_id) // units.Mi,
+        QUOTA_IMAGE_COUNT_TOTAL: lambda: db.user_get_image_count(
+            context, project_id),
+        QUOTA_IMAGE_COUNT_UPLOADING: lambda: db.user_get_uploading_count(
+            context, project_id),
+    }
+
+    def callback(project_id, resource_names):
+        return {name: usages[name]()
+                for name in resource_names}
+
+    enforcer = limit.Enforcer(callback)
+    return enforcer.calculate_usage(project_id, list(usages.keys()))

glance/tests/functional/v2/test_discovery.py

@@ -0,0 +1,98 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import fixtures
+
+from oslo_utils import units
+
+from glance.quota import keystone as ks_quota
+from glance.tests import functional
+from glance.tests.functional.v2.test_images import get_enforcer_class
+from glance.tests import utils as test_utils
+
+
+class TestDiscovery(functional.SynchronousAPIBase):
+    def setUp(self):
+        super(TestDiscovery, self).setUp()
+        self.config(use_keystone_limits=True)
+
+        self.enforcer_mock = self.useFixture(
+            fixtures.MockPatchObject(ks_quota, 'limit')).mock
+
+    def set_limit(self, limits):
+        self.enforcer_mock.Enforcer = get_enforcer_class(limits)
+
+    def _assert_usage(self, expected):
+        usage = self.api_get('/v2/info/usage')
+        usage = usage.json['usage']
+        for item in ('count', 'size', 'stage'):
+            key = 'image_%s_total' % item
+            self.assertEqual(expected[key], usage[key],
+                             'Mismatch in %s' % key)
+        self.assertEqual(expected['image_count_uploading'],
+                         usage['image_count_uploading'])
+
+    def test_quota_with_usage(self):
+        self.set_limit({'image_size_total': 5,
+                        'image_count_total': 10,
+                        'image_stage_total': 15,
+                        'image_count_uploading': 20})
+
+        self.start_server()
+
+        # Initially we expect no usage, but our limits in place.
+        expected = {
+            'image_size_total': {'limit': 5, 'usage': 0},
+            'image_count_total': {'limit': 10, 'usage': 0},
+            'image_stage_total': {'limit': 15, 'usage': 0},
+            'image_count_uploading': {'limit': 20, 'usage': 0},
+        }
+        self._assert_usage(expected)
+
+        # Stage 1MiB and see our total count, uploading count, and
+        # staging area usage increase.
+        data = test_utils.FakeData(1 * units.Mi)
+        image_id = self._create_and_stage(data_iter=data)
+        expected['image_count_uploading']['usage'] = 1
+        expected['image_count_total']['usage'] = 1
+        expected['image_stage_total']['usage'] = 1
+        self._assert_usage(expected)
+
+        # Doing the import does not change anything (since we are
+        # synchronous and the task will not have run yet).
+        self._import_direct(image_id, ['store1'])
+        self._assert_usage(expected)
+
+        # After the import is complete, our usage of the staging area
+        # drops to zero, and our consumption of actual store space
+        # reflects the new active image.
+        self._wait_for_import(image_id)
+        expected['image_count_uploading']['usage'] = 0
+        expected['image_stage_total']['usage'] = 0
+        expected['image_size_total']['usage'] = 1
+        self._assert_usage(expected)
+
+        # Upload also yields a new active image and store usage.
+        data = test_utils.FakeData(1 * units.Mi)
+        image_id = self._create_and_upload(data_iter=data)
+        expected['image_count_total']['usage'] = 2
+        expected['image_size_total']['usage'] = 2
+        self._assert_usage(expected)
+
+        # Deleting an image drops the usage down.
+        self.api_delete('/v2/images/%s' % image_id)
+        expected['image_count_total']['usage'] = 1
+        expected['image_size_total']['usage'] = 1
+        self._assert_usage(expected)

glance/tests/functional/v2/test_images.py

@@ -22,6 +22,7 @@
 
 import fixtures
 from oslo_limit import exception as ol_exc
+from oslo_limit import limit
 from oslo_serialization import jsonutils
 from oslo_utils.secretutils import md5
 from oslo_utils import units
@@ -7037,6 +7038,13 @@ def enforce(self, project_id, values):
                         over_limit_info_list=[ol_exc.OverLimitInfo(
                             name, limits.get(name), current.get(name), delta)])
 
+        def calculate_usage(self, project_id, names):
+            return {
+                name: limit.ProjectUsage(
+                    limits.get(name, 0),
+                    self._callback(project_id, [name])[name])
+                for name in names}
+
     return FakeEnforcer
 
 

glance/tests/unit/test_versions.py

@@ -29,6 +29,12 @@
 # functional tests
 def get_versions_list(url, enabled_backends=False):
     image_versions = [
+        {
+            'id': 'v2.13',
+            'status': 'CURRENT',
+            'links': [{'rel': 'self',
+                       'href': '%s/v2/' % url}],
+        },
         {
             'id': 'v2.7',
             'status': 'SUPPORTED',
@@ -82,7 +88,7 @@ def get_versions_list(url, enabled_backends=False):
         image_versions = [
             {
                 'id': 'v2.12',
-                'status': 'CURRENT',
+                'status': 'SUPPORTED',
                 'links': [{'rel': 'self',
                            'href': '%s/v2/' % url}],
             },
@@ -114,7 +120,7 @@ def get_versions_list(url, enabled_backends=False):
     else:
         image_versions.insert(0, {
             'id': 'v2.9',
-            'status': 'CURRENT',
+            'status': 'SUPPORTED',
             'links': [{'rel': 'self',
                        'href': '%s/v2/' % url}],
         })
@@ -321,15 +327,20 @@ def test_request_url_v2_12_enabled_supported(self):
         self.middleware.process_request(request)
         self.assertEqual('/v2/images', request.path_info)
 
-    # version 2.13 does not exist
-    def test_request_url_v2_13_default_unsupported(self):
+    def test_request_url_v2_13_enabled_supported(self):
         request = webob.Request.blank('/v2.13/images')
+        self.middleware.process_request(request)
+        self.assertEqual('/v2/images', request.path_info)
+
+    # version 2.14 does not exist
+    def test_request_url_v2_14_default_unsupported(self):
+        request = webob.Request.blank('/v2.14/images')
         resp = self.middleware.process_request(request)
         self.assertIsInstance(resp, versions.Controller)
 
-    def test_request_url_v2_13_enabled_unsupported(self):
+    def test_request_url_v2_14_enabled_unsupported(self):
         self.config(enabled_backends='slow:one,fast:two')
-        request = webob.Request.blank('/v2.13/images')
+        request = webob.Request.blank('/v2.14/images')
         resp = self.middleware.process_request(request)
         self.assertIsInstance(resp, versions.Controller)
 

releasenotes/notes/added-quota-usage-api-f1914054132f2021.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    This release brings additional functionality to the unified quota
+    work done in the previous release. A usage API is now available,
+    which provides a way for users to see their current quota limits
+    and their active resource usage towards them. For more
+    information, see the discovery section in the `api-ref
+    <https://developer.openstack.org/api-ref/image/v2/index.html#image-service-info-discovery>`_.

requirements.txt

@@ -38,7 +38,7 @@ six>=1.11.0 # MIT
 
 oslo.db>=5.0.0 # Apache-2.0
 oslo.i18n>=5.0.0 # Apache-2.0
-oslo.limit>=1.0.0 # Apache-2.0
+oslo.limit>=1.4.0 # Apache-2.0
 oslo.log>=4.5.0 # Apache-2.0
 oslo.messaging>=5.29.0,!=9.0.0 # Apache-2.0
 oslo.middleware>=3.31.0 # Apache-2.0