placement: Add GET /usages to placement API

This adds GET /usages as part of a new microversion 1.9 of the
placement API. Usages can be queried by project or project/user:

  GET /usages?project_id=<project id>
  GET /usages?project_id=<project id>&user_id=<user id>

and will be returned as a sum of usages, for example:

  200 OK
  Content-Type: application/json

  {
    "usages": {
      "VCPU": 2,
      "MEMORY_MB": 1024,
      "DISK_GB": 50,
      ...
    }
  }

A new method UsageList.get_all_by_project_user() has been added
for usage queries.

Part of blueprint placement-project-user

Change-Id: I8b948a4dfe6a50bea053b5dcae8f039229e2e364

13 files changed, 251 insertions, 25 deletions

diff --git a/nova/api/openstack/placement/handler.py b/nova/api/openstack/placement/handler.py
index fa9ebfa5d5..b13517481d 100644
--- a/nova/api/openstack/placement/handler.py
+++ b/nova/api/openstack/placement/handler.py
@@ -117,6 +117,9 @@ ROUTE_DECLARATIONS = {
         'PUT': trait.update_traits_for_resource_provider,
         'DELETE': trait.delete_traits_for_resource_provider
     },
+    '/usages': {
+        'GET': usage.get_total_usages,
+    },
 }
 
 
diff --git a/nova/api/openstack/placement/handlers/resource_provider.py b/nova/api/openstack/placement/handlers/resource_provider.py
index 1eef16f4a6..199d53b8fb 100644
--- a/nova/api/openstack/placement/handlers/resource_provider.py
+++ b/nova/api/openstack/placement/handlers/resource_provider.py
@@ -13,7 +13,6 @@
 
 import copy
 
-import jsonschema
 from oslo_db import exception as db_exc
 from oslo_serialization import jsonutils
 from oslo_utils import encodeutils
@@ -267,13 +266,8 @@ def list_resource_providers(req):
         schema = GET_RPS_SCHEMA_1_3
     if want_version >= (1, 4):
         schema = GET_RPS_SCHEMA_1_4
-    try:
-        jsonschema.validate(dict(req.GET), schema,
-                            format_checker=jsonschema.FormatChecker())
-    except jsonschema.ValidationError as exc:
-        raise webob.exc.HTTPBadRequest(
-            _('Invalid query string parameters: %(exc)s') %
-            {'exc': exc})
+
+    util.validate_query_params(req, schema)
 
     filters = {}
     for attr in ['uuid', 'name', 'member_of']:
diff --git a/nova/api/openstack/placement/handlers/usage.py b/nova/api/openstack/placement/handlers/usage.py
index d0fea2d358..68c3eb3989 100644
--- a/nova/api/openstack/placement/handlers/usage.py
+++ b/nova/api/openstack/placement/handlers/usage.py
@@ -15,6 +15,7 @@ from oslo_serialization import jsonutils
 from oslo_utils import encodeutils
 import webob
 
+from nova.api.openstack.placement import microversion
 from nova.api.openstack.placement import util
 from nova.api.openstack.placement import wsgi_wrapper
 from nova import exception
@@ -22,6 +23,28 @@ from nova.i18n import _
 from nova import objects
 
 
+# Represents the allowed query string parameters to GET /usages
+GET_USAGES_SCHEMA_1_9 = {
+    "type": "object",
+    "properties": {
+        "project_id": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 255,
+        },
+        "user_id": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 255,
+        },
+    },
+    "required": [
+        "project_id"
+     ],
+    "additionalProperties": False,
+}
+
+
 def _serialize_usages(resource_provider, usage):
     usage_dict = {resource.resource_class: resource.usage
                   for resource in usage}
@@ -63,3 +86,33 @@ def list_usages(req):
         _serialize_usages(resource_provider, usage)))
     req.response.content_type = 'application/json'
     return req.response
+
+
+@wsgi_wrapper.PlacementWsgify
+@microversion.version_handler('1.9')
+@util.check_accept('application/json')
+def get_total_usages(req):
+    """GET the sum of usages for a project or a project/user.
+
+    On success return a 200 and an application/json body representing the
+    sum/total of usages.
+    Return 404 Not Found if the wanted microversion does not match.
+    """
+    context = req.environ['placement.context']
+
+    schema = GET_USAGES_SCHEMA_1_9
+
+    util.validate_query_params(req, schema)
+
+    project_id = req.GET.get('project_id')
+    user_id = req.GET.get('user_id')
+
+    usages = objects.UsageList.get_all_by_project_user(context, project_id,
+                                                       user_id=user_id)
+
+    response = req.response
+    usages_dict = {'usages': {resource.resource_class: resource.usage
+                   for resource in usages}}
+    response.body = encodeutils.to_utf8(jsonutils.dumps(usages_dict))
+    req.response.content_type = 'application/json'
+    return req.response
diff --git a/nova/api/openstack/placement/microversion.py b/nova/api/openstack/placement/microversion.py
index 2f7ccaa2c8..8d80e3b02e 100644
--- a/nova/api/openstack/placement/microversion.py
+++ b/nova/api/openstack/placement/microversion.py
@@ -46,6 +46,7 @@ VERSIONS = [
     '1.7',  # PUT /resource_classes/{name} is bodiless create or update
     '1.8',  # Adds 'project_id' and 'user_id' required request parameters to
             # PUT /allocations
+    '1.9',  # Adds GET /usages
 ]
 
 
diff --git a/nova/api/openstack/placement/util.py b/nova/api/openstack/placement/util.py
index d716723094..91df481650 100644
--- a/nova/api/openstack/placement/util.py
+++ b/nova/api/openstack/placement/util.py
@@ -170,6 +170,16 @@ def trait_url(environ, trait):
     return '%s/traits/%s' % (prefix, trait.name)
 
 
+def validate_query_params(req, schema):
+    try:
+        jsonschema.validate(dict(req.GET), schema,
+                            format_checker=jsonschema.FormatChecker())
+    except jsonschema.ValidationError as exc:
+        raise webob.exc.HTTPBadRequest(
+            _('Invalid query string parameters: %(exc)s') %
+            {'exc': exc})
+
+
 def wsgi_path_item(environ, name):
     """Extract the value of a named field in a URL.
 
diff --git a/nova/objects/resource_provider.py b/nova/objects/resource_provider.py
index d33d027eae..0317fa2b24 100644
--- a/nova/objects/resource_provider.py
+++ b/nova/objects/resource_provider.py
@@ -1894,7 +1894,8 @@ class Usage(base.NovaObject):
 class UsageList(base.ObjectListBase, base.NovaObject):
     # Version 1.0: Initial version
     # Version 1.1: Turn off remotable
-    VERSION = '1.1'
+    # Version 1.2: Add get_all_by_project_user()
+    VERSION = '1.2'
 
     fields = {
         'objects': fields.ListOfObjectsField('Usage'),
@@ -1919,11 +1920,36 @@ class UsageList(base.ObjectListBase, base.NovaObject):
                   for item in query.all()]
         return result
 
+    @staticmethod
+    @db_api.api_context_manager.reader
+    def _get_all_by_project_user(context, project_id, user_id=None):
+        query = (context.session.query(models.Allocation.resource_class_id,
+                 func.coalesce(func.sum(models.Allocation.used), 0))
+                 .join(models.Consumer,
+                       models.Allocation.consumer_id == models.Consumer.uuid)
+                 .join(models.Project,
+                       models.Consumer.project_id == models.Project.id)
+                 .filter(models.Project.external_id == project_id))
+        if user_id:
+            query = query.join(models.User,
+                               models.Consumer.user_id == models.User.id)
+            query = query.filter(models.User.external_id == user_id)
+        query = query.group_by(models.Allocation.resource_class_id)
+        result = [dict(resource_class_id=item[0], usage=item[1])
+                  for item in query.all()]
+        return result
+
     @classmethod
     def get_all_by_resource_provider_uuid(cls, context, rp_uuid):
         usage_list = cls._get_all_by_resource_provider_uuid(context, rp_uuid)
         return base.obj_make_list(context, cls(context), Usage, usage_list)
 
+    @classmethod
+    def get_all_by_project_user(cls, context, project_id, user_id=None):
+        usage_list = cls._get_all_by_project_user(context, project_id,
+                                                  user_id=user_id)
+        return base.obj_make_list(context, cls(context), Usage, usage_list)
+
     def __repr__(self):
         strings = [repr(x) for x in self.objects]
         return "UsageList[" + ", ".join(strings) + "]"
diff --git a/nova/tests/functional/api/openstack/placement/fixtures.py b/nova/tests/functional/api/openstack/placement/fixtures.py
index 3c31f21b47..3b72723e32 100644
--- a/nova/tests/functional/api/openstack/placement/fixtures.py
+++ b/nova/tests/functional/api/openstack/placement/fixtures.py
@@ -95,6 +95,13 @@ class AllocationFixture(APIFixture):
     def start_fixture(self):
         super(AllocationFixture, self).start_fixture()
         self.context = context.get_admin_context()
+
+        # For use creating and querying allocations/usages
+        os.environ['ALT_USER_ID'] = uuidutils.generate_uuid()
+        project_id = os.environ['PROJECT_ID']
+        user_id = os.environ['USER_ID']
+        alt_user_id = os.environ['ALT_USER_ID']
+
         # Stealing from the super
         rp_name = os.environ['RP_NAME']
         rp_uuid = os.environ['RP_UUID']
@@ -103,6 +110,9 @@ class AllocationFixture(APIFixture):
         rp.create()
 
         # Create some DISK_GB inventory and allocations.
+        # Each set of allocations must have the same consumer_id because only
+        # the first allocation is used for the project/user association.
+        consumer_id = uuidutils.generate_uuid()
         inventory = objects.Inventory(
             self.context, resource_provider=rp,
             resource_class='DISK_GB', total=2048,
@@ -112,36 +122,67 @@ class AllocationFixture(APIFixture):
         alloc1 = objects.Allocation(
             self.context, resource_provider=rp,
             resource_class='DISK_GB',
-            consumer_id=uuidutils.generate_uuid(),
+            consumer_id=consumer_id,
             used=500)
         alloc2 = objects.Allocation(
             self.context, resource_provider=rp,
             resource_class='DISK_GB',
-            consumer_id=uuidutils.generate_uuid(),
+            consumer_id=consumer_id,
             used=500)
-        alloc_list = objects.AllocationList(self.context,
-                objects=[alloc1, alloc2])
+        alloc_list = objects.AllocationList(
+            self.context,
+            objects=[alloc1, alloc2],
+            project_id=project_id,
+            user_id=user_id,
+        )
         alloc_list.create_all()
 
         # Create some VCPU inventory and allocations.
+        # Each set of allocations must have the same consumer_id because only
+        # the first allocation is used for the project/user association.
+        consumer_id = uuidutils.generate_uuid()
         inventory = objects.Inventory(
             self.context, resource_provider=rp,
-            resource_class='VCPU', total=8,
+            resource_class='VCPU', total=10,
             max_unit=4)
         inventory.obj_set_defaults()
         rp.add_inventory(inventory)
         alloc1 = objects.Allocation(
             self.context, resource_provider=rp,
             resource_class='VCPU',
-            consumer_id=uuidutils.generate_uuid(),
+            consumer_id=consumer_id,
             used=2)
         alloc2 = objects.Allocation(
             self.context, resource_provider=rp,
             resource_class='VCPU',
-            consumer_id=uuidutils.generate_uuid(),
+            consumer_id=consumer_id,
             used=4)
-        alloc_list = objects.AllocationList(self.context,
-                objects=[alloc1, alloc2])
+        alloc_list = objects.AllocationList(
+                self.context,
+                objects=[alloc1, alloc2],
+                project_id=project_id,
+                user_id=user_id)
+        alloc_list.create_all()
+
+        # Create a couple of allocations for a different user.
+        # Each set of allocations must have the same consumer_id because only
+        # the first allocation is used for the project/user association.
+        consumer_id = uuidutils.generate_uuid()
+        alloc1 = objects.Allocation(
+            self.context, resource_provider=rp,
+            resource_class='DISK_GB',
+            consumer_id=consumer_id,
+            used=20)
+        alloc2 = objects.Allocation(
+            self.context, resource_provider=rp,
+            resource_class='VCPU',
+            consumer_id=consumer_id,
+            used=1)
+        alloc_list = objects.AllocationList(
+                self.context,
+                objects=[alloc1, alloc2],
+                project_id=project_id,
+                user_id=alt_user_id)
         alloc_list.create_all()
 
         # The ALT_RP_XXX variables are for a resource provider that has
diff --git a/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml b/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml
index 33cdd3338d..d721c9be67 100644
--- a/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml
+++ b/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml
@@ -39,13 +39,13 @@ tests:
   response_json_paths:
       $.errors[0].title: Not Acceptable
 
-- name: latest microversion is 1.8
+- name: latest microversion is 1.9
   GET: /
   request_headers:
       openstack-api-version: placement latest
   response_headers:
       vary: /OpenStack-API-Version/
-      openstack-api-version: placement 1.8
+      openstack-api-version: placement 1.9
 
 - name: other accept header bad version
   GET: /
diff --git a/nova/tests/functional/api/openstack/placement/gabbits/usage.yaml b/nova/tests/functional/api/openstack/placement/gabbits/usage.yaml
index dbe3a767e3..b34c025572 100644
--- a/nova/tests/functional/api/openstack/placement/gabbits/usage.yaml
+++ b/nova/tests/functional/api/openstack/placement/gabbits/usage.yaml
@@ -37,3 +37,47 @@ tests:
       content-type: application/json
   response_json_paths:
       usages: {}
+
+- name: get total usages earlier version
+  GET: /usages?project_id=$ENVIRON['PROJECT_ID']
+  request_headers:
+      openstack-api-version: placement 1.8
+  status: 404
+
+- name: get total usages no project or user
+  GET: /usages
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 400
+
+- name: get total usages project_id less than min length
+  GET: /usages?project_id=
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 400
+  response_strings:
+      - "Failed validating 'minLength'"
+
+- name: get total usages user_id less than min length
+  GET: /usages?project_id=$ENVIRON['PROJECT_ID']&user_id=
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 400
+  response_strings:
+      - "Failed validating 'minLength'"
+
+- name: get total usages project_id exceeds max length
+  GET: /usages?project_id=78725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b1
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 400
+  response_strings:
+      - "Failed validating 'maxLength'"
+
+- name: get total usages user_id exceeds max length
+  GET: /usages?project_id=$ENVIRON['PROJECT_ID']&user_id=78725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b178725f09-5c01-4c9e-97a5-98d75e1e32b1
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 400
+  response_strings:
+      - "Failed validating 'maxLength'"
diff --git a/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml b/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml
index a2ffc01c2f..13f112bd7b 100644
--- a/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml
+++ b/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml
@@ -21,9 +21,9 @@ tests:
       # required but superfluous, is present
       content-type: /application/json/
   response_json_paths:
-      $.resource_provider_generation: 4
-      $.usages.DISK_GB: 1000
-      $.usages.VCPU: 6
+      $.resource_provider_generation: 5
+      $.usages.DISK_GB: 1020
+      $.usages.VCPU: 7
 
 - name: fail to delete resource provider
   DELETE: /resource_providers/$ENVIRON['RP_UUID']
@@ -41,3 +41,30 @@ tests:
       content-type: /application/json/
   response_strings:
       - Unable to delete inventory for resource provider $ENVIRON['RP_UUID'] because the inventory is in use.
+
+- name: get total usages by project
+  GET: /usages?project_id=$ENVIRON['PROJECT_ID']
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 200
+  response_json_paths:
+      $.usages.DISK_GB: 1020
+      $.usages.VCPU: 7
+
+- name: get total usages by project and user
+  GET: /usages?project_id=$ENVIRON['PROJECT_ID']&user_id=$ENVIRON['USER_ID']
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 200
+  response_json_paths:
+      $.usages.DISK_GB: 1000
+      $.usages.VCPU: 6
+
+- name: get total usages by project and alt user
+  GET: /usages?project_id=$ENVIRON['PROJECT_ID']&user_id=$ENVIRON['ALT_USER_ID']
+  request_headers:
+      openstack-api-version: placement 1.9
+  status: 200
+  response_json_paths:
+      $.usages.DISK_GB: 20
+      $.usages.VCPU: 1
diff --git a/nova/tests/functional/db/test_resource_provider.py b/nova/tests/functional/db/test_resource_provider.py
index 9fd9bd2995..44e416feb0 100644
--- a/nova/tests/functional/db/test_resource_provider.py
+++ b/nova/tests/functional/db/test_resource_provider.py
@@ -1258,6 +1258,33 @@ class TestAllocationListCreateDelete(ResourceProviderBaseCase):
             res = conn.execute(sel).fetchall()
             self.assertEqual(1, len(res), "consumer lookup not created.")
 
+        # Create allocation for a different user in the project
+        other_consumer_uuid = uuidsentinel.other_consumer
+        allocation3 = objects.Allocation(resource_provider=rp,
+                                         consumer_id=other_consumer_uuid,
+                                         resource_class=rp_class,
+                                         used=200)
+        allocation_list = objects.AllocationList(
+            self.context,
+            objects=[allocation3],
+            project_id=self.context.project_id,
+            user_id=uuidsentinel.other_user,
+        )
+        allocation_list.create_all()
+
+        # Get usages back by project
+        usage_list = objects.UsageList.get_all_by_project_user(
+            self.context, self.context.project_id)
+        self.assertEqual(1, len(usage_list))
+        self.assertEqual(500, usage_list[0].usage)
+
+        # Get usages back by project and user
+        usage_list = objects.UsageList.get_all_by_project_user(
+            self.context, self.context.project_id,
+            user_id=uuidsentinel.other_user)
+        self.assertEqual(1, len(usage_list))
+        self.assertEqual(200, usage_list[0].usage)
+
 
 class UsageListTestCase(ResourceProviderBaseCase):
 
diff --git a/nova/tests/unit/api/openstack/placement/test_microversion.py b/nova/tests/unit/api/openstack/placement/test_microversion.py
index 2a98fa9049..11f1d49cf5 100644
--- a/nova/tests/unit/api/openstack/placement/test_microversion.py
+++ b/nova/tests/unit/api/openstack/placement/test_microversion.py
@@ -74,7 +74,7 @@ class TestMicroversionIntersection(test.NoDBTestCase):
     # if you add two different versions of method 'foobar' the
     # number only goes up by one if no other version foobar yet
     # exists. This operates as a simple sanity check.
-    TOTAL_VERSIONED_METHODS = 13
+    TOTAL_VERSIONED_METHODS = 14
 
     def test_methods_versioned(self):
         methods_data = microversion.VERSIONED_METHODS
diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py
index e64cd73be5..0911e40dc8 100644
--- a/nova/tests/unit/objects/test_objects.py
+++ b/nova/tests/unit/objects/test_objects.py
@@ -1171,7 +1171,7 @@ object_data = {
     'Trait': '1.0-2b58dd7c5037153cb4bfc94c0ae5dd3a',
     'TraitList': '1.0-ff48fc1575f20800796b48266114c608',
     'Usage': '1.1-b738dbebeb20e3199fc0ebca6e292a47',
-    'UsageList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',
+    'UsageList': '1.2-15ecf022a68ddbb8c2a6739cfc9f8f5e',
     'USBDeviceBus': '1.0-e4c7dd6032e46cd74b027df5eb2d4750',
     'VirtCPUFeature': '1.0-ea2464bdd09084bd388e5f61d5d4fc86',
     'VirtCPUModel': '1.0-5e1864af9227f698326203d7249796b5',