Implements regions resource in 3.2 Catalog API

Adds CRUD implementation in SQL and KVS catalog drivers
for hierarchical regions support. The SQL driver implements
the region hierarchy using a simple adjacency list model
since it is expected that typical deployments will only
have a handful of regions, and the current regions API does
not offer any complex hierarchy querying that would make
a nested set model more appropriate.

Implements: blueprint first-class-regions

Change-Id: I2d4cca19008b92ef5758181b1792726834db7f7a

11 files changed, 473 insertions, 1 deletions

diff --git a/keystone/catalog/backends/kvs.py b/keystone/catalog/backends/kvs.py
index 07953a02f..bb9c758f4 100644
--- a/keystone/catalog/backends/kvs.py
+++ b/keystone/catalog/backends/kvs.py
@@ -25,6 +25,60 @@ class Catalog(kvs.Base, catalog.Driver):
     def get_catalog(self, user_id, tenant_id, metadata=None):
         return self.db.get('catalog-%s-%s' % (tenant_id, user_id))
 
+    # region crud
+
+    def _delete_child_regions(self, region_id):
+        """Delete all child regions.
+
+        Recursively delete any region that has the supplied region
+        as its parent.
+        """
+        children = [r for r in self.list_regions()
+                    if r['parent_region_id'] == region_id]
+        for child in children:
+            self._delete_child_regions(child['id'])
+            self.delete_region(child['id'])
+
+    def _check_parent_region(self, region_ref):
+        """Raise a NotFound if the parent region does not exist.
+
+        If the region_ref has a specified parent_region_id, check that
+        the parent exists, otherwise, raise a NotFound.
+        """
+        parent_region_id = region_ref.get('parent_region_id')
+        if parent_region_id is not None:
+            # This will raise NotFound if the parent doesn't exist,
+            # which is the behavior we want.
+            self.get_region(parent_region_id)
+
+    def create_region(self, region_id, region):
+        region.setdefault('parent_region_id')
+        self._check_parent_region(region)
+        self.db.set('region-%s' % region_id, region)
+        region_list = set(self.db.get('region_list', []))
+        region_list.add(region_id)
+        self.db.set('region_list', list(region_list))
+        return region
+
+    def list_regions(self):
+        return [self.get_region(x) for x in self.db.get('region_list', [])]
+
+    def get_region(self, region_id):
+        return self.db.get('region-%s' % region_id)
+
+    def update_region(self, region_id, region):
+        region.setdefault('parent_region_id')
+        self._check_parent_region(region)
+        self.db.set('region-%s' % region_id, region)
+        return region
+
+    def delete_region(self, region_id):
+        self._delete_child_regions(region_id)
+        self.db.delete('region-%s' % region_id)
+        region_list = set(self.db.get('region_list', []))
+        region_list.remove(region_id)
+        self.db.set('region_list', list(region_list))
+
     # service crud
 
     def create_service(self, service_id, service):
diff --git a/keystone/catalog/backends/sql.py b/keystone/catalog/backends/sql.py
index 7baa8ea6f..d29fec68e 100644
--- a/keystone/catalog/backends/sql.py
+++ b/keystone/catalog/backends/sql.py
@@ -26,6 +26,31 @@ from keystone import exception
 CONF = config.CONF
 
 
+class Region(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'region'
+    attributes = ['id', 'description', 'parent_region_id']
+    id = sql.Column(sql.String(64), primary_key=True)
+    description = sql.Column(sql.String(255))
+    # NOTE(jaypipes): Right now, using an adjacency list model for
+    #                 storing the hierarchy of regions is fine, since
+    #                 the API does not support any kind of querying for
+    #                 more complex hierarchical queries such as "get me only
+    #                 the regions that are subchildren of this region", etc.
+    #                 If, in the future, such queries are needed, then it
+    #                 would be possible to add in columns to this model for
+    #                 "left" and "right" and provide support for a nested set
+    #                 model.
+    parent_region_id = sql.Column(sql.String(64), nullable=True)
+
+    # TODO(jaypipes): I think it's absolutely stupid that every single model
+    #                 is required to have an "extra" column because of the
+    #                 DictBase in the keystone.common.sql.core module. Forcing
+    #                 tables to have pointless columns in the database is just
+    #                 bad. Remove all of this extra JSON blob stuff.
+    #                 See: https://bugs.launchpad.net/keystone/+bug/1265071
+    extra = sql.Column(sql.JsonBlob())
+
+
 class Service(sql.ModelBase, sql.DictBase):
     __tablename__ = 'service'
     attributes = ['id', 'type']
@@ -54,6 +79,77 @@ class Catalog(sql.Base, catalog.Driver):
     def db_sync(self, version=None):
         migration.db_sync(version=version)
 
+    # Regions
+    def list_regions(self):
+        session = self.get_session()
+        regions = session.query(Region).all()
+        return [s.to_dict() for s in list(regions)]
+
+    def _get_region(self, session, region_id):
+        ref = session.query(Region).get(region_id)
+        if not ref:
+            raise exception.RegionNotFound(region_id=region_id)
+        return ref
+
+    def _delete_child_regions(self, session, region_id):
+        """Delete all child regions.
+
+        Recursively delete any region that has the supplied region
+        as its parent.
+        """
+        children = session.query(Region).filter_by(parent_region_id=region_id)
+        for child in children:
+            self._delete_child_regions(session, child.id)
+            session.delete(child)
+
+    def _check_parent_region(self, session, region_ref):
+        """Raise a NotFound if the parent region does not exist.
+
+        If the region_ref has a specified parent_region_id, check that
+        the parent exists, otherwise, raise a NotFound.
+        """
+        parent_region_id = region_ref.get('parent_region_id')
+        if parent_region_id is not None:
+            # This will raise NotFound if the parent doesn't exist,
+            # which is the behavior we want.
+            self._get_region(session, parent_region_id)
+
+    def get_region(self, region_id):
+        session = self.get_session()
+        return self._get_region(session, region_id).to_dict()
+
+    def delete_region(self, region_id):
+        session = self.get_session()
+        with session.begin():
+            ref = self._get_region(session, region_id)
+            self._delete_child_regions(session, region_id)
+            session.query(Region).filter_by(id=region_id).delete()
+            session.delete(ref)
+            session.flush()
+
+    def create_region(self, region_id, region_ref):
+        session = self.get_session()
+        with session.begin():
+            self._check_parent_region(session, region_ref)
+            region = Region.from_dict(region_ref)
+            session.add(region)
+            session.flush()
+        return region.to_dict()
+
+    def update_region(self, region_id, region_ref):
+        session = self.get_session()
+        with session.begin():
+            self._check_parent_region(session, region_ref)
+            ref = self._get_region(session, region_id)
+            old_dict = ref.to_dict()
+            old_dict.update(region_ref)
+            new_region = Region.from_dict(old_dict)
+            for attr in Region.attributes:
+                if attr != 'id':
+                    setattr(ref, attr, getattr(new_region, attr))
+            session.flush()
+        return ref.to_dict()
+
     # Services
     def list_services(self):
         session = self.get_session()
diff --git a/keystone/catalog/controllers.py b/keystone/catalog/controllers.py
index ccf4dfd9a..c055185fc 100644
--- a/keystone/catalog/controllers.py
+++ b/keystone/catalog/controllers.py
@@ -137,6 +137,43 @@ class Endpoint(controller.V2Controller):
 
 
 @dependency.requires('catalog_api')
+class RegionV3(controller.V3Controller):
+    collection_name = 'regions'
+    member_name = 'region'
+
+    def __init__(self):
+        super(RegionV3, self).__init__()
+        self.get_member_from_driver = self.catalog_api.get_region
+
+    @controller.protected()
+    def create_region(self, context, region):
+        ref = self._assign_unique_id(self._normalize_dict(region))
+
+        ref = self.catalog_api.create_region(ref['id'], ref)
+        return RegionV3.wrap_member(context, ref)
+
+    def list_regions(self, context):
+        refs = self.catalog_api.list_regions()
+        return RegionV3.wrap_collection(context, refs)
+
+    @controller.protected()
+    def get_region(self, context, region_id):
+        ref = self.catalog_api.get_region(region_id)
+        return RegionV3.wrap_member(context, ref)
+
+    @controller.protected()
+    def update_region(self, context, region_id, region):
+        self._require_matching_id(region_id, region)
+
+        ref = self.catalog_api.update_region(region_id, region)
+        return RegionV3.wrap_member(context, ref)
+
+    @controller.protected()
+    def delete_region(self, context, region_id):
+        return self.catalog_api.delete_region(region_id)
+
+
+@dependency.requires('catalog_api')
 class ServiceV3(controller.V3Controller):
     collection_name = 'services'
     member_name = 'service'
diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py
index 901b26aa1..02785a732 100644
--- a/keystone/catalog/core.py
+++ b/keystone/catalog/core.py
@@ -68,6 +68,25 @@ class Manager(manager.Manager):
     def __init__(self):
         super(Manager, self).__init__(CONF.catalog.driver)
 
+    def create_region(self, region_id, region_ref):
+        try:
+            return self.driver.create_region(region_id, region_ref)
+        except exception.NotFound:
+            parent_region_id = region_ref.get('parent_region_id')
+            raise exception.RegionNotFound(region_id=parent_region_id)
+
+    def get_region(self, region_id):
+        try:
+            return self.driver.get_region(region_id)
+        except exception.NotFound:
+            raise exception.RegionNotFound(region_id=region_id)
+
+    def delete_region(self, region_id):
+        try:
+            return self.driver.delete_region(region_id)
+        except exception.NotFound:
+            raise exception.RegionNotFound(region_id=region_id)
+
     def get_service(self, service_id):
         try:
             return self.driver.get_service(service_id)
@@ -111,6 +130,54 @@ class Driver(object):
     """Interface description for an Catalog driver."""
 
     @abc.abstractmethod
+    def create_region(self, region_id, region_ref):
+        """Creates a new region.
+
+        :raises: keystone.exception.Conflict
+        :raises: keystone.exception.RegionNotFound (if parent region invalid)
+
+        """
+        raise exception.NotImplemented()
+
+    @abc.abstractmethod
+    def list_regions(self):
+        """List all regions.
+
+        :returns: list of region_refs or an empty list.
+
+        """
+        raise exception.NotImplemented()
+
+    @abc.abstractmethod
+    def get_region(self, region_id):
+        """Get region by id.
+
+        :returns: region_ref dict
+        :raises: keystone.exception.RegionNotFound
+
+        """
+        raise exception.NotImplemented()
+
+    @abc.abstractmethod
+    def update_region(self, region_id):
+        """Update region by id.
+
+        :returns: region_ref dict
+        :raises: keystone.exception.RegionNotFound
+
+        """
+        raise exception.NotImplemented()
+
+    @abc.abstractmethod
+    def delete_region(self, region_id):
+        """Deletes an existing region.
+
+        :raises: keystone.exception.RegionNotFound
+
+        """
+        raise exception.NotImplemented()
+
+    @abc.abstractmethod
     def create_service(self, service_id, service_ref):
         """Creates a new service.
 
diff --git a/keystone/catalog/routers.py b/keystone/catalog/routers.py
index ad76b41e4..80740aa75 100644
--- a/keystone/catalog/routers.py
+++ b/keystone/catalog/routers.py
@@ -19,6 +19,8 @@ from keystone.common import router
 
 
 def append_v3_routers(mapper, routers):
+    routers.append(router.Router(controllers.RegionV3(),
+                                 'regions', 'region'))
     routers.append(router.Router(controllers.ServiceV3(),
                                  'services', 'service'))
     routers.append(router.Router(controllers.EndpointV3(),
diff --git a/keystone/common/sql/migrate_repo/versions/037_add_region_table.py b/keystone/common/sql/migrate_repo/versions/037_add_region_table.py
new file mode 100644
index 000000000..e1f2e659f
--- /dev/null
+++ b/keystone/common/sql/migrate_repo/versions/037_add_region_table.py
@@ -0,0 +1,40 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    region_table = sql.Table(
+        'region',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('description', sql.String(255), unique=True,
+                   nullable=False),
+        sql.Column('parent_region_id', sql.String(64), nullable=True),
+        sql.Column('extra', sql.Text()))
+    region_table.create(migrate_engine, checkfirst=True)
+
+
+def downgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    region = sql.Table('region', meta, autoload=True)
+    region.drop(migrate_engine, checkfirst=True)
diff --git a/keystone/exception.py b/keystone/exception.py
index 66378c541..ceb22cdb7 100644
--- a/keystone/exception.py
+++ b/keystone/exception.py
@@ -189,6 +189,10 @@ class RoleNotFound(NotFound):
     message_format = _("Could not find role, %(role_id)s.")
 
 
+class RegionNotFound(NotFound):
+    message_format = _("Could not find region, %(region_id)s.")
+
+
 class ServiceNotFound(NotFound):
     message_format = _("Could not find service, %(service_id)s.")
 
diff --git a/keystone/tests/test_backend.py b/keystone/tests/test_backend.py
index 43b4894e7..b555dc34c 100644
--- a/keystone/tests/test_backend.py
+++ b/keystone/tests/test_backend.py
@@ -21,6 +21,7 @@ import mock
 import uuid
 
 from six import moves
+from testtools import matchers
 
 from keystone.catalog import core
 from keystone import config
@@ -3437,6 +3438,80 @@ class CommonHelperTests(tests.TestCase):
 
 
 class CatalogTests(object):
+    def test_region_crud(self):
+        # create
+        region_id = uuid.uuid4().hex
+        new_region = {
+            'id': region_id,
+            'description': uuid.uuid4().hex,
+        }
+        res = self.catalog_api.create_region(
+            region_id,
+            new_region.copy())
+        # Ensure that we don't need to have a
+        # parent_region_id in the original supplied
+        # ref dict, but that it will be returned from
+        # the endpoint, with None value.
+        expected_region = new_region.copy()
+        expected_region['parent_region_id'] = None
+        self.assertDictEqual(res, expected_region)
+
+        # Test adding another region with the one above
+        # as its parent. We will check below whether deleting
+        # the parent successfully deletes any child regions.
+        parent_region_id = region_id
+        region_id = uuid.uuid4().hex
+        new_region = {
+            'id': region_id,
+            'description': uuid.uuid4().hex,
+            'parent_region_id': parent_region_id
+        }
+        self.catalog_api.create_region(
+            region_id,
+            new_region.copy())
+
+        # list
+        regions = self.catalog_api.list_regions()
+        self.assertThat(regions, matchers.HasLength(2))
+        region_ids = [x['id'] for x in regions]
+        self.assertIn(parent_region_id, region_ids)
+        self.assertIn(region_id, region_ids)
+
+        # delete
+        self.catalog_api.delete_region(parent_region_id)
+        self.assertRaises(exception.RegionNotFound,
+                          self.catalog_api.delete_region,
+                          parent_region_id)
+        self.assertRaises(exception.RegionNotFound,
+                          self.catalog_api.get_region,
+                          parent_region_id)
+        # Ensure the child is also gone...
+        self.assertRaises(exception.RegionNotFound,
+                          self.catalog_api.get_region,
+                          region_id)
+
+    def test_get_region_404(self):
+        self.assertRaises(exception.RegionNotFound,
+                          self.catalog_api.get_region,
+                          uuid.uuid4().hex)
+
+    def test_delete_region_404(self):
+        self.assertRaises(exception.RegionNotFound,
+                          self.catalog_api.delete_region,
+                          uuid.uuid4().hex)
+
+    def test_create_region_invalid_parent_region_404(self):
+        region_id = uuid.uuid4().hex
+        new_region = {
+            'id': region_id,
+            'description': uuid.uuid4().hex,
+            'parent_region_id': 'nonexisting'
+        }
+        self.assertRaises(exception.RegionNotFound,
+                          self.catalog_api.create_region,
+                          region_id,
+                          new_region)
+
     def test_service_crud(self):
         # create
         service_id = uuid.uuid4().hex
diff --git a/keystone/tests/test_sql_upgrade.py b/keystone/tests/test_sql_upgrade.py
index babefe0b4..a412c1227 100644
--- a/keystone/tests/test_sql_upgrade.py
+++ b/keystone/tests/test_sql_upgrade.py
@@ -1674,6 +1674,14 @@ class SqlUpgradeTests(SqlMigrateBase):
         self.assertEqual(user1['default_project_id'],
                          new_json_data['tenant_id'])
 
+    def test_region_migration(self):
+        self.upgrade(36)
+        self.assertTableDoesNotExist('region')
+        self.upgrade(37)
+        self.assertTableExists('region')
+        self.downgrade(36)
+        self.assertTableDoesNotExist('region')
+
     def populate_user_table(self, with_pass_enab=False,
                             with_pass_enab_domain=False):
         # Populate the appropriate fields in the user
diff --git a/keystone/tests/test_v3.py b/keystone/tests/test_v3.py
index f45e75f6a..18be2eaf1 100644
--- a/keystone/tests/test_v3.py
+++ b/keystone/tests/test_v3.py
@@ -152,6 +152,13 @@ class RestfulTestCase(rest.RestfulTestCase):
             self.default_domain_user_id, self.project_id,
             self.role_id)
 
+        self.region_id = uuid.uuid4().hex
+        self.region = self.new_region_ref()
+        self.region['id'] = self.region_id
+        self.catalog_api.create_region(
+            self.region_id,
+            self.region.copy())
+
         self.service_id = uuid.uuid4().hex
         self.service = self.new_service_ref()
         self.service['id'] = self.service_id
@@ -174,6 +181,14 @@ class RestfulTestCase(rest.RestfulTestCase):
             'description': uuid.uuid4().hex,
             'enabled': True}
 
+    def new_region_ref(self):
+        ref = self.new_ref()
+        # Region doesn't have name or enabled.
+        del ref['name']
+        del ref['enabled']
+        ref['parent_region_id'] = None
+        return ref
+
     def new_service_ref(self):
         ref = self.new_ref()
         ref['type'] = uuid.uuid4().hex
@@ -449,7 +464,7 @@ class RestfulTestCase(rest.RestfulTestCase):
         If a reference is provided, the entity will also be compared against
         the reference.
         """
-        if keys_to_check:
+        if keys_to_check is not None:
             keys = keys_to_check
         else:
             keys = ['name', 'description', 'enabled']
@@ -599,6 +614,40 @@ class RestfulTestCase(rest.RestfulTestCase):
 
         return self.assertDictEqual(normalize(a), normalize(b))
 
+    # region validation
+
+    def assertValidRegionListResponse(self, resp, *args, **kwargs):
+        #NOTE(jaypipes): I have to pass in a blank keys_to_check parameter
+        #                below otherwise the base assertValidEntity method
+        #                tries to find a "name" and an "enabled" key in the
+        #                returned ref dicts. The issue is, I don't understand
+        #                how the service and endpoint entity assertions below
+        #                actually work (they don't raise assertions), since
+        #                AFAICT, the service and endpoint tables don't have
+        #                a "name" column either... :(
+        return self.assertValidListResponse(
+            resp,
+            'regions',
+            self.assertValidRegion,
+            keys_to_check=[],
+            *args,
+            **kwargs)
+
+    def assertValidRegionResponse(self, resp, *args, **kwargs):
+        return self.assertValidResponse(
+            resp,
+            'region',
+            self.assertValidRegion,
+            keys_to_check=[],
+            *args,
+            **kwargs)
+
+    def assertValidRegion(self, entity, ref=None):
+        self.assertIsNotNone(entity.get('description'))
+        if ref:
+            self.assertEqual(ref['description'], entity['description'])
+        return entity
+
     # service validation
 
     def assertValidServiceListResponse(self, resp, *args, **kwargs):
diff --git a/keystone/tests/test_v3_catalog.py b/keystone/tests/test_v3_catalog.py
index 56f75151a..8edc45a8b 100644
--- a/keystone/tests/test_v3_catalog.py
+++ b/keystone/tests/test_v3_catalog.py
@@ -23,6 +23,46 @@ class CatalogTestCase(test_v3.RestfulTestCase):
     def setUp(self):
         super(CatalogTestCase, self).setUp()
 
+    # region crud tests
+
+    def test_create_region(self):
+        """Call ``POST /regions``."""
+        ref = self.new_region_ref()
+        r = self.post(
+            '/regions',
+            body={'region': ref})
+        return self.assertValidRegionResponse(r, ref)
+
+    def test_list_regions(self):
+        """Call ``GET /regions``."""
+        r = self.get('/regions')
+        self.assertValidRegionListResponse(r, ref=self.region)
+
+    def test_list_regions_xml(self):
+        """Call ``GET /regions (xml data)``."""
+        r = self.get('/regions', content_type='xml')
+        self.assertValidRegionListResponse(r, ref=self.region)
+
+    def test_get_region(self):
+        """Call ``GET /regions/{region_id}``."""
+        r = self.get('/regions/%(region_id)s' % {
+            'region_id': self.region_id})
+        self.assertValidRegionResponse(r, self.region)
+
+    def test_update_region(self):
+        """Call ``PATCH /regions/{region_id}``."""
+        region = self.new_region_ref()
+        del region['id']
+        r = self.patch('/regions/%(region_id)s' % {
+            'region_id': self.region_id},
+            body={'region': region})
+        self.assertValidRegionResponse(r, region)
+
+    def test_delete_region(self):
+        """Call ``DELETE /regions/{region_id}``."""
+        self.delete('/regions/%(region_id)s' % {
+            'region_id': self.region_id})
+
     # service crud tests
 
     def test_create_service(self):