path: root/nova/objects/build_request.py

#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import re

from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import versionutils
from oslo_versionedobjects import exception as ovoo_exc
import six
from sqlalchemy.sql import null

from nova.db.sqlalchemy import api as db
from nova.db.sqlalchemy import api_models
from nova import exception
from nova.i18n import _LE
from nova import objects
from nova.objects import base
from nova.objects import fields

LOG = logging.getLogger(__name__)


@base.NovaObjectRegistry.register
class BuildRequest(base.NovaObject):
    # Version 1.0: Initial version
    # Version 1.1: Added block_device_mappings
    # Version 1.2: Added save() method
    VERSION = '1.2'

    fields = {
        'id': fields.IntegerField(),
        'instance_uuid': fields.UUIDField(),
        'project_id': fields.StringField(),
        'instance': fields.ObjectField('Instance'),
        'block_device_mappings': fields.ObjectField('BlockDeviceMappingList'),
        # NOTE(alaski): Normally these would come from the NovaPersistentObject
        # mixin but they're being set explicitly because we only need
        # created_at/updated_at. There is no soft delete for this object.
        'created_at': fields.DateTimeField(nullable=True),
        'updated_at': fields.DateTimeField(nullable=True),
    }

    def obj_make_compatible(self, primitive, target_version):
        super(BuildRequest, self).obj_make_compatible(primitive,
                                                      target_version)
        target_version = versionutils.convert_version_to_tuple(target_version)
        if target_version < (1, 1) and 'block_device_mappings' in primitive:
            del primitive['block_device_mappings']

    def _load_instance(self, db_instance):
        # NOTE(alaski): Be very careful with instance loading because it
        # changes more than most objects.
        try:
            self.instance = objects.Instance.obj_from_primitive(
                    jsonutils.loads(db_instance))
        except TypeError:
            LOG.debug('Failed to load instance from BuildRequest with uuid '
                      '%s because it is None', self.instance_uuid)
            raise exception.BuildRequestNotFound(uuid=self.instance_uuid)
        except ovoo_exc.IncompatibleObjectVersion:
            # This should only happen if proper service upgrade strategies are
            # not followed. Log the exception and raise BuildRequestNotFound.
            # If the instance can't be loaded this object is useless and may
            # as well not exist.
            LOG.debug('Could not deserialize instance store in BuildRequest '
                      'with uuid %(instance_uuid)s. Found version %(version)s '
                      'which is not supported here.',
                      dict(instance_uuid=self.instance_uuid,
                           version=jsonutils.loads(
                               db_instance)["nova_object.version"]))
            LOG.exception(_LE('Could not deserialize instance in '
                              'BuildRequest'))
            raise exception.BuildRequestNotFound(uuid=self.instance_uuid)
        # NOTE(sbauza): The instance primitive should already have the deleted
        # field being set, so when hydrating it back here, we should get the
        # right value but in case we don't have it, let's suppose that the
        # instance is not deleted, which is the default value for that field.
        self.instance.obj_set_defaults('deleted')
        # NOTE(alaski): Set some fields on instance that are needed by the api,
        # not lazy-loadable, and don't change.
        self.instance.disable_terminate = False
        self.instance.terminated_at = None
        self.instance.host = None
        self.instance.node = None
        self.instance.launched_at = None
        self.instance.launched_on = None
        self.instance.cell_name = None
        # The fields above are not set until the instance is in a cell at
        # which point this BuildRequest will be gone. locked_by could
        # potentially be set by an update so it should not be overwritten.
        if not self.instance.obj_attr_is_set('locked_by'):
            self.instance.locked_by = None
        # created_at/updated_at are not on the serialized instance because it
        # was never persisted.
        self.instance.created_at = self.created_at
        self.instance.updated_at = self.updated_at
        self.instance.tags = objects.TagList([])

    def _load_block_device_mappings(self, db_bdms):
        # 'db_bdms' is a serialized BlockDeviceMappingList object. If it's None
        # we're in a mixed version nova-api scenario and can't retrieve the
        # actual list. Set it to an empty list here which will cause a
        # temporary API inconsistency that will be resolved as soon as the
        # instance is scheduled and on a compute.
        if db_bdms is None:
            LOG.debug('Failed to load block_device_mappings from BuildRequest '
                      'for instance %s because it is None', self.instance_uuid)
            self.block_device_mappings = objects.BlockDeviceMappingList()
            return

        self.block_device_mappings = (
            objects.BlockDeviceMappingList.obj_from_primitive(
                jsonutils.loads(db_bdms)))

    @staticmethod
    def _from_db_object(context, req, db_req):
        # Set this up front so that it can be pulled for error messages or
        # logging at any point.
        req.instance_uuid = db_req['instance_uuid']

        for key in req.fields:
            if key == 'instance':
                continue
            elif isinstance(req.fields[key], fields.ObjectField):
                try:
                    getattr(req, '_load_%s' % key)(db_req[key])
                except AttributeError:
                    LOG.exception(_LE('No load handler for %s'), key)
            else:
                setattr(req, key, db_req[key])
        # Load instance last because other fields on req may be referenced
        req._load_instance(db_req['instance'])
        req.obj_reset_changes(recursive=True)
        req._context = context
        return req

    @staticmethod
    @db.api_context_manager.reader
    def _get_by_instance_uuid_from_db(context, instance_uuid):
        db_req = context.session.query(api_models.BuildRequest).filter_by(
                    instance_uuid=instance_uuid).first()
        if not db_req:
            raise exception.BuildRequestNotFound(uuid=instance_uuid)
        return db_req

    @base.remotable_classmethod
    def get_by_instance_uuid(cls, context, instance_uuid):
        db_req = cls._get_by_instance_uuid_from_db(context, instance_uuid)
        return cls._from_db_object(context, cls(), db_req)

    @staticmethod
    @db.api_context_manager.writer
    def _create_in_db(context, updates):
        db_req = api_models.BuildRequest()
        db_req.update(updates)
        db_req.save(context.session)
        return db_req

    def _get_update_primitives(self):
        updates = self.obj_get_changes()
        for key, value in updates.items():
            if isinstance(self.fields[key], fields.ObjectField):
                updates[key] = jsonutils.dumps(value.obj_to_primitive())
        return updates

    @base.remotable
    def create(self):
        if self.obj_attr_is_set('id'):
            raise exception.ObjectActionError(action='create',
                                              reason='already created')
        if not self.obj_attr_is_set('instance_uuid'):
            # We can't guarantee this is not null in the db so check here
            raise exception.ObjectActionError(action='create',
                    reason='instance_uuid must be set')

        updates = self._get_update_primitives()
        db_req = self._create_in_db(self._context, updates)
        self._from_db_object(self._context, self, db_req)

    @staticmethod
    @db.api_context_manager.writer
    def _destroy_in_db(context, instance_uuid):
        result = context.session.query(api_models.BuildRequest).filter_by(
                instance_uuid=instance_uuid).delete()
        if not result:
            raise exception.BuildRequestNotFound(uuid=instance_uuid)

    @base.remotable
    def destroy(self):
        self._destroy_in_db(self._context, self.instance_uuid)

    @db.api_context_manager.writer
    def _save_in_db(self, context, req_id, updates):
        db_req = context.session.query(
            api_models.BuildRequest).filter_by(id=req_id).first()
        if not db_req:
            raise exception.BuildRequestNotFound(uuid=self.instance_uuid)

        db_req.update(updates)
        context.session.add(db_req)
        return db_req

    @base.remotable
    def save(self):
        updates = self._get_update_primitives()
        db_req = self._save_in_db(self._context, self.id, updates)
        self._from_db_object(self._context, self, db_req)

    def get_new_instance(self, context):
        # NOTE(danms): This is a hack to make sure that the returned
        # instance has all dirty fields. There are probably better
        # ways to do this, but they kinda involve o.vo internals
        # so this is okay for the moment.
        instance = objects.Instance(context)
        for field in self.instance.obj_fields:
            # NOTE(danms): Don't copy the defaulted tags field
            # as instance.create() won't handle it properly.
            if field == 'tags':
                continue
            if self.instance.obj_attr_is_set(field):
                setattr(instance, field, getattr(self.instance, field))
        return instance


@base.NovaObjectRegistry.register
class BuildRequestList(base.ObjectListBase, base.NovaObject):
    # Version 1.0: Initial version
    VERSION = '1.0'

    fields = {
        'objects': fields.ListOfObjectsField('BuildRequest'),
    }

    @staticmethod
    @db.api_context_manager.reader
    def _get_all_from_db(context):
        query = context.session.query(api_models.BuildRequest)

        if not context.is_admin:
            query = query.filter_by(project_id=context.project_id)

        db_reqs = query.all()
        return db_reqs

    @base.remotable_classmethod
    def get_all(cls, context):
        db_build_reqs = cls._get_all_from_db(context)
        return base.obj_make_list(context, cls(context), objects.BuildRequest,
                                  db_build_reqs)

    @staticmethod
    def _pass_exact_filters(instance, filters):
        for filter_key, filter_val in filters.items():
            if filter_key in ('metadata', 'system_metadata'):
                if isinstance(filter_val, list):
                    for item in filter_val:
                        for k, v in item.items():
                            if (k not in instance.metadata or
                                    v != instance.metadata[k]):
                                return False
                else:
                    for k, v in filter_val.items():
                        if (k not in instance.metadata or
                                v != instance.metadata[k]):
                            return False
            elif isinstance(filter_val, (list, tuple, set, frozenset)):
                if not filter_val:
                    # Special value to indicate that nothing will match.
                    return None
                if instance.get(filter_key, None) not in filter_val:
                    return False
            else:
                if instance.get(filter_key, None) != filter_val:
                    return False

        return True

    @staticmethod
    def _pass_regex_filters(instance, filters):
        for filter_name, filter_val in filters.items():
            try:
                instance_attr = getattr(instance, filter_name)
            except AttributeError:
                continue

            # Sometimes the REGEX filter value is not a string
            if not isinstance(filter_val, six.string_types):
                filter_val = str(filter_val)
            filter_re = re.compile(filter_val)
            if instance_attr and not filter_re.search(str(instance_attr)):
                return False

        return True

    @staticmethod
    def _sort_build_requests(build_req_list, sort_keys, sort_dirs):
        # build_req_list is a [] of build_reqs
        sort_keys.reverse()
        sort_dirs.reverse()

        def sort_attr(sort_key, build_req):
            if sort_key == 'id':
                # 'id' is not set on the instance yet. Use the BuildRequest
                # 'id' instead.
                return build_req.id
            return getattr(build_req.instance, sort_key)

        for sort_key, sort_dir in zip(sort_keys, sort_dirs):
            reverse = False if sort_dir.lower().startswith('asc') else True
            build_req_list.sort(key=functools.partial(sort_attr, sort_key),
                                reverse=reverse)
        return build_req_list

    @base.remotable_classmethod
    def get_by_filters(cls, context, filters, limit=None, marker=None,
                       sort_keys=None, sort_dirs=None):
        # Short-circuit on anything that will not yield results.
        # 'deleted' records can not be returned from here since build_requests
        # are not soft deleted.
        # 'cleaned' records won't exist as they would need to be deleted.
        if (limit == 0 or
                filters.get('deleted', False) or
                filters.get('cleaned', False)):
            # If we have a marker honor the MarkerNotFound semantics.
            if marker:
                raise exception.MarkerNotFound(marker=marker)
            return cls(context, objects=[])

        # Because the build_requests table stores an instance as a serialized
        # versioned object it is not feasible to do the filtering and sorting
        # in the database. Just get all potentially relevant records and
        # process them here. It should be noted that build requests are short
        # lived so there should not be a lot of results to deal with.
        build_requests = cls.get_all(context)

        # Fortunately some filters do not apply here.
        # 'tags' can not be applied at boot time so will not be set for an
        # instance here.
        # 'changes-since' works off of the updated_at field which has not yet
        # been set at the point in the boot process where build_request still
        # exists. So it can be ignored.
        # 'deleted' and 'cleaned' are handled above.

        sort_keys, sort_dirs = db.process_sort_params(sort_keys, sort_dirs,
                                                      default_dir='desc')

        # For other filters that don't match this, we will do regexp matching
        # Taken from db/sqlalchemy/api.py
        exact_match_filter_names = ['project_id', 'user_id', 'image_ref',
                                    'vm_state', 'instance_type_id', 'uuid',
                                    'metadata', 'host', 'task_state',
                                    'system_metadata']
        exact_filters = {}
        regex_filters = {}
        for key, value in filters.items():
            if key in exact_match_filter_names:
                exact_filters[key] = value
            else:
                regex_filters[key] = value

        # As much as possible this copies the logic from db/sqlalchemy/api.py
        # instance_get_all_by_filters_sort. The main difference is that method
        # builds a sql query and this filters in python.
        filtered_build_reqs = []

        for build_req in build_requests:
            instance = build_req.instance

            filter_result = cls._pass_exact_filters(instance, exact_filters)
            if filter_result is None:
                # The filter condition is such that nothing will match.
                # Bail early.
                return cls(context, objects=[])
            if filter_result is False:
                continue

            if not cls._pass_regex_filters(instance, regex_filters):
                continue

            filtered_build_reqs.append(build_req)

        if (((len(filtered_build_reqs) < 2) or (not sort_keys))
                and not marker):
            # No need to sort
            return cls(context, objects=filtered_build_reqs)

        sorted_build_reqs = cls._sort_build_requests(filtered_build_reqs,
                                                     sort_keys, sort_dirs)

        marker_index = 0
        if marker:
            for i, build_req in enumerate(sorted_build_reqs):
                if build_req.instance.uuid == marker:
                    # The marker is the last seen item in the last page, so
                    # we increment the index to the next item immediately
                    # after the marker so the marker is not returned.
                    marker_index = i + 1
                    break
            else:
                raise exception.MarkerNotFound(marker=marker)
        len_build_reqs = len(sorted_build_reqs)
        limit_index = len_build_reqs
        if limit:
            limit_index = marker_index + limit
            if limit_index > len_build_reqs:
                limit_index = len_build_reqs

        return cls(context,
                   objects=sorted_build_reqs[marker_index:limit_index])


@db.api_context_manager.reader
def _get_build_requests_with_no_instance_uuid(context, limit):
    """Returns up to $limit build_requests where instance_uuid is null"""
    # build_requests don't use the SoftDeleteMixin so we don't have to filter
    # on the deleted column.
    return context.session.query(api_models.BuildRequest).\
        filter_by(instance_uuid=null()).\
        limit(limit).\
        all()


@db.api_context_manager.writer
def _destroy_in_db(context, id):
    return context.session.query(api_models.BuildRequest).filter_by(
        id=id).delete()


def delete_build_requests_with_no_instance_uuid(context, count):
    """Online data migration which cleans up failed build requests from Mitaka

    build_requests were initially a mirror of instances and had similar fields
    to satisfy listing/showing instances while they were building. In Mitaka
    if an instance failed to build we'd delete the instance but didn't delete
    the associated BuildRequest. In the Newton release we changed the schema
    on the build_requests table to just store a serialized Instance object and
    added an instance_uuid field which is expected to not be None as seen how
    it's used in _from_db_object. However, failed build requests created before
    that schema migration won't have the instance_uuid set and fail to load
    as an object when calling BuildRequestList.get_all(). So we need to perform
    a cleanup routine here where we search for build requests which do not have
    the instance_uuid field set and delete them.

    :param context: The auth context used to query the database.
    :type context: nova.context.RequestContext
    :param count: The max number of build requests to delete.
    :type count: int
    :returns: 2-item tuple of
        (number of orphaned build requests read from DB, number deleted)
    """
    orphaned_build_requests = (
        _get_build_requests_with_no_instance_uuid(context, count))
    done = 0
    for orphan_buildreq in orphaned_build_requests:
        result = _destroy_in_db(context, orphan_buildreq.id)
        if result:
            done += 1
    return len(orphaned_build_requests), done