diff options
64 files changed, 1781 insertions, 940 deletions
@@ -16,6 +16,8 @@ <jmckenty@gmail.com> <jmckenty@joshua-mckentys-macbook-pro.local> <jmckenty@gmail.com> <joshua.mckenty@nasa.gov> <justin@fathomdb.com> <justinsb@justinsb-desktop> +<masumotok@nttdata.co.jp> <root@openstack2-api> +<masumotok@nttdata.co.jp> Masumoto<masumotok@nttdata.co.jp> <mordred@inaugust.com> <mordred@hudson> <paul@openstack.org> <pvoccio@castor.local> <paul@openstack.org> <paul.voccio@rackspace.com> @@ -26,6 +26,7 @@ Josh Durgin <joshd@hq.newdream.net> Josh Kearney <josh.kearney@rackspace.com> Joshua McKenty <jmckenty@gmail.com> Justin Santa Barbara <justin@fathomdb.com> +Kei Masumoto <masumotok@nttdata.co.jp> Ken Pepple <ken.pepple@gmail.com> Koji Iida <iida.koji@lab.ntt.co.jp> Lorin Hochstein <lorin@isi.edu> @@ -34,6 +35,7 @@ Michael Gundlach <michael.gundlach@rackspace.com> Monsyne Dragon <mdragon@rackspace.com> Monty Taylor <mordred@inaugust.com> MORITA Kazutaka <morita.kazutaka@gmail.com> +Muneyuki Noguchi <noguchimn@nttdata.co.jp> Nachi Ueno <ueno.nachi@lab.ntt.co.jp> <openstack@lab.ntt.co.jp> <nati.ueno@gmail.com> <nova@u4> Paul Voccio <paul@openstack.org> Rick Clark <rick@openstack.org> diff --git a/MANIFEST.in b/MANIFEST.in index 199ce30b6d..07e4dd516a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,7 @@ include nova/cloudpipe/bootscript.sh include nova/cloudpipe/client.ovpn.template include nova/compute/fakevirtinstance.xml include nova/compute/interfaces.template +include nova/db/sqlalchemy/migrate_repo/migrate.cfg include nova/virt/interfaces.template include nova/virt/libvirt*.xml.template include nova/tests/CA/ diff --git a/bin/nova-api b/bin/nova-api index c7cbb3ab42..7b4fbeab14 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -34,22 +34,53 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) -from nova import api from nova import flags -from nova import utils +from nova import log as logging from nova import wsgi +logging.basicConfig() +LOG = logging.getLogger('nova.api') +LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS -flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') -flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') -flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') +API_ENDPOINTS = ['ec2', 'osapi'] -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) + +def run_app(paste_config_file): + LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) + apps = [] + for api in API_ENDPOINTS: + config = wsgi.load_paste_configuration(paste_config_file, api) + if config is None: + LOG.debug(_("No paste configuration for app: %s"), api) + continue + LOG.debug(_("App Config: %s\n%r"), api, config) + wsgi.paste_config_to_flags(config, { + "verbose": FLAGS.verbose, + "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) + LOG.info(_("Running %s API"), api) + app = wsgi.load_paste_app(paste_config_file, api) + apps.append((app, getattr(FLAGS, "%s_port" % api), + getattr(FLAGS, "%s_host" % api))) + if len(apps) == 0: + LOG.error(_("No known API applications configured in %s."), + paste_config_file) + return + + # NOTE(todd): redo logging config, verbose could be set in paste config + logging.basicConfig() server = wsgi.Server() - server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) - server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) + for app in apps: + server.start(*app) server.wait() + + +if __name__ == '__main__': + FLAGS(sys.argv) + conf = wsgi.paste_config_file('nova-api.conf') + if conf: + run_app(conf) + else: + LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') diff --git a/bin/nova-api-paste b/bin/nova-api-paste deleted file mode 100755 index 419f0bbdc2..0000000000 --- a/bin/nova-api-paste +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -# pylint: disable-msg=C0103 -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Starter script for Nova API.""" - -import gettext -import os -import sys - -from paste import deploy - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import wsgi - -LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) -LOG.addHandler(logging.StreamHandler()) - -FLAGS = flags.FLAGS - -API_ENDPOINTS = ['ec2', 'openstack'] - - -def load_configuration(paste_config): - """Load the paste configuration from the config file and return it.""" - config = None - # Try each known name to get the global DEFAULTS, which will give ports - for name in API_ENDPOINTS: - try: - config = deploy.appconfig("config:%s" % paste_config, name=name) - except LookupError: - pass - if config: - verbose = config.get('verbose', None) - if verbose: - FLAGS.verbose = int(verbose) == 1 - if FLAGS.verbose: - logging.getLogger().setLevel(logging.DEBUG) - return config - LOG.debug(_("Paste config at %s has no secion for known apis"), - paste_config) - print _("Paste config at %s has no secion for any known apis") % \ - paste_config - os.exit(1) - - -def launch_api(paste_config_file, section, server, port, host): - """Launch an api server from the specified port and IP.""" - LOG.debug(_("Launching %s api on %s:%s"), section, host, port) - app = deploy.loadapp('config:%s' % paste_config_file, name=section) - server.start(app, int(port), host) - - -def run_app(paste_config_file): - LOG.debug(_("Using paste.deploy config at: %s"), configfile) - config = load_configuration(paste_config_file) - LOG.debug(_("Configuration: %r"), config) - server = wsgi.Server() - ip = config.get('host', '0.0.0.0') - for api in API_ENDPOINTS: - port = config.get("%s_port" % api, None) - if not port: - continue - host = config.get("%s_host" % api, ip) - launch_api(configfile, api, server, port, host) - LOG.debug(_("All api servers launched, now waiting")) - server.wait() - - -if __name__ == '__main__': - FLAGS(sys.argv) - configfiles = ['/etc/nova/nova-api.conf'] - if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - configfiles.insert(0, - os.path.join(possible_topdir, 'etc', 'nova-api.conf')) - for configfile in configfiles: - if os.path.exists(configfile): - run_app(configfile) - break - else: - LOG.debug(_("Skipping missing configuration: %s"), configfile) diff --git a/bin/nova-combined b/bin/nova-combined index f932fdfd57..913c866bf2 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -36,22 +36,20 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) -from nova import api from nova import flags +from nova import log as logging from nova import service from nova import utils from nova import wsgi FLAGS = flags.FLAGS -flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') -flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') -flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.basicConfig() compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') @@ -61,7 +59,22 @@ if __name__ == '__main__': service.serve(compute, network, volume, scheduler) - server = wsgi.Server() - server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) - server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) - server.wait() + apps = [] + paste_config_file = wsgi.paste_config_file('nova-api.conf') + for api in ['osapi', 'ec2']: + config = wsgi.load_paste_configuration(paste_config_file, api) + if config is None: + continue + wsgi.paste_config_to_flags(config, { + "verbose": FLAGS.verbose, + "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) + app = wsgi.load_paste_app(paste_config_file, api) + apps.append((app, getattr(FLAGS, "%s_port" % api), + getattr(FLAGS, "%s_host" % api))) + if len(apps) > 0: + logging.basicConfig() + server = wsgi.Server() + for app in apps: + server.start(*app) + server.wait() diff --git a/bin/nova-direct-api b/bin/nova-direct-api new file mode 100755 index 0000000000..e7dd14fb27 --- /dev/null +++ b/bin/nova-direct-api @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# pylint: disable-msg=C0103 +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starter script for Nova Direct API.""" + +import gettext +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +gettext.install('nova', unicode=1) + +from nova import flags +from nova import utils +from nova import wsgi +from nova.api import direct +from nova.compute import api as compute_api + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('direct_port', 8001, 'Direct API port') +flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') + +if __name__ == '__main__': + utils.default_flagfile() + FLAGS(sys.argv) + + direct.register_service('compute', compute_api.ComputeAPI()) + direct.register_service('reflect', direct.Reflection()) + router = direct.Router() + with_json = direct.JsonParamsMiddleware(router) + with_req = direct.PostParamsMiddleware(with_json) + with_auth = direct.DelegatedAuthMiddleware(with_req) + + server = wsgi.Server() + server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host) + server.wait() diff --git a/bin/nova-manage b/bin/nova-manage index 7f241d7fa4..d0901ddfca 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -527,12 +527,11 @@ class DbCommands(object): pass def sync(self, version=None): - """adds role to user - if project is specified, adds project specific role - arguments: user, role [project]""" + """Sync the database up to the most recent version.""" return migration.db_sync(version) def version(self): + """Print the current database version.""" print migration.db_version() diff --git a/bin/stack b/bin/stack new file mode 100755 index 0000000000..7a6ce5960d --- /dev/null +++ b/bin/stack @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""CLI for the Direct API.""" + +import eventlet +eventlet.monkey_patch() + +import os +import pprint +import sys +import textwrap +import urllib +import urllib2 + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +import gflags +from nova import utils + + +FLAGS = gflags.FLAGS +gflags.DEFINE_string('host', '127.0.0.1', 'Direct API host') +gflags.DEFINE_integer('port', 8001, 'Direct API host') +gflags.DEFINE_string('user', 'user1', 'Direct API username') +gflags.DEFINE_string('project', 'proj1', 'Direct API project') + + +USAGE = """usage: stack [options] <controller> <method> [arg1=value arg2=value] + + `stack help` should output the list of available controllers + `stack <controller>` should output the available methods for that controller + `stack help <controller>` should do the same + `stack help <controller> <method>` should output info for a method +""" + + +def format_help(d): + """Format help text, keys are labels and values are descriptions.""" + indent = max([len(k) for k in d]) + out = [] + for k, v in d.iteritems(): + t = textwrap.TextWrapper(initial_indent=' %s ' % k.ljust(indent), + subsequent_indent=' ' * (indent + 6)) + out.extend(t.wrap(v)) + return out + + +def help_all(): + rv = do_request('reflect', 'get_controllers') + out = format_help(rv) + return (USAGE + str(FLAGS.MainModuleHelp()) + + '\n\nAvailable controllers:\n' + + '\n'.join(out) + '\n') + + +def help_controller(controller): + rv = do_request('reflect', 'get_methods') + methods = dict([(k.split('/')[2], v) for k, v in rv.iteritems() + if k.startswith('/%s' % controller)]) + return ('Available methods for %s:\n' % controller + + '\n'.join(format_help(methods))) + + +def help_method(controller, method): + rv = do_request('reflect', + 'get_method_info', + {'method': '/%s/%s' % (controller, method)}) + + sig = '%s(%s):' % (method, ', '.join(['='.join(x) for x in rv['args']])) + out = textwrap.wrap(sig, subsequent_indent=' ' * len('%s(' % method)) + out.append('\n' + rv['doc']) + return '\n'.join(out) + + +def do_request(controller, method, params=None): + if params: + data = urllib.urlencode(params) + else: + data = None + + url = 'http://%s:%s/%s/%s' % (FLAGS.host, FLAGS.port, controller, method) + headers = {'X-OpenStack-User': FLAGS.user, + 'X-OpenStack-Project': FLAGS.project} + + req = urllib2.Request(url, data, headers) + resp = urllib2.urlopen(req) + return utils.loads(resp.read()) + + +if __name__ == '__main__': + args = FLAGS(sys.argv) + + cmd = args.pop(0) + if not args: + print help_all() + sys.exit() + + first = args.pop(0) + if first == 'help': + action = help_all + params = [] + if args: + params.append(args.pop(0)) + action = help_controller + if args: + params.append(args.pop(0)) + action = help_method + print action(*params) + sys.exit(0) + + controller = first + if not args: + print help_controller(controller) + sys.exit() + + method = args.pop(0) + params = {} + for x in args: + key, value = x.split('=', 1) + params[key] = value + + pprint.pprint(do_request(controller, method, params)) diff --git a/etc/nova-api.conf b/etc/nova-api.conf index c5dd0aaec4..4873e465d1 100644 --- a/etc/nova-api.conf +++ b/etc/nova-api.conf @@ -1,9 +1,5 @@ [DEFAULT] verbose = 1 -ec2_port = 8773 -ec2_address = 0.0.0.0 -openstack_port = 8774 -openstack_address = 0.0.0.0 ####### # EC2 # @@ -12,52 +8,80 @@ openstack_address = 0.0.0.0 [composite:ec2] use = egg:Paste#urlmap /: ec2versions -/services: ec2api +/services/Cloud: ec2cloud +/services/Admin: ec2admin /latest: ec2metadata -/200: ec2metadata +/20: ec2metadata /1.0: ec2metadata -[pipeline:ec2api] -pipeline = authenticate router authorizer ec2executor +[pipeline:ec2cloud] +pipeline = logrequest authenticate cloudrequest authorizer ec2executor +#pipeline = logrequest ec2lockout authenticate cloudrequest authorizer ec2executor + +[pipeline:ec2admin] +pipeline = logrequest authenticate adminrequest authorizer ec2executor + +[pipeline:ec2metadata] +pipeline = logrequest ec2md + +[pipeline:ec2versions] +pipeline = logrequest ec2ver + +[filter:logrequest] +paste.filter_factory = nova.api.ec2:RequestLogging.factory + +[filter:ec2lockout] +paste.filter_factory = nova.api.ec2:Lockout.factory [filter:authenticate] -paste.filter_factory = nova.api.ec2:authenticate_factory +paste.filter_factory = nova.api.ec2:Authenticate.factory + +[filter:cloudrequest] +controller = nova.api.ec2.cloud.CloudController +paste.filter_factory = nova.api.ec2:Requestify.factory -[filter:router] -paste.filter_factory = nova.api.ec2:router_factory +[filter:adminrequest] +controller = nova.api.ec2.admin.AdminController +paste.filter_factory = nova.api.ec2:Requestify.factory [filter:authorizer] -paste.filter_factory = nova.api.ec2:authorizer_factory +paste.filter_factory = nova.api.ec2:Authorizer.factory [app:ec2executor] -paste.app_factory = nova.api.ec2:executor_factory +paste.app_factory = nova.api.ec2:Executor.factory -[app:ec2versions] -paste.app_factory = nova.api.ec2:versions_factory +[app:ec2ver] +paste.app_factory = nova.api.ec2:Versions.factory -[app:ec2metadata] -paste.app_factory = nova.api.ec2.metadatarequesthandler:metadata_factory +[app:ec2md] +paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.factory ############# # Openstack # ############# -[composite:openstack] +[composite:osapi] use = egg:Paste#urlmap /: osversions /v1.0: openstackapi [pipeline:openstackapi] -pipeline = auth ratelimit osapi +pipeline = faultwrap auth ratelimit osapiapp + +[filter:faultwrap] +paste.filter_factory = nova.api.openstack:FaultWrapper.factory [filter:auth] -paste.filter_factory = nova.api.openstack.auth:auth_factory +paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory [filter:ratelimit] -paste.filter_factory = nova.api.openstack.ratelimiting:ratelimit_factory +paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory + +[app:osapiapp] +paste.app_factory = nova.api.openstack:APIRouter.factory -[app:osapi] -paste.app_factory = nova.api.openstack:router_factory +[pipeline:osversions] +pipeline = faultwrap osversionapp -[app:osversions] -paste.app_factory = nova.api.openstack:versions_factory +[app:osversionapp] +paste.app_factory = nova.api.openstack:Versions.factory diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 8034705700..0fedbbfadc 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -15,96 +15,5 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" -Root WSGI middleware for all API controllers. -**Related Flags** - -:osapi_subdomain: subdomain running the OpenStack API (default: api) -:ec2api_subdomain: subdomain running the EC2 API (default: ec2) - -""" - -import routes -import webob.dec - -from nova import flags -from nova import wsgi -from nova.api import ec2 -from nova.api import openstack -from nova.api.ec2 import metadatarequesthandler - - -flags.DEFINE_string('osapi_subdomain', 'api', - 'subdomain running the OpenStack API') -flags.DEFINE_string('ec2api_subdomain', 'ec2', - 'subdomain running the EC2 API') - -FLAGS = flags.FLAGS - - -class API(wsgi.Router): - """Routes top-level requests to the appropriate controller.""" - - def __init__(self, default_api): - osapi_subdomain = {'sub_domain': [FLAGS.osapi_subdomain]} - ec2api_subdomain = {'sub_domain': [FLAGS.ec2api_subdomain]} - if default_api == 'os': - osapi_subdomain = {} - elif default_api == 'ec2': - ec2api_subdomain = {} - mapper = routes.Mapper() - mapper.sub_domains = True - - mapper.connect("/", controller=self.osapi_versions, - conditions=osapi_subdomain) - mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), - conditions=osapi_subdomain) - - mapper.connect("/", controller=self.ec2api_versions, - conditions=ec2api_subdomain) - mapper.connect("/services/{path_info:.*}", controller=ec2.API(), - conditions=ec2api_subdomain) - mrh = metadatarequesthandler.MetadataRequestHandler() - for s in ['/latest', - '/2009-04-04', - '/2008-09-01', - '/2008-02-01', - '/2007-12-15', - '/2007-10-10', - '/2007-08-29', - '/2007-03-01', - '/2007-01-19', - '/1.0']: - mapper.connect('%s/{path_info:.*}' % s, controller=mrh, - conditions=ec2api_subdomain) - - super(API, self).__init__(mapper) - - @webob.dec.wsgify - def osapi_versions(self, req): - """Respond to a request for all OpenStack API versions.""" - response = { - "versions": [ - dict(status="CURRENT", id="v1.0")]} - metadata = { - "application/xml": { - "attributes": dict(version=["status", "id"])}} - return wsgi.Serializer(req.environ, metadata).to_content_type(response) - - @webob.dec.wsgify - def ec2api_versions(self, req): - """Respond to a request for all EC2 versions.""" - # available api versions - versions = [ - '1.0', - '2007-01-19', - '2007-03-01', - '2007-08-29', - '2007-10-10', - '2007-12-15', - '2008-02-01', - '2008-09-01', - '2009-04-04', - ] - return ''.join('%s\n' % v for v in versions) +"""No-op __init__ for directory full of api goodies.""" diff --git a/nova/api/direct.py b/nova/api/direct.py new file mode 100644 index 0000000000..81b3ae2026 --- /dev/null +++ b/nova/api/direct.py @@ -0,0 +1,232 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Public HTTP interface that allows services to self-register. + +The general flow of a request is: + - Request is parsed into WSGI bits. + - Some middleware checks authentication. + - Routing takes place based on the URL to find a controller. + (/controller/method) + - Parameters are parsed from the request and passed to a method on the + controller as keyword arguments. + - Optionally 'json' is decoded to provide all the parameters. + - Actual work is done and a result is returned. + - That result is turned into json and returned. + +""" + +import inspect +import urllib + +import routes +import webob + +from nova import context +from nova import flags +from nova import utils +from nova import wsgi + + +ROUTES = {} + + +def register_service(path, handle): + ROUTES[path] = handle + + +class Router(wsgi.Router): + def __init__(self, mapper=None): + if mapper is None: + mapper = routes.Mapper() + + self._load_registered_routes(mapper) + super(Router, self).__init__(mapper=mapper) + + def _load_registered_routes(self, mapper): + for route in ROUTES: + mapper.connect('/%s/{action}' % route, + controller=ServiceWrapper(ROUTES[route])) + + +class DelegatedAuthMiddleware(wsgi.Middleware): + def process_request(self, request): + os_user = request.headers['X-OpenStack-User'] + os_project = request.headers['X-OpenStack-Project'] + context_ref = context.RequestContext(user=os_user, project=os_project) + request.environ['openstack.context'] = context_ref + + +class JsonParamsMiddleware(wsgi.Middleware): + def process_request(self, request): + if 'json' not in request.params: + return + + params_json = request.params['json'] + params_parsed = utils.loads(params_json) + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ['openstack.params'] = params + + +class PostParamsMiddleware(wsgi.Middleware): + def process_request(self, request): + params_parsed = request.params + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ['openstack.params'] = params + + +class Reflection(object): + """Reflection methods to list available methods.""" + def __init__(self): + self._methods = {} + self._controllers = {} + + def _gather_methods(self): + methods = {} + controllers = {} + for route, handler in ROUTES.iteritems(): + controllers[route] = handler.__doc__.split('\n')[0] + for k in dir(handler): + if k.startswith('_'): + continue + f = getattr(handler, k) + if not callable(f): + continue + + # bunch of ugly formatting stuff + argspec = inspect.getargspec(f) + args = [x for x in argspec[0] + if x != 'self' and x != 'context'] + defaults = argspec[3] and argspec[3] or [] + args_r = list(reversed(args)) + defaults_r = list(reversed(defaults)) + + args_out = [] + while args_r: + if defaults_r: + args_out.append((args_r.pop(0), + repr(defaults_r.pop(0)))) + else: + args_out.append((str(args_r.pop(0)),)) + + # if the method accepts keywords + if argspec[2]: + args_out.insert(0, ('**%s' % argspec[2],)) + + methods['/%s/%s' % (route, k)] = { + 'short_doc': f.__doc__.split('\n')[0], + 'doc': f.__doc__, + 'name': k, + 'args': list(reversed(args_out))} + + self._methods = methods + self._controllers = controllers + + def get_controllers(self, context): + """List available controllers.""" + if not self._controllers: + self._gather_methods() + + return self._controllers + + def get_methods(self, context): + """List available methods.""" + if not self._methods: + self._gather_methods() + + method_list = self._methods.keys() + method_list.sort() + methods = {} + for k in method_list: + methods[k] = self._methods[k]['short_doc'] + return methods + + def get_method_info(self, context, method): + """Get detailed information about a method.""" + if not self._methods: + self._gather_methods() + return self._methods[method] + + +class ServiceWrapper(wsgi.Controller): + def __init__(self, service_handle): + self.service_handle = service_handle + + @webob.dec.wsgify + def __call__(self, req): + arg_dict = req.environ['wsgiorg.routing_args'][1] + action = arg_dict['action'] + del arg_dict['action'] + + context = req.environ['openstack.context'] + # allow middleware up the stack to override the params + params = {} + if 'openstack.params' in req.environ: + params = req.environ['openstack.params'] + + # TODO(termie): do some basic normalization on methods + method = getattr(self.service_handle, action) + + result = method(context, **params) + if type(result) is dict or type(result) is list: + return self._serialize(result, req) + else: + return result + + +class Proxy(object): + """Pretend a Direct API endpoint is an object.""" + def __init__(self, app, prefix=None): + self.app = app + self.prefix = prefix + + def __do_request(self, path, context, **kwargs): + req = webob.Request.blank(path) + req.method = 'POST' + req.body = urllib.urlencode({'json': utils.dumps(kwargs)}) + req.environ['openstack.context'] = context + resp = req.get_response(self.app) + try: + return utils.loads(resp.body) + except Exception: + return resp.body + + def __getattr__(self, key): + if self.prefix is None: + return self.__class__(self.app, prefix=key) + + def _wrapper(context, **kwargs): + return self.__do_request('/%s/%s' % (self.prefix, key), + context, + **kwargs) + _wrapper.func_name = key + return _wrapper diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 2fa1f636ce..238cb0f383 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -30,10 +30,9 @@ from nova import context from nova import exception from nova import flags from nova import log as logging +from nova import utils from nova import wsgi from nova.api.ec2 import apirequest -from nova.api.ec2 import admin -from nova.api.ec2 import cloud from nova.auth import manager @@ -42,8 +41,6 @@ LOG = logging.getLogger("nova.api") flags.DEFINE_boolean('use_forwarded_for', False, 'Treat X-Forwarded-For as the canonical remote address. ' 'Only enable this if you have a sanitizing proxy.') -flags.DEFINE_boolean('use_lockout', False, - 'Whether or not to use lockout middleware.') flags.DEFINE_integer('lockout_attempts', 5, 'Number of failed auths before lockout.') flags.DEFINE_integer('lockout_minutes', 15, @@ -54,13 +51,8 @@ flags.DEFINE_list('lockout_memcached_servers', None, 'Memcached servers or None for in process cache.') -class API(wsgi.Middleware): - """Routing for all EC2 API requests.""" - - def __init__(self): - self.application = Authenticate(Router(Authorizer(Executor()))) - if FLAGS.use_lockout: - self.application = Lockout(self.application) +class RequestLogging(wsgi.Middleware): + """Access-Log akin logging for all EC2 API requests.""" @webob.dec.wsgify def __call__(self, req): @@ -192,26 +184,14 @@ class Authenticate(wsgi.Middleware): return self.application -class Router(wsgi.Middleware): - - """Add ec2.'controller', .'action', and .'action_args' to WSGI environ.""" +class Requestify(wsgi.Middleware): - def __init__(self, application): - super(Router, self).__init__(application) - self.map = routes.Mapper() - self.map.connect("/{controller_name}/") - self.controllers = dict(Cloud=cloud.CloudController(), - Admin=admin.AdminController()) + def __init__(self, app, controller): + super(Requestify, self).__init__(app) + self.controller = utils.import_class(controller)() @webob.dec.wsgify def __call__(self, req): - # Obtain the appropriate controller and action for this request. - try: - match = self.map.match(req.path_info) - controller_name = match['controller_name'] - controller = self.controllers[controller_name] - except: - raise webob.exc.HTTPNotFound() non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Version', 'Timestamp'] args = dict(req.params) @@ -229,8 +209,8 @@ class Router(wsgi.Middleware): LOG.debug(_('arg: %s\t\tval: %s'), key, value) # Success! - req.environ['ec2.controller'] = controller - req.environ['ec2.action'] = action + api_request = apirequest.APIRequest(self.controller, action, args) + req.environ['ec2.request'] = api_request req.environ['ec2.action_args'] = args return self.application @@ -291,16 +271,14 @@ class Authorizer(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): context = req.environ['ec2.context'] - controller_name = req.environ['ec2.controller'].__class__.__name__ - action = req.environ['ec2.action'] - allowed_roles = self.action_roles[controller_name].get(action, - ['none']) + controller = req.environ['ec2.request'].controller.__class__.__name__ + action = req.environ['ec2.request'].action + allowed_roles = self.action_roles[controller].get(action, ['none']) if self._matches_any_role(context, allowed_roles): return self.application else: LOG.audit(_("Unauthorized request for controller=%s " - "and action=%s"), controller_name, action, - context=context) + "and action=%s"), controller, action, context=context) raise webob.exc.HTTPUnauthorized() def _matches_any_role(self, context, roles): @@ -327,14 +305,10 @@ class Executor(wsgi.Application): @webob.dec.wsgify def __call__(self, req): context = req.environ['ec2.context'] - controller = req.environ['ec2.controller'] - action = req.environ['ec2.action'] - args = req.environ['ec2.action_args'] - - api_request = apirequest.APIRequest(controller, action) + api_request = req.environ['ec2.request'] result = None try: - result = api_request.send(context, **args) + result = api_request.invoke(context) except exception.NotFound as ex: LOG.info(_('NotFound raised: %s'), str(ex), context=context) return self._error(req, context, type(ex).__name__, str(ex)) @@ -391,29 +365,3 @@ class Versions(wsgi.Application): '2009-04-04', ] return ''.join('%s\n' % v for v in versions) - - -def authenticate_factory(global_args, **local_args): - def authenticator(app): - return Authenticate(app) - return authenticator - - -def router_factory(global_args, **local_args): - def router(app): - return Router(app) - return router - - -def authorizer_factory(global_args, **local_args): - def authorizer(app): - return Authorizer(app) - return authorizer - - -def executor_factory(global_args, **local_args): - return Executor() - - -def versions_factory(global_args, **local_args): - return Versions() diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index d0b417db11..78576470a5 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -83,11 +83,12 @@ def _try_convert(value): class APIRequest(object): - def __init__(self, controller, action): + def __init__(self, controller, action, args): self.controller = controller self.action = action + self.args = args - def send(self, context, **kwargs): + def invoke(self, context): try: method = getattr(self.controller, _camelcase_to_underscore(self.action)) @@ -100,7 +101,7 @@ class APIRequest(object): raise Exception(_error) args = {} - for key, value in kwargs.items(): + for key, value in self.args.items(): parts = key.split(".") key = _camelcase_to_underscore(parts[0]) if isinstance(value, str) or isinstance(value, unicode): diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index fb7e6a59f5..57d41ed67b 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -92,8 +92,11 @@ class CloudController(object): self.image_service = utils.import_object(FLAGS.image_service) self.network_api = network.API() self.volume_api = volume.API() - self.compute_api = compute.API(self.image_service, self.network_api, - self.volume_api) + self.compute_api = compute.API( + network_api=self.network_api, + image_service=self.image_service, + volume_api=self.volume_api, + hostname_factory=id_to_ec2_id) self.setup() def __str__(self): @@ -251,15 +254,15 @@ class CloudController(object): name, _sep, host = region.partition('=') endpoint = '%s://%s:%s%s' % (FLAGS.ec2_prefix, host, - FLAGS.cc_port, + FLAGS.ec2_port, FLAGS.ec2_suffix) regions.append({'regionName': name, 'regionEndpoint': endpoint}) else: regions = [{'regionName': 'nova', 'regionEndpoint': '%s://%s:%s%s' % (FLAGS.ec2_prefix, - FLAGS.cc_host, - FLAGS.cc_port, + FLAGS.ec2_host, + FLAGS.ec2_port, FLAGS.ec2_suffix)}] return {'regionInfo': regions} @@ -512,7 +515,8 @@ class CloudController(object): # instance_id is passed in as a list of instances ec2_id = instance_id[0] instance_id = ec2_id_to_id(ec2_id) - output = self.compute_api.get_console_output(context, instance_id) + output = self.compute_api.get_console_output( + context, instance_id=instance_id) now = datetime.datetime.utcnow() return {"InstanceId": ec2_id, "Timestamp": now, @@ -580,7 +584,7 @@ class CloudController(object): def delete_volume(self, context, volume_id, **kwargs): volume_id = ec2_id_to_id(volume_id) - self.volume_api.delete(context, volume_id) + self.volume_api.delete(context, volume_id=volume_id) return True def update_volume(self, context, volume_id, **kwargs): @@ -597,9 +601,12 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_id = ec2_id_to_id(volume_id) instance_id = ec2_id_to_id(instance_id) - LOG.audit(_("Attach volume %s to instacne %s at %s"), volume_id, + LOG.audit(_("Attach volume %s to instance %s at %s"), volume_id, instance_id, device, context=context) - self.compute_api.attach_volume(context, instance_id, volume_id, device) + self.compute_api.attach_volume(context, + instance_id=instance_id, + volume_id=volume_id, + device=device) volume = self.volume_api.get(context, volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], @@ -612,7 +619,7 @@ class CloudController(object): volume_id = ec2_id_to_id(volume_id) LOG.audit(_("Detach volume %s"), volume_id, context=context) volume = self.volume_api.get(context, volume_id) - instance = self.compute_api.detach_volume(context, volume_id) + instance = self.compute_api.detach_volume(context, volume_id=volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], 'instanceId': id_to_ec2_id(instance['id']), @@ -643,6 +650,10 @@ class CloudController(object): return i[0] def _format_instances(self, context, instance_id=None, **kwargs): + # TODO(termie): this method is poorly named as its name does not imply + # that it will be making a variety of database calls + # rather than simply formatting a bunch of instances that + # were handed to it reservations = {} # NOTE(vish): instance_id is an optional list of ids to filter by if instance_id: @@ -743,7 +754,9 @@ class CloudController(object): LOG.audit(_("Associate address %s to instance %s"), public_ip, instance_id, context=context) instance_id = ec2_id_to_id(instance_id) - self.compute_api.associate_floating_ip(context, instance_id, public_ip) + self.compute_api.associate_floating_ip(context, + instance_id=instance_id, + address=public_ip) return {'associateResponse': ["Address associated."]} def disassociate_address(self, context, public_ip, **kwargs): @@ -754,8 +767,9 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) instances = self.compute_api.create(context, - instance_types.get_by_type(kwargs.get('instance_type', None)), - kwargs['image_id'], + instance_type=instance_types.get_by_type( + kwargs.get('instance_type', None)), + image_id=kwargs['image_id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id', None), @@ -766,8 +780,7 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone'), - generate_hostname=id_to_ec2_id) + 'AvailabilityZone')) return self._format_run_instances(context, instances[0]['reservation_id']) @@ -777,7 +790,7 @@ class CloudController(object): LOG.debug(_("Going to start terminating instances")) for ec2_id in instance_id: instance_id = ec2_id_to_id(ec2_id) - self.compute_api.delete(context, instance_id) + self.compute_api.delete(context, instance_id=instance_id) return True def reboot_instances(self, context, instance_id, **kwargs): @@ -785,19 +798,19 @@ class CloudController(object): LOG.audit(_("Reboot instance %r"), instance_id, context=context) for ec2_id in instance_id: instance_id = ec2_id_to_id(ec2_id) - self.compute_api.reboot(context, instance_id) + self.compute_api.reboot(context, instance_id=instance_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" instance_id = ec2_id_to_id(instance_id) - self.compute_api.rescue(context, instance_id) + self.compute_api.rescue(context, instance_id=instance_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" instance_id = ec2_id_to_id(instance_id) - self.compute_api.unrescue(context, instance_id) + self.compute_api.unrescue(context, instance_id=instance_id) return True def update_instance(self, context, ec2_id, **kwargs): @@ -808,7 +821,7 @@ class CloudController(object): changes[field] = kwargs[field] if changes: instance_id = ec2_id_to_id(ec2_id) - self.compute_api.update(context, instance_id, **kwargs) + self.compute_api.update(context, instance_id=instance_id, **kwargs) return True def describe_images(self, context, image_id=None, **kwargs): diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 848f0b0343..6fb441656b 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -23,6 +23,7 @@ import webob.exc from nova import log as logging from nova import flags +from nova import wsgi from nova.api.ec2 import cloud @@ -30,7 +31,7 @@ LOG = logging.getLogger('nova.api.ec2.metadata') FLAGS = flags.FLAGS -class MetadataRequestHandler(object): +class MetadataRequestHandler(wsgi.Application): """Serve metadata from the EC2 API.""" def print_data(self, data): @@ -78,7 +79,3 @@ class MetadataRequestHandler(object): if data is None: raise webob.exc.HTTPNotFound() return self.print_data(data) - - -def metadata_factory(global_args, **local_args): - return MetadataRequestHandler() diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index f96e2af911..f2caac4836 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -23,11 +23,9 @@ WSGI middleware for OpenStack API controllers. import routes import webob.dec import webob.exc -import webob from nova import flags from nova import log as logging -from nova import utils from nova import wsgi from nova.api.openstack import faults from nova.api.openstack import backup_schedules @@ -40,32 +38,16 @@ from nova.api.openstack import shared_ip_groups LOG = logging.getLogger('nova.api.openstack') FLAGS = flags.FLAGS -flags.DEFINE_string('os_api_auth', - 'nova.api.openstack.auth.AuthMiddleware', - 'The auth mechanism to use for the OpenStack API implemenation') - -flags.DEFINE_string('os_api_ratelimiting', - 'nova.api.openstack.ratelimiting.RateLimitingMiddleware', - 'Default ratelimiting implementation for the Openstack API') - flags.DEFINE_string('os_krm_mapping_file', 'krm_mapping.json', 'Location of OpenStack Flavor/OS:EC2 Kernel/Ramdisk/Machine JSON file.') - flags.DEFINE_bool('allow_admin_api', False, 'When True, this API service will accept admin operations.') -class API(wsgi.Middleware): - """WSGI entry point for all OpenStack API requests.""" - - def __init__(self): - auth_middleware = utils.import_class(FLAGS.os_api_auth) - ratelimiting_middleware = \ - utils.import_class(FLAGS.os_api_ratelimiting) - app = auth_middleware(ratelimiting_middleware(APIRouter())) - super(API, self).__init__(app) +class FaultWrapper(wsgi.Middleware): + """Calls down the middleware stack, making exceptions into faults.""" @webob.dec.wsgify def __call__(self, req): @@ -83,6 +65,11 @@ class APIRouter(wsgi.Router): and method. """ + @classmethod + def factory(cls, global_config, **local_config): + """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" + return cls() + def __init__(self): mapper = routes.Mapper() @@ -132,11 +119,3 @@ class Versions(wsgi.Application): "application/xml": { "attributes": dict(version=["status", "id"])}} return wsgi.Serializer(req.environ, metadata).to_content_type(response) - - -def router_factory(global_cof, **local_conf): - return APIRouter() - - -def versions_factory(global_conf, **local_conf): - return Versions() diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 00e817c8de..1dfdd5318c 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -134,9 +134,3 @@ class AuthMiddleware(wsgi.Middleware): token = self.db.auth_create_token(ctxt, token_dict) return token, user return None, None - - -def auth_factory(global_conf, **local_conf): - def auth(app): - return AuthMiddleware(app) - return auth diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index a5f55a4890..9d56bc508a 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -78,7 +78,14 @@ def _translate_status(item): 'decrypting': 'preparing', 'untarring': 'saving', 'available': 'active'} - item['status'] = status_mapping[item['status']] + try: + item['status'] = status_mapping[item['status']] + except KeyError: + # TODO(sirp): Performing translation of status (if necessary) here for + # now. Perhaps this should really be done in EC2 API and + # S3ImageService + pass + return item @@ -92,9 +99,11 @@ def _filter_keys(item, keys): def _convert_image_id_to_hash(image): - image_id = abs(hash(image['imageId'])) - image['imageId'] = image_id - image['id'] = image_id + if 'imageId' in image: + # Convert EC2-style ID (i-blah) to Rackspace-style (int) + image_id = abs(hash(image['imageId'])) + image['imageId'] = image_id + image['id'] = image_id class Controller(wsgi.Controller): @@ -147,7 +156,11 @@ class Controller(wsgi.Controller): env = self._deserialize(req.body, req) instance_id = env["image"]["serverId"] name = env["image"]["name"] - return compute.API().snapshot(context, instance_id, name) + + image_meta = compute.API().snapshot( + context, instance_id, name) + + return dict(image=image_meta) def update(self, req, id): # Users may not modify public images, and that's all that diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index 81b83142ff..cbb4b897eb 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -219,9 +219,3 @@ class WSGIAppProxy(object): # No delay return None return float(resp.getheader('X-Wait-Seconds')) - - -def ratelimit_factory(global_conf, **local_conf): - def rl(app): - return RateLimitingMiddleware(app) - return rl diff --git a/nova/auth/manager.py b/nova/auth/manager.py index de18a3838e..1652e24e1c 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -682,7 +682,7 @@ class AuthManager(object): region, _sep, region_host = item.partition("=") regions[region] = region_host else: - regions = {'nova': FLAGS.cc_host} + regions = {'nova': FLAGS.ec2_host} for region, host in regions.iteritems(): rc = self.__generate_rc(user, pid, @@ -727,28 +727,28 @@ class AuthManager(object): def __generate_rc(user, pid, use_dmz=True, host=None): """Generate rc file for user""" if use_dmz: - cc_host = FLAGS.cc_dmz + ec2_host = FLAGS.ec2_dmz_host else: - cc_host = FLAGS.cc_host + ec2_host = FLAGS.ec2_host # NOTE(vish): Always use the dmz since it is used from inside the # instance s3_host = FLAGS.s3_dmz if host: s3_host = host - cc_host = host + ec2_host = host rc = open(FLAGS.credentials_template).read() rc = rc % {'access': user.access, 'project': pid, 'secret': user.secret, - 'ec2': '%s://%s:%s%s' % (FLAGS.ec2_prefix, - cc_host, - FLAGS.cc_port, - FLAGS.ec2_suffix), + 'ec2': '%s://%s:%s%s' % (FLAGS.ec2_scheme, + ec2_host, + FLAGS.ec2_port, + FLAGS.ec2_path), 's3': 'http://%s:%s' % (s3_host, FLAGS.s3_port), - 'os': '%s://%s:%s%s' % (FLAGS.os_prefix, - cc_host, + 'os': '%s://%s:%s%s' % (FLAGS.osapi_scheme, + ec2_host, FLAGS.osapi_port, - FLAGS.os_suffix), + FLAGS.osapi_path), 'user': user.name, 'nova': FLAGS.ca_file, 'cert': FLAGS.credential_cert_file, diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 8aefd341f8..dc6f55af29 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -68,8 +68,8 @@ class CloudPipe(object): shellfile = open(FLAGS.boot_script_template, "r") s = string.Template(shellfile.read()) shellfile.close() - boot_script = s.substitute(cc_dmz=FLAGS.cc_dmz, - cc_port=FLAGS.cc_port, + boot_script = s.substitute(cc_dmz=FLAGS.ec2_dmz_host, + cc_port=FLAGS.ec2_port, dmz_net=FLAGS.dmz_net, dmz_mask=FLAGS.dmz_mask, num_vpn=FLAGS.cnt_vpn_clients) diff --git a/nova/compute/api.py b/nova/compute/api.py index 791cd9d1fd..a6b99c1cb8 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -48,7 +48,8 @@ def generate_default_hostname(instance_id): class API(base.Base): """API for interacting with the compute manager.""" - def __init__(self, image_service=None, network_api=None, volume_api=None, + def __init__(self, image_service=None, network_api=None, + volume_api=None, hostname_factory=generate_default_hostname, **kwargs): if not image_service: image_service = utils.import_object(FLAGS.image_service) @@ -59,9 +60,11 @@ class API(base.Base): if not volume_api: volume_api = volume.API() self.volume_api = volume_api + self.hostname_factory = hostname_factory super(API, self).__init__(**kwargs) def get_network_topic(self, context, instance_id): + """Get the network topic for an instance.""" try: instance = self.get(context, instance_id) except exception.NotFound as e: @@ -82,8 +85,7 @@ class API(base.Base): min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', - availability_zone=None, user_data=None, - generate_hostname=generate_default_hostname): + availability_zone=None, user_data=None): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -173,9 +175,9 @@ class API(base.Base): security_group_id) # Set sane defaults if not specified - updates = dict(hostname=generate_hostname(instance_id)) - if (not hasattr(instance, 'display_name')) or \ - instance.display_name == None: + updates = dict(hostname=self.hostname_factory(instance_id)) + if (not hasattr(instance, 'display_name') or + instance.display_name == None): updates['display_name'] = "Server %s" % instance_id instance = self.update(context, instance_id, **updates) @@ -193,7 +195,7 @@ class API(base.Base): for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) - return instances + return [dict(x.iteritems()) for x in instances] def ensure_default_security_group(self, context): """ Create security group for the security context if it @@ -278,7 +280,8 @@ class API(base.Base): :retval None """ - return self.db.instance_update(context, instance_id, kwargs) + rv = self.db.instance_update(context, instance_id, kwargs) + return dict(rv.iteritems()) def delete(self, context, instance_id): LOG.debug(_("Going to try to terminate %s"), instance_id) @@ -309,7 +312,8 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - return self.db.instance_get_by_id(context, instance_id) + rv = self.db.instance_get_by_id(context, instance_id) + return dict(rv.iteritems()) def get_all(self, context, project_id=None, reservation_id=None, fixed_ip=None): @@ -318,7 +322,7 @@ class API(base.Base): an admin, it will retreive all instances in the system.""" if reservation_id is not None: return self.db.instance_get_all_by_reservation(context, - reservation_id) + reservation_id) if fixed_ip is not None: return self.db.fixed_ip_get_instance(context, fixed_ip) if project_id or not context.is_admin: @@ -331,27 +335,55 @@ class API(base.Base): project_id) return self.db.instance_get_all(context) - def _cast_compute_message(self, method, context, instance_id, host=None): - """Generic handler for RPC casts to compute.""" + def _cast_compute_message(self, method, context, instance_id, host=None, + params=None): + """Generic handler for RPC casts to compute. + + :param params: Optional dictionary of arguments to be passed to the + compute worker + + :retval None + """ + if not params: + params = {} if not host: instance = self.get(context, instance_id) host = instance['host'] queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - kwargs = {'method': method, 'args': {'instance_id': instance_id}} + params['instance_id'] = instance_id + kwargs = {'method': method, 'args': params} rpc.cast(context, queue, kwargs) - def _call_compute_message(self, method, context, instance_id, host=None): - """Generic handler for RPC calls to compute.""" + def _call_compute_message(self, method, context, instance_id, host=None, + params=None): + """Generic handler for RPC calls to compute. + + :param params: Optional dictionary of arguments to be passed to the + compute worker + + :retval: Result returned by compute worker + """ + if not params: + params = {} if not host: instance = self.get(context, instance_id) host = instance["host"] queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - kwargs = {"method": method, "args": {"instance_id": instance_id}} + params['instance_id'] = instance_id + kwargs = {'method': method, 'args': params} return rpc.call(context, queue, kwargs) def snapshot(self, context, instance_id, name): - """Snapshot the given instance.""" - self._cast_compute_message('snapshot_instance', context, instance_id) + """Snapshot the given instance. + + :retval: A dict containing image metadata + """ + data = {'name': name, 'is_public': False} + image_meta = self.image_service.create(context, data) + params = {'image_id': image_meta['id']} + self._cast_compute_message('snapshot_instance', context, instance_id, + params=params) + return image_meta def reboot(self, context, instance_id): """Reboot the given instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 613ee45f63..6f09ce674a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -294,7 +294,7 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) @exception.wrap_exception - def snapshot_instance(self, context, instance_id, name): + def snapshot_instance(self, context, instance_id, image_id): """Snapshot an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -311,7 +311,7 @@ class ComputeManager(manager.Manager): 'instance: %s (state: %s excepted: %s)'), instance_id, instance_ref['state'], power_state.RUNNING) - self.driver.snapshot(instance_ref, name) + self.driver.snapshot(instance_ref, image_id) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 6c9989b7b0..b63b84bed5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -611,9 +611,9 @@ def fixed_ip_get_instance_v6(context, address): session = get_session() mac = utils.to_mac(address) - result = session.query(models.Instance - ).filter_by(mac_address=mac - ).first() + result = session.query(models.Instance).\ + filter_by(mac_address=mac).\ + first() return result @@ -775,6 +775,7 @@ def instance_get_by_id(context, instance_id): if is_admin_context(context): result = session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.floating_ips')).\ filter_by(id=instance_id).\ diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index f71b087b8d..c54ebe3ba5 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -90,8 +90,14 @@ class NovaBase(object): setattr(self, k, v) def iteritems(self): - """Make the model object behave like a dict""" - return iter(self) + """Make the model object behave like a dict. + + Includes attributes from joins.""" + local = dict(self) + joined = dict([(k, v) for k, v in self.__dict__.iteritems() + if not k[0] == '_']) + local.update(joined) + return local.iteritems() class Service(BASE, NovaBase): diff --git a/nova/flags.py b/nova/flags.py index ef66c3f3a6..81e2e36f99 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -254,14 +254,15 @@ DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('ec2_prefix', 'http', 'prefix for ec2') -DEFINE_string('os_prefix', 'http', 'prefix for openstack') -DEFINE_string('cc_host', '$my_ip', 'ip of api server') -DEFINE_string('cc_dmz', '$my_ip', 'internal ip of api server') -DEFINE_integer('cc_port', 8773, 'cloud controller port') +DEFINE_string('ec2_host', '$my_ip', 'ip of api server') +DEFINE_string('ec2_dmz_host', '$my_ip', 'internal ip of api server') +DEFINE_integer('ec2_port', 8773, 'cloud controller port') +DEFINE_string('ec2_scheme', 'http', 'prefix for ec2') +DEFINE_string('ec2_path', '/services/Cloud', 'suffix for ec2') +DEFINE_string('osapi_host', '$my_ip', 'ip of api server') +DEFINE_string('osapi_scheme', 'http', 'prefix for openstack') DEFINE_integer('osapi_port', 8774, 'OpenStack API port') -DEFINE_string('ec2_suffix', '/services/Cloud', 'suffix for ec2') -DEFINE_string('os_suffix', '/v1.0/', 'suffix for openstack') +DEFINE_string('osapi_path', '/v1.0/', 'suffix for openstack') DEFINE_string('default_project', 'openstack', 'default project for openstack') DEFINE_string('default_image', 'ami-11111', diff --git a/nova/image/glance.py b/nova/image/glance.py index a3a2f43081..593c4bce6f 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -14,9 +14,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """Implementation of an image service that uses Glance as the backend""" +from __future__ import absolute_import import httplib import json import urlparse @@ -24,171 +24,40 @@ import urlparse from nova import exception from nova import flags from nova import log as logging +from nova import utils from nova.image import service LOG = logging.getLogger('nova.image.glance') FLAGS = flags.FLAGS -flags.DEFINE_string('glance_teller_address', 'http://127.0.0.1', - 'IP address or URL where Glance\'s Teller service resides') -flags.DEFINE_string('glance_teller_port', '9191', - 'Port for Glance\'s Teller service') -flags.DEFINE_string('glance_parallax_address', 'http://127.0.0.1', - 'IP address or URL where Glance\'s Parallax service ' - 'resides') -flags.DEFINE_string('glance_parallax_port', '9292', - 'Port for Glance\'s Parallax service') - - -class TellerClient(object): - - def __init__(self): - self.address = FLAGS.glance_teller_address - self.port = FLAGS.glance_teller_port - url = urlparse.urlparse(self.address) - self.netloc = url.netloc - self.connection_type = {'http': httplib.HTTPConnection, - 'https': httplib.HTTPSConnection}[url.scheme] - -class ParallaxClient(object): - - def __init__(self): - self.address = FLAGS.glance_parallax_address - self.port = FLAGS.glance_parallax_port - url = urlparse.urlparse(self.address) - self.netloc = url.netloc - self.connection_type = {'http': httplib.HTTPConnection, - 'https': httplib.HTTPSConnection}[url.scheme] - - def get_image_index(self): - """ - Returns a list of image id/name mappings from Parallax - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("GET", "images") - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(images=image_list) - data = json.loads(res.read())['images'] - return data - else: - LOG.warn(_("Parallax returned HTTP error %d from " - "request for /images"), res.status_int) - return [] - finally: - c.close() - - def get_image_details(self): - """ - Returns a list of detailed image data mappings from Parallax - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("GET", "images/detail") - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(images=image_list) - data = json.loads(res.read())['images'] - return data - else: - LOG.warn(_("Parallax returned HTTP error %d from " - "request for /images/detail"), res.status_int) - return [] - finally: - c.close() - - def get_image_metadata(self, image_id): - """ - Returns a mapping of image metadata from Parallax - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("GET", "images/%s" % image_id) - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(image=image_info) - data = json.loads(res.read())['image'] - return data - else: - # TODO(jaypipes): log the error? - return None - finally: - c.close() - - def add_image_metadata(self, image_metadata): - """ - Tells parallax about an image's metadata - """ - try: - c = self.connection_type(self.netloc, self.port) - body = json.dumps(image_metadata) - c.request("POST", "images", body) - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(image=image_info) - data = json.loads(res.read())['image'] - return data['id'] - else: - # TODO(jaypipes): log the error? - return None - finally: - c.close() - - def update_image_metadata(self, image_id, image_metadata): - """ - Updates Parallax's information about an image - """ - try: - c = self.connection_type(self.netloc, self.port) - body = json.dumps(image_metadata) - c.request("PUT", "images/%s" % image_id, body) - res = c.getresponse() - return res.status == 200 - finally: - c.close() - - def delete_image_metadata(self, image_id): - """ - Deletes Parallax's information about an image - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("DELETE", "images/%s" % image_id) - res = c.getresponse() - return res.status == 200 - finally: - c.close() +GlanceClient = utils.import_class('glance.client.Client') class GlanceImageService(service.BaseImageService): """Provides storage and retrieval of disk image objects within Glance.""" def __init__(self): - self.teller = TellerClient() - self.parallax = ParallaxClient() + self.client = GlanceClient(FLAGS.glance_host, FLAGS.glance_port) def index(self, context): """ - Calls out to Parallax for a list of images available + Calls out to Glance for a list of images available """ - images = self.parallax.get_image_index() - return images + return self.client.get_images() def detail(self, context): """ - Calls out to Parallax for a list of detailed image information + Calls out to Glance for a list of detailed image information """ - images = self.parallax.get_image_details() - return images + return self.client.get_images_detailed() def show(self, context, id): """ Returns a dict containing image data for the given opaque image id. """ - image = self.parallax.get_image_metadata(id) + image = self.client.get_image_meta(id) if image: return image raise exception.NotFound @@ -200,7 +69,7 @@ class GlanceImageService(service.BaseImageService): :raises AlreadyExists if the image already exist. """ - return self.parallax.add_image_metadata(data) + return self.client.add_image(image_meta=data) def update(self, context, image_id, data): """Replace the contents of the given image with the new data. @@ -208,7 +77,7 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - self.parallax.update_image_metadata(image_id, data) + return self.client.update_image(image_id, data) def delete(self, context, image_id): """ @@ -217,7 +86,7 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - self.parallax.delete_image_metadata(image_id) + return self.client.delete_image(image_id) def delete_all(self): """ diff --git a/nova/log.py b/nova/log.py index c1428c051d..4997d3f283 100644 --- a/nova/log.py +++ b/nova/log.py @@ -116,6 +116,8 @@ def basicConfig(): handler.setFormatter(_formatter) if FLAGS.verbose: logging.root.setLevel(logging.DEBUG) + else: + logging.root.setLevel(logging.INFO) if FLAGS.use_syslog: syslog = SysLogHandler(address='/dev/log') syslog.setFormatter(_formatter) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 891e9bc1d7..d29e17603c 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -61,7 +61,7 @@ def metadata_forward(): """Create forwarding rule for metadata""" _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.cc_dmz, FLAGS.cc_port)) + "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): diff --git a/nova/network/manager.py b/nova/network/manager.py index a92bd3f997..61de8055ab 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -92,7 +92,7 @@ flags.DEFINE_bool('update_dhcp_on_disassociate', False, flags.DEFINE_integer('fixed_ip_disassociate_timeout', 600, 'Seconds after which a deallocated ip is disassociated') -flags.DEFINE_bool('use_ipv6', True, +flags.DEFINE_bool('use_ipv6', False, 'use the ipv6') flags.DEFINE_string('network_host', socket.gethostname(), 'Network host to use for ip allocation in flat modes') @@ -517,10 +517,8 @@ class VlanManager(NetworkManager): net['vlan'] = vlan net['bridge'] = 'br%s' % vlan if(FLAGS.use_ipv6): - cidr_v6 = "%s/%s" % ( - fixed_net_v6[start_v6], - significant_bits_v6 - ) + cidr_v6 = "%s/%s" % (fixed_net_v6[start_v6], + significant_bits_v6) net['cidr_v6'] = cidr_v6 # NOTE(vish): This makes ports unique accross the cloud, a more diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index 9e183bd0d4..14eaaa62c7 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -15,23 +15,28 @@ # License for the specific language governing permissions and limitations # under the License. +import webob.dec import unittest from nova import context from nova import flags from nova.api.openstack.ratelimiting import RateLimitingMiddleware from nova.api.openstack.common import limited -from nova.tests.api.fakes import APIStub -from nova import utils +from nova.tests.api.openstack import fakes from webob import Request FLAGS = flags.FLAGS +@webob.dec.wsgify +def simple_wsgi(req): + return "" + + class RateLimitingMiddlewareTest(unittest.TestCase): def test_get_action_name(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) def verify(method, url, action_name): req = Request.blank(url) @@ -61,19 +66,19 @@ class RateLimitingMiddlewareTest(unittest.TestCase): self.assertTrue('Retry-After' in resp.headers) def test_single_action(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) self.exhaust(middleware, 'DELETE', '/servers/4', 'usr1', 100) self.exhaust(middleware, 'DELETE', '/servers/4', 'usr2', 100) def test_POST_servers_action_implies_POST_action(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) self.exhaust(middleware, 'POST', '/images/4', 'usr2', 10) self.assertTrue(set(middleware.limiter._levels) == \ set(['usr1:POST', 'usr1:POST servers', 'usr2:POST'])) def test_POST_servers_action_correctly_ratelimited(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) # Use up all of our "POST" allowance for the minute, 5 times for i in range(5): self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) @@ -83,9 +88,9 @@ class RateLimitingMiddlewareTest(unittest.TestCase): self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 0) def test_proxy_ctor_works(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) self.assertEqual(middleware.limiter.__class__.__name__, "Limiter") - middleware = RateLimitingMiddleware(APIStub(), service_host='foobar') + middleware = RateLimitingMiddleware(simple_wsgi, service_host='foobar') self.assertEqual(middleware.limiter.__class__.__name__, "WSGIAppProxy") diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 194304e790..fb282f1c97 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -22,6 +22,9 @@ import string import webob import webob.dec +from paste import urlmap + +from glance import client as glance_client from nova import auth from nova import context @@ -29,6 +32,7 @@ from nova import exception as exc from nova import flags from nova import utils import nova.api.openstack.auth +from nova.api import openstack from nova.api.openstack import auth from nova.api.openstack import ratelimiting from nova.image import glance @@ -69,6 +73,17 @@ def fake_wsgi(self, req): return self.application +def wsgi_app(inner_application=None): + if not inner_application: + inner_application = openstack.APIRouter() + mapper = urlmap.URLMap() + api = openstack.FaultWrapper(auth.AuthMiddleware( + ratelimiting.RateLimitingMiddleware(inner_application))) + mapper['/v1.0'] = api + mapper['/'] = openstack.FaultWrapper(openstack.Versions()) + return mapper + + def stub_out_key_pair_funcs(stubs): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] @@ -116,64 +131,60 @@ def stub_out_compute_api_snapshot(stubs): stubs.Set(nova.compute.API, 'snapshot', snapshot) -def stub_out_glance(stubs, initial_fixtures=[]): +def stub_out_glance(stubs, initial_fixtures=None): - class FakeParallaxClient: + class FakeGlanceClient: def __init__(self, initial_fixtures): - self.fixtures = initial_fixtures + self.fixtures = initial_fixtures or [] - def fake_get_image_index(self): + def fake_get_images(self): return [dict(id=f['id'], name=f['name']) for f in self.fixtures] - def fake_get_image_details(self): + def fake_get_images_detailed(self): return self.fixtures - def fake_get_image_metadata(self, image_id): + def fake_get_image_meta(self, image_id): for f in self.fixtures: if f['id'] == image_id: return f return None - def fake_add_image_metadata(self, image_data): + def fake_add_image(self, image_meta): id = ''.join(random.choice(string.letters) for _ in range(20)) - image_data['id'] = id - self.fixtures.append(image_data) + image_meta['id'] = id + self.fixtures.append(image_meta) return id - def fake_update_image_metadata(self, image_id, image_data): - f = self.fake_get_image_metadata(image_id) + def fake_update_image(self, image_id, image_meta): + f = self.fake_get_image_meta(image_id) if not f: raise exc.NotFound - f.update(image_data) + f.update(image_meta) - def fake_delete_image_metadata(self, image_id): - f = self.fake_get_image_metadata(image_id) + def fake_delete_image(self, image_id): + f = self.fake_get_image_meta(image_id) if not f: raise exc.NotFound self.fixtures.remove(f) - def fake_delete_all(self): - self.fixtures = [] - - fake_parallax_client = FakeParallaxClient(initial_fixtures) - stubs.Set(nova.image.glance.ParallaxClient, 'get_image_index', - fake_parallax_client.fake_get_image_index) - stubs.Set(nova.image.glance.ParallaxClient, 'get_image_details', - fake_parallax_client.fake_get_image_details) - stubs.Set(nova.image.glance.ParallaxClient, 'get_image_metadata', - fake_parallax_client.fake_get_image_metadata) - stubs.Set(nova.image.glance.ParallaxClient, 'add_image_metadata', - fake_parallax_client.fake_add_image_metadata) - stubs.Set(nova.image.glance.ParallaxClient, 'update_image_metadata', - fake_parallax_client.fake_update_image_metadata) - stubs.Set(nova.image.glance.ParallaxClient, 'delete_image_metadata', - fake_parallax_client.fake_delete_image_metadata) - stubs.Set(nova.image.glance.GlanceImageService, 'delete_all', - fake_parallax_client.fake_delete_all) + ##def fake_delete_all(self): + ## self.fixtures = [] + + GlanceClient = glance_client.Client + fake = FakeGlanceClient(initial_fixtures) + + stubs.Set(GlanceClient, 'get_images', fake.fake_get_images) + stubs.Set(GlanceClient, 'get_images_detailed', + fake.fake_get_images_detailed) + stubs.Set(GlanceClient, 'get_image_meta', fake.fake_get_image_meta) + stubs.Set(GlanceClient, 'add_image', fake.fake_add_image) + stubs.Set(GlanceClient, 'update_image', fake.fake_update_image) + stubs.Set(GlanceClient, 'delete_image', fake.fake_delete_image) + #stubs.Set(GlanceClient, 'delete_all', fake.fake_delete_all) class FakeToken(object): diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 1b2e1654d5..73120c31dd 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -19,15 +19,19 @@ import unittest import stubout import webob +from paste import urlmap -import nova.api from nova import flags +from nova.api import openstack +from nova.api.openstack import ratelimiting +from nova.api.openstack import auth from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS class AdminAPITest(unittest.TestCase): + def setUp(self): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} @@ -45,7 +49,7 @@ class AdminAPITest(unittest.TestCase): FLAGS.allow_admin_api = True # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are available. @@ -53,7 +57,7 @@ class AdminAPITest(unittest.TestCase): FLAGS.allow_admin_api = False # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are unavailable. diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index d8b202e212..db0fe1060b 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -19,14 +19,18 @@ import unittest import webob.exc import webob.dec -import nova.api.openstack -from nova.api.openstack import API -from nova.api.openstack import faults from webob import Request +from nova.api import openstack +from nova.api.openstack import faults + class APITest(unittest.TestCase): + def _wsgi_app(self, inner_app): + # simpler version of the app than fakes.wsgi_app + return openstack.FaultWrapper(inner_app) + def test_exceptions_are_converted_to_faults(self): @webob.dec.wsgify @@ -46,29 +50,32 @@ class APITest(unittest.TestCase): exc = webob.exc.HTTPNotFound(explanation='Raised a webob.exc') return faults.Fault(exc) - api = API() - - api.application = succeed + #api.application = succeed + api = self._wsgi_app(succeed) resp = Request.blank('/').get_response(api) self.assertFalse('computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 200, resp.body) - api.application = raise_webob_exc + #api.application = raise_webob_exc + api = self._wsgi_app(raise_webob_exc) resp = Request.blank('/').get_response(api) self.assertFalse('computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 404, resp.body) - api.application = raise_api_fault + #api.application = raise_api_fault + api = self._wsgi_app(raise_api_fault) resp = Request.blank('/').get_response(api) self.assertTrue('itemNotFound' in resp.body, resp.body) self.assertEqual(resp.status_int, 404, resp.body) - api.application = fail + #api.application = fail + api = self._wsgi_app(fail) resp = Request.blank('/').get_response(api) self.assertTrue('{"computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 500, resp.body) - api.application = fail + #api.application = fail + api = self._wsgi_app(fail) resp = Request.blank('/.xml').get_response(api) self.assertTrue('<computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 500, resp.body) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 489a1dfbfd..0dd65d321c 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -53,7 +53,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-CDN-Management-Url'], @@ -67,7 +67,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], @@ -81,7 +81,7 @@ class Test(unittest.TestCase): fakes.FakeRouter) req = webob.Request.blank('/v1.0/fake') req.headers['X-Auth-Token'] = token - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') @@ -105,7 +105,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-Token'] = 'bacon' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) @@ -113,18 +113,18 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_no_user(self): req = webob.Request.blank('/v1.0/') - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_bad_token(self): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-Token'] = 'baconbaconbacon' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -149,7 +149,7 @@ class TestLimiter(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(len(result.headers['X-Auth-Token']), 40) token = result.headers['X-Auth-Token'] @@ -158,7 +158,7 @@ class TestLimiter(unittest.TestCase): req = webob.Request.blank('/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 41018afdf1..1bdaea161a 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -39,7 +39,7 @@ class FlavorsTest(unittest.TestCase): def test_get_flavor_list(self): req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_get_flavor_by_id(self): pass diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 00ca739a57..5d9ddefbea 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -210,7 +210,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def test_get_image_index(self): req = webob.Request.blank('/v1.0/images') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) fixture_index = [dict(id=f['id'], name=f['name']) for f @@ -222,7 +222,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def test_get_image_details(self): req = webob.Request.blank('/v1.0/images/detail') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) def _is_equivalent_subset(x, y): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 0396daf986..29883e7c89 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -100,14 +100,14 @@ class ServersTest(unittest.TestCase): def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') self.assertEqual(res_dict['server']['name'], 'server1') def test_get_server_list(self): req = webob.Request.blank('/v1.0/servers') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) i = 0 @@ -160,14 +160,14 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 422) def test_update_bad_params(self): @@ -186,7 +186,7 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body - req.get_response(nova.api.API('os')) + req.get_response(fakes.wsgi_app()) def test_update_server(self): inst_dict = dict(name='server_test', adminPass='bacon') @@ -202,28 +202,28 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body - req.get_response(nova.api.API('os')) + req.get_response(fakes.wsgi_app()) def test_create_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'POST' - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'DELETE' - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): req = webob.Request.blank('/v1.0/servers/detail') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) i = 0 @@ -242,7 +242,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_unpause(self): @@ -254,7 +254,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_suspend(self): @@ -266,7 +266,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_resume(self): @@ -278,19 +278,19 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_diagnostics(self): req = webob.Request.blank("/v1.0/servers/1/diagnostics") req.method = "GET" - res = req.get_response(nova.api.API("os")) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_actions(self): req = webob.Request.blank("/v1.0/servers/1/actions") req.method = "GET" - res = req.get_response(nova.api.API("os")) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_reboot(self): @@ -301,7 +301,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_server_rebuild(self): body = dict(server=dict( @@ -311,7 +311,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_server_resize(self): body = dict(server=dict( @@ -321,7 +321,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_delete_server_instance(self): req = webob.Request.blank('/v1.0/servers/1') @@ -335,7 +335,7 @@ class ServersTest(unittest.TestCase): self.stubs.Set(nova.db.api, 'instance_destroy', instance_destroy_mock) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) diff --git a/nova/tests/api/test.py b/nova/tests/api/test.py deleted file mode 100644 index 9caa8c9d0d..0000000000 --- a/nova/tests/api/test.py +++ /dev/null @@ -1,81 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Test for the root WSGI middleware for all API controllers. -""" - -import unittest - -import stubout -import webob -import webob.dec - -import nova.exception -from nova import api -from nova.tests.api.fakes import APIStub - - -class Test(unittest.TestCase): - - def setUp(self): - self.stubs = stubout.StubOutForTesting() - - def tearDown(self): - self.stubs.UnsetAll() - - def _request(self, url, subdomain, **kwargs): - environ_keys = {'HTTP_HOST': '%s.example.com' % subdomain} - environ_keys.update(kwargs) - req = webob.Request.blank(url, environ_keys) - return req.get_response(api.API('ec2')) - - def test_openstack(self): - self.stubs.Set(api.openstack, 'API', APIStub) - result = self._request('/v1.0/cloud', 'api') - self.assertEqual(result.body, "/cloud") - - def test_ec2(self): - self.stubs.Set(api.ec2, 'API', APIStub) - result = self._request('/services/cloud', 'ec2') - self.assertEqual(result.body, "/cloud") - - def test_not_found(self): - self.stubs.Set(api.ec2, 'API', APIStub) - self.stubs.Set(api.openstack, 'API', APIStub) - result = self._request('/test/cloud', 'ec2') - self.assertNotEqual(result.body, "/cloud") - - def test_query_api_versions(self): - result = self._request('/', 'api') - self.assertTrue('CURRENT' in result.body) - - def test_metadata(self): - def go(url): - result = self._request(url, 'ec2', REMOTE_ADDR='128.192.151.2') - # Each should get to the ORM layer and fail to find the IP - self.assertRaises(nova.exception.NotFound, go, '/latest/') - self.assertRaises(nova.exception.NotFound, go, '/2009-04-04/') - self.assertRaises(nova.exception.NotFound, go, '/1.0/') - - def test_ec2_root(self): - result = self._request('/', 'ec2') - self.assertTrue('2007-12-15\n' in result.body) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/fakes.py b/nova/tests/glance/__init__.py index 0aedcaff0b..ef9fa05a7c 100644 --- a/nova/tests/api/fakes.py +++ b/nova/tests/glance/__init__.py @@ -1,7 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. +# Copyright (c) 2011 Citrix Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,12 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import webob.dec -from nova import wsgi - - -class APIStub(object): - """Class to verify request and mark it was called.""" - @webob.dec.wsgify - def __call__(self, req): - return req.path_info +""" +:mod:`glance` -- Stubs for Glance +================================= +""" diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py new file mode 100644 index 0000000000..f182b857a5 --- /dev/null +++ b/nova/tests/glance/stubs.py @@ -0,0 +1,37 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import StringIO + +import glance.client + + +def stubout_glance_client(stubs, cls): + """Stubs out glance.client.Client""" + stubs.Set(glance.client, 'Client', + lambda *args, **kwargs: cls(*args, **kwargs)) + + +class FakeGlance(object): + def __init__(self, host, port=None, use_ssl=False): + pass + + def get_image(self, image): + meta = { + 'size': 0, + } + image_file = StringIO.StringIO('') + return meta, image_file diff --git a/nova/tests/test_access.py b/nova/tests/test_access.py index 0929903cf0..e170ccee62 100644 --- a/nova/tests/test_access.py +++ b/nova/tests/test_access.py @@ -20,21 +20,31 @@ import unittest import webob from nova import context -from nova import exception from nova import flags from nova import test from nova.api import ec2 from nova.auth import manager - FLAGS = flags.FLAGS -class Context(object): +class FakeControllerClass(object): pass +class FakeApiRequest(object): + def __init__(self, action): + self.controller = FakeControllerClass() + self.action = action + + class AccessTestCase(test.TestCase): + def _env_for(self, ctxt, action): + env = {} + env['ec2.context'] = ctxt + env['ec2.request'] = FakeApiRequest(action) + return env + def setUp(self): super(AccessTestCase, self).setUp() um = manager.AuthManager() @@ -64,7 +74,7 @@ class AccessTestCase(test.TestCase): return [''] self.mw = ec2.Authorizer(noopWSGIApp) - self.mw.action_roles = {'str': { + self.mw.action_roles = {'FakeControllerClass': { '_allow_all': ['all'], '_allow_none': [], '_allow_project_manager': ['projectmanager'], @@ -84,9 +94,7 @@ class AccessTestCase(test.TestCase): def response_status(self, user, methodName): ctxt = context.RequestContext(user, self.project) - environ = {'ec2.context': ctxt, - 'ec2.controller': 'some string', - 'ec2.action': methodName} + environ = self._env_for(ctxt, methodName) req = webob.Request.blank('/', environ) resp = req.get_response(self.mw) return resp.status_int diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 17789c25c7..66a16b0cb7 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -26,9 +26,8 @@ import StringIO import webob from nova import context -from nova import flags from nova import test -from nova import api +from nova.api import ec2 from nova.api.ec2 import cloud from nova.api.ec2 import apirequest from nova.auth import manager @@ -100,12 +99,10 @@ class ApiEc2TestCase(test.TestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() - self.manager = manager.AuthManager() - self.host = '127.0.0.1' - - self.app = api.API('ec2') + self.app = ec2.Authenticate(ec2.Requestify(ec2.Executor(), + 'nova.api.ec2.cloud.CloudController')) def expect_http(self, host=None, is_secure=False): """Returns a new EC2 connection""" diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 2e350cd5ad..771b1fcc07 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -21,6 +21,7 @@ import json from M2Crypto import BIO from M2Crypto import RSA import os +import shutil import tempfile import time @@ -50,6 +51,8 @@ IMAGES_PATH = os.path.join(OSS_TEMPDIR, 'images') os.makedirs(IMAGES_PATH) +# TODO(termie): these tests are rather fragile, they should at the lest be +# wiping database state after each run class CloudTestCase(test.TestCase): def setUp(self): super(CloudTestCase, self).setUp() @@ -287,6 +290,7 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, comp1['id']) def test_instance_update_state(self): + # TODO(termie): what is this code even testing? def instance(num): return { 'reservation_id': 'r-1', @@ -305,7 +309,8 @@ class CloudTestCase(test.TestCase): 'state': 0x01, 'user_data': ''} rv = self.cloud._format_describe_instances(self.context) - self.assert_(len(rv['reservationSet']) == 0) + logging.error(str(rv)) + self.assertEqual(len(rv['reservationSet']), 0) # simulate launch of 5 instances # self.cloud.instances['pending'] = {} @@ -368,6 +373,7 @@ class CloudTestCase(test.TestCase): self.assertEqual('Foo Img', img.metadata['description']) self._fake_set_image_description(self.context, 'ami-testing', '') self.assertEqual('', img.metadata['description']) + shutil.rmtree(pathdir) def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index a7d47961cc..09f6ee94a0 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -32,8 +32,9 @@ from nova import utils from nova.auth import manager -FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.compute') +FLAGS = flags.FLAGS +flags.DECLARE('stub_network', 'nova.compute.manager') class ComputeTestCase(test.TestCase): @@ -75,7 +76,7 @@ class ComputeTestCase(test.TestCase): ref = self.compute_api.create(self.context, FLAGS.default_instance_type, None, **instance) try: - self.assertNotEqual(ref[0].display_name, None) + self.assertNotEqual(ref[0]['display_name'], None) finally: db.instance_destroy(self.context, ref[0]['id']) @@ -86,10 +87,14 @@ class ComputeTestCase(test.TestCase): 'user_id': self.user.id, 'project_id': self.project.id} group = db.security_group_create(self.context, values) - ref = self.compute_api.create(self.context, - FLAGS.default_instance_type, None, security_group=['default']) + ref = self.compute_api.create( + self.context, + instance_type=FLAGS.default_instance_type, + image_id=None, + security_group=['default']) try: - self.assertEqual(len(ref[0]['security_groups']), 1) + self.assertEqual(len(db.security_group_get_by_instance( + self.context, ref[0]['id'])), 1) finally: db.security_group_destroy(self.context, group['id']) db.instance_destroy(self.context, ref[0]['id']) diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 31b5ca79c4..85bf944582 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -111,12 +111,14 @@ class ConsoleTestCase(test.TestCase): console_instances = [con['instance_id'] for con in pool.consoles] self.assert_(instance_id in console_instances) + db.instance_destroy(self.context, instance_id) def test_add_console_does_not_duplicate(self): instance_id = self._create_instance() cons1 = self.console.add_console(self.context, instance_id) cons2 = self.console.add_console(self.context, instance_id) self.assertEqual(cons1, cons2) + db.instance_destroy(self.context, instance_id) def test_remove_console(self): instance_id = self._create_instance() @@ -127,3 +129,4 @@ class ConsoleTestCase(test.TestCase): db.console_get, self.context, console_id) + db.instance_destroy(self.context, instance_id) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py new file mode 100644 index 0000000000..8a74b22967 --- /dev/null +++ b/nova/tests/test_direct.py @@ -0,0 +1,103 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for Direct API.""" + +import json +import logging + +import webob + +from nova import compute +from nova import context +from nova import exception +from nova import test +from nova import utils +from nova.api import direct +from nova.tests import test_cloud + + +class FakeService(object): + def echo(self, context, data): + return {'data': data} + + def context(self, context): + return {'user': context.user_id, + 'project': context.project_id} + + +class DirectTestCase(test.TestCase): + def setUp(self): + super(DirectTestCase, self).setUp() + direct.register_service('fake', FakeService()) + self.router = direct.PostParamsMiddleware( + direct.JsonParamsMiddleware( + direct.Router())) + self.auth_router = direct.DelegatedAuthMiddleware(self.router) + self.context = context.RequestContext('user1', 'proj1') + + def tearDown(self): + direct.ROUTES = {} + + def test_delegated_auth(self): + req = webob.Request.blank('/fake/context') + req.headers['X-OpenStack-User'] = 'user1' + req.headers['X-OpenStack-Project'] = 'proj1' + resp = req.get_response(self.auth_router) + data = json.loads(resp.body) + self.assertEqual(data['user'], 'user1') + self.assertEqual(data['project'], 'proj1') + + def test_json_params(self): + req = webob.Request.blank('/fake/echo') + req.environ['openstack.context'] = self.context + req.method = 'POST' + req.body = 'json=%s' % json.dumps({'data': 'foo'}) + resp = req.get_response(self.router) + resp_parsed = json.loads(resp.body) + self.assertEqual(resp_parsed['data'], 'foo') + + def test_post_params(self): + req = webob.Request.blank('/fake/echo') + req.environ['openstack.context'] = self.context + req.method = 'POST' + req.body = 'data=foo' + resp = req.get_response(self.router) + resp_parsed = json.loads(resp.body) + self.assertEqual(resp_parsed['data'], 'foo') + + def test_proxy(self): + proxy = direct.Proxy(self.router) + rv = proxy.fake.echo(self.context, data='baz') + self.assertEqual(rv['data'], 'baz') + + +class DirectCloudTestCase(test_cloud.CloudTestCase): + def setUp(self): + super(DirectCloudTestCase, self).setUp() + compute_handle = compute.API(image_service=self.cloud.image_service, + network_api=self.cloud.network_api, + volume_api=self.cloud.volume_api) + direct.register_service('compute', compute_handle) + self.router = direct.JsonParamsMiddleware(direct.Router()) + proxy = direct.Proxy(self.router) + self.cloud.compute_api = proxy.compute + + def tearDown(self): + super(DirectCloudTestCase, self).tearDown() + direct.ROUTES = {} diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index afdc89ba2a..556fe561c5 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -122,10 +122,10 @@ class LibvirtConnTestCase(test.TestCase): if rescue: check = (lambda t: t.find('./os/kernel').text.split('/')[1], - 'rescue-kernel') + 'kernel.rescue') check_list.append(check) check = (lambda t: t.find('./os/initrd').text.split('/')[1], - 'rescue-ramdisk') + 'ramdisk.rescue') check_list.append(check) else: if expect_kernel: @@ -161,13 +161,16 @@ class LibvirtConnTestCase(test.TestCase): if rescue: common_checks += [ (lambda t: t.findall('./devices/disk/source')[0].get( - 'file').split('/')[1], 'rescue-disk'), + 'file').split('/')[1], 'disk.rescue'), (lambda t: t.findall('./devices/disk/source')[1].get( 'file').split('/')[1], 'disk')] else: common_checks += [(lambda t: t.findall( './devices/disk/source')[0].get('file').split('/')[1], 'disk')] + common_checks += [(lambda t: t.findall( + './devices/disk/source')[1].get('file').split('/')[1], + 'disk.local')] for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 261ee0fde9..9f5b266f33 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -34,6 +34,7 @@ from nova.virt.xenapi import volume_utils from nova.virt.xenapi.vmops import SimpleDH from nova.tests.db import fakes as db_fakes from nova.tests.xenapi import stubs +from nova.tests.glance import stubs as glance_stubs FLAGS = flags.FLAGS @@ -108,18 +109,16 @@ class XenAPIVolumeTestCase(test.TestCase): conn = xenapi_conn.get_connection(False) volume = self._create_volume() instance = db.instance_create(self.values) - xenapi_fake.create_vm(instance.name, 'Running') + vm = xenapi_fake.create_vm(instance.name, 'Running') result = conn.attach_volume(instance.name, volume['id'], '/dev/sdc') def check(): # check that the VM has a VBD attached to it - # Get XenAPI reference for the VM - vms = xenapi_fake.get_all('VM') # Get XenAPI record for VBD vbds = xenapi_fake.get_all('VBD') vbd = xenapi_fake.get_record('VBD', vbds[0]) vm_ref = vbd['VM'] - self.assertEqual(vm_ref, vms[0]) + self.assertEqual(vm_ref, vm) check() @@ -157,9 +156,14 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' xenapi_fake.reset() + xenapi_fake.create_local_srs() db_fakes.stub_out_db_instance_api(self.stubs) xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + stubs.stubout_get_this_vm_uuid(self.stubs) + stubs.stubout_stream_disk(self.stubs) + glance_stubs.stubout_glance_client(self.stubs, + glance_stubs.FakeGlance) self.conn = xenapi_conn.get_connection(False) def test_list_instances_0(self): @@ -207,40 +211,70 @@ class XenAPIVMTestCase(test.TestCase): check() - def test_spawn(self): - instance = self._create_instance() + def check_vm_record(self, conn): + instances = conn.list_instances() + self.assertEquals(instances, [1]) + + # Get Nova record for VM + vm_info = conn.get_info(1) + + # Get XenAPI record for VM + vms = [rec for ref, rec + in xenapi_fake.get_all_records('VM').iteritems() + if not rec['is_control_domain']] + vm = vms[0] + + # Check that m1.large above turned into the right thing. + instance_type = instance_types.INSTANCE_TYPES['m1.large'] + mem_kib = long(instance_type['memory_mb']) << 10 + mem_bytes = str(mem_kib << 10) + vcpus = instance_type['vcpus'] + self.assertEquals(vm_info['max_mem'], mem_kib) + self.assertEquals(vm_info['mem'], mem_kib) + self.assertEquals(vm['memory_static_max'], mem_bytes) + self.assertEquals(vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(vm['VCPUs_max'], str(vcpus)) + self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + + # Check that the VM is running according to Nova + self.assertEquals(vm_info['state'], power_state.RUNNING) + + # Check that the VM is running according to XenAPI. + self.assertEquals(vm['power_state'], 'Running') + + def _test_spawn(self, image_id, kernel_id, ramdisk_id): + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + values = {'name': 1, + 'id': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': image_id, + 'kernel_id': kernel_id, + 'ramdisk_id': ramdisk_id, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } + conn = xenapi_conn.get_connection(False) + instance = db.instance_create(values) + conn.spawn(instance) + self.check_vm_record(conn) - def check(): - instances = self.conn.list_instances() - self.assertEquals(instances, [1]) - - # Get Nova record for VM - vm_info = self.conn.get_info(1) - - # Get XenAPI record for VM - vms = xenapi_fake.get_all('VM') - vm = xenapi_fake.get_record('VM', vms[0]) - - # Check that m1.large above turned into the right thing. - instance_type = instance_types.INSTANCE_TYPES['m1.large'] - mem_kib = long(instance_type['memory_mb']) << 10 - mem_bytes = str(mem_kib << 10) - vcpus = instance_type['vcpus'] - self.assertEquals(vm_info['max_mem'], mem_kib) - self.assertEquals(vm_info['mem'], mem_kib) - self.assertEquals(vm['memory_static_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_min'], mem_bytes) - self.assertEquals(vm['VCPUs_max'], str(vcpus)) - self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) - - # Check that the VM is running according to Nova - self.assertEquals(vm_info['state'], power_state.RUNNING) - - # Check that the VM is running according to XenAPI. - self.assertEquals(vm['power_state'], 'Running') + def test_spawn_raw_objectstore(self): + FLAGS.xenapi_image_service = 'objectstore' + self._test_spawn(1, None, None) - check() + def test_spawn_objectstore(self): + FLAGS.xenapi_image_service = 'objectstore' + self._test_spawn(1, 2, 3) + + def test_spawn_raw_glance(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(1, None, None) + + def test_spawn_glance(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(1, 2, 3) def tearDown(self): super(XenAPIVMTestCase, self).tearDown() diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 292bd9ba9a..624995ada1 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -115,6 +115,21 @@ def stub_out_get_target(stubs): stubs.Set(volume_utils, '_get_target', fake_get_target) +def stubout_get_this_vm_uuid(stubs): + def f(): + vms = [rec['uuid'] for ref, rec + in fake.get_all_records('VM').iteritems() + if rec['is_control_domain']] + return vms[0] + stubs.Set(vm_utils, 'get_this_vm_uuid', f) + + +def stubout_stream_disk(stubs): + def f(_1, _2, _3, _4): + pass + stubs.Set(vm_utils, '_stream_disk', f) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri): @@ -124,7 +139,10 @@ class FakeSessionForVMTests(fake.SessionBase): return self.xenapi.network.get_all_records() def host_call_plugin(self, _1, _2, _3, _4, _5): - return '' + sr_ref = fake.get_all('SR')[0] + vdi_ref = fake.create_vdi('', False, sr_ref, False) + vdi_rec = fake.get_record('VDI', vdi_ref) + return '<string>%s</string>' % vdi_rec['uuid'] def VM_start(self, _1, ref, _2, _3): vm = fake.get_record('VM', ref) @@ -159,10 +177,6 @@ class FakeSessionForVolumeTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForVolumeTests, self).__init__(uri) - def VBD_plug(self, _1, ref): - rec = fake.get_record('VBD', ref) - rec['currently-attached'] = True - def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11): valid_vdi = False diff --git a/nova/utils.py b/nova/utils.py index 27589c30ce..6d3ddd092a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -22,6 +22,7 @@ System-level utilities and helper functions. import datetime import inspect +import json import os import random import subprocess @@ -333,6 +334,20 @@ class LazyPluggable(object): return getattr(backend, key) +class LoopingCallDone(Exception): + """The poll-function passed to LoopingCall can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCall.wait() + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCall.wait() should return""" + self.retvalue = retvalue + + class LoopingCall(object): def __init__(self, f=None, *args, **kw): self.args = args @@ -351,12 +366,15 @@ class LoopingCall(object): while self._running: self.f(*self.args, **self.kw) greenthread.sleep(interval) + except LoopingCallDone, e: + self.stop() + done.send(e.retvalue) except Exception: logging.exception('in looping call') done.send_exception(*sys.exc_info()) return - - done.send(True) + else: + done.send(True) self.done = done @@ -391,3 +409,36 @@ def utf8(value): return value.encode("utf-8") assert isinstance(value, str) return value + + +def to_primitive(value): + if type(value) is type([]) or type(value) is type((None,)): + o = [] + for v in value: + o.append(to_primitive(v)) + return o + elif type(value) is type({}): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v) + return o + elif isinstance(value, datetime.datetime): + return str(value) + elif hasattr(value, 'iteritems'): + return to_primitive(dict(value.iteritems())) + elif hasattr(value, '__iter__'): + return to_primitive(list(value)) + else: + return value + + +def dumps(value): + try: + return json.dumps(value) + except TypeError: + pass + return json.dumps(to_primitive(value)) + + +def loads(s): + return json.loads(s) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index a57a8f43b3..f8b3c78071 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -310,6 +310,54 @@ class FakeConnection(object): 'username': 'fakeuser', 'password': 'fakepassword'} + def refresh_security_group_rules(self, security_group_id): + """This method is called after a change to security groups. + + All security groups and their associated rules live in the datastore, + and calling this method should apply the updated rules to instances + running the specified security group. + + An error should be raised if the operation cannot complete. + + """ + return True + + def refresh_security_group_members(self, security_group_id): + """This method is called when a security group is added to an instance. + + This message is sent to the virtualization drivers on hosts that are + running an instance that belongs to a security group that has a rule + that references the security group identified by `security_group_id`. + It is the responsiblity of this method to make sure any rules + that authorize traffic flow with members of the security group are + updated and any new members can communicate, and any removed members + cannot. + + Scenario: + * we are running on host 'H0' and we have an instance 'i-0'. + * instance 'i-0' is a member of security group 'speaks-b' + * group 'speaks-b' has an ingress rule that authorizes group 'b' + * another host 'H1' runs an instance 'i-1' + * instance 'i-1' is a member of security group 'b' + + When 'i-1' launches or terminates we will recieve the message + to update members of group 'b', at which time we will make + any changes needed to the rules for instance 'i-0' to allow + or deny traffic coming from 'i-1', depending on if it is being + added or removed from the group. + + In this scenario, 'i-1' could just as easily have been running on our + host 'H0' and this method would still have been called. The point was + that this method isn't called on the host where instances of that + group are running (as is the case with + :method:`refresh_security_group_rules`) but is called where references + are made to authorizing those instances. + + An error should be raised if the operation cannot complete. + + """ + return True + class FakeInstance(object): diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index de06a1eb04..8139c3620b 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -18,10 +18,10 @@ #set $disk_prefix = 'vd' #set $disk_bus = 'virtio' <type>hvm</type> - #end if + #end if #if $getVar('rescue', False) - <kernel>${basepath}/rescue-kernel</kernel> - <initrd>${basepath}/rescue-ramdisk</initrd> + <kernel>${basepath}/kernel.rescue</kernel> + <initrd>${basepath}/ramdisk.rescue</initrd> #else #if $getVar('kernel', None) <kernel>${kernel}</kernel> @@ -47,7 +47,7 @@ #if $getVar('rescue', False) <disk type='file'> <driver type='${driver_type}'/> - <source file='${basepath}/rescue-disk'/> + <source file='${basepath}/disk.rescue'/> <target dev='${disk_prefix}a' bus='${disk_bus}'/> </disk> <disk type='file'> @@ -64,7 +64,7 @@ #if $getVar('local', False) <disk type='file'> <driver type='${driver_type}'/> - <source file='${basepath}/local'/> + <source file='${basepath}/disk.local'/> <target dev='${disk_prefix}b' bus='${disk_bus}'/> </disk> #end if diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 073a8e5bb3..f5b0bd365e 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -295,7 +295,7 @@ class LibvirtConnection(object): virt_dom.detachDevice(xml) @exception.wrap_exception - def snapshot(self, instance, name): + def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ raise NotImplementedError( _("Instance snapshotting is not supported for libvirt" @@ -350,7 +350,7 @@ class LibvirtConnection(object): rescue_images = {'image_id': FLAGS.rescue_image_id, 'kernel_id': FLAGS.rescue_kernel_id, 'ramdisk_id': FLAGS.rescue_ramdisk_id} - self._create_image(instance, xml, 'rescue-', rescue_images) + self._create_image(instance, xml, '.rescue', rescue_images) self._conn.createXML(xml, 0) timer = utils.LoopingCall(f=None) @@ -532,23 +532,23 @@ class LibvirtConnection(object): utils.execute('truncate %s -s %dG' % (target, local_gb)) # TODO(vish): should we format disk by default? - def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): + def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): # syntactic nicety - def basepath(fname='', prefix=prefix): + def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, inst['name'], - prefix + fname) + fname + suffix) # ensure directories exist and are writable - utils.execute('mkdir -p %s' % basepath(prefix='')) - utils.execute('chmod 0777 %s' % basepath(prefix='')) + utils.execute('mkdir -p %s' % basepath(suffix='')) + utils.execute('chmod 0777 %s' % basepath(suffix='')) LOG.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') f.write(libvirt_xml) f.close() - # NOTE(vish): No need add the prefix to console.log + # NOTE(vish): No need add the suffix to console.log os.close(os.open(basepath('console.log', ''), os.O_CREAT | os.O_WRONLY, 0660)) @@ -577,7 +577,7 @@ class LibvirtConnection(object): root_fname = disk_images['image_id'] size = FLAGS.minimum_root_size - if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': + if inst['instance_type'] == 'm1.tiny' or suffix == '.rescue': size = None root_fname += "_sm" @@ -593,7 +593,7 @@ class LibvirtConnection(object): if type_data['local_gb']: self._cache_image(fn=self._create_local, - target=basepath('local'), + target=basepath('disk.local'), fname="local_%s" % type_data['local_gb'], cow=FLAGS.use_cow_images, local_gb=type_data['local_gb']) @@ -995,8 +995,7 @@ class NWFilterFirewall(FirewallDriver): ['no-mac-spoofing', 'no-ip-spoofing', 'no-arp-spoofing', - 'allow-dhcp-server' - ])) + 'allow-dhcp-server'])) self._define_filter(self.nova_base_ipv4_filter) self._define_filter(self.nova_base_ipv6_filter) self._define_filter(self.nova_dhcp_filter) diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index 96d8f5fc8c..4bfaf4b57b 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -76,6 +76,7 @@ def reset(): for c in _CLASSES: _db_content[c] = {} create_host('fake') + create_vm('fake', 'Running', is_a_template=False, is_control_domain=True) def create_host(name_label): @@ -136,14 +137,21 @@ def create_vdi(name_label, read_only, sr_ref, sharable): def create_vbd(vm_ref, vdi_ref): - vbd_rec = {'VM': vm_ref, 'VDI': vdi_ref} + vbd_rec = { + 'VM': vm_ref, + 'VDI': vdi_ref, + 'currently_attached': False, + } vbd_ref = _create_object('VBD', vbd_rec) after_VBD_create(vbd_ref, vbd_rec) return vbd_ref def after_VBD_create(vbd_ref, vbd_rec): - """Create backref from VM to VBD when VBD is created""" + """Create read-only fields and backref from VM to VBD when VBD is + created.""" + vbd_rec['currently_attached'] = False + vbd_rec['device'] = '' vm_ref = vbd_rec['VM'] vm_rec = _db_content['VM'][vm_ref] vm_rec['VBDs'] = [vbd_ref] @@ -152,9 +160,10 @@ def after_VBD_create(vbd_ref, vbd_rec): vbd_rec['vm_name_label'] = vm_name_label -def create_pbd(config, sr_ref, attached): +def create_pbd(config, host_ref, sr_ref, attached): return _create_object('PBD', { 'device-config': config, + 'host': host_ref, 'SR': sr_ref, 'currently-attached': attached, }) @@ -167,6 +176,33 @@ def create_task(name_label): }) +def create_local_srs(): + """Create an SR that looks like the one created on the local disk by + default by the XenServer installer. Do this one per host.""" + for host_ref in _db_content['host'].keys(): + _create_local_sr(host_ref) + + +def _create_local_sr(host_ref): + sr_ref = _create_object('SR', { + 'name_label': 'Local storage', + 'type': 'lvm', + 'content_type': 'user', + 'shared': False, + 'physical_size': str(1 << 30), + 'physical_utilisation': str(0), + 'virtual_allocation': str(0), + 'other_config': { + 'i18n-original-value-name_label': 'Local storage', + 'i18n-key': 'local-storage', + }, + 'VDIs': [] + }) + pbd_ref = create_pbd('', host_ref, sr_ref, True) + _db_content['SR'][sr_ref]['PBDs'] = [pbd_ref] + return sr_ref + + def _create_object(table, obj): ref = str(uuid.uuid4()) obj['uuid'] = str(uuid.uuid4()) @@ -179,9 +215,10 @@ def _create_sr(table, obj): # Forces fake to support iscsi only if sr_type != 'iscsi': raise Failure(['SR_UNKNOWN_DRIVER', sr_type]) + host_ref = _db_content['host'].keys()[0] sr_ref = _create_object(table, obj[2]) vdi_ref = create_vdi('', False, sr_ref, False) - pbd_ref = create_pbd('', sr_ref, True) + pbd_ref = create_pbd('', host_ref, sr_ref, True) _db_content['SR'][sr_ref]['VDIs'] = [vdi_ref] _db_content['SR'][sr_ref]['PBDs'] = [pbd_ref] _db_content['VDI'][vdi_ref]['SR'] = sr_ref @@ -233,6 +270,20 @@ class SessionBase(object): def __init__(self, uri): self._session = None + def VBD_plug(self, _1, ref): + rec = get_record('VBD', ref) + if rec['currently_attached']: + raise Failure(['DEVICE_ALREADY_ATTACHED', ref]) + rec['currently_attached'] = True + rec['device'] = rec['userdevice'] + + def VBD_unplug(self, _1, ref): + rec = get_record('VBD', ref) + if not rec['currently_attached']: + raise Failure(['DEVICE_ALREADY_DETACHED', ref]) + rec['currently_attached'] = False + rec['device'] = '' + def xenapi_request(self, methodname, params): if methodname.startswith('login'): self._login(methodname, params) @@ -289,6 +340,8 @@ class SessionBase(object): return lambda *params: self._getter(name, params) elif self._is_create(name): return lambda *params: self._create(name, params) + elif self._is_destroy(name): + return lambda *params: self._destroy(name, params) else: return None @@ -299,10 +352,16 @@ class SessionBase(object): bits[1].startswith(getter and 'get_' or 'set_')) def _is_create(self, name): + return self._is_method(name, 'create') + + def _is_destroy(self, name): + return self._is_method(name, 'destroy') + + def _is_method(self, name, meth): bits = name.split('.') return (len(bits) == 2 and bits[0] in _CLASSES and - bits[1] == 'create') + bits[1] == meth) def _getter(self, name, params): self._check_session(params) @@ -370,10 +429,9 @@ class SessionBase(object): _create_sr(cls, params) or _create_object(cls, params[1]) # Call hook to provide any fixups needed (ex. creating backrefs) - try: - globals()["after_%s_create" % cls](ref, params[1]) - except KeyError: - pass + after_hook = 'after_%s_create' % cls + if after_hook in globals(): + globals()[after_hook](ref, params[1]) obj = get_record(cls, ref) @@ -383,6 +441,15 @@ class SessionBase(object): return ref + def _destroy(self, name, params): + self._check_session(params) + self._check_arg_count(params, 2) + table, _ = name.split('.') + ref = params[1] + if ref not in _db_content[table]: + raise Failure(['HANDLE_INVALID', table, ref]) + del _db_content[table][ref] + def _async(self, name, params): task_ref = create_task(name) task = _db_content['task'][task_ref] @@ -420,7 +487,7 @@ class SessionBase(object): try: return result[0] except IndexError: - return None + raise Failure(['UUID_INVALID', v, result, recs, k]) return result diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a91c8ea27b..b80ff4dbad 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -19,11 +19,14 @@ Helper methods for operations related to the management of VM records and their attributes like VDIs, VIFs, as well as their lookup functions. """ +import os import pickle +import re import urllib from xml.dom import minidom from eventlet import event +import glance.client from nova import exception from nova import flags from nova import log as logging @@ -47,17 +50,23 @@ XENAPI_POWER_STATE = { 'Crashed': power_state.CRASHED} +SECTOR_SIZE = 512 +MBR_SIZE_SECTORS = 63 +MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE +KERNEL_DIR = '/boot/guest' + + class ImageType: - """ - Enumeration class for distinguishing different image types - 0 - kernel/ramdisk image (goes on dom0's filesystem) - 1 - disk image (local SR, partitioned by objectstore plugin) - 2 - raw disk image (local SR, NOT partitioned by plugin) - """ + """ + Enumeration class for distinguishing different image types + 0 - kernel/ramdisk image (goes on dom0's filesystem) + 1 - disk image (local SR, partitioned by objectstore plugin) + 2 - raw disk image (local SR, NOT partitioned by plugin) + """ - KERNEL_RAMDISK = 0 - DISK = 1 - DISK_RAW = 2 + KERNEL_RAMDISK = 0 + DISK = 1 + DISK_RAW = 2 class VMHelper(HelperBase): @@ -207,6 +216,25 @@ class VMHelper(HelperBase): return vif_ref @classmethod + def create_vdi(cls, session, sr_ref, name_label, virtual_size, read_only): + """Create a VDI record and returns its reference.""" + vdi_ref = session.get_xenapi().VDI.create( + {'name_label': name_label, + 'name_description': '', + 'SR': sr_ref, + 'virtual_size': str(virtual_size), + 'type': 'User', + 'sharable': False, + 'read_only': read_only, + 'xenstore_data': {}, + 'other_config': {}, + 'sm_config': {}, + 'tags': []}) + LOG.debug(_('Created VDI %s (%s, %s, %s) on %s.'), vdi_ref, + name_label, virtual_size, read_only, sr_ref) + return vdi_ref + + @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): """ Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI, Snapshot VHD @@ -236,14 +264,15 @@ class VMHelper(HelperBase): return template_vm_ref, [template_vdi_uuid, parent_uuid] @classmethod - def upload_image(cls, session, instance_id, vdi_uuids, image_name): + def upload_image(cls, session, instance_id, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and push them into Glance using the specified human-friendly name. """ - LOG.debug(_("Asking xapi to upload %s as '%s'"), vdi_uuids, image_name) + logging.debug(_("Asking xapi to upload %s as ID %s"), + vdi_uuids, image_id) params = {'vdi_uuids': vdi_uuids, - 'image_name': image_name, + 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port} @@ -255,15 +284,71 @@ class VMHelper(HelperBase): def fetch_image(cls, session, instance_id, image, user, project, type): """ type is interpreted as an ImageType instance + Related flags: + xenapi_image_service = ['glance', 'objectstore'] + glance_address = 'address for glance services' + glance_port = 'port for glance services' """ - url = images.image_url(image) access = AuthManager().get_access_key(user, project) + + if FLAGS.xenapi_image_service == 'glance': + return cls._fetch_image_glance(session, instance_id, image, + access, type) + else: + return cls._fetch_image_objectstore(session, instance_id, image, + access, user.secret, type) + + @classmethod + def _fetch_image_glance(cls, session, instance_id, image, access, type): + sr = find_sr(session) + if sr is None: + raise exception.NotFound('Cannot find SR to write VDI to') + + c = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) + + meta, image_file = c.get_image(image) + virtual_size = int(meta['size']) + vdi_size = virtual_size + LOG.debug(_("Size for image %s:%d"), image, virtual_size) + if type == ImageType.DISK: + # Make room for MBR. + vdi_size += MBR_SIZE_BYTES + + vdi = cls.create_vdi(session, sr, _('Glance image %s') % image, + vdi_size, False) + + with_vdi_attached_here(session, vdi, False, + lambda dev: + _stream_disk(dev, type, + virtual_size, image_file)) + if (type == ImageType.KERNEL_RAMDISK): + #we need to invoke a plugin for copying VDI's + #content into proper path + LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi) + fn = "copy_kernel_vdi" + args = {} + args['vdi-ref'] = vdi + #let the plugin copy the correct number of bytes + args['image-size'] = str(vdi_size) + task = session.async_call_plugin('glance', fn, args) + filename = session.wait_for_task(instance_id, task) + #remove the VDI as it is not needed anymore + session.get_xenapi().VDI.destroy(vdi) + LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi) + return filename + else: + return session.get_xenapi().VDI.get_uuid(vdi) + + @classmethod + def _fetch_image_objectstore(cls, session, instance_id, image, access, + secret, type): + url = images.image_url(image) LOG.debug(_("Asking xapi to fetch %s as %s"), url, access) fn = (type != ImageType.KERNEL_RAMDISK) and 'get_vdi' or 'get_kernel' args = {} args['src_url'] = url args['username'] = access - args['password'] = user.secret + args['password'] = secret args['add_partition'] = 'false' args['raw'] = 'false' if type != ImageType.KERNEL_RAMDISK: @@ -275,14 +360,21 @@ class VMHelper(HelperBase): return uuid @classmethod - def lookup_image(cls, session, vdi_ref): + def lookup_image(cls, session, instance_id, vdi_ref): + if FLAGS.xenapi_image_service == 'glance': + return cls._lookup_image_glance(session, vdi_ref) + else: + return cls._lookup_image_objectstore(session, instance_id, vdi_ref) + + @classmethod + def _lookup_image_objectstore(cls, session, instance_id, vdi_ref): LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) fn = "is_vdi_pv" args = {} args['vdi-ref'] = vdi_ref - #TODO: Call proper function in plugin task = session.async_call_plugin('objectstore', fn, args) - pv_str = session.wait_for_task(task) + pv_str = session.wait_for_task(instance_id, task) + pv = None if pv_str.lower() == 'true': pv = True elif pv_str.lower() == 'false': @@ -291,6 +383,23 @@ class VMHelper(HelperBase): return pv @classmethod + def _lookup_image_glance(cls, session, vdi_ref): + LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) + + def is_vdi_pv(dev): + LOG.debug(_("Running pygrub against %s"), dev) + output = os.popen('pygrub -qn /dev/%s' % dev) + for line in output.readlines(): + #try to find kernel string + m = re.search('(?<=kernel:)/.*(?:>)', line) + if m and m.group(0).find('xen') != -1: + LOG.debug(_("Found Xen kernel %s") % m.group(0)) + return True + LOG.debug(_("No Xen kernel found. Booting HVM.")) + return False + return with_vdi_attached_here(session, vdi_ref, True, is_vdi_pv) + + @classmethod def lookup(cls, session, i): """Look the instance i up, and returns it if available""" vms = session.get_xenapi().VM.get_by_name_label(i) @@ -424,9 +533,16 @@ def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, * parent_vhd snapshot """ - #TODO(sirp): we need to timeout this req after a while + max_attempts = FLAGS.xenapi_vhd_coalesce_max_attempts + attempts = {'counter': 0} def _poll_vhds(): + attempts['counter'] += 1 + if attempts['counter'] > max_attempts: + msg = (_("VHD coalesce attempts exceeded (%d > %d), giving up...") + % (attempts['counter'], max_attempts)) + raise exception.Error(msg) + scan_sr(session, instance_id, sr_ref) parent_uuid = get_vhd_parent_uuid(session, vdi_ref) if original_parent_uuid and (parent_uuid != original_parent_uuid): @@ -434,13 +550,12 @@ def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, "waiting for coalesce..."), parent_uuid, original_parent_uuid) else: - done.send(parent_uuid) + # Breakout of the loop (normally) and return the parent_uuid + raise utils.LoopingCallDone(parent_uuid) - done = event.Event() loop = utils.LoopingCall(_poll_vhds) loop.start(FLAGS.xenapi_vhd_coalesce_poll_interval, now=True) - parent_uuid = done.wait() - loop.stop() + parent_uuid = loop.wait() return parent_uuid @@ -457,3 +572,123 @@ def get_vdi_for_vm_safely(session, vm_ref): vdi_ref = vdi_refs[0] vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) return vdi_ref, vdi_rec + + +def find_sr(session): + host = session.get_xenapi_host() + srs = session.get_xenapi().SR.get_all() + for sr in srs: + sr_rec = session.get_xenapi().SR.get_record(sr) + if not ('i18n-key' in sr_rec['other_config'] and + sr_rec['other_config']['i18n-key'] == 'local-storage'): + continue + for pbd in sr_rec['PBDs']: + pbd_rec = session.get_xenapi().PBD.get_record(pbd) + if pbd_rec['host'] == host: + return sr + return None + + +def with_vdi_attached_here(session, vdi, read_only, f): + this_vm_ref = get_this_vm_ref(session) + vbd_rec = {} + vbd_rec['VM'] = this_vm_ref + vbd_rec['VDI'] = vdi + vbd_rec['userdevice'] = 'autodetect' + vbd_rec['bootable'] = False + vbd_rec['mode'] = read_only and 'RO' or 'RW' + vbd_rec['type'] = 'disk' + vbd_rec['unpluggable'] = True + vbd_rec['empty'] = False + vbd_rec['other_config'] = {} + vbd_rec['qos_algorithm_type'] = '' + vbd_rec['qos_algorithm_params'] = {} + vbd_rec['qos_supported_algorithms'] = [] + LOG.debug(_('Creating VBD for VDI %s ... '), vdi) + vbd = session.get_xenapi().VBD.create(vbd_rec) + LOG.debug(_('Creating VBD for VDI %s done.'), vdi) + try: + LOG.debug(_('Plugging VBD %s ... '), vbd) + session.get_xenapi().VBD.plug(vbd) + LOG.debug(_('Plugging VBD %s done.'), vbd) + return f(session.get_xenapi().VBD.get_device(vbd)) + finally: + LOG.debug(_('Destroying VBD for VDI %s ... '), vdi) + vbd_unplug_with_retry(session, vbd) + ignore_failure(session.get_xenapi().VBD.destroy, vbd) + LOG.debug(_('Destroying VBD for VDI %s done.'), vdi) + + +def vbd_unplug_with_retry(session, vbd): + """Call VBD.unplug on the given VBD, with a retry if we get + DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're + seeing the device still in use, even when all processes using the device + should be dead.""" + while True: + try: + session.get_xenapi().VBD.unplug(vbd) + LOG.debug(_('VBD.unplug successful first time.')) + return + except VMHelper.XenAPI.Failure, e: + if (len(e.details) > 0 and + e.details[0] == 'DEVICE_DETACH_REJECTED'): + LOG.debug(_('VBD.unplug rejected: retrying...')) + time.sleep(1) + elif (len(e.details) > 0 and + e.details[0] == 'DEVICE_ALREADY_DETACHED'): + LOG.debug(_('VBD.unplug successful eventually.')) + return + else: + LOG.error(_('Ignoring XenAPI.Failure in VBD.unplug: %s'), + e) + return + + +def ignore_failure(func, *args, **kwargs): + try: + return func(*args, **kwargs) + except VMHelper.XenAPI.Failure, e: + LOG.error(_('Ignoring XenAPI.Failure %s'), e) + return None + + +def get_this_vm_uuid(): + with file('/sys/hypervisor/uuid') as f: + return f.readline().strip() + + +def get_this_vm_ref(session): + return session.get_xenapi().VM.get_by_uuid(get_this_vm_uuid()) + + +def _stream_disk(dev, type, virtual_size, image_file): + offset = 0 + if type == ImageType.DISK: + offset = MBR_SIZE_BYTES + _write_partition(virtual_size, dev) + + with open('/dev/%s' % dev, 'wb') as f: + f.seek(offset) + for chunk in image_file: + f.write(chunk) + + +def _write_partition(virtual_size, dev): + dest = '/dev/%s' % dev + mbr_last = MBR_SIZE_SECTORS - 1 + primary_first = MBR_SIZE_SECTORS + primary_last = MBR_SIZE_SECTORS + (virtual_size / SECTOR_SIZE) - 1 + + LOG.debug(_('Writing partition table %d %d to %s...'), + primary_first, primary_last, dest) + + def execute(cmd, process_input=None, check_exit_code=True): + return utils.execute(cmd=cmd, + process_input=process_input, + check_exit_code=check_exit_code) + + execute('parted --script %s mklabel msdos' % dest) + execute('parted --script %s mkpart primary %ds %ds' % + (dest, primary_first, primary_last)) + + LOG.debug(_('Writing partition table %s done.'), dest) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6e359ef82b..6c2fd6a687 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -85,7 +85,8 @@ class VMOps(object): #Have a look at the VDI and see if it has a PV kernel pv_kernel = False if not instance.kernel_id: - pv_kernel = VMHelper.lookup_image(self._session, vdi_ref) + pv_kernel = VMHelper.lookup_image(self._session, instance.id, + vdi_ref) kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, @@ -161,11 +162,11 @@ class VMOps(object): raise Exception(_('Instance not present %s') % instance_name) return vm - def snapshot(self, instance, name): + def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance :param instance: instance to be snapshotted - :param name: name/label to be given to the snapshot + :param image_id: id of image to upload to Steps involved in a XenServer snapshot: @@ -201,7 +202,7 @@ class VMOps(object): try: # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, name) + self._session, instance.id, template_vdi_uuids, image_id) finally: self._destroy(instance, template_vm_ref, shutdown=False) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 689844f346..c57c883c97 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -89,10 +89,17 @@ flags.DEFINE_float('xenapi_task_poll_interval', 'The interval used for polling of remote tasks ' '(Async.VM.start, etc). Used only if ' 'connection_type=xenapi.') +flags.DEFINE_string('xenapi_image_service', + 'glance', + 'Where to get VM images: glance or objectstore.') flags.DEFINE_float('xenapi_vhd_coalesce_poll_interval', 5.0, 'The interval used for polling of coalescing vhds.' ' Used only if connection_type=xenapi.') +flags.DEFINE_integer('xenapi_vhd_coalesce_max_attempts', + 5, + 'Max number of times to poll for VHD to coalesce.' + ' Used only if connection_type=xenapi.') flags.DEFINE_string('target_host', None, 'iSCSI Target Host') @@ -141,9 +148,9 @@ class XenAPIConnection(object): """Create VM instance""" self._vmops.spawn(instance) - def snapshot(self, instance, name): + def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ - self._vmops.snapshot(instance, name) + self._vmops.snapshot(instance, image_id) def reboot(self, instance): """Reboot VM instance""" diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 71fe18a403..5fefa10cf2 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -47,7 +47,7 @@ flags.DEFINE_integer('iscsi_num_targets', 'Number of iscsi target ids per host') flags.DEFINE_string('iscsi_target_prefix', 'iqn.2010-10.org.openstack:', 'prefix for iscsi volumes') -flags.DEFINE_string('iscsi_ip_prefix', '127.0', +flags.DEFINE_string('iscsi_ip_prefix', '$my_ip', 'discover volumes on the ip that starts with this prefix') flags.DEFINE_string('rbd_pool', 'rbd', 'the rbd pool in which volumes are stored') @@ -285,7 +285,8 @@ class ISCSIDriver(VolumeDriver): self._execute("sudo iscsiadm -m node -T %s -p %s --op update " "-n node.startup -v automatic" % (iscsi_name, iscsi_portal)) - return "/dev/iscsi/%s" % volume['name'] + return "/dev/disk/by-path/ip-%s-iscsi-%s-lun-0" % (iscsi_portal, + iscsi_name) def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" diff --git a/nova/wsgi.py b/nova/wsgi.py index e999f76a3a..4f5307d80a 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -21,7 +21,7 @@ Utility methods for working with WSGI servers """ -import json +import os import sys from xml.dom import minidom @@ -34,7 +34,14 @@ import webob import webob.dec import webob.exc +from paste import deploy + +from nova import flags from nova import log as logging +from nova import utils + + +FLAGS = flags.FLAGS class WritableLogger(object): @@ -76,10 +83,33 @@ class Server(object): class Application(object): -# TODO(gundlach): I think we should toss this class, now that it has no -# purpose. """Base WSGI application wrapper. Subclasses need to implement __call__.""" + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config fles. + + Any local configuration (that is, values under the [app:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [app:wadl] + latest_version = 1.3 + paste.app_factory = nova.api.fancy_api:Wadl.factory + + which would result in a call to the `Wadl` class as + + import nova.api.fancy_api + fancy_api.Wadl(latest_version='1.3') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + return cls(**local_config) + def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: @@ -117,20 +147,65 @@ class Application(object): class Middleware(Application): - """ - Base WSGI middleware wrapper. These classes require an application to be + """Base WSGI middleware. + + These classes require an application to be initialized that will be called next. By default the middleware will simply call its wrapped app, or you can override __call__ to customize its behavior. """ - def __init__(self, application): # pylint: disable-msg=W0231 + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config fles. + + Any local configuration (that is, values under the [filter:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [filter:analytics] + redis_host = 127.0.0.1 + paste.filter_factory = nova.api.analytics:Analytics.factory + + which would result in a call to the `Analytics` class as + + import nova.api.analytics + analytics.Analytics(app_from_paste, redis_host='127.0.0.1') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + def _factory(app): + return cls(app, **local_config) + return _factory + + def __init__(self, application): self.application = application + def process_request(self, req): + """Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + @webob.dec.wsgify - def __call__(self, req): # pylint: disable-msg=W0221 - """Override to implement middleware behavior.""" - return self.application + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) class Debug(Middleware): @@ -316,7 +391,7 @@ class Serializer(object): try: is_xml = (datastring[0] == '<') if not is_xml: - return json.loads(datastring) + return utils.loads(datastring) return self._from_xml(datastring) except: return None @@ -349,7 +424,7 @@ class Serializer(object): return result def _to_json(self, data): - return json.dumps(data) + return utils.dumps(data) def _to_xml(self, data): metadata = self.metadata.get('application/xml', {}) @@ -385,3 +460,64 @@ class Serializer(object): node = doc.createTextNode(str(data)) result.appendChild(node) return result + + +def paste_config_file(basename): + """Find the best location in the system for a paste config file. + + Search Order + ------------ + + The search for a paste config file honors `FLAGS.state_path`, which in a + version checked out from bzr will be the `nova` directory in the top level + of the checkout, and in an installation for a package for your distribution + will likely point to someplace like /etc/nova. + + This method tries to load places likely to be used in development or + experimentation before falling back to the system-wide configuration + in `/etc/nova/`. + + * Current working directory + * the `etc` directory under state_path, because when working on a checkout + from bzr this will point to the default + * top level of FLAGS.state_path, for distributions + * /etc/nova, which may not be diffrerent from state_path on your distro + + """ + + configfiles = [basename, + os.path.join(FLAGS.state_path, 'etc', basename), + os.path.join(FLAGS.state_path, basename), + '/etc/nova/%s' % basename] + for configfile in configfiles: + if os.path.exists(configfile): + return configfile + + +def load_paste_configuration(filename, appname): + """Returns a paste configuration dict, or None.""" + filename = os.path.abspath(filename) + config = None + try: + config = deploy.appconfig("config:%s" % filename, name=appname) + except LookupError: + pass + return config + + +def load_paste_app(filename, appname): + """Builds a wsgi app from a paste config, None if app not configured.""" + filename = os.path.abspath(filename) + app = None + try: + app = deploy.loadapp("config:%s" % filename, name=appname) + except LookupError: + pass + return app + + +def paste_config_to_flags(config, mixins): + for k, v in mixins.iteritems(): + value = config.get(k, v) + converted_value = FLAGS[k].parser.Parse(value) + setattr(FLAGS, k, converted_value) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 5e648b9702..aadacce574 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -18,7 +18,7 @@ # under the License. # -# XenAPI plugin for putting images into glance +# XenAPI plugin for managing glance images # import base64 @@ -40,29 +40,57 @@ from pluginlib_nova import * configure_logging('glance') CHUNK_SIZE = 8192 +KERNEL_DIR = '/boot/guest' FILE_SR_PATH = '/var/run/sr-mount' +def copy_kernel_vdi(session,args): + vdi = exists(args, 'vdi-ref') + size = exists(args,'image-size') + #Use the uuid as a filename + vdi_uuid=session.xenapi.VDI.get_uuid(vdi) + copy_args={'vdi_uuid':vdi_uuid,'vdi_size':int(size)} + filename=with_vdi_in_dom0(session, vdi, False, + lambda dev: + _copy_kernel_vdi('/dev/%s' % dev,copy_args)) + return filename + +def _copy_kernel_vdi(dest,copy_args): + vdi_uuid=copy_args['vdi_uuid'] + vdi_size=copy_args['vdi_size'] + logging.debug("copying kernel/ramdisk file from %s to /boot/guest/%s",dest,vdi_uuid) + filename=KERNEL_DIR + '/' + vdi_uuid + #read data from /dev/ and write into a file on /boot/guest + of=open(filename,'wb') + f=open(dest,'rb') + #copy only vdi_size bytes + data=f.read(vdi_size) + of.write(data) + f.close() + of.close() + logging.debug("Done. Filename: %s",filename) + return filename + def put_vdis(session, args): params = pickle.loads(exists(args, 'params')) vdi_uuids = params["vdi_uuids"] - image_name = params["image_name"] + image_id = params["image_id"] glance_host = params["glance_host"] glance_port = params["glance_port"] sr_path = get_sr_path(session) #FIXME(sirp): writing to a temp file until Glance supports chunked-PUTs - tmp_file = "%s.tar.gz" % os.path.join('/tmp', image_name) + tmp_file = "%s.tar.gz" % os.path.join('/tmp', str(image_id)) tar_cmd = ['tar', '-zcf', tmp_file, '--directory=%s' % sr_path] paths = [ "%s.vhd" % vdi_uuid for vdi_uuid in vdi_uuids ] tar_cmd.extend(paths) logging.debug("Bundling image with cmd: %s", tar_cmd) subprocess.call(tar_cmd) logging.debug("Writing to test file %s", tmp_file) - put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port) + put_bundle_in_glance(tmp_file, image_id, glance_host, glance_port) return "" # FIXME(sirp): return anything useful here? -def put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port): +def put_bundle_in_glance(tmp_file, image_id, glance_host, glance_port): size = os.path.getsize(tmp_file) basename = os.path.basename(tmp_file) @@ -72,7 +100,6 @@ def put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port): 'x-image-meta-store': 'file', 'x-image-meta-is_public': 'True', 'x-image-meta-type': 'raw', - 'x-image-meta-name': image_name, 'x-image-meta-size': size, 'content-length': size, 'content-type': 'application/octet-stream', @@ -80,7 +107,7 @@ def put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port): conn = httplib.HTTPConnection(glance_host, glance_port) #NOTE(sirp): httplib under python2.4 won't accept a file-like object # to request - conn.putrequest('POST', '/images') + conn.putrequest('PUT', '/images/%s' % image_id) for header, value in headers.iteritems(): conn.putheader(header, value) @@ -129,4 +156,5 @@ def find_sr(session): if __name__ == '__main__': - XenAPIPlugin.dispatch({'put_vdis': put_vdis}) + XenAPIPlugin.dispatch({'put_vdis': put_vdis, + 'copy_kernel_vdi': copy_kernel_vdi}) diff --git a/tools/iscsidev.sh b/tools/iscsidev.sh deleted file mode 100755 index 6f5b572df8..0000000000 --- a/tools/iscsidev.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# NOTE(vish): This script helps udev create common names for discovered iscsi -# volumes under /dev/iscsi. To use it, create /dev/iscsi and add -# a file to /etc/udev/rules.d like so: -# mkdir /dev/iscsi -# echo 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/path/to/iscsidev.sh -# %b",SYMLINK+="iscsi/%c%n"' > /etc/udev/rules.d/55-openiscsi.rules - -BUS=${1} -HOST=${BUS%%:*} - -if [ ! -e /sys/class/iscsi_host ]; then - exit 1 -fi - -file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" - -target_name=$(cat ${file}) - -if [ -z "${target_name}" ]; then - exit 1 -fi - -echo "${target_name##*:}" diff --git a/tools/pip-requires b/tools/pip-requires index dedd973dca..895e81eb3f 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -27,3 +27,4 @@ PasteDeploy paste sqlalchemy-migrate netaddr +glance |