diff options
-rw-r--r-- | heatclient/common/template_utils.py | 39 | ||||
-rw-r--r-- | heatclient/common/utils.py | 82 | ||||
-rw-r--r-- | heatclient/tests/test_shell.py | 82 | ||||
-rw-r--r-- | heatclient/tests/test_template_utils.py | 75 | ||||
-rw-r--r-- | heatclient/tests/test_utils.py | 119 | ||||
-rw-r--r-- | heatclient/v1/shell.py | 48 |
6 files changed, 333 insertions, 112 deletions
diff --git a/heatclient/common/template_utils.py b/heatclient/common/template_utils.py index 5b87b1f..3c05687 100644 --- a/heatclient/common/template_utils.py +++ b/heatclient/common/template_utils.py @@ -13,18 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. -import base64 import collections -import os - from oslo_serialization import jsonutils import six -from six.moves.urllib import error from six.moves.urllib import parse from six.moves.urllib import request from heatclient.common import environment_format from heatclient.common import template_format +from heatclient.common import utils from heatclient import exc from heatclient.openstack.common._i18n import _ @@ -35,7 +32,7 @@ def get_template_contents(template_file=None, template_url=None, # Transform a bare file path to a file:// URL. if template_file: - template_url = normalise_file_path_to_url(template_file) + template_url = utils.normalise_file_path_to_url(template_file) if template_url: tpl = request.urlopen(template_url).read() @@ -65,7 +62,7 @@ def get_template_contents(template_file=None, template_url=None, raise exc.CommandError(_('Error parsing template %(url)s %(error)s') % {'url': template_url, 'error': e}) - tmpl_base_url = base_url_for_url(template_url) + tmpl_base_url = utils.base_url_for_url(template_url) if files is None: files = {} resolve_template_get_files(template, files, tmpl_base_url) @@ -133,37 +130,25 @@ def get_file_contents(from_data, files, base_url=None, template_url=str_url, files=files)[1] file_content = jsonutils.dumps(template) else: - file_content = read_url_content(str_url) + file_content = utils.read_url_content(str_url) files[str_url] = file_content # replace the data value with the normalised absolute URL from_data[key] = str_url def read_url_content(url): - try: - content = request.urlopen(url).read() - except error.URLError: - raise exc.CommandError(_('Could not fetch contents for %s') % url) - - if content: - try: - content.decode('utf-8') - except ValueError: - content = base64.encodestring(content) - return content + '''DEPRECATED! Use 'utils.read_url_content' instead.''' + return utils.read_url_content(url) def base_url_for_url(url): - parsed = parse.urlparse(url) - parsed_dir = os.path.dirname(parsed.path) - return parse.urljoin(url, parsed_dir) + '''DEPRECATED! Use 'utils.base_url_for_url' instead.''' + return utils.base_url_for_url(url) def normalise_file_path_to_url(path): - if parse.urlparse(path).scheme: - return path - path = os.path.abspath(path) - return parse.urljoin('file:', request.pathname2url(path)) + '''DEPRECATED! Use 'utils.normalise_file_path_to_url' instead.''' + return utils.normalise_file_path_to_url(path) def deep_update(old, new): @@ -204,8 +189,8 @@ def process_environment_and_files(env_path=None, template=None, env = {} if env_path: - env_url = normalise_file_path_to_url(env_path) - env_base_url = base_url_for_url(env_url) + env_url = utils.normalise_file_path_to_url(env_path) + env_base_url = utils.base_url_for_url(env_url) raw_env = request.urlopen(env_url).read() env = environment_format.parse(raw_env) diff --git a/heatclient/common/utils.py b/heatclient/common/utils.py index 28c0ca1..0afdd81 100644 --- a/heatclient/common/utils.py +++ b/heatclient/common/utils.py @@ -12,16 +12,18 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from __future__ import print_function -import sys +import base64 +import os import textwrap import uuid from oslo_serialization import jsonutils from oslo_utils import importutils import prettytable +from six.moves.urllib import error from six.moves.urllib import parse +from six.moves.urllib import request import yaml from heatclient import exc @@ -111,12 +113,6 @@ def import_versioned_module(version, submodule=None): return importutils.import_module(module) -def exit(msg=''): - if msg: - print(msg, file=sys.stderr) - sys.exit(1) - - def format_parameters(params, parse_semicolon=True): '''Reformat parameters into dict of format expected by the API.''' @@ -147,6 +143,43 @@ def format_parameters(params, parse_semicolon=True): return parameters +def format_all_parameters(params, param_files, + template_file=None, template_url=None): + parameters = {} + parameters.update(format_parameters(params)) + parameters.update(format_parameter_file( + param_files, + template_file, + template_url)) + return parameters + + +def format_parameter_file(param_files, template_file=None, + template_url=None): + '''Reformat file parameters into dict of format expected by the API.''' + if not param_files: + return {} + params = format_parameters(param_files, False) + + template_base_url = None + if template_file or template_url: + template_base_url = base_url_for_url(get_template_url( + template_file, template_url)) + + param_file = {} + for key, value in iter(params.items()): + param_file[key] = resolve_param_get_file(value, + template_base_url) + return param_file + + +def resolve_param_get_file(file, base_url): + if base_url and not base_url.endswith('/'): + base_url = base_url + '/' + str_url = parse.urljoin(base_url, file) + return read_url_content(str_url) + + def format_output(output, format='yaml'): """Format the supplied dict as specified.""" output_format = format.lower() @@ -160,3 +193,36 @@ def format_output(output, format='yaml'): def parse_query_url(url): base_url, query_params = url.split('?') return base_url, parse.parse_qs(query_params) + + +def get_template_url(template_file=None, template_url=None): + if template_file: + template_url = normalise_file_path_to_url(template_file) + return template_url + + +def read_url_content(url): + try: + content = request.urlopen(url).read() + except error.URLError: + raise exc.CommandError(_('Could not fetch contents for %s') % url) + + if content: + try: + content.decode('utf-8') + except ValueError: + content = base64.encodestring(content) + return content + + +def base_url_for_url(url): + parsed = parse.urlparse(url) + parsed_dir = os.path.dirname(parsed.path) + return parse.urljoin(url, parsed_dir) + + +def normalise_file_path_to_url(path): + if parse.urlparse(path).scheme: + return path + path = os.path.abspath(path) + return parse.urljoin('file:', request.pathname2url(path)) diff --git a/heatclient/tests/test_shell.py b/heatclient/tests/test_shell.py index 7bab129..474d222 100644 --- a/heatclient/tests/test_shell.py +++ b/heatclient/tests/test_shell.py @@ -273,6 +273,17 @@ class ShellValidationTest(TestCase): 'LinuxDistribution=F17"', 'Need to specify exactly one of') + def test_stack_create_with_paramfile_validation(self): + self.register_keystone_auth_fixture() + self.set_fake_env(FAKE_ENV_KEYSTONE_V2) + self.shell_error( + 'stack-create teststack ' + '--parameter-file private_key=private_key.env ' + '--parameters="InstanceType=m1.large;DBUsername=wp;' + 'DBPassword=verybadpassword;KeyName=heat_key;' + 'LinuxDistribution=F17"', + 'Need to specify exactly one of') + def test_stack_create_validation_keystone_v3(self): self.register_keystone_auth_fixture() self.set_fake_env(FAKE_ENV_KEYSTONE_V3) @@ -1126,13 +1137,49 @@ class ShellTestUserPass(ShellBase): headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} ).AndReturn((resp, None)) fakes.script_heat_list() + self.m.ReplayAll() + + template_file = os.path.join(TEST_VAR_DIR, 'minimal.template') + create_text = self.shell( + 'stack-create teststack ' + '--template-file=%s ' + '--parameters="InstanceType=m1.large;DBUsername=wp;' + 'DBPassword=verybadpassword;KeyName=heat_key;' + 'LinuxDistribution=F17"' % template_file) + required = [ + 'stack_name', + 'id', + 'teststack', + '1' + ] + + for r in required: + self.assertRegexpMatches(create_text, r) + + def test_stack_create_param_file(self): + self.register_keystone_auth_fixture() + resp = fakes.FakeHTTPResponse( + 201, + 'Created', + {'location': 'http://no.where/v1/tenant_id/stacks/teststack2/2'}, + None) + http.HTTPClient.json_request( + 'POST', '/stacks', data=mox.IgnoreArg(), + headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} + ).AndReturn((resp, None)) + fakes.script_heat_list() + + self.m.StubOutWithMock(utils, 'read_url_content') + url = 'file://%s/private_key.env' % TEST_VAR_DIR + utils.read_url_content(url).AndReturn('xxxxxx') self.m.ReplayAll() template_file = os.path.join(TEST_VAR_DIR, 'minimal.template') create_text = self.shell( 'stack-create teststack ' '--template-file=%s ' + '--parameter-file private_key=private_key.env ' '--parameters="InstanceType=m1.large;DBUsername=wp;' 'DBPassword=verybadpassword;KeyName=heat_key;' 'LinuxDistribution=F17"' % template_file) @@ -1147,6 +1194,41 @@ class ShellTestUserPass(ShellBase): for r in required: self.assertRegexpMatches(create_text, r) + def test_stack_create_only_param_file(self): + self.register_keystone_auth_fixture() + resp = fakes.FakeHTTPResponse( + 201, + 'Created', + {'location': 'http://no.where/v1/tenant_id/stacks/teststack2/2'}, + None) + http.HTTPClient.json_request( + 'POST', '/stacks', data=mox.IgnoreArg(), + headers={'X-Auth-Key': 'password', 'X-Auth-User': 'username'} + ).AndReturn((resp, None)) + fakes.script_heat_list() + + self.m.StubOutWithMock(utils, 'read_url_content') + url = 'file://%s/private_key.env' % TEST_VAR_DIR + utils.read_url_content(url).AndReturn('xxxxxx') + self.m.ReplayAll() + + template_file = os.path.join(TEST_VAR_DIR, 'minimal.template') + create_text = self.shell( + 'stack-create teststack ' + '--template-file=%s ' + '--parameter-file private_key=private_key.env ' + % template_file) + + required = [ + 'stack_name', + 'id', + 'teststack', + '1' + ] + + for r in required: + self.assertRegexpMatches(create_text, r) + def test_stack_create_timeout(self): self.register_keystone_auth_fixture() template_file = os.path.join(TEST_VAR_DIR, 'minimal.template') diff --git a/heatclient/tests/test_template_utils.py b/heatclient/tests/test_template_utils.py index bf38c38..f486154 100644 --- a/heatclient/tests/test_template_utils.py +++ b/heatclient/tests/test_template_utils.py @@ -14,7 +14,6 @@ import base64 import json from mox3 import mox -import os import six from six.moves.urllib import request import tempfile @@ -23,6 +22,7 @@ from testtools import matchers import yaml from heatclient.common import template_utils +from heatclient.common import utils from heatclient import exc @@ -93,10 +93,10 @@ class ShellEnvironmentTest(testtools.TestCase): self.assertEqual( env_url, - template_utils.normalise_file_path_to_url(env_file)) + utils.normalise_file_path_to_url(env_file)) self.assertEqual( 'file:///home/my/dir', - template_utils.base_url_for_url(env_url)) + utils.base_url_for_url(env_url)) files, env_dict = template_utils.process_environment_and_files( env_file) @@ -127,10 +127,10 @@ class ShellEnvironmentTest(testtools.TestCase): env_url = 'file://%s' % env_file self.assertEqual( env_url, - template_utils.normalise_file_path_to_url(env_file)) + utils.normalise_file_path_to_url(env_file)) self.assertEqual( 'file:///home/my/dir', - template_utils.base_url_for_url(env_url)) + utils.base_url_for_url(env_url)) files, env_dict = template_utils.process_environment_and_files( env_file) @@ -897,68 +897,3 @@ parameters: files.get(three_url)) self.m.VerifyAll() - - -class TestURLFunctions(testtools.TestCase): - - def setUp(self): - super(TestURLFunctions, self).setUp() - self.m = mox.Mox() - - self.addCleanup(self.m.VerifyAll) - self.addCleanup(self.m.UnsetStubs) - - def test_normalise_file_path_to_url_relative(self): - self.assertEqual( - 'file://%s/foo' % os.getcwd(), - template_utils.normalise_file_path_to_url( - 'foo')) - - def test_normalise_file_path_to_url_absolute(self): - self.assertEqual( - 'file:///tmp/foo', - template_utils.normalise_file_path_to_url( - '/tmp/foo')) - - def test_normalise_file_path_to_url_file(self): - self.assertEqual( - 'file:///tmp/foo', - template_utils.normalise_file_path_to_url( - 'file:///tmp/foo')) - - def test_normalise_file_path_to_url_http(self): - self.assertEqual( - 'http://localhost/foo', - template_utils.normalise_file_path_to_url( - 'http://localhost/foo')) - - def test_base_url_for_url(self): - self.assertEqual( - 'file:///foo/bar', - template_utils.base_url_for_url( - 'file:///foo/bar/baz')) - self.assertEqual( - 'file:///foo/bar', - template_utils.base_url_for_url( - 'file:///foo/bar/baz.txt')) - self.assertEqual( - 'file:///foo/bar', - template_utils.base_url_for_url( - 'file:///foo/bar/')) - self.assertEqual( - 'file:///', - template_utils.base_url_for_url( - 'file:///')) - self.assertEqual( - 'file:///', - template_utils.base_url_for_url( - 'file:///foo')) - - self.assertEqual( - 'http://foo/bar', - template_utils.base_url_for_url( - 'http://foo/bar/')) - self.assertEqual( - 'http://foo/bar', - template_utils.base_url_for_url( - 'http://foo/bar/baz.template')) diff --git a/heatclient/tests/test_utils.py b/heatclient/tests/test_utils.py index d7a786b..f5ec218 100644 --- a/heatclient/tests/test_utils.py +++ b/heatclient/tests/test_utils.py @@ -14,6 +14,8 @@ # under the License. from heatclient.common import utils from heatclient import exc +import mock +import os import testtools @@ -137,3 +139,120 @@ class shellTest(testtools.TestCase): self.assertEqual('', utils.newline_list_formatter([])) self.assertEqual('one\ntwo', utils.newline_list_formatter(['one', 'two'])) + + +class shellTestParameterFiles(testtools.TestCase): + + def test_format_parameter_file_none(self): + self.assertEqual({}, utils.format_parameter_file(None)) + + def test_format_parameter_file(self): + tmpl_file = '/opt/stack/template.yaml' + contents = 'DBUsername=wp\nDBPassword=verybadpassword' + utils.read_url_content = mock.MagicMock() + utils.read_url_content.return_value = 'DBUsername=wp\n' \ + 'DBPassword=verybadpassword' + + p = utils.format_parameter_file([ + 'env_file1=test_file1'], tmpl_file) + self.assertEqual({'env_file1': contents + }, p) + + def test_format_parameter_file_no_template(self): + tmpl_file = None + contents = 'DBUsername=wp\nDBPassword=verybadpassword' + utils.read_url_content = mock.MagicMock() + utils.read_url_content.return_value = 'DBUsername=wp\n' \ + 'DBPassword=verybadpassword' + p = utils.format_parameter_file([ + 'env_file1=test_file1'], tmpl_file) + self.assertEqual({'env_file1': contents + }, p) + + def test_format_all_parameters(self): + tmpl_file = '/opt/stack/template.yaml' + contents = 'DBUsername=wp\nDBPassword=verybadpassword' + params = ['KeyName=heat_key;UpstreamDNS=8.8.8.8'] + utils.read_url_content = mock.MagicMock() + utils.read_url_content.return_value = 'DBUsername=wp\n' \ + 'DBPassword=verybadpassword' + p = utils.format_all_parameters(params, [ + 'env_file1=test_file1'], template_file=tmpl_file) + self.assertEqual({'KeyName': 'heat_key', + 'UpstreamDNS': '8.8.8.8', + 'env_file1': contents}, p) + + +class TestURLFunctions(testtools.TestCase): + + def setUp(self): + super(TestURLFunctions, self).setUp() + self.m = mock.MagicMock() + + self.addCleanup(self.m.VerifyAll) + self.addCleanup(self.m.UnsetStubs) + + def test_normalise_file_path_to_url_relative(self): + self.assertEqual( + 'file://%s/foo' % os.getcwd(), + utils.normalise_file_path_to_url( + 'foo')) + + def test_normalise_file_path_to_url_absolute(self): + self.assertEqual( + 'file:///tmp/foo', + utils.normalise_file_path_to_url( + '/tmp/foo')) + + def test_normalise_file_path_to_url_file(self): + self.assertEqual( + 'file:///tmp/foo', + utils.normalise_file_path_to_url( + 'file:///tmp/foo')) + + def test_normalise_file_path_to_url_http(self): + self.assertEqual( + 'http://localhost/foo', + utils.normalise_file_path_to_url( + 'http://localhost/foo')) + + def test_get_template_url(self): + tmpl_file = '/opt/stack/template.yaml' + tmpl_url = 'file:///opt/stack/template.yaml' + self.assertEqual(utils.get_template_url(tmpl_file, None), + tmpl_url) + self.assertEqual(utils.get_template_url(None, tmpl_url), + tmpl_url) + self.assertEqual(utils.get_template_url(None, None), + None) + + def test_base_url_for_url(self): + self.assertEqual( + 'file:///foo/bar', + utils.base_url_for_url( + 'file:///foo/bar/baz')) + self.assertEqual( + 'file:///foo/bar', + utils.base_url_for_url( + 'file:///foo/bar/baz.txt')) + self.assertEqual( + 'file:///foo/bar', + utils.base_url_for_url( + 'file:///foo/bar/')) + self.assertEqual( + 'file:///', + utils.base_url_for_url( + 'file:///')) + self.assertEqual( + 'file:///', + utils.base_url_for_url( + 'file:///foo')) + + self.assertEqual( + 'http://foo/bar', + utils.base_url_for_url( + 'http://foo/bar/')) + self.assertEqual( + 'http://foo/bar', + utils.base_url_for_url( + 'http://foo/bar/baz.template')) diff --git a/heatclient/v1/shell.py b/heatclient/v1/shell.py index e8b5ba4..32636bc 100644 --- a/heatclient/v1/shell.py +++ b/heatclient/v1/shell.py @@ -68,6 +68,11 @@ def _authenticated_fetcher(hc): 'This can be specified multiple times, or once with parameters ' 'separated by a semicolon.'), action='append') +@utils.arg('-Pf', '--parameter-file', metavar='<KEY=VALUE>', + help=_('Parameter values from file used to create the stack. ' + 'This can be specified multiple times. Parameter value ' + 'would be the content of the file'), + action='append') @utils.arg('name', metavar='<STACK_NAME>', help=_('Name of the stack to create.')) def do_create(hc, args): @@ -102,6 +107,11 @@ def do_create(hc, args): 'This can be specified multiple times, or once with parameters ' 'separated by a semicolon.'), action='append') +@utils.arg('-Pf', '--parameter-file', metavar='<KEY=VALUE>', + help=_('Parameter values from file used to create the stack. ' + 'This can be specified multiple times. Parameter value ' + 'would be the content of the file'), + action='append') @utils.arg('name', metavar='<STACK_NAME>', help=_('Name of the stack to create.')) def do_stack_create(hc, args): @@ -125,7 +135,10 @@ def do_stack_create(hc, args): fields = { 'stack_name': args.name, 'disable_rollback': not(args.enable_rollback), - 'parameters': utils.format_parameters(args.parameters), + 'parameters': utils.format_all_parameters(args.parameters, + args.parameter_file, + args.template_file, + args.template_url), 'template': template, 'files': dict(list(tpl_files.items()) + list(env_files.items())), 'environment': env @@ -171,7 +184,7 @@ def do_stack_adopt(hc, args): raise exc.CommandError(_('Need to specify %(arg)s') % {'arg': '--adopt-file'}) - adopt_url = template_utils.normalise_file_path_to_url(args.adopt_file) + adopt_url = utils.normalise_file_path_to_url(args.adopt_file) adopt_data = request.urlopen(adopt_url).read() if args.create_timeout: @@ -221,6 +234,11 @@ def do_stack_adopt(hc, args): 'This can be specified multiple times, or once with parameters ' 'separated by semicolon.'), action='append') +@utils.arg('-Pf', '--parameter-file', metavar='<KEY=VALUE>', + help=_('Parameter values from file used to create the stack. ' + 'This can be specified multiple times. Parameter value ' + 'would be the content of the file'), + action='append') @utils.arg('name', metavar='<STACK_NAME>', help=_('Name of the stack to preview.')) def do_stack_preview(hc, args): @@ -237,7 +255,10 @@ def do_stack_preview(hc, args): 'stack_name': args.name, 'disable_rollback': not(args.enable_rollback), 'timeout_mins': args.timeout, - 'parameters': utils.format_parameters(args.parameters), + 'parameters': utils.format_all_parameters(args.parameters, + args.parameter_file, + args.template_file, + args.template_url), 'template': template, 'files': dict(list(tpl_files.items()) + list(env_files.items())), 'environment': env @@ -413,6 +434,11 @@ def do_stack_show(hc, args): 'This can be specified multiple times, or once with parameters ' 'separated by a semicolon.'), action='append') +@utils.arg('-Pf', '--parameter-file', metavar='<KEY=VALUE>', + help=_('Parameter values from file used to create the stack. ' + 'This can be specified multiple times. Parameter value ' + 'would be the content of the file'), + action='append') @utils.arg('-x', '--existing', default=False, action="store_true", help=_('Re-use the set of parameters of the current stack. ' 'Parameters specified in %(arg)s will patch over the existing ' @@ -463,6 +489,11 @@ def do_update(hc, args): 'This can be specified multiple times, or once with parameters ' 'separated by a semicolon.'), action='append') +@utils.arg('-Pf', '--parameter-file', metavar='<KEY=VALUE>', + help=_('Parameter values from file used to create the stack. ' + 'This can be specified multiple times. Parameter value ' + 'would be the content of the file'), + action='append') @utils.arg('-x', '--existing', default=False, action="store_true", help=_('Re-use the set of parameters of the current stack. ' 'Parameters specified in %(arg)s will patch over the existing ' @@ -491,7 +522,10 @@ def do_stack_update(hc, args): fields = { 'stack_id': args.id, - 'parameters': utils.format_parameters(args.parameters), + 'parameters': utils.format_all_parameters(args.parameters, + args.parameter_file, + args.template_file, + args.template_url), 'existing': args.existing, 'template': template, 'files': dict(list(tpl_files.items()) + list(env_files.items())), @@ -865,7 +899,7 @@ def do_resource_signal(hc, args): if data and data_file: raise exc.CommandError(_('Can only specify one of data and data-file')) if data_file: - data_url = template_utils.normalise_file_path_to_url(data_file) + data_url = utils.normalise_file_path_to_url(data_file) data = request.urlopen(data_url).read() if data: if isinstance(data, six.binary_type): @@ -979,7 +1013,7 @@ def do_config_create(hc, args): defn = {} if args.definition_file: - defn_url = template_utils.normalise_file_path_to_url( + defn_url = utils.normalise_file_path_to_url( args.definition_file) defn_raw = request.urlopen(defn_url).read() or '{}' defn = yaml.load(defn_raw, Loader=template_format.yaml_loader) @@ -989,7 +1023,7 @@ def do_config_create(hc, args): config['options'] = defn.get('options', {}) if args.config_file: - config_url = template_utils.normalise_file_path_to_url( + config_url = utils.normalise_file_path_to_url( args.config_file) config['config'] = request.urlopen(config_url).read() |