diff options
35 files changed, 1274 insertions, 218 deletions
@@ -1,4 +1,4 @@ [gerrit] -host=review.openstack.org +host=review.opendev.org port=29418 project=openstack/python-glanceclient.git @@ -1,43 +1,4 @@ - job: - name: glanceclient-dsvm-functional-v1 - parent: devstack-tox-functional - description: | - Devstack-based functional tests for glanceclient - against the Image API v1. - - The Image API v1 is removed from glance in Rocky, but - is still supported by glanceclient until the S cycle, - so we test it against glance stable/queens. - - THIS JOB SHOULD BE REMOVED AT THE BEGINNING OF THE S - CYCLE. - override-checkout: stable/queens - required-projects: - - name: openstack/python-glanceclient - override-checkout: master - timeout: 4200 - vars: - tox_envlist: functional-v1 - devstack_localrc: - GLANCE_V1_ENABLED: true - devstack_services: - # turn off ceilometer - ceilometer-acentral: false - ceilometer-acompute: false - ceilometer-alarm-evaluator: false - ceilometer-alarm-notifier: false - ceilometer-anotification: false - ceilometer-api: false - ceilometer-collector: false - # turn on swift - s-account: true - s-container: true - s-object: true - s-proxy: true - # Hardcode glanceclient path so the job can be run on glance patches - zuul_work_dir: src/git.openstack.org/openstack/python-glanceclient - -- job: name: glanceclient-dsvm-functional parent: devstack-tox-functional description: | @@ -65,54 +26,48 @@ s-object: true s-proxy: true # Hardcode glanceclient path so the job can be run on glance patches - zuul_work_dir: src/git.openstack.org/openstack/python-glanceclient + zuul_work_dir: src/opendev.org/openstack/python-glanceclient + irrelevant-files: + - ^doc/.*$ + - ^releasenotes/.*$ + - ^.*\.rst$ + - ^(test-|)requirements.txt$ + - ^lower-constraints.txt$ + - ^setup.cfg$ + - ^tox.ini$ - job: name: glanceclient-tox-keystone-tips-base parent: tox + abstract: true description: Abstract job for glanceclient vs. keystone required-projects: - name: openstack/keystoneauth - job: - name: glanceclient-tox-py27-keystone-tips + name: glanceclient-tox-py3-keystone-tips parent: glanceclient-tox-keystone-tips-base description: | - glanceclient py27 unit tests vs. keystone masters + glanceclient py3 unit tests vs. keystone masters vars: - tox_envlist: py27 - -- job: - name: glanceclient-tox-py35-keystone-tips - parent: glanceclient-tox-keystone-tips-base - description: | - glanceclient py35 unit tests vs. keystone masters - vars: - tox_envlist: py35 + tox_envlist: py3 - job: name: glanceclient-tox-oslo-tips-base parent: tox + abstract: true description: Abstract job for glanceclient vs. oslo required-projects: - name: openstack/oslo.i18n - name: openstack/oslo.utils - job: - name: glanceclient-tox-py27-oslo-tips - parent: glanceclient-tox-oslo-tips-base - description: | - glanceclient py27 unit tests vs. oslo masters - vars: - tox_envlist: py27 - -- job: - name: glanceclient-tox-py35-oslo-tips + name: glanceclient-tox-py3-oslo-tips parent: glanceclient-tox-oslo-tips-base description: | - glanceclient py35 unit tests vs. oslo masters + glanceclient py3 unit tests vs. oslo masters vars: - tox_envlist: py35 + tox_envlist: py3 - job: name: glanceclient-dsvm-functional-py3 @@ -122,22 +77,38 @@ USE_PYTHON3: true - project: + templates: + - check-requirements + - lib-forward-testing-python3 + - openstack-cover-jobs + - openstack-lower-constraints-jobs + - openstack-python3-ussuri-jobs + - publish-openstack-docs-pti + - release-notes-jobs-python3 check: jobs: - - glanceclient-dsvm-functional-v1 - glanceclient-dsvm-functional - - openstack-tox-lower-constraints gate: jobs: - - glanceclient-dsvm-functional-v1 - glanceclient-dsvm-functional - - openstack-tox-lower-constraints periodic: jobs: - - glanceclient-tox-py27-keystone-tips - - glanceclient-tox-py35-keystone-tips - - glanceclient-tox-py27-oslo-tips - - glanceclient-tox-py35-oslo-tips + # NOTE(rosmaita): we only want the "tips" jobs to be run against + # master, hence the 'branches' qualifiers below. Without them, when + # a stable branch is cut, the tests would be run against the stable + # branch as well, which is pointless because these libraries are + # frozen (more or less) in the stable branches. + # + # The "tips" jobs can be removed from the stable branch .zuul.yaml + # files if someone is so inclined, but that would require manual + # maintenance, so we do not do it by default. Another option is + # to define these jobs in the openstack/project-config repo. + # That would make us less agile in adjusting these tests, so we + # aren't doing that either. + - glanceclient-tox-py3-keystone-tips: + branches: master + - glanceclient-tox-py3-oslo-tips: + branches: master experimental: jobs: - glanceclient-dsvm-functional-py3 @@ -26,7 +26,7 @@ Python bindings to the OpenStack Images API This is a client library for Glance built on the OpenStack Images API. It provides a Python API (the ``glanceclient`` module) and a command-line tool (``glance``). This library fully supports the v1 Images API, while support for the v2 API is in progress. -Development takes place via the usual OpenStack processes as outlined in the `developer guide <https://docs.openstack.org/infra/manual/developers.html>`_. The master repository is in `Git <https://git.openstack.org/cgit/openstack/python-glanceclient>`_. +Development takes place via the usual OpenStack processes as outlined in the `developer guide <https://docs.openstack.org/infra/manual/developers.html>`_. The master repository is in `Git <https://opendev.org/openstack/python-glanceclient>`_. See release notes and more at `<https://docs.openstack.org/python-glanceclient/latest/>`_. @@ -45,7 +45,7 @@ See release notes and more at `<https://docs.openstack.org/python-glanceclient/l .. _Launchpad project: https://launchpad.net/python-glanceclient .. _Blueprints: https://blueprints.launchpad.net/python-glanceclient .. _Bugs: https://bugs.launchpad.net/python-glanceclient -.. _Source: https://git.openstack.org/cgit/openstack/python-glanceclient +.. _Source: https://opendev.org/openstack/python-glanceclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/glance-specs/ .. _Release notes: https://docs.openstack.org/releasenotes/python-glanceclient diff --git a/doc/requirements.txt b/doc/requirements.txt index 4faabc1..4c33153 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,7 +1,8 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -openstackdocstheme>=1.18.1 # Apache-2.0 +openstackdocstheme>=1.20.0 # Apache-2.0 reno>=2.5.0 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD sphinxcontrib-apidoc>=0.2.0 # BSD diff --git a/doc/source/conf.py b/doc/source/conf.py index 7d18119..74d6595 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,7 +84,6 @@ html_theme_path = [openstackdocstheme.get_html_theme_path()] # Output file base name for HTML help builder. htmlhelp_basename = '%sdoc' % project -html_last_updated_fmt = '%Y-%m-%d %H:%M' # -- Options for man page output ---------------------------------------------- diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index a5fb153..78c4bc5 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -66,7 +66,11 @@ def encode_headers(headers): for h, v in headers.items(): if v is not None: # if the item is token, do not quote '+' as well. - safe = '=+/' if h in TOKEN_HEADERS else '/' + # NOTE(imacdonn): urlparse.quote() is intended for quoting the + # path part of a URL, but headers like x-image-meta-location + # include an entire URL. We should avoid encoding the colon in + # this case (bug #1788942) + safe = '=+/' if h in TOKEN_HEADERS else '/:' if six.PY2: # incoming items may be unicode, so get them into something # the py2 version of urllib can handle before percent encoding @@ -179,6 +183,15 @@ class HTTPClient(_BaseHTTPClient): self.session.cert = (kwargs.get('cert_file'), kwargs.get('key_file')) + def __del__(self): + if self.session: + try: + self.session.close() + except Exception as e: + LOG.exception(e) + finally: + self.session = None + @staticmethod def parse_endpoint(endpoint): return netutils.urlsplit(endpoint) @@ -266,6 +279,7 @@ class HTTPClient(_BaseHTTPClient): conn_url, data=data, headers=headers, + timeout=self.timeout, **kwargs) except requests.exceptions.Timeout as e: message = ("Error communicating with %(url)s: %(e)s" % diff --git a/glanceclient/common/utils.py b/glanceclient/common/utils.py index dee9978..bc0c0eb 100644 --- a/glanceclient/common/utils.py +++ b/glanceclient/common/utils.py @@ -28,10 +28,10 @@ import uuid import six -if os.name == 'nt': - import msvcrt -else: - msvcrt = None +if os.name == 'nt': # noqa + import msvcrt # noqa +else: # noqa + msvcrt = None # noqa from oslo_utils import encodeutils from oslo_utils import strutils @@ -449,6 +449,26 @@ def integrity_iter(iter, checksum): (md5sum, checksum)) +def serious_integrity_iter(iter, hasher, hash_value): + """Check image data integrity using the Glance "multihash". + + :param iter: iterable containing the image data + :param hasher: a hashlib object + :param hash_value: hexdigest of the image data + :raises: IOError if the hashdigest of the data is not hash_value + """ + for chunk in iter: + yield chunk + if isinstance(chunk, six.string_types): + chunk = six.b(chunk) + hasher.update(chunk) + computed = hasher.hexdigest() + if computed != hash_value: + raise IOError(errno.EPIPE, + 'Corrupt image download. Hash was %s expected %s' % + (computed, hash_value)) + + def memoized_property(fn): attr_name = '_lazy_once_' + fn.__name__ diff --git a/glanceclient/exc.py b/glanceclient/exc.py index c8616c3..eee47ca 100644 --- a/glanceclient/exc.py +++ b/glanceclient/exc.py @@ -52,7 +52,7 @@ class HTTPException(ClientException): self.details = details or self.__class__.__name__ def __str__(self): - return "%s (HTTP %s)" % (self.details, self.code) + return "HTTP %s" % (self.details) class HTTPMultipleChoices(HTTPException): diff --git a/glanceclient/tests/unit/test_http.py b/glanceclient/tests/unit/test_http.py index 6eee005..2f72b9a 100644 --- a/glanceclient/tests/unit/test_http.py +++ b/glanceclient/tests/unit/test_http.py @@ -216,7 +216,11 @@ class TestClient(testtools.TestCase): def test_headers_encoding(self): value = u'ni\xf1o' - headers = {"test": value, "none-val": None, "Name": "value"} + fake_location = b'http://web_server:80/images/fake.img' + headers = {"test": value, + "none-val": None, + "Name": "value", + "x-image-meta-location": fake_location} encoded = http.encode_headers(headers) # Bug #1766235: According to RFC 8187, headers must be # encoded as 7-bit ASCII, so expect to see only displayable @@ -225,6 +229,8 @@ class TestClient(testtools.TestCase): self.assertNotIn("none-val", encoded) self.assertNotIn(b"none-val", encoded) self.assertEqual(b"value", encoded[b"Name"]) + # Bug #1788942: Colons in URL should not get percent-encoded + self.assertEqual(fake_location, encoded[b"x-image-meta-location"]) @mock.patch('keystoneauth1.adapter.Adapter.request') def test_http_duplicate_content_type_headers(self, mock_ksarq): diff --git a/glanceclient/tests/unit/test_shell.py b/glanceclient/tests/unit/test_shell.py index 0f15007..3027e46 100644 --- a/glanceclient/tests/unit/test_shell.py +++ b/glanceclient/tests/unit/test_shell.py @@ -965,7 +965,9 @@ class ShellTestRequests(testutils.TestCase): self.requests = self.useFixture(rm_fixture.Fixture()) self.requests.get('http://example.com/v2/images/%s/file' % id, headers=headers, raw=fake) - + self.requests.get('http://example.com/v2/images/%s' % id, + headers={'Content-type': 'application/json'}, + json=image_show_fixture) shell = openstack_shell.OpenStackImagesShell() argstr = ('--os-image-api-version 2 --os-auth-token faketoken ' '--os-image-url http://example.com ' diff --git a/glanceclient/tests/unit/v1/test_shell.py b/glanceclient/tests/unit/v1/test_shell.py index 95bbd07..a3bd29b 100644 --- a/glanceclient/tests/unit/v1/test_shell.py +++ b/glanceclient/tests/unit/v1/test_shell.py @@ -334,8 +334,8 @@ class ShellInvalidEndpointandParameterTest(utils.TestCase): e = self.assertRaises(exc.CommandError, self.run_command, '--os-image-api-version 1 image-create ' + origin + ' fake_src --container-format bare') - self.assertEqual('error: Must provide --disk-format when using ' - + origin + '.', e.message) + self.assertEqual('error: Must provide --disk-format when using ' + + origin + '.', e.message) @mock.patch('sys.stderr') def test_image_create_missing_container_format(self, __): @@ -536,8 +536,8 @@ class ShellStdinHandlingTests(testtools.TestCase): self._do_update() self.assertTrue( - 'data' not in self.collected_args[1] - or self.collected_args[1]['data'] is None + 'data' not in self.collected_args[1] or + self.collected_args[1]['data'] is None ) def test_image_update_opened_stdin(self): diff --git a/glanceclient/tests/unit/v2/fixtures.py b/glanceclient/tests/unit/v2/fixtures.py index 5a603c0..22e1ff7 100644 --- a/glanceclient/tests/unit/v2/fixtures.py +++ b/glanceclient/tests/unit/v2/fixtures.py @@ -14,6 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib + + UUID = "3fc2ba62-9a02-433e-b565-d493ffc69034" image_list_fixture = { @@ -65,7 +68,9 @@ image_show_fixture = { "tags": [], "updated_at": "2015-07-24T12:18:13Z", "virtual_size": "null", - "visibility": "shared" + "visibility": "shared", + "os_hash_algo": "sha384", + "os_hash_value": hashlib.sha384(b'DATA').hexdigest() } image_create_fixture = { diff --git a/glanceclient/tests/unit/v2/test_images.py b/glanceclient/tests/unit/v2/test_images.py index 23cbb43..99926de 100644 --- a/glanceclient/tests/unit/v2/test_images.py +++ b/glanceclient/tests/unit/v2/test_images.py @@ -14,6 +14,7 @@ # under the License. import errno +import hashlib import mock import testtools @@ -25,6 +26,10 @@ from glanceclient.v2 import images _CHKSUM = '93264c3edf5972c9f1cb309543d38a5c' _CHKSUM1 = '54264c3edf5972c9f1cb309453d38a46' +_HASHVAL = '54264c3edf93264c3edf5972c9f1cb309543d38a5c5972c9f1cb309453d38a46' +_HASHVAL1 = 'cb309543d38a5c5972c9f1cb309453d38a4654264c3edf93264c3edf5972c9f1' +_HASHBAD = '93264c3edf597254264c3edf5972c9f1cb309453d38a46c9f1cb309543d38a5c' + _TAG1 = 'power' _TAG2 = '64bit' @@ -193,7 +198,27 @@ data_fixtures = { 'A', ), }, - '/v2/images/66fb18d6-db27-11e1-a1eb-080027cbe205/file': { + '/v2/images/5cc4bebc-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + {}, + ), + }, + '/v2/images/headeronly-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': 'wrong' + }, + 'BB', + ), + }, + '/v2/images/headeronly-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + {}, + ), + }, + '/v2/images/chkonly-db27-11e1-a1eb-080027cbe205/file': { 'GET': ( { 'content-md5': 'wrong' @@ -201,7 +226,83 @@ data_fixtures = { 'BB', ), }, - '/v2/images/1b1c6366-dd57-11e1-af0f-02163e68b1d8/file': { + '/v2/images/chkonly-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + { + 'checksum': 'wrong', + }, + ), + }, + '/v2/images/multihash-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': 'wrong' + }, + 'BB', + ), + }, + '/v2/images/multihash-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + { + 'checksum': 'wrong', + 'os_hash_algo': 'md5', + 'os_hash_value': 'junk' + }, + ), + }, + '/v2/images/badalgo-db27-11e1-a1eb-080027cbe205/file': { + 'GET': ( + { + 'content-md5': hashlib.md5(b'BB').hexdigest() + }, + 'BB', + ), + }, + '/v2/images/badalgo-db27-11e1-a1eb-080027cbe205': { + 'GET': ( + {}, + { + 'checksum': hashlib.md5(b'BB').hexdigest(), + 'os_hash_algo': 'not_an_algo', + 'os_hash_value': 'whatever' + }, + ), + }, + '/v2/images/bad-multihash-value-good-checksum/file': { + 'GET': ( + { + 'content-md5': hashlib.md5(b'GOODCHECKSUM').hexdigest() + }, + 'GOODCHECKSUM', + ), + }, + '/v2/images/bad-multihash-value-good-checksum': { + 'GET': ( + {}, + { + 'checksum': hashlib.md5(b'GOODCHECKSUM').hexdigest(), + 'os_hash_algo': 'sha512', + 'os_hash_value': 'badmultihashvalue' + }, + ), + }, + '/v2/images/headeronly-dd57-11e1-af0f-02163e68b1d8/file': { + 'GET': ( + { + 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' + }, + 'CCC', + ), + }, + '/v2/images/headeronly-dd57-11e1-af0f-02163e68b1d8': { + 'GET': ( + {}, + {}, + ), + }, + '/v2/images/chkonly-dd57-11e1-af0f-02163e68b1d8/file': { 'GET': ( { 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' @@ -209,6 +310,32 @@ data_fixtures = { 'CCC', ), }, + '/v2/images/chkonly-dd57-11e1-af0f-02163e68b1d8': { + 'GET': ( + {}, + { + 'checksum': 'defb99e69a9f1f6e06f15006b1f166ae', + }, + ), + }, + '/v2/images/multihash-dd57-11e1-af0f-02163e68b1d8/file': { + 'GET': ( + { + 'content-md5': 'defb99e69a9f1f6e06f15006b1f166ae' + }, + 'CCC', + ), + }, + '/v2/images/multihash-dd57-11e1-af0f-02163e68b1d8': { + 'GET': ( + {}, + { + 'checksum': 'defb99e69a9f1f6e06f15006b1f166ae', + 'os_hash_algo': 'sha384', + 'os_hash_value': hashlib.sha384(b'CCC').hexdigest() + }, + ), + }, '/v2/images/87b634c1-f893-33c9-28a9-e5673c99239a/actions/reactivate': { 'POST': ({}, None) }, @@ -334,6 +461,41 @@ data_fixtures = { {'images': []}, ), }, + '/v2/images?limit=%d&os_hash_value=%s' % (images.DEFAULT_PAGE_SIZE, + _HASHVAL): { + 'GET': ( + {}, + {'images': [ + { + 'id': '3a4560a1-e585-443e-9b39-553b46ec92d1', + 'name': 'image-1', + } + ]}, + ), + }, + '/v2/images?limit=%d&os_hash_value=%s' % (images.DEFAULT_PAGE_SIZE, + _HASHVAL1): { + 'GET': ( + {}, + {'images': [ + { + 'id': '2a4560b2-e585-443e-9b39-553b46ec92d1', + 'name': 'image-1', + }, + { + 'id': '6f99bf80-2ee6-47cf-acfe-1f1fabb7e810', + 'name': 'image-2', + }, + ]}, + ), + }, + '/v2/images?limit=%d&os_hash_value=%s' % (images.DEFAULT_PAGE_SIZE, + _HASHBAD): { + 'GET': ( + {}, + {'images': []}, + ), + }, '/v2/images?limit=%d&tag=%s' % (images.DEFAULT_PAGE_SIZE, _TAG1): { 'GET': ( {}, @@ -631,6 +793,27 @@ class TestController(testtools.TestCase): images = self.controller.list(**filters) self.assertEqual(0, len(images)) + def test_list_images_for_hash_single_image(self): + fake_id = '3a4560a1-e585-443e-9b39-553b46ec92d1' + filters = {'filters': {'os_hash_value': _HASHVAL}} + images = self.controller.list(**filters) + self.assertEqual(1, len(images)) + self.assertEqual('%s' % fake_id, images[0].id) + + def test_list_images_for_hash_multiple_images(self): + fake_id1 = '2a4560b2-e585-443e-9b39-553b46ec92d1' + fake_id2 = '6f99bf80-2ee6-47cf-acfe-1f1fabb7e810' + filters = {'filters': {'os_hash_value': _HASHVAL1}} + images = self.controller.list(**filters) + self.assertEqual(2, len(images)) + self.assertEqual('%s' % fake_id1, images[0].id) + self.assertEqual('%s' % fake_id2, images[1].id) + + def test_list_images_for_wrong_hash(self): + filters = {'filters': {'os_hash_value': _HASHBAD}} + images = self.controller.list(**filters) + self.assertEqual(0, len(images)) + def test_list_images_for_bogus_owner(self): filters = {'filters': {'owner': _BOGUS_ID}} images = self.controller.list(**filters) @@ -846,12 +1029,24 @@ class TestController(testtools.TestCase): self.assertEqual('A', body) def test_data_with_wrong_checksum(self): - body = self.controller.data('66fb18d6-db27-11e1-a1eb-080027cbe205', + body = self.controller.data('headeronly-db27-11e1-a1eb-080027cbe205', do_checksum=False) body = ''.join([b for b in body]) self.assertEqual('BB', body) + body = self.controller.data('headeronly-db27-11e1-a1eb-080027cbe205') + try: + body = ''.join([b for b in body]) + self.fail('data did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected wrong' + self.assertIn(msg, str(e)) - body = self.controller.data('66fb18d6-db27-11e1-a1eb-080027cbe205') + body = self.controller.data('chkonly-db27-11e1-a1eb-080027cbe205', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) + body = self.controller.data('chkonly-db27-11e1-a1eb-080027cbe205') try: body = ''.join([b for b in body]) self.fail('data did not raise an error.') @@ -860,15 +1055,103 @@ class TestController(testtools.TestCase): msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected wrong' self.assertIn(msg, str(e)) - def test_data_with_checksum(self): - body = self.controller.data('1b1c6366-dd57-11e1-af0f-02163e68b1d8', + body = self.controller.data('multihash-db27-11e1-a1eb-080027cbe205', do_checksum=False) body = ''.join([b for b in body]) - self.assertEqual('CCC', body) + self.assertEqual('BB', body) + body = self.controller.data('multihash-db27-11e1-a1eb-080027cbe205') + try: + body = ''.join([b for b in body]) + self.fail('data did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'was 9d3d9048db16a7eee539e93e3618cbe7 expected junk' + self.assertIn(msg, str(e)) + + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) + try: + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205') + self.fail('bad os_hash_algo did not raise an error.') + except ValueError as e: + msg = 'unsupported hash type not_an_algo' + self.assertIn(msg, str(e)) + + def test_data_with_checksum(self): + for prefix in ['headeronly', 'chkonly', 'multihash']: + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8', + do_checksum=False) + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8') + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + def test_data_with_checksum_and_fallback(self): + # make sure the allow_md5_fallback option does not cause any + # incorrect behavior when fallback is not needed + for prefix in ['headeronly', 'chkonly', 'multihash']: + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8', + do_checksum=False, + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + body = self.controller.data(prefix + + '-dd57-11e1-af0f-02163e68b1d8', + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('CCC', body) + + def test_data_with_bad_hash_algo_and_fallback(self): + # shouldn't matter when do_checksum is False + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205', + do_checksum=False, + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) + + # default value for do_checksum is True + body = self.controller.data('badalgo-db27-11e1-a1eb-080027cbe205', + allow_md5_fallback=True) + body = ''.join([b for b in body]) + self.assertEqual('BB', body) - body = self.controller.data('1b1c6366-dd57-11e1-af0f-02163e68b1d8') + def test_neg_data_with_bad_hash_value_and_fallback_enabled(self): + # make sure download fails when good hash_algo but bad hash_value + # even when correct checksum is present regardless of + # allow_md5_fallback setting + body = self.controller.data('bad-multihash-value-good-checksum', + allow_md5_fallback=False) + try: + body = ''.join([b for b in body]) + self.fail('bad os_hash_value did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'expected badmultihashvalue' + self.assertIn(msg, str(e)) + + body = self.controller.data('bad-multihash-value-good-checksum', + allow_md5_fallback=True) + try: + body = ''.join([b for b in body]) + self.fail('bad os_hash_value did not raise an error.') + except IOError as e: + self.assertEqual(errno.EPIPE, e.errno) + msg = 'expected badmultihashvalue' + self.assertIn(msg, str(e)) + + # download should succeed when do_checksum is off, though + body = self.controller.data('bad-multihash-value-good-checksum', + do_checksum=False) body = ''.join([b for b in body]) - self.assertEqual('CCC', body) + self.assertEqual('GOODCHECKSUM', body) def test_image_import(self): uri = 'http://example.com/image.qcow' @@ -883,7 +1166,7 @@ class TestController(testtools.TestCase): def test_download_no_data(self): resp = utils.FakeResponse(headers={}, status_code=204) self.controller.controller.http_client.get = mock.Mock( - return_value=(resp, None)) + return_value=(resp, {})) self.controller.data('image_id') def test_download_forbidden(self): diff --git a/glanceclient/tests/unit/v2/test_shell_v2.py b/glanceclient/tests/unit/v2/test_shell_v2.py index 6eeca83..c43f606 100644 --- a/glanceclient/tests/unit/v2/test_shell_v2.py +++ b/glanceclient/tests/unit/v2/test_shell_v2.py @@ -57,7 +57,7 @@ def schema_args(schema_getter, omit=None): return original_schema_args(my_schema_getter, omit) utils.schema_args = schema_args -from glanceclient.v2 import shell as test_shell +from glanceclient.v2 import shell as test_shell # noqa # Return original decorator. utils.schema_args = original_schema_args @@ -95,7 +95,7 @@ class ShellV2Test(testtools.TestCase): # dict directly, it throws an AttributeError. class Args(object): def __init__(self, entries): - self.backend = None + self.store = None self.__dict__.update(entries) return Args(args) @@ -264,6 +264,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc', 'asc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -286,6 +288,86 @@ class ShellV2Test(testtools.TestCase): filters=exp_img_filters) utils.print_list.assert_called_once_with({}, ['ID', 'Name']) + def test_do_image_list_verbose(self): + input = { + 'limit': None, + 'page_size': 18, + 'visibility': True, + 'member_status': 'Fake', + 'owner': 'test', + 'checksum': 'fake_checksum', + 'tag': 'fake tag', + 'properties': [], + 'sort_key': ['name', 'id'], + 'sort_dir': ['desc', 'asc'], + 'sort': None, + 'verbose': True, + 'include_stores': False, + 'os_hash_value': None, + 'os_hidden': False + } + args = self._make_args(input) + with mock.patch.object(self.gc.images, 'list') as mocked_list: + mocked_list.return_value = {} + + test_shell.do_image_list(self.gc, args) + utils.print_list.assert_called_once_with( + {}, ['ID', 'Name', 'Disk_format', 'Container_format', + 'Size', 'Status', 'Owner']) + + def test_do_image_list_with_include_stores_true(self): + input = { + 'limit': None, + 'page_size': 18, + 'visibility': True, + 'member_status': 'Fake', + 'owner': 'test', + 'checksum': 'fake_checksum', + 'tag': 'fake tag', + 'properties': [], + 'sort_key': ['name', 'id'], + 'sort_dir': ['desc', 'asc'], + 'sort': None, + 'verbose': False, + 'include_stores': True, + 'os_hash_value': None, + 'os_hidden': False + } + args = self._make_args(input) + with mock.patch.object(self.gc.images, 'list') as mocked_list: + mocked_list.return_value = {} + + test_shell.do_image_list(self.gc, args) + utils.print_list.assert_called_once_with( + {}, ['ID', 'Name', 'Stores']) + + def test_do_image_list_verbose_with_include_stores_true(self): + input = { + 'limit': None, + 'page_size': 18, + 'visibility': True, + 'member_status': 'Fake', + 'owner': 'test', + 'checksum': 'fake_checksum', + 'tag': 'fake tag', + 'properties': [], + 'sort_key': ['name', 'id'], + 'sort_dir': ['desc', 'asc'], + 'sort': None, + 'verbose': True, + 'include_stores': True, + 'os_hash_value': None, + 'os_hidden': False + } + args = self._make_args(input) + with mock.patch.object(self.gc.images, 'list') as mocked_list: + mocked_list.return_value = {} + + test_shell.do_image_list(self.gc, args) + utils.print_list.assert_called_once_with( + {}, ['ID', 'Name', 'Disk_format', 'Container_format', + 'Size', 'Status', 'Owner', 'Stores']) + def test_do_image_list_with_hidden_true(self): input = { 'limit': None, @@ -300,6 +382,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc', 'asc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': True } args = self._make_args(input) @@ -336,6 +420,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -372,6 +458,8 @@ class ShellV2Test(testtools.TestCase): 'sort_key': [], 'sort_dir': [], 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -408,6 +496,8 @@ class ShellV2Test(testtools.TestCase): 'sort_dir': ['desc'], 'sort': None, 'verbose': False, + 'include_stores': False, + 'os_hash_value': None, 'os_hidden': False } args = self._make_args(input) @@ -692,9 +782,9 @@ class ShellV2Test(testtools.TestCase): @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) - def test_neg_do_image_create_no_file_and_stdin_with_backend( + def test_neg_do_image_create_no_file_and_stdin_with_store( self, mock_stdin, mock_access, mock_utils_exit): - expected_msg = ('--backend option should only be provided with --file ' + expected_msg = ('--store option should only be provided with --file ' 'option or stdin.') mock_utils_exit.side_effect = self._mock_utils_exit mock_stdin.isatty = lambda: True @@ -702,7 +792,7 @@ class ShellV2Test(testtools.TestCase): args = self._make_args({'name': 'IMG-01', 'property': ['myprop=myval'], 'file': None, - 'backend': 'file1', + 'store': 'file1', 'container_format': 'bare', 'disk_format': 'qcow2'}) @@ -715,16 +805,16 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') - def test_neg_do_image_create_invalid_backend( + def test_neg_do_image_create_invalid_store( self, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info " "command.") mock_utils_exit.side_effect = self._mock_utils_exit args = self._make_args({'name': 'IMG-01', 'property': ['myprop=myval'], 'file': "somefile.txt", - 'backend': 'dummy', + 'store': 'dummy', 'container_format': 'bare', 'disk_format': 'qcow2'}) @@ -752,7 +842,7 @@ class ShellV2Test(testtools.TestCase): import_info_response = {'import-methods': { 'type': 'array', 'description': 'Import methods available.', - 'value': ['glance-direct', 'web-download']}} + 'value': ['glance-direct', 'web-download', 'copy-image']}} def _mock_utils_exit(self, msg): sys.exit(msg) @@ -781,15 +871,120 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_copy_image( + self, mock_utils_exit): + expected_msg = ("Import method 'copy-image' cannot be used " + "while creating the image.") + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'copy-image'}) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_stores_all_stores_specified( + self, mock_utils_exit): + expected_msg = ('Only one of --store, --stores and --all-stores can ' + 'be provided') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'stores': 'file1,file2', 'os_all_stores': True, + 'file': 'some.mufile', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + @mock.patch('sys.stdin', autospec=True) + def test_neg_image_create_via_import_stores_without_file( + self, mock_stdin, mock_utils_exit): + expected_msg = ('--stores option should only be provided with --file ' + 'option or stdin for the glance-direct import method.') + mock_utils_exit.side_effect = self._mock_utils_exit + mock_stdin.isatty = lambda: True + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'stores': 'file1,file2', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + with mock.patch.object(self.gc.images, + 'get_stores_info') as mocked_stores_info: + mocked_stores_info.return_value = self.stores_info_response + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + @mock.patch('sys.stdin', autospec=True) + def test_neg_image_create_via_import_all_stores_without_file( + self, mock_stdin, mock_utils_exit): + expected_msg = ('--all-stores option should only be provided with ' + '--file option or stdin for the glance-direct import ' + 'method.') + mock_utils_exit.side_effect = self._mock_utils_exit + mock_stdin.isatty = lambda: True + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'glance-direct', + 'os_all_stores': True, + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) - def test_neg_image_create_via_import_no_file_and_stdin_with_backend( + def test_neg_image_create_via_import_no_file_and_stdin_with_store( self, mock_stdin, mock_access, mock_utils_exit): - expected_msg = ('--backend option should only be provided with --file ' + expected_msg = ('--store option should only be provided with --file ' 'option or stdin for the glance-direct import method.') my_args = self.base_args.copy() my_args['import_method'] = 'glance-direct' - my_args['backend'] = 'file1' + my_args['store'] = 'file1' args = self._make_args(my_args) mock_stdin.isatty = lambda: True @@ -810,13 +1005,13 @@ class ShellV2Test(testtools.TestCase): @mock.patch('glanceclient.common.utils.exit') @mock.patch('sys.stdin', autospec=True) - def test_neg_image_create_via_import_no_uri_with_backend( + def test_neg_image_create_via_import_no_uri_with_store( self, mock_stdin, mock_utils_exit): - expected_msg = ('--backend option should only be provided with --uri ' + expected_msg = ('--store option should only be provided with --uri ' 'option for the web-download import method.') my_args = self.base_args.copy() my_args['import_method'] = 'web-download' - my_args['backend'] = 'file1' + my_args['store'] = 'file1' args = self._make_args(my_args) mock_utils_exit.side_effect = self._mock_utils_exit with mock.patch.object(self.gc.images, @@ -835,14 +1030,14 @@ class ShellV2Test(testtools.TestCase): @mock.patch('glanceclient.common.utils.exit') @mock.patch('os.access') @mock.patch('sys.stdin', autospec=True) - def test_neg_image_create_via_import_invalid_backend( + def test_neg_image_create_via_import_invalid_store( self, mock_stdin, mock_access, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info" " command.") my_args = self.base_args.copy() my_args['import_method'] = 'glance-direct' - my_args['backend'] = 'dummy' + my_args['store'] = 'dummy' args = self._make_args(my_args) mock_stdin.isatty = lambda: True @@ -994,6 +1189,60 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_stores_without_uri( + self, mock_utils_exit): + expected_msg = ('--stores option should only be provided with --uri ' + 'option for the web-download import method.') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'web-download', + 'stores': 'file1,file2', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + with mock.patch.object(self.gc.images, + 'get_stores_info') as mocked_stores_info: + mocked_stores_info.return_value = self.stores_info_response + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_create_via_import_all_stores_without_uri( + self, mock_utils_exit): + expected_msg = ('--all-stores option should only be provided with ' + '--uri option for the web-download import ' + 'method.') + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'web-download', + 'os_all_stores': True, + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_create_via_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') @mock.patch('sys.stdin', autospec=True) def test_neg_image_create_via_import_web_download_no_uri_with_file( self, mock_stdin, mock_utils_exit): @@ -1428,18 +1677,25 @@ class ShellV2Test(testtools.TestCase): def test_do_location_add(self): gc = self.gc - loc = {'url': 'http://foo.com/', 'metadata': {'foo': 'bar'}} - args = self._make_args({'id': 'pass', - 'url': loc['url'], - 'metadata': json.dumps(loc['metadata'])}) + loc = {'url': 'http://foo.com/', + 'metadata': {'foo': 'bar'}, + 'validation_data': {'checksum': 'csum', + 'os_hash_algo': 'algo', + 'os_hash_value': 'value'}} + args = {'id': 'pass', + 'url': loc['url'], + 'metadata': json.dumps(loc['metadata']), + 'checksum': 'csum', + 'hash_algo': 'algo', + 'hash_value': 'value'} with mock.patch.object(gc.images, 'add_location') as mocked_addloc: expect_image = {'id': 'pass', 'locations': [loc]} mocked_addloc.return_value = expect_image - test_shell.do_location_add(self.gc, args) - mocked_addloc.assert_called_once_with('pass', - loc['url'], - loc['metadata']) + test_shell.do_location_add(self.gc, self._make_args(args)) + mocked_addloc.assert_called_once_with( + 'pass', loc['url'], loc['metadata'], + validation_data=loc['validation_data']) utils.print_dict.assert_called_once_with(expect_image) def test_do_location_delete(self): @@ -1479,15 +1735,15 @@ class ShellV2Test(testtools.TestCase): backend=None) @mock.patch('glanceclient.common.utils.exit') - def test_image_upload_invalid_backend(self, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + def test_image_upload_invalid_store(self, mock_utils_exit): + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info " "command.") mock_utils_exit.side_effect = self._mock_utils_exit args = self._make_args( {'id': 'IMG-01', 'file': 'test', 'size': 1024, 'progress': False, - 'backend': 'dummy'}) + 'store': 'dummy'}) with mock.patch.object(self.gc.images, 'get_stores_info') as mock_stores_info: @@ -1646,15 +1902,15 @@ class ShellV2Test(testtools.TestCase): mock_utils_exit.assert_called_once_with(expected_msg) @mock.patch('glanceclient.common.utils.exit') - def test_image_import_invalid_backend(self, mock_utils_exit): - expected_msg = ("Backend 'dummy' is not valid for this cloud. " + def test_image_import_invalid_store(self, mock_utils_exit): + expected_msg = ("Store 'dummy' is not valid for this cloud. " "Valid values can be retrieved with stores-info " "command.") mock_utils_exit.side_effect = self._mock_utils_exit args = self._make_args( {'id': 'IMG-01', 'import_method': 'glance-direct', 'uri': None, - 'backend': 'dummy'}) + 'store': 'dummy'}) with mock.patch.object(self.gc.images, 'get') as mocked_get: with mock.patch.object(self.gc.images, @@ -1688,7 +1944,8 @@ class ShellV2Test(testtools.TestCase): mock_import.return_value = None test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( - 'IMG-01', 'glance-direct', None, backend=None) + 'IMG-01', 'glance-direct', None, backend=None, + all_stores=None, allow_failure=True, stores=None) def test_image_import_web_download(self): args = self._make_args( @@ -1706,7 +1963,9 @@ class ShellV2Test(testtools.TestCase): test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( 'IMG-01', 'web-download', - 'http://example.com/image.qcow', backend=None) + 'http://example.com/image.qcow', + all_stores=None, allow_failure=True, + backend=None, stores=None) @mock.patch('glanceclient.common.utils.print_image') def test_image_import_no_print_image(self, mocked_utils_print_image): @@ -1724,12 +1983,112 @@ class ShellV2Test(testtools.TestCase): mock_import.return_value = None test_shell.do_image_import(self.gc, args) mock_import.assert_called_once_with( - 'IMG-02', 'glance-direct', None, backend=None) + 'IMG-02', 'glance-direct', None, stores=None, + all_stores=None, allow_failure=True, backend=None) mocked_utils_print_image.assert_not_called() + @mock.patch('glanceclient.common.utils.print_image') + @mock.patch('glanceclient.v2.shell._validate_backend') + def test_image_import_multiple_stores(self, mocked_utils_print_image, + msvb): + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'glance-direct', + 'from_create': False, 'stores': 'site1,site2'}) + with mock.patch.object(self.gc.images, 'image_import') as mock_import: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_get.return_value = {'status': 'uploading', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + mock_import.return_value = None + test_shell.do_image_import(self.gc, args) + mock_import.assert_called_once_with( + 'IMG-02', 'glance-direct', None, all_stores=None, + allow_failure=True, stores=['site1', 'site2'], + backend=None) + + @mock.patch('glanceclient.common.utils.print_image') + @mock.patch('glanceclient.v2.shell._validate_backend') + def test_image_import_copy_image(self, mocked_utils_print_image, + msvb): + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'copy-image', + 'from_create': False, 'stores': 'file1,file2'}) + with mock.patch.object(self.gc.images, 'image_import') as mock_import: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_get.return_value = {'status': 'active', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + mock_import.return_value = None + test_shell.do_image_import(self.gc, args) + mock_import.assert_called_once_with( + 'IMG-02', 'copy-image', None, all_stores=None, + allow_failure=True, stores=['file1', 'file2'], + backend=None) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_import_copy_image_not_active( + self, mock_utils_exit): + expected_msg = ("The 'copy-image' import method can only be used on " + "an image with status 'active'.") + mock_utils_exit.side_effect = self._mock_utils_exit + args = self._make_args( + {'id': 'IMG-02', 'uri': None, 'import_method': 'copy-image', + 'disk_format': 'raw', + 'container_format': 'bare', + 'from_create': False, 'stores': 'file1,file2'}) + with mock.patch.object( + self.gc.images, + 'get_stores_info') as mocked_stores_info: + with mock.patch.object(self.gc.images, 'get') as mocked_get: + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + + mocked_stores_info.return_value = self.stores_info_response + mocked_get.return_value = {'status': 'uploading', + 'container_format': 'bare', + 'disk_format': 'raw'} + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + + @mock.patch('glanceclient.common.utils.exit') + def test_neg_image_import_stores_all_stores_not_specified( + self, mock_utils_exit): + expected_msg = ("Provide either --stores or --all-stores for " + "'copy-image' import method.") + mock_utils_exit.side_effect = self._mock_utils_exit + my_args = self.base_args.copy() + my_args.update( + {'id': 'IMG-01', 'import_method': 'copy-image', + 'disk_format': 'raw', + 'container_format': 'bare', + }) + args = self._make_args(my_args) + + with mock.patch.object(self.gc.images, + 'get_import_info') as mocked_info: + mocked_info.return_value = self.import_info_response + try: + test_shell.do_image_import(self.gc, args) + self.fail("utils.exit should have been called") + except SystemExit: + pass + mock_utils_exit.assert_called_once_with(expected_msg) + def test_image_download(self): args = self._make_args( - {'id': 'IMG-01', 'file': 'test', 'progress': True}) + {'id': 'IMG-01', 'file': 'test', 'progress': True, + 'allow_md5_fallback': False}) with mock.patch.object(self.gc.images, 'data') as mocked_data, \ mock.patch.object(utils, '_extract_request_id'): @@ -1737,14 +2096,27 @@ class ShellV2Test(testtools.TestCase): [c for c in 'abcdef']) test_shell.do_image_download(self.gc, args) - mocked_data.assert_called_once_with('IMG-01') + mocked_data.assert_called_once_with('IMG-01', + allow_md5_fallback=False) + + # check that non-default value is being passed correctly + args.allow_md5_fallback = True + with mock.patch.object(self.gc.images, 'data') as mocked_data, \ + mock.patch.object(utils, '_extract_request_id'): + mocked_data.return_value = utils.RequestIdProxy( + [c for c in 'abcdef']) + + test_shell.do_image_download(self.gc, args) + mocked_data.assert_called_once_with('IMG-01', + allow_md5_fallback=True) @mock.patch.object(utils, 'exit') @mock.patch('sys.stdout', autospec=True) def test_image_download_no_file_arg(self, mocked_stdout, mocked_utils_exit): # Indicate that no file name was given as command line argument - args = self._make_args({'id': '1234', 'file': None, 'progress': False}) + args = self._make_args({'id': '1234', 'file': None, 'progress': False, + 'allow_md5_fallback': False}) # Indicate that no file is specified for output redirection mocked_stdout.isatty = lambda: True test_shell.do_image_download(self.gc, args) @@ -1794,6 +2166,29 @@ class ShellV2Test(testtools.TestCase): mocked_utils_exit.assert_called_once_with() @mock.patch.object(utils, 'exit') + def test_do_image_delete_from_store_not_found(self, mocked_utils_exit): + args = argparse.Namespace(id='image1', store='store1') + with mock.patch.object(self.gc.images, + 'delete_from_store') as mocked_delete: + mocked_delete.side_effect = exc.HTTPNotFound + + test_shell.do_stores_delete(self.gc, args) + + self.assertEqual(1, mocked_delete.call_count) + mocked_utils_exit.assert_called_once_with('Multi Backend support ' + 'is not enabled or ' + 'Image/store not found.') + + def test_do_image_delete_from_store(self): + args = argparse.Namespace(id='image1', store='store1') + with mock.patch.object(self.gc.images, + 'delete_from_store') as mocked_delete: + test_shell.do_stores_delete(self.gc, args) + + mocked_delete.assert_called_once_with('store1', + 'image1') + + @mock.patch.object(utils, 'exit') @mock.patch.object(utils, 'print_err') def test_do_image_delete_with_forbidden_ids(self, mocked_print_err, mocked_utils_exit): @@ -1835,7 +2230,8 @@ class ShellV2Test(testtools.TestCase): def test_do_image_download_with_forbidden_id(self, mocked_print_err, mocked_stdout): args = self._make_args({'id': 'IMG-01', 'file': None, - 'progress': False}) + 'progress': False, + 'allow_md5_fallback': False}) mocked_stdout.isatty = lambda: False with mock.patch.object(self.gc.images, 'data') as mocked_data: mocked_data.side_effect = exc.HTTPForbidden @@ -1852,7 +2248,8 @@ class ShellV2Test(testtools.TestCase): @mock.patch.object(utils, 'print_err') def test_do_image_download_with_500(self, mocked_print_err, mocked_stdout): args = self._make_args({'id': 'IMG-01', 'file': None, - 'progress': False}) + 'progress': False, + 'allow_md5_fallback': False}) mocked_stdout.isatty = lambda: False with mock.patch.object(self.gc.images, 'data') as mocked_data: mocked_data.side_effect = exc.HTTPInternalServerError diff --git a/glanceclient/v1/shell.py b/glanceclient/v1/shell.py index fff7490..8a4d29d 100644 --- a/glanceclient/v1/shell.py +++ b/glanceclient/v1/shell.py @@ -30,7 +30,7 @@ import glanceclient.v1.images CONTAINER_FORMATS = ('Acceptable formats: ami, ari, aki, bare, ovf, ova,' 'docker.') -DISK_FORMATS = ('Acceptable formats: ami, ari, aki, vhd, vdhx, vmdk, raw, ' +DISK_FORMATS = ('Acceptable formats: ami, ari, aki, vhd, vhdx, vmdk, raw, ' 'qcow2, vdi, iso, and ploop.') DATA_FIELDS = ('location', 'copy_from', 'file') @@ -45,11 +45,11 @@ _bool_strict = functools.partial(strutils.bool_from_string, strict=True) help='Filter images to those that changed since the given time' ', which will include the deleted images.') @utils.arg('--container-format', metavar='<CONTAINER_FORMAT>', - help='Filter images to those that have this container format. ' - + CONTAINER_FORMATS) + help='Filter images to those that have this container format. ' + + CONTAINER_FORMATS) @utils.arg('--disk-format', metavar='<DISK_FORMAT>', - help='Filter images to those that have this disk format. ' - + DISK_FORMATS) + help='Filter images to those that have this disk format. ' + + DISK_FORMATS) @utils.arg('--size-min', metavar='<SIZE>', type=int, help='Filter images to those with a size greater than this.') @utils.arg('--size-max', metavar='<SIZE>', type=int, diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index be804a2..1e8e621 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib import json from oslo_utils import encodeutils from requests import codes @@ -197,13 +198,39 @@ class Controller(object): return self._get(image_id) @utils.add_req_id_to_object() - def data(self, image_id, do_checksum=True): + def data(self, image_id, do_checksum=True, allow_md5_fallback=False): """Retrieve data of an image. - :param image_id: ID of the image to download. - :param do_checksum: Enable/disable checksum validation. - :returns: An iterable body or None + When do_checksum is enabled, validation proceeds as follows: + + 1. if the image has a 'os_hash_value' property, the algorithm + specified in the image's 'os_hash_algo' property will be used + to validate against the 'os_hash_value' value. If the + specified hash algorithm is not available AND allow_md5_fallback + is True, then continue to step #2 + 2. else if the image has a checksum property, MD5 is used to + validate against the 'checksum' value + 3. else if the download response has a 'content-md5' header, MD5 + is used to validate against the header value + 4. if none of 1-3 obtain, the data is **not validated** (this is + compatible with legacy behavior) + + :param image_id: ID of the image to download + :param do_checksum: Enable/disable checksum validation + :param allow_md5_fallback: + Use the MD5 checksum for validation if the algorithm specified by + the image's 'os_hash_algo' property is not available + :returns: An iterable body or ``None`` """ + if do_checksum: + # doing this first to prevent race condition if image record + # is deleted during the image download + url = '/v2/images/%s' % image_id + resp, image_meta = self.http_client.get(url) + meta_checksum = image_meta.get('checksum', None) + meta_hash_value = image_meta.get('os_hash_value', None) + meta_hash_algo = image_meta.get('os_hash_algo', None) + url = '/v2/images/%s/file' % image_id resp, body = self.http_client.get(url) if resp.status_code == codes.no_content: @@ -212,8 +239,32 @@ class Controller(object): checksum = resp.headers.get('content-md5', None) content_length = int(resp.headers.get('content-length', 0)) - if do_checksum and checksum is not None: - body = utils.integrity_iter(body, checksum) + check_md5sum = do_checksum + if do_checksum and meta_hash_value is not None: + try: + hasher = hashlib.new(str(meta_hash_algo)) + body = utils.serious_integrity_iter(body, + hasher, + meta_hash_value) + check_md5sum = False + except ValueError as ve: + if (str(ve).startswith('unsupported hash type') and + allow_md5_fallback): + check_md5sum = True + else: + raise + + if do_checksum and check_md5sum: + if meta_checksum is not None: + body = utils.integrity_iter(body, meta_checksum) + elif checksum is not None: + body = utils.integrity_iter(body, checksum) + else: + # NOTE(rosmaita): this preserves legacy behavior to return the + # image data when checksumming is requested but there's no + # 'content-md5' header in the response. Just want to make it + # clear that we're doing this on purpose. + pass return utils.IterableWithLength(body, content_length), resp @@ -252,6 +303,14 @@ class Controller(object): return body, resp @utils.add_req_id_to_object() + def delete_from_store(self, store_id, image_id): + """Delete image data from specific store.""" + url = ('/v2/stores/%(store)s/%(image)s' % {'store': store_id, + 'image': image_id}) + resp, body = self.http_client.delete(url) + return body, resp + + @utils.add_req_id_to_object() def stage(self, image_id, image_data, image_size=None): """Upload the data to image staging. @@ -267,13 +326,22 @@ class Controller(object): @utils.add_req_id_to_object() def image_import(self, image_id, method='glance-direct', uri=None, - backend=None): + backend=None, stores=None, allow_failure=True, + all_stores=None): """Import Image via method.""" headers = {} url = '/v2/images/%s/import' % image_id data = {'method': {'name': method}} + if stores: + data['stores'] = stores + if allow_failure: + data['all_stores_must_succeed'] = 'false' if backend is not None: headers['x-image-meta-store'] = backend + if all_stores: + data['all_stores'] = 'true' + if allow_failure: + data['all_stores_must_succeed'] = 'false' if uri: if method == 'web-download': @@ -381,7 +449,7 @@ class Controller(object): data=json.dumps(patch_body)) return (resp, body), resp - def add_location(self, image_id, url, metadata): + def add_location(self, image_id, url, metadata, validation_data=None): """Add a new location entry to an image's list of locations. It is an error to add a URL that is already present in the list of @@ -390,10 +458,13 @@ class Controller(object): :param image_id: ID of image to which the location is to be added. :param url: URL of the location to add. :param metadata: Metadata associated with the location. + :param validation_data: Validation data for the image. :returns: The updated image """ add_patch = [{'op': 'add', 'path': '/locations/-', 'value': {'url': url, 'metadata': metadata}}] + if validation_data: + add_patch[0]['value']['validation_data'] = validation_data response = self._send_image_update_request(image_id, add_patch) # Get request id from the above update request and pass the same to # following get request diff --git a/glanceclient/v2/shell.py b/glanceclient/v2/shell.py index aaa85bb..b4dc811 100644 --- a/glanceclient/v2/shell.py +++ b/glanceclient/v2/shell.py @@ -71,8 +71,8 @@ def get_image_schema(): 'passed to the client via stdin.')) @utils.arg('--progress', action='store_true', default=False, help=_('Show upload progress bar.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') @utils.on_data_require_fields(DATA_FIELDS) def do_image_create(gc, args): @@ -90,12 +90,12 @@ def do_image_create(gc, args): key, value = datum.split('=', 1) fields[key] = value - backend = args.backend + backend = args.store file_name = fields.pop('file', None) using_stdin = not sys.stdin.isatty() - if args.backend and not (file_name or using_stdin): - utils.exit("--backend option should only be provided with --file " + if args.store and not (file_name or using_stdin): + utils.exit("--store option should only be provided with --file " "option or stdin.") if backend: @@ -108,7 +108,7 @@ def do_image_create(gc, args): image = gc.images.create(**fields) try: if utils.get_data_file(args) is not None: - backend = fields.get('backend', None) + backend = fields.get('store', None) args.id = image['id'] args.size = None do_image_upload(gc, args) @@ -147,9 +147,31 @@ def do_image_create(gc, args): 'record if no import-method and no data is supplied')) @utils.arg('--uri', metavar='<IMAGE_URL>', default=None, help=_('URI to download the external image.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') +@utils.arg('--stores', metavar='<STORES>', + default=utils.env('OS_IMAGE_STORES', default=None), + help=_('Stores to upload image to if multi-stores import ' + 'available. Comma separated list. Available stores can be ' + 'listed with "stores-info" call.')) +@utils.arg('--all-stores', type=strutils.bool_from_string, + metavar='[True|False]', + default=None, + dest='os_all_stores', + help=_('"all-stores" can be ued instead of "stores"-list to ' + 'indicate that image should be imported into all available ' + 'stores.')) +@utils.arg('--allow-failure', type=strutils.bool_from_string, + metavar='[True|False]', + dest='os_allow_failure', + default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True), + help=_('Indicator if all stores listed (or available) must ' + 'succeed. "True" by default meaning that we allow some ' + 'stores to fail and the status can be monitored from the ' + 'image metadata. If this is set to "False" the import will ' + 'be reverted should any of the uploads fail. Only usable ' + 'with "stores" or "all-stores".')) @utils.on_data_require_fields(DATA_FIELDS) def do_image_create_via_import(gc, args): """EXPERIMENTAL: Create a new image via image import. @@ -188,6 +210,10 @@ def do_image_create_via_import(gc, args): if args.import_method is None and (file_name or using_stdin): args.import_method = 'glance-direct' + if args.import_method == 'copy-image': + utils.exit("Import method 'copy-image' cannot be used " + "while creating the image.") + # determine whether the requested import method is valid import_methods = gc.images.get_import_info().get('import-methods') if args.import_method and args.import_method not in import_methods.get( @@ -198,9 +224,21 @@ def do_image_create_via_import(gc, args): # determine if backend is valid backend = None - if args.backend: - backend = args.backend + stores = getattr(args, "stores", None) + all_stores = getattr(args, "os_all_stores", None) + + if (args.store and (stores or all_stores)) or (stores and all_stores): + utils.exit("Only one of --store, --stores and --all-stores can be " + "provided") + elif args.store: + backend = args.store + # determine if backend is valid _validate_backend(backend, gc) + elif stores: + stores = str(stores).split(',') + for store in stores: + # determine if backend is valid + _validate_backend(store, gc) # make sure we have all and only correct inputs for the requested method if args.import_method is None: @@ -209,8 +247,16 @@ def do_image_create_via_import(gc, args): "method.") if args.import_method == 'glance-direct': if backend and not (file_name or using_stdin): - utils.exit("--backend option should only be provided with --file " + utils.exit("--store option should only be provided with --file " + "option or stdin for the glance-direct import method.") + if stores and not (file_name or using_stdin): + utils.exit("--stores option should only be provided with --file " "option or stdin for the glance-direct import method.") + if all_stores and not (file_name or using_stdin): + utils.exit("--all-stores option should only be provided with " + "--file option or stdin for the glance-direct import " + "method.") + if args.uri: utils.exit("You cannot specify a --uri with the glance-direct " "import method.") @@ -225,8 +271,14 @@ def do_image_create_via_import(gc, args): "for the glance-direct import method.") if args.import_method == 'web-download': if backend and not args.uri: - utils.exit("--backend option should only be provided with --uri " + utils.exit("--store option should only be provided with --uri " "option for the web-download import method.") + if stores and not args.uri: + utils.exit("--stores option should only be provided with --uri " + "option for the web-download import method.") + if all_stores and not args.uri: + utils.exit("--all-stores option should only be provided with " + "--uri option for the web-download import method.") if not args.uri: utils.exit("URI is required for web-download import method. " "Please use '--uri <uri>'.") @@ -246,6 +298,7 @@ def do_image_create_via_import(gc, args): args.size = None do_image_stage(gc, args) args.from_create = True + args.stores = stores do_image_import(gc, args) image = gc.images.get(args.id) finally: @@ -267,7 +320,7 @@ def _validate_backend(backend, gc): break if not valid_backend: - utils.exit("Backend '%s' is not valid for this cloud. Valid " + utils.exit("Store '%s' is not valid for this cloud. Valid " "values can be retrieved with stores-info command." % backend) @@ -327,6 +380,9 @@ def do_image_update(gc, args): action='append', dest='properties', default=[]) @utils.arg('--checksum', metavar='<CHECKSUM>', help=_('Displays images that match the MD5 checksum.')) +@utils.arg('--hash', dest='os_hash_value', default=None, + metavar='<HASH_VALUE>', + help=_('Displays images that match the specified hash value.')) @utils.arg('--tag', metavar='<TAG>', action='append', help=_("Filter images by a user-defined tag.")) @utils.arg('--sort-key', default=[], action='append', @@ -348,10 +404,17 @@ def do_image_update(gc, args): const=True, nargs='?', help="Filters results by hidden status. Default=None.") +@utils.arg('--include-stores', + metavar='[True|False]', + default=None, + type=strutils.bool_from_string, + const=True, + nargs='?', + help="Print backend store id.") def do_image_list(gc, args): """List images you can access.""" filter_keys = ['visibility', 'member_status', 'owner', 'checksum', 'tag', - 'os_hidden'] + 'os_hidden', 'os_hash_value'] filter_items = [(key, getattr(args, key)) for key in filter_keys] if args.properties: @@ -384,6 +447,9 @@ def do_image_list(gc, args): columns += ['Disk_format', 'Container_format', 'Size', 'Status', 'Owner'] + if args.include_stores: + columns += ['Stores'] + images = gc.images.list(**kwargs) utils.print_list(images, columns) @@ -490,6 +556,35 @@ def do_stores_info(gc, args): utils.print_dict(stores_info) +@utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to update.')) +@utils.arg('--store', metavar='<STORE_ID>', required=True, + help=_('Store to delete image from.')) +def do_stores_delete(gc, args): + """Delete image from specific store.""" + try: + gc.images.delete_from_store(args.store, args.id) + except exc.HTTPNotFound: + utils.exit('Multi Backend support is not enabled or Image/store not ' + 'found.') + except (exc.HTTPForbidden, exc.HTTPException) as e: + msg = ("Unable to delete image '%s' from store '%s'. (%s)" % ( + args.id, + args.store, + e)) + utils.exit(msg) + + +@utils.arg('--allow-md5-fallback', action='store_true', + default=utils.env('OS_IMAGE_ALLOW_MD5_FALLBACK', default=False), + help=_('If os_hash_algo and os_hash_value properties are available ' + 'on the image, they will be used to validate the downloaded ' + 'image data. If the indicated secure hash algorithm is not ' + 'available on the client, the download will fail. Use this ' + 'flag to indicate that in such a case the legacy MD5 image ' + 'checksum should be used to validate the downloaded data. ' + 'You can also set the environment variable ' + 'OS_IMAGE_ALLOW_MD5_FALLBACK to any value to activate this ' + 'option.')) @utils.arg('--file', metavar='<FILE>', help=_('Local file to save downloaded image data to. ' 'If this is not specified and there is no redirection ' @@ -506,7 +601,8 @@ def do_image_download(gc, args): utils.exit(msg) try: - body = gc.images.data(args.id) + body = gc.images.data(args.id, + allow_md5_fallback=args.allow_md5_fallback) except (exc.HTTPForbidden, exc.HTTPException) as e: msg = "Unable to download image '%s'. (%s)" % (args.id, e) utils.exit(msg) @@ -534,14 +630,14 @@ def do_image_download(gc, args): help=_('Show upload progress bar.')) @utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to upload data to.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') def do_image_upload(gc, args): """Upload data for a specific image.""" backend = None - if args.backend: - backend = args.backend + if args.store: + backend = args.store # determine if backend is valid _validate_backend(backend, gc) @@ -589,22 +685,59 @@ def do_image_stage(gc, args): help=_('URI to download the external image.')) @utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to import.')) -@utils.arg('--backend', metavar='<STORE>', - default=utils.env('OS_IMAGE_BACKEND', default=None), +@utils.arg('--store', metavar='<STORE>', + default=utils.env('OS_IMAGE_STORE', default=None), help='Backend store to upload image to.') +@utils.arg('--stores', metavar='<STORES>', + default=utils.env('OS_IMAGE_STORES', default=None), + help='Stores to upload image to if multi-stores import available.') +@utils.arg('--all-stores', type=strutils.bool_from_string, + metavar='[True|False]', + default=None, + dest='os_all_stores', + help=_('"all-stores" can be ued instead of "stores"-list to ' + 'indicate that image should be imported all available ' + 'stores.')) +@utils.arg('--allow-failure', type=strutils.bool_from_string, + metavar='[True|False]', + dest='os_allow_failure', + default=utils.env('OS_IMAGE_ALLOW_FAILURE', default=True), + help=_('Indicator if all stores listed (or available) must ' + 'succeed. "True" by default meaning that we allow some ' + 'stores to fail and the status can be monitored from the ' + 'image metadata. If this is set to "False" the import will ' + 'be reverted should any of the uploads fail. Only usable ' + 'with "stores" or "all-stores".')) def do_image_import(gc, args): """Initiate the image import taskflow.""" - backend = None - if args.backend: - backend = args.backend + backend = getattr(args, "store", None) + stores = getattr(args, "stores", None) + all_stores = getattr(args, "os_all_stores", None) + allow_failure = getattr(args, "os_allow_failure", True) + + if not getattr(args, 'from_create', False): + if (args.store and (stores or all_stores)) or (stores and all_stores): + utils.exit("Only one of --store, --stores and --all-stores can be " + "provided") + elif args.store: + backend = args.store + # determine if backend is valid + _validate_backend(backend, gc) + elif stores: + stores = str(stores).split(',') + # determine if backend is valid - _validate_backend(backend, gc) + if stores: + for store in stores: + _validate_backend(store, gc) if getattr(args, 'from_create', False): # this command is being called "internally" so we can skip # validation -- just do the import and get out of here gc.images.image_import(args.id, args.import_method, args.uri, - backend=backend) + backend=backend, + stores=stores, all_stores=all_stores, + allow_failure=allow_failure) return # do input validation @@ -624,6 +757,10 @@ def do_image_import(gc, args): utils.exit("Import method should be 'web-download' if URI is " "provided.") + if args.import_method == 'copy-image' and not (stores or all_stores): + utils.exit("Provide either --stores or --all-stores for " + "'copy-image' import method.") + # check image properties image = gc.images.get(args.id) container_format = image.get('container_format') @@ -641,10 +778,16 @@ def do_image_import(gc, args): if image_status != 'queued': utils.exit("The 'web-download' import method can only be applied " "to an image in status 'queued'") + if args.import_method == 'copy-image': + if image_status != 'active': + utils.exit("The 'copy-image' import method can only be used on " + "an image with status 'active'.") # finally, do the import gc.images.image_import(args.id, args.import_method, args.uri, - backend=backend) + backend=backend, + stores=stores, all_stores=all_stores, + allow_failure=allow_failure) image = gc.images.get(args.id) utils.print_image(image) @@ -725,16 +868,30 @@ def do_image_tag_delete(gc, args): @utils.arg('--metadata', metavar='<STRING>', default='{}', help=_('Metadata associated with the location. ' 'Must be a valid JSON object (default: %(default)s)')) +@utils.arg('--checksum', metavar='<STRING>', + help=_('md5 checksum of image contents')) +@utils.arg('--hash-algo', metavar='<STRING>', + help=_('Multihash algorithm')) +@utils.arg('--hash-value', metavar='<STRING>', + help=_('Multihash value')) @utils.arg('id', metavar='<IMAGE_ID>', help=_('ID of image to which the location is to be added.')) def do_location_add(gc, args): """Add a location (and related metadata) to an image.""" + validation_data = {} + if args.checksum: + validation_data['checksum'] = args.checksum + if args.hash_algo: + validation_data['os_hash_algo'] = args.hash_algo + if args.hash_value: + validation_data['os_hash_value'] = args.hash_value try: metadata = json.loads(args.metadata) except ValueError: utils.exit('Metadata is not a valid JSON object.') else: - image = gc.images.add_location(args.id, args.url, metadata) + image = gc.images.add_location(args.id, args.url, metadata, + validation_data=validation_data) utils.print_dict(image) diff --git a/lower-constraints.txt b/lower-constraints.txt index a02f7fb..61be707 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -32,7 +32,7 @@ monotonic==0.6 msgpack-python==0.4.0 netaddr==0.7.18 netifaces==0.10.4 -openstackdocstheme==1.18.1 +openstackdocstheme==1.20.0 ordereddict==1.1 os-client-config==1.28.0 os-testr==1.0.0 diff --git a/releasenotes/notes/2.16.0_Release-43ebe06b74a272ba.yaml b/releasenotes/notes/2.16.0_Release-43ebe06b74a272ba.yaml new file mode 100644 index 0000000..47b61e7 --- /dev/null +++ b/releasenotes/notes/2.16.0_Release-43ebe06b74a272ba.yaml @@ -0,0 +1,12 @@ +--- +prelude: > + This version of python-glanceclient adds Python 3.6 classifier and gating + on Python 3.7 environment. +fixes: + - | + * Bug 1788271_: Add image-list filter for multihash + * Bug 1598714_: Remove redundant information from error message + + .. _1788271: https://code.launchpad.net/bugs/1788271 + .. _1598714: https://code.launchpad.net/bugs/1598714 + diff --git a/releasenotes/notes/2.17.0_Release-c67392be3b428d10.yaml b/releasenotes/notes/2.17.0_Release-c67392be3b428d10.yaml new file mode 100644 index 0000000..d52fcc0 --- /dev/null +++ b/releasenotes/notes/2.17.0_Release-c67392be3b428d10.yaml @@ -0,0 +1,35 @@ +--- +prelude: | + This version of python-glanceclient finalizes client-side support for + the Glance multiple stores feature. See the `Multi Store Support + <https://docs.openstack.org/glance/latest/admin/multistores.html>`_ + section of the Glance documentation for more information. + + Support for Glance multiple stores has been available on an EXPERIMENTAL + basis since release 2.12.0. For the Train release, the Image service + has finalized how API users interact with multiple stores. See the + "Upgrade Notes" section of this document for information about changes this + has necessitated in multistore support in the glanceclient. +fixes: + - | + Bug 1822052_: HTTPClient: actually set a timeout for requests + + .. _1822052: https://code.launchpad.net/bugs/1822052 +upgrade: + - | + The following Command Line Interface calls now take a ``--store`` + option: + + * ``glance image-create`` + * ``glance image-create-via-import`` + * ``glance image-upload`` + * ``glance image-import`` + + The value for this option is a store identifier. The list of + available stores may be obtained from the ``glance stores-info`` + command. + + - | + The ``--backend`` option, available on some commands on an experimental + basis since release 2.12.0, is no longer available. Use ``--store`` + instead. diff --git a/releasenotes/notes/copy-existing-image-619b7e6bc3394446.yaml b/releasenotes/notes/copy-existing-image-619b7e6bc3394446.yaml new file mode 100644 index 0000000..fc7833c --- /dev/null +++ b/releasenotes/notes/copy-existing-image-619b7e6bc3394446.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for copy-image import method which will copy existing + images into multiple stores. diff --git a/releasenotes/notes/del_from_store-2d807c3038283907.yaml b/releasenotes/notes/del_from_store-2d807c3038283907.yaml new file mode 100644 index 0000000..7c330a8 --- /dev/null +++ b/releasenotes/notes/del_from_store-2d807c3038283907.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Support for deleting the image data from single store. diff --git a/releasenotes/notes/drop-py-2-7-f10417b8d1dd38fb.yaml b/releasenotes/notes/drop-py-2-7-f10417b8d1dd38fb.yaml new file mode 100644 index 0000000..8d2d16b --- /dev/null +++ b/releasenotes/notes/drop-py-2-7-f10417b8d1dd38fb.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + Python 2.7 support has been dropped. Last release of python-glanceclient + to support py2.7 is OpenStack Train. The minimum version of Python now + supported by python-glanceclient is Python 3.6. diff --git a/releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml b/releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml new file mode 100644 index 0000000..8b483e3 --- /dev/null +++ b/releasenotes/notes/multi-store-import-45d05a6193ef2c04.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for multi-store import where user can import + image into multiple backend stores with single command. diff --git a/releasenotes/notes/multihash-download-verification-596e91bf7b68e7db.yaml b/releasenotes/notes/multihash-download-verification-596e91bf7b68e7db.yaml new file mode 100644 index 0000000..f32b4a9 --- /dev/null +++ b/releasenotes/notes/multihash-download-verification-596e91bf7b68e7db.yaml @@ -0,0 +1,41 @@ +--- +features: + - | + This release adds verification of image data downloads using the Glance + "multihash" feature introduced in the OpenStack Rocky release. When + the ``os_hash_value`` is populated on an image, the glanceclient will + verify this value by computing the hexdigest of the downloaded data + using the algorithm specified by the image's ``os_hash_algo`` property. + + Because the secure hash algorithm specified is determined by the cloud + provider, it is possible that the ``os_hash_algo`` may identify an + algorithm not available in the version of the Python ``hashlib`` library + used by the client. In such a case the download will fail due to an + unsupported hash type. In the event this occurs, a new option, + ``--allow-md5-fallback``, is introduced to the ``image-download`` command. + When present, this option will allow the glanceclient to use the legacy + MD5 checksum to verify the downloaded data if the secure hash algorithm + specified by the ``os_hash_algo`` image property is not supported. + + Note that the fallback is *not* used in the case where the algorithm is + supported but the hexdigest of the downloaded data does not match the + ``os_hash_value``. In that case the download fails regardless of whether + the option is present or not. + + Whether using the ``--allow-md5-fallback`` option is a good idea depends + upon the user's expectations for the verification. MD5 is an insecure + hashing algorithm, so if you are interested in making sure that the + downloaded image data has not been replaced by a datastream carefully + crafted to have the same MD5 checksum, then you should not use the + fallback. If, however, you are using Glance in a trusted environment + and your interest is simply to verify that no bits have flipped during + the data transfer, the MD5 fallback is sufficient for that purpose. That + being said, it is our recommendation that the multihash should be used + whenever possible. +security: + - | + This release of the glanceclient uses the Glance "multihash" feature, + introduced in Rocky, to use a secure hashing algorithm to verify the + integrity of downloaded data. Legacy images without the "multihash" + image properties (``os_hash_algo`` and ``os_hash_value``) are verified + using the MD5 ``checksum`` image property. diff --git a/releasenotes/notes/multihash-filter-ef2a48dc48fae9dc.yaml b/releasenotes/notes/multihash-filter-ef2a48dc48fae9dc.yaml new file mode 100644 index 0000000..6bb22aa --- /dev/null +++ b/releasenotes/notes/multihash-filter-ef2a48dc48fae9dc.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + For parity with the old ``checksum`` field, this release adds the + ability for CLI users to filter the image list based upon a particular + multihash value using the ``--hash <HASH_VALUE>`` option. Issue the + command: + + .. code-block:: none + + glance help image-list + + for more information. diff --git a/releasenotes/notes/pike-relnote-2c77b01aa8799f35.yaml b/releasenotes/notes/pike-relnote-2c77b01aa8799f35.yaml index a35dca5..a057fa1 100644 --- a/releasenotes/notes/pike-relnote-2c77b01aa8799f35.yaml +++ b/releasenotes/notes/pike-relnote-2c77b01aa8799f35.yaml @@ -70,7 +70,7 @@ other: may now be specified_ by setting the value of the ``OS_PROFILE`` environment variable. - .. _removed: https://git.openstack.org/cgit/openstack/python-glanceclient/commit/?id=28c003dc1179ddb3124fd30c6f525dd341ae9213 + .. _removed: https://opendev.org/openstack/python-glanceclient/commit/28c003dc1179ddb3124fd30c6f525dd341ae9213 .. _inoperative: https://specs.openstack.org/openstack/glance-specs/specs/liberty/approved/remove-special-client-ssl-handling.html - .. _optimization: https://git.openstack.org/cgit/openstack/python-glanceclient/commit/?id=1df55dd952fe52c1c1fc2583326d017275b01ddc - .. _specified: https://git.openstack.org/cgit/openstack/python-glanceclient/commit/?id=c0f88d5fc0fd947319e022cfeba21bcb15635316 + .. _optimization: https://opendev.org/openstack/python-glanceclient/commit/1df55dd952fe52c1c1fc2583326d017275b01ddc + .. _specified: https://opendev.org/openstack/python-glanceclient/commit/c0f88d5fc0fd947319e022cfeba21bcb15635316 diff --git a/releasenotes/notes/validation-data-support-dfb2463914818cd2.yaml b/releasenotes/notes/validation-data-support-dfb2463914818cd2.yaml new file mode 100644 index 0000000..499c1fb --- /dev/null +++ b/releasenotes/notes/validation-data-support-dfb2463914818cd2.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Support for embedding validation data (checksum and multihash) when adding + a location to an image. Requires the Stein release server-side. + + The ``glance.images.add_location()`` method now accepts an optional + argument ``validation_data``, in the form of a dictionary containing + ``checksum``, ``os_hash_algo`` and ``os_hash_value``. + + The ``location-add`` command now accepts optional arguments ``--checksum``, + ``--hash-algo`` and ``--hash-value``. diff --git a/releasenotes/source/earlier.rst b/releasenotes/source/earlier.rst index e6a9eaa..f0c9d83 100644 --- a/releasenotes/source/earlier.rst +++ b/releasenotes/source/earlier.rst @@ -107,14 +107,14 @@ A lot of effort has been invested to make the transition as smooth as possible, * 5e85d61 cleanup openstack-common.conf and sync updated files * 1432701_: Add parameter 'changes-since' for image-list of v1 -.. _remcustssl: https://review.openstack.org/#/c/187674 +.. _remcustssl: https://review.opendev.org/#/c/187674 .. _1309272: https://bugs.launchpad.net/python-glanceclient/+bug/1309272 .. _1481729: https://bugs.launchpad.net/python-glanceclient/+bug/1481729 .. _1477910: https://bugs.launchpad.net/python-glanceclient/+bug/1477910 .. _1475769: https://bugs.launchpad.net/python-glanceclient/+bug/1475769 .. _1479020: https://bugs.launchpad.net/python-glanceclient/+bug/1479020 .. _1433637: https://bugs.launchpad.net/python-glanceclient/+bug/1433637 -.. _metatags: https://review.openstack.org/#/c/179674/ +.. _metatags: https://review.opendev.org/#/c/179674/ .. _1472234: https://bugs.launchpad.net/python-glanceclient/+bug/1472234 .. _1473454: https://bugs.launchpad.net/python-cinderclient/+bug/1473454 .. _1468485: https://bugs.launchpad.net/python-glanceclient/+bug/1468485 diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 97fc0eb..1567c44 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,8 @@ glanceclient Release Notes :maxdepth: 1 unreleased + train + stein rocky queens pike diff --git a/releasenotes/source/stein.rst b/releasenotes/source/stein.rst new file mode 100644 index 0000000..efaceb6 --- /dev/null +++ b/releasenotes/source/stein.rst @@ -0,0 +1,6 @@ +=================================== + Stein Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/stein diff --git a/releasenotes/source/train.rst b/releasenotes/source/train.rst new file mode 100644 index 0000000..7fa1088 --- /dev/null +++ b/releasenotes/source/train.rst @@ -0,0 +1,6 @@ +=================================== + Train Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/train @@ -5,8 +5,9 @@ description-file = README.rst license = Apache License, Version 2.0 author = OpenStack -author-email = openstack-dev@lists.openstack.org +author-email = openstack-discuss@lists.openstack.org home-page = https://docs.openstack.org/python-glanceclient/latest/ +python-requires = >=3.6 classifier = Development Status :: 5 - Production/Stable Environment :: Console @@ -16,22 +17,14 @@ classifier = License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 [files] packages = glanceclient -[global] -setup-hooks = - pbr.hooks.setup_hook - [entry_points] console_scripts = glance = glanceclient.shell:main - -[wheel] -universal = 1 diff --git a/test-requirements.txt b/test-requirements.txt index 0424393..8e8541c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,8 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 + +hacking>=1.1.0,<1.2.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 mock>=2.0.0 # BSD @@ -1,17 +1,15 @@ [tox] -envlist = py35,py27,pep8 -minversion = 1.6 +envlist = py37,pep8 +minversion = 2.0 skipsdist = True [testenv] usedevelop = True -install_command = pip install {opts} {packages} -setenv = VIRTUAL_ENV={envdir} - OS_STDOUT_NOCAPTURE=False +setenv = OS_STDOUT_NOCAPTURE=False OS_STDERR_NOCAPTURE=False deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} @@ -39,18 +37,6 @@ commands = bash tools/fix_ca_bundle.sh stestr run --slowest {posargs} -[testenv:functional-v1] -# TODO(rosmaita): remove this testenv at the beginning -# of the 'S' cycle -setenv = - OS_TEST_PATH = ./glanceclient/tests/functional/v1 - OS_TESTENV_NAME = {envname} -whitelist_externals = - bash -commands = - bash tools/fix_ca_bundle.sh - stestr run --slowest {posargs} - [testenv:cover] basepython = python3 setenv = @@ -69,12 +55,15 @@ commands = [testenv:releasenotes] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [flake8] -ignore = F403,F812,F821 +# E731 skipped as assign a lambda expression +ignore = E731,F403,F812,F821 show-source = True exclude = .venv*,.tox,dist,*egg,build,.git,doc,*lib/python*,.update-venv |