summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/source/howto_installmultinode.rst31
-rwxr-xr-xswift/cli/recon.py56
-rw-r--r--swift/common/middleware/tempauth.py3
-rw-r--r--swift/obj/replicator.py7
-rw-r--r--test/functional/__init__.py5
-rw-r--r--test/unit/cli/test_recon.py112
-rw-r--r--test/unit/common/middleware/test_tempauth.py6
-rw-r--r--test/unit/obj/test_replicator.py81
8 files changed, 287 insertions, 14 deletions
diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst
index d7e363413..7b37cb077 100644
--- a/doc/source/howto_installmultinode.rst
+++ b/doc/source/howto_installmultinode.rst
@@ -2,9 +2,28 @@
Instructions for a Multiple Server Swift Installation
=====================================================
-Please refer to the installation guides at:
- http://docs.openstack.org, including
-* Debian 7.0: http://docs.openstack.org/icehouse/install-guide/install/apt-debian/content/ch_swift.html
-* openSUSE and SUSE Linux Enterprise Server: http://docs.openstack.org/icehouse/install-guide/install/zypper/content/ch_swift.html
-* Red Hat Enterprise Linux, CentOS, and Fedora: http://docs.openstack.org/icehouse/install-guide/install/yum/content/ch_swift.html
-* Ubuntu 12.04/14.04: http://docs.openstack.org/trunk/install-guide/install/apt/content/ch_swift.html
+Please refer to the latest offical
+`Openstack Installation Guides <http://docs.openstack.org/#install-guides>`_
+for the most up-to-date documentation.
+
+Object Storage installation guide for Openstack Juno
+----------------------------------------------------
+
+ * `openSUSE 13.1 and SUSE Linux Enterprise Server 11 <http://docs.openstack.org/juno/install-guide/install/zypper/content/ch_swift.html>`_
+ * `RHEL 7, CentOS 7, and Fedora 20 <http://docs.openstack.org/juno/install-guide/install/yum/content/ch_swift.html>`_
+ * `Ubuntu 14.04 <http://docs.openstack.org/juno/install-guide/install/apt/content/ch_swift.html>`_
+
+Object Storage installation guide for Openstack Icehouse
+--------------------------------------------------------
+
+ * `openSUSE and SUSE Linux Enterprise Server <http://docs.openstack.org/icehouse/install-guide/install/zypper/content/ch_swift.html>`_
+ * `Red Hat Enterprise Linux, CentOS, and Fedora <http://docs.openstack.org/icehouse/install-guide/install/yum/content/ch_swift.html>`_
+ * `Ubuntu 12.04/14.04 (LTS) <http://docs.openstack.org/icehouse/install-guide/install/apt/content/ch_swift.html>`_
+
+Object Storage installation guide for Openstack Havana
+------------------------------------------------------
+
+ * `Debian 7.0 <http://docs.openstack.org/havana/install-guide/install/apt-debian/content/ch_swift.html>`_
+ * `openSUSE and SUSE Linux Enterprise Server <http://docs.openstack.org/havana/install-guide/install/zypper/content/ch_swift.html>`_
+ * `Red Hat Enterprise Linux, CentOS, and Fedora <http://docs.openstack.org/havana/install-guide/install/yum/content/ch_swift.html>`_
+ * `Ubuntu 12.04 (LTS) <http://docs.openstack.org/havana/install-guide/install/apt/content/ch_swift.html>`_
diff --git a/swift/cli/recon.py b/swift/cli/recon.py
index b47bee948..676973c41 100755
--- a/swift/cli/recon.py
+++ b/swift/cli/recon.py
@@ -109,6 +109,34 @@ class Scout(object):
url, content, status = self.scout_host(base_url, self.recon_type)
return url, content, status
+ def scout_server_type(self, host):
+ """
+ Obtain Server header by calling OPTIONS.
+
+ :param host: host to check
+ :returns: Server type, status
+ """
+ try:
+ url = "http://%s:%s/" % (host[0], host[1])
+ req = urllib2.Request(url)
+ req.get_method = lambda: 'OPTIONS'
+ conn = urllib2.urlopen(req)
+ header = conn.info().getheader('Server')
+ server_header = header.split('/')
+ content = server_header[0]
+ status = 200
+ except urllib2.HTTPError as err:
+ if not self.suppress_errors or self.verbose:
+ print("-> %s: %s" % (url, err))
+ content = err
+ status = err.code
+ except urllib2.URLError as err:
+ if not self.suppress_errors or self.verbose:
+ print("-> %s: %s" % (url, err))
+ content = err
+ status = -1
+ return url, content, status
+
class SwiftRecon(object):
"""
@@ -334,6 +362,29 @@ class SwiftRecon(object):
print("Device errors: %s on %s" % (entry, node))
print("=" * 79)
+ def server_type_check(self, hosts):
+ """
+ Check for server types on the ring
+
+ :param hosts: set of hosts to check. in the format of:
+ set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+ """
+ errors = {}
+ recon = Scout("server_type_check", self.verbose, self.suppress_errors,
+ self.timeout)
+ print("[%s] Validating server type '%s' on %s hosts..." %
+ (self._ptime(), self.server_type, len(hosts)))
+ for url, response, status in self.pool.imap(
+ recon.scout_server_type, hosts):
+ if status == 200:
+ if response != self.server_type + '-server':
+ errors[url] = response
+ print("%s/%s hosts ok, %s error[s] while checking hosts." % (
+ len(hosts) - len(errors), len(hosts), len(errors)))
+ for host in errors:
+ print("Invalid: %s is %s" % (host, errors[host]))
+ print("=" * 79)
+
def expirer_check(self, hosts):
"""
Obtain and print expirer statistics
@@ -872,6 +923,8 @@ class SwiftRecon(object):
help="Get cluster load average stats")
args.add_option('--quarantined', '-q', action="store_true",
help="Get cluster quarantine stats")
+ args.add_option('--validate-servers', action="store_true",
+ help="Validate servers on the ring")
args.add_option('--md5', action="store_true",
help="Get md5sum of servers ring and compare to "
"local copy")
@@ -938,6 +991,7 @@ class SwiftRecon(object):
self.get_ringmd5(hosts, swift_dir)
self.quarantine_check(hosts)
self.socket_usage(hosts)
+ self.server_type_check(hosts)
else:
if options.async:
if self.server_type == 'object':
@@ -966,6 +1020,8 @@ class SwiftRecon(object):
self.expirer_check(hosts)
else:
print("Error: Can't check expired on non object servers.")
+ if options.validate_servers:
+ self.server_type_check(hosts)
if options.loadstats:
self.load_check(hosts)
if options.diskusage:
diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py
index 837368fdd..a2b07128a 100644
--- a/swift/common/middleware/tempauth.py
+++ b/swift/common/middleware/tempauth.py
@@ -32,7 +32,7 @@ from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.middleware.acl import (
clean_acl, parse_acl, referrer_allowed, acls_from_account_info)
from swift.common.utils import cache_from_env, get_logger, \
- split_path, config_true_value
+ split_path, config_true_value, register_swift_info
from swift.common.utils import config_read_reseller_options
from swift.proxy.controllers.base import get_account_info
@@ -769,6 +769,7 @@ def filter_factory(global_conf, **local_conf):
"""Returns a WSGI filter app for use with paste.deploy."""
conf = global_conf.copy()
conf.update(local_conf)
+ register_swift_info('tempauth', account_acls=True)
def auth_filter(app):
return TempAuth(app, conf)
diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py
index b3df0ce28..5ee32884c 100644
--- a/swift/obj/replicator.py
+++ b/swift/obj/replicator.py
@@ -242,7 +242,7 @@ class ObjectReplicator(Daemon):
for node in job['nodes']:
kwargs = {}
if node['region'] in synced_remote_regions and \
- self.conf.get('sync_method') == 'ssync':
+ self.conf.get('sync_method', 'rsync') == 'ssync':
kwargs['remote_check_objs'] = \
synced_remote_regions[node['region']]
# cand_objs is a list of objects for deletion
@@ -273,11 +273,12 @@ class ObjectReplicator(Daemon):
delete_handoff = len(responses) == len(job['nodes']) and \
all(responses)
if delete_handoff:
- if delete_objs:
+ if (self.conf.get('sync_method', 'rsync') == 'ssync' and
+ delete_objs is not None):
self.logger.info(_("Removing %s objects"),
len(delete_objs))
self.delete_handoff_objs(job, delete_objs)
- elif self.conf.get('sync_method') == 'rsync':
+ else:
self.delete_partition(job['path'])
elif not suffixes:
self.delete_partition(job['path'])
diff --git a/test/functional/__init__.py b/test/functional/__init__.py
index 5ad204b9b..c4d764268 100644
--- a/test/functional/__init__.py
+++ b/test/functional/__init__.py
@@ -625,6 +625,7 @@ def retry(func, *args, **kwargs):
os_options = {'user_domain_name': swift_test_domain[use_account],
'project_domain_name': swift_test_domain[use_account]}
while attempts <= retries:
+ auth_failure = False
attempts += 1
try:
if not url[use_account] or not token[use_account]:
@@ -654,13 +655,15 @@ def retry(func, *args, **kwargs):
if service_user:
service_token[service_user] = None
except AuthError:
+ auth_failure = True
url[use_account] = token[use_account] = None
if service_user:
service_token[service_user] = None
except InternalServerError:
pass
if attempts <= retries:
- sleep(backoff)
+ if not auth_failure:
+ sleep(backoff)
backoff *= 2
raise Exception('No result after %s retries.' % retries)
diff --git a/test/unit/cli/test_recon.py b/test/unit/cli/test_recon.py
index 4f836e7d5..e9ad45d2c 100644
--- a/test/unit/cli/test_recon.py
+++ b/test/unit/cli/test_recon.py
@@ -56,6 +56,7 @@ class TestScout(unittest.TestCase):
def setUp(self, *_args, **_kwargs):
self.scout_instance = recon.Scout("type", suppress_errors=True)
self.url = 'http://127.0.0.1:8080/recon/type'
+ self.server_type_url = 'http://127.0.0.1:8080/'
@mock.patch('eventlet.green.urllib2.urlopen')
def test_scout_ok(self, mock_urlopen):
@@ -85,6 +86,37 @@ class TestScout(unittest.TestCase):
self.assertTrue(isinstance(content, urllib2.HTTPError))
self.assertEqual(status, 404)
+ @mock.patch('eventlet.green.urllib2.urlopen')
+ def test_scout_server_type_ok(self, mock_urlopen):
+ def getheader(name):
+ d = {'Server': 'server-type'}
+ return d.get(name)
+ mock_urlopen.return_value.info.return_value.getheader = getheader
+ url, content, status = self.scout_instance.scout_server_type(
+ ("127.0.0.1", "8080"))
+ self.assertEqual(url, self.server_type_url)
+ self.assertEqual(content, 'server-type')
+ self.assertEqual(status, 200)
+
+ @mock.patch('eventlet.green.urllib2.urlopen')
+ def test_scout_server_type_url_error(self, mock_urlopen):
+ mock_urlopen.side_effect = urllib2.URLError("")
+ url, content, status = self.scout_instance.scout_server_type(
+ ("127.0.0.1", "8080"))
+ self.assertTrue(isinstance(content, urllib2.URLError))
+ self.assertEqual(url, self.server_type_url)
+ self.assertEqual(status, -1)
+
+ @mock.patch('eventlet.green.urllib2.urlopen')
+ def test_scout_server_type_http_error(self, mock_urlopen):
+ mock_urlopen.side_effect = urllib2.HTTPError(
+ self.server_type_url, 404, "Internal error", None, None)
+ url, content, status = self.scout_instance.scout_server_type(
+ ("127.0.0.1", "8080"))
+ self.assertEqual(url, self.server_type_url)
+ self.assertTrue(isinstance(content, urllib2.HTTPError))
+ self.assertEqual(status, 404)
+
class TestRecon(unittest.TestCase):
def setUp(self, *_args, **_kwargs):
@@ -289,6 +321,86 @@ class TestReconCommands(unittest.TestCase):
return mock.patch('eventlet.green.urllib2.urlopen', fake_urlopen)
+ def test_server_type_check(self):
+ hosts = [('127.0.0.1', 6010), ('127.0.0.1', 6011),
+ ('127.0.0.1', 6012)]
+
+ # sample json response from http://<host>:<port>/
+ responses = {6010: 'object-server', 6011: 'container-server',
+ 6012: 'account-server'}
+
+ def mock_scout_server_type(app, host):
+ url = 'http://%s:%s/' % (host[0], host[1])
+ response = responses[host[1]]
+ status = 200
+ return url, response, status
+
+ stdout = StringIO()
+ patches = [
+ mock.patch('swift.cli.recon.Scout.scout_server_type',
+ mock_scout_server_type),
+ mock.patch('sys.stdout', new=stdout),
+ ]
+
+ res_object = 'Invalid: http://127.0.0.1:6010/ is object-server'
+ res_container = 'Invalid: http://127.0.0.1:6011/ is container-server'
+ res_account = 'Invalid: http://127.0.0.1:6012/ is account-server'
+ valid = "1/1 hosts ok, 0 error[s] while checking hosts."
+
+ #Test for object server type - default
+ with nested(*patches):
+ self.recon.server_type_check(hosts)
+
+ output = stdout.getvalue()
+ self.assertTrue(res_container in output.splitlines())
+ self.assertTrue(res_account in output.splitlines())
+ stdout.truncate(0)
+
+ #Test ok for object server type - default
+ with nested(*patches):
+ self.recon.server_type_check([hosts[0]])
+
+ output = stdout.getvalue()
+ self.assertTrue(valid in output.splitlines())
+ stdout.truncate(0)
+
+ #Test for account server type
+ with nested(*patches):
+ self.recon.server_type = 'account'
+ self.recon.server_type_check(hosts)
+
+ output = stdout.getvalue()
+ self.assertTrue(res_container in output.splitlines())
+ self.assertTrue(res_object in output.splitlines())
+ stdout.truncate(0)
+
+ #Test ok for account server type
+ with nested(*patches):
+ self.recon.server_type = 'account'
+ self.recon.server_type_check([hosts[2]])
+
+ output = stdout.getvalue()
+ self.assertTrue(valid in output.splitlines())
+ stdout.truncate(0)
+
+ #Test for container server type
+ with nested(*patches):
+ self.recon.server_type = 'container'
+ self.recon.server_type_check(hosts)
+
+ output = stdout.getvalue()
+ self.assertTrue(res_account in output.splitlines())
+ self.assertTrue(res_object in output.splitlines())
+ stdout.truncate(0)
+
+ #Test ok for container server type
+ with nested(*patches):
+ self.recon.server_type = 'container'
+ self.recon.server_type_check([hosts[1]])
+
+ output = stdout.getvalue()
+ self.assertTrue(valid in output.splitlines())
+
def test_get_swiftconfmd5(self):
hosts = set([('10.1.1.1', 10000),
('10.2.2.2', 10000)])
diff --git a/test/unit/common/middleware/test_tempauth.py b/test/unit/common/middleware/test_tempauth.py
index df847ae28..394668b47 100644
--- a/test/unit/common/middleware/test_tempauth.py
+++ b/test/unit/common/middleware/test_tempauth.py
@@ -21,7 +21,7 @@ from time import time
from swift.common.middleware import tempauth as auth
from swift.common.middleware.acl import format_acl
from swift.common.swob import Request, Response
-from swift.common.utils import split_path
+from swift.common.utils import split_path, get_swift_info
NO_CONTENT_RESP = (('204 No Content', {}, ''),) # mock server response
@@ -110,6 +110,10 @@ class TestAuth(unittest.TestCase):
def setUp(self):
self.test_auth = auth.filter_factory({})(FakeApp())
+ def test_swift_info(self):
+ info = get_swift_info()
+ self.assertTrue(info['tempauth']['account_acls'])
+
def _make_request(self, path, **kwargs):
req = Request.blank(path, **kwargs)
req.environ['swift.cache'] = FakeMemcache()
diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py
index bf1c5bcb5..ab89e4925 100644
--- a/test/unit/obj/test_replicator.py
+++ b/test/unit/obj/test_replicator.py
@@ -117,14 +117,14 @@ def _mock_process(ret):
object_replicator.subprocess.Popen = orig_process
-def _create_test_rings(path):
+def _create_test_rings(path, devs=None):
testgz = os.path.join(path, 'object.ring.gz')
intended_replica2part2dev_id = [
[0, 1, 2, 3, 4, 5, 6],
[1, 2, 3, 0, 5, 6, 4],
[2, 3, 0, 1, 6, 4, 5],
]
- intended_devs = [
+ intended_devs = devs or [
{'id': 0, 'device': 'sda', 'zone': 0,
'region': 1, 'ip': '127.0.0.0', 'port': 6000},
{'id': 1, 'device': 'sda', 'zone': 1,
@@ -153,6 +153,8 @@ def _create_test_rings(path):
ring.RingData(intended_replica2part2dev_id,
intended_devs, intended_part_shift),
f)
+ for policy in POLICIES:
+ policy.object_ring = None # force reload
return
@@ -418,6 +420,81 @@ class TestObjectReplicator(unittest.TestCase):
self.replicator.replicate()
self.assertFalse(os.access(part_path, os.F_OK))
+ def test_delete_partition_default_sync_method(self):
+ self.replicator.conf.pop('sync_method')
+ with mock.patch('swift.obj.replicator.http_connect',
+ mock_http_connect(200)):
+ df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o')
+ mkdirs(df._datadir)
+ f = open(os.path.join(df._datadir,
+ normalize_timestamp(time.time()) + '.data'),
+ 'wb')
+ f.write('1234567890')
+ f.close()
+ ohash = hash_path('a', 'c', 'o')
+ data_dir = ohash[-3:]
+ whole_path_from = os.path.join(self.objects, '1', data_dir)
+ part_path = os.path.join(self.objects, '1')
+ self.assertTrue(os.access(part_path, os.F_OK))
+ ring = self.replicator.get_object_ring(0)
+ nodes = [node for node in
+ ring.get_part_nodes(1)
+ if node['ip'] not in _ips()]
+ process_arg_checker = []
+ for node in nodes:
+ rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1)
+ process_arg_checker.append(
+ (0, '', ['rsync', whole_path_from, rsync_mod]))
+ with _mock_process(process_arg_checker):
+ self.replicator.replicate()
+ self.assertFalse(os.access(part_path, os.F_OK))
+
+ def test_delete_partition_ssync_single_region(self):
+ devs = [
+ {'id': 0, 'device': 'sda', 'zone': 0,
+ 'region': 1, 'ip': '127.0.0.0', 'port': 6000},
+ {'id': 1, 'device': 'sda', 'zone': 1,
+ 'region': 1, 'ip': '127.0.0.1', 'port': 6000},
+ {'id': 2, 'device': 'sda', 'zone': 2,
+ 'region': 1, 'ip': '127.0.0.2', 'port': 6000},
+ {'id': 3, 'device': 'sda', 'zone': 4,
+ 'region': 1, 'ip': '127.0.0.3', 'port': 6000},
+ {'id': 4, 'device': 'sda', 'zone': 5,
+ 'region': 1, 'ip': '127.0.0.4', 'port': 6000},
+ {'id': 5, 'device': 'sda', 'zone': 6,
+ 'region': 1, 'ip': 'fe80::202:b3ff:fe1e:8329', 'port': 6000},
+ {'id': 6, 'device': 'sda', 'zone': 7, 'region': 1,
+ 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334', 'port': 6000},
+ ]
+ _create_test_rings(self.testdir, devs=devs)
+ self.conf['sync_method'] = 'ssync'
+ self.replicator = object_replicator.ObjectReplicator(self.conf)
+ self.replicator.logger = debug_logger()
+
+ with mock.patch('swift.obj.replicator.http_connect',
+ mock_http_connect(200)):
+ df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o')
+ mkdirs(df._datadir)
+ f = open(os.path.join(df._datadir,
+ normalize_timestamp(time.time()) + '.data'),
+ 'wb')
+ f.write('1234567890')
+ f.close()
+ ohash = hash_path('a', 'c', 'o')
+ whole_path_from = storage_directory(self.objects, 1, ohash)
+ suffix_dir_path = os.path.dirname(whole_path_from)
+ part_path = os.path.join(self.objects, '1')
+ self.assertTrue(os.access(part_path, os.F_OK))
+
+ def _fake_ssync(node, job, suffixes, **kwargs):
+ return True, set([ohash])
+
+ self.replicator.sync_method = _fake_ssync
+ self.replicator.replicate()
+ self.assertFalse(os.access(whole_path_from, os.F_OK))
+ self.assertFalse(os.access(suffix_dir_path, os.F_OK))
+ self.assertFalse(os.access(part_path, os.F_OK))
+
def test_delete_partition_1(self):
with mock.patch('swift.obj.replicator.http_connect',
mock_http_connect(200)):
View Lorry Controller Status