summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatthew Oliver <matt@oliver.net.au>2022-07-07 17:27:49 +1000
committerMatthew Oliver <matt@oliver.net.au>2022-07-12 12:30:43 +1000
commita548da916f5bb3097c42de54b53e54a57853220b (patch)
tree119d293adef8732951c4602fa5ff0b48696ed04f
parent475cdba65bcd29871b840395576572c0d8399a6b (diff)
downloadswift-a548da916f5bb3097c42de54b53e54a57853220b.tar.gz
Remove :memory: from DatabaseBrokers and unittests
The SQLite in-memory databases have been great for testing but as the swift DatabaseBroker's have become more complex, the limitations of in memory databases are being reached. Mostly due to the introduction of container sharding where a broker sometimes needs to make multiple connections to the same database as the same time. Rather then rework the real broker logic to better support in-memory testing, it's actually easier to just remove the in-memory broker tests and use a "real" broker in a tempdir. This allows us to better test how brokers behave in real life, pending files and all. This patch replaces all the :memory: brokers in the tests with real ones placed in a tempdir. To achieve this, we new base unittest class `TestDBBase` has been added that creates, cleans up and provides some helper methods to manage the db path and location. Further, all references to :memory: in the Database brokers have been removed. Change-Id: I5983132f776b84db634fef39c833d5cfdce11980
Diffstat
-rw-r--r--swift/common/db.py38
-rw-r--r--swift/container/backend.py11
-rw-r--r--test/unit/__init__.py7
-rw-r--r--test/unit/account/test_auditor.py7
-rw-r--r--test/unit/account/test_backend.py109
-rw-r--r--test/unit/account/test_utils.py13
-rw-r--r--test/unit/common/test_db.py121
-rw-r--r--test/unit/container/test_backend.py233
8 files changed, 345 insertions, 194 deletions
diff --git a/swift/common/db.py b/swift/common/db.py
index 3e42d952a..c5ecc4152 100644
--- a/swift/common/db.py
+++ b/swift/common/db.py
@@ -205,7 +205,7 @@ def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
factory=GreenDBConnection, timeout=timeout)
if QUERY_LOGGING and logger and not six.PY2:
conn.set_trace_callback(logger.debug)
- if path != ':memory:' and not okay_to_create:
+ if not okay_to_create:
# attempt to detect and fail when connect creates the db file
stat = os.stat(path)
if stat.st_size == 0 and stat.st_ctime >= connect_time:
@@ -359,17 +359,13 @@ class DatabaseBroker(object):
:param put_timestamp: internalized timestamp of initial PUT request
:param storage_policy_index: only required for containers
"""
- if self._db_file == ':memory:':
- tmp_db_file = None
- conn = get_db_connection(self._db_file, self.timeout, self.logger)
- else:
- mkdirs(self.db_dir)
- fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
- os.close(fd)
- conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
- factory=GreenDBConnection, timeout=0)
- if QUERY_LOGGING and not six.PY2:
- conn.set_trace_callback(self.logger.debug)
+ mkdirs(self.db_dir)
+ fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
+ os.close(fd)
+ conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
+ factory=GreenDBConnection, timeout=0)
+ if QUERY_LOGGING and not six.PY2:
+ conn.set_trace_callback(self.logger.debug)
# creating dbs implicitly does a lot of transactions, so we
# pick fast, unsafe options here and do a big fsync at the end.
with closing(conn.cursor()) as cur:
@@ -541,7 +537,7 @@ class DatabaseBroker(object):
def get(self):
"""Use with the "with" statement; returns a database connection."""
if not self.conn:
- if self.db_file != ':memory:' and os.path.exists(self.db_file):
+ if os.path.exists(self.db_file):
try:
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
@@ -569,7 +565,7 @@ class DatabaseBroker(object):
def lock(self):
"""Use with the "with" statement; locks a database."""
if not self.conn:
- if self.db_file != ':memory:' and os.path.exists(self.db_file):
+ if os.path.exists(self.db_file):
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
else:
@@ -637,7 +633,7 @@ class DatabaseBroker(object):
:returns: True if the DB is considered to be deleted, False otherwise
"""
- if self.db_file != ':memory:' and not os.path.exists(self.db_file):
+ if not os.path.exists(self.db_file):
return True
self._commit_puts_stale_ok()
with self.get() as conn:
@@ -770,8 +766,8 @@ class DatabaseBroker(object):
"""
Put a record into the DB. If the DB has an associated pending file with
space then the record is appended to that file and a commit to the DB
- is deferred. If the DB is in-memory or its pending file is full then
- the record will be committed immediately.
+ is deferred. If its pending file is full then the record will be
+ committed immediately.
:param record: a record to be added to the DB.
:raises DatabaseConnectionError: if the DB file does not exist or if
@@ -779,9 +775,6 @@ class DatabaseBroker(object):
:raises LockTimeout: if a timeout occurs while waiting to take a lock
to write to the pending file.
"""
- if self._db_file == ':memory:':
- self.merge_items([record])
- return
if not os.path.exists(self.db_file):
raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
if self.skip_commits:
@@ -807,8 +800,7 @@ class DatabaseBroker(object):
fp.flush()
def _skip_commit_puts(self):
- return (self._db_file == ':memory:' or self.skip_commits or not
- os.path.exists(self.pending_file))
+ return self.skip_commits or not os.path.exists(self.pending_file)
def _commit_puts(self, item_list=None):
"""
@@ -922,7 +914,7 @@ class DatabaseBroker(object):
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
- if not DB_PREALLOCATION or self._db_file == ':memory:':
+ if not DB_PREALLOCATION:
return
MB = (1024 * 1024)
diff --git a/swift/container/backend.py b/swift/container/backend.py
index 17546a887..ea8d33c82 100644
--- a/swift/container/backend.py
+++ b/swift/container/backend.py
@@ -348,10 +348,7 @@ class ContainerBroker(DatabaseBroker):
stale_reads_ok=False, skip_commits=False,
force_db_file=False):
self._init_db_file = db_file
- if db_file == ':memory:':
- base_db_file = db_file
- else:
- base_db_file = make_db_file_path(db_file, None)
+ base_db_file = make_db_file_path(db_file, None)
super(ContainerBroker, self).__init__(
base_db_file, timeout, logger, account, container, pending_timeout,
stale_reads_ok, skip_commits=skip_commits)
@@ -396,8 +393,6 @@ class ContainerBroker(DatabaseBroker):
"""
Returns the current state of on disk db files.
"""
- if self._db_file == ':memory:':
- return UNSHARDED
if not self.db_files:
return NOTFOUND
if len(self.db_files) > 1:
@@ -441,8 +436,6 @@ class ContainerBroker(DatabaseBroker):
"""
Reloads the cached list of valid on disk db files for this broker.
"""
- if self._db_file == ':memory:':
- return
# reset connection so the next access will use the correct DB file
self.conn = None
self._db_files = get_db_files(self._init_db_file)
@@ -849,7 +842,7 @@ class ContainerBroker(DatabaseBroker):
:returns: a tuple, in the form (info, is_deleted) info is a dict as
returned by get_info and is_deleted is a boolean.
"""
- if self.db_file != ':memory:' and not os.path.exists(self.db_file):
+ if not os.path.exists(self.db_file):
return {}, True
info = self.get_info()
return info, self._is_deleted_info(**info)
diff --git a/test/unit/__init__.py b/test/unit/__init__.py
index 266763e0a..0dc3d6a83 100644
--- a/test/unit/__init__.py
+++ b/test/unit/__init__.py
@@ -42,6 +42,7 @@ import random
import errno
import xattr
from io import BytesIO
+from uuid import uuid4
import six
import six.moves.cPickle as pickle
@@ -1408,3 +1409,9 @@ def group_by_byte(contents):
return [
(char, sum(1 for _ in grp))
for char, grp in itertools.groupby(byte_iter)]
+
+
+def generate_db_path(tempdir, server_type):
+ return os.path.join(
+ tempdir, '%ss' % server_type, 'part', 'suffix', 'hash',
+ '%s-%s.db' % (server_type, uuid4()))
diff --git a/test/unit/account/test_auditor.py b/test/unit/account/test_auditor.py
index f0ecc9d8f..f869cc717 100644
--- a/test/unit/account/test_auditor.py
+++ b/test/unit/account/test_auditor.py
@@ -53,14 +53,15 @@ class TestAuditorRealBrokerMigration(
self.assertUnmigrated(self.broker)
# run auditor, and validate migration
- conf = {'devices': self.tempdir, 'mount_check': False,
- 'recon_cache_path': self.tempdir}
+ conf = {'devices': self.testdir, 'mount_check': False,
+ 'recon_cache_path': self.testdir}
test_auditor = auditor.AccountAuditor(conf, logger=debug_logger())
test_auditor.run_once()
self.restore_account_broker()
- broker = auditor.AccountBroker(self.db_path)
+ broker = auditor.AccountBroker(self.db_path, account='a')
+ broker.initialize(Timestamp('1').internal, 0)
# go after rows directly to avoid unintentional migration
with broker.get() as conn:
rows = conn.execute('''
diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py
index 9e6cb55c3..ca8ef30e6 100644
--- a/test/unit/account/test_backend.py
+++ b/test/unit/account/test_backend.py
@@ -17,13 +17,10 @@
from collections import defaultdict
import json
-import unittest
import pickle
import os
from time import sleep, time
from uuid import uuid4
-from tempfile import mkdtemp
-from shutil import rmtree
import sqlite3
import itertools
from contextlib import contextmanager
@@ -46,23 +43,25 @@ from test.unit.common import test_db
@patch_policies
-class TestAccountBroker(unittest.TestCase):
+class TestAccountBroker(test_db.TestDbBase):
"""Tests for AccountBroker"""
def setUp(self):
+ super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
def test_creation(self):
# Test AccountBroker.__init__
- broker = AccountBroker(':memory:', account='a')
- self.assertEqual(broker.db_file, ':memory:')
+ db_file = self.get_db_path()
+ broker = AccountBroker(db_file, account='a')
+ self.assertEqual(broker.db_file, db_file)
try:
with broker.get() as conn:
pass
except DatabaseConnectionError as e:
self.assertTrue(hasattr(e, 'path'))
- self.assertEqual(e.path, ':memory:')
+ self.assertEqual(e.path, db_file)
self.assertTrue(hasattr(e, 'msg'))
self.assertEqual(e.msg, "DB doesn't exist")
except Exception as e:
@@ -76,7 +75,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(curs.fetchall()[0][0], 1)
def test_initialize_fail(self):
- broker = AccountBroker(':memory:')
+ broker = AccountBroker(self.get_db_path())
with self.assertRaises(ValueError) as cm:
broker.initialize(Timestamp('1').internal)
self.assertEqual(str(cm.exception), 'Attempting to create a new'
@@ -85,7 +84,7 @@ class TestAccountBroker(unittest.TestCase):
def test_exception(self):
# Test AccountBroker throwing a conn away after exception
first_conn = None
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
first_conn = conn
@@ -99,7 +98,7 @@ class TestAccountBroker(unittest.TestCase):
def test_empty(self):
# Test AccountBroker.empty
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
self.assertTrue(broker.empty())
broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
@@ -112,12 +111,12 @@ class TestAccountBroker(unittest.TestCase):
def test_is_status_deleted(self):
# Test AccountBroker.is_status_deleted
- broker1 = AccountBroker(':memory:', account='a')
+ broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp.now().internal)
self.assertFalse(broker1.is_status_deleted())
broker1.delete_db(Timestamp.now().internal)
self.assertTrue(broker1.is_status_deleted())
- broker2 = AccountBroker(':memory:', account='a')
+ broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp.now().internal)
# Set delete_timestamp greater than put_timestamp
broker2.merge_timestamps(
@@ -126,10 +125,12 @@ class TestAccountBroker(unittest.TestCase):
self.assertTrue(broker2.is_status_deleted())
def test_reclaim(self):
- broker = AccountBroker(':memory:', account='test_account')
+ broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal)
broker.put_container('c', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -148,6 +149,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001)
broker.put_container('c', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -176,6 +178,7 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
+ broker._commit_puts()
broker.reclaim(Timestamp.now().internal, time())
# Now delete the account
broker.delete_db(Timestamp.now().internal)
@@ -193,7 +196,7 @@ class TestAccountBroker(unittest.TestCase):
random.seed(now)
random.shuffle(container_specs)
policy_indexes = list(p.idx for p in POLICIES)
- broker = AccountBroker(':memory:', account='test_account')
+ broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal)
for i, container_spec in enumerate(container_specs):
# with container12 before container2 and shuffled ts.internal we
@@ -206,6 +209,9 @@ class TestAccountBroker(unittest.TestCase):
else:
broker.put_container(name, ts.internal, 0, 0, 0, pidx)
+ # commit pending file into db
+ broker._commit_puts()
+
def count_reclaimable(conn, reclaim_age):
return conn.execute(
"SELECT count(*) FROM container "
@@ -251,7 +257,7 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_db_status(self):
start = next(self.ts)
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(start.internal)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], start.internal)
@@ -273,10 +279,12 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_container(self):
# Test AccountBroker.delete_container
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -287,6 +295,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001)
broker.put_container('o', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -297,13 +306,15 @@ class TestAccountBroker(unittest.TestCase):
def test_put_container(self):
# Test AccountBroker.put_container
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
# Create initial container
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -317,6 +328,7 @@ class TestAccountBroker(unittest.TestCase):
# Reput same event
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -332,6 +344,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -346,6 +359,7 @@ class TestAccountBroker(unittest.TestCase):
otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -360,6 +374,7 @@ class TestAccountBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -378,6 +393,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -393,6 +409,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -405,7 +422,7 @@ class TestAccountBroker(unittest.TestCase):
def test_get_info(self):
# Test AccountBroker.get_info
- broker = AccountBroker(':memory:', account='test1')
+ broker = AccountBroker(self.get_db_path(), account='test1')
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
@@ -452,7 +469,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_containers_iter(self):
# Test AccountBroker.list_containers_iter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
for cont1 in range(4):
for cont2 in range(125):
@@ -582,7 +599,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal, 0)
broker.put_container(
@@ -617,7 +634,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_container_iter_with_reserved_name(self):
# Test ContainerBroker.list_objects_iter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0)
broker.put_container(
@@ -754,11 +771,13 @@ class TestAccountBroker(unittest.TestCase):
}
failures = []
for expected in expectations:
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0)
for name in expected['containers']:
broker.put_container(name, next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(c[0] for c in broker.list_containers_iter(**params))
@@ -773,7 +792,7 @@ class TestAccountBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
@@ -814,7 +833,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_chexor(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(1).internal,
Timestamp(0).internal, 0, 0,
@@ -842,9 +861,9 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_info()['hash'], hashc)
def test_merge_items(self):
- broker1 = AccountBroker(':memory:', account='a')
+ broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal)
- broker2 = AccountBroker(':memory:', account='a')
+ broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal)
broker1.put_container('a', Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx)
@@ -869,10 +888,10 @@ class TestAccountBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}'
if six.PY2:
snowman = snowman.encode('utf-8')
- broker1 = AccountBroker(':memory:', account='a')
+ broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal, 0)
id1 = broker1.get_info()['id']
- broker2 = AccountBroker(':memory:', account='a')
+ broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_container(snowman, Timestamp(2).internal, 0, 1, 100,
POLICIES.default.idx)
@@ -974,7 +993,7 @@ class TestAccountBroker(unittest.TestCase):
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)])
def test_get_policy_stats(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal)
# check empty policy_stats
self.assertTrue(broker.empty())
@@ -1041,7 +1060,7 @@ class TestAccountBroker(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
def test_policy_stats_tracking(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal)
# policy 0
@@ -1162,6 +1181,7 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp):
class TestCommonAccountBroker(test_db.TestExampleBroker):
broker_class = AccountBroker
+ server_type = 'account'
def setUp(self):
super(TestCommonAccountBroker, self).setUp()
@@ -1183,13 +1203,14 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
"""
def setUp(self):
+ super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_account_stat_table = \
AccountBroker.create_account_stat_table
AccountBroker.create_account_stat_table = \
premetadata_create_account_stat_table
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
@@ -1202,10 +1223,11 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
def tearDown(self):
AccountBroker.create_account_stat_table = \
self._imported_create_account_stat_table
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT metadata FROM account_stat')
+ super(TestAccountBrokerBeforeMetadata, self).tearDown()
def prespi_create_container_table(self, conn):
@@ -1270,6 +1292,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
"""
def setUp(self):
+ super(TestAccountBrokerBeforeSPI, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_container_table = \
@@ -1278,7 +1301,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
prespi_create_container_table
self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
@@ -1299,10 +1322,11 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
AccountBroker.create_container_table = \
self._imported_create_container_table
AccountBroker._initialize = self._imported_initialize
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container')
+ super(TestAccountBrokerBeforeSPI, self).tearDown()
@with_tempdir
def test_policy_table_migration(self, tempdir):
@@ -1681,7 +1705,7 @@ def pre_track_containers_create_container_table(self, conn):
""" + OLD_POLICY_STAT_TRIGGER_SCRIPT)
-class AccountBrokerPreTrackContainerCountSetup(object):
+class AccountBrokerPreTrackContainerCountSetup(test_db.TestDbBase):
def assertUnmigrated(self, broker):
with broker.get() as conn:
try:
@@ -1696,6 +1720,7 @@ class AccountBrokerPreTrackContainerCountSetup(object):
'trying to select container_count from policy_stat!')
def setUp(self):
+ super(AccountBrokerPreTrackContainerCountSetup, self).setUp()
# use old version of policy_stat
self._imported_create_policy_stat_table = \
AccountBroker.create_policy_stat_table
@@ -1708,17 +1733,16 @@ class AccountBrokerPreTrackContainerCountSetup(object):
AccountBroker.create_container_table = \
pre_track_containers_create_container_table
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
self.assertUnmigrated(broker)
- self.tempdir = mkdtemp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
- self.db_path = os.path.join(self.tempdir, 'sda', 'accounts',
+ self.db_path = os.path.join(self.testdir, 'sda', 'accounts',
'0', '0', '0', 'test.db')
- self.broker = AccountBroker(self.db_path, account='a')
+ self.broker = AccountBroker(self.get_db_path(), account='a')
self.broker.initialize(next(self.ts).internal)
# Common sanity-check that our starting, pre-migration state correctly
@@ -1726,14 +1750,13 @@ class AccountBrokerPreTrackContainerCountSetup(object):
self.assertUnmigrated(self.broker)
def tearDown(self):
- rmtree(self.tempdir, ignore_errors=True)
-
self.restore_account_broker()
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT container_count FROM policy_stat')
+ super(AccountBrokerPreTrackContainerCountSetup, self).tearDown()
def restore_account_broker(self):
AccountBroker.create_policy_stat_table = \
@@ -1925,7 +1948,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
self.assertEqual(policy_stat['container_count'], 1)
def test_migrate_add_storage_policy_index_fail(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.db_path, account='a')
broker.initialize(Timestamp('1').internal)
with mock.patch.object(
broker, 'create_policy_stat_table',
diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py
index eb332395a..473b3b053 100644
--- a/test/unit/account/test_utils.py
+++ b/test/unit/account/test_utils.py
@@ -26,6 +26,7 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_reserved_name
from test.unit import patch_policies, make_timestamp_iter
+from test.unit.common.test_db import TestDbBase
class TestFakeAccountBroker(unittest.TestCase):
@@ -58,9 +59,11 @@ class TestFakeAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_policy_stats(), {})
-class TestAccountUtils(unittest.TestCase):
+class TestAccountUtils(TestDbBase):
+ server_type = 'account'
def setUp(self):
+ super(TestAccountUtils, self).setUp()
self.ts = make_timestamp_iter()
def test_get_response_headers_fake_broker(self):
@@ -78,7 +81,7 @@ class TestAccountUtils(unittest.TestCase):
self.assertEqual(resp_headers, expected)
def test_get_response_headers_empty_memory_broker(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@@ -94,7 +97,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies
def test_get_response_headers_with_data(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@@ -141,7 +144,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies
def test_get_response_headers_with_legacy_data(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@@ -213,7 +216,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', is_default=True)])
def test_account_listing_reserved_names(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts)
now = time.time()
with mock.patch('time.time', new=lambda: now):
diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py
index b7b4aa114..d8bbf7e7c 100644
--- a/test/unit/common/test_db.py
+++ b/test/unit/common/test_db.py
@@ -46,7 +46,7 @@ from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException
-from test.unit import with_tempdir, make_timestamp_iter
+from test.unit import make_timestamp_iter, generate_db_path
class TestHelperFunctions(unittest.TestCase):
@@ -180,10 +180,35 @@ class TestGreenDBConnection(unittest.TestCase):
InterceptConnection.commit.call_count))
-class TestGetDBConnection(unittest.TestCase):
+class TestDbBase(unittest.TestCase):
+ server_type = 'container'
+ testdir = None
+
+ def setUp(self):
+ self.testdir = mkdtemp()
+ self.db_path = self.get_db_path()
+
+ def tearDown(self):
+ rmtree(self.testdir, ignore_errors=True)
+
+ def get_db_path(self):
+ return generate_db_path(self.testdir, self.server_type)
+
+
+class TestGetDBConnection(TestDbBase):
+ def setUp(self):
+ super(TestGetDBConnection, self).setUp()
+ self.db_path = self.init_db_path()
+
+ def init_db_path(self):
+ # Test ContainerBroker.empty
+ db_path = self.get_db_path()
+ broker = ExampleBroker(db_path, account='a')
+ broker.initialize(Timestamp.now().internal, 0)
+ return db_path
def test_normal_case(self):
- conn = get_db_connection(':memory:')
+ conn = get_db_connection(self.db_path)
self.assertTrue(hasattr(conn, 'execute'))
def test_invalid_path(self):
@@ -201,8 +226,8 @@ class TestGetDBConnection(unittest.TestCase):
InterceptCursor.execute = mock_db_cmd
with patch('sqlite3.Cursor', new=InterceptCursor):
- self.assertRaises(Timeout, get_db_connection, ':memory:',
- timeout=0.1)
+ self.assertRaises(Timeout, get_db_connection,
+ self.db_path, timeout=0.1)
self.assertTrue(mock_db_cmd.called)
self.assertEqual(mock_db_cmd.call_args_list,
list((mock_db_cmd.call_args,) *
@@ -319,7 +344,7 @@ class ExampleBroker(DatabaseBroker):
Timestamp(info['put_timestamp']))
-class TestExampleBroker(unittest.TestCase):
+class TestExampleBroker(TestDbBase):
"""
Tests that use the mostly Concrete enough ExampleBroker to exercise some
of the abstract methods on DatabaseBroker.
@@ -327,19 +352,21 @@ class TestExampleBroker(unittest.TestCase):
broker_class = ExampleBroker
policy = 0
+ server_type = 'example'
def setUp(self):
+ super(TestExampleBroker, self).setUp()
self.ts = make_timestamp_iter()
def test_delete_db(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal)
broker.delete_db(next(self.ts).internal)
self.assertTrue(broker.is_deleted())
def test_merge_timestamps_simple_delete(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp)
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
@@ -366,7 +393,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_delete_with_objects(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
@@ -397,7 +424,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_simple_recreate(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
virgin_status_changed_at = broker.get_info()['status_changed_at']
created_at = broker.get_info()['created_at']
@@ -425,7 +452,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_recreate_with_objects(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
# delete
@@ -459,7 +486,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_put_no_status_change(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
info = broker.get_info()
orig_status_changed_at = info['status_changed_at']
@@ -472,7 +499,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_delete_no_status_change(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp,
@@ -486,19 +513,23 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_get_max_row(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
self.assertEqual(-1, broker.get_max_row())
self.put_item(broker, next(self.ts).internal)
+ # commit pending file into db
+ broker._commit_puts()
self.assertEqual(1, broker.get_max_row())
self.delete_item(broker, next(self.ts).internal)
+ broker._commit_puts()
self.assertEqual(2, broker.get_max_row())
self.put_item(broker, next(self.ts).internal)
+ broker._commit_puts()
self.assertEqual(3, broker.get_max_row())
def test_get_info(self):
- broker = self.broker_class(':memory:', account='test', container='c')
+ broker = self.broker_class(self.db_path, account='test', container='c')
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(Timestamp(1).internal,
@@ -518,7 +549,7 @@ class TestExampleBroker(unittest.TestCase):
k, info[k], v))
def test_get_raw_metadata(self):
- broker = self.broker_class(':memory:', account='test', container='c')
+ broker = self.broker_class(self.db_path, account='test', container='c')
broker.initialize(Timestamp(0).internal,
storage_policy_index=int(self.policy))
self.assertEqual(broker.metadata, {})
@@ -543,7 +574,7 @@ class TestExampleBroker(unittest.TestCase):
json.dumps(metadata))
def test_put_timestamp(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
orig_put_timestamp = next(self.ts).internal
broker.initialize(orig_put_timestamp,
storage_policy_index=int(self.policy))
@@ -564,7 +595,7 @@ class TestExampleBroker(unittest.TestCase):
newer_put_timestamp)
def test_status_changed_at(self):
- broker = self.broker_class(':memory:', account='test', container='c')
+ broker = self.broker_class(self.db_path, account='test', container='c')
put_timestamp = next(self.ts).internal
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
@@ -590,7 +621,7 @@ class TestExampleBroker(unittest.TestCase):
status_changed_at)
def test_get_syncs(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(Timestamp.now().internal,
storage_policy_index=int(self.policy))
self.assertEqual([], broker.get_syncs())
@@ -603,9 +634,8 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual([{'sync_point': 2, 'remote_id': 'remote2'}],
broker.get_syncs(incoming=False))
- @with_tempdir
- def test_commit_pending(self, tempdir):
- broker = self.broker_class(os.path.join(tempdir, 'test.db'),
+ def test_commit_pending(self):
+ broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
@@ -616,12 +646,12 @@ class TestExampleBroker(unittest.TestCase):
info = rows[0]
count_key = '%s_count' % broker.db_contains_type
self.assertEqual(0, info[count_key])
- broker.get_info()
+ # commit pending file into db
+ broker._commit_puts()
self.assertEqual(1, broker.get_info()[count_key])
- @with_tempdir
- def test_maybe_get(self, tempdir):
- broker = self.broker_class(os.path.join(tempdir, 'test.db'),
+ def test_maybe_get(self):
+ broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
@@ -639,13 +669,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(broker.conn, conn)
-class TestDatabaseBroker(unittest.TestCase):
-
- def setUp(self):
- self.testdir = mkdtemp()
-
- def tearDown(self):
- rmtree(self.testdir, ignore_errors=1)
+class TestDatabaseBroker(TestDbBase):
def test_DB_PREALLOCATION_setting(self):
u = uuid4().hex
@@ -656,8 +680,8 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertRaises(OSError, b._preallocate)
def test_memory_db_init(self):
- broker = DatabaseBroker(':memory:')
- self.assertEqual(broker.db_file, ':memory:')
+ broker = DatabaseBroker(self.db_path)
+ self.assertEqual(broker.db_file, self.db_path)
self.assertRaises(AttributeError, broker.initialize,
normalize_timestamp('0'))
@@ -686,7 +710,7 @@ class TestDatabaseBroker(unittest.TestCase):
def test_initialize(self):
self.assertRaises(AttributeError,
- DatabaseBroker(':memory:').initialize,
+ DatabaseBroker(self.db_path).initialize,
normalize_timestamp('1'))
stub_dict = {}
@@ -694,7 +718,7 @@ class TestDatabaseBroker(unittest.TestCase):
stub_dict.clear()
stub_dict['args'] = args
stub_dict.update(kwargs)
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker._initialize = stub
broker.initialize(normalize_timestamp('1'))
self.assertTrue(hasattr(stub_dict['args'][0], 'execute'))
@@ -734,7 +758,7 @@ class TestDatabaseBroker(unittest.TestCase):
def do_test(expected_metadata, delete_meta_whitelist=None):
if not delete_meta_whitelist:
delete_meta_whitelist = []
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.get_db_path())
broker.delete_meta_whitelist = delete_meta_whitelist
broker.db_type = 'test'
broker._initialize = init_stub
@@ -790,13 +814,13 @@ class TestDatabaseBroker(unittest.TestCase):
['x-container-meta-test', 'x-something-else'])
def test_get(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
with self.assertRaises(DatabaseConnectionError) as raised, \
broker.get() as conn:
conn.execute('SELECT 1')
self.assertEqual(
str(raised.exception),
- "DB connection error (:memory:, 0):\nDB doesn't exist")
+ "DB connection error (%s, 0):\nDB doesn't exist" % self.db_path)
broker = DatabaseBroker(os.path.join(self.testdir, '1.db'))
with self.assertRaises(DatabaseConnectionError) as raised, \
@@ -932,7 +956,7 @@ class TestDatabaseBroker(unittest.TestCase):
pass
def test_newid(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
@@ -983,7 +1007,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(points[0][1], uuid2)
def test_get_items_since(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
@@ -1005,7 +1029,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_items_since(999, 2), [])
def test_get_sync(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
@@ -1045,7 +1069,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_sync(uuid3, incoming=False), 2)
def test_merge_syncs(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
def stub(*args, **kwargs):
pass
@@ -1090,7 +1114,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.get_replication_info_tester(metadata=True)
def get_replication_info_tester(self, metadata=False):
- broker = DatabaseBroker(':memory:', account='a')
+ broker = DatabaseBroker(self.db_path, account='a')
broker.db_type = 'test'
broker.db_contains_type = 'test'
broker.db_reclaim_timestamp = 'created_at'
@@ -1413,7 +1437,7 @@ class TestDatabaseBroker(unittest.TestCase):
(dbpath, qpath, hint))
def test_skip_commits(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
self.assertTrue(broker._skip_commit_puts())
broker._initialize = MagicMock()
broker.initialize(Timestamp.now())
@@ -1570,7 +1594,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertFalse(pending)
-class TestTombstoneReclaimer(unittest.TestCase):
+class TestTombstoneReclaimer(TestDbBase):
def _make_object(self, broker, obj_name, ts, deleted):
if deleted:
broker.delete_test(obj_name, ts.internal)
@@ -1588,7 +1612,8 @@ class TestTombstoneReclaimer(unittest.TestCase):
return self._count_reclaimable(conn, reclaim_age)
def _setup_tombstones(self, reverse_names=True):
- broker = ExampleBroker(':memory:', account='test_account',
+ broker = ExampleBroker(self.db_path,
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
now = time.time()
diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py
index c1ed3ca6d..8037d6793 100644
--- a/test/unit/container/test_backend.py
+++ b/test/unit/container/test_backend.py
@@ -53,13 +53,15 @@ from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
from test.unit.common import test_db
-class TestContainerBroker(unittest.TestCase):
+class TestContainerBroker(test_db.TestDbBase):
"""Tests for ContainerBroker"""
expected_db_tables = {'outgoing_sync', 'incoming_sync', 'object',
'sqlite_sequence', 'policy_stat',
'container_info', 'shard_range'}
+ server_type = 'container'
def setUp(self):
+ super(TestContainerBroker, self).setUp()
self.ts = make_timestamp_iter()
def _assert_shard_ranges(self, broker, expected, include_own=False):
@@ -70,8 +72,9 @@ class TestContainerBroker(unittest.TestCase):
def test_creation(self):
# Test ContainerBroker.__init__
- broker = ContainerBroker(':memory:', account='a', container='c')
- self.assertEqual(broker._db_file, ':memory:')
+ db_file = self.get_db_path()
+ broker = ContainerBroker(db_file, account='a', container='c')
+ self.assertEqual(broker._db_file, db_file)
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
curs = conn.cursor()
@@ -83,6 +86,8 @@ class TestContainerBroker(unittest.TestCase):
# check the update trigger
broker.put_object('blah', Timestamp.now().internal, 0, 'text/plain',
'etag', 0, 0)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
with self.assertRaises(sqlite3.DatabaseError) as cm:
conn.execute('UPDATE object SET name="blah";')
@@ -98,7 +103,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies
def test_storage_policy_property(self):
for policy in POLICIES:
- broker = ContainerBroker(':memory:', account='a',
+ broker = ContainerBroker(self.get_db_path(), account='a',
container='policy_%s' % policy.name)
broker.initialize(next(self.ts).internal, policy.idx)
with broker.get() as conn:
@@ -121,7 +126,8 @@ class TestContainerBroker(unittest.TestCase):
# Test ContainerBroker throwing a conn away after
# unhandled exception
first_conn = None
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
first_conn = conn
@@ -618,11 +624,14 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual('.shards_a/cc', broker.root_path)
def test_reclaim(self):
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -640,6 +649,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.delete_object('o', Timestamp.now().internal)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -673,6 +683,7 @@ class TestContainerBroker(unittest.TestCase):
'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('z', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ broker._commit_puts()
# Test before deletion
broker.reclaim(Timestamp.now().internal, time())
broker.delete_db(Timestamp.now().internal)
@@ -689,7 +700,8 @@ class TestContainerBroker(unittest.TestCase):
random.seed(now)
random.shuffle(obj_specs)
policy_indexes = list(p.idx for p in POLICIES)
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
for i, obj_spec in enumerate(obj_specs):
@@ -703,6 +715,8 @@ class TestContainerBroker(unittest.TestCase):
else:
broker.put_object(obj_name, ts.internal, 0, 'text/plain',
'etag', storage_policy_index=pidx)
+ # commit pending file into db
+ broker._commit_puts()
def count_reclaimable(conn, reclaim_age):
return conn.execute(
@@ -749,7 +763,8 @@ class TestContainerBroker(unittest.TestCase):
trace[1][2] > trace[2][2])
def test_reclaim_with_duplicate_names(self):
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
now = time()
@@ -758,6 +773,8 @@ class TestContainerBroker(unittest.TestCase):
for spidx in range(10):
obj_name = 'object%s' % i
broker.delete_object(obj_name, ages_ago.internal, spidx)
+ # commit pending file into db
+ broker._commit_puts()
reclaim_age = now - (2 * 7 * 24 * 60 * 60)
with broker.get() as conn:
self.assertEqual(conn.execute(
@@ -844,7 +861,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info_is_deleted(self):
ts = make_timestamp_iter()
start = next(ts)
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
# create it
broker.initialize(start.internal, POLICIES.default.idx)
@@ -893,10 +911,13 @@ class TestContainerBroker(unittest.TestCase):
def test_delete_object(self):
# Test ContainerBroker.delete_object
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -906,6 +927,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.delete_object('o', Timestamp.now().internal)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -916,7 +938,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object(self):
# Test ContainerBroker.put_object
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
# Create initial object
@@ -924,6 +947,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -945,6 +970,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -968,6 +994,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 124,
'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -990,6 +1017,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', otimestamp, 124,
'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1011,6 +1039,7 @@ class TestContainerBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_object('"{<object \'&\' name>}"', dtimestamp, 0, '', '',
deleted=1)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1033,6 +1062,7 @@ class TestContainerBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_object('"{<object \'&\' name>}"', timestamp, 0, '', '',
deleted=1)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1048,6 +1078,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1096,6 +1127,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 456,
'application/x-test3',
'6af83e3196bf99f440f31f2e1a6c9afe')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1115,7 +1147,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_range_single_record(self):
# Test ContainerBroker.merge_shard_range
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
# Stash these for later
@@ -1420,7 +1453,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_ranges_deleted(self):
# Test ContainerBroker.merge_shard_ranges sets deleted attribute
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
# put shard range
broker.merge_shard_ranges(ShardRange('a/o', next(self.ts).internal))
@@ -1453,7 +1487,8 @@ class TestContainerBroker(unittest.TestCase):
'storage_policy_index': '2',
'ctype_timestamp': None,
'meta_timestamp': None}
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
expect = ('obj', '1234567890.12345', 42, 'text/plain', 'hash_test',
'1', '2', None, None)
@@ -1627,7 +1662,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_encoded_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
self._test_put_object_multiple_encoded_timestamps(broker)
@with_tempdir
@@ -1961,7 +1997,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_explicit_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps passed as explicit args
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
self._test_put_object_multiple_explicit_timestamps(broker)
@with_tempdir
@@ -1977,7 +2014,8 @@ class TestContainerBroker(unittest.TestCase):
# Test container listing reports the most recent of data or metadata
# timestamp as last-modified time
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(next(ts).internal, 0)
# simple 'single' timestamp case
@@ -2030,7 +2068,7 @@ class TestContainerBroker(unittest.TestCase):
def test_put_misplaced_object_does_not_effect_container_stats(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@@ -2057,7 +2095,7 @@ class TestContainerBroker(unittest.TestCase):
def test_has_multiple_policies(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@@ -2069,18 +2107,21 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx)
+ # commit pending file into db
+ broker._commit_puts()
self.assertFalse(broker.has_multiple_policies())
other_policy = [p for p in POLICIES if p is not policy][0]
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx)
+ broker._commit_puts()
self.assertTrue(broker.has_multiple_policies())
@patch_policies
def test_get_policy_info(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@@ -2097,6 +2138,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx)
+ # commit pending file into db
+ broker._commit_puts()
policy_stats = broker.get_policy_stats()
expected = {policy.idx: {'bytes_used': 123, 'object_count': 1}}
self.assertEqual(policy_stats, expected)
@@ -2107,6 +2150,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx)
+ broker._commit_puts()
policy_stats = broker.get_policy_stats()
expected = {
policy.idx: {'bytes_used': 123, 'object_count': 1},
@@ -2117,7 +2161,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies
def test_policy_stat_tracking(self):
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
# Note: in subclasses of this TestCase that inherit the
# ContainerBrokerMigrationMixin, passing POLICIES.default.idx here has
@@ -2149,6 +2193,8 @@ class TestContainerBroker(unittest.TestCase):
# track the size of the latest timestamp put for each object
# in each storage policy
stats[policy_index][name] = size
+ # commit pending file into db
+ broker._commit_puts()
policy_stats = broker.get_policy_stats()
if POLICIES.default.idx not in stats:
# unlikely, but check empty default index still in policy stats
@@ -2161,7 +2207,7 @@ class TestContainerBroker(unittest.TestCase):
sum(stats[policy_index].values()))
def test_initialize_container_broker_in_default(self):
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
# initialize with no storage_policy_index argument
@@ -2200,7 +2246,7 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info(self):
# Test ContainerBroker.get_info
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@@ -2341,7 +2387,7 @@ class TestContainerBroker(unittest.TestCase):
'db_state': 'collapsed'})
def test_set_x_syncs(self):
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@@ -2355,7 +2401,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(info['x_container_sync_point2'], 2)
def test_get_report_info(self):
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@@ -2527,7 +2573,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertFalse(get_rows(broker))
def test_get_objects(self):
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
objects_0 = [{'name': 'obj_0_%d' % i,
'created_at': next(self.ts).normal,
@@ -2575,7 +2622,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(objects_0 + objects_1, actual)
def test_get_objects_since_row(self):
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
obj_names = ['obj%03d' % i for i in range(20)]
timestamps = [next(self.ts) for o in obj_names]
@@ -2625,7 +2673,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter(self):
# Test ContainerBroker.list_objects_iter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4):
for obj2 in range(125):
@@ -2775,7 +2824,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
def test_list_objects_iter_with_reserved_name(self):
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(next(self.ts).internal, 0)
broker.put_object(
@@ -2934,10 +2984,13 @@ class TestContainerBroker(unittest.TestCase):
}
failures = []
for expected in expectations:
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker.initialize(next(ts).internal, 0)
for name in expected['objects']:
broker.put_object(name, next(ts).internal, **obj_create_params)
+ # commit pending file into db
+ broker._commit_puts()
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(o[0] for o in broker.list_objects_iter(**params))
@@ -2952,7 +3005,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_non_slash(self):
# Test ContainerBroker.list_objects_iter using a
# delimiter that is not a slash
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4):
for obj2 in range(125):
@@ -3069,7 +3123,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_prefix_delim(self):
# Test ContainerBroker.list_objects_iter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
@@ -3108,7 +3163,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
@@ -3150,7 +3206,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3230,7 +3287,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter_non_slash(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3313,7 +3371,8 @@ class TestContainerBroker(unittest.TestCase):
s = s.encode('utf8')
return md5(s, usedforsecurity=False).hexdigest()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3365,7 +3424,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_items_since(self):
# test DatabaseBroker.get_items_since
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3378,9 +3438,11 @@ class TestContainerBroker(unittest.TestCase):
def test_sync_merging(self):
# exercise the DatabaseBroker sync functions a bit
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
self.assertEqual(broker2.get_sync('12345'), -1)
broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}])
@@ -3388,14 +3450,18 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(broker2.get_sync('12345'), 3)
def test_merge_items(self):
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
id = broker1.get_info()['id']
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
@@ -3404,6 +3470,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_object('c', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3431,19 +3498,24 @@ class TestContainerBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}'
if six.PY2:
snowman = snowman.encode('utf-8')
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object(snowman, Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id)
broker1.put_object(snowman, Timestamp(4).internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id)
items = broker2.get_items_since(-1, 1000)
@@ -3457,19 +3529,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_overwrite(self):
# test DatabaseBroker.merge_items
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3482,19 +3559,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_post_overwrite_out_of_order(self):
# test DatabaseBroker.merge_items
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3514,6 +3596,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(rec['created_at'], Timestamp(3).internal)
broker1.put_object('b', Timestamp(5).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3527,7 +3610,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index(self):
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
timestamp = next(ts)
broker.initialize(timestamp.internal, 0)
@@ -3584,7 +3668,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index_empty(self):
# Putting an object may trigger migrations, so test with a
# never-had-an-object container to make sure we handle it
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
info = broker.get_info()
@@ -3595,7 +3680,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(2, info['storage_policy_index'])
def test_reconciler_sync(self):
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
self.assertEqual(-1, broker.get_reconciler_sync())
@@ -5234,6 +5320,7 @@ class TestContainerBroker(unittest.TestCase):
class TestCommonContainerBroker(test_db.TestExampleBroker):
broker_class = ContainerBroker
+ server_type = 'container'
def setUp(self):
super(TestCommonContainerBroker, self).setUp()
@@ -5248,7 +5335,7 @@ class TestCommonContainerBroker(test_db.TestExampleBroker):
storage_policy_index=int(self.policy))
-class ContainerBrokerMigrationMixin(object):
+class ContainerBrokerMigrationMixin(test_db.TestDbBase):
"""
Mixin for running ContainerBroker against databases created with
older schemas.
@@ -5263,6 +5350,7 @@ class ContainerBrokerMigrationMixin(object):
return self.func.__get__(obj, obj_type)
def setUp(self):
+ super(ContainerBrokerMigrationMixin, self).setUp()
self._imported_create_object_table = \
ContainerBroker.create_object_table
ContainerBroker.create_object_table = \
@@ -5303,6 +5391,7 @@ class ContainerBrokerMigrationMixin(object):
self._imported_create_shard_range_table
ContainerBroker.create_policy_stat_table = \
self._imported_create_policy_stat_table
+ # We need to manually teardown and clean the self.tempdir
def premetadata_create_container_info_table(self, conn, put_timestamp,
@@ -5359,7 +5448,8 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def setUp(self):
super(TestContainerBrokerBeforeMetadata, self).setUp()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
exc = None
with broker.get() as conn:
@@ -5371,10 +5461,12 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeMetadata, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT metadata FROM container_stat')
+ test_db.TestDbBase.tearDown(self)
def prexsync_create_container_info_table(self, conn, put_timestamp,
@@ -5435,7 +5527,8 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
super(TestContainerBrokerBeforeXSync, self).setUp()
ContainerBroker.create_container_info_table = \
prexsync_create_container_info_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
exc = None
with broker.get() as conn:
@@ -5448,10 +5541,12 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeXSync, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT x_container_sync_point1 FROM container_stat')
+ test_db.TestDbBase.tearDown(self)
def prespi_create_object_table(self, conn, *args, **kwargs):
@@ -5552,7 +5647,8 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
ContainerBroker.create_container_info_table = \
prespi_create_container_info_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -5563,10 +5659,12 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeSPI, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container_stat')
+ test_db.TestDbBase.tearDown(self)
@patch_policies
@with_tempdir
@@ -5760,7 +5858,8 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def setUp(self):
super(TestContainerBrokerBeforeShardRanges, self).setUp()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -5770,11 +5869,13 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeShardRanges, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT *
FROM shard_range''')
+ test_db.TestDbBase.tearDown(self)
def pre_reported_create_shard_range_table(self, conn):
@@ -5828,7 +5929,8 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
ContainerBroker.create_shard_range_table = \
pre_reported_create_shard_range_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -5839,11 +5941,13 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
def tearDown(self):
super(TestContainerBrokerBeforeShardRangeReportedColumn,
self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT reported
FROM shard_range''')
+ test_db.TestDbBase.tearDown(self)
@with_tempdir
def test_get_shard_ranges_attempts(self, tempdir):
@@ -6056,7 +6160,8 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
ContainerBroker.create_shard_range_table = \
pre_tombstones_create_shard_range_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -6067,11 +6172,13 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
def tearDown(self):
super(TestContainerBrokerBeforeShardRangeTombstonesColumn,
self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT tombstones
FROM shard_range''')
+ test_db.TestDbBase.tearDown(self)
class TestUpdateNewItemFromExisting(unittest.TestCase):
View Lorry Controller Status