summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoshua Harlow <harlowja@gmail.com>2014-08-12 19:11:30 -0700
committerJoshua Harlow <harlowja@yahoo-inc.com>2014-09-16 11:27:39 -0700
commit37a80126f2be04f11dd3d56465de103bd8578782 (patch)
treee3495477876b2591d35ee30f3e6d6a94dc5fc1e4
parentc5e1c3f96f016c9ddfff8206d1bebeaf4ea44e6c (diff)
downloadtooz-37a80126f2be04f11dd3d56465de103bd8578782.tar.gz
Use the more reliable sysv_ipc instead of posix_ipc+lockutils
Remove the usage of lockutils (and surronding oslo modules) and keep it much simpler by just using the sysv_ipc module directly. This adjustment reduces the complexity, test exposure and API complexity of tooz; removing much of the complex functionality that was pulled in when sucking in the lockutils module for its simple (but still broken) posix_ipc logic. Change-Id: I46db9bda252473317a0f3d65fe0329ac2151f332
-rw-r--r--openstack-common.conf1
-rw-r--r--requirements.txt3
-rw-r--r--tooz/drivers/ipc.py79
-rw-r--r--tooz/drivers/memcached.py4
-rw-r--r--tooz/drivers/zookeeper.py4
-rw-r--r--tooz/locking.py24
-rw-r--r--tooz/openstack/common/excutils.py113
-rw-r--r--tooz/openstack/common/fileutils.py146
-rw-r--r--tooz/openstack/common/importutils.py73
-rw-r--r--tooz/openstack/common/jsonutils.py190
-rw-r--r--tooz/openstack/common/local.py45
-rw-r--r--tooz/openstack/common/lockutils.py379
-rw-r--r--tooz/openstack/common/log.py703
-rw-r--r--tooz/openstack/common/strutils.py295
-rw-r--r--tooz/openstack/common/timeutils.py210
-rw-r--r--tooz/tests/test_coordination.py3
16 files changed, 101 insertions, 2171 deletions
diff --git a/openstack-common.conf b/openstack-common.conf
index 6393b3e..0735b97 100644
--- a/openstack-common.conf
+++ b/openstack-common.conf
@@ -1,4 +1,3 @@
[DEFAULT]
-module=lockutils
module=network_utils
base=tooz
diff --git a/requirements.txt b/requirements.txt
index c46fda1..64a171a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,9 +4,8 @@ stevedore>=0.14
six>=1.7.0
iso8601
kazoo>=1.3.1
-oslo.config
pymemcache>=1.2
zake>=0.1.6
-posix_ipc
+sysv_ipc>=0.6.8
msgpack-python
retrying
diff --git a/tooz/drivers/ipc.py b/tooz/drivers/ipc.py
index 0e2a013..1f1c4f7 100644
--- a/tooz/drivers/ipc.py
+++ b/tooz/drivers/ipc.py
@@ -15,32 +15,93 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
-import posix_ipc
+
+import hashlib
+
+import six
+import sysv_ipc
import tooz
from tooz import coordination
from tooz import locking
-from tooz.openstack.common import lockutils
+
+if sysv_ipc.KEY_MIN <= 0:
+ _KEY_RANGE = abs(sysv_ipc.KEY_MIN) + sysv_ipc.KEY_MAX
+else:
+ _KEY_RANGE = sysv_ipc.KEY_MAX - sysv_ipc.KEY_MIN
class IPCLock(locking.Lock):
- _LOCK_PREFIX = b'_tooz_'
+ """A sysv IPC based lock.
+
+ Please ensure you have read over (and understand) the limitations of sysv
+ IPC locks, and especially have tried and used $ ipcs -l (note the maximum
+ number of semaphores system wide field that command outputs). To ensure
+ that you do not reach that limit it is recommended to use destroy() at
+ the correct program exit/entry points.
+ """
+ _LOCK_PROJECT = b'__TOOZ_LOCK_'
def __init__(self, name, timeout):
- self.lock = lockutils.external_lock(
- (self._LOCK_PREFIX + name).decode('ascii'))
+ super(IPCLock, self).__init__(name)
+ self.key = self.ftok(name, self._LOCK_PROJECT)
+ try:
+ self.lock = sysv_ipc.Semaphore(self.key,
+ flags=sysv_ipc.IPC_CREX,
+ initial_value=1)
+ except sysv_ipc.ExistentialError:
+ self.lock = sysv_ipc.Semaphore(self.key)
+ self.lock.undo = True
self.timeout = timeout
+ @staticmethod
+ def ftok(name, project):
+ # Similar to ftok & http://semanchuk.com/philip/sysv_ipc/#ftok_weakness
+ # but hopefully without as many weaknesses...
+ h = hashlib.md5()
+ if not isinstance(project, six.binary_type):
+ project = project.encode('ascii')
+ h.update(project)
+ if not isinstance(name, six.binary_type):
+ name = name.encode('ascii')
+ h.update(name)
+ return (int(h.hexdigest(), 16) % _KEY_RANGE) + sysv_ipc.KEY_MIN
+
def acquire(self, blocking=True):
timeout = self.timeout if blocking else 0
try:
- return bool(self.lock.acquire(timeout=timeout))
- # TODO(jd) This should be encapsulated in lockutils!
- except posix_ipc.BusyError:
+ self.lock.acquire(timeout=timeout)
+ except (sysv_ipc.BusyError, sysv_ipc.ExistentialError):
return False
+ else:
+ return True
def release(self):
- return self.lock.release()
+ try:
+ self.lock.release()
+ except sysv_ipc.ExistentialError:
+ return False
+ else:
+ return True
+
+ def destroy(self):
+ """This will destroy the lock.
+
+ NOTE(harlowja): this will destroy the lock, and if it is being shared
+ across processes this can have unintended consquences, so it *must*
+ only be used when it is *safe* to remove it (ie at a known program
+ exit point, where it can be ensured that no other process will be
+ using it, or that if other processes are using it they can tolerate
+ it being destroyed).
+
+ Read your man pages for `semctl(IPC_RMID)` before using this to
+ understand its side-effects on other programs that *may* be
+ concurrently using the same lock while it is being destroyed...
+ """
+ try:
+ self.lock.remove()
+ except sysv_ipc.ExistentialError:
+ pass
class IPCDriver(coordination.CoordinationDriver):
diff --git a/tooz/drivers/memcached.py b/tooz/drivers/memcached.py
index 085923c..e92abb2 100644
--- a/tooz/drivers/memcached.py
+++ b/tooz/drivers/memcached.py
@@ -45,8 +45,8 @@ class MemcachedLock(locking.Lock):
_LOCK_PREFIX = b'__TOOZ_LOCK_'
def __init__(self, coord, name, timeout):
+ super(MemcachedLock, self).__init__(self._LOCK_PREFIX + name)
self.coord = coord
- self.name = self._LOCK_PREFIX + name
self.timeout = timeout
@retry
@@ -64,7 +64,7 @@ class MemcachedLock(locking.Lock):
def release(self):
self.coord._acquired_locks.remove(self)
- self.coord.client.delete(self.name)
+ return bool(self.coord.client.delete(self.name))
def heartbeat(self):
"""Keep the lock alive."""
diff --git a/tooz/drivers/zookeeper.py b/tooz/drivers/zookeeper.py
index 9b110e1..a164458 100644
--- a/tooz/drivers/zookeeper.py
+++ b/tooz/drivers/zookeeper.py
@@ -27,7 +27,8 @@ from tooz import locking
class ZooKeeperLock(locking.Lock):
- def __init__(self, lock):
+ def __init__(self, name, lock):
+ super(ZooKeeperLock, self).__init__(name)
self._lock = lock
def acquire(self, blocking=True, timeout=None):
@@ -330,6 +331,7 @@ class KazooDriver(BaseZooKeeperDriver):
def get_lock(self, name):
return ZooKeeperLock(
+ name,
self._coord.Lock(
self.paths_join(b"/", self._TOOZ_NAMESPACE, b"locks", name),
self._member_id.decode('ascii')))
diff --git a/tooz/locking.py b/tooz/locking.py
index 0d246e7..43b43f7 100644
--- a/tooz/locking.py
+++ b/tooz/locking.py
@@ -20,6 +20,11 @@ import six
@six.add_metaclass(abc.ABCMeta)
class Lock(object):
+ def __init__(self, name):
+ if not name:
+ raise ValueError("Locks must be provided a name")
+ self.name = name
+
def __enter__(self):
self.acquire()
@@ -28,8 +33,23 @@ class Lock(object):
@abc.abstractmethod
def release(self):
- pass
+ """Attempts to release the lock, returns true if released.
+
+ The behavior of releasing a lock which was not acquired in the first
+ place is undefined (it can range from harmless to releasing some other
+ users lock)..
+
+ :returns: returns true if released (false if not)
+ :rtype: bool
+ """
@abc.abstractmethod
def acquire(self):
- pass
+ """Attempts to acquire the lock.
+
+ :returns: returns true if acquired (false if not)
+ :rtype: bool
+ """
+
+ def destroy(self):
+ """Removes the lock + any resources associated with the lock."""
diff --git a/tooz/openstack/common/excutils.py b/tooz/openstack/common/excutils.py
deleted file mode 100644
index a7d8bb0..0000000
--- a/tooz/openstack/common/excutils.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# Copyright 2012, Red Hat, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""
-Exception related utilities.
-"""
-
-import logging
-import sys
-import time
-import traceback
-
-import six
-
-from tooz.openstack.common.gettextutils import _LE
-
-
-class save_and_reraise_exception(object):
- """Save current exception, run some code and then re-raise.
-
- In some cases the exception context can be cleared, resulting in None
- being attempted to be re-raised after an exception handler is run. This
- can happen when eventlet switches greenthreads or when running an
- exception handler, code raises and catches an exception. In both
- cases the exception context will be cleared.
-
- To work around this, we save the exception state, run handler code, and
- then re-raise the original exception. If another exception occurs, the
- saved exception is logged and the new exception is re-raised.
-
- In some cases the caller may not want to re-raise the exception, and
- for those circumstances this context provides a reraise flag that
- can be used to suppress the exception. For example::
-
- except Exception:
- with save_and_reraise_exception() as ctxt:
- decide_if_need_reraise()
- if not should_be_reraised:
- ctxt.reraise = False
-
- If another exception occurs and reraise flag is False,
- the saved exception will not be logged.
-
- If the caller wants to raise new exception during exception handling
- he/she sets reraise to False initially with an ability to set it back to
- True if needed::
-
- except Exception:
- with save_and_reraise_exception(reraise=False) as ctxt:
- [if statements to determine whether to raise a new exception]
- # Not raising a new exception, so reraise
- ctxt.reraise = True
- """
- def __init__(self, reraise=True):
- self.reraise = reraise
-
- def __enter__(self):
- self.type_, self.value, self.tb, = sys.exc_info()
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- if exc_type is not None:
- if self.reraise:
- logging.error(_LE('Original exception being dropped: %s'),
- traceback.format_exception(self.type_,
- self.value,
- self.tb))
- return False
- if self.reraise:
- six.reraise(self.type_, self.value, self.tb)
-
-
-def forever_retry_uncaught_exceptions(infunc):
- def inner_func(*args, **kwargs):
- last_log_time = 0
- last_exc_message = None
- exc_count = 0
- while True:
- try:
- return infunc(*args, **kwargs)
- except Exception as exc:
- this_exc_message = six.u(str(exc))
- if this_exc_message == last_exc_message:
- exc_count += 1
- else:
- exc_count = 1
- # Do not log any more frequently than once a minute unless
- # the exception message changes
- cur_time = int(time.time())
- if (cur_time - last_log_time > 60 or
- this_exc_message != last_exc_message):
- logging.exception(
- _LE('Unexpected exception occurred %d time(s)... '
- 'retrying.') % exc_count)
- last_log_time = cur_time
- last_exc_message = this_exc_message
- exc_count = 0
- # This should be a very rare event. In case it isn't, do
- # a sleep.
- time.sleep(1)
- return inner_func
diff --git a/tooz/openstack/common/fileutils.py b/tooz/openstack/common/fileutils.py
deleted file mode 100644
index ae77acd..0000000
--- a/tooz/openstack/common/fileutils.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import contextlib
-import errno
-import os
-import tempfile
-
-from tooz.openstack.common import excutils
-from tooz.openstack.common import log as logging
-
-LOG = logging.getLogger(__name__)
-
-_FILE_CACHE = {}
-
-
-def ensure_tree(path):
- """Create a directory (and any ancestor directories required)
-
- :param path: Directory to create
- """
- try:
- os.makedirs(path)
- except OSError as exc:
- if exc.errno == errno.EEXIST:
- if not os.path.isdir(path):
- raise
- else:
- raise
-
-
-def read_cached_file(filename, force_reload=False):
- """Read from a file if it has been modified.
-
- :param force_reload: Whether to reload the file.
- :returns: A tuple with a boolean specifying if the data is fresh
- or not.
- """
- global _FILE_CACHE
-
- if force_reload:
- delete_cached_file(filename)
-
- reloaded = False
- mtime = os.path.getmtime(filename)
- cache_info = _FILE_CACHE.setdefault(filename, {})
-
- if not cache_info or mtime > cache_info.get('mtime', 0):
- LOG.debug("Reloading cached file %s" % filename)
- with open(filename) as fap:
- cache_info['data'] = fap.read()
- cache_info['mtime'] = mtime
- reloaded = True
- return (reloaded, cache_info['data'])
-
-
-def delete_cached_file(filename):
- """Delete cached file if present.
-
- :param filename: filename to delete
- """
- global _FILE_CACHE
-
- if filename in _FILE_CACHE:
- del _FILE_CACHE[filename]
-
-
-def delete_if_exists(path, remove=os.unlink):
- """Delete a file, but ignore file not found error.
-
- :param path: File to delete
- :param remove: Optional function to remove passed path
- """
-
- try:
- remove(path)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
-
-
-@contextlib.contextmanager
-def remove_path_on_error(path, remove=delete_if_exists):
- """Protect code that wants to operate on PATH atomically.
- Any exception will cause PATH to be removed.
-
- :param path: File to work with
- :param remove: Optional function to remove passed path
- """
-
- try:
- yield
- except Exception:
- with excutils.save_and_reraise_exception():
- remove(path)
-
-
-def file_open(*args, **kwargs):
- """Open file
-
- see built-in open() documentation for more details
-
- Note: The reason this is kept in a separate module is to easily
- be able to provide a stub module that doesn't alter system
- state at all (for unit tests)
- """
- return open(*args, **kwargs)
-
-
-def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):
- """Create temporary file or use existing file.
-
- This util is needed for creating temporary file with
- specified content, suffix and prefix. If path is not None,
- it will be used for writing content. If the path doesn't
- exist it'll be created.
-
- :param content: content for temporary file.
- :param path: same as parameter 'dir' for mkstemp
- :param suffix: same as parameter 'suffix' for mkstemp
- :param prefix: same as parameter 'prefix' for mkstemp
-
- For example: it can be used in database tests for creating
- configuration files.
- """
- if path:
- ensure_tree(path)
-
- (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
- try:
- os.write(fd, content)
- finally:
- os.close(fd)
- return path
diff --git a/tooz/openstack/common/importutils.py b/tooz/openstack/common/importutils.py
deleted file mode 100644
index 1c276d3..0000000
--- a/tooz/openstack/common/importutils.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""
-Import related utilities and helper functions.
-"""
-
-import sys
-import traceback
-
-
-def import_class(import_str):
- """Returns a class from a string including module and class."""
- mod_str, _sep, class_str = import_str.rpartition('.')
- __import__(mod_str)
- try:
- return getattr(sys.modules[mod_str], class_str)
- except AttributeError:
- raise ImportError('Class %s cannot be found (%s)' %
- (class_str,
- traceback.format_exception(*sys.exc_info())))
-
-
-def import_object(import_str, *args, **kwargs):
- """Import a class and return an instance of it."""
- return import_class(import_str)(*args, **kwargs)
-
-
-def import_object_ns(name_space, import_str, *args, **kwargs):
- """Tries to import object from default namespace.
-
- Imports a class and return an instance of it, first by trying
- to find the class in a default namespace, then failing back to
- a full path if not found in the default namespace.
- """
- import_value = "%s.%s" % (name_space, import_str)
- try:
- return import_class(import_value)(*args, **kwargs)
- except ImportError:
- return import_class(import_str)(*args, **kwargs)
-
-
-def import_module(import_str):
- """Import a module."""
- __import__(import_str)
- return sys.modules[import_str]
-
-
-def import_versioned_module(version, submodule=None):
- module = 'tooz.v%s' % version
- if submodule:
- module = '.'.join((module, submodule))
- return import_module(module)
-
-
-def try_import(import_str, default=None):
- """Try to import a module and if it fails return default."""
- try:
- return import_module(import_str)
- except ImportError:
- return default
diff --git a/tooz/openstack/common/jsonutils.py b/tooz/openstack/common/jsonutils.py
deleted file mode 100644
index ba79323..0000000
--- a/tooz/openstack/common/jsonutils.py
+++ /dev/null
@@ -1,190 +0,0 @@
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# Copyright 2011 Justin Santa Barbara
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-'''
-JSON related utilities.
-
-This module provides a few things:
-
- 1) A handy function for getting an object down to something that can be
- JSON serialized. See to_primitive().
-
- 2) Wrappers around loads() and dumps(). The dumps() wrapper will
- automatically use to_primitive() for you if needed.
-
- 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
- is available.
-'''
-
-
-import codecs
-import datetime
-import functools
-import inspect
-import itertools
-import sys
-
-if sys.version_info < (2, 7):
- # On Python <= 2.6, json module is not C boosted, so try to use
- # simplejson module if available
- try:
- import simplejson as json
- except ImportError:
- import json
-else:
- import json
-
-import six
-import six.moves.xmlrpc_client as xmlrpclib
-
-from tooz.openstack.common import gettextutils
-from tooz.openstack.common import importutils
-from tooz.openstack.common import strutils
-from tooz.openstack.common import timeutils
-
-netaddr = importutils.try_import("netaddr")
-
-_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
- inspect.isfunction, inspect.isgeneratorfunction,
- inspect.isgenerator, inspect.istraceback, inspect.isframe,
- inspect.iscode, inspect.isbuiltin, inspect.isroutine,
- inspect.isabstract]
-
-_simple_types = (six.string_types + six.integer_types
- + (type(None), bool, float))
-
-
-def to_primitive(value, convert_instances=False, convert_datetime=True,
- level=0, max_depth=3):
- """Convert a complex object into primitives.
-
- Handy for JSON serialization. We can optionally handle instances,
- but since this is a recursive function, we could have cyclical
- data structures.
-
- To handle cyclical data structures we could track the actual objects
- visited in a set, but not all objects are hashable. Instead we just
- track the depth of the object inspections and don't go too deep.
-
- Therefore, convert_instances=True is lossy ... be aware.
-
- """
- # handle obvious types first - order of basic types determined by running
- # full tests on nova project, resulting in the following counts:
- # 572754 <type 'NoneType'>
- # 460353 <type 'int'>
- # 379632 <type 'unicode'>
- # 274610 <type 'str'>
- # 199918 <type 'dict'>
- # 114200 <type 'datetime.datetime'>
- # 51817 <type 'bool'>
- # 26164 <type 'list'>
- # 6491 <type 'float'>
- # 283 <type 'tuple'>
- # 19 <type 'long'>
- if isinstance(value, _simple_types):
- return value
-
- if isinstance(value, datetime.datetime):
- if convert_datetime:
- return timeutils.strtime(value)
- else:
- return value
-
- # value of itertools.count doesn't get caught by nasty_type_tests
- # and results in infinite loop when list(value) is called.
- if type(value) == itertools.count:
- return six.text_type(value)
-
- # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
- # tests that raise an exception in a mocked method that
- # has a @wrap_exception with a notifier will fail. If
- # we up the dependency to 0.5.4 (when it is released) we
- # can remove this workaround.
- if getattr(value, '__module__', None) == 'mox':
- return 'mock'
-
- if level > max_depth:
- return '?'
-
- # The try block may not be necessary after the class check above,
- # but just in case ...
- try:
- recursive = functools.partial(to_primitive,
- convert_instances=convert_instances,
- convert_datetime=convert_datetime,
- level=level,
- max_depth=max_depth)
- if isinstance(value, dict):
- return dict((k, recursive(v)) for k, v in six.iteritems(value))
- elif isinstance(value, (list, tuple)):
- return [recursive(lv) for lv in value]
-
- # It's not clear why xmlrpclib created their own DateTime type, but
- # for our purposes, make it a datetime type which is explicitly
- # handled
- if isinstance(value, xmlrpclib.DateTime):
- value = datetime.datetime(*tuple(value.timetuple())[:6])
-
- if convert_datetime and isinstance(value, datetime.datetime):
- return timeutils.strtime(value)
- elif isinstance(value, gettextutils.Message):
- return value.data
- elif hasattr(value, 'iteritems'):
- return recursive(dict(value.iteritems()), level=level + 1)
- elif hasattr(value, '__iter__'):
- return recursive(list(value))
- elif convert_instances and hasattr(value, '__dict__'):
- # Likely an instance of something. Watch for cycles.
- # Ignore class member vars.
- return recursive(value.__dict__, level=level + 1)
- elif netaddr and isinstance(value, netaddr.IPAddress):
- return six.text_type(value)
- else:
- if any(test(value) for test in _nasty_type_tests):
- return six.text_type(value)
- return value
- except TypeError:
- # Class objects are tricky since they may define something like
- # __iter__ defined but it isn't callable as list().
- return six.text_type(value)
-
-
-def dumps(value, default=to_primitive, **kwargs):
- return json.dumps(value, default=default, **kwargs)
-
-
-def dump(obj, fp, *args, **kwargs):
- return json.dump(obj, fp, *args, **kwargs)
-
-
-def loads(s, encoding='utf-8', **kwargs):
- return json.loads(strutils.safe_decode(s, encoding), **kwargs)
-
-
-def load(fp, encoding='utf-8', **kwargs):
- return json.load(codecs.getreader(encoding)(fp), **kwargs)
-
-
-try:
- import anyjson
-except ImportError:
- pass
-else:
- anyjson._modules.append((__name__, 'dumps', TypeError,
- 'loads', ValueError, 'load'))
- anyjson.force_implementation(__name__)
diff --git a/tooz/openstack/common/local.py b/tooz/openstack/common/local.py
deleted file mode 100644
index 0819d5b..0000000
--- a/tooz/openstack/common/local.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""Local storage of variables using weak references"""
-
-import threading
-import weakref
-
-
-class WeakLocal(threading.local):
- def __getattribute__(self, attr):
- rval = super(WeakLocal, self).__getattribute__(attr)
- if rval:
- # NOTE(mikal): this bit is confusing. What is stored is a weak
- # reference, not the value itself. We therefore need to lookup
- # the weak reference and return the inner value here.
- rval = rval()
- return rval
-
- def __setattr__(self, attr, value):
- value = weakref.ref(value)
- return super(WeakLocal, self).__setattr__(attr, value)
-
-
-# NOTE(mikal): the name "store" should be deprecated in the future
-store = WeakLocal()
-
-# A "weak" store uses weak references and allows an object to fall out of scope
-# when it falls out of scope in the code that uses the thread local storage. A
-# "strong" store will hold a reference to the object so that it never falls out
-# of scope.
-weak_store = WeakLocal()
-strong_store = threading.local()
diff --git a/tooz/openstack/common/lockutils.py b/tooz/openstack/common/lockutils.py
deleted file mode 100644
index e9012f7..0000000
--- a/tooz/openstack/common/lockutils.py
+++ /dev/null
@@ -1,379 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import contextlib
-import errno
-import functools
-import logging
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import threading
-import time
-import weakref
-
-from oslo.config import cfg
-
-from tooz.openstack.common import fileutils
-from tooz.openstack.common.gettextutils import _, _LE, _LI
-
-
-LOG = logging.getLogger(__name__)
-
-
-util_opts = [
- cfg.BoolOpt('disable_process_locking', default=False,
- help='Enables or disables inter-process locks.'),
- cfg.StrOpt('lock_path',
- default=os.environ.get("TOOZ_LOCK_PATH"),
- help='Directory to use for lock files.')
-]
-
-
-CONF = cfg.CONF
-CONF.register_opts(util_opts)
-
-
-def set_defaults(lock_path):
- cfg.set_defaults(util_opts, lock_path=lock_path)
-
-
-class _FileLock(object):
- """Lock implementation which allows multiple locks, working around
- issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
- not require any cleanup. Since the lock is always held on a file
- descriptor rather than outside of the process, the lock gets dropped
- automatically if the process crashes, even if __exit__ is not executed.
-
- There are no guarantees regarding usage by multiple green threads in a
- single process here. This lock works only between processes. Exclusive
- access between local threads should be achieved using the semaphores
- in the @synchronized decorator.
-
- Note these locks are released when the descriptor is closed, so it's not
- safe to close the file descriptor while another green thread holds the
- lock. Just opening and closing the lock file can break synchronisation,
- so lock files must be accessed only using this abstraction.
- """
-
- def __init__(self, name):
- self.lockfile = None
- self.fname = name
-
- def acquire(self):
- basedir = os.path.dirname(self.fname)
-
- if not os.path.exists(basedir):
- fileutils.ensure_tree(basedir)
- LOG.info(_LI('Created lock path: %s'), basedir)
-
- self.lockfile = open(self.fname, 'w')
-
- while True:
- try:
- # Using non-blocking locks since green threads are not
- # patched to deal with blocking locking calls.
- # Also upon reading the MSDN docs for locking(), it seems
- # to have a laughable 10 attempts "blocking" mechanism.
- self.trylock()
- LOG.debug('Got file lock "%s"', self.fname)
- return True
- except IOError as e:
- if e.errno in (errno.EACCES, errno.EAGAIN):
- # external locks synchronise things like iptables
- # updates - give it some time to prevent busy spinning
- time.sleep(0.01)
- else:
- raise threading.ThreadError(_("Unable to acquire lock on"
- " `%(filename)s` due to"
- " %(exception)s") %
- {
- 'filename': self.fname,
- 'exception': e,
- })
-
- def __enter__(self):
- self.acquire()
- return self
-
- def release(self):
- try:
- self.unlock()
- self.lockfile.close()
- LOG.debug('Released file lock "%s"', self.fname)
- except IOError:
- LOG.exception(_LE("Could not release the acquired lock `%s`"),
- self.fname)
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.release()
-
- def exists(self):
- return os.path.exists(self.fname)
-
- def trylock(self):
- raise NotImplementedError()
-
- def unlock(self):
- raise NotImplementedError()
-
-
-class _WindowsLock(_FileLock):
- def trylock(self):
- msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
-
- def unlock(self):
- msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
-
-
-class _FcntlLock(_FileLock):
- def trylock(self):
- fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
-
- def unlock(self):
- fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
-
-
-class _PosixLock(object):
- def __init__(self, name):
- # Hash the name because it's not valid to have POSIX semaphore
- # names with things like / in them. Then use base64 to encode
- # the digest() instead taking the hexdigest() because the
- # result is shorter and most systems can't have shm sempahore
- # names longer than 31 characters.
- h = hashlib.sha1()
- h.update(name.encode('ascii'))
- self.name = str((b'/' + base64.urlsafe_b64encode(
- h.digest())).decode('ascii'))
-
- def acquire(self, timeout=None):
- self.semaphore = posix_ipc.Semaphore(self.name,
- flags=posix_ipc.O_CREAT,
- initial_value=1)
- self.semaphore.acquire(timeout)
- return self
-
- def __enter__(self):
- self.acquire()
- return self
-
- def release(self):
- self.semaphore.release()
- self.semaphore.close()
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.release()
-
- def exists(self):
- try:
- semaphore = posix_ipc.Semaphore(self.name)
- except posix_ipc.ExistentialError:
- return False
- else:
- semaphore.close()
- return True
-
-
-if os.name == 'nt':
- import msvcrt
- InterProcessLock = _WindowsLock
- FileLock = _WindowsLock
-else:
- import base64
- import fcntl
- import hashlib
-
- import posix_ipc
- InterProcessLock = _PosixLock
- FileLock = _FcntlLock
-
-_semaphores = weakref.WeakValueDictionary()
-_semaphores_lock = threading.Lock()
-
-
-def _get_lock_path(name, lock_file_prefix, lock_path=None):
- # NOTE(mikal): the lock name cannot contain directory
- # separators
- name = name.replace(os.sep, '_')
- if lock_file_prefix:
- sep = '' if lock_file_prefix.endswith('-') else '-'
- name = '%s%s%s' % (lock_file_prefix, sep, name)
-
- local_lock_path = lock_path or CONF.lock_path
-
- if not local_lock_path:
- # NOTE(bnemec): Create a fake lock path for posix locks so we don't
- # unnecessarily raise the RequiredOptError below.
- if InterProcessLock is not _PosixLock:
- raise cfg.RequiredOptError('lock_path')
- local_lock_path = 'posixlock:/'
-
- return os.path.join(local_lock_path, name)
-
-
-def external_lock(name, lock_file_prefix=None, lock_path=None):
- LOG.debug('Attempting to grab external lock "%(lock)s"',
- {'lock': name})
-
- lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
-
- # NOTE(bnemec): If an explicit lock_path was passed to us then it
- # means the caller is relying on file-based locking behavior, so
- # we can't use posix locks for those calls.
- if lock_path:
- return FileLock(lock_file_path)
- return InterProcessLock(lock_file_path)
-
-
-def remove_external_lock_file(name, lock_file_prefix=None):
- """Remove an external lock file when it's not used anymore
- This will be helpful when we have a lot of lock files
- """
- with internal_lock(name):
- lock_file_path = _get_lock_path(name, lock_file_prefix)
- try:
- os.remove(lock_file_path)
- except OSError:
- LOG.info(_LI('Failed to remove file %(file)s'),
- {'file': lock_file_path})
-
-
-def internal_lock(name):
- with _semaphores_lock:
- try:
- sem = _semaphores[name]
- except KeyError:
- sem = threading.Semaphore()
- _semaphores[name] = sem
-
- LOG.debug('Got semaphore "%(lock)s"', {'lock': name})
- return sem
-
-
-@contextlib.contextmanager
-def lock(name, lock_file_prefix=None, external=False, lock_path=None):
- """Context based lock
-
- This function yields a `threading.Semaphore` instance (if we don't use
- eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
- True, in which case, it'll yield an InterProcessLock instance.
-
- :param lock_file_prefix: The lock_file_prefix argument is used to provide
- lock files on disk with a meaningful prefix.
-
- :param external: The external keyword argument denotes whether this lock
- should work across multiple processes. This means that if two different
- workers both run a method decorated with @synchronized('mylock',
- external=True), only one of them will execute at a time.
- """
- int_lock = internal_lock(name)
- with int_lock:
- if external and not CONF.disable_process_locking:
- ext_lock = external_lock(name, lock_file_prefix, lock_path)
- with ext_lock:
- yield ext_lock
- else:
- yield int_lock
- LOG.debug('Released semaphore "%(lock)s"', {'lock': name})
-
-
-def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
- """Synchronization decorator.
-
- Decorating a method like so::
-
- @synchronized('mylock')
- def foo(self, *args):
- ...
-
- ensures that only one thread will execute the foo method at a time.
-
- Different methods can share the same lock::
-
- @synchronized('mylock')
- def foo(self, *args):
- ...
-
- @synchronized('mylock')
- def bar(self, *args):
- ...
-
- This way only one of either foo or bar can be executing at a time.
- """
-
- def wrap(f):
- @functools.wraps(f)
- def inner(*args, **kwargs):
- try:
- with lock(name, lock_file_prefix, external, lock_path):
- LOG.debug('Got semaphore / lock "%(function)s"',
- {'function': f.__name__})
- return f(*args, **kwargs)
- finally:
- LOG.debug('Semaphore / lock released "%(function)s"',
- {'function': f.__name__})
- return inner
- return wrap
-
-
-def synchronized_with_prefix(lock_file_prefix):
- """Partial object generator for the synchronization decorator.
-
- Redefine @synchronized in each project like so::
-
- (in nova/utils.py)
- from nova.openstack.common import lockutils
-
- synchronized = lockutils.synchronized_with_prefix('nova-')
-
-
- (in nova/foo.py)
- from nova import utils
-
- @utils.synchronized('mylock')
- def bar(self, *args):
- ...
-
- The lock_file_prefix argument is used to provide lock files on disk with a
- meaningful prefix.
- """
-
- return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
-
-
-def main(argv):
- """Create a dir for locks and pass it to command from arguments
-
- If you run this:
- python -m openstack.common.lockutils python setup.py testr <etc>
-
- a temporary directory will be created for all your locks and passed to all
- your tests in an environment variable. The temporary dir will be deleted
- afterwards and the return value will be preserved.
- """
-
- lock_dir = tempfile.mkdtemp()
- os.environ["TOOZ_LOCK_PATH"] = lock_dir
- try:
- ret_val = subprocess.call(argv[1:])
- finally:
- shutil.rmtree(lock_dir, ignore_errors=True)
- return ret_val
-
-
-if __name__ == '__main__':
- sys.exit(main(sys.argv))
diff --git a/tooz/openstack/common/log.py b/tooz/openstack/common/log.py
deleted file mode 100644
index a6764ac..0000000
--- a/tooz/openstack/common/log.py
+++ /dev/null
@@ -1,703 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""OpenStack logging handler.
-
-This module adds to logging functionality by adding the option to specify
-a context object when calling the various log methods. If the context object
-is not specified, default formatting is used. Additionally, an instance uuid
-may be passed as part of the log message, which is intended to make it easier
-for admins to find messages related to a specific instance.
-
-It also allows setting of formatting information through conf.
-
-"""
-
-import inspect
-import itertools
-import logging
-import logging.config
-import logging.handlers
-import os
-import sys
-import traceback
-
-from oslo.config import cfg
-import six
-from six import moves
-
-_PY26 = sys.version_info[0:2] == (2, 6)
-
-from tooz.openstack.common.gettextutils import _
-from tooz.openstack.common import importutils
-from tooz.openstack.common import jsonutils
-from tooz.openstack.common import local
-# NOTE(flaper87): Pls, remove when graduating this module
-# from the incubator.
-from tooz.openstack.common.strutils import mask_password # noqa
-
-
-_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
-
-
-common_cli_opts = [
- cfg.BoolOpt('debug',
- short='d',
- default=False,
- help='Print debugging output (set logging level to '
- 'DEBUG instead of default WARNING level).'),
- cfg.BoolOpt('verbose',
- short='v',
- default=False,
- help='Print more verbose output (set logging level to '
- 'INFO instead of default WARNING level).'),
-]
-
-logging_cli_opts = [
- cfg.StrOpt('log-config-append',
- metavar='PATH',
- deprecated_name='log-config',
- help='The name of a logging configuration file. This file '
- 'is appended to any existing logging configuration '
- 'files. For details about logging configuration files, '
- 'see the Python logging module documentation.'),
- cfg.StrOpt('log-format',
- metavar='FORMAT',
- help='DEPRECATED. '
- 'A logging.Formatter log message format string which may '
- 'use any of the available logging.LogRecord attributes. '
- 'This option is deprecated. Please use '
- 'logging_context_format_string and '
- 'logging_default_format_string instead.'),
- cfg.StrOpt('log-date-format',
- default=_DEFAULT_LOG_DATE_FORMAT,
- metavar='DATE_FORMAT',
- help='Format string for %%(asctime)s in log records. '
- 'Default: %(default)s .'),
- cfg.StrOpt('log-file',
- metavar='PATH',
- deprecated_name='logfile',
- help='(Optional) Name of log file to output to. '
- 'If no default is set, logging will go to stdout.'),
- cfg.StrOpt('log-dir',
- deprecated_name='logdir',
- help='(Optional) The base directory used for relative '
- '--log-file paths.'),
- cfg.BoolOpt('use-syslog',
- default=False,
- help='Use syslog for logging. '
- 'Existing syslog format is DEPRECATED during I, '
- 'and will change in J to honor RFC5424.'),
- cfg.BoolOpt('use-syslog-rfc-format',
- # TODO(bogdando) remove or use True after existing
- # syslog format deprecation in J
- default=False,
- help='(Optional) Enables or disables syslog rfc5424 format '
- 'for logging. If enabled, prefixes the MSG part of the '
- 'syslog message with APP-NAME (RFC5424). The '
- 'format without the APP-NAME is deprecated in I, '
- 'and will be removed in J.'),
- cfg.StrOpt('syslog-log-facility',
- default='LOG_USER',
- help='Syslog facility to receive log lines.')
-]
-
-generic_log_opts = [
- cfg.BoolOpt('use_stderr',
- default=True,
- help='Log output to standard error.')
-]
-
-DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
- 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
- 'oslo.messaging=INFO', 'iso8601=WARN',
- 'requests.packages.urllib3.connectionpool=WARN',
- 'urllib3.connectionpool=WARN', 'websocket=WARN']
-
-log_opts = [
- cfg.StrOpt('logging_context_format_string',
- default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
- '%(name)s [%(request_id)s %(user_identity)s] '
- '%(instance)s%(message)s',
- help='Format string to use for log messages with context.'),
- cfg.StrOpt('logging_default_format_string',
- default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
- '%(name)s [-] %(instance)s%(message)s',
- help='Format string to use for log messages without context.'),
- cfg.StrOpt('logging_debug_format_suffix',
- default='%(funcName)s %(pathname)s:%(lineno)d',
- help='Data to append to log format when level is DEBUG.'),
- cfg.StrOpt('logging_exception_prefix',
- default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
- '%(instance)s',
- help='Prefix each line of exception output with this format.'),
- cfg.ListOpt('default_log_levels',
- default=DEFAULT_LOG_LEVELS,
- help='List of logger=LEVEL pairs.'),
- cfg.BoolOpt('publish_errors',
- default=False,
- help='Enables or disables publication of error events.'),
- cfg.BoolOpt('fatal_deprecations',
- default=False,
- help='Enables or disables fatal status of deprecations.'),
-
- # NOTE(mikal): there are two options here because sometimes we are handed
- # a full instance (and could include more information), and other times we
- # are just handed a UUID for the instance.
- cfg.StrOpt('instance_format',
- default='[instance: %(uuid)s] ',
- help='The format for an instance that is passed with the log '
- 'message.'),
- cfg.StrOpt('instance_uuid_format',
- default='[instance: %(uuid)s] ',
- help='The format for an instance UUID that is passed with the '
- 'log message.'),
-]
-
-CONF = cfg.CONF
-CONF.register_cli_opts(common_cli_opts)
-CONF.register_cli_opts(logging_cli_opts)
-CONF.register_opts(generic_log_opts)
-CONF.register_opts(log_opts)
-
-# our new audit level
-# NOTE(jkoelker) Since we synthesized an audit level, make the logging
-# module aware of it so it acts like other levels.
-logging.AUDIT = logging.INFO + 1
-logging.addLevelName(logging.AUDIT, 'AUDIT')
-
-
-try:
- NullHandler = logging.NullHandler
-except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7
- class NullHandler(logging.Handler):
- def handle(self, record):
- pass
-
- def emit(self, record):
- pass
-
- def createLock(self):
- self.lock = None
-
-
-def _dictify_context(context):
- if context is None:
- return None
- if not isinstance(context, dict) and getattr(context, 'to_dict', None):
- context = context.to_dict()
- return context
-
-
-def _get_binary_name():
- return os.path.basename(inspect.stack()[-1][1])
-
-
-def _get_log_file_path(binary=None):
- logfile = CONF.log_file
- logdir = CONF.log_dir
-
- if logfile and not logdir:
- return logfile
-
- if logfile and logdir:
- return os.path.join(logdir, logfile)
-
- if logdir:
- binary = binary or _get_binary_name()
- return '%s.log' % (os.path.join(logdir, binary),)
-
- return None
-
-
-class BaseLoggerAdapter(logging.LoggerAdapter):
-
- def audit(self, msg, *args, **kwargs):
- self.log(logging.AUDIT, msg, *args, **kwargs)
-
- def isEnabledFor(self, level):
- if _PY26:
- # This method was added in python 2.7 (and it does the exact
- # same logic, so we need to do the exact same logic so that
- # python 2.6 has this capability as well).
- return self.logger.isEnabledFor(level)
- else:
- return super(BaseLoggerAdapter, self).isEnabledFor(level)
-
-
-class LazyAdapter(BaseLoggerAdapter):
- def __init__(self, name='unknown', version='unknown'):
- self._logger = None
- self.extra = {}
- self.name = name
- self.version = version
-
- @property
- def logger(self):
- if not self._logger:
- self._logger = getLogger(self.name, self.version)
- if six.PY3:
- # In Python 3, the code fails because the 'manager' attribute
- # cannot be found when using a LoggerAdapter as the
- # underlying logger. Work around this issue.
- self._logger.manager = self._logger.logger.manager
- return self._logger
-
-
-class ContextAdapter(BaseLoggerAdapter):
- warn = logging.LoggerAdapter.warning
-
- def __init__(self, logger, project_name, version_string):
- self.logger = logger
- self.project = project_name
- self.version = version_string
- self._deprecated_messages_sent = dict()
-
- @property
- def handlers(self):
- return self.logger.handlers
-
- def deprecated(self, msg, *args, **kwargs):
- """Call this method when a deprecated feature is used.
-
- If the system is configured for fatal deprecations then the message
- is logged at the 'critical' level and :class:`DeprecatedConfig` will
- be raised.
-
- Otherwise, the message will be logged (once) at the 'warn' level.
-
- :raises: :class:`DeprecatedConfig` if the system is configured for
- fatal deprecations.
-
- """
- stdmsg = _("Deprecated: %s") % msg
- if CONF.fatal_deprecations:
- self.critical(stdmsg, *args, **kwargs)
- raise DeprecatedConfig(msg=stdmsg)
-
- # Using a list because a tuple with dict can't be stored in a set.
- sent_args = self._deprecated_messages_sent.setdefault(msg, list())
-
- if args in sent_args:
- # Already logged this message, so don't log it again.
- return
-
- sent_args.append(args)
- self.warn(stdmsg, *args, **kwargs)
-
- def process(self, msg, kwargs):
- # NOTE(mrodden): catch any Message/other object and
- # coerce to unicode before they can get
- # to the python logging and possibly
- # cause string encoding trouble
- if not isinstance(msg, six.string_types):
- msg = six.text_type(msg)
-
- if 'extra' not in kwargs:
- kwargs['extra'] = {}
- extra = kwargs['extra']
-
- context = kwargs.pop('context', None)
- if not context:
- context = getattr(local.store, 'context', None)
- if context:
- extra.update(_dictify_context(context))
-
- instance = kwargs.pop('instance', None)
- instance_uuid = (extra.get('instance_uuid') or
- kwargs.pop('instance_uuid', None))
- instance_extra = ''
- if instance:
- instance_extra = CONF.instance_format % instance
- elif instance_uuid:
- instance_extra = (CONF.instance_uuid_format
- % {'uuid': instance_uuid})
- extra['instance'] = instance_extra
-
- extra.setdefault('user_identity', kwargs.pop('user_identity', None))
-
- extra['project'] = self.project
- extra['version'] = self.version
- extra['extra'] = extra.copy()
- return msg, kwargs
-
-
-class JSONFormatter(logging.Formatter):
- def __init__(self, fmt=None, datefmt=None):
- # NOTE(jkoelker) we ignore the fmt argument, but its still there
- # since logging.config.fileConfig passes it.
- self.datefmt = datefmt
-
- def formatException(self, ei, strip_newlines=True):
- lines = traceback.format_exception(*ei)
- if strip_newlines:
- lines = [moves.filter(
- lambda x: x,
- line.rstrip().splitlines()) for line in lines]
- lines = list(itertools.chain(*lines))
- return lines
-
- def format(self, record):
- message = {'message': record.getMessage(),
- 'asctime': self.formatTime(record, self.datefmt),
- 'name': record.name,
- 'msg': record.msg,
- 'args': record.args,
- 'levelname': record.levelname,
- 'levelno': record.levelno,
- 'pathname': record.pathname,
- 'filename': record.filename,
- 'module': record.module,
- 'lineno': record.lineno,
- 'funcname': record.funcName,
- 'created': record.created,
- 'msecs': record.msecs,
- 'relative_created': record.relativeCreated,
- 'thread': record.thread,
- 'thread_name': record.threadName,
- 'process_name': record.processName,
- 'process': record.process,
- 'traceback': None}
-
- if hasattr(record, 'extra'):
- message['extra'] = record.extra
-
- if record.exc_info:
- message['traceback'] = self.formatException(record.exc_info)
-
- return jsonutils.dumps(message)
-
-
-def _create_logging_excepthook(product_name):
- def logging_excepthook(exc_type, value, tb):
- extra = {'exc_info': (exc_type, value, tb)}
- getLogger(product_name).critical(
- "".join(traceback.format_exception_only(exc_type, value)),
- **extra)
- return logging_excepthook
-
-
-class LogConfigError(Exception):
-
- message = _('Error loading logging config %(log_config)s: %(err_msg)s')
-
- def __init__(self, log_config, err_msg):
- self.log_config = log_config
- self.err_msg = err_msg
-
- def __str__(self):
- return self.message % dict(log_config=self.log_config,
- err_msg=self.err_msg)
-
-
-def _load_log_config(log_config_append):
- try:
- logging.config.fileConfig(log_config_append,
- disable_existing_loggers=False)
- except (moves.configparser.Error, KeyError) as exc:
- raise LogConfigError(log_config_append, six.text_type(exc))
-
-
-def setup(product_name, version='unknown'):
- """Setup logging."""
- if CONF.log_config_append:
- _load_log_config(CONF.log_config_append)
- else:
- _setup_logging_from_conf(product_name, version)
- sys.excepthook = _create_logging_excepthook(product_name)
-
-
-def set_defaults(logging_context_format_string=None,
- default_log_levels=None):
- # Just in case the caller is not setting the
- # default_log_level. This is insurance because
- # we introduced the default_log_level parameter
- # later in a backwards in-compatible change
- if default_log_levels is not None:
- cfg.set_defaults(
- log_opts,
- default_log_levels=default_log_levels)
- if logging_context_format_string is not None:
- cfg.set_defaults(
- log_opts,
- logging_context_format_string=logging_context_format_string)
-
-
-def _find_facility_from_conf():
- facility_names = logging.handlers.SysLogHandler.facility_names
- facility = getattr(logging.handlers.SysLogHandler,
- CONF.syslog_log_facility,
- None)
-
- if facility is None and CONF.syslog_log_facility in facility_names:
- facility = facility_names.get(CONF.syslog_log_facility)
-
- if facility is None:
- valid_facilities = facility_names.keys()
- consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
- 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
- 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
- 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
- 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
- valid_facilities.extend(consts)
- raise TypeError(_('syslog facility must be one of: %s') %
- ', '.join("'%s'" % fac
- for fac in valid_facilities))
-
- return facility
-
-
-class RFCSysLogHandler(logging.handlers.SysLogHandler):
- def __init__(self, *args, **kwargs):
- self.binary_name = _get_binary_name()
- # Do not use super() unless type(logging.handlers.SysLogHandler)
- # is 'type' (Python 2.7).
- # Use old style calls, if the type is 'classobj' (Python 2.6)
- logging.handlers.SysLogHandler.__init__(self, *args, **kwargs)
-
- def format(self, record):
- # Do not use super() unless type(logging.handlers.SysLogHandler)
- # is 'type' (Python 2.7).
- # Use old style calls, if the type is 'classobj' (Python 2.6)
- msg = logging.handlers.SysLogHandler.format(self, record)
- msg = self.binary_name + ' ' + msg
- return msg
-
-
-def _setup_logging_from_conf(project, version):
- log_root = getLogger(None).logger
- for handler in log_root.handlers:
- log_root.removeHandler(handler)
-
- if CONF.use_syslog:
- facility = _find_facility_from_conf()
- # TODO(bogdando) use the format provided by RFCSysLogHandler
- # after existing syslog format deprecation in J
- if CONF.use_syslog_rfc_format:
- syslog = RFCSysLogHandler(address='/dev/log',
- facility=facility)
- else:
- syslog = logging.handlers.SysLogHandler(address='/dev/log',
- facility=facility)
- log_root.addHandler(syslog)
-
- logpath = _get_log_file_path()
- if logpath:
- filelog = logging.handlers.WatchedFileHandler(logpath)
- log_root.addHandler(filelog)
-
- if CONF.use_stderr:
- streamlog = ColorHandler()
- log_root.addHandler(streamlog)
-
- elif not logpath:
- # pass sys.stdout as a positional argument
- # python2.6 calls the argument strm, in 2.7 it's stream
- streamlog = logging.StreamHandler(sys.stdout)
- log_root.addHandler(streamlog)
-
- if CONF.publish_errors:
- try:
- handler = importutils.import_object(
- "tooz.openstack.common.log_handler.PublishErrorsHandler",
- logging.ERROR)
- except ImportError:
- handler = importutils.import_object(
- "oslo.messaging.notify.log_handler.PublishErrorsHandler",
- logging.ERROR)
- log_root.addHandler(handler)
-
- datefmt = CONF.log_date_format
- for handler in log_root.handlers:
- # NOTE(alaski): CONF.log_format overrides everything currently. This
- # should be deprecated in favor of context aware formatting.
- if CONF.log_format:
- handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
- datefmt=datefmt))
- log_root.info('Deprecated: log_format is now deprecated and will '
- 'be removed in the next release')
- else:
- handler.setFormatter(ContextFormatter(project=project,
- version=version,
- datefmt=datefmt))
-
- if CONF.debug:
- log_root.setLevel(logging.DEBUG)
- elif CONF.verbose:
- log_root.setLevel(logging.INFO)
- else:
- log_root.setLevel(logging.WARNING)
-
- for pair in CONF.default_log_levels:
- mod, _sep, level_name = pair.partition('=')
- logger = logging.getLogger(mod)
- # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name
- # to integer code.
- if sys.version_info < (2, 7):
- level = logging.getLevelName(level_name)
- logger.setLevel(level)
- else:
- logger.setLevel(level_name)
-
-
-_loggers = {}
-
-
-def getLogger(name='unknown', version='unknown'):
- if name not in _loggers:
- _loggers[name] = ContextAdapter(logging.getLogger(name),
- name,
- version)
- return _loggers[name]
-
-
-def getLazyLogger(name='unknown', version='unknown'):
- """Returns lazy logger.
-
- Creates a pass-through logger that does not create the real logger
- until it is really needed and delegates all calls to the real logger
- once it is created.
- """
- return LazyAdapter(name, version)
-
-
-class WritableLogger(object):
- """A thin wrapper that responds to `write` and logs."""
-
- def __init__(self, logger, level=logging.INFO):
- self.logger = logger
- self.level = level
-
- def write(self, msg):
- self.logger.log(self.level, msg.rstrip())
-
-
-class ContextFormatter(logging.Formatter):
- """A context.RequestContext aware formatter configured through flags.
-
- The flags used to set format strings are: logging_context_format_string
- and logging_default_format_string. You can also specify
- logging_debug_format_suffix to append extra formatting if the log level is
- debug.
-
- For information about what variables are available for the formatter see:
- http://docs.python.org/library/logging.html#formatter
-
- If available, uses the context value stored in TLS - local.store.context
-
- """
-
- def __init__(self, *args, **kwargs):
- """Initialize ContextFormatter instance
-
- Takes additional keyword arguments which can be used in the message
- format string.
-
- :keyword project: project name
- :type project: string
- :keyword version: project version
- :type version: string
-
- """
-
- self.project = kwargs.pop('project', 'unknown')
- self.version = kwargs.pop('version', 'unknown')
-
- logging.Formatter.__init__(self, *args, **kwargs)
-
- def format(self, record):
- """Uses contextstring if request_id is set, otherwise default."""
-
- # store project info
- record.project = self.project
- record.version = self.version
-
- # store request info
- context = getattr(local.store, 'context', None)
- if context:
- d = _dictify_context(context)
- for k, v in d.items():
- setattr(record, k, v)
-
- # NOTE(sdague): default the fancier formatting params
- # to an empty string so we don't throw an exception if
- # they get used
- for key in ('instance', 'color', 'user_identity'):
- if key not in record.__dict__:
- record.__dict__[key] = ''
-
- if record.__dict__.get('request_id'):
- fmt = CONF.logging_context_format_string
- else:
- fmt = CONF.logging_default_format_string
-
- if (record.levelno == logging.DEBUG and
- CONF.logging_debug_format_suffix):
- fmt += " " + CONF.logging_debug_format_suffix
-
- if sys.version_info < (3, 2):
- self._fmt = fmt
- else:
- self._style = logging.PercentStyle(fmt)
- self._fmt = self._style._fmt
- # Cache this on the record, Logger will respect our formatted copy
- if record.exc_info:
- record.exc_text = self.formatException(record.exc_info, record)
- return logging.Formatter.format(self, record)
-
- def formatException(self, exc_info, record=None):
- """Format exception output with CONF.logging_exception_prefix."""
- if not record:
- return logging.Formatter.formatException(self, exc_info)
-
- stringbuffer = moves.StringIO()
- traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
- None, stringbuffer)
- lines = stringbuffer.getvalue().split('\n')
- stringbuffer.close()
-
- if CONF.logging_exception_prefix.find('%(asctime)') != -1:
- record.asctime = self.formatTime(record, self.datefmt)
-
- formatted_lines = []
- for line in lines:
- pl = CONF.logging_exception_prefix % record.__dict__
- fl = '%s%s' % (pl, line)
- formatted_lines.append(fl)
- return '\n'.join(formatted_lines)
-
-
-class ColorHandler(logging.StreamHandler):
- LEVEL_COLORS = {
- logging.DEBUG: '\033[00;32m', # GREEN
- logging.INFO: '\033[00;36m', # CYAN
- logging.AUDIT: '\033[01;36m', # BOLD CYAN
- logging.WARN: '\033[01;33m', # BOLD YELLOW
- logging.ERROR: '\033[01;31m', # BOLD RED
- logging.CRITICAL: '\033[01;31m', # BOLD RED
- }
-
- def format(self, record):
- record.color = self.LEVEL_COLORS[record.levelno]
- return logging.StreamHandler.format(self, record)
-
-
-class DeprecatedConfig(Exception):
- message = _("Fatal call to deprecated config: %(msg)s")
-
- def __init__(self, msg):
- super(Exception, self).__init__(self.message % dict(msg=msg))
diff --git a/tooz/openstack/common/strutils.py b/tooz/openstack/common/strutils.py
deleted file mode 100644
index 994c91a..0000000
--- a/tooz/openstack/common/strutils.py
+++ /dev/null
@@ -1,295 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""
-System-level utilities and helper functions.
-"""
-
-import math
-import re
-import sys
-import unicodedata
-
-import six
-
-from tooz.openstack.common.gettextutils import _
-
-
-UNIT_PREFIX_EXPONENT = {
- 'k': 1,
- 'K': 1,
- 'Ki': 1,
- 'M': 2,
- 'Mi': 2,
- 'G': 3,
- 'Gi': 3,
- 'T': 4,
- 'Ti': 4,
-}
-UNIT_SYSTEM_INFO = {
- 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
- 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
-}
-
-TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
-FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
-
-SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
-SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
-
-
-# NOTE(flaper87): The following 3 globals are used by `mask_password`
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
- r'(<%(key)s>).*?(</%(key)s>)',
- r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
- r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
- r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
- '.*?([\'"])',
- r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
-
-for key in _SANITIZE_KEYS:
- for pattern in _FORMAT_PATTERNS:
- reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
- _SANITIZE_PATTERNS.append(reg_ex)
-
-
-def int_from_bool_as_string(subject):
- """Interpret a string as a boolean and return either 1 or 0.
-
- Any string value in:
-
- ('True', 'true', 'On', 'on', '1')
-
- is interpreted as a boolean True.
-
- Useful for JSON-decoded stuff and config file parsing
- """
- return bool_from_string(subject) and 1 or 0
-
-
-def bool_from_string(subject, strict=False, default=False):
- """Interpret a string as a boolean.
-
- A case-insensitive match is performed such that strings matching 't',
- 'true', 'on', 'y', 'yes', or '1' are considered True and, when
- `strict=False`, anything else returns the value specified by 'default'.
-
- Useful for JSON-decoded stuff and config file parsing.
-
- If `strict=True`, unrecognized values, including None, will raise a
- ValueError which is useful when parsing values passed in from an API call.
- Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
- """
- if not isinstance(subject, six.string_types):
- subject = six.text_type(subject)
-
- lowered = subject.strip().lower()
-
- if lowered in TRUE_STRINGS:
- return True
- elif lowered in FALSE_STRINGS:
- return False
- elif strict:
- acceptable = ', '.join(
- "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
- msg = _("Unrecognized value '%(val)s', acceptable values are:"
- " %(acceptable)s") % {'val': subject,
- 'acceptable': acceptable}
- raise ValueError(msg)
- else:
- return default
-
-
-def safe_decode(text, incoming=None, errors='strict'):
- """Decodes incoming text/bytes string using `incoming` if they're not
- already unicode.
-
- :param incoming: Text's current encoding
- :param errors: Errors handling policy. See here for valid
- values http://docs.python.org/2/library/codecs.html
- :returns: text or a unicode `incoming` encoded
- representation of it.
- :raises TypeError: If text is not an instance of str
- """
- if not isinstance(text, (six.string_types, six.binary_type)):
- raise TypeError("%s can't be decoded" % type(text))
-
- if isinstance(text, six.text_type):
- return text
-
- if not incoming:
- incoming = (sys.stdin.encoding or
- sys.getdefaultencoding())
-
- try:
- return text.decode(incoming, errors)
- except UnicodeDecodeError:
- # Note(flaper87) If we get here, it means that
- # sys.stdin.encoding / sys.getdefaultencoding
- # didn't return a suitable encoding to decode
- # text. This happens mostly when global LANG
- # var is not set correctly and there's no
- # default encoding. In this case, most likely
- # python will use ASCII or ANSI encoders as
- # default encodings but they won't be capable
- # of decoding non-ASCII characters.
- #
- # Also, UTF-8 is being used since it's an ASCII
- # extension.
- return text.decode('utf-8', errors)
-
-
-def safe_encode(text, incoming=None,
- encoding='utf-8', errors='strict'):
- """Encodes incoming text/bytes string using `encoding`.
-
- If incoming is not specified, text is expected to be encoded with
- current python's default encoding. (`sys.getdefaultencoding`)
-
- :param incoming: Text's current encoding
- :param encoding: Expected encoding for text (Default UTF-8)
- :param errors: Errors handling policy. See here for valid
- values http://docs.python.org/2/library/codecs.html
- :returns: text or a bytestring `encoding` encoded
- representation of it.
- :raises TypeError: If text is not an instance of str
- """
- if not isinstance(text, (six.string_types, six.binary_type)):
- raise TypeError("%s can't be encoded" % type(text))
-
- if not incoming:
- incoming = (sys.stdin.encoding or
- sys.getdefaultencoding())
-
- if isinstance(text, six.text_type):
- return text.encode(encoding, errors)
- elif text and encoding != incoming:
- # Decode text before encoding it with `encoding`
- text = safe_decode(text, incoming, errors)
- return text.encode(encoding, errors)
- else:
- return text
-
-
-def string_to_bytes(text, unit_system='IEC', return_int=False):
- """Converts a string into an float representation of bytes.
-
- The units supported for IEC ::
-
- Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
- KB, KiB, MB, MiB, GB, GiB, TB, TiB
-
- The units supported for SI ::
-
- kb(it), Mb(it), Gb(it), Tb(it)
- kB, MB, GB, TB
-
- Note that the SI unit system does not support capital letter 'K'
-
- :param text: String input for bytes size conversion.
- :param unit_system: Unit system for byte size conversion.
- :param return_int: If True, returns integer representation of text
- in bytes. (default: decimal)
- :returns: Numerical representation of text in bytes.
- :raises ValueError: If text has an invalid value.
-
- """
- try:
- base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
- except KeyError:
- msg = _('Invalid unit system: "%s"') % unit_system
- raise ValueError(msg)
- match = reg_ex.match(text)
- if match:
- magnitude = float(match.group(1))
- unit_prefix = match.group(2)
- if match.group(3) in ['b', 'bit']:
- magnitude /= 8
- else:
- msg = _('Invalid string format: %s') % text
- raise ValueError(msg)
- if not unit_prefix:
- res = magnitude
- else:
- res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
- if return_int:
- return int(math.ceil(res))
- return res
-
-
-def to_slug(value, incoming=None, errors="strict"):
- """Normalize string.
-
- Convert to lowercase, remove non-word characters, and convert spaces
- to hyphens.
-
- Inspired by Django's `slugify` filter.
-
- :param value: Text to slugify
- :param incoming: Text's current encoding
- :param errors: Errors handling policy. See here for valid
- values http://docs.python.org/2/library/codecs.html
- :returns: slugified unicode representation of `value`
- :raises TypeError: If text is not an instance of str
- """
- value = safe_decode(value, incoming, errors)
- # NOTE(aababilov): no need to use safe_(encode|decode) here:
- # encodings are always "ascii", error handling is always "ignore"
- # and types are always known (first: unicode; second: str)
- value = unicodedata.normalize("NFKD", value).encode(
- "ascii", "ignore").decode("ascii")
- value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
- return SLUGIFY_HYPHENATE_RE.sub("-", value)
-
-
-def mask_password(message, secret="***"):
- """Replace password with 'secret' in message.
-
- :param message: The string which includes security information.
- :param secret: value with which to replace passwords.
- :returns: The unicode value of message with the password fields masked.
-
- For example:
-
- >>> mask_password("'adminPass' : 'aaaaa'")
- "'adminPass' : '***'"
- >>> mask_password("'admin_pass' : 'aaaaa'")
- "'admin_pass' : '***'"
- >>> mask_password('"password" : "aaaaa"')
- '"password" : "***"'
- >>> mask_password("'original_password' : 'aaaaa'")
- "'original_password' : '***'"
- >>> mask_password("u'original_password' : u'aaaaa'")
- "u'original_password' : u'***'"
- """
- message = six.text_type(message)
-
- # NOTE(ldbragst): Check to see if anything in message contains any key
- # specified in _SANITIZE_KEYS, if not then just return the message since
- # we don't have to mask any passwords.
- if not any(key in message for key in _SANITIZE_KEYS):
- return message
-
- secret = r'\g<1>' + secret + r'\g<2>'
- for pattern in _SANITIZE_PATTERNS:
- message = re.sub(pattern, secret, message)
- return message
diff --git a/tooz/openstack/common/timeutils.py b/tooz/openstack/common/timeutils.py
deleted file mode 100644
index c48da95..0000000
--- a/tooz/openstack/common/timeutils.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""
-Time related utilities and helper functions.
-"""
-
-import calendar
-import datetime
-import time
-
-import iso8601
-import six
-
-
-# ISO 8601 extended time format with microseconds
-_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
-_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
-PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
-
-
-def isotime(at=None, subsecond=False):
- """Stringify time in ISO 8601 format."""
- if not at:
- at = utcnow()
- st = at.strftime(_ISO8601_TIME_FORMAT
- if not subsecond
- else _ISO8601_TIME_FORMAT_SUBSECOND)
- tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
- st += ('Z' if tz == 'UTC' else tz)
- return st
-
-
-def parse_isotime(timestr):
- """Parse time from ISO 8601 format."""
- try:
- return iso8601.parse_date(timestr)
- except iso8601.ParseError as e:
- raise ValueError(six.text_type(e))
- except TypeError as e:
- raise ValueError(six.text_type(e))
-
-
-def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
- """Returns formatted utcnow."""
- if not at:
- at = utcnow()
- return at.strftime(fmt)
-
-
-def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
- """Turn a formatted time back into a datetime."""
- return datetime.datetime.strptime(timestr, fmt)
-
-
-def normalize_time(timestamp):
- """Normalize time in arbitrary timezone to UTC naive object."""
- offset = timestamp.utcoffset()
- if offset is None:
- return timestamp
- return timestamp.replace(tzinfo=None) - offset
-
-
-def is_older_than(before, seconds):
- """Return True if before is older than seconds."""
- if isinstance(before, six.string_types):
- before = parse_strtime(before).replace(tzinfo=None)
- else:
- before = before.replace(tzinfo=None)
-
- return utcnow() - before > datetime.timedelta(seconds=seconds)
-
-
-def is_newer_than(after, seconds):
- """Return True if after is newer than seconds."""
- if isinstance(after, six.string_types):
- after = parse_strtime(after).replace(tzinfo=None)
- else:
- after = after.replace(tzinfo=None)
-
- return after - utcnow() > datetime.timedelta(seconds=seconds)
-
-
-def utcnow_ts():
- """Timestamp version of our utcnow function."""
- if utcnow.override_time is None:
- # NOTE(kgriffs): This is several times faster
- # than going through calendar.timegm(...)
- return int(time.time())
-
- return calendar.timegm(utcnow().timetuple())
-
-
-def utcnow():
- """Overridable version of utils.utcnow."""
- if utcnow.override_time:
- try:
- return utcnow.override_time.pop(0)
- except AttributeError:
- return utcnow.override_time
- return datetime.datetime.utcnow()
-
-
-def iso8601_from_timestamp(timestamp):
- """Returns an iso8601 formatted date from timestamp."""
- return isotime(datetime.datetime.utcfromtimestamp(timestamp))
-
-
-utcnow.override_time = None
-
-
-def set_time_override(override_time=None):
- """Overrides utils.utcnow.
-
- Make it return a constant time or a list thereof, one at a time.
-
- :param override_time: datetime instance or list thereof. If not
- given, defaults to the current UTC time.
- """
- utcnow.override_time = override_time or datetime.datetime.utcnow()
-
-
-def advance_time_delta(timedelta):
- """Advance overridden time using a datetime.timedelta."""
- assert utcnow.override_time is not None
- try:
- for dt in utcnow.override_time:
- dt += timedelta
- except TypeError:
- utcnow.override_time += timedelta
-
-
-def advance_time_seconds(seconds):
- """Advance overridden time by seconds."""
- advance_time_delta(datetime.timedelta(0, seconds))
-
-
-def clear_time_override():
- """Remove the overridden time."""
- utcnow.override_time = None
-
-
-def marshall_now(now=None):
- """Make an rpc-safe datetime with microseconds.
-
- Note: tzinfo is stripped, but not required for relative times.
- """
- if not now:
- now = utcnow()
- return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
- minute=now.minute, second=now.second,
- microsecond=now.microsecond)
-
-
-def unmarshall_time(tyme):
- """Unmarshall a datetime dict."""
- return datetime.datetime(day=tyme['day'],
- month=tyme['month'],
- year=tyme['year'],
- hour=tyme['hour'],
- minute=tyme['minute'],
- second=tyme['second'],
- microsecond=tyme['microsecond'])
-
-
-def delta_seconds(before, after):
- """Return the difference between two timing objects.
-
- Compute the difference in seconds between two date, time, or
- datetime objects (as a float, to microsecond resolution).
- """
- delta = after - before
- return total_seconds(delta)
-
-
-def total_seconds(delta):
- """Return the total seconds of datetime.timedelta object.
-
- Compute total seconds of datetime.timedelta, datetime.timedelta
- doesn't have method total_seconds in Python2.6, calculate it manually.
- """
- try:
- return delta.total_seconds()
- except AttributeError:
- return ((delta.days * 24 * 3600) + delta.seconds +
- float(delta.microseconds) / (10 ** 6))
-
-
-def is_soon(dt, window):
- """Determines if time is going to happen in the next window seconds.
-
- :param dt: the time
- :param window: minimum seconds to remain to consider the time not soon
-
- :return: True if expiration is within the given duration
- """
- soon = (utcnow() + datetime.timedelta(seconds=window))
- return normalize_time(dt) <= soon
diff --git a/tooz/tests/test_coordination.py b/tooz/tests/test_coordination.py
index 7b119e9..d974908 100644
--- a/tooz/tests/test_coordination.py
+++ b/tooz/tests/test_coordination.py
@@ -437,6 +437,7 @@ class TestAPI(testscenarios.TestWithScenarios,
def test_get_lock(self):
lock = self._coord.get_lock(self._get_random_uuid())
+ self.addCleanup(lock.destroy)
self.assertEqual(True, lock.acquire())
lock.release()
with lock:
@@ -450,9 +451,11 @@ class TestAPI(testscenarios.TestWithScenarios,
lock_name = self._get_random_uuid()
lock = self._coord.get_lock(lock_name)
+ self.addCleanup(lock.destroy)
self.assertEqual(True, lock.acquire())
lock2 = client2.get_lock(lock_name)
+ self.addCleanup(lock2.destroy)
self.assertEqual(False, lock2.acquire(blocking=False))
lock.release()
self.assertEqual(True, lock2.acquire(blocking=True))