From e0237aac2ba992a5c31206fea52f55969db65ba0 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Aug 2013 15:03:50 -0400 Subject: - don't need resolve, don't need import for this. just look in sys.modules, since we are dealing with cycles in any case. --- lib/sqlalchemy/util/langhelpers.py | 58 +++++++++++--------------------------- 1 file changed, 16 insertions(+), 42 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index ef1ff881d..430c630a8 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -688,62 +688,36 @@ class importlater(object): except evaluted upon attribute access to "somesubmod". - importlater() currently requires that resolve_all() be - called, typically at the bottom of a package's __init__.py. - This is so that __import__ still called only at - module import time, and not potentially within - a non-main thread later on. + importlater() relies on sys.modules being populated with the + target package, and the target package containing the target module, + by time the attribute is evaulated. """ - _unresolved = set() - - def __init__(self, path, addtl=None): + def __init__(self, path, addtl): self._il_path = path self._il_addtl = addtl - importlater._unresolved.add(self) - - @classmethod - def resolve_all(cls): - for m in list(importlater._unresolved): - m._resolve() @property def _full_path(self): - if self._il_addtl: - return self._il_path + "." + self._il_addtl - else: - return self._il_path + return self._il_path + "." + self._il_addtl @memoized_property def module(self): - if self in importlater._unresolved: - raise ImportError( - "importlater.resolve_all() hasn't " - "been called (this is %s %s)" - % (self._il_path, self._il_addtl)) - - m = self._initial_import - if self._il_addtl: - m = getattr(m, self._il_addtl) - else: - for token in self._il_path.split(".")[1:]: - m = getattr(m, token) - return m - - def _resolve(self): - importlater._unresolved.discard(self) - if self._il_addtl: - self._initial_import = compat.import_( - self._il_path, globals(), locals(), - [self._il_addtl]) + try: + m = sys.modules[self._il_path] + except KeyError: + raise KeyError("Module %s hasn't been installed" % self._il_path) else: - self._initial_import = compat.import_(self._il_path) + try: + return getattr(m, self._il_addtl) + except AttributeError: + raise KeyError( + "Module %s hasn't been installed into %s" % + (self._il_addtl, self._il_path) + ) def __getattr__(self, key): - if key == 'module': - raise ImportError("Could not resolve module %s" - % self._full_path) try: attr = getattr(self.module, key) except AttributeError: -- cgit v1.2.1 From fe34b4ee63d498d385ab029dcee966de9f0203ec Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Aug 2013 16:21:37 -0400 Subject: ok forget it, that approach didn't really cover every base, so we are pretty much back to the beginning, nothing to see here --- lib/sqlalchemy/util/langhelpers.py | 45 ++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 430c630a8..d8e469a2b 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -688,15 +688,25 @@ class importlater(object): except evaluted upon attribute access to "somesubmod". - importlater() relies on sys.modules being populated with the - target package, and the target package containing the target module, - by time the attribute is evaulated. + importlater() currently requires that resolve_all() be + called, typically at the bottom of a package's __init__.py. + This is so that __import__ still called only at + module import time, and not potentially within + a non-main thread later on. """ + _unresolved = set() + def __init__(self, path, addtl): self._il_path = path self._il_addtl = addtl + importlater._unresolved.add(self) + + @classmethod + def resolve_all(cls): + for m in list(importlater._unresolved): + m._resolve() @property def _full_path(self): @@ -704,20 +714,24 @@ class importlater(object): @memoized_property def module(self): - try: - m = sys.modules[self._il_path] - except KeyError: - raise KeyError("Module %s hasn't been installed" % self._il_path) - else: - try: - return getattr(m, self._il_addtl) - except AttributeError: - raise KeyError( - "Module %s hasn't been installed into %s" % - (self._il_addtl, self._il_path) - ) + if self in importlater._unresolved: + raise ImportError( + "importlater.resolve_all() hasn't " + "been called (this is %s %s)" + % (self._il_path, self._il_addtl)) + + return getattr(self._initial_import, self._il_addtl) + + def _resolve(self): + importlater._unresolved.discard(self) + self._initial_import = compat.import_( + self._il_path, globals(), locals(), + [self._il_addtl]) def __getattr__(self, key): + if key == 'module': + raise ImportError("Could not resolve module %s" + % self._full_path) try: attr = getattr(self.module, key) except AttributeError: @@ -728,7 +742,6 @@ class importlater(object): self.__dict__[key] = attr return attr - # from paste.deploy.converters def asbool(obj): if isinstance(obj, compat.string_types): -- cgit v1.2.1 From f6198d9abf453182f4b111e0579a7a4ef1614e79 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 12 Aug 2013 17:50:37 -0400 Subject: - A large refactoring of the ``sqlalchemy.sql`` package has reorganized the import structure of many core modules. ``sqlalchemy.schema`` and ``sqlalchemy.types`` remain in the top-level package, but are now just lists of names that pull from within ``sqlalchemy.sql``. Their implementations are now broken out among ``sqlalchemy.sql.type_api``, ``sqlalchemy.sql.sqltypes``, ``sqlalchemy.sql.schema`` and ``sqlalchemy.sql.ddl``, the last of which was moved from ``sqlalchemy.engine``. ``sqlalchemy.sql.expression`` is also a namespace now which pulls implementations mostly from ``sqlalchemy.sql.elements``, ``sqlalchemy.sql.selectable``, and ``sqlalchemy.sql.dml``. Most of the "factory" functions used to create SQL expression objects have been moved to classmethods or constructors, which are exposed in ``sqlalchemy.sql.expression`` using a programmatic system. Care has been taken such that all the original import namespaces remain intact and there should be no impact on any existing applications. The rationale here was to break out these very large modules into smaller ones, provide more manageable lists of function names, to greatly reduce "import cycles" and clarify the up-front importing of names, and to remove the need for redundant functions and documentation throughout the expression package. --- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/langhelpers.py | 102 +++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 5 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 104566215..2237e1f13 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -25,7 +25,7 @@ from .langhelpers import iterate_attributes, class_hierarchy, \ getargspec_init, format_argspec_init, format_argspec_plus, \ get_func_kwargs, get_cls_kwargs, decorator, as_interface, \ memoized_property, memoized_instancemethod, md5_hex, \ - group_expirable_memoized_property, importlater, decode_slice, \ + group_expirable_memoized_property, importlater, dependencies, decode_slice, \ monkeypatch_proxied_specials, asbool, bool_or_str, coerce_kw_type,\ duck_type_collection, assert_arg_type, symbol, dictlike_iteritems,\ classproperty, set_creation_order, warn_exception, warn, NoneType,\ diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index d8e469a2b..e10a10e27 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -107,6 +107,30 @@ def decorator(target): return update_wrapper(decorate, target) +def public_factory(target): + """Produce a wrapping function for the given cls or classmethod. + + Rationale here is so that the __init__ method of the + class can serve as documentation for the function. + + """ + if isinstance(target, type): + fn = target.__init__ + callable_ = target + else: + fn = callable_ = target + spec = compat.inspect_getfullargspec(fn) + del spec[0][0] + #import pdb + #pdb.set_trace() + metadata = format_argspec_plus(spec, grouped=False) + code = 'lambda %(args)s: cls(%(apply_kw)s)' % metadata + decorated = eval(code, {'cls': callable_, 'symbol': symbol}) + decorated.__doc__ = fn.__doc__ + return decorated + #return update_wrapper(decorated, fn) + + class PluginLoader(object): def __init__(self, group, auto_fn=None): @@ -619,7 +643,11 @@ class memoized_property(object): return result def _reset(self, obj): - obj.__dict__.pop(self.__name__, None) + memoized_property.reset(obj, self.__name__) + + @classmethod + def reset(cls, obj, name): + obj.__dict__.pop(name, None) class memoized_instancemethod(object): @@ -698,15 +726,26 @@ class importlater(object): _unresolved = set() + _by_key = {} + + def __new__(cls, path, addtl): + key = path + "." + addtl + if key in importlater._by_key: + return importlater._by_key[key] + else: + importlater._by_key[key] = imp = object.__new__(cls) + return imp + def __init__(self, path, addtl): self._il_path = path self._il_addtl = addtl importlater._unresolved.add(self) @classmethod - def resolve_all(cls): + def resolve_all(cls, path): for m in list(importlater._unresolved): - m._resolve() + if m._full_path.startswith(path): + m._resolve() @property def _full_path(self): @@ -742,6 +781,61 @@ class importlater(object): self.__dict__[key] = attr return attr +def dependencies(*deps): + """Apply imported dependencies as arguments to a function. + + E.g.:: + + @util.dependencies( + "sqlalchemy.sql.widget", + "sqlalchemy.engine.default" + ); + def some_func(self, widget, default, arg1, arg2, **kw): + # ... + + Rationale is so that the impact of a dependency cycle can be + associated directly with the few functions that cause the cycle, + and not pollute the module-level namespace. + + """ + import_deps = [] + for dep in deps: + tokens = dep.split(".") + import_deps.append( + importlater( + ".".join(tokens[0:-1]), + tokens[-1] + ) + ) + + def decorate(fn): + spec = compat.inspect_getfullargspec(fn) + + spec_zero = list(spec[0]) + hasself = spec_zero[0] in ('self', 'cls') + + for i in range(len(import_deps)): + spec[0][i + (1 if hasself else 0)] = "import_deps[%r]" % i + + inner_spec = format_argspec_plus(spec, grouped=False) + + for impname in import_deps: + del spec_zero[1 if hasself else 0] + spec[0][:] = spec_zero + + outer_spec = format_argspec_plus(spec, grouped=False) + + code = 'lambda %(args)s: fn(%(apply_kw)s)' % { + "args": outer_spec['args'], + "apply_kw": inner_spec['apply_kw'] + } + + decorated = eval(code, locals()) + decorated.__defaults__ = getattr(fn, 'im_func', fn).__defaults__ + return update_wrapper(decorated, fn) + return decorate + + # from paste.deploy.converters def asbool(obj): if isinstance(obj, compat.string_types): @@ -943,7 +1037,7 @@ class _symbol(int): return repr(self) def __repr__(self): - return "" % self.name + return "symbol(%r)" % self.name _symbol.__name__ = 'symbol' -- cgit v1.2.1 From 59141d360e70d1a762719206e3cb0220b4c53fef Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 14 Aug 2013 19:58:34 -0400 Subject: - apply an import refactoring to the ORM as well - rework the event system so that event modules load after their targets, dependencies are reversed - create an improved strategy lookup system for the ORM - rework the ORM to have very few import cycles - move out "importlater" to just util.dependency - other tricks to cross-populate modules in as clear a way as possible --- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/langhelpers.py | 188 +++++++++++++++++++------------------ 2 files changed, 98 insertions(+), 92 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 2237e1f13..968421c8b 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -25,7 +25,7 @@ from .langhelpers import iterate_attributes, class_hierarchy, \ getargspec_init, format_argspec_init, format_argspec_plus, \ get_func_kwargs, get_cls_kwargs, decorator, as_interface, \ memoized_property, memoized_instancemethod, md5_hex, \ - group_expirable_memoized_property, importlater, dependencies, decode_slice, \ + group_expirable_memoized_property, dependencies, decode_slice, \ monkeypatch_proxied_specials, asbool, bool_or_str, coerce_kw_type,\ duck_type_collection, assert_arg_type, symbol, dictlike_iteritems,\ classproperty, set_creation_order, warn_exception, warn, NoneType,\ diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index e10a10e27..1fd5ccf30 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -107,7 +107,7 @@ def decorator(target): return update_wrapper(decorate, target) -def public_factory(target): +def public_factory(target, location): """Produce a wrapping function for the given cls or classmethod. Rationale here is so that the __init__ method of the @@ -117,18 +117,26 @@ def public_factory(target): if isinstance(target, type): fn = target.__init__ callable_ = target + doc = "Construct a new :class:`.%s` object. \n\n"\ + "This constructor is mirrored as a public API function; see :func:`~%s` "\ + "for a full usage and argument description." % ( + target.__name__, location, ) else: fn = callable_ = target + doc = "This function is mirrored; see :func:`~%s` "\ + "for a description of arguments." % location + spec = compat.inspect_getfullargspec(fn) del spec[0][0] - #import pdb - #pdb.set_trace() metadata = format_argspec_plus(spec, grouped=False) code = 'lambda %(args)s: cls(%(apply_kw)s)' % metadata decorated = eval(code, {'cls': callable_, 'symbol': symbol}) decorated.__doc__ = fn.__doc__ + if compat.py2k or hasattr(fn, '__func__'): + fn.__func__.__doc__ = doc + else: + fn.__doc__ = doc return decorated - #return update_wrapper(decorated, fn) class PluginLoader(object): @@ -703,85 +711,19 @@ class group_expirable_memoized_property(object): return memoized_instancemethod(fn) -class importlater(object): - """Deferred import object. - - e.g.:: - - somesubmod = importlater("mypackage.somemodule", "somesubmod") - - is equivalent to:: - - from mypackage.somemodule import somesubmod - - except evaluted upon attribute access to "somesubmod". - - importlater() currently requires that resolve_all() be - called, typically at the bottom of a package's __init__.py. - This is so that __import__ still called only at - module import time, and not potentially within - a non-main thread later on. - - """ - - _unresolved = set() - - _by_key = {} - def __new__(cls, path, addtl): - key = path + "." + addtl - if key in importlater._by_key: - return importlater._by_key[key] - else: - importlater._by_key[key] = imp = object.__new__(cls) - return imp - - def __init__(self, path, addtl): - self._il_path = path - self._il_addtl = addtl - importlater._unresolved.add(self) - - @classmethod - def resolve_all(cls, path): - for m in list(importlater._unresolved): - if m._full_path.startswith(path): - m._resolve() +def dependency_for(modulename): + def decorate(obj): + # TODO: would be nice to improve on this import silliness, + # unfortunately importlib doesn't work that great either + tokens = modulename.split(".") + mod = compat.import_(".".join(tokens[0:-1]), globals(), locals(), tokens[-1]) + mod = getattr(mod, tokens[-1]) + setattr(mod, obj.__name__, obj) + return obj + return decorate - @property - def _full_path(self): - return self._il_path + "." + self._il_addtl - - @memoized_property - def module(self): - if self in importlater._unresolved: - raise ImportError( - "importlater.resolve_all() hasn't " - "been called (this is %s %s)" - % (self._il_path, self._il_addtl)) - - return getattr(self._initial_import, self._il_addtl) - - def _resolve(self): - importlater._unresolved.discard(self) - self._initial_import = compat.import_( - self._il_path, globals(), locals(), - [self._il_addtl]) - - def __getattr__(self, key): - if key == 'module': - raise ImportError("Could not resolve module %s" - % self._full_path) - try: - attr = getattr(self.module, key) - except AttributeError: - raise AttributeError( - "Module %s has no attribute '%s'" % - (self._full_path, key) - ) - self.__dict__[key] = attr - return attr - -def dependencies(*deps): +class dependencies(object): """Apply imported dependencies as arguments to a function. E.g.:: @@ -798,17 +740,20 @@ def dependencies(*deps): and not pollute the module-level namespace. """ - import_deps = [] - for dep in deps: - tokens = dep.split(".") - import_deps.append( - importlater( - ".".join(tokens[0:-1]), - tokens[-1] + + def __init__(self, *deps): + self.import_deps = [] + for dep in deps: + tokens = dep.split(".") + self.import_deps.append( + dependencies._importlater( + ".".join(tokens[0:-1]), + tokens[-1] + ) ) - ) - def decorate(fn): + def __call__(self, fn): + import_deps = self.import_deps spec = compat.inspect_getfullargspec(fn) spec_zero = list(spec[0]) @@ -833,7 +778,68 @@ def dependencies(*deps): decorated = eval(code, locals()) decorated.__defaults__ = getattr(fn, 'im_func', fn).__defaults__ return update_wrapper(decorated, fn) - return decorate + + @classmethod + def resolve_all(cls, path): + for m in list(dependencies._unresolved): + if m._full_path.startswith(path): + m._resolve() + + _unresolved = set() + _by_key = {} + + class _importlater(object): + _unresolved = set() + + _by_key = {} + + def __new__(cls, path, addtl): + key = path + "." + addtl + if key in dependencies._by_key: + return dependencies._by_key[key] + else: + dependencies._by_key[key] = imp = object.__new__(cls) + return imp + + def __init__(self, path, addtl): + self._il_path = path + self._il_addtl = addtl + dependencies._unresolved.add(self) + + + @property + def _full_path(self): + return self._il_path + "." + self._il_addtl + + @memoized_property + def module(self): + if self in dependencies._unresolved: + raise ImportError( + "importlater.resolve_all() hasn't " + "been called (this is %s %s)" + % (self._il_path, self._il_addtl)) + + return getattr(self._initial_import, self._il_addtl) + + def _resolve(self): + dependencies._unresolved.discard(self) + self._initial_import = compat.import_( + self._il_path, globals(), locals(), + [self._il_addtl]) + + def __getattr__(self, key): + if key == 'module': + raise ImportError("Could not resolve module %s" + % self._full_path) + try: + attr = getattr(self.module, key) + except AttributeError: + raise AttributeError( + "Module %s has no attribute '%s'" % + (self._full_path, key) + ) + self.__dict__[key] = attr + return attr # from paste.deploy.converters -- cgit v1.2.1 From b83a1564f1f4bc8da2af2d33bfb4f524b620382b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 17 Aug 2013 00:31:57 -0400 Subject: Improved support for the cymysql driver, supporting version 0.6.5, courtesy Hajime Nakagami. --- lib/sqlalchemy/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 968421c8b..c52b86f0a 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -10,7 +10,7 @@ from .compat import callable, cmp, reduce, \ raise_from_cause, text_type, string_types, int_types, binary_type, \ quote_plus, with_metaclass, print_, itertools_filterfalse, u, ue, b,\ unquote_plus, b64decode, b64encode, byte_buffer, itertools_filter,\ - StringIO, inspect_getargspec + iterbytes, StringIO, inspect_getargspec from ._collections import KeyedTuple, ImmutableContainer, immutabledict, \ Properties, OrderedProperties, ImmutableProperties, OrderedDict, \ -- cgit v1.2.1 From 2038837fa4c03c6da4b6d4c389fd062abc4e844c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 17 Aug 2013 12:14:58 -0400 Subject: - spot checking of imports, obsolete functions --- lib/sqlalchemy/util/_collections.py | 1 - lib/sqlalchemy/util/langhelpers.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index 86a90828a..c3b44abae 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -6,7 +6,6 @@ """Collection classes and helpers.""" -import itertools import weakref import operator from .compat import threading, itertools_filterfalse diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 1fd5ccf30..8cf3db1bb 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -166,7 +166,6 @@ class PluginLoader(object): self.impls[name] = impl.load return impl.load() - from sqlalchemy import exc raise exc.ArgumentError( "Can't load plugin: %s:%s" % (self.group, name)) @@ -219,6 +218,7 @@ def get_cls_kwargs(cls, _set=None): try: + # TODO: who doesn't have this constant? from inspect import CO_VARKEYWORDS def inspect_func_args(fn): -- cgit v1.2.1 From 676876f4668520af267d7db883d9486a8924b320 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 18 Aug 2013 15:34:23 -0400 Subject: Fixed a potential issue in an ordered sequence implementation used by the ORM to iterate mapper hierarchies; under the Jython interpreter this implementation wasn't ordered, even though cPython and Pypy maintained ordering. Also in 0.8.3. [ticket:2794] --- lib/sqlalchemy/util/_collections.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index c3b44abae..b2e5c6250 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -650,18 +650,23 @@ class IdentitySet(object): class WeakSequence(object): def __init__(self, elements): - self._storage = weakref.WeakValueDictionary( - (idx, element) for idx, element in enumerate(elements) - ) + self._storage = [ + weakref.ref(element) for element in elements + ] + + def _remove(self, ref): + self._storage.remove(ref) def __iter__(self): - return iter(self._storage.values()) + return (obj for obj in (ref() for ref in self._storage) if obj is not None) def __getitem__(self, index): try: - return self._storage[index] + obj = self._storage[index] except KeyError: raise IndexError("Index %s out of range" % index) + else: + return obj() class OrderedIdentitySet(IdentitySet): -- cgit v1.2.1 From 9769628ac1c080dc69b3812bc546ff6df3318cbb Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 18 Aug 2013 16:05:38 -0400 Subject: - add WeakSequence.append() - fix and test weakref cleanout for WeakSequence, [ticket:2794] --- lib/sqlalchemy/util/_collections.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index b2e5c6250..a43115203 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -649,16 +649,23 @@ class IdentitySet(object): class WeakSequence(object): - def __init__(self, elements): + def __init__(self, __elements=()): self._storage = [ - weakref.ref(element) for element in elements + weakref.ref(element, self._remove) for element in __elements ] + def append(self, item): + self._storage.append(weakref.ref(item, self._remove)) + def _remove(self, ref): self._storage.remove(ref) + def __len__(self): + return len(self._storage) + def __iter__(self): - return (obj for obj in (ref() for ref in self._storage) if obj is not None) + return (obj for obj in + (ref() for ref in self._storage) if obj is not None) def __getitem__(self, index): try: -- cgit v1.2.1 From 633a86bfd4b68bb52e429f53ec34b3a47e787315 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 18 Aug 2013 16:16:25 -0400 Subject: and we don't need weakset --- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/compat.py | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index c52b86f0a..c68a64866 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -6,7 +6,7 @@ from .compat import callable, cmp, reduce, \ threading, py3k, py33, py2k, jython, pypy, cpython, win32, \ - pickle, dottedgetter, parse_qsl, namedtuple, next, WeakSet, reraise, \ + pickle, dottedgetter, parse_qsl, namedtuple, next, reraise, \ raise_from_cause, text_type, string_types, int_types, binary_type, \ quote_plus, with_metaclass, print_, itertools_filterfalse, u, ue, b,\ unquote_plus, b64decode, b64encode, byte_buffer, itertools_filter,\ diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index a89762b4e..7c2bc88d4 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -149,29 +149,6 @@ else: itertools_imap = itertools.imap - -try: - from weakref import WeakSet -except: - import weakref - - class WeakSet(object): - """Implement the small subset of set() which SQLAlchemy needs - here. """ - def __init__(self, values=None): - self._storage = weakref.WeakKeyDictionary() - if values is not None: - self._storage.update((value, None) for value in values) - - def __iter__(self): - return iter(self._storage) - - def union(self, other): - return WeakSet(set(self).union(other)) - - def add(self, other): - self._storage[other] = True - import time if win32 or jython: time_func = time.clock -- cgit v1.2.1 From 08a6a8b51916ab1d084a0070bbb07001cabb1c38 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 22 Sep 2013 20:35:40 -0400 Subject: - Removed some now unneeded version checks [ticket:2829] courtesy alex gaynor --- lib/sqlalchemy/util/queue.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/queue.py b/lib/sqlalchemy/util/queue.py index 537526bef..b66738aff 100644 --- a/lib/sqlalchemy/util/queue.py +++ b/lib/sqlalchemy/util/queue.py @@ -25,14 +25,7 @@ within QueuePool. from collections import deque from time import time as _time from .compat import threading -import sys -if sys.version_info < (2, 6): - def notify_all(condition): - condition.notify() -else: - def notify_all(condition): - condition.notify_all() __all__ = ['Empty', 'Full', 'Queue', 'SAAbort'] @@ -195,7 +188,7 @@ class Queue: if not self.not_full.acquire(False): return try: - notify_all(self.not_empty) + self.not_empty.notify_all() finally: self.not_full.release() -- cgit v1.2.1 From 6029496bd3fb78caeab349ef8df5b58f058db16e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Mon, 25 Nov 2013 14:46:58 -0500 Subject: - adjustment, the spec says: "Within the user and password field, any ":", "@", or "/" must be encoded." - so re-apply encoding to both password and username, don't encode spaces as plus signs, don't encode any chars outside of :, @, / on stringification - but we still parse for any %XX character (is that right?) --- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/compat.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index c68a64866..77339e56a 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -9,7 +9,7 @@ from .compat import callable, cmp, reduce, \ pickle, dottedgetter, parse_qsl, namedtuple, next, reraise, \ raise_from_cause, text_type, string_types, int_types, binary_type, \ quote_plus, with_metaclass, print_, itertools_filterfalse, u, ue, b,\ - unquote_plus, b64decode, b64encode, byte_buffer, itertools_filter,\ + unquote_plus, unquote, b64decode, b64encode, byte_buffer, itertools_filter,\ iterbytes, StringIO, inspect_getargspec from ._collections import KeyedTuple, ImmutableContainer, immutabledict, \ diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index 7c2bc88d4..ff434df43 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -40,7 +40,7 @@ if py3k: import builtins from inspect import getfullargspec as inspect_getfullargspec - from urllib.parse import quote_plus, unquote_plus, parse_qsl + from urllib.parse import quote_plus, unquote_plus, parse_qsl, quote, unquote import configparser from io import StringIO @@ -95,7 +95,7 @@ if py3k: else: from inspect import getargspec as inspect_getfullargspec inspect_getargspec = inspect_getfullargspec - from urllib import quote_plus, unquote_plus + from urllib import quote_plus, unquote_plus, quote, unquote from urlparse import parse_qsl import ConfigParser as configparser from StringIO import StringIO -- cgit v1.2.1 From 003b288b7644d8c39a13ac488ba56356c6375233 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 6 Dec 2013 15:53:59 -0500 Subject: - Made a slight adjustment to the logic which waits for a pooled connection to be available, such that for a connection pool with no timeout specified, it will every half a second break out of the wait to check for the so-called "abort" flag, which allows the waiter to break out in case the whole connection pool was dumped; normally the waiter should break out due to a notify_all() but it's possible this notify_all() is missed in very slim cases. This is an extension of logic first introduced in 0.8.0, and the issue has only been observed occasionally in stress tests. --- lib/sqlalchemy/util/queue.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/queue.py b/lib/sqlalchemy/util/queue.py index b66738aff..639b23a93 100644 --- a/lib/sqlalchemy/util/queue.py +++ b/lib/sqlalchemy/util/queue.py @@ -151,7 +151,6 @@ class Queue: return an item if one is immediately available, else raise the ``Empty`` exception (`timeout` is ignored in that case). """ - self.not_empty.acquire() try: if not block: @@ -159,7 +158,11 @@ class Queue: raise Empty elif timeout is None: while self._empty(): - self.not_empty.wait() + # wait for only half a second, then + # loop around, so that we can see a change in + # _sqla_abort_context in case we missed the notify_all() + # called by abort() + self.not_empty.wait(.5) if self._sqla_abort_context: raise SAAbort(self._sqla_abort_context) else: @@ -188,6 +191,9 @@ class Queue: if not self.not_full.acquire(False): return try: + # note that this is now optional + # as the waiters in get() both loop around + # to check the _sqla_abort_context flag periodically self.not_empty.notify_all() finally: self.not_full.release() -- cgit v1.2.1 From 2692238f45ae4d2f46949dfa52b16132bd266e0e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 18 Dec 2013 18:26:15 -0500 Subject: - Improvements to the system by which SQL types generate within ``__repr__()``, particularly with regards to the MySQL integer/numeric/ character types which feature a wide variety of keyword arguments. The ``__repr__()`` is important for use with Alembic autogenerate for when Python code is rendered in a migration script. [ticket:2893] --- lib/sqlalchemy/util/langhelpers.py | 85 ++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 31 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 8cf3db1bb..1a66426d0 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -19,6 +19,7 @@ from functools import update_wrapper from .. import exc import hashlib from . import compat +from . import _collections def md5_hex(x): if compat.py3k: @@ -392,44 +393,66 @@ def generic_repr(obj, additional_kw=(), to_inspect=None): """ if to_inspect is None: - to_inspect = obj + to_inspect = [obj] + else: + to_inspect = _collections.to_list(to_inspect) missing = object() - def genargs(): + pos_args = [] + kw_args = _collections.OrderedDict() + vargs = None + for i, insp in enumerate(to_inspect): try: - (args, vargs, vkw, defaults) = \ - inspect.getargspec(to_inspect.__init__) + (_args, _vargs, vkw, defaults) = \ + inspect.getargspec(insp.__init__) except TypeError: - return + continue + else: + default_len = defaults and len(defaults) or 0 + if i == 0: + if _vargs: + vargs = _vargs + if default_len: + pos_args.extend(_args[1:-default_len]) + else: + pos_args.extend(_args[1:]) + else: + kw_args.update([ + (arg, missing) for arg in _args[1:-default_len] + ]) - default_len = defaults and len(defaults) or 0 + if default_len: + kw_args.update([ + (arg, default) + for arg, default + in zip(_args[-default_len:], defaults) + ]) + output = [] - if not default_len: - for arg in args[1:]: - yield repr(getattr(obj, arg, None)) - if vargs is not None and hasattr(obj, vargs): - yield ', '.join(repr(val) for val in getattr(obj, vargs)) - else: - for arg in args[1:-default_len]: - yield repr(getattr(obj, arg, None)) - for (arg, defval) in zip(args[-default_len:], defaults): - try: - val = getattr(obj, arg, missing) - if val is not missing and val != defval: - yield '%s=%r' % (arg, val) - except: - pass - if additional_kw: - for arg, defval in additional_kw: - try: - val = getattr(obj, arg, missing) - if val is not missing and val != defval: - yield '%s=%r' % (arg, val) - except: - pass - - return "%s(%s)" % (obj.__class__.__name__, ", ".join(genargs())) + output.extend(repr(getattr(obj, arg, None)) for arg in pos_args) + + if vargs is not None and hasattr(obj, vargs): + output.extend([repr(val) for val in getattr(obj, vargs)]) + + for arg, defval in kw_args.items(): + try: + val = getattr(obj, arg, missing) + if val is not missing and val != defval: + output.append('%s=%r' % (arg, val)) + except: + pass + + if additional_kw: + for arg, defval in additional_kw: + try: + val = getattr(obj, arg, missing) + if val is not missing and val != defval: + output.append('%s=%r' % (arg, val)) + except: + pass + + return "%s(%s)" % (obj.__class__.__name__, ", ".join(output)) class portable_instancemethod(object): -- cgit v1.2.1 From 282a19d95251ecb033d30300b2e21ad6c57fbd77 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 21 Dec 2013 19:51:25 -0500 Subject: - alter the decorator function, and the newer public_factory function, to use a named def instead of a lambda. this so that TypeError on wrong arguments are more legible. [ticket:2884] --- lib/sqlalchemy/util/langhelpers.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 1a66426d0..7e261e38f 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -99,14 +99,21 @@ def decorator(target): metadata = dict(target=targ_name, fn=fn_name) metadata.update(format_argspec_plus(spec, grouped=False)) - - code = 'lambda %(args)s: %(target)s(%(fn)s, %(apply_kw)s)' % ( - metadata) - decorated = eval(code, {targ_name: target, fn_name: fn}) + metadata['name'] = fn.__name__ + code = """\ +def %(name)s(%(args)s): + return %(target)s(%(fn)s, %(apply_kw)s) +""" % metadata + decorated = _exec_code_in_env(code, + {targ_name: target, fn_name: fn}, + fn.__name__) decorated.__defaults__ = getattr(fn, 'im_func', fn).__defaults__ return update_wrapper(decorated, fn) return update_wrapper(decorate, target) +def _exec_code_in_env(code, env, fn_name): + exec(code, env) + return env[fn_name] def public_factory(target, location): """Produce a wrapping function for the given cls or classmethod. @@ -127,11 +134,18 @@ def public_factory(target, location): doc = "This function is mirrored; see :func:`~%s` "\ "for a description of arguments." % location + location_name = location.split(".")[-1] spec = compat.inspect_getfullargspec(fn) del spec[0][0] metadata = format_argspec_plus(spec, grouped=False) - code = 'lambda %(args)s: cls(%(apply_kw)s)' % metadata - decorated = eval(code, {'cls': callable_, 'symbol': symbol}) + metadata['name'] = location_name + code = """\ +def %(name)s(%(args)s): + return cls(%(apply_kw)s) +""" % metadata + env = {'cls': callable_, 'symbol': symbol} + exec(code, env) + decorated = env[location_name] decorated.__doc__ = fn.__doc__ if compat.py2k or hasattr(fn, '__func__'): fn.__func__.__doc__ = doc -- cgit v1.2.1 From c450cd6cb6d0c8fed110abcec5bc17ec4e0e8c5e Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 4 Jan 2014 00:35:48 -0500 Subject: - Fixed regression where using a ``functools.partial()`` with the event system would cause a recursion overflow due to usage of inspect.getargspec() on it in order to detect a legacy calling signature for certain events, and apparently there's no way to do this with a partial object. Instead we skip the legacy check and assume the modern style; the check itself now only occurs for the SessionEvents.after_bulk_update and SessionEvents.after_bulk_delete events. Those two events will require the new signature style if assigned to a "partial" event listener. [ticket:2905] --- lib/sqlalchemy/util/langhelpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 7e261e38f..105b64c6b 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -268,7 +268,9 @@ def get_callable_argspec(fn, no_self=False): return compat.ArgSpec(spec.args[1:], spec.varargs, spec.keywords, spec.defaults) elif hasattr(fn, '__func__'): return compat.inspect_getargspec(fn.__func__) - elif hasattr(fn, '__call__'): + elif hasattr(fn, '__call__') and \ + not hasattr(fn.__call__, '__call__'): # functools.partial does this; + # not much we can do return get_callable_argspec(fn.__call__) else: raise ValueError("Can't inspect function: %s" % fn) -- cgit v1.2.1 From 196f7ee6cc132aa0f31741af80fa5c0ba77efcf2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 5 Jan 2014 14:11:12 -0500 Subject: - conjunctions like and_() and or_() can now accept generators as arguments. --- lib/sqlalchemy/util/__init__.py | 3 ++- lib/sqlalchemy/util/_collections.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 77339e56a..fdf0c9dac 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -18,7 +18,8 @@ from ._collections import KeyedTuple, ImmutableContainer, immutabledict, \ column_dict, ordered_column_set, populate_column_dict, unique_list, \ UniqueAppender, PopulateDict, EMPTY_SET, to_list, to_set, \ to_column_set, update_copy, flatten_iterator, \ - LRUCache, ScopedRegistry, ThreadLocalRegistry, WeakSequence + LRUCache, ScopedRegistry, ThreadLocalRegistry, WeakSequence, \ + coerce_generator_arg from .langhelpers import iterate_attributes, class_hierarchy, \ portable_instancemethod, unbound_method_to_callable, \ diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index a43115203..24a3c1767 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -6,10 +6,12 @@ """Collection classes and helpers.""" +from __future__ import absolute_import import weakref import operator from .compat import threading, itertools_filterfalse from . import py2k +import types EMPTY_SET = frozenset() @@ -754,6 +756,11 @@ class UniqueAppender(object): def __iter__(self): return iter(self.data) +def coerce_generator_arg(arg): + if len(arg) == 1 and isinstance(arg[0], types.GeneratorType): + return list(arg[0]) + else: + return arg def to_list(x, default=None): if x is None: -- cgit v1.2.1 From f89d4d216bd7605c920b7b8a10ecde6bfea2238c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 5 Jan 2014 16:57:05 -0500 Subject: - happy new year --- lib/sqlalchemy/util/__init__.py | 2 +- lib/sqlalchemy/util/_collections.py | 2 +- lib/sqlalchemy/util/compat.py | 2 +- lib/sqlalchemy/util/deprecations.py | 2 +- lib/sqlalchemy/util/langhelpers.py | 2 +- lib/sqlalchemy/util/queue.py | 2 +- lib/sqlalchemy/util/topological.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index fdf0c9dac..eba64ed15 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -1,5 +1,5 @@ # util/__init__.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index 24a3c1767..c0a24ba4f 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -1,5 +1,5 @@ # util/_collections.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index ff434df43..f1346406e 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -1,5 +1,5 @@ # util/compat.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/lib/sqlalchemy/util/deprecations.py b/lib/sqlalchemy/util/deprecations.py index c315d2da6..c8854dc32 100644 --- a/lib/sqlalchemy/util/deprecations.py +++ b/lib/sqlalchemy/util/deprecations.py @@ -1,5 +1,5 @@ # util/deprecations.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 105b64c6b..b6ca7eb2a 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -1,5 +1,5 @@ # util/langhelpers.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/lib/sqlalchemy/util/queue.py b/lib/sqlalchemy/util/queue.py index 639b23a93..82ff55a5d 100644 --- a/lib/sqlalchemy/util/queue.py +++ b/lib/sqlalchemy/util/queue.py @@ -1,5 +1,5 @@ # util/queue.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/lib/sqlalchemy/util/topological.py b/lib/sqlalchemy/util/topological.py index de3dfd0ae..fe7e76896 100644 --- a/lib/sqlalchemy/util/topological.py +++ b/lib/sqlalchemy/util/topological.py @@ -1,5 +1,5 @@ # util/topological.py -# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors # # This module is part of SQLAlchemy and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -- cgit v1.2.1 From 1af8e2491dcbed723d2cdafd44fd37f1a6908e91 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 18 Jan 2014 19:26:56 -0500 Subject: - implement kwarg validation and type system for dialect-specific arguments; [ticket:2866] - add dialect specific kwarg functionality to ForeignKeyConstraint, ForeignKey --- lib/sqlalchemy/util/langhelpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/sqlalchemy/util') diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index b6ca7eb2a..82e37ce99 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -181,7 +181,7 @@ class PluginLoader(object): self.impls[name] = impl.load return impl.load() - raise exc.ArgumentError( + raise exc.NoSuchModuleError( "Can't load plugin: %s:%s" % (self.group, name)) -- cgit v1.2.1