Lazy load the all_timezones and common_timezones data structuresrelease_2013d

3 files changed, 159 insertions, 40 deletions

diff --git a/gen_tzinfo.py b/gen_tzinfo.py
index 052cbe6..a1ae7e5 100644
--- a/gen_tzinfo.py
+++ b/gen_tzinfo.py
@@ -122,17 +122,17 @@ def add_allzones(filename):
 
     print >> outf, 'all_timezones = \\'
     pprint(sorted(allzones()), outf)
-    print >> outf, '''all_timezones = [
-        tz for tz in all_timezones if resource_exists(tz)]
+    print >> outf, '''all_timezones = LazyList(
+        tz for tz in all_timezones if resource_exists(tz))
         '''
-    print >> outf, 'all_timezones_set = set(all_timezones)'
+    print >> outf, 'all_timezones_set = LazySet(all_timezones)'
 
     print >> outf, 'common_timezones = \\'
     pprint(cz, outf)
-    print >> outf, '''common_timezones = [
-        tz for tz in common_timezones if tz in all_timezones]
+    print >> outf, '''common_timezones = LazyList(
+            tz for tz in common_timezones if tz in all_timezones)
         '''
-    print >> outf, 'common_timezones_set = set(common_timezones)'
+    print >> outf, 'common_timezones_set = LazySet(common_timezones)'
 
     outf.close()
 
diff --git a/src/pytz/__init__.py b/src/pytz/__init__.py
index 59fc4a4..a6e5282 100644
--- a/src/pytz/__init__.py
+++ b/src/pytz/__init__.py
@@ -26,10 +26,6 @@ __all__ = [
     ]
 
 import sys, datetime, os.path, gettext
-try:
-    from UserDict import DictMixin
-except ImportError:
-    from collections import Mapping as DictMixin
 
 try:
     from pkg_resources import resource_stream
@@ -40,6 +36,7 @@ from pytz.exceptions import AmbiguousTimeError
 from pytz.exceptions import InvalidTimeError
 from pytz.exceptions import NonExistentTimeError
 from pytz.exceptions import UnknownTimeZoneError
+from pytz.lazy import LazyDict, LazyList, LazySet
 from pytz.tzinfo import unpickler
 from pytz.tzfile import build_tzinfo, _byte_string
 
@@ -292,36 +289,8 @@ def _p(*args):
 _p.__safe_for_unpickling__ = True
 
 
-class _LazyDict(DictMixin):
-    """Dictionary populated on first use."""
-    data = None
-    def __getitem__(self, key):
-        if self.data is None:
-            self._fill()
-        return self.data[key.upper()]
-
-    def __contains__(self, key):
-        if self.data is None:
-            self._fill()
-        return key in self.data
-
-    def __iter__(self):
-        if self.data is None:
-            self._fill()
-        return iter(self.data)
-
-    def __len__(self):
-        if self.data is None:
-            self._fill()
-        return len(self.data)
-
-    def keys(self):
-        if self.data is None:
-            self._fill()
-        return self.data.keys()
-
 
-class _CountryTimezoneDict(_LazyDict):
+class _CountryTimezoneDict(LazyDict):
     """Map ISO 3166 country code to a list of timezone names commonly used
     in that country.
 
@@ -379,7 +348,7 @@ class _CountryTimezoneDict(_LazyDict):
 country_timezones = _CountryTimezoneDict()
 
 
-class _CountryNameDict(_LazyDict):
+class _CountryNameDict(LazyDict):
     '''Dictionary proving ISO3166 code -> English name.
 
     >>> print(country_names['au'])
diff --git a/src/pytz/lazy.py b/src/pytz/lazy.py
new file mode 100644
index 0000000..aed7e10
--- /dev/null
+++ b/src/pytz/lazy.py
@@ -0,0 +1,150 @@
+from threading import RLock
+try:
+    from UserDict import DictMixin
+except ImportError:
+    from collections import Mapping as DictMixin
+
+
+# With lazy loading, we might end up with multiple threads triggering
+# it at the same time. We need a lock.
+_fill_lock = RLock()
+
+
+class LazyDict(DictMixin):
+    """Dictionary populated on first use."""
+    data = None
+    def __getitem__(self, key):
+        if self.data is None:
+            _fill_lock.acquire()
+            try:
+                if self.data is None:
+                    self._fill()
+            finally:
+                _fill_lock.release()
+        return self.data[key.upper()]
+
+    def __contains__(self, key):
+        if self.data is None:
+            _fill_lock.acquire()
+            try:
+                if self.data is None:
+                    self._fill()
+            finally:
+                _fill_lock_release()
+        return key in self.data
+
+    def __iter__(self):
+        if self.data is None:
+            _fill_lock.acquire()
+            try:
+                if self.data is None:
+                    self._fill()
+            finally:
+                _fill_lock.release()
+        return iter(self.data)
+
+    def __len__(self):
+        if self.data is None:
+            _fill_lock.acquire()
+            try:
+                if self.data is None:
+                    self._fill()
+            finally:
+                _fill_lock.release()
+        return len(self.data)
+
+    def keys(self):
+        if self.data is None:
+            _fill_lock.acquire()
+            try:
+                if self.data is None:
+                    self._fill()
+            finally:
+                _fill_lock.release()
+        return self.data.keys()
+
+
+class LazyList(list):
+    """List populated on first use."""
+    def __new__(cls, fill_iter):
+
+        class LazyList(list):
+            _fill_iter = None
+
+        _props = (
+            '__str__', '__repr__', '__unicode__',
+            '__hash__', '__sizeof__', '__cmp__', '__nonzero__',
+            '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__',
+            'append', 'count', 'index', 'extend', 'insert', 'pop', 'remove',
+            'reverse', 'sort', '__add__', '__radd__', '__iadd__', '__mul__',
+            '__rmul__', '__imul__', '__contains__', '__len__', '__nonzero__',
+            '__getitem__', '__setitem__', '__delitem__', '__iter__',
+            '__reversed__', '__getslice__', '__setslice__', '__delslice__')
+
+        def lazy(name):
+            def _lazy(self, *args, **kw):
+                if self._fill_iter is not None:
+                    _fill_lock.acquire()
+                    try:
+                        if self._fill_iter is not None:
+                            list.extend(self, self._fill_iter)
+                            self._fill_iter = None
+                    finally:
+                        _fill_lock.release()
+                real = getattr(list, name)
+                setattr(self.__class__, name, real)
+                return real(self, *args, **kw)
+            return _lazy
+
+        for name in _props:
+            setattr(LazyList, name, lazy(name))
+
+        new_list = LazyList()
+        new_list._fill_iter = fill_iter
+        return new_list
+
+
+class LazySet(set):
+    """Set populated on first use."""
+    def __new__(cls, fill_iter):
+
+        class LazySet(set):
+            _fill_iter = None
+
+        _props = (
+            '__str__', '__repr__', '__unicode__',
+            '__hash__', '__sizeof__', '__cmp__', '__nonzero__',
+            '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__',
+            '__contains__', '__len__', '__nonzero__',
+            '__getitem__', '__setitem__', '__delitem__', '__iter__',
+            '__sub__', '__and__', '__xor__', '__or__',
+            '__rsub__', '__rand__', '__rxor__', '__ror__',
+            '__isub__', '__iand__', '__ixor__', '__ior__',
+            'add', 'clear', 'copy', 'difference', 'difference_update',
+            'discard', 'intersection', 'intersection_update', 'isdisjoint',
+            'issubset', 'issuperset', 'pop', 'remove',
+            'symmetric_difference', 'symmetric_difference_update',
+            'union', 'update')
+
+        def lazy(name):
+            def _lazy(self, *args, **kw):
+                if self._fill_iter is not None:
+                    _fill_lock.acquire()
+                    try:
+                        if self._fill_iter is not None:
+                            for i in self._fill_iter:
+                                set.add(self, i)
+                            self._fill_iter = None
+                    finally:
+                        _fill_lock.release()
+                real = getattr(set, name)
+                setattr(self.__class__, name, real)
+                return real(self, *args, **kw)
+            return _lazy
+
+        for name in _props:
+            setattr(LazySet, name, lazy(name))
+
+        new_set = LazySet()
+        new_set._fill_iter = fill_iter
+        return new_set