Fix test suite failures and edge cases not being tested for

6 files changed, 227 insertions, 68 deletions

diff --git a/gen_tests.py b/gen_tests.py
index 14d8512..a195c1c 100644
--- a/gen_tests.py
+++ b/gen_tests.py
@@ -23,7 +23,11 @@ def main():
         print 'Collecting zdump(1) output for %s in zdump.out' % (zone,)
         tname = zone.replace(
                 '+', '_plus_').replace('-', '_minus_').replace('/','_')
-        zd_out, zd_in = popen2.popen2('%s -v -c 1800,2038 %s' % (zdump, zone))
+        # We don't yet support v2 format tzfile(5) files, so limit
+        # the daterange we test against - zdump understands v2 format
+        # files and will output historical records we can't cope with
+        # otherwise.
+        zd_out, zd_in = popen2.popen2('%s -v -c 1902,2038 %s' % (zdump, zone))
         zd_in.close()
         # Skip bogus output on 64bit architectures, per Bug #213816
         lines = [
diff --git a/gen_tzinfo.py b/gen_tzinfo.py
index ddeb7e5..5db2792 100644
--- a/gen_tzinfo.py
+++ b/gen_tzinfo.py
@@ -27,7 +27,7 @@ def allzones():
                 ])
     stripnum = len(os.path.commonprefix(zones))
     zones = [z[stripnum:] for z in zones]
-   
+
     if target:
         wanted = target + ['US/Eastern', 'UTC']
         zones = [z for z in zones if z in wanted]
@@ -128,7 +128,7 @@ def add_allzones(filename):
 
 def main(destdir):
     _destdir = os.path.join(os.path.abspath(destdir), 'dist')
-   
+
     dupe_src(_destdir)
     add_allzones(os.path.join(_destdir, 'pytz', '__init__.py'))
 
diff --git a/src/pytz/tests/test_tzinfo.py b/src/pytz/tests/test_tzinfo.py
index d79b26f..e9d2b7b 100644
--- a/src/pytz/tests/test_tzinfo.py
+++ b/src/pytz/tests/test_tzinfo.py
@@ -26,6 +26,22 @@ NOTIME = timedelta(0)
 UTC = pytz.timezone('UTC')
 GMT = pytz.timezone('GMT')
 
+
+def prettydt(dt):
+    """datetime as a string using a known format.
+
+    We don't use strftime as it doesn't handle years earlier than 1900
+    per http://bugs.python.org/issue1777412
+    """
+    if dt.utcoffset() >= timedelta(0):
+        offset = '+%s' % (dt.utcoffset(),)
+    else:
+        offset = '-%s' % (-1 * dt.utcoffset(),)
+    return '%04d-%02d-%02d %02d:%02d:%02d %s %s' % (
+        dt.year, dt.month, dt.day,
+        dt.hour, dt.minute, dt.second,
+        dt.tzname(), offset)
+
 class BasicTest(unittest.TestCase):
 
     def testVersion(self):
@@ -194,13 +210,15 @@ class USEasternDSTStartTestCase(unittest.TestCase):
 
             # Make sure arithmetic crossing DST boundaries ends
             # up in the correct timezone after normalization
+            utc_plus_delta = (utc_dt + delta).astimezone(self.tzinfo)
+            local_plus_delta = self.tzinfo.normalize(dt + delta)
             self.failUnlessEqual(
-                    (utc_dt + delta).astimezone(self.tzinfo).strftime(fmt),
-                    self.tzinfo.normalize(dt + delta).strftime(fmt),
+                    prettydt(utc_plus_delta),
+                    prettydt(local_plus_delta),
                     'Incorrect result for delta==%d days.  Wanted %r. Got %r'%(
                         days,
-                        (utc_dt + delta).astimezone(self.tzinfo).strftime(fmt),
-                        self.tzinfo.normalize(dt + delta).strftime(fmt),
+                        prettydt(utc_plus_delta),
+                        prettydt(local_plus_delta),
                         )
                     )
 
@@ -340,7 +358,6 @@ class VilniusCESTStartTestCase(USEasternDSTStartTestCase):
     # causing the clocks to go backwards for this summer time
     # switchover.
     tzinfo = pytz.timezone('Europe/Vilnius')
-    instant = timedelta(seconds=31)
     transition_time = datetime(1941, 6, 23, 21, 00, 00, tzinfo=UTC)
     before = {
         'tzname': 'MSK',
@@ -354,6 +371,117 @@ class VilniusCESTStartTestCase(USEasternDSTStartTestCase):
         }
 
 
+class LondonHistoryStartTestCase(USEasternDSTStartTestCase):
+    # The first known timezone transition in London was in 1847 when
+    # clocks where synchronized to GMT. However, we currently only
+    # understand v1 format tzfile(5) files which does handle years
+    # this far in the past, so our earliest known transition is in
+    # 1916.
+    tzinfo = pytz.timezone('Europe/London')
+    # transition_time = datetime(1847, 12, 1, 1, 15, 00, tzinfo=UTC)
+    # before = {
+    #     'tzname': 'LMT',
+    #     'utcoffset': timedelta(minutes=-75),
+    #     'dst': timedelta(0),
+    #     }
+    # after = {
+    #     'tzname': 'GMT',
+    #     'utcoffset': timedelta(0),
+    #     'dst': timedelta(0),
+    #     }
+    transition_time = datetime(1916, 5, 21, 2, 00, 00, tzinfo=UTC)
+    before = {
+        'tzname': 'GMT',
+        'utcoffset': timedelta(0),
+        'dst': timedelta(0),
+        }
+    after = {
+        'tzname': 'BST',
+        'utcoffset': timedelta(hours=1),
+        'dst': timedelta(hours=1),
+        }
+
+
+class LondonHistoryEndTestCase(USEasternDSTStartTestCase):
+    # Timezone switchovers are projected into the future, even
+    # though no official statements exist or could be believed even
+    # if they did exist. We currently only check the last known
+    # transition in 2037, as we are still using v1 format tzfile(5)
+    # files.
+    tzinfo = pytz.timezone('Europe/London')
+    # transition_time = datetime(2499, 10, 25, 1, 0, 0, tzinfo=UTC)
+    transition_time = datetime(2037, 10, 25, 1, 0, 0, tzinfo=UTC)
+    before = {
+        'tzname': 'BST',
+        'utcoffset': timedelta(hours=1),
+        'dst': timedelta(hours=1),
+        }
+    after = {
+        'tzname': 'GMT',
+        'utcoffset': timedelta(0),
+        'dst': timedelta(0),
+        }
+
+
+class NoumeaHistoryStartTestCase(USEasternDSTStartTestCase):
+    # Noumea adopted a whole hour offset in 1912. Previously
+    # it was 11 hours, 5 minutes and 48 seconds off UTC. However,
+    # due to limitations of the Python datetime library, we need
+    # to round that to 11 hours 6 minutes.
+    tzinfo = pytz.timezone('Pacific/Noumea')
+    transition_time = datetime(1912, 1, 12, 12, 54, 12, tzinfo=UTC)
+    before = {
+        'tzname': 'LMT',
+        'utcoffset': timedelta(hours=11, minutes=6),
+        'dst': timedelta(0),
+        }
+    after = {
+        'tzname': 'NCT',
+        'utcoffset': timedelta(hours=11),
+        'dst': timedelta(0),
+        }
+
+
+class NoumeaDSTEndTestCase(USEasternDSTStartTestCase):
+    # Noumea dropped DST in 1997.
+    tzinfo = pytz.timezone('Pacific/Noumea')
+    transition_time = datetime(1997, 3, 1, 15, 00, 00, tzinfo=UTC)
+    before = {
+        'tzname': 'NCST',
+        'utcoffset': timedelta(hours=12),
+        'dst': timedelta(hours=1),
+        }
+    after = {
+        'tzname': 'NCT',
+        'utcoffset': timedelta(hours=11),
+        'dst': timedelta(0),
+        }
+
+
+class NoumeaNoMoreDSTTestCase(NoumeaDSTEndTestCase):
+    # Noumea dropped DST in 1997. Here we test that it stops occuring.
+    transition_time = (
+        NoumeaDSTEndTestCase.transition_time + timedelta(days=365*10))
+    before = NoumeaDSTEndTestCase.after
+    after = NoumeaDSTEndTestCase.after
+
+
+class TahitiTestCase(USEasternDSTStartTestCase):
+    # Tahiti has had a single transition in its history.
+    tzinfo = pytz.timezone('Pacific/Tahiti')
+    transition_time = datetime(1912, 10, 1, 9, 58, 16, tzinfo=UTC)
+    before = {
+        'tzname': 'LMT',
+        'utcoffset': timedelta(hours=-9, minutes=-58),
+        'dst': timedelta(0),
+        }
+    after = {
+        'tzname': 'TAHT',
+        'utcoffset': timedelta(hours=-10),
+        'dst': timedelta(0),
+        }
+
+
 class ReferenceUSEasternDSTStartTestCase(USEasternDSTStartTestCase):
     tzinfo = reference.Eastern
     def test_arithmetic(self):
diff --git a/src/pytz/tzfile.py b/src/pytz/tzfile.py
index b5f818f..7ea00c1 100644
--- a/src/pytz/tzfile.py
+++ b/src/pytz/tzfile.py
@@ -12,12 +12,12 @@ from pytz.tzinfo import memorized_datetime, memorized_timedelta
 
 
 def build_tzinfo(zone, fp):
-    head_fmt = '>4s 16x 6l'
+    head_fmt = '>4s c 15x 6l'
     head_size = calcsize(head_fmt)
-    (magic,ttisgmtcnt,ttisstdcnt,leapcnt,
-     timecnt,typecnt,charcnt) =  unpack(head_fmt, fp.read(head_size))
+    (magic, format, ttisgmtcnt, ttisstdcnt,leapcnt, timecnt,
+        typecnt, charcnt) =  unpack(head_fmt, fp.read(head_size))
 
-    # Make sure it is a tzinfo(5) file
+    # Make sure it is a tzfile(5) file
     assert magic == 'TZif'
 
     # Read out the transition times, localtime indices and ttinfo structures.
@@ -84,12 +84,13 @@ def build_tzinfo(zone, fp):
                         break
                 dst = inf[0] - prev_inf[0] # dst offset
 
-                if dst < 0: # Negative dst? Look further.
+                if dst <= 0: # Bad dst? Look further.
                     for j in range(i+1, len(transitions)):
                         stdinf = ttinfo[lindexes[j]]
                         if not stdinf[1]:
-                            break # Found std time.
-                    dst = inf[0] - stdinf[0]
+                            dst = inf[0] - stdinf[0]
+                            if dst > 0:
+                                break # Found a useful std time.
 
             tzname = inf[2]
 
diff --git a/src/pytz/tzinfo.py b/src/pytz/tzinfo.py
index 55afcb2..f31dc90 100644
--- a/src/pytz/tzinfo.py
+++ b/src/pytz/tzinfo.py
@@ -68,13 +68,13 @@ class BaseTzInfo(tzinfo):
 class StaticTzInfo(BaseTzInfo):
     '''A timezone that has a constant offset from UTC
 
-    These timezones are rare, as most regions have changed their
-    offset from UTC at some point in their history
+    These timezones are rare, as most locations have changed their
+    offset at some point in their history
     '''
     def fromutc(self, dt):
         '''See datetime.tzinfo.fromutc'''
         return (dt + self._utcoffset).replace(tzinfo=self)
-    
+
     def utcoffset(self,dt):
         '''See datetime.tzinfo.utcoffset'''
         return self._utcoffset
@@ -110,11 +110,10 @@ class StaticTzInfo(BaseTzInfo):
 
 class DstTzInfo(BaseTzInfo):
     '''A timezone that has a variable offset from UTC
-   
-    The offset might change if daylight savings time comes into effect,
-    or at a point in history when the region decides to change their 
-    timezone definition. 
 
+    The offset might change if daylight savings time comes into effect,
+    or at a point in history when the region decides to change their
+    timezone definition.
     '''
     # Overridden in subclass
     _utc_transition_times = None # Sorted list of DST transition times in UTC
@@ -180,7 +179,6 @@ class DstTzInfo(BaseTzInfo):
         >>> before = eastern.normalize(before)
         >>> before.strftime(fmt)
         '2002-10-27 01:50:00 EDT (-0400)'
-
         '''
         if dt.tzinfo is None:
             raise ValueError, 'Naive time - no tzinfo set'
@@ -194,13 +192,13 @@ class DstTzInfo(BaseTzInfo):
 
     def localize(self, dt, is_dst=False):
         '''Convert naive time to local time.
-        
+
         This method should be used to construct localtimes, rather
         than passing a tzinfo argument to a datetime constructor.
 
         is_dst is used to determine the correct timezone in the ambigous
         period at the end of daylight savings time.
-        
+
         >>> from pytz import timezone
         >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)'
         >>> amdam = timezone('Europe/Amsterdam')
@@ -223,7 +221,7 @@ class DstTzInfo(BaseTzInfo):
         AmbiguousTimeError: 2004-10-31 02:00:00
 
         is_dst defaults to False
-        
+
         >>> amdam.localize(dt) == amdam.localize(dt, False)
         True
 
@@ -252,12 +250,14 @@ class DstTzInfo(BaseTzInfo):
         if dt.tzinfo is not None:
             raise ValueError, 'Not naive datetime (tzinfo is already set)'
 
-        # Find the possibly correct timezones. We probably just have one,
-        # but we might end up with two if we are in the end-of-DST
-        # transition period. Or possibly more in some particularly confused
-        # location...
+        # Find the two best possibilities.
         possible_loc_dt = set()
-        for tzinfo in self._tzinfos.values():
+        for delta in [timedelta(days=-1), timedelta(days=1)]:
+            loc_dt = dt + delta
+            idx = max(0, bisect_right(
+                self._utc_transition_times, loc_dt) - 1)
+            inf = self._transition_info[idx]
+            tzinfo = self._tzinfos[inf]
             loc_dt = tzinfo.normalize(dt.replace(tzinfo=tzinfo))
             if loc_dt.replace(tzinfo=None) == dt:
                 possible_loc_dt.add(loc_dt)
@@ -324,7 +324,7 @@ class DstTzInfo(BaseTzInfo):
                     )
         filtered_possible_loc_dt.sort(mycmp)
         return filtered_possible_loc_dt[0]
-        
+
     def utcoffset(self, dt):
         '''See datetime.tzinfo.utcoffset'''
         return self._utcoffset
@@ -388,7 +388,7 @@ class NonExistentTimeError(InvalidTimeError):
 
 def unpickler(zone, utcoffset=None, dstoffset=None, tzname=None):
     """Factory function for unpickling pytz tzinfo instances.
-    
+
     This is shared for both StaticTzInfo and DstTzInfo instances, because
     database changes could cause a zones implementation to switch between
     these two base classes and we can't break pickles on a pytz version
diff --git a/test_zdump.py b/test_zdump.py
index 0cb91ba..c815a28 100644
--- a/test_zdump.py
+++ b/test_zdump.py
@@ -19,7 +19,6 @@ class ZdumpTestCase(unittest.TestCase):
             loc_dt.replace(tzinfo=None))
 
     def local_to_utc_check(self, zone, utc_dt, loc_dt, loc_tzname, is_dst):
-        loc_tz = pytz.timezone(zone)
         self.failUnlessEqual(
             loc_dt.astimezone(pytz.utc).replace(tzinfo=None),
             utc_dt.replace(tzinfo=None))
@@ -29,19 +28,18 @@ def test_suite():
     testcases = []
     raw_data = open(
         os.path.join(os.path.dirname(__file__), 'zdump.out'), 'r').readlines()
-    raw_data.reverse() # Keep tests running in alphabetical order
     last_zone = None
     test_class = None
-    for line in raw_data:
-        m = re.match(
-        r'^([^\s]+) \s+ (.+) \s UTC \s+ = \s+ (.+) \s ([^\s]+) \s+ '
-            r'isdst=(0|1)$',
-            line, re.X
-            )
-        if m:
-            zone, utc_string, loc_string, tzname, is_dst = m.groups()
-        else:
+    zdump_line_re = re.compile(r'''(?x)
+        ^([^\s]+) \s+ (.+) \s UTC \s+ = \s+ (.+) \s ([^\s]+) \s+ isdst=(0|1)$
+        ''')
+    for i in range(0, len(raw_data)):
+        line = raw_data[i]
+        m = zdump_line_re.search(line)
+        if m is None:
             raise RuntimeError, 'Dud line %r' % (line,)
+        zone, utc_string, loc_string, tzname, is_dst = m.groups()
+        is_dst = bool(int(is_dst))
 
         if zone != last_zone:
             classname = zone.replace(
@@ -49,31 +47,49 @@ def test_suite():
             test_class = type(classname, (ZdumpTestCase,), {})
             testcases.append(test_class)
             last_zone = zone
-            prev_loc_dt = None
-            prev_is_dst = False
+            skip_next_local = False
 
         utc_dt = datetime(
             *strptime(utc_string, '%a %b %d %H:%M:%S %Y')[:6])
         loc_dt = datetime(
             *strptime(loc_string, '%a %b %d %H:%M:%S %Y')[:6])
 
-        # Urgh - utcoffset() and dst() have to be rounded to the nearest
-        # minute, so we need to break our tests to match this limitation
-        real_offset = loc_dt - utc_dt
-        secs = real_offset.seconds + real_offset.days*86400
-        fake_offset = timedelta(seconds=int((secs+30)/60)*60)
-        loc_dt = utc_dt + fake_offset
+        def round_dt(loc_dt, utc_dt):
+            # Urgh - utcoffset() and dst() have to be rounded to the nearest
+            # minute, so we need to break our tests to match this limitation
+            real_offset = loc_dt - utc_dt
+            secs = real_offset.seconds + real_offset.days*86400
+            fake_offset = timedelta(seconds=int((secs+30)/60)*60)
+            return utc_dt + fake_offset
+
+        loc_dt = round_dt(loc_dt, utc_dt)
 
-        # If the naive time on the previous line is greater than on this
+        # If the naive time on the next line is less than on this
         # line, and we arn't seeing an end-of-dst transition, then
-        # we can't do our local->utc test since we are in an ambiguous
-        # time period (ie. we have wound back the clock but don't have
-        # differing is_dst flags to resolve the ambiguity)
-        skip_local = (
-            prev_loc_dt is not None and prev_loc_dt > loc_dt and
-            bool(prev_is_dst) == bool(is_dst))
-        prev_loc_dt = loc_dt
-        prev_is_dst = is_dst
+        # we can't do our local->utc tests for either this nor the
+        # next line since we are in an ambiguous time period (ie.
+        # we have wound back the clock but don't have differing
+        # is_dst flags to resolve the ambiguity)
+        skip_local = skip_next_local
+        skip_next_local = False
+        try:
+            m = zdump_line_re.match(raw_data[i+1])
+        except IndexError:
+            m = None
+        if m is not None:
+            (next_zone, next_utc_string, next_loc_string,
+                next_tzname, next_is_dst) = m.groups()
+            next_is_dst = bool(int(next_is_dst))
+            if next_zone == zone and next_is_dst == is_dst:
+                next_utc_dt = datetime(
+                    *strptime(next_utc_string, '%a %b %d %H:%M:%S %Y')[:6])
+                next_loc_dt = round_dt(
+                    datetime(*strptime(
+                        next_loc_string, '%a %b %d %H:%M:%S %Y')[:6]),
+                    next_utc_dt)
+                if next_loc_dt <= loc_dt:
+                    skip_local = True
+                    skip_next_local = True
 
         loc_tz = pytz.timezone(zone)
         loc_dt = loc_tz.localize(loc_dt, is_dst)
@@ -83,22 +99,32 @@ def test_suite():
         test_name = 'test_utc_to_local_%04d_%02d_%02d_%02d_%02d_%02d' % (
             utc_dt.year, utc_dt.month, utc_dt.day,
             utc_dt.hour, utc_dt.minute, utc_dt.second)
-        def test_utc_to_local(self):
+        def test_utc_to_local(
+            self, zone=zone, utc_dt=utc_dt, loc_dt=loc_dt, tzname=tzname,
+            is_dst=is_dst):
             self.utc_to_local_check(zone, utc_dt, loc_dt, tzname, is_dst)
         test_utc_to_local.__name__ = test_name
-        #test_utc_to_local.__doc__ = line
         setattr(test_class, test_name, test_utc_to_local)
 
         if not skip_local:
             test_name = 'test_local_to_utc_%04d_%02d_%02d_%02d_%02d_%02d' % (
-                utc_dt.year, utc_dt.month, utc_dt.day,
-                utc_dt.hour, utc_dt.minute, utc_dt.second)
-            def test_local_to_utc(self):
-                self.utc_to_local_check(zone, utc_dt, loc_dt, tzname, is_dst)
+                loc_dt.year, loc_dt.month, loc_dt.day,
+                loc_dt.hour, loc_dt.minute, loc_dt.second)
+            if is_dst:
+                test_name += '_dst'
+            else:
+                test_name += '_nodst'
+            def test_local_to_utc(
+                self, zone=zone, utc_dt=utc_dt, loc_dt=loc_dt, tzname=tzname,
+                is_dst=is_dst):
+                self.local_to_utc_check(zone, utc_dt, loc_dt, tzname, is_dst)
             test_local_to_utc.__name__ = test_name
-            #test_local_to_utc.__doc__ = line
             setattr(test_class, test_name, test_local_to_utc)
 
+    classname = zone.replace(
+            '+', '_plus_').replace('-', '_minus_').replace('/','_')
+    test_class = type(classname, (ZdumpTestCase,), {})
+    testcases.append(test_class)
 
     suite = unittest.TestSuite()
     while testcases: