Merge pull request #29 from zopefoundation/issue9

Fix issue 9: add default checkers for itertools.groupby

2 files changed, 162 insertions, 43 deletions

diff --git a/src/zope/security/checker.py b/src/zope/security/checker.py
index f870a13..e5e23c3 100644
--- a/src/zope/security/checker.py
+++ b/src/zope/security/checker.py
@@ -860,6 +860,74 @@ _fixup_zope_interface()
 del _fixup_zope_interface
 
 
+def _fixup_itertools():
+    # itertools.groupby is a built-in custom iterator type introduced
+    # in python2.4. It should have the same checker as other built-in
+    # iterators.
+
+    # Also, itertools._grouper also needs to be exposed as an
+    # iterator. Its type is not exposed by name, but can be accessed
+    # like so: type(list(itertools.groupby([0]))[0][1])
+
+    import itertools
+
+    group = itertools.groupby([0])
+    type_group = type(group)
+    if type_group not in _default_checkers:
+        _default_checkers[type_group] = _iteratorChecker
+
+    type_grouper = type(list(group)[0][1])
+    if type_grouper not in _default_checkers:
+        _default_checkers[type_grouper] = _iteratorChecker
+
+    # There are also many other custom types in itertools that need the
+    # same treatment. See a similar list in test_checker.py:test_itertools_checkers
+    pred = lambda x: x
+    iterable = (1, 2, 3)
+    pred_iterable = (pred, iterable)
+    missing_in_py3 = {'ifilter', 'ifilterfalse', 'imap',
+                      'izip', 'izip_longest'}
+    missing_in_py2 = {'zip_longest', 'accumulate', 'compress',
+                      'combinations', 'combinations_with_replacement'}
+    missing = missing_in_py3 if sys.version_info[0] >= 3 else missing_in_py2
+    for func, args in (('count', ()),
+                       ('cycle', ((),)),
+                       ('dropwhile', pred_iterable),
+                       ('ifilter', pred_iterable),
+                       ('ifilterfalse', pred_iterable),
+                       ('imap', pred_iterable),
+                       ('islice', (iterable, 2)),
+                       ('izip', (iterable,)),
+                       ('izip_longest', (iterable,)),
+                       ('permutations', (iterable,)),
+                       ('product', (iterable,)),
+                       ('repeat', (1, 2)),
+                       ('starmap', pred_iterable),
+                       ('takewhile', pred_iterable),
+                       ('tee', (iterable,)),
+                       # Python 3 additions
+                       ('zip_longest', (iterable,)),
+                       ('accumulate', (iterable,)),
+                       ('compress', (iterable, ())),
+                       ('combinations', (iterable, 1)),
+                       ('combinations_with_replacement', (iterable, 1)),
+    ):
+        try:
+            func = getattr(itertools, func)
+        except AttributeError:
+            if func in missing:
+                continue
+            raise
+        result = func(*args)
+        if func == itertools.tee:
+            result = result[0]
+        tresult = type(result)
+        if tresult not in _default_checkers:
+            _default_checkers[tresult] = _iteratorChecker
+
+_fixup_itertools()
+del _fixup_itertools
+
 def _clear():
     _checkers.clear()
     _checkers.update(_default_checkers)
diff --git a/src/zope/security/tests/test_checker.py b/src/zope/security/tests/test_checker.py
index b82799f..692cbde 100644
--- a/src/zope/security/tests/test_checker.py
+++ b/src/zope/security/tests/test_checker.py
@@ -14,6 +14,7 @@
 """Tests for zope.security.checker
 """
 import unittest
+from zope.security import checker
 
 def _skip_if_not_Py2(testfunc):
     import sys
@@ -201,8 +202,7 @@ _marker = []
 class CheckerTestsBase(object):
 
     def _getTargetClass(self):
-        from zope.security.checker import Checker
-        return Checker
+        raise NotImplementedError("Subclasses must define")
 
     def _makeOne(self, get_permissions=_marker, set_permissions=_marker):
         if get_permissions is _marker:
@@ -553,21 +553,51 @@ class CheckerTestsBase(object):
         self.assertTrue(i.hint_called, "__length_hint__ should be called")
 
 
-class CheckerPyTests(unittest.TestCase, CheckerTestsBase):
+    def test_iteration_of_itertools_groupby(self):
+        # itertools.groupby is a custom iterator type.
+        # The groups it returns are also custom.
+        from zope.security.checker import ProxyFactory
+        from zope.security.checker import Checker
+
+        from itertools import groupby
+
+        group = groupby([0])
+        list_group = list(group)
+        self.assertEqual(1, len(list_group))
+        self.assertEqual(0, list_group[0][0])
+
+        proxy = ProxyFactory(groupby([0]))
+        list_group = list(proxy)
+        self.assertEqual(1, len(list_group))
+        self.assertEqual(0, list_group[0][0])
+
+        # Note that groupby docs say: "The returned group is itself an
+        # iterator that shares the underlying iterable with groupby().
+        # Because the source is shared, when the groupby() object is
+        # advanced, the previous group is no longer visible."
+        # For a one-item list, this doesn't make a difference on CPython,
+        # but it does on PyPy (if we use list(group), the list_group[0][0] is
+        # empty); probably this has to do with GC
+        proxy = ProxyFactory(groupby([0]))
+        _key, subiter = next(proxy)
+        self.assertEqual([0], list(subiter))
+
+
+class TestCheckerPy(CheckerTestsBase, unittest.TestCase):
 
     def _getTargetClass(self):
         from zope.security.checker import CheckerPy
         return CheckerPy
 
 
-class CheckerTests(unittest.TestCase, CheckerTestsBase):
+class TestChecker(CheckerTestsBase, unittest.TestCase):
 
     def _getTargetClass(self):
         from zope.security.checker import Checker
         return Checker
 
 
-class TracebackSupplementTests(unittest.TestCase):
+class TestTracebackSupplement(unittest.TestCase):
 
     def _getTargetClass(self):
         from zope.security.checker import TracebackSupplement
@@ -610,7 +640,7 @@ class TracebackSupplementTests(unittest.TestCase):
                          ])
 
 
-class GlobalTests(unittest.TestCase):
+class TestGlobal(unittest.TestCase):
 
     def _getTargetClass(self):
         from zope.security.checker import Global
@@ -907,9 +937,55 @@ class _SelectCheckerBase(object):
         _checkers[Foo] = _factory_factory
         self.assertTrue(self._callFUT(Foo()) is checker)
 
-
-
-class Test_selectCheckerPy(unittest.TestCase, _SelectCheckerBase):
+    def test_itertools_checkers(self):
+        from zope.security.checker import _iteratorChecker
+        import sys
+        import itertools
+        pred = lambda x: x
+        iterable = (1, 2, 3)
+        pred_iterable = (pred, iterable)
+        missing_in_py3 = {'ifilter', 'ifilterfalse', 'imap',
+                          'izip', 'izip_longest'}
+        missing_in_py2 = {'zip_longest', 'accumulate', 'compress',
+                          'combinations', 'combinations_with_replacement'}
+        missing = missing_in_py3 if sys.version_info[0] >= 3 else missing_in_py2
+        for func, args in (('count', ()),
+                           ('cycle', ((),)),
+                           ('dropwhile', pred_iterable),
+                           ('ifilter', pred_iterable),
+                           ('ifilterfalse', pred_iterable),
+                           ('imap', pred_iterable),
+                           ('islice', (iterable, 2)),
+                           ('izip', (iterable,)),
+                           ('izip_longest', (iterable,)),
+                           ('permutations', (iterable,)),
+                           ('product', (iterable,)),
+                           ('repeat', (1, 2)),
+                           ('starmap', pred_iterable),
+                           ('takewhile', pred_iterable),
+                           ('tee', (iterable,)),
+                           # Python 3 additions
+                           ('zip_longest', (iterable,)),
+                           ('accumulate', (iterable,)),
+                           ('compress', (iterable, ())),
+                           ('combinations', (iterable, 1)),
+                           ('combinations_with_replacement', (iterable, 1)),
+        ):
+            try:
+                func = getattr(itertools, func)
+            except AttributeError:
+                if func in missing:
+                    continue
+                raise
+            __traceback_info__ = func
+            result = func(*args)
+            if func == itertools.tee:
+                result = result[0]
+
+            self.assertIs(self._callFUT(result), _iteratorChecker)
+
+
+class Test_selectCheckerPy(_SelectCheckerBase, unittest.TestCase):
 
     def _callFUT(self, obj):
         from zope.security.checker import selectCheckerPy
@@ -917,7 +993,9 @@ class Test_selectCheckerPy(unittest.TestCase, _SelectCheckerBase):
 
 
 
-class Test_selectChecker(unittest.TestCase, _SelectCheckerBase):
+@unittest.skipIf(checker.selectChecker is checker.selectCheckerPy,
+                 "Pure Python")
+class Test_selectChecker(_SelectCheckerBase, unittest.TestCase):
 
     def _callFUT(self, obj):
         from zope.security.checker import selectChecker
@@ -1040,7 +1118,7 @@ class Test_undefineChecker(unittest.TestCase):
         self.assertFalse(Foo in _checkers)
 
 
-class CombinedCheckerTests(unittest.TestCase):
+class TestCombinedChecker(unittest.TestCase):
 
     def _getTargetClass(self):
         from zope.security.checker import CombinedChecker
@@ -1264,7 +1342,7 @@ class CombinedCheckerTests(unittest.TestCase):
             del thread_local.interaction
 
 
-class CheckerLoggingMixinTests(unittest.TestCase):
+class TestCheckerLoggingMixin(unittest.TestCase):
 
     def _getTargetClass(self):
         from zope.security.checker import CheckerLoggingMixin
@@ -1481,7 +1559,7 @@ class Test_moduleChecker(unittest.TestCase):
         self.assertTrue(self._callFUT(verify) is checker)
 
 
-class BasicTypesTests(unittest.TestCase):
+class TestBasicTypes(unittest.TestCase):
 
     def setUp(self):
         from zope.security.checker import _clear
@@ -1529,7 +1607,7 @@ class BasicTypesTests(unittest.TestCase):
 
 # Pre-geddon tests start here
 
-class Test(unittest.TestCase):
+class TestSecurityPolicy(unittest.TestCase):
 
     def setUp(self):
         from zope.security.management import newInteraction
@@ -2118,7 +2196,7 @@ class TestMixinDecoratedChecker(unittest.TestCase):
         from zope.security.checker import Checker
         return Checker(self.decorationGetMap, self.decorationSetMap)
 
-class TestCombinedChecker(TestMixinDecoratedChecker, unittest.TestCase):
+class TestCombinedCheckerMixin(TestMixinDecoratedChecker, unittest.TestCase):
 
     def setUp(self):
         unittest.TestCase.setUp(self)
@@ -2215,31 +2293,4 @@ class TestBasicTypes(unittest.TestCase):
         self.assertRaises(NotImplementedError, BasicTypes.clear)
 
 def test_suite():
-    return unittest.TestSuite((
-        unittest.makeSuite(Test_ProxyFactory),
-        unittest.makeSuite(Test_canWrite),
-        unittest.makeSuite(Test_canAccess),
-        unittest.makeSuite(Test_canAccess),
-        unittest.makeSuite(CheckerPyTests),
-        unittest.makeSuite(CheckerTests),
-        unittest.makeSuite(TracebackSupplementTests),
-        unittest.makeSuite(GlobalTests),
-        unittest.makeSuite(Test_NamesChecker),
-        unittest.makeSuite(Test_InterfaceChecker),
-        unittest.makeSuite(Test_MultiChecker),
-        unittest.makeSuite(Test_selectCheckerPy),
-        unittest.makeSuite(Test_selectChecker),
-        unittest.makeSuite(Test_getCheckerForInstancesOf),
-        unittest.makeSuite(Test_defineChecker),
-        unittest.makeSuite(Test_undefineChecker),
-        unittest.makeSuite(CombinedCheckerTests),
-        unittest.makeSuite(CheckerLoggingMixinTests),
-        unittest.makeSuite(Test__instanceChecker),
-        unittest.makeSuite(Test_moduleChecker),
-        unittest.makeSuite(BasicTypesTests),
-        # pre-geddon fossils
-        unittest.makeSuite(Test),
-        unittest.makeSuite(TestCheckerPublic),
-        unittest.makeSuite(TestCombinedChecker),
-        unittest.makeSuite(TestBasicTypes),
-    ))
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)