ENH: always use zip64, upgrade pickle protocol to 3

3 files changed, 19 insertions, 8 deletions

diff --git a/numpy/lib/format.py b/numpy/lib/format.py
index cd8700051..a14f3e4b2 100644
--- a/numpy/lib/format.py
+++ b/numpy/lib/format.py
@@ -650,14 +650,13 @@ def write_array(fp, array, version=None, allow_pickle=True, pickle_kwargs=None):
 
     if array.dtype.hasobject:
         # We contain Python objects so we cannot write out the data
-        # directly.  Instead, we will pickle it out with version 2 of the
-        # pickle protocol.
+        # directly.  Instead, we will pickle it out
         if not allow_pickle:
             raise ValueError("Object arrays cannot be saved when "
                              "allow_pickle=False")
         if pickle_kwargs is None:
             pickle_kwargs = {}
-        pickle.dump(array, fp, protocol=2, **pickle_kwargs)
+        pickle.dump(array, fp, protocol=3, **pickle_kwargs)
     elif array.flags.f_contiguous and not array.flags.c_contiguous:
         if isfileobj(fp):
             array.T.tofile(fp)
diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py
index 1845305d1..e1808f7bd 100644
--- a/numpy/lib/npyio.py
+++ b/numpy/lib/npyio.py
@@ -734,8 +734,8 @@ def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None):
         for key, val in namedict.items():
             fname = key + '.npy'
             val = np.asanyarray(val)
-            force_zip64 = val.nbytes >= 2**30
-            with zipf.open(fname, 'w', force_zip64=force_zip64) as fid:
+            # always force zip64, gh-10776
+            with zipf.open(fname, 'w', force_zip64=True) as fid:
                 format.write_array(fid, val,
                                    allow_pickle=allow_pickle,
                                    pickle_kwargs=pickle_kwargs)
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
index c6193d79f..0151b5118 100644
--- a/numpy/lib/tests/test_io.py
+++ b/numpy/lib/tests/test_io.py
@@ -504,8 +504,6 @@ class TestSaveTxt(object):
              b' (3.142e+00-2.718e+00j)  (3.142e+00-2.718e+00j)\n'])
 
 
-        
-
     def test_custom_writer(self):
 
         class CustomWriter(list):
@@ -574,6 +572,20 @@ class TestSaveTxt(object):
         else:
             assert_equal(s.read(), b"%f\n" % 1.)
 
+    @pytest.mark.skipif(sys.platform=='win32',
+                        reason="large files cause problems")
+    @pytest.mark.slow
+    def test_large_zip(self):
+        # The test takes at least 6GB of memory, writes a file larger than 4GB
+        try:
+            a = 'a' * 6 * 1024 * 1024 * 1024
+            del a
+        except (MemoryError, OverflowError):
+            pytest.skip("Cannot allocate enough memory for test")
+        test_data = np.asarray([np.random.rand(np.random.randint(50,100),4)
+                               for i in range(800000)])
+        with tempdir() as tmpdir:
+            np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data)
 
 class LoadTxtBase(object):
     def check_compressed(self, fopen, suffixes):
@@ -2378,7 +2390,7 @@ class TestPathUsage(object):
             np.savez(path, lab='place holder')
             with np.load(path) as data:
                 assert_array_equal(data['lab'], 'place holder')
-    
+
     def test_savez_compressed_load(self):
         # Test that pathlib.Path instances can be used with savez.
         with temppath(suffix='.npz') as path: