BUG: numpy.ndarray.tofile creates a corrupt file for large arrays in append file mode

Fixes by using the flag `FALLOC_FL_KEEP_SIZE` (=1) in the `fappend` call.

Fixes #8329

2 files changed, 29 insertions, 3 deletions

diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c
index 5499160be..bb46aa94e 100644
--- a/numpy/core/src/multiarray/convert.c
+++ b/numpy/core/src/multiarray/convert.c
@@ -43,10 +43,21 @@ npy_fallocate(npy_intp nbytes, FILE * fp)
     if (nbytes < 16 * 1024 * 1024) {
         return 0;
     }
+
     /* btrfs can take a while to allocate making release worthwhile */
     NPY_BEGIN_ALLOW_THREADS;
-    r = fallocate(fileno(fp), 0, npy_ftell(fp), nbytes);
+    /*
+     * flush in case there might be some unexpected interactions between the
+     * fallocate call and unwritten data in the descriptor
+     */
+    fflush(fp);
+    /*
+     * the flag "1" (=FALLOC_FL_KEEP_SIZE) is needed for the case of files
+     * opened in append mode (issue #8329)
+     */
+    r = fallocate(fileno(fp), 1, npy_ftell(fp), nbytes);
     NPY_END_ALLOW_THREADS;
+
     /*
      * early exit on no space, other errors will also get found during fwrite
      */
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index 7d14186d8..7e3d1b7d0 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -3724,6 +3724,15 @@ class TestIO(object):
             f.seek(d.nbytes)
             d.tofile(f)
             assert_equal(os.path.getsize(self.filename), d.nbytes * 2)
+        # check append mode (gh-8329)
+        open(self.filename, "w").close() # delete file contents
+        with open(self.filename, "ab") as f:
+            d.tofile(f)
+        assert_array_equal(d, np.fromfile(self.filename))
+        with open(self.filename, "ab") as f:
+            d.tofile(f)
+        assert_equal(os.path.getsize(self.filename), d.nbytes * 2)
+
 
     def test_file_position_after_fromfile(self):
         # gh-4118
@@ -3982,7 +3991,10 @@ class TestResize(TestCase):
 
     def test_int_shape(self):
         x = np.eye(3)
-        x.resize(3)
+        if IS_PYPY:
+            x.resize(3, refcheck=False)
+        else:
+            x.resize(3)
         assert_array_equal(x, np.eye(3)[0,:])
 
     def test_none_shape(self):
@@ -4005,7 +4017,10 @@ class TestResize(TestCase):
 
     def test_zeros_appended(self):
         x = np.eye(3)
-        x.resize(2, 3, 3)
+        if IS_PYPY:
+            x.resize(2, 3, 3, refcheck=False)
+        else:
+            x.resize(2, 3, 3)
         assert_array_equal(x[0], np.eye(3))
         assert_array_equal(x[1], np.zeros((3, 3)))