From a9f57a8919f264fa184320d1f5c3ed7a35b7576a Mon Sep 17 00:00:00 2001 From: Dhruva Krishnamurthy Date: Mon, 23 Nov 2020 10:14:00 -0800 Subject: Remove broken support for write in emulated mmap * Emulated mmap based write without pagefault handling is not possible since IO happens outside of call to mmap and data is written to mapped memory * Potential emulation using userfaultfd() might be possible --- src/posix.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/posix.c b/src/posix.c index 9cd96d2c8..debd8d2c8 100644 --- a/src/posix.c +++ b/src/posix.c @@ -240,14 +240,15 @@ int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, off64_t offset { GIT_MMAP_VALIDATE(out, len, prot, flags); - out->data = NULL; - out->len = 0; - - if ((prot & GIT_PROT_WRITE) && ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED)) { - git_error_set(GIT_ERROR_OS, "trying to map shared-writeable"); + /* writes cannot be emulated without handling pagefaults since write happens by + * writing to mapped memory */ + if (prot & GIT_PROT_WRITE) { + git_error_set(GIT_ERROR_OS, "trying to map %s-writeable", + ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED) ? "shared": "private"); return -1; } + out->len = 0; out->data = git__malloc(len); GIT_ERROR_CHECK_ALLOC(out->data); -- cgit v1.2.1 From 4ce8e01a7ada3c75f7407dc9fafd25b4ac38d037 Mon Sep 17 00:00:00 2001 From: Dhruva Krishnamurthy Date: Wed, 17 Jun 2020 14:31:11 -0700 Subject: Support build with NO_MMAP to disable use of system mmap * Use pread/pwrite to avoid updating position in file descriptor * Emulate missing pread/pwrite on win32 using overlapped file IO --- src/indexer.c | 18 +++++++++++++ src/posix.c | 32 +++++++++++++++++++---- src/posix.h | 9 +++++++ src/unix/posix.h | 3 +++ src/win32/posix_w32.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 5 deletions(-) diff --git a/src/indexer.c b/src/indexer.c index 453bb3df7..ba82d716d 100644 --- a/src/indexer.c +++ b/src/indexer.c @@ -603,6 +603,23 @@ static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size) static int write_at(git_indexer *idx, const void *data, off64_t offset, size_t size) { +#ifdef NO_MMAP + size_t remaining_size = size; + const char *ptr = (const char *)data; + + /* Handle data size larger that ssize_t */ + while (remaining_size > 0) { + ssize_t nb; + HANDLE_EINTR(nb, p_pwrite(idx->pack->mwf.fd, (void *)ptr, + remaining_size, offset)); + if (nb <= 0) + return -1; + + ptr += nb; + offset += nb; + remaining_size -= nb; + } +#else git_file fd = idx->pack->mwf.fd; size_t mmap_alignment; size_t page_offset; @@ -627,6 +644,7 @@ static int write_at(git_indexer *idx, const void *data, off64_t offset, size_t s map_data = (unsigned char *)map.data; memcpy(map_data + page_offset, data, size); p_munmap(&map); +#endif return 0; } diff --git a/src/posix.c b/src/posix.c index debd8d2c8..bf764ae6b 100644 --- a/src/posix.c +++ b/src/posix.c @@ -238,6 +238,9 @@ int git__mmap_alignment(size_t *alignment) int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, off64_t offset) { + const char *ptr; + size_t remaining_len; + GIT_MMAP_VALIDATE(out, len, prot, flags); /* writes cannot be emulated without handling pagefaults since write happens by @@ -248,15 +251,30 @@ int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, off64_t offset return -1; } + if (!git__is_ssizet(len)) { + errno = EINVAL; + return -1; + } + out->len = 0; out->data = git__malloc(len); GIT_ERROR_CHECK_ALLOC(out->data); - if (!git__is_ssizet(len) || - (p_lseek(fd, offset, SEEK_SET) < 0) || - (p_read(fd, out->data, len) != (ssize_t)len)) { - git_error_set(GIT_ERROR_OS, "mmap emulation failed"); - return -1; + remaining_len = len; + ptr = (const char *)out->data; + while (remaining_len > 0) { + ssize_t nb; + HANDLE_EINTR(nb, p_pread(fd, (void *)ptr, remaining_len, offset)); + if (nb <= 0) { + git_error_set(GIT_ERROR_OS, "mmap emulation failed"); + git__free(out->data); + out->data = NULL; + return -1; + } + + ptr += nb; + offset += nb; + remaining_len -= nb; } out->len = len; @@ -268,6 +286,10 @@ int p_munmap(git_map *map) GIT_ASSERT_ARG(map); git__free(map->data); + /* Initializing will help debug use-after-free */ + map->len = 0; + map->data = NULL; + return 0; } diff --git a/src/posix.h b/src/posix.h index eef667762..d98bc82ca 100644 --- a/src/posix.h +++ b/src/posix.h @@ -89,6 +89,12 @@ #define EAFNOSUPPORT (INT_MAX-1) #endif +/* Compiler independent macro to handle signal interrpted system calls */ +#define HANDLE_EINTR(result, x) do { \ + result = (x); \ + } while (result == -1 && errno == EINTR); + + /* Provide a 64-bit size for offsets. */ #if defined(_MSC_VER) @@ -119,6 +125,9 @@ typedef int git_file; extern ssize_t p_read(git_file fd, void *buf, size_t cnt); extern int p_write(git_file fd, const void *buf, size_t cnt); +extern ssize_t p_pread(int fd, void *data, size_t size, off64_t offset); +extern ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset); + #define p_close(fd) close(fd) #define p_umask(m) umask(m) diff --git a/src/unix/posix.h b/src/unix/posix.h index b5527a4a8..7b3325e78 100644 --- a/src/unix/posix.h +++ b/src/unix/posix.h @@ -101,4 +101,7 @@ GIT_INLINE(int) p_futimes(int f, const struct p_timeval t[2]) # define p_futimes futimes #endif +#define p_pread(f, d, s, o) pread(f, d, s, o) +#define p_pwrite(f, d, s, o) pwrite(f, d, s, o) + #endif diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c index 5a5e92158..0a8f2bee0 100644 --- a/src/win32/posix_w32.c +++ b/src/win32/posix_w32.c @@ -981,3 +981,73 @@ int p_inet_pton(int af, const char *src, void *dst) errno = EINVAL; return -1; } + +ssize_t p_pread(int fd, void *data, size_t size, off64_t offset) +{ + HANDLE fh; + DWORD rsize = 0; + OVERLAPPED ov = {0}; + LARGE_INTEGER pos = {0}; + off64_t final_offset = 0; + + /* Fail if the final offset would have overflowed to match POSIX semantics. */ + if (!git__is_ssizet(size) || git__add_int64_overflow(&final_offset, offset, (int64_t)size)) { + errno = EINVAL; + return -1; + } + + /* + * Truncate large writes to the maximum allowable size: the caller + * needs to always call this in a loop anyways. + */ + if (size > INT32_MAX) { + size = INT32_MAX; + } + + pos.QuadPart = offset; + ov.Offset = pos.LowPart; + ov.OffsetHigh = pos.HighPart; + fh = (HANDLE)_get_osfhandle(fd); + + if (ReadFile(fh, data, (DWORD)size, &rsize, &ov)) { + return (ssize_t)rsize; + } + + set_errno(); + return -1; +} + +ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset) +{ + HANDLE fh; + DWORD wsize = 0; + OVERLAPPED ov = {0}; + LARGE_INTEGER pos = {0}; + off64_t final_offset = 0; + + /* Fail if the final offset would have overflowed to match POSIX semantics. */ + if (!git__is_ssizet(size) || git__add_int64_overflow(&final_offset, offset, (int64_t)size)) { + errno = EINVAL; + return -1; + } + + /* + * Truncate large writes to the maximum allowable size: the caller + * needs to always call this in a loop anyways. + */ + if (size > INT32_MAX) { + size = INT32_MAX; + } + + pos.QuadPart = offset; + ov.Offset = pos.LowPart; + ov.OffsetHigh = pos.HighPart; + fh = (HANDLE)_get_osfhandle(fd); + + if (WriteFile(fh, data, (DWORD)size, &wsize, &ov)) { + return (ssize_t)wsize; + } + + set_errno(); + return -1; +} -- cgit v1.2.1 From 4fafe9f6a15ccf540d23c96d37ca1c3127a7ddac Mon Sep 17 00:00:00 2001 From: Dhruva Krishnamurthy Date: Sun, 22 Nov 2020 13:23:46 -0800 Subject: Add github action to build and test with mmap emulation --- .github/workflows/nightly.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8e35726d7..fa83491ba 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -96,6 +96,17 @@ jobs: ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10 TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1 os: ubuntu-latest + - # Focal, Clang 10, mmap emulation (NO_MMAP) + container: + name: focal + env: + CC: clang-10 + CFLAGS: -DNO_MMAP + CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local + CMAKE_GENERATOR: Ninja + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true + os: ubuntu-latest - # macOS os: macos-10.15 env: @@ -114,6 +125,15 @@ jobs: CMAKE_OPTIONS: -A x64 -DWIN32_LEAKCHECK=ON -DDEPRECATE_HARD=ON SKIP_SSH_TESTS: true SKIP_NEGOTIATE_TESTS: true + - # Windows amd64 Visual Studio (NO_MMAP) + os: windows-2019 + env: + ARCH: amd64 + CMAKE_GENERATOR: Visual Studio 16 2019 + CFLAGS: -DNO_MMAP + CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON + SKIP_SSH_TESTS: true + SKIP_NEGOTIATE_TESTS: true - # Windows x86 Visual Studio os: windows-2019 env: -- cgit v1.2.1