From 6c0d5b11c05254d91a2d0025d556251404f8f05f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:19 +0100 Subject: util: make monotonic time fn return ms `git__timer` is now `git_time_monotonic`, and returns milliseconds since an arbitrary epoch. Using a floating point to store the number of seconds elapsed was clever, as it better supports the wide range of precision from the different monotonic clocks of different systems. But we're a version control system, not a real-time clock. Milliseconds is a good enough precision for our work _and_ it's the units that system calls like `poll` take and that our users interact with. Make `git_time_monotonic` return the monotonically increasing number of milliseconds "ticked" since some arbitrary epoch. --- src/cli/progress.c | 17 +++++++------- src/cli/progress.h | 8 +++---- src/libgit2/pack-objects.c | 12 +++++----- src/libgit2/pack-objects.h | 4 +++- src/libgit2/transports/smart_protocol.c | 8 +++---- src/util/rand.c | 10 +++++---- src/util/util.h | 40 ++++++++++++++++++++------------- tests/clar/clar_libgit2_timer.c | 8 +++---- tests/clar/clar_libgit2_timer.h | 10 ++++----- tests/clar/clar_libgit2_trace.c | 2 +- 10 files changed, 66 insertions(+), 53 deletions(-) diff --git a/src/cli/progress.c b/src/cli/progress.c index ba52655e7..ddfbafb73 100644 --- a/src/cli/progress.c +++ b/src/cli/progress.c @@ -15,10 +15,10 @@ /* * Show updates to the percentage and number of objects received * separately from the throughput to give an accurate progress while - * avoiding too much noise on the screen. + * avoiding too much noise on the screen. (In milliseconds.) */ -#define PROGRESS_UPDATE_TIME 0.10 -#define THROUGHPUT_UPDATE_TIME 1.00 +#define PROGRESS_UPDATE_TIME 60 +#define THROUGHPUT_UPDATE_TIME 500 #define is_nl(c) ((c) == '\r' || (c) == '\n') @@ -54,7 +54,7 @@ static int progress_write(cli_progress *progress, bool force, git_str *line) bool has_nl; size_t no_nl = no_nl_len(line->ptr, line->size); size_t nl = nl_len(&has_nl, line->ptr + no_nl, line->size - no_nl); - double now = git__timer(); + uint64_t now = git_time_monotonic(); size_t i; /* Avoid spamming the console with progress updates */ @@ -191,20 +191,21 @@ static int fetch_receiving( { char *recv_units[] = { "B", "KiB", "MiB", "GiB", "TiB", NULL }; char *rate_units[] = { "B/s", "KiB/s", "MiB/s", "GiB/s", "TiB/s", NULL }; + uint64_t now, elapsed; - double now, recv_len, rate, elapsed; + double recv_len, rate; size_t recv_unit_idx = 0, rate_unit_idx = 0; bool done = (stats->received_objects == stats->total_objects); if (!progress->action_start) - progress->action_start = git__timer(); + progress->action_start = git_time_monotonic(); if (done && progress->action_finish) now = progress->action_finish; else if (done) - progress->action_finish = now = git__timer(); + progress->action_finish = now = git_time_monotonic(); else - now = git__timer(); + now = git_time_monotonic(); if (progress->throughput_update && now - progress->throughput_update < THROUGHPUT_UPDATE_TIME) { diff --git a/src/cli/progress.h b/src/cli/progress.h index 7a445ec29..886fef89d 100644 --- a/src/cli/progress.h +++ b/src/cli/progress.h @@ -30,11 +30,11 @@ typedef struct { cli_progress_t action; /* Actions may time themselves (eg fetch) but are not required to */ - double action_start; - double action_finish; + uint64_t action_start; + uint64_t action_finish; /* Last console update, avoid too frequent updates. */ - double last_update; + uint64_t last_update; /* Accumulators for partial output and deferred updates. */ git_str sideband; @@ -42,7 +42,7 @@ typedef struct { git_str deferred; /* Last update about throughput */ - double throughput_update; + uint64_t throughput_update; double throughput_bytes; } cli_progress; diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c index d6fd60326..fc8efc65f 100644 --- a/src/libgit2/pack-objects.c +++ b/src/libgit2/pack-objects.c @@ -255,10 +255,10 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid, pb->done = false; if (pb->progress_cb) { - double current_time = git__timer(); - double elapsed = current_time - pb->last_progress_report_time; + uint64_t current_time = git_time_monotonic(); + uint64_t elapsed = current_time - pb->last_progress_report_time; - if (elapsed < 0 || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) { + if (elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) { pb->last_progress_report_time = current_time; ret = pb->progress_cb( @@ -934,10 +934,10 @@ static int report_delta_progress( int ret; if (pb->progress_cb) { - double current_time = git__timer(); - double elapsed = current_time - pb->last_progress_report_time; + uint64_t current_time = git_time_monotonic(); + uint64_t elapsed = current_time - pb->last_progress_report_time; - if (force || elapsed < 0 || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) { + if (force || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) { pb->last_progress_report_time = current_time; ret = pb->progress_cb( diff --git a/src/libgit2/pack-objects.h b/src/libgit2/pack-objects.h index c6bc52fdc..0ceebc23c 100644 --- a/src/libgit2/pack-objects.h +++ b/src/libgit2/pack-objects.h @@ -96,7 +96,9 @@ struct git_packbuilder { git_packbuilder_progress progress_cb; void *progress_cb_payload; - double last_progress_report_time; /* the time progress was last reported */ + + /* the time progress was last reported, in millisecond ticks */ + uint64_t last_progress_report_time; bool done; }; diff --git a/src/libgit2/transports/smart_protocol.c b/src/libgit2/transports/smart_protocol.c index 488ef07c0..f7a567829 100644 --- a/src/libgit2/transports/smart_protocol.c +++ b/src/libgit2/transports/smart_protocol.c @@ -1114,7 +1114,7 @@ struct push_packbuilder_payload git_push_transfer_progress_cb cb; void *cb_payload; size_t last_bytes; - double last_progress_report_time; + uint64_t last_progress_report_time; }; static int stream_thunk(void *buf, size_t size, void *data) @@ -1126,11 +1126,11 @@ static int stream_thunk(void *buf, size_t size, void *data) return error; if (payload->cb) { - double current_time = git__timer(); - double elapsed = current_time - payload->last_progress_report_time; + uint64_t current_time = git_time_monotonic(); + uint64_t elapsed = current_time - payload->last_progress_report_time; payload->last_bytes += size; - if (elapsed < 0 || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) { + if (elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) { payload->last_progress_report_time = current_time; error = payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload); } diff --git a/src/util/rand.c b/src/util/rand.c index 940faf947..2ed060573 100644 --- a/src/util/rand.c +++ b/src/util/rand.c @@ -32,7 +32,6 @@ GIT_INLINE(int) getseed(uint64_t *seed) HCRYPTPROV provider; SYSTEMTIME systemtime; FILETIME filetime, idletime, kerneltime, usertime; - bits convert; if (CryptAcquireContext(&provider, 0, 0, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT|CRYPT_SILENT)) { @@ -67,7 +66,7 @@ GIT_INLINE(int) getseed(uint64_t *seed) *seed ^= ((uint64_t)GetCurrentProcessId() << 32); *seed ^= ((uint64_t)GetCurrentThreadId() << 48); - convert.f = git__timer(); *seed ^= (convert.d); + *seed ^= git_time_monotonic(); /* Mix in the addresses of some functions and variables */ *seed ^= (((uint64_t)((uintptr_t)seed) << 32)); @@ -82,9 +81,12 @@ GIT_INLINE(int) getseed(uint64_t *seed) { struct timeval tv; double loadavg[3]; - bits convert; int fd; +# if defined(GIT_RAND_GETLOADAVG) + bits convert; +# endif + # if defined(GIT_RAND_GETENTROPY) GIT_UNUSED((fd = 0)); @@ -131,7 +133,7 @@ GIT_INLINE(int) getseed(uint64_t *seed) GIT_UNUSED(loadavg[0]); # endif - convert.f = git__timer(); *seed ^= (convert.d); + *seed ^= git_time_monotonic(); /* Mix in the addresses of some variables */ *seed ^= ((uint64_t)((size_t)((void *)seed)) << 32); diff --git a/src/util/util.h b/src/util/util.h index 63d6080f7..7f178b169 100644 --- a/src/util/util.h +++ b/src/util/util.h @@ -319,59 +319,67 @@ GIT_INLINE(void) git__memzero(void *data, size_t size) #ifdef GIT_WIN32 -GIT_INLINE(double) git__timer(void) +GIT_INLINE(uint64_t) git_time_monotonic(void) { /* GetTickCount64 returns the number of milliseconds that have * elapsed since the system was started. */ - return (double) GetTickCount64() / (double) 1000; + return GetTickCount64(); } #elif __APPLE__ #include +#include -GIT_INLINE(double) git__timer(void) +GIT_INLINE(uint64_t) git_time_monotonic(void) { - uint64_t time = mach_absolute_time(); - static double scaling_factor = 0; + static double scaling_factor = 0; + + if (scaling_factor == 0) { + mach_timebase_info_data_t info; - if (scaling_factor == 0) { - mach_timebase_info_data_t info; - (void)mach_timebase_info(&info); - scaling_factor = (double)info.numer / (double)info.denom; - } + scaling_factor = mach_timebase_info(&info) == KERN_SUCCESS ? + ((double)info.numer / (double)info.denom) / 1.0E6 : + -1; + } else if (scaling_factor < 0) { + struct timeval tv; + + /* mach_timebase_info failed; fall back to gettimeofday */ + gettimeofday(&tv, NULL); + return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); + } - return (double)time * scaling_factor / 1.0E9; + return (uint64_t)(mach_absolute_time() * scaling_factor); } #elif defined(__amigaos4__) #include -GIT_INLINE(double) git__timer(void) +GIT_INLINE(uint64_t) git_time_monotonic(void) { struct TimeVal tv; ITimer->GetUpTime(&tv); - return (double)tv.Seconds + (double)tv.Microseconds / 1.0E6; + return (tv.Seconds * 1000) + (tv.Microseconds / 1000); } #else #include -GIT_INLINE(double) git__timer(void) +GIT_INLINE(uint64_t) git_time_monotonic(void) { struct timeval tv; #ifdef CLOCK_MONOTONIC struct timespec tp; if (clock_gettime(CLOCK_MONOTONIC, &tp) == 0) - return (double) tp.tv_sec + (double) tp.tv_nsec / 1.0E9; + return (tp.tv_sec * 1000) + (tp.tv_nsec / 1.0E6); #endif /* Fall back to using gettimeofday */ gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1.0E6; + return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); } #endif diff --git a/tests/clar/clar_libgit2_timer.c b/tests/clar/clar_libgit2_timer.c index 2330f9351..6c75413be 100644 --- a/tests/clar/clar_libgit2_timer.c +++ b/tests/clar/clar_libgit2_timer.c @@ -8,23 +8,23 @@ void cl_perf_timer__init(cl_perf_timer *t) void cl_perf_timer__start(cl_perf_timer *t) { - t->time_started = git__timer(); + t->time_started = git_time_monotonic(); } void cl_perf_timer__stop(cl_perf_timer *t) { - double time_now = git__timer(); + uint64_t time_now = git_time_monotonic(); t->last = time_now - t->time_started; t->sum += t->last; } -double cl_perf_timer__last(const cl_perf_timer *t) +uint64_t cl_perf_timer__last(const cl_perf_timer *t) { return t->last; } -double cl_perf_timer__sum(const cl_perf_timer *t) +uint64_t cl_perf_timer__sum(const cl_perf_timer *t) { return t->sum; } diff --git a/tests/clar/clar_libgit2_timer.h b/tests/clar/clar_libgit2_timer.h index 7571a52e9..887067278 100644 --- a/tests/clar/clar_libgit2_timer.h +++ b/tests/clar/clar_libgit2_timer.h @@ -4,13 +4,13 @@ struct cl_perf_timer { /* cumulative running time across all start..stop intervals */ - double sum; + uint64_t sum; /* value of last start..stop interval */ - double last; + uint64_t last; /* clock value at start */ - double time_started; + uint64_t time_started; }; #define CL_PERF_TIMER_INIT {0} @@ -24,12 +24,12 @@ void cl_perf_timer__stop(cl_perf_timer *t); /** * return value of last start..stop interval in seconds. */ -double cl_perf_timer__last(const cl_perf_timer *t); +uint64_t cl_perf_timer__last(const cl_perf_timer *t); /** * return cumulative running time across all start..stop * intervals in seconds. */ -double cl_perf_timer__sum(const cl_perf_timer *t); +uint64_t cl_perf_timer__sum(const cl_perf_timer *t); #endif /* __CLAR_LIBGIT2_TIMER__ */ diff --git a/tests/clar/clar_libgit2_trace.c b/tests/clar/clar_libgit2_trace.c index ebb0f41dd..814a5fa9e 100644 --- a/tests/clar/clar_libgit2_trace.c +++ b/tests/clar/clar_libgit2_trace.c @@ -197,7 +197,7 @@ static void _cl_trace_cb__event_handler( case CL_TRACE__TEST__END: cl_perf_timer__stop(&s_timer_test); - git_trace(GIT_TRACE_TRACE, "%s::%s: End Test (%.3f %.3f)", suite_name, test_name, + git_trace(GIT_TRACE_TRACE, "%s::%s: End Test (%" PRIuZ " %" PRIuZ ")", suite_name, test_name, cl_perf_timer__last(&s_timer_run), cl_perf_timer__last(&s_timer_test)); break; -- cgit v1.2.1 From dbe343b6e3e957b5cffbd04832c6e7364b496ae7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:28 +0100 Subject: stransport: store error information We lose some error information from the read / write callbacks to stransport. Store our own error value in the object so that we can ensure that we rely upon it. --- src/libgit2/streams/stransport.c | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/libgit2/streams/stransport.c b/src/libgit2/streams/stransport.c index 3f31d2541..74ee0d1ee 100644 --- a/src/libgit2/streams/stransport.c +++ b/src/libgit2/streams/stransport.c @@ -44,6 +44,7 @@ typedef struct { git_stream parent; git_stream *io; int owned; + int error; SSLContextRef ctx; CFDataRef der_data; git_cert_x509 cert_info; @@ -61,7 +62,10 @@ static int stransport_connect(git_stream *stream) return error; ret = SSLHandshake(st->ctx); - if (ret != errSSLServerAuthCompleted) { + + if (ret != errSSLServerAuthCompleted && st->error != 0) + return -1; + else if (ret != errSSLServerAuthCompleted) { git_error_set(GIT_ERROR_SSL, "unexpected return value from ssl handshake %d", (int)ret); return -1; } @@ -147,10 +151,18 @@ static int stransport_set_proxy( */ static OSStatus write_cb(SSLConnectionRef conn, const void *data, size_t *len) { - git_stream *io = (git_stream *) conn; + stransport_stream *st = (stransport_stream *)conn; + git_stream *io = st->io; + OSStatus ret; + + st->error = 0; + + ret = git_stream__write_full(io, data, *len, 0); - if (git_stream__write_full(io, data, *len, 0) < 0) - return -36; /* "ioErr" from MacErrors.h which is not available on iOS */ + if (ret < 0) { + st->error = ret; + return -36; /* ioErr */ + } return noErr; } @@ -182,18 +194,22 @@ static ssize_t stransport_write(git_stream *stream, const char *data, size_t len */ static OSStatus read_cb(SSLConnectionRef conn, void *data, size_t *len) { - git_stream *io = (git_stream *) conn; + stransport_stream *st = (stransport_stream *)conn; + git_stream *io = st->io; OSStatus error = noErr; size_t off = 0; ssize_t ret; + st->error = 0; + do { ret = git_stream_read(io, data + off, *len - off); + if (ret < 0) { - error = -36; /* "ioErr" from MacErrors.h which is not available on iOS */ + st->error = ret; + error = -36; /* ioErr */ break; - } - if (ret == 0) { + } else if (ret == 0) { error = errSSLClosedGraceful; break; } @@ -207,12 +223,13 @@ static OSStatus read_cb(SSLConnectionRef conn, void *data, size_t *len) static ssize_t stransport_read(git_stream *stream, void *data, size_t len) { - stransport_stream *st = (stransport_stream *) stream; + stransport_stream *st = (stransport_stream *)stream; size_t processed; OSStatus ret; - if ((ret = SSLRead(st->ctx, data, len, &processed)) != noErr) + if ((ret = SSLRead(st->ctx, data, len, &processed)) != noErr) { return stransport_error(ret); + } return processed; } @@ -269,7 +286,7 @@ static int stransport_wrap( } if ((ret = SSLSetIOFuncs(st->ctx, read_cb, write_cb)) != noErr || - (ret = SSLSetConnection(st->ctx, st->io)) != noErr || + (ret = SSLSetConnection(st->ctx, st)) != noErr || (ret = SSLSetSessionOption(st->ctx, kSSLSessionOptionBreakOnServerAuth, true)) != noErr || (ret = SSLSetProtocolVersionMin(st->ctx, kTLSProtocol1)) != noErr || (ret = SSLSetProtocolVersionMax(st->ctx, kTLSProtocol12)) != noErr || -- cgit v1.2.1 From 6e4bbf222d8c4babaff90aef40615546c8bc9cde Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:30 +0100 Subject: net: move rfc2818 hostname / wildcard matching to util --- src/libgit2/netops.c | 39 ---------------------------------- src/libgit2/netops.h | 13 ------------ src/libgit2/streams/openssl.c | 19 ++++++----------- src/util/net.c | 44 +++++++++++++++++++++++++++++++++++++++ src/util/net.h | 17 +++++++++++++++ tests/libgit2/network/matchhost.c | 13 ------------ tests/util/hostname.c | 13 ++++++++++++ 7 files changed, 80 insertions(+), 78 deletions(-) delete mode 100644 tests/libgit2/network/matchhost.c create mode 100644 tests/util/hostname.c diff --git a/src/libgit2/netops.c b/src/libgit2/netops.c index 00640c600..5cae374ad 100644 --- a/src/libgit2/netops.c +++ b/src/libgit2/netops.c @@ -83,42 +83,3 @@ void gitno_consume_n(gitno_buffer *buf, size_t cons) memset(buf->data + cons, 0x0, buf->len - buf->offset); buf->offset -= cons; } - -/* Match host names according to RFC 2818 rules */ -int gitno__match_host(const char *pattern, const char *host) -{ - for (;;) { - char c = git__tolower(*pattern++); - - if (c == '\0') - return *host ? -1 : 0; - - if (c == '*') { - c = *pattern; - /* '*' at the end matches everything left */ - if (c == '\0') - return 0; - - /* - * We've found a pattern, so move towards the next matching - * char. The '.' is handled specially because wildcards aren't - * allowed to cross subdomains. - */ - - while(*host) { - char h = git__tolower(*host); - if (c == h) - return gitno__match_host(pattern, host++); - if (h == '.') - return gitno__match_host(pattern, host); - host++; - } - return -1; - } - - if (c != git__tolower(*host++)) - return -1; - } - - return -1; -} diff --git a/src/libgit2/netops.h b/src/libgit2/netops.h index 56f968534..a3f4a0f95 100644 --- a/src/libgit2/netops.h +++ b/src/libgit2/netops.h @@ -45,19 +45,6 @@ enum { GITNO_CONNECT_SSL = 1 }; -/** - * Check if the name in a cert matches the wanted hostname - * - * Check if a pattern from a certificate matches the hostname we - * wanted to connect to according to RFC2818 rules (which specifies - * HTTP over TLS). Mainly, an asterisk matches anything, but is - * limited to a single url component. - * - * Note that this does not set an error message. It expects the user - * to provide the message for the user. - */ -int gitno__match_host(const char *pattern, const char *host); - void gitno_buffer_setup_fromstream(git_stream *st, gitno_buffer *buf, char *data, size_t len); void gitno_buffer_setup_callback(gitno_buffer *buf, char *data, size_t len, int (*recv)(gitno_buffer *buf), void *cb_data); int gitno_recv(gitno_buffer *buf); diff --git a/src/libgit2/streams/openssl.c b/src/libgit2/streams/openssl.c index 5e0e2c939..58b2d1b23 100644 --- a/src/libgit2/streams/openssl.c +++ b/src/libgit2/streams/openssl.c @@ -18,6 +18,7 @@ #include "settings.h" #include "posix.h" #include "stream.h" +#include "net.h" #include "streams/socket.h" #include "netops.h" #include "git2/transport.h" @@ -357,15 +358,10 @@ static int ssl_teardown(SSL *ssl) return ret; } -static int check_host_name(const char *name, const char *host) +static bool check_host_name(const char *host, const char *name) { - if (!strcasecmp(name, host)) - return 0; - - if (gitno__match_host(name, host) < 0) - return -1; - - return 0; + return !strcasecmp(host, name) || + git_net_hostname_matches_cert(host, name); } static int verify_server_cert(SSL *ssl, const char *host) @@ -425,10 +421,7 @@ static int verify_server_cert(SSL *ssl, const char *host) if (memchr(name, '\0', namelen)) continue; - if (check_host_name(name, host) < 0) - matched = 0; - else - matched = 1; + matched = !!check_host_name(host, name); } else if (type == GEN_IPADD) { /* Here name isn't so much a name but a binary representation of the IP */ matched = addr && !!memcmp(name, addr, namelen); @@ -481,7 +474,7 @@ static int verify_server_cert(SSL *ssl, const char *host) goto cert_fail_name; } - if (check_host_name((char *)peer_cn, host) < 0) + if (!check_host_name(host, (char *)peer_cn)) goto cert_fail_name; goto cleanup; diff --git a/src/util/net.c b/src/util/net.c index ac7befe07..dd8a1ba46 100644 --- a/src/util/net.c +++ b/src/util/net.c @@ -19,6 +19,50 @@ #define DEFAULT_PORT_GIT "9418" #define DEFAULT_PORT_SSH "22" +bool git_net_hostname_matches_cert( + const char *hostname, + const char *pattern) +{ + for (;;) { + char c = git__tolower(*pattern++); + + if (c == '\0') + return *hostname ? false : true; + + if (c == '*') { + c = *pattern; + + /* '*' at the end matches everything left */ + if (c == '\0') + return true; + + /* + * We've found a pattern, so move towards the + * next matching char. The '.' is handled + * specially because wildcards aren't allowed + * to cross subdomains. + */ + while(*hostname) { + char h = git__tolower(*hostname); + + if (h == c) + return git_net_hostname_matches_cert(hostname++, pattern); + else if (h == '.') + return git_net_hostname_matches_cert(hostname, pattern); + + hostname++; + } + + return false; + } + + if (c != git__tolower(*hostname++)) + return false; + } + + return false; +} + bool git_net_str_is_url(const char *str) { const char *c; diff --git a/src/util/net.h b/src/util/net.h index 17f0bc4f0..c9a84cb6c 100644 --- a/src/util/net.h +++ b/src/util/net.h @@ -9,6 +9,23 @@ #include "git2_util.h" +/* + * Hostname handling + */ + +/* + * See if a given hostname matches a certificate name pattern, according + * to RFC2818 rules (which specifies HTTP over TLS). Mainly, an asterisk + * matches anything, but is limited to a single url component. + */ +extern bool git_net_hostname_matches_cert( + const char *hostname, + const char *pattern); + +/* + * URL handling + */ + typedef struct git_net_url { char *scheme; char *host; diff --git a/tests/libgit2/network/matchhost.c b/tests/libgit2/network/matchhost.c deleted file mode 100644 index 3100dc21d..000000000 --- a/tests/libgit2/network/matchhost.c +++ /dev/null @@ -1,13 +0,0 @@ -#include "clar_libgit2.h" -#include "netops.h" - -void test_network_matchhost__match(void) -{ - cl_git_pass(gitno__match_host("*.example.org", "www.example.org")); - cl_git_pass(gitno__match_host("*.foo.example.org", "www.foo.example.org")); - cl_git_fail(gitno__match_host("*.foo.example.org", "foo.example.org")); - cl_git_fail(gitno__match_host("*.foo.example.org", "www.example.org")); - cl_git_fail(gitno__match_host("*.example.org", "example.org")); - cl_git_fail(gitno__match_host("*.example.org", "www.foo.example.org")); - cl_git_fail(gitno__match_host("*.example.org", "blah.www.www.example.org")); -} diff --git a/tests/util/hostname.c b/tests/util/hostname.c new file mode 100644 index 000000000..5d8bfe2ac --- /dev/null +++ b/tests/util/hostname.c @@ -0,0 +1,13 @@ +#include "clar_libgit2.h" +#include "net.h" + +void test_hostname__matches_cert(void) +{ + cl_assert_equal_b(true, git_net_hostname_matches_cert("www.example.org", "*.example.org")); + cl_assert_equal_b(true, git_net_hostname_matches_cert("www.foo.example.org", "*.foo.example.org")); + cl_assert_equal_b(false, git_net_hostname_matches_cert("foo.example.org", "*.foo.example.org")); + cl_assert_equal_b(false, git_net_hostname_matches_cert("www.example.org", "*.foo.example.org")); + cl_assert_equal_b(false, git_net_hostname_matches_cert("example.org", "*.example.org")); + cl_assert_equal_b(false, git_net_hostname_matches_cert("www.foo.example.org", "*.example.org")); + cl_assert_equal_b(false, git_net_hostname_matches_cert("blah.www.www.example.org", "*.example.org")); +} -- cgit v1.2.1 From 11c89c386c8862f834097cdc5d9ed79634cf00f3 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:30 +0100 Subject: net: move `gitno` buffer to `staticstr` The `gitno` buffer interface is another layer on top of socket reads. Abstract it a bit into a "static string" that has `git_str` like semantics but without heap allocation which moves the actual reading logic into the socket / stream code, and allows for easier future usage of a static / stack-allocated `git_str`-like interface. --- src/libgit2/fetch.c | 1 - src/libgit2/fetch.h | 2 - src/libgit2/netops.c | 85 --------------------------------- src/libgit2/netops.h | 55 --------------------- src/libgit2/pack-objects.c | 1 - src/libgit2/pack-objects.h | 1 - src/libgit2/streams/mbedtls.c | 1 - src/libgit2/streams/openssl.c | 5 +- src/libgit2/streams/socket.c | 1 - src/libgit2/streams/socket.h | 2 +- src/libgit2/transports/auth.h | 3 +- src/libgit2/transports/git.c | 15 +++--- src/libgit2/transports/http.c | 1 - src/libgit2/transports/smart.c | 57 +++++++++++----------- src/libgit2/transports/smart.h | 9 ++-- src/libgit2/transports/smart_pkt.c | 1 - src/libgit2/transports/smart_protocol.c | 78 +++++++++++++++--------------- src/libgit2/transports/ssh.c | 1 - src/libgit2/transports/winhttp.c | 1 - src/util/staticstr.h | 66 +++++++++++++++++++++++++ tests/util/url/joinpath.c | 1 - tests/util/url/redirect.c | 1 - 22 files changed, 151 insertions(+), 237 deletions(-) delete mode 100644 src/libgit2/netops.c delete mode 100644 src/libgit2/netops.h create mode 100644 src/util/staticstr.h diff --git a/src/libgit2/fetch.c b/src/libgit2/fetch.c index 5bbef87f4..d74abb4a8 100644 --- a/src/libgit2/fetch.c +++ b/src/libgit2/fetch.c @@ -17,7 +17,6 @@ #include "remote.h" #include "refspec.h" #include "pack.h" -#include "netops.h" #include "repository.h" #include "refs.h" #include "transports/smart.h" diff --git a/src/libgit2/fetch.h b/src/libgit2/fetch.h index 10b6731f0..493366ded 100644 --- a/src/libgit2/fetch.h +++ b/src/libgit2/fetch.h @@ -11,8 +11,6 @@ #include "git2/remote.h" -#include "netops.h" - int git_fetch_negotiate(git_remote *remote, const git_fetch_options *opts); int git_fetch_download_pack(git_remote *remote); diff --git a/src/libgit2/netops.c b/src/libgit2/netops.c deleted file mode 100644 index 5cae374ad..000000000 --- a/src/libgit2/netops.c +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) the libgit2 contributors. All rights reserved. - * - * This file is part of libgit2, distributed under the GNU GPL v2 with - * a Linking Exception. For full terms see the included COPYING file. - */ - -#include "netops.h" - -#include -#include "git2/errors.h" - -#include "posix.h" -#include "str.h" -#include "runtime.h" - -int gitno_recv(gitno_buffer *buf) -{ - return buf->recv(buf); -} - -void gitno_buffer_setup_callback( - gitno_buffer *buf, - char *data, - size_t len, - int (*recv)(gitno_buffer *buf), void *cb_data) -{ - memset(data, 0x0, len); - buf->data = data; - buf->len = len; - buf->offset = 0; - buf->recv = recv; - buf->cb_data = cb_data; -} - -static int recv_stream(gitno_buffer *buf) -{ - git_stream *io = (git_stream *) buf->cb_data; - size_t readlen = buf->len - buf->offset; - ssize_t ret; - - readlen = min(readlen, INT_MAX); - - ret = git_stream_read(io, buf->data + buf->offset, (int)readlen); - if (ret < 0) - return -1; - - buf->offset += ret; - return (int)ret; -} - -void gitno_buffer_setup_fromstream(git_stream *st, gitno_buffer *buf, char *data, size_t len) -{ - memset(data, 0x0, len); - buf->data = data; - buf->len = len; - buf->offset = 0; - buf->recv = recv_stream; - buf->cb_data = st; -} - -/* Consume up to ptr and move the rest of the buffer to the beginning */ -int gitno_consume(gitno_buffer *buf, const char *ptr) -{ - size_t consumed; - - GIT_ASSERT(ptr - buf->data >= 0); - GIT_ASSERT(ptr - buf->data <= (int) buf->len); - - consumed = ptr - buf->data; - - memmove(buf->data, ptr, buf->offset - consumed); - memset(buf->data + buf->offset, 0x0, buf->len - buf->offset); - buf->offset -= consumed; - - return 0; -} - -/* Consume const bytes and move the rest of the buffer to the beginning */ -void gitno_consume_n(gitno_buffer *buf, size_t cons) -{ - memmove(buf->data, buf->data + cons, buf->len - buf->offset); - memset(buf->data + cons, 0x0, buf->len - buf->offset); - buf->offset -= cons; -} diff --git a/src/libgit2/netops.h b/src/libgit2/netops.h deleted file mode 100644 index a3f4a0f95..000000000 --- a/src/libgit2/netops.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) the libgit2 contributors. All rights reserved. - * - * This file is part of libgit2, distributed under the GNU GPL v2 with - * a Linking Exception. For full terms see the included COPYING file. - */ -#ifndef INCLUDE_netops_h__ -#define INCLUDE_netops_h__ - -#include "common.h" - -#include "posix.h" -#include "stream.h" -#include "net.h" - -#ifdef GIT_OPENSSL -# include "streams/openssl.h" -#endif - -typedef struct gitno_ssl { -#ifdef GIT_OPENSSL - SSL *ssl; -#else - size_t dummy; -#endif -} gitno_ssl; - -/* Represents a socket that may or may not be using SSL */ -typedef struct gitno_socket { - GIT_SOCKET socket; - gitno_ssl ssl; -} gitno_socket; - -typedef struct gitno_buffer { - char *data; - size_t len; - size_t offset; - int (*recv)(struct gitno_buffer *buffer); - void *cb_data; -} gitno_buffer; - -/* Flags to gitno_connect */ -enum { - /* Attempt to create an SSL connection. */ - GITNO_CONNECT_SSL = 1 -}; - -void gitno_buffer_setup_fromstream(git_stream *st, gitno_buffer *buf, char *data, size_t len); -void gitno_buffer_setup_callback(gitno_buffer *buf, char *data, size_t len, int (*recv)(gitno_buffer *buf), void *cb_data); -int gitno_recv(gitno_buffer *buf); - -int gitno_consume(gitno_buffer *buf, const char *ptr); -void gitno_consume_n(gitno_buffer *buf, size_t cons); - -#endif diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c index fc8efc65f..b2d80cba9 100644 --- a/src/libgit2/pack-objects.c +++ b/src/libgit2/pack-objects.c @@ -11,7 +11,6 @@ #include "zstream.h" #include "delta.h" #include "iterator.h" -#include "netops.h" #include "pack.h" #include "thread.h" #include "tree.h" diff --git a/src/libgit2/pack-objects.h b/src/libgit2/pack-objects.h index 0ceebc23c..bbc8b9430 100644 --- a/src/libgit2/pack-objects.h +++ b/src/libgit2/pack-objects.h @@ -13,7 +13,6 @@ #include "str.h" #include "hash.h" #include "oidmap.h" -#include "netops.h" #include "zstream.h" #include "pool.h" #include "indexer.h" diff --git a/src/libgit2/streams/mbedtls.c b/src/libgit2/streams/mbedtls.c index 0cf5c8af1..49aa76c3e 100644 --- a/src/libgit2/streams/mbedtls.c +++ b/src/libgit2/streams/mbedtls.c @@ -14,7 +14,6 @@ #include "runtime.h" #include "stream.h" #include "streams/socket.h" -#include "netops.h" #include "git2/transport.h" #include "util.h" diff --git a/src/libgit2/streams/openssl.c b/src/libgit2/streams/openssl.c index 58b2d1b23..9db911e39 100644 --- a/src/libgit2/streams/openssl.c +++ b/src/libgit2/streams/openssl.c @@ -20,7 +20,6 @@ #include "stream.h" #include "net.h" #include "streams/socket.h" -#include "netops.h" #include "git2/transport.h" #include "git2/sys/openssl.h" @@ -71,14 +70,14 @@ static void *git_openssl_malloc(size_t bytes, const char *file, int line) GIT_UNUSED(line); return git__calloc(1, bytes); } - + static void *git_openssl_realloc(void *mem, size_t size, const char *file, int line) { GIT_UNUSED(file); GIT_UNUSED(line); return git__realloc(mem, size); } - + static void git_openssl_free(void *mem, const char *file, int line) { GIT_UNUSED(file); diff --git a/src/libgit2/streams/socket.c b/src/libgit2/streams/socket.c index 8f23e746e..6994d58f2 100644 --- a/src/libgit2/streams/socket.c +++ b/src/libgit2/streams/socket.c @@ -8,7 +8,6 @@ #include "streams/socket.h" #include "posix.h" -#include "netops.h" #include "registry.h" #include "runtime.h" #include "stream.h" diff --git a/src/libgit2/streams/socket.h b/src/libgit2/streams/socket.h index 300e70893..73e8de099 100644 --- a/src/libgit2/streams/socket.h +++ b/src/libgit2/streams/socket.h @@ -9,7 +9,7 @@ #include "common.h" -#include "netops.h" +#include "stream.h" typedef struct { git_stream parent; diff --git a/src/libgit2/transports/auth.h b/src/libgit2/transports/auth.h index 64680cc53..9f6f8fd3b 100644 --- a/src/libgit2/transports/auth.h +++ b/src/libgit2/transports/auth.h @@ -9,8 +9,7 @@ #define INCLUDE_transports_auth_h__ #include "common.h" - -#include "netops.h" +#include "net.h" typedef enum { GIT_HTTP_AUTH_BASIC = 1, diff --git a/src/libgit2/transports/git.c b/src/libgit2/transports/git.c index 591e2ab03..53611f2a7 100644 --- a/src/libgit2/transports/git.c +++ b/src/libgit2/transports/git.c @@ -7,7 +7,7 @@ #include "common.h" -#include "netops.h" +#include "net.h" #include "stream.h" #include "streams/socket.h" #include "git2/sys/transport.h" @@ -95,22 +95,21 @@ static int git_proto_stream_read( size_t buf_size, size_t *bytes_read) { - int error; git_proto_stream *s = (git_proto_stream *)stream; - gitno_buffer buf; + ssize_t ret; + int error; *bytes_read = 0; if (!s->sent_command && (error = send_command(s)) < 0) return error; - gitno_buffer_setup_fromstream(s->io, &buf, buffer, buf_size); + ret = git_stream_read(s->io, buffer, min(buf_size, INT_MAX)); - if ((error = gitno_recv(&buf)) < 0) - return error; - - *bytes_read = buf.offset; + if (ret < 0) + return -1; + *bytes_read = (size_t)ret; return 0; } diff --git a/src/libgit2/transports/http.c b/src/libgit2/transports/http.c index cda76ae61..0534503bf 100644 --- a/src/libgit2/transports/http.c +++ b/src/libgit2/transports/http.c @@ -11,7 +11,6 @@ #include "http_parser.h" #include "net.h" -#include "netops.h" #include "remote.h" #include "smart.h" #include "auth.h" diff --git a/src/libgit2/transports/smart.c b/src/libgit2/transports/smart.c index a56524bff..537272828 100644 --- a/src/libgit2/transports/smart.c +++ b/src/libgit2/transports/smart.c @@ -13,30 +13,42 @@ #include "refspec.h" #include "proxy.h" -static int git_smart__recv_cb(gitno_buffer *buf) +int git_smart__recv(transport_smart *t) { - transport_smart *t = (transport_smart *) buf->cb_data; - size_t old_len, bytes_read; - int error; + size_t bytes_read; + int ret; + GIT_ASSERT_ARG(t); GIT_ASSERT(t->current_stream); - old_len = buf->offset; + if (git_staticstr_remain(&t->buffer) == 0) { + git_error_set(GIT_ERROR_NET, "out of buffer space"); + return -1; + } - if ((error = t->current_stream->read(t->current_stream, buf->data + buf->offset, buf->len - buf->offset, &bytes_read)) < 0) - return error; + ret = t->current_stream->read(t->current_stream, + git_staticstr_offset(&t->buffer), + git_staticstr_remain(&t->buffer), + &bytes_read); - buf->offset += bytes_read; + if (ret < 0) + return ret; + + GIT_ASSERT(bytes_read <= INT_MAX); + GIT_ASSERT(bytes_read <= git_staticstr_remain(&t->buffer)); + + git_staticstr_increase(&t->buffer, bytes_read); if (t->packetsize_cb && !t->cancelled.val) { - error = t->packetsize_cb(bytes_read, t->packetsize_payload); - if (error) { + ret = t->packetsize_cb(bytes_read, t->packetsize_payload); + + if (ret) { git_atomic32_set(&t->cancelled, 1); return GIT_EUSER; } } - return (int)(buf->offset - old_len); + return (int)bytes_read; } GIT_INLINE(int) git_smart__reset_stream(transport_smart *t, bool close_subtransport) @@ -155,8 +167,6 @@ static int git_smart__connect( /* Save off the current stream (i.e. socket) that we are working with */ t->current_stream = stream; - gitno_buffer_setup_callback(&t->buffer, t->buffer_data, sizeof(t->buffer_data), git_smart__recv_cb, t); - /* 2 flushes for RPC; 1 for stateful */ if ((error = git_smart__store_refs(t, t->rpc ? 2 : 1)) < 0) return error; @@ -313,8 +323,6 @@ int git_smart__negotiation_step(git_transport *transport, void *data, size_t len if ((error = stream->write(stream, (const char *)data, len)) < 0) return error; - gitno_buffer_setup_callback(&t->buffer, t->buffer_data, sizeof(t->buffer_data), git_smart__recv_cb, t); - return 0; } @@ -339,8 +347,6 @@ int git_smart__get_push_stream(transport_smart *t, git_smart_subtransport_stream /* Save off the current stream (i.e. socket) that we are working with */ t->current_stream = *stream; - gitno_buffer_setup_callback(&t->buffer, t->buffer_data, sizeof(t->buffer_data), git_smart__recv_cb, t); - return 0; } @@ -502,20 +508,17 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param) t->owner = owner; t->rpc = definition->rpc; - if (git_vector_init(&t->refs, 16, ref_name_cmp) < 0) { + if (git_vector_init(&t->refs, 16, ref_name_cmp) < 0 || + git_vector_init(&t->heads, 16, ref_name_cmp) < 0 || + definition->callback(&t->wrapped, &t->parent, definition->param) < 0) { + git_vector_free(&t->refs); + git_vector_free(&t->heads); + t->wrapped->free(t->wrapped); git__free(t); return -1; } - if (git_vector_init(&t->heads, 16, ref_name_cmp) < 0) { - git__free(t); - return -1; - } - - if (definition->callback(&t->wrapped, &t->parent, definition->param) < 0) { - git__free(t); - return -1; - } + git_staticstr_init(&t->buffer, GIT_SMART_BUFFER_SIZE); *out = (git_transport *) t; return 0; diff --git a/src/libgit2/transports/smart.h b/src/libgit2/transports/smart.h index 34e27ea8e..52c7553a1 100644 --- a/src/libgit2/transports/smart.h +++ b/src/libgit2/transports/smart.h @@ -11,12 +11,14 @@ #include "git2.h" #include "vector.h" -#include "netops.h" #include "push.h" #include "str.h" #include "oidarray.h" +#include "staticstr.h" #include "git2/sys/transport.h" +#define GIT_SMART_BUFFER_SIZE 65536 + #define GIT_SIDE_BAND_DATA 1 #define GIT_SIDE_BAND_PROGRESS 2 #define GIT_SIDE_BAND_ERROR 3 @@ -170,8 +172,7 @@ typedef struct { unsigned rpc : 1, have_refs : 1, connected : 1; - gitno_buffer buffer; - char buffer_data[65536]; + git_staticstr_with_size(GIT_SMART_BUFFER_SIZE) buffer; } transport_smart; /* smart_protocol.c */ @@ -192,6 +193,8 @@ int git_smart__download_pack( git_indexer_progress *stats); /* smart.c */ +int git_smart__recv(transport_smart *t); + int git_smart__negotiation_step(git_transport *transport, void *data, size_t len); int git_smart__get_push_stream(transport_smart *t, git_smart_subtransport_stream **out); diff --git a/src/libgit2/transports/smart_pkt.c b/src/libgit2/transports/smart_pkt.c index 9127ad5fe..7805f3323 100644 --- a/src/libgit2/transports/smart_pkt.c +++ b/src/libgit2/transports/smart_pkt.c @@ -9,7 +9,6 @@ #include "smart.h" #include "util.h" -#include "netops.h" #include "posix.h" #include "str.h" #include "oid.h" diff --git a/src/libgit2/transports/smart_protocol.c b/src/libgit2/transports/smart_protocol.c index f7a567829..c9c422d4c 100644 --- a/src/libgit2/transports/smart_protocol.c +++ b/src/libgit2/transports/smart_protocol.c @@ -27,7 +27,6 @@ bool git_smart__ofs_delta_enabled = true; int git_smart__store_refs(transport_smart *t, int flushes) { - gitno_buffer *buf = &t->buffer; git_vector *refs = &t->refs; int error, flush = 0, recvd; const char *line_end = NULL; @@ -45,8 +44,10 @@ int git_smart__store_refs(transport_smart *t, int flushes) pkt = NULL; do { - if (buf->offset > 0) - error = git_pkt_parse_line(&pkt, &line_end, buf->data, buf->offset, &pkt_parse_data); + if (t->buffer.len > 0) + error = git_pkt_parse_line(&pkt, &line_end, + t->buffer.data, t->buffer.len, + &pkt_parse_data); else error = GIT_EBUFS; @@ -54,7 +55,7 @@ int git_smart__store_refs(transport_smart *t, int flushes) return error; if (error == GIT_EBUFS) { - if ((recvd = gitno_recv(buf)) < 0) + if ((recvd = git_smart__recv(t)) < 0) return recvd; if (recvd == 0) { @@ -65,8 +66,7 @@ int git_smart__store_refs(transport_smart *t, int flushes) continue; } - if (gitno_consume(buf, line_end) < 0) - return -1; + git_staticstr_consume(&t->buffer, line_end); if (pkt->type == GIT_PKT_ERR) { git_error_set(GIT_ERROR_NET, "remote error: %s", ((git_pkt_err *)pkt)->error); @@ -259,10 +259,9 @@ int git_smart__detect_caps( static int recv_pkt( git_pkt **out_pkt, git_pkt_type *out_type, - transport_smart *t, - gitno_buffer *buf) + transport_smart *t) { - const char *ptr = buf->data, *line_end = ptr; + const char *ptr = t->buffer.data, *line_end = ptr; git_pkt *pkt = NULL; git_pkt_parse_data pkt_parse_data = { 0 }; int error = 0, ret; @@ -271,8 +270,9 @@ static int recv_pkt( pkt_parse_data.seen_capabilities = 1; do { - if (buf->offset > 0) - error = git_pkt_parse_line(&pkt, &line_end, ptr, buf->offset, &pkt_parse_data); + if (t->buffer.len > 0) + error = git_pkt_parse_line(&pkt, &line_end, ptr, + t->buffer.len, &pkt_parse_data); else error = GIT_EBUFS; @@ -282,7 +282,7 @@ static int recv_pkt( if (error < 0 && error != GIT_EBUFS) return error; - if ((ret = gitno_recv(buf)) < 0) { + if ((ret = git_smart__recv(t)) < 0) { return ret; } else if (ret == 0) { git_error_set(GIT_ERROR_NET, "early EOF"); @@ -290,8 +290,7 @@ static int recv_pkt( } } while (error); - if (gitno_consume(buf, line_end) < 0) - return -1; + git_staticstr_consume(&t->buffer, line_end); if (out_type != NULL) *out_type = pkt->type; @@ -306,11 +305,10 @@ static int recv_pkt( static int store_common(transport_smart *t) { git_pkt *pkt = NULL; - gitno_buffer *buf = &t->buffer; int error; do { - if ((error = recv_pkt(&pkt, NULL, t, buf)) < 0) + if ((error = recv_pkt(&pkt, NULL, t)) < 0) return error; if (pkt->type != GIT_PKT_ACK) { @@ -327,7 +325,7 @@ static int store_common(transport_smart *t) return 0; } -static int wait_while_ack(transport_smart *t, gitno_buffer *buf) +static int wait_while_ack(transport_smart *t) { int error; git_pkt *pkt = NULL; @@ -336,7 +334,7 @@ static int wait_while_ack(transport_smart *t, gitno_buffer *buf) while (1) { git_pkt_free(pkt); - if ((error = recv_pkt(&pkt, NULL, t, buf)) < 0) + if ((error = recv_pkt(&pkt, NULL, t)) < 0) return error; if (pkt->type == GIT_PKT_NAK) @@ -402,7 +400,6 @@ int git_smart__negotiate_fetch( { transport_smart *t = (transport_smart *)transport; git_revwalk__push_options opts = GIT_REVWALK__PUSH_OPTIONS_INIT; - gitno_buffer *buf = &t->buffer; git_str data = GIT_STR_INIT; git_revwalk *walk = NULL; int error = -1; @@ -430,7 +427,7 @@ int git_smart__negotiate_fetch( if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0) goto on_error; - while ((error = recv_pkt((git_pkt **)&pkt, NULL, t, buf)) == 0) { + while ((error = recv_pkt((git_pkt **)&pkt, NULL, t)) == 0) { bool complete = false; if (pkt->type == GIT_PKT_SHALLOW) { @@ -495,7 +492,7 @@ int git_smart__negotiate_fetch( if ((error = store_common(t)) < 0) goto on_error; } else { - if ((error = recv_pkt(NULL, &pkt_type, t, buf)) < 0) + if ((error = recv_pkt(NULL, &pkt_type, t)) < 0) goto on_error; if (pkt_type == GIT_PKT_ACK) { @@ -568,7 +565,7 @@ int git_smart__negotiate_fetch( /* Now let's eat up whatever the server gives us */ if (!t->caps.multi_ack && !t->caps.multi_ack_detailed) { - if ((error = recv_pkt(NULL, &pkt_type, t, buf)) < 0) + if ((error = recv_pkt(NULL, &pkt_type, t)) < 0) return error; if (pkt_type != GIT_PKT_ACK && pkt_type != GIT_PKT_NAK) { @@ -576,7 +573,7 @@ int git_smart__negotiate_fetch( return -1; } } else { - error = wait_while_ack(t, buf); + error = wait_while_ack(t); } return error; @@ -606,7 +603,10 @@ int git_smart__shallow_roots(git_oidarray *out, git_transport *transport) return 0; } -static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack, gitno_buffer *buf, git_indexer_progress *stats) +static int no_sideband( + transport_smart *t, + struct git_odb_writepack *writepack, + git_indexer_progress *stats) { int recvd; @@ -616,12 +616,12 @@ static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack, return GIT_EUSER; } - if (writepack->append(writepack, buf->data, buf->offset, stats) < 0) + if (writepack->append(writepack, t->buffer.data, t->buffer.len, stats) < 0) return -1; - gitno_consume_n(buf, buf->offset); + git_staticstr_clear(&t->buffer); - if ((recvd = gitno_recv(buf)) < 0) + if ((recvd = git_smart__recv(t)) < 0) return recvd; } while(recvd > 0); @@ -663,7 +663,6 @@ int git_smart__download_pack( git_indexer_progress *stats) { transport_smart *t = (transport_smart *)transport; - gitno_buffer *buf = &t->buffer; git_odb *odb; struct git_odb_writepack *writepack = NULL; int error = 0; @@ -682,9 +681,10 @@ int git_smart__download_pack( t->packetsize_payload = &npp; /* We might have something in the buffer already from negotiate_fetch */ - if (t->buffer.offset > 0 && !t->cancelled.val) - if (t->packetsize_cb(t->buffer.offset, t->packetsize_payload)) + if (t->buffer.len > 0 && !t->cancelled.val) { + if (t->packetsize_cb(t->buffer.len, t->packetsize_payload)) git_atomic32_set(&t->cancelled, 1); + } } if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 || @@ -697,7 +697,7 @@ int git_smart__download_pack( * check which one belongs there. */ if (!t->caps.side_band && !t->caps.side_band_64k) { - error = no_sideband(t, writepack, buf, stats); + error = no_sideband(t, writepack, stats); goto done; } @@ -711,7 +711,7 @@ int git_smart__download_pack( goto done; } - if ((error = recv_pkt(&pkt, NULL, t, buf)) >= 0) { + if ((error = recv_pkt(&pkt, NULL, t)) >= 0) { /* Check cancellation after network call */ if (t->cancelled.val) { git_error_clear(); @@ -916,15 +916,15 @@ static int parse_report(transport_smart *transport, git_push *push) git_pkt *pkt = NULL; git_pkt_parse_data pkt_parse_data = { 0 }; const char *line_end = NULL; - gitno_buffer *buf = &transport->buffer; int error, recvd; git_str data_pkt_buf = GIT_STR_INIT; for (;;) { - if (buf->offset > 0) + if (transport->buffer.len > 0) error = git_pkt_parse_line(&pkt, &line_end, - buf->data, buf->offset, - &pkt_parse_data); + transport->buffer.data, + transport->buffer.len, + &pkt_parse_data); else error = GIT_EBUFS; @@ -934,7 +934,7 @@ static int parse_report(transport_smart *transport, git_push *push) } if (error == GIT_EBUFS) { - if ((recvd = gitno_recv(buf)) < 0) { + if ((recvd = git_smart__recv(transport)) < 0) { error = recvd; goto done; } @@ -947,9 +947,7 @@ static int parse_report(transport_smart *transport, git_push *push) continue; } - if (gitno_consume(buf, line_end) < 0) - return -1; - + git_staticstr_consume(&transport->buffer, line_end); error = 0; switch (pkt->type) { diff --git a/src/libgit2/transports/ssh.c b/src/libgit2/transports/ssh.c index 5500ea100..af618e1a6 100644 --- a/src/libgit2/transports/ssh.c +++ b/src/libgit2/transports/ssh.c @@ -13,7 +13,6 @@ #include "runtime.h" #include "net.h" -#include "netops.h" #include "smart.h" #include "streams/socket.h" #include "sysdir.h" diff --git a/src/libgit2/transports/winhttp.c b/src/libgit2/transports/winhttp.c index de24a2a41..27e0fb6f7 100644 --- a/src/libgit2/transports/winhttp.c +++ b/src/libgit2/transports/winhttp.c @@ -13,7 +13,6 @@ #include "git2/transport.h" #include "posix.h" #include "str.h" -#include "netops.h" #include "smart.h" #include "remote.h" #include "repository.h" diff --git a/src/util/staticstr.h b/src/util/staticstr.h new file mode 100644 index 000000000..b7d0790c4 --- /dev/null +++ b/src/util/staticstr.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +#ifndef INCLUDE_stackstr_h__ +#define INCLUDE_stackstr_h__ + +#include "git2_util.h" + +typedef struct { + /* Length of / number of bytes used by `data`. */ + size_t len; + + /* Size of the allocated `data` buffer. */ + size_t size; + + /* The actual string buffer data. */ + char data[GIT_FLEX_ARRAY]; +} git_staticstr; + +#define git_staticstr_with_size(__size) \ + struct { \ + size_t len; \ + size_t size; \ + char data[__size]; \ + } + +#define git_staticstr_init(__str, __size) \ + do { \ + (__str)->len = 0; \ + (__str)->size = __size; \ + (__str)->data[0] = '\0'; \ + } while(0) + +#define git_staticstr_offset(__str) \ + ((__str)->data + (__str)->len) + +#define git_staticstr_remain(__str) \ + ((__str)->len > (__str)->size ? 0 : ((__str)->size - (__str)->len)) + +#define git_staticstr_increase(__str, __len) \ + do { ((__str)->len += __len); } while(0) + +#define git_staticstr_consume_bytes(__str, __len) \ + do { git_staticstr_consume(__str, (__str)->data + __len); } while(0) + +#define git_staticstr_consume(__str, __end) \ + do { \ + if (__end > (__str)->data && \ + __end <= (__str)->data + (__str)->len) { \ + size_t __consumed = __end - (__str)->data; \ + memmove((__str)->data, __end, (__str)->len - __consumed); \ + (__str)->len -= __consumed; \ + (__str)->data[(__str)->len] = '\0'; \ + } \ + } while(0) + +#define git_staticstr_clear(__str) \ + do { \ + (__str)->len = 0; \ + (__str)->data[0] = 0; \ + } while(0) + +#endif diff --git a/tests/util/url/joinpath.c b/tests/util/url/joinpath.c index 9fc02cde4..6027093e4 100644 --- a/tests/util/url/joinpath.c +++ b/tests/util/url/joinpath.c @@ -1,6 +1,5 @@ #include "clar_libgit2.h" #include "net.h" -#include "netops.h" static git_net_url source, target; diff --git a/tests/util/url/redirect.c b/tests/util/url/redirect.c index 540177861..a6f99dcdf 100644 --- a/tests/util/url/redirect.c +++ b/tests/util/url/redirect.c @@ -1,6 +1,5 @@ #include "clar_libgit2.h" #include "net.h" -#include "netops.h" static git_net_url conndata; -- cgit v1.2.1 From 933b04c219b49d0f9cd779d024bfa39d806839ac Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:30 +0100 Subject: ci: update to poxygit v0.6.0 v0.6.0 of poxygit add support for throttling connections to test timeouts and low-bandwidth situations. --- ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test.sh b/ci/test.sh index c46cf0dc4..a6735231f 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -171,7 +171,7 @@ if [ -z "$SKIP_PROXY_TESTS" ]; then fi if [ -z "$SKIP_NTLM_TESTS" -o -z "$SKIP_ONLINE_TESTS" ]; then - curl --location --silent --show-error https://github.com/ethomson/poxygit/releases/download/v0.5.1/poxygit-0.5.1.jar >poxygit.jar + curl --location --silent --show-error https://github.com/ethomson/poxygit/releases/download/v0.6.0/poxygit-0.6.0.jar >poxygit.jar echo "Starting HTTP server..." HTTP_DIR=`mktemp -d ${TMPDIR}/http.XXXXXXXX` -- cgit v1.2.1 From fad90428970e332153027773b517a1606c0efa1f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:30 +0100 Subject: streams: sockets are non-blocking and can timeout Make socket I/O non-blocking and add optional timeouts. Users may now set `GIT_OPT_SET_SERVER_CONNECT_TIMEOUT` to set a shorter connection timeout. (The connect timeout cannot be longer than the operating system default.) Users may also now configure the socket read and write timeouts with `GIT_OPT_SET_SERVER_TIMEOUT`. By default, connects still timeout based on the operating system defaults (typically 75 seconds) and socket read and writes block. Add a test against our custom testing git server that ensures that we can timeout reads against a slow server. --- ci/test.sh | 4 + include/git2/common.h | 27 +++++- include/git2/errors.h | 3 +- include/git2/sys/stream.h | 18 +++- src/libgit2/libgit2.c | 36 +++++++ src/libgit2/streams/socket.c | 198 ++++++++++++++++++++++++++++++++++----- src/libgit2/streams/stransport.c | 17 +++- tests/libgit2/CMakeLists.txt | 2 +- tests/libgit2/online/clone.c | 60 ++++++++++++ 9 files changed, 335 insertions(+), 30 deletions(-) diff --git a/ci/test.sh b/ci/test.sh index a6735231f..ee6801a79 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -271,9 +271,13 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then export GITTEST_REMOTE_REDIRECT_INITIAL="http://localhost:9000/initial-redirect/libgit2/TestGitRepository" export GITTEST_REMOTE_REDIRECT_SUBSEQUENT="http://localhost:9000/subsequent-redirect/libgit2/TestGitRepository" + export GITTEST_REMOTE_SPEED_SLOW="http://localhost:9000/speed-9600/test.git" + export GITTEST_REMOTE_SPEED_TIMESOUT="http://localhost:9000/speed-0.5/test.git" run_test online unset GITTEST_REMOTE_REDIRECT_INITIAL unset GITTEST_REMOTE_REDIRECT_SUBSEQUENT + unset GITTEST_REMOTE_SPEED_SLOW + unset GITTEST_REMOTE_SPEED_TIMESOUT # Run the online tests that immutably change global state separately # to avoid polluting the test environment. diff --git a/include/git2/common.h b/include/git2/common.h index f968deb23..ab6bc1333 100644 --- a/include/git2/common.h +++ b/include/git2/common.h @@ -224,7 +224,11 @@ typedef enum { GIT_OPT_GET_OWNER_VALIDATION, GIT_OPT_SET_OWNER_VALIDATION, GIT_OPT_GET_HOMEDIR, - GIT_OPT_SET_HOMEDIR + GIT_OPT_SET_HOMEDIR, + GIT_OPT_SET_SERVER_CONNECT_TIMEOUT, + GIT_OPT_GET_SERVER_CONNECT_TIMEOUT, + GIT_OPT_SET_SERVER_TIMEOUT, + GIT_OPT_GET_SERVER_TIMEOUT } git_libgit2_opt_t; /** @@ -480,6 +484,27 @@ typedef enum { * > * > - `path` directory of home directory. * + * opts(GIT_OPT_GET_SERVER_CONNECT_TIMEOUT, int *timeout) + * > Gets the timeout (in milliseconds) to attempt connections to + * > a remote server. + * + * opts(GIT_OPT_SET_SERVER_CONNECT_TIMEOUT, int timeout) + * > Sets the timeout (in milliseconds) to attempt connections to + * > a remote server. This is supported only for HTTP(S) connections + * > and is not supported by SSH. Set to 0 to use the system default. + * > Note that this may not be able to be configured longer than the + * > system default, typically 75 seconds. + * + * opts(GIT_OPT_GET_SERVER_TIMEOUT, int *timeout) + * > Gets the timeout (in milliseconds) for reading from and writing + * > to a remote server. + * + * opts(GIT_OPT_SET_SERVER_TIMEOUT, int timeout) + * > Sets the timeout (in milliseconds) for reading from and writing + * > to a remote server. This is supported only for HTTP(S) + * > connections and is not supported by SSH. Set to 0 to use the + * > system default. + * * @param option Option key * @param ... value to set the option * @return 0 on success, <0 on failure diff --git a/include/git2/errors.h b/include/git2/errors.h index e634a97c1..7180852f9 100644 --- a/include/git2/errors.h +++ b/include/git2/errors.h @@ -58,7 +58,8 @@ typedef enum { GIT_EMISMATCH = -33, /**< Hashsum mismatch in object */ GIT_EINDEXDIRTY = -34, /**< Unsaved changes in the index would be overwritten */ GIT_EAPPLYFAIL = -35, /**< Patch application failed */ - GIT_EOWNER = -36 /**< The object is not owned by the current user */ + GIT_EOWNER = -36, /**< The object is not owned by the current user */ + GIT_TIMEOUT = -37 /**< The operation timed out */ } git_error_code; /** diff --git a/include/git2/sys/stream.h b/include/git2/sys/stream.h index e0e03a2d7..3d28d09b3 100644 --- a/include/git2/sys/stream.h +++ b/include/git2/sys/stream.h @@ -29,8 +29,22 @@ GIT_BEGIN_DECL typedef struct git_stream { int version; - int encrypted; - int proxy_support; + int encrypted : 1, + proxy_support : 1; + + /** + * Timeout for read and write operations; can be set to `0` to + * block indefinitely. + */ + int timeout; + + /** + * Timeout to connect to the remote server; can be set to `0` + * to use the system defaults. This can be shorter than the + * system default - often 75 seconds - but cannot be longer. + */ + int connect_timeout; + int GIT_CALLBACK(connect)(struct git_stream *); int GIT_CALLBACK(certificate)(git_cert **, struct git_stream *); int GIT_CALLBACK(set_proxy)(struct git_stream *, const git_proxy_options *proxy_opts); diff --git a/src/libgit2/libgit2.c b/src/libgit2/libgit2.c index 178880c9e..ce287147a 100644 --- a/src/libgit2/libgit2.c +++ b/src/libgit2/libgit2.c @@ -48,6 +48,8 @@ extern size_t git_indexer__max_objects; extern bool git_disable_pack_keep_file_checks; extern int git_odb__packed_priority; extern int git_odb__loose_priority; +extern int git_socket_stream__connect_timeout; +extern int git_socket_stream__timeout; char *git__user_agent; char *git__ssl_ciphers; @@ -436,6 +438,40 @@ int git_libgit2_opts(int key, ...) error = git_sysdir_set(GIT_SYSDIR_HOME, va_arg(ap, const char *)); break; + case GIT_OPT_GET_SERVER_CONNECT_TIMEOUT: + *(va_arg(ap, int *)) = git_socket_stream__connect_timeout; + break; + + case GIT_OPT_SET_SERVER_CONNECT_TIMEOUT: + { + int timeout = va_arg(ap, int); + + if (timeout < 0) { + git_error_set(GIT_ERROR_INVALID, "invalid connect timeout"); + error = -1; + } else { + git_socket_stream__connect_timeout = timeout; + } + } + break; + + case GIT_OPT_GET_SERVER_TIMEOUT: + *(va_arg(ap, int *)) = git_socket_stream__timeout; + break; + + case GIT_OPT_SET_SERVER_TIMEOUT: + { + int timeout = va_arg(ap, int); + + if (timeout < 0) { + git_error_set(GIT_ERROR_INVALID, "invalid timeout"); + error = -1; + } else { + git_socket_stream__timeout = timeout; + } + } + break; + default: git_error_set(GIT_ERROR_INVALID, "invalid option key"); error = -1; diff --git a/src/libgit2/streams/socket.c b/src/libgit2/streams/socket.c index 6994d58f2..0e0aa6934 100644 --- a/src/libgit2/streams/socket.c +++ b/src/libgit2/streams/socket.c @@ -20,6 +20,7 @@ # include # include # include +# include #else # include # include @@ -28,6 +29,9 @@ # endif #endif +int git_socket_stream__connect_timeout = 0; +int git_socket_stream__timeout = 0; + #ifdef GIT_WIN32 static void net_set_error(const char *str) { @@ -66,21 +70,105 @@ static int close_socket(GIT_SOCKET s) } +static int set_nonblocking(GIT_SOCKET s) +{ +#ifdef GIT_WIN32 + unsigned long nonblocking = 1; + + if (ioctlsocket(s, FIONBIO, &nonblocking) != 0) { + net_set_error("could not set socket non-blocking"); + return -1; + } +#else + int flags; + + if ((flags = fcntl(s, F_GETFL, 0)) == -1) { + net_set_error("could not query socket flags"); + return -1; + } + + flags |= O_NONBLOCK; + + if (fcntl(s, F_SETFL, flags) != 0) { + net_set_error("could not set socket non-blocking"); + return -1; + } +#endif + + return 0; +} + +/* Promote a sockerr to an errno for our error handling routines */ +static int handle_sockerr(GIT_SOCKET socket) +{ + int sockerr; + socklen_t errlen = sizeof(sockerr); + + if (getsockopt(socket, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) < 0) + return -1; + + if (sockerr == ETIMEDOUT) + return GIT_TIMEOUT; + + errno = sockerr; + return -1; +} + +static int connect_with_timeout( + GIT_SOCKET socket, + const struct sockaddr *address, + socklen_t address_len, + int timeout) +{ + struct pollfd fd; + int error; + + if (timeout && (error = set_nonblocking(socket)) < 0) + return error; + + error = connect(socket, address, address_len); + + if (error == 0 || (error == -1 && errno != EINPROGRESS)) + return error; + + fd.fd = socket; + fd.events = POLLOUT; + fd.revents = 0; + + error = poll(&fd, 1, timeout); + + if (error == 0) { + return GIT_TIMEOUT; + } else if (error != 1) { + return -1; + } else if ((fd.revents & (POLLHUP | POLLERR))) { + return handle_sockerr(socket); + } else if ((fd.revents & POLLOUT) != POLLOUT) { + git_error_set(GIT_ERROR_NET, + "unknown error while polling for connect: %d", + fd.revents); + return -1; + } + + return 0; +} + static int socket_connect(git_stream *stream) { - struct addrinfo *info = NULL, *p; - struct addrinfo hints; git_socket_stream *st = (git_socket_stream *) stream; GIT_SOCKET s = INVALID_SOCKET; - int ret; + struct addrinfo *info = NULL, *p; + struct addrinfo hints; + int error; memset(&hints, 0x0, sizeof(struct addrinfo)); hints.ai_socktype = SOCK_STREAM; hints.ai_family = AF_UNSPEC; - if ((ret = p_getaddrinfo(st->host, st->port, &hints, &info)) != 0) { + if ((error = p_getaddrinfo(st->host, st->port, &hints, &info)) != 0) { git_error_set(GIT_ERROR_NET, - "failed to resolve address for %s: %s", st->host, p_gai_strerror(ret)); + "failed to resolve address for %s: %s", + st->host, p_gai_strerror(error)); return -1; } @@ -90,51 +178,115 @@ static int socket_connect(git_stream *stream) if (s == INVALID_SOCKET) continue; - if (connect(s, p->ai_addr, (socklen_t)p->ai_addrlen) == 0) + error = connect_with_timeout(s, p->ai_addr, + (socklen_t)p->ai_addrlen, + st->parent.connect_timeout); + + if (error == 0) break; /* If we can't connect, try the next one */ close_socket(s); s = INVALID_SOCKET; + + if (error == GIT_TIMEOUT) + break; } /* Oops, we couldn't connect to any address */ - if (s == INVALID_SOCKET && p == NULL) { - git_error_set(GIT_ERROR_OS, "failed to connect to %s", st->host); - p_freeaddrinfo(info); - return -1; + if (s == INVALID_SOCKET) { + if (error == GIT_TIMEOUT) + git_error_set(GIT_ERROR_NET, "failed to connect to %s: Operation timed out", st->host); + else + git_error_set(GIT_ERROR_OS, "failed to connect to %s", st->host); + error = -1; + goto done; } + if (st->parent.timeout && !st->parent.connect_timeout && + (error = set_nonblocking(s)) < 0) + return error; + st->s = s; + error = 0; + +done: p_freeaddrinfo(info); - return 0; + return error; } -static ssize_t socket_write(git_stream *stream, const char *data, size_t len, int flags) +static ssize_t socket_write( + git_stream *stream, + const char *data, + size_t len, + int flags) { git_socket_stream *st = (git_socket_stream *) stream; - ssize_t written; + struct pollfd fd; + ssize_t ret; GIT_ASSERT(flags == 0); GIT_UNUSED(flags); - errno = 0; + ret = p_send(st->s, data, len, 0); + + if (st->parent.timeout && ret < 0 && + (errno == EAGAIN || errno != EWOULDBLOCK)) { + fd.fd = st->s; + fd.events = POLLOUT; + fd.revents = 0; + + ret = poll(&fd, 1, st->parent.timeout); - if ((written = p_send(st->s, data, len, 0)) < 0) { - net_set_error("error sending data"); + if (ret == 1) { + ret = p_send(st->s, data, len, 0); + } else if (ret == 0) { + git_error_set(GIT_ERROR_NET, + "could not write to socket: timed out"); + return GIT_TIMEOUT; + } + } + + if (ret < 0) { + net_set_error("error receiving data from socket"); return -1; } - return written; + return ret; } -static ssize_t socket_read(git_stream *stream, void *data, size_t len) +static ssize_t socket_read( + git_stream *stream, + void *data, + size_t len) { - ssize_t ret; git_socket_stream *st = (git_socket_stream *) stream; + struct pollfd fd; + ssize_t ret; + + ret = p_recv(st->s, data, len, 0); + + if (st->parent.timeout && ret < 0 && + (errno == EAGAIN || errno != EWOULDBLOCK)) { + fd.fd = st->s; + fd.events = POLLIN; + fd.revents = 0; - if ((ret = p_recv(st->s, data, len, 0)) < 0) - net_set_error("error receiving socket data"); + ret = poll(&fd, 1, st->parent.timeout); + + if (ret == 1) { + ret = p_recv(st->s, data, len, 0); + } else if (ret == 0) { + git_error_set(GIT_ERROR_NET, + "could not read from socket: timed out"); + return GIT_TIMEOUT; + } + } + + if (ret < 0) { + net_set_error("error receiving data from socket"); + return -1; + } return ret; } @@ -182,6 +334,8 @@ static int default_socket_stream_new( } st->parent.version = GIT_STREAM_VERSION; + st->parent.timeout = git_socket_stream__timeout; + st->parent.connect_timeout = git_socket_stream__connect_timeout; st->parent.connect = socket_connect; st->parent.write = socket_write; st->parent.read = socket_read; @@ -248,7 +402,7 @@ int git_socket_stream_global_init(void) return git_runtime_shutdown_register(socket_stream_global_shutdown); } - + #else #include "stream.h" diff --git a/src/libgit2/streams/stransport.c b/src/libgit2/streams/stransport.c index 74ee0d1ee..d956df84d 100644 --- a/src/libgit2/streams/stransport.c +++ b/src/libgit2/streams/stransport.c @@ -161,7 +161,9 @@ static OSStatus write_cb(SSLConnectionRef conn, const void *data, size_t *len) if (ret < 0) { st->error = ret; - return -36; /* ioErr */ + return (ret == GIT_TIMEOUT) ? + errSSLNetworkTimeout : + -36 /* ioErr */; } return noErr; @@ -176,8 +178,12 @@ static ssize_t stransport_write(git_stream *stream, const char *data, size_t len GIT_UNUSED(flags); data_len = min(len, SSIZE_MAX); - if ((ret = SSLWrite(st->ctx, data, data_len, &processed)) != noErr) + if ((ret = SSLWrite(st->ctx, data, data_len, &processed)) != noErr) { + if (st->error == GIT_TIMEOUT) + return GIT_TIMEOUT; + return stransport_error(ret); + } GIT_ASSERT(processed < SSIZE_MAX); return (ssize_t)processed; @@ -207,7 +213,9 @@ static OSStatus read_cb(SSLConnectionRef conn, void *data, size_t *len) if (ret < 0) { st->error = ret; - error = -36; /* ioErr */ + error = (ret == GIT_TIMEOUT) ? + errSSLNetworkTimeout : + -36 /* ioErr */; break; } else if (ret == 0) { error = errSSLClosedGraceful; @@ -228,6 +236,9 @@ static ssize_t stransport_read(git_stream *stream, void *data, size_t len) OSStatus ret; if ((ret = SSLRead(st->ctx, data, len, &processed)) != noErr) { + if (st->error == GIT_TIMEOUT) + return GIT_TIMEOUT; + return stransport_error(ret); } diff --git a/tests/libgit2/CMakeLists.txt b/tests/libgit2/CMakeLists.txt index 866122d1c..af70f55a7 100644 --- a/tests/libgit2/CMakeLists.txt +++ b/tests/libgit2/CMakeLists.txt @@ -65,7 +65,7 @@ endif() include(AddClarTest) add_clar_test(libgit2_tests offline -v -xonline) -add_clar_test(libgit2_tests invasive -v -sfilter::stream::bigfile -sodb::largefiles -siterator::workdir::filesystem_gunk -srepo::init -srepo::init::at_filesystem_root) +add_clar_test(libgit2_tests invasive -v -sfilter::stream::bigfile -sodb::largefiles -siterator::workdir::filesystem_gunk -srepo::init -srepo::init::at_filesystem_root -sonline::clone::connect_timeout_default) add_clar_test(libgit2_tests online -v -sonline -xonline::customcert) add_clar_test(libgit2_tests online_customcert -v -sonline::customcert) add_clar_test(libgit2_tests gitdaemon -v -sonline::push) diff --git a/tests/libgit2/online/clone.c b/tests/libgit2/online/clone.c index b635739b6..92fa4d351 100644 --- a/tests/libgit2/online/clone.c +++ b/tests/libgit2/online/clone.c @@ -35,6 +35,8 @@ static char *_remote_proxy_selfsigned = NULL; static char *_remote_expectcontinue = NULL; static char *_remote_redirect_initial = NULL; static char *_remote_redirect_subsequent = NULL; +static char *_remote_speed_timesout = NULL; +static char *_remote_speed_slow = NULL; static char *_github_ssh_pubkey = NULL; static char *_github_ssh_privkey = NULL; @@ -89,6 +91,8 @@ void test_online_clone__initialize(void) _remote_expectcontinue = cl_getenv("GITTEST_REMOTE_EXPECTCONTINUE"); _remote_redirect_initial = cl_getenv("GITTEST_REMOTE_REDIRECT_INITIAL"); _remote_redirect_subsequent = cl_getenv("GITTEST_REMOTE_REDIRECT_SUBSEQUENT"); + _remote_speed_timesout = cl_getenv("GITTEST_REMOTE_SPEED_TIMESOUT"); + _remote_speed_slow = cl_getenv("GITTEST_REMOTE_SPEED_SLOW"); _github_ssh_pubkey = cl_getenv("GITTEST_GITHUB_SSH_PUBKEY"); _github_ssh_privkey = cl_getenv("GITTEST_GITHUB_SSH_KEY"); @@ -128,6 +132,8 @@ void test_online_clone__cleanup(void) git__free(_remote_expectcontinue); git__free(_remote_redirect_initial); git__free(_remote_redirect_subsequent); + git__free(_remote_speed_timesout); + git__free(_remote_speed_slow); git__free(_github_ssh_pubkey); git__free(_github_ssh_privkey); @@ -145,6 +151,8 @@ void test_online_clone__cleanup(void) } git_libgit2_opts(GIT_OPT_SET_SSL_CERT_LOCATIONS, NULL, NULL); + git_libgit2_opts(GIT_OPT_SET_SERVER_TIMEOUT, 0); + git_libgit2_opts(GIT_OPT_SET_SERVER_CONNECT_TIMEOUT, 0); } void test_online_clone__network_full(void) @@ -1207,3 +1215,55 @@ void test_online_clone__sha256(void) git_reference_free(head); #endif } + +void test_online_clone__connect_timeout_configurable(void) +{ + uint64_t start, finish; + + start = git_time_monotonic(); + + cl_git_pass(git_libgit2_opts(GIT_OPT_SET_SERVER_CONNECT_TIMEOUT, 1)); + cl_git_fail(git_clone(&g_repo, "http://www.google.com:8000/", "./timedout", NULL)); + cl_assert(git_error_last() && strstr(git_error_last()->message, "timed out")); + + finish = git_time_monotonic(); + + cl_assert(finish - start < 1000); +} + +void test_online_clone__connect_timeout_default(void) +{ + /* This test takes ~ 75 seconds on Unix. */ + if (!cl_is_env_set("GITTEST_INVASIVE_SPEED")) + cl_skip(); + + /* + * Use a host/port pair that blackholes packets and does not + * send an RST. + */ + cl_git_fail_with(GIT_TIMEOUT, git_clone(&g_repo, "http://www.google.com:8000/", "./refused", NULL)); + cl_assert(git_error_last() && strstr(git_error_last()->message, "timed out")); +} + +void test_online_clone__timeout_configurable_times_out(void) +{ + git_repository *failed_repo; + + if (!_remote_speed_timesout) + cl_skip(); + + cl_git_pass(git_libgit2_opts(GIT_OPT_SET_SERVER_TIMEOUT, 1000)); + + cl_git_fail_with(GIT_TIMEOUT, git_clone(&failed_repo, _remote_speed_timesout, "./timedout", NULL)); + cl_assert(git_error_last() && strstr(git_error_last()->message, "timed out")); +} + +void test_online_clone__timeout_configurable_succeeds_slowly(void) +{ + if (!_remote_speed_slow) + cl_skip(); + + cl_git_pass(git_libgit2_opts(GIT_OPT_SET_SERVER_TIMEOUT, 1000)); + + cl_git_pass(git_clone(&g_repo, _remote_speed_slow, "./slow-but-successful", NULL)); +} -- cgit v1.2.1 From 94f98400bf950aa9835ff9b04c724f2aa3ebd0fc Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:31 +0100 Subject: posix: introduce p_poll emulation with select Not all systems have poll(2); emulate it with select(2). --- src/CMakeLists.txt | 5 ++++ src/libgit2/streams/socket.c | 9 ++++---- src/util/git2_features.h.in | 3 +++ src/util/posix.c | 54 ++++++++++++++++++++++++++++++++++++++++++++ src/util/posix.h | 19 ++++++++++++++++ 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 88d616cec..a00ab600a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -87,6 +87,11 @@ check_prototype_definition_safe(qsort_s check_function_exists(getentropy GIT_RAND_GETENTROPY) check_function_exists(getloadavg GIT_RAND_GETLOADAVG) +# poll + +check_symbol_exists(poll poll.h GIT_IO_POLL) +check_symbol_exists(select sys/select.h GIT_IO_SELECT) + # determine architecture of the machine if(CMAKE_SIZEOF_VOID_P EQUAL 8) diff --git a/src/libgit2/streams/socket.c b/src/libgit2/streams/socket.c index 0e0aa6934..ecbdf7b7d 100644 --- a/src/libgit2/streams/socket.c +++ b/src/libgit2/streams/socket.c @@ -20,7 +20,6 @@ # include # include # include -# include #else # include # include @@ -135,13 +134,13 @@ static int connect_with_timeout( fd.events = POLLOUT; fd.revents = 0; - error = poll(&fd, 1, timeout); + error = p_poll(&fd, 1, timeout); if (error == 0) { return GIT_TIMEOUT; } else if (error != 1) { return -1; - } else if ((fd.revents & (POLLHUP | POLLERR))) { + } else if ((fd.revents & (POLLPRI | POLLHUP | POLLERR))) { return handle_sockerr(socket); } else if ((fd.revents & POLLOUT) != POLLOUT) { git_error_set(GIT_ERROR_NET, @@ -236,7 +235,7 @@ static ssize_t socket_write( fd.events = POLLOUT; fd.revents = 0; - ret = poll(&fd, 1, st->parent.timeout); + ret = p_poll(&fd, 1, st->parent.timeout); if (ret == 1) { ret = p_send(st->s, data, len, 0); @@ -272,7 +271,7 @@ static ssize_t socket_read( fd.events = POLLIN; fd.revents = 0; - ret = poll(&fd, 1, st->parent.timeout); + ret = p_poll(&fd, 1, st->parent.timeout); if (ret == 1) { ret = p_recv(st->s, data, len, 0); diff --git a/src/util/git2_features.h.in b/src/util/git2_features.h.in index 18644c3cc..cf955122c 100644 --- a/src/util/git2_features.h.in +++ b/src/util/git2_features.h.in @@ -61,4 +61,7 @@ #cmakedefine GIT_RAND_GETENTROPY 1 #cmakedefine GIT_RAND_GETLOADAVG 1 +#cmakedefine GIT_IO_POLL 1 +#cmakedefine GIT_IO_SELECT 1 + #endif diff --git a/src/util/posix.c b/src/util/posix.c index b1f85dc94..cfc0e0751 100644 --- a/src/util/posix.c +++ b/src/util/posix.c @@ -301,3 +301,57 @@ int p_munmap(git_map *map) } #endif + +#if defined(GIT_IO_POLL) || defined(GIT_IO_WSAPOLL) + +/* Handled by posix.h; this test simplifies the final else */ + +#elif defined(GIT_IO_SELECT) + +int p_poll(struct pollfd *fds, unsigned int nfds, int timeout_ms) +{ + fd_set read_fds, write_fds, except_fds; + struct timeval timeout = { 0, 0 }; + unsigned int i; + int max_fd = -1, ret; + + FD_ZERO(&read_fds); + FD_ZERO(&write_fds); + FD_ZERO(&except_fds); + + for (i = 0; i < nfds; i++) { + if ((fds[i].events & POLLIN)) + FD_SET(fds[i].fd, &read_fds); + + if ((fds[i].events & POLLOUT)) + FD_SET(fds[i].fd, &write_fds); + + if ((fds[i].events & POLLPRI)) + FD_SET(fds[i].fd, &except_fds); + + max_fd = MAX(max_fd, fds[i].fd); + } + + if (timeout_ms > 0) { + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_usec = (timeout_ms % 1000) * 1000; + } + + if ((ret = select(max_fd + 1, &read_fds, &write_fds, &except_fds, + timeout_ms < 0 ? NULL : &timeout)) < 0) + goto done; + + for (i = 0; i < nfds; i++) { + fds[i].revents = 0 | + FD_ISSET(fds[i].fd, &read_fds) ? POLLIN : 0 | + FD_ISSET(fds[i].fd, &write_fds) ? POLLOUT : 0 | + FD_ISSET(fds[i].fd, &except_fds) ? POLLPRI : 0; + } + +done: + return ret; +} + +#else +# error no poll compatible implementation +#endif diff --git a/src/util/posix.h b/src/util/posix.h index 607aa9dce..f516bb3cc 100644 --- a/src/util/posix.h +++ b/src/util/posix.h @@ -195,4 +195,23 @@ extern const char *p_gai_strerror(int ret); # define p_gai_strerror(c) gai_strerror(c) #endif /* NO_ADDRINFO */ +#ifdef GIT_IO_POLL +# include +# define p_poll poll +#else +# define POLLIN 0x01 +# define POLLPRI 0x02 +# define POLLOUT 0x04 +# define POLLERR 0x08 +# define POLLHUP 0x10 + +struct pollfd { + int fd; + short events; + short revents; +}; + +extern int p_poll(struct pollfd *fds, unsigned int nfds, int timeout); +#endif + #endif -- cgit v1.2.1 From 9908c076ab3fafa18f00c866840819d52bc6b00f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:31 +0100 Subject: posix: use WSAPoll on win32 --- src/CMakeLists.txt | 8 ++++++-- src/util/git2_features.h.in | 1 + src/util/posix.h | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a00ab600a..e9de94b52 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,8 +89,12 @@ check_function_exists(getloadavg GIT_RAND_GETLOADAVG) # poll -check_symbol_exists(poll poll.h GIT_IO_POLL) -check_symbol_exists(select sys/select.h GIT_IO_SELECT) +if(WIN32) + set(GIT_IO_WSAPOLL 1) +else() + check_symbol_exists(poll poll.h GIT_IO_POLL) + check_symbol_exists(select sys/select.h GIT_IO_SELECT) +endif() # determine architecture of the machine diff --git a/src/util/git2_features.h.in b/src/util/git2_features.h.in index cf955122c..a84ea8956 100644 --- a/src/util/git2_features.h.in +++ b/src/util/git2_features.h.in @@ -62,6 +62,7 @@ #cmakedefine GIT_RAND_GETLOADAVG 1 #cmakedefine GIT_IO_POLL 1 +#cmakedefine GIT_IO_WSAPOLL 1 #cmakedefine GIT_IO_SELECT 1 #endif diff --git a/src/util/posix.h b/src/util/posix.h index f516bb3cc..74707453a 100644 --- a/src/util/posix.h +++ b/src/util/posix.h @@ -198,6 +198,9 @@ extern const char *p_gai_strerror(int ret); #ifdef GIT_IO_POLL # include # define p_poll poll +#elif GIT_IO_WSAPOLL +# include +# define p_poll WSAPoll #else # define POLLIN 0x01 # define POLLPRI 0x02 -- cgit v1.2.1 From 02ab53ce051df9a6b70b8f075956ed3a41f9979e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:31 +0100 Subject: cmake: use `check_symbol_exists` for rand functions `check_symbol_exists` is superior to `check_function_exists`; use it consistently in our cmake configuration --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e9de94b52..8525acdd8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -84,8 +84,8 @@ check_prototype_definition_safe(qsort_s # random / entropy data -check_function_exists(getentropy GIT_RAND_GETENTROPY) -check_function_exists(getloadavg GIT_RAND_GETLOADAVG) +check_symbol_exists(getentropy unistd.h GIT_RAND_GETENTROPY) +check_symbol_exists(getloadavg stdlib.h GIT_RAND_GETLOADAVG) # poll -- cgit v1.2.1 From 03eebab8a9fc353e96b2f6932d28defd5d868bca Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 12 May 2023 20:48:32 +0100 Subject: win32: getsockopt takes a `char *` instead of a `void *` Monsters. --- src/libgit2/streams/socket.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libgit2/streams/socket.c b/src/libgit2/streams/socket.c index ecbdf7b7d..3bfc3fd0f 100644 --- a/src/libgit2/streams/socket.c +++ b/src/libgit2/streams/socket.c @@ -103,7 +103,8 @@ static int handle_sockerr(GIT_SOCKET socket) int sockerr; socklen_t errlen = sizeof(sockerr); - if (getsockopt(socket, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) < 0) + if (getsockopt(socket, SOL_SOCKET, SO_ERROR, + (void *)&sockerr, &errlen) < 0) return -1; if (sockerr == ETIMEDOUT) -- cgit v1.2.1 From 14c820b191a7083bf9a54792e09afcd83e0f0206 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 11 Apr 2023 20:55:22 +0100 Subject: win32: use WSAGetLastError to determine blocking --- src/libgit2/streams/socket.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libgit2/streams/socket.c b/src/libgit2/streams/socket.c index 3bfc3fd0f..a463312fd 100644 --- a/src/libgit2/streams/socket.c +++ b/src/libgit2/streams/socket.c @@ -114,6 +114,19 @@ static int handle_sockerr(GIT_SOCKET socket) return -1; } +GIT_INLINE(bool) connect_would_block(int error) +{ +#ifdef GIT_WIN32 + if (error == SOCKET_ERROR && WSAGetLastError() == WSAEWOULDBLOCK) + return true; +#endif + + if (error == -1 && errno == EINPROGRESS) + return true; + + return false; +} + static int connect_with_timeout( GIT_SOCKET socket, const struct sockaddr *address, @@ -128,7 +141,7 @@ static int connect_with_timeout( error = connect(socket, address, address_len); - if (error == 0 || (error == -1 && errno != EINPROGRESS)) + if (error == 0 || !connect_would_block(error)) return error; fd.fd = socket; -- cgit v1.2.1 From 8f695c806bac2b044f102c55b802e44f2d30ca01 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 11 Apr 2023 20:59:30 +0100 Subject: clone: skip timeout tests on WinHTTP --- tests/libgit2/online/clone.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/libgit2/online/clone.c b/tests/libgit2/online/clone.c index 92fa4d351..dbcac50ae 100644 --- a/tests/libgit2/online/clone.c +++ b/tests/libgit2/online/clone.c @@ -1218,6 +1218,9 @@ void test_online_clone__sha256(void) void test_online_clone__connect_timeout_configurable(void) { +#ifdef GIT_WINHTTP + cl_skip(); +#else uint64_t start, finish; start = git_time_monotonic(); @@ -1229,10 +1232,14 @@ void test_online_clone__connect_timeout_configurable(void) finish = git_time_monotonic(); cl_assert(finish - start < 1000); +#endif } void test_online_clone__connect_timeout_default(void) { +#ifdef GIT_WINHTTP + cl_skip(); +#else /* This test takes ~ 75 seconds on Unix. */ if (!cl_is_env_set("GITTEST_INVASIVE_SPEED")) cl_skip(); @@ -1243,10 +1250,14 @@ void test_online_clone__connect_timeout_default(void) */ cl_git_fail_with(GIT_TIMEOUT, git_clone(&g_repo, "http://www.google.com:8000/", "./refused", NULL)); cl_assert(git_error_last() && strstr(git_error_last()->message, "timed out")); +#endif } void test_online_clone__timeout_configurable_times_out(void) { +#ifdef GIT_WINHTTP + cl_skip(); +#else git_repository *failed_repo; if (!_remote_speed_timesout) @@ -1256,14 +1267,19 @@ void test_online_clone__timeout_configurable_times_out(void) cl_git_fail_with(GIT_TIMEOUT, git_clone(&failed_repo, _remote_speed_timesout, "./timedout", NULL)); cl_assert(git_error_last() && strstr(git_error_last()->message, "timed out")); +#endif } void test_online_clone__timeout_configurable_succeeds_slowly(void) { +#ifdef GIT_WINHTTP + cl_skip(); +#else if (!_remote_speed_slow) cl_skip(); cl_git_pass(git_libgit2_opts(GIT_OPT_SET_SERVER_TIMEOUT, 1000)); cl_git_pass(git_clone(&g_repo, _remote_speed_slow, "./slow-but-successful", NULL)); +#endif } -- cgit v1.2.1