diff options
-rw-r--r-- | ChangeLog | 4 | ||||
-rw-r--r-- | configure.ac | 6 | ||||
-rw-r--r-- | debuginfod/ChangeLog | 6 | ||||
-rw-r--r-- | debuginfod/debuginfod.cxx | 59 | ||||
-rw-r--r-- | doc/ChangeLog | 4 | ||||
-rw-r--r-- | doc/debuginfod.8 | 11 |
6 files changed, 83 insertions, 7 deletions
@@ -1,3 +1,7 @@ +2023-01-11 Frank Ch. Eigler <fche@redhat.com> + + * configure.ac: Add some rlimit/affinity checks. + 2022-11-02 Mark Wielaard <mark@klomp.org> * configure.ac (AC_INIT): Set version to 0.188. diff --git a/configure.ac b/configure.ac index b2597f8b..8fe8baee 100644 --- a/configure.ac +++ b/configure.ac @@ -455,6 +455,12 @@ AS_IF([test "x$ac_cv_func_mremap" = "xno"], AC_CHECK_HEADERS([error.h]) AC_CHECK_HEADERS([err.h]) +dnl for debuginfod concurrency heuristics +AC_CHECK_HEADERS([sched.h]) +AC_CHECK_FUNCS([sched_getaffinity]) +AC_CHECK_HEADERS([sys/resource.h]) +AC_CHECK_FUNCS([getrlimit]) + old_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS -D_GNU_SOURCE" AC_FUNC_STRERROR_R() diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog index 8f9637f5..fc44e039 100644 --- a/debuginfod/ChangeLog +++ b/debuginfod/ChangeLog @@ -3,6 +3,12 @@ * debuginfod-client.c (debuginfod_query_server): Use CURL_AT_LEAST_VERSION(7.85.0) for CURLOPT_PROTOCOLS_STR. +2023-01-11 Frank Ch. Eigler <fche@redhat.com> + + PR29975 & PR29976 + * debuginfod.cxx (default_concurrency): New function to guess a + reasonable default for -c/-C on large but constrained machines. + 2022-12-21 Mark Wielaard <mark@klomp.org> * debuginfod-client.c: Define CURL_AT_LEAST_VERSION. diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx index 02a11477..4271acf4 100644 --- a/debuginfod/debuginfod.cxx +++ b/debuginfod/debuginfod.cxx @@ -32,6 +32,18 @@ #include "config.h" #endif +// #define _GNU_SOURCE +#ifdef HAVE_SCHED_H +extern "C" { +#include <sched.h> +} +#endif +#ifdef HAVE_SYS_RESOURCE_H +extern "C" { +#include <sys/resource.h> +} +#endif + extern "C" { #include "printversion.h" #include "system.h" @@ -398,6 +410,8 @@ static const char args_doc[] = "[PATH ...]"; /* Prototype for option handler. */ static error_t parse_opt (int key, char *arg, struct argp_state *state); +static unsigned default_concurrency(); + /* Data structure to communicate with argp functions. */ static struct argp argp = { @@ -418,7 +432,7 @@ static unsigned http_port = 8002; static unsigned rescan_s = 300; static unsigned groom_s = 86400; static bool maxigroom = false; -static unsigned concurrency = std::thread::hardware_concurrency() ?: 1; +static unsigned concurrency = default_concurrency(); static int connection_pool = 0; static set<string> source_paths; static bool scan_files = false; @@ -4082,6 +4096,47 @@ static void sqlite3_sharedprefix_fn (sqlite3_context* c, int argc, sqlite3_value } +static unsigned +default_concurrency() // guaranteed >= 1 +{ + // Prior to PR29975 & PR29976, we'd just use this: + unsigned sth = std::thread::hardware_concurrency(); + // ... but on many-CPU boxes, admins or distros may throttle + // resources in such a way that debuginfod would mysteriously fail. + // So we reduce the defaults: + + unsigned aff = 0; +#ifdef HAVE_SCHED_GETAFFINITY + { + int ret; + cpu_set_t mask; + CPU_ZERO(&mask); + ret = sched_getaffinity(0, sizeof(mask), &mask); + if (ret == 0) + aff = CPU_COUNT(&mask); + } +#endif + + unsigned fn = 0; +#ifdef HAVE_GETRLIMIT + { + struct rlimit rlim; + int rc = getrlimit(RLIMIT_NOFILE, &rlim); + if (rc == 0) + fn = max((rlim_t)1, (rlim.rlim_cur - 100) / 4); + // at least 2 fds are used by each listener thread etc. + // plus a bunch to account for shared libraries and such + } +#endif + + unsigned d = min(max(sth, 1U), + min(max(aff, 1U), + max(fn, 1U))); + return d; +} + + + int main (int argc, char *argv[]) { @@ -4207,7 +4262,7 @@ main (int argc, char *argv[]) /* If '-C' wasn't given or was given with no arg, pick a reasonable default for the number of worker threads. */ if (connection_pool == 0) - connection_pool = std::thread::hardware_concurrency() * 2 ?: 2; + connection_pool = default_concurrency(); /* Note that MHD_USE_EPOLL and MHD_USE_THREAD_PER_CONNECTION don't work together. */ diff --git a/doc/ChangeLog b/doc/ChangeLog index 7b18dbef..b2ef357a 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,7 @@ +2023-01-11 Frank Ch. Eigler <fche@redhat.com> + + * debuginfod.8: Tweak -c/-C documentation for heuristic defaults. + 2022-10-31 Aaron Merey <amerey@redhat.com> * Makefile.am (notrans_dist_man3_MANS): Add debuginfod_find_section.3. diff --git a/doc/debuginfod.8 b/doc/debuginfod.8 index 93db47e1..1aa8bc14 100644 --- a/doc/debuginfod.8 +++ b/doc/debuginfod.8 @@ -202,8 +202,9 @@ do maximal-grooming. See also the \fIDATA MANAGEMENT\fP section. Set the concurrency limit for the scanning queue threads, which work together to process archives & files located by the traversal thread. This important for controlling CPU-intensive operations like parsing -an ELF file and especially decompressing archives. The default is the -number of processors on the system; the minimum is 1. +an ELF file and especially decompressing archives. The default is +related to the number of processors on the system and other +constraints; the minimum is 1. .TP .B "\-C" "\-C=NUM" "\-\-connection\-pool" "\-\-connection\-pool=NUM" @@ -216,9 +217,9 @@ no option, \-C use a fixed thread pool sized automatically \-C=NUM use a fixed thread pool sized NUM, minimum 2 .TE -The first mode is a simple and safe configuration based on the number -of processors. The second mode is suitable for tuned load-limiting -configurations facing unruly traffic. +The first mode is a simple and safe configuration related to the +number of processors and other constraints. The second mode is +suitable for tuned load-limiting configurations facing unruly traffic. .TP .B "\-L" |