diff options
Diffstat (limited to 'debuginfod')
| -rw-r--r-- | debuginfod/ChangeLog | 22 | ||||
| -rw-r--r-- | debuginfod/Makefile.am | 2 | ||||
| -rw-r--r-- | debuginfod/debuginfod-client.c | 732 | ||||
| -rw-r--r-- | debuginfod/debuginfod-find.c | 131 | ||||
| -rw-r--r-- | debuginfod/debuginfod.cxx | 437 | ||||
| -rw-r--r-- | debuginfod/debuginfod.h.in | 30 | ||||
| -rw-r--r-- | debuginfod/libdebuginfod.map | 5 |
7 files changed, 1085 insertions, 274 deletions
diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog index 79f827d9..81ff4d4d 100644 --- a/debuginfod/ChangeLog +++ b/debuginfod/ChangeLog @@ -22,6 +22,28 @@ functions used in place of identical previously inline code. (debuginfod_find_metadata): New function. +2022-10-31 Aaron Merey <amerey@redhat.com> + + * Makefile.am (libdebuginfod_so_LDLIBS): Add libelf. + * debuginfod-client.c (debuginfod_find_section): New function. + (path_escape): New function. + (extract_section): New function. + (cache_find_section): New function. + (debuginfod_query_server): Add support for section queries. + * debuginfod-find.c (main): Add support for section queries. + * debuginfod.cxx (extract_section): New function. + (handle_buildid_f_match): Add section parameter. When non-empty, + try to create response from section contents. + (handle_buildid_r_match): Add section parameter. When non-empty, + try to create response from section contents. + (handle_buildid_match): Add section parameter. Pass to + handle_buildid_{f,r}_match. + (handle_buildid): Handle section name when artifacttype is set to + "section". Query upstream servers via debuginfod_find_section + when necessary. + (debuginfod.h.in): Add declaration for debuginfod_find_section. + (libdebuginfod.map): Add debuginfod_find_section. + 2022-10-18 Daniel Thornburgh <dthorn@google.com> * debuginfod-client.c (debuginfod_query_server): Add DEBUGINFOD_HEADERS_FILE diff --git a/debuginfod/Makefile.am b/debuginfod/Makefile.am index 3d6bc26e..602feb6c 100644 --- a/debuginfod/Makefile.am +++ b/debuginfod/Makefile.am @@ -97,7 +97,7 @@ libdebuginfod_so_LIBS = libdebuginfod_pic.a if DUMMY_LIBDEBUGINFOD libdebuginfod_so_LDLIBS = else -libdebuginfod_so_LDLIBS = -lpthread $(libcurl_LIBS) $(fts_LIBS) $(jsonc_LIBS) +libdebuginfod_so_LDLIBS = -lpthread $(libcurl_LIBS) $(fts_LIBS) $(libelf) $(jsonc_LIBS) endif $(LIBDEBUGINFOD_SONAME): $(srcdir)/libdebuginfod.map $(libdebuginfod_so_LIBS) $(AM_V_CCLD)$(LINK) $(dso_LDFLAGS) -o $@ \ diff --git a/debuginfod/debuginfod-client.c b/debuginfod/debuginfod-client.c index 4a75eef2..8167911f 100644 --- a/debuginfod/debuginfod-client.c +++ b/debuginfod/debuginfod-client.c @@ -1,5 +1,5 @@ /* Retrieve ELF / DWARF / source files from the debuginfod. - Copyright (C) 2019-2021 Red Hat, Inc. + Copyright (C) 2019-2023 Red Hat, Inc. Copyright (C) 2021, 2022 Mark J. Wielaard <mark@klomp.org> This file is part of elfutils. @@ -45,6 +45,7 @@ #include <ctype.h> #include <errno.h> #include <stdlib.h> +#include <gelf.h> /* We might be building a bootstrap dummy library, which is really simple. */ #ifdef DUMMY_LIBDEBUGINFOD @@ -56,8 +57,11 @@ int debuginfod_find_executable (debuginfod_client *c, const unsigned char *b, int s, char **p) { return -ENOSYS; } int debuginfod_find_source (debuginfod_client *c, const unsigned char *b, int s, const char *f, char **p) { return -ENOSYS; } +int debuginfod_find_section (debuginfod_client *c, const unsigned char *b, + int s, const char *scn, char **p) + { return -ENOSYS; } int debuginfod_find_metadata (debuginfod_client *c, - const char *k, const char* v, char** m) { return -ENOSYS; } + const char *k, const char *v, char **p) { return -ENOSYS; } void debuginfod_set_progressfn(debuginfod_client *c, debuginfod_progressfn_t fn) { } void debuginfod_set_verbose_fd(debuginfod_client *c, int fd) { } @@ -136,6 +140,9 @@ struct debuginfod_client debuginfod_end needs to terminate. */ int default_progressfn_printed_p; + /* Indicates whether the last query was cancelled by progressfn. */ + bool progressfn_cancel; + /* File descriptor to output any verbose messages if > 0. */ int verbose_fd; @@ -166,6 +173,11 @@ static const char *cache_miss_filename = "cache_miss_s"; static const char *cache_max_unused_age_filename = "max_unused_age_s"; static const long cache_default_max_unused_age_s = 604800; /* 1 week */ +/* The metadata_retention_default_s file within the debuginfod cache + specifies how long metadata query results should be cached. */ +static const long metadata_retention_default_s = 86400; /* 1 day */ +static const char *metadata_retention_filename = "metadata_retention_s"; + /* Location of the cache of files downloaded from debuginfods. The default parent directory is $HOME, or '/' if $HOME doesn't exist. */ static const char *cache_default_name = ".debuginfod_client_cache"; @@ -241,6 +253,8 @@ debuginfod_write_callback (char *ptr, size_t size, size_t nmemb, void *data) return (size_t) write(d->fd, (void*)ptr, count); } + + /* handle config file read and write */ static int debuginfod_config_cache(char *config_path, @@ -328,7 +342,8 @@ debuginfod_clean_cache(debuginfod_client *c, return -errno; regex_t re; - const char * pattern = ".*/[a-f0-9]+(/debuginfo|/executable|/source.*|)$"; /* include dirs */ + const char * pattern = ".*/(metadata.*|[a-f0-9]+(/debuginfo|/executable|/source.*|))$"; /* include dirs */ + /* NB: also includes .../section/ subdirs */ if (regcomp (&re, pattern, REG_EXTENDED | REG_NOSUB) != 0) return -ENOMEM; @@ -509,6 +524,7 @@ add_headers_from_file(debuginfod_client *client, const char* filename) } while (0) + /* Offer a basic form of progress tracing */ static int default_progressfn (debuginfod_client *c, long a, long b) @@ -576,6 +592,7 @@ header_callback (char * buffer, size_t size, size_t numitems, void * userdata) return numitems; } + #ifdef HAVE_JSON_C static size_t metadata_callback (char * buffer, size_t size, size_t numitems, void * userdata) @@ -588,7 +605,7 @@ metadata_callback (char * buffer, size_t size, size_t numitems, void * userdata) temp = realloc(data->metadata, data->metadata_size + numitems + 1); if (temp == NULL) return 0; - + memcpy(temp + data->metadata_size, buffer, numitems); data->metadata = temp; data->metadata_size += numitems; @@ -908,21 +925,336 @@ perform_queries(CURLM *curlm, CURL **target_handle, struct handle_data *data, de } +/* Copy SRC to DEST, s,/,#,g */ + +static void +path_escape (const char *src, char *dest) +{ + unsigned q = 0; + + for (unsigned fi=0; q < PATH_MAX-2; fi++) /* -2, escape is 2 chars. */ + switch (src[fi]) + { + case '\0': + dest[q] = '\0'; + q = PATH_MAX-1; /* escape for loop too */ + break; + case '/': /* escape / to prevent dir escape */ + dest[q++]='#'; + dest[q++]='#'; + break; + case '#': /* escape # to prevent /# vs #/ collisions */ + dest[q++]='#'; + dest[q++]='_'; + break; + default: + dest[q++]=src[fi]; + } + + dest[q] = '\0'; +} + +/* Attempt to read an ELF/DWARF section with name SECTION from FD and write + it to a separate file in the debuginfod cache. If successful the absolute + path of the separate file containing SECTION will be stored in USR_PATH. + FD_PATH is the absolute path for FD. + + If the section cannot be extracted, then return a negative error code. + -ENOENT indicates that the parent file was able to be read but the + section name was not found. -EEXIST indicates that the section was + found but had type SHT_NOBITS. */ + +int +extract_section (int fd, const char *section, char *fd_path, char **usr_path) +{ + elf_version (EV_CURRENT); + + Elf *elf = elf_begin (fd, ELF_C_READ_MMAP_PRIVATE, NULL); + if (elf == NULL) + return -EIO; + + size_t shstrndx; + int rc = elf_getshdrstrndx (elf, &shstrndx); + if (rc < 0) + { + rc = -EIO; + goto out; + } + + int sec_fd = -1; + char *escaped_name = NULL; + char *sec_path_tmp = NULL; + Elf_Scn *scn = NULL; + + /* Try to find the target section and copy the contents into a + separate file. */ + while (true) + { + scn = elf_nextscn (elf, scn); + if (scn == NULL) + { + rc = -ENOENT; + goto out; + } + GElf_Shdr shdr_storage; + GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage); + if (shdr == NULL) + { + rc = -EIO; + goto out; + } + + const char *scn_name = elf_strptr (elf, shstrndx, shdr->sh_name); + if (scn_name == NULL) + { + rc = -EIO; + goto out; + } + if (strcmp (scn_name, section) == 0) + { + /* We found the desired section. */ + if (shdr->sh_type == SHT_NOBITS) + { + rc = -EEXIST; + goto out; + } + + Elf_Data *data = NULL; + data = elf_rawdata (scn, NULL); + if (data == NULL) + { + rc = -EIO; + goto out; + } + + if (data->d_buf == NULL) + { + rc = -EIO; + goto out; + } + + /* Compute the absolute filename we'll write the section to. + Replace the last component of FD_PATH with the path-escaped + section filename. */ + int i = strlen (fd_path); + while (i >= 0) + { + if (fd_path[i] == '/') + { + fd_path[i] = '\0'; + break; + } + --i; + } + + escaped_name = malloc (strlen (section) * 2 + 1); + if (escaped_name == NULL) + { + rc = -ENOMEM; + goto out; + } + path_escape (section, escaped_name); + + rc = asprintf (&sec_path_tmp, "%s/section-%s.XXXXXX", + fd_path, escaped_name); + if (rc == -1) + { + rc = -ENOMEM; + goto out1; + } + + sec_fd = mkstemp (sec_path_tmp); + if (sec_fd < 0) + { + rc = -EIO; + goto out2; + } + + ssize_t res = write_retry (sec_fd, data->d_buf, data->d_size); + if (res < 0 || (size_t) res != data->d_size) + { + rc = -EIO; + goto out3; + } + + /* Success. Rename tmp file and update USR_PATH. */ + char *sec_path; + if (asprintf (&sec_path, "%s/section-%s", fd_path, section) == -1) + { + rc = -ENOMEM; + goto out3; + } + + rc = rename (sec_path_tmp, sec_path); + if (rc < 0) + { + free (sec_path); + rc = -EIO; + goto out3; + } + + if (usr_path != NULL) + *usr_path = sec_path; + else + free (sec_path); + rc = sec_fd; + goto out2; + } + } + +out3: + close (sec_fd); + unlink (sec_path_tmp); + +out2: + free (sec_path_tmp); + +out1: + free (escaped_name); + +out: + elf_end (elf); + return rc; +} + +/* Search TARGET_CACHE_DIR for a debuginfo or executable file containing + an ELF/DWARF section with name SCN_NAME. If found, extract the section + to a separate file in TARGET_CACHE_DIR and return a file descriptor + for the section file. The path for this file will be stored in USR_PATH. + Return a negative errno if unsuccessful. */ + +static int +cache_find_section (const char *scn_name, const char *target_cache_dir, + char **usr_path) +{ + int fd; + int rc = -EEXIST; + char parent_path[PATH_MAX]; + + /* Check the debuginfo first. */ + snprintf (parent_path, PATH_MAX, "%s/debuginfo", target_cache_dir); + fd = open (parent_path, O_RDONLY); + if (fd >= 0) + { + rc = extract_section (fd, scn_name, parent_path, usr_path); + close (fd); + } + + /* If the debuginfo file couldn't be found or the section type was + SHT_NOBITS, check the executable. */ + if (rc == -EEXIST) + { + snprintf (parent_path, PATH_MAX, "%s/executable", target_cache_dir); + fd = open (parent_path, O_RDONLY); + + if (fd >= 0) + { + rc = extract_section (fd, scn_name, parent_path, usr_path); + close (fd); + } + } + + return rc; +} + + +/* Helper function to create client cache directory. + $XDG_CACHE_HOME takes priority over $HOME/.cache. + $DEBUGINFOD_CACHE_PATH takes priority over $HOME/.cache and $XDG_CACHE_HOME. + + Return resulting path name or NULL on error. Caller must free resulting string. + */ +static char * +make_cache_path(void) +{ + char* cache_path = NULL; + int rc = 0; + /* Determine location of the cache. The path specified by the debuginfod + cache environment variable takes priority. */ + char *cache_var = getenv(DEBUGINFOD_CACHE_PATH_ENV_VAR); + if (cache_var != NULL && strlen (cache_var) > 0) + xalloc_str (cache_path, "%s", cache_var); + else + { + /* If a cache already exists in $HOME ('/' if $HOME isn't set), then use + that. Otherwise use the XDG cache directory naming format. */ + xalloc_str (cache_path, "%s/%s", getenv ("HOME") ?: "/", cache_default_name); + + struct stat st; + if (stat (cache_path, &st) < 0) + { + char cachedir[PATH_MAX]; + char *xdg = getenv ("XDG_CACHE_HOME"); + + if (xdg != NULL && strlen (xdg) > 0) + snprintf (cachedir, PATH_MAX, "%s", xdg); + else + snprintf (cachedir, PATH_MAX, "%s/.cache", getenv ("HOME") ?: "/"); + + /* Create XDG cache directory if it doesn't exist. */ + if (stat (cachedir, &st) == 0) + { + if (! S_ISDIR (st.st_mode)) + { + rc = -EEXIST; + goto out1; + } + } + else + { + rc = mkdir (cachedir, 0700); + + /* Also check for EEXIST and S_ISDIR in case another client just + happened to create the cache. */ + if (rc < 0 + && (errno != EEXIST + || stat (cachedir, &st) != 0 + || ! S_ISDIR (st.st_mode))) + { + rc = -errno; + goto out1; + } + } + + free (cache_path); + xalloc_str (cache_path, "%s/%s", cachedir, cache_xdg_name); + } + } + + goto out; + + out1: + (void) rc; + free (cache_path); + cache_path = NULL; + + out: + if (cache_path != NULL) + (void) mkdir (cache_path, 0700); + return cache_path; +} + + + /* Query each of the server URLs found in $DEBUGINFOD_URLS for the file - with the specified build-id, type (debuginfo, executable or source) - and filename. filename may be NULL. If found, return a file - descriptor for the target, otherwise return an error code. + with the specified build-id and type (debuginfo, executable, source or + section). If type is source, then type_arg should be a filename. If + type is section, then type_arg should be the name of an ELF/DWARF + section. Otherwise type_arg may be NULL. Return a file descriptor + for the target if successful, otherwise return an error code. */ static int debuginfod_query_server_by_buildid (debuginfod_client *c, const unsigned char *build_id, int build_id_len, const char *type, - const char *filename, + const char *type_arg, char **path) { char *server_urls; char *urls_envvar; + const char *section = NULL; + const char *filename = NULL; char *cache_path = NULL; char *maxage_path = NULL; char *interval_path = NULL; @@ -935,6 +1267,17 @@ debuginfod_query_server_by_buildid (debuginfod_client *c, int vfd = c->verbose_fd; int rc, r; + c->progressfn_cancel = false; + + if (strcmp (type, "source") == 0) + filename = type_arg; + else if (strcmp (type, "section") == 0) + { + section = type_arg; + if (section == NULL) + return -EINVAL; + } + if (vfd >= 0) { dprintf (vfd, "debuginfod_find_%s ", type); @@ -1033,31 +1376,13 @@ debuginfod_query_server_by_buildid (debuginfod_client *c, goto out; } - /* copy the filename to suffix, s,/,#,g */ - unsigned q = 0; - for (unsigned fi=0; q < PATH_MAX-2; fi++) /* -2, escape is 2 chars. */ - switch (filename[fi]) - { - case '\0': - suffix[q] = '\0'; - q = PATH_MAX-1; /* escape for loop too */ - break; - case '/': /* escape / to prevent dir escape */ - suffix[q++]='#'; - suffix[q++]='#'; - break; - case '#': /* escape # to prevent /# vs #/ collisions */ - suffix[q++]='#'; - suffix[q++]='_'; - break; - default: - suffix[q++]=filename[fi]; - } - suffix[q] = '\0'; + path_escape (filename, suffix); /* If the DWARF filenames are super long, this could exceed PATH_MAX and truncate/collide. Oh well, that'll teach them! */ } + else if (section != NULL) + path_escape (section, suffix); else suffix[0] = '\0'; @@ -1065,71 +1390,26 @@ debuginfod_query_server_by_buildid (debuginfod_client *c, dprintf (vfd, "suffix %s\n", suffix); /* set paths needed to perform the query - - example format + example format: cache_path: $HOME/.cache target_cache_dir: $HOME/.cache/0123abcd target_cache_path: $HOME/.cache/0123abcd/debuginfo target_cache_path: $HOME/.cache/0123abcd/source#PATH#TO#SOURCE ? - - $XDG_CACHE_HOME takes priority over $HOME/.cache. - $DEBUGINFOD_CACHE_PATH takes priority over $HOME/.cache and $XDG_CACHE_HOME. */ - /* Determine location of the cache. The path specified by the debuginfod - cache environment variable takes priority. */ - char *cache_var = getenv(DEBUGINFOD_CACHE_PATH_ENV_VAR); - if (cache_var != NULL && strlen (cache_var) > 0) - xalloc_str (cache_path, "%s", cache_var); - else + cache_path = make_cache_path(); + if (!cache_path) { - /* If a cache already exists in $HOME ('/' if $HOME isn't set), then use - that. Otherwise use the XDG cache directory naming format. */ - xalloc_str (cache_path, "%s/%s", getenv ("HOME") ?: "/", cache_default_name); - - struct stat st; - if (stat (cache_path, &st) < 0) - { - char cachedir[PATH_MAX]; - char *xdg = getenv ("XDG_CACHE_HOME"); - - if (xdg != NULL && strlen (xdg) > 0) - snprintf (cachedir, PATH_MAX, "%s", xdg); - else - snprintf (cachedir, PATH_MAX, "%s/.cache", getenv ("HOME") ?: "/"); - - /* Create XDG cache directory if it doesn't exist. */ - if (stat (cachedir, &st) == 0) - { - if (! S_ISDIR (st.st_mode)) - { - rc = -EEXIST; - goto out; - } - } - else - { - rc = mkdir (cachedir, 0700); - - /* Also check for EEXIST and S_ISDIR in case another client just - happened to create the cache. */ - if (rc < 0 - && (errno != EEXIST - || stat (cachedir, &st) != 0 - || ! S_ISDIR (st.st_mode))) - { - rc = -errno; - goto out; - } - } - - free (cache_path); - xalloc_str (cache_path, "%s/%s", cachedir, cache_xdg_name); - } + rc = -ENOMEM; + goto out; } - xalloc_str (target_cache_dir, "%s/%s", cache_path, build_id_bytes); - xalloc_str (target_cache_path, "%s/%s%s", target_cache_dir, type, suffix); + (void) mkdir (target_cache_dir, 0700); // failures with this mkdir would be caught later too + + if (section != NULL) + xalloc_str (target_cache_path, "%s/%s-%s", target_cache_dir, type, suffix); + else + xalloc_str (target_cache_path, "%s/%s%s", target_cache_dir, type, suffix); xalloc_str (target_cache_tmppath, "%s.XXXXXX", target_cache_path); /* XXX combine these */ @@ -1213,6 +1493,18 @@ debuginfod_query_server_by_buildid (debuginfod_client *c, /* Ensure old 000-permission files are not lingering in the cache. */ unlink(target_cache_path); + if (section != NULL) + { + /* Try to extract the section from a cached file before querying + any servers. */ + rc = cache_find_section (section, target_cache_dir, path); + + /* If the section was found or confirmed to not exist, then we + are done. */ + if (rc >= 0 || rc == -ENOENT) + goto out; + } + long timeout = default_timeout; const char* timeout_envvar = getenv(DEBUGINFOD_TIMEOUT_ENV_VAR); if (timeout_envvar != NULL) @@ -1332,6 +1624,9 @@ debuginfod_query_server_by_buildid (debuginfod_client *c, snprintf(data[i].url, PATH_MAX, "%s/%s/%s/%s", server_url, build_id_bytes, type, escaped_string); } + else if (section) + snprintf(data[i].url, PATH_MAX, "%s/%s/%s/%s", server_url, + build_id_bytes, type, section); else snprintf(data[i].url, PATH_MAX, "%s/%s/%s", server_url, build_id_bytes, type); @@ -1412,6 +1707,8 @@ debuginfod_query_server_by_buildid (debuginfod_client *c, /* 406 signals that the requested file was too large */ if ( ok0 == CURLE_OK && resp_code == 406) rc = -EFBIG; + else if (section != NULL && resp_code == 503) + rc = -EINVAL; else rc = -ENOENT; break; @@ -1737,23 +2034,80 @@ int debuginfod_find_source(debuginfod_client *client, "source", filename, path); } +int +debuginfod_find_section (debuginfod_client *client, + const unsigned char *build_id, int build_id_len, + const char *section, char **path) +{ + int rc = debuginfod_query_server_by_buildid(client, build_id, build_id_len, + "section", section, path); + if (rc != -EINVAL) + return rc; + + /* The servers may have lacked support for section queries. Attempt to + download the debuginfo or executable containing the section in order + to extract it. */ + rc = -EEXIST; + int fd = -1; + char *tmp_path = NULL; + + fd = debuginfod_find_debuginfo (client, build_id, build_id_len, &tmp_path); + if (client->progressfn_cancel) + { + if (fd >= 0) + { + /* This shouldn't happen, but we'll check this condition + just in case. */ + close (fd); + free (tmp_path); + } + return -ENOENT; + } + if (fd > 0) + { + rc = extract_section (fd, section, tmp_path, path); + close (fd); + } + + if (rc == -EEXIST) + { + /* The section should be found in the executable. */ + fd = debuginfod_find_executable (client, build_id, + build_id_len, &tmp_path); + if (fd > 0) + { + rc = extract_section (fd, section, tmp_path, path); + close (fd); + } + } + + free (tmp_path); + return rc; +} + int debuginfod_find_metadata (debuginfod_client *client, - const char* key, const char* value, char** metadata) + const char* key, const char* value, char **path) { (void) client; (void) key; (void) value; + (void) path; - if (NULL == metadata) return -EINVAL; #ifdef HAVE_JSON_C - char *server_urls; - char *urls_envvar; - json_object *json_metadata = json_object_new_array(); + char *server_urls = NULL; + char *urls_envvar = NULL; + char *cache_path = NULL; + char *target_cache_dir = NULL; + char *target_cache_path = NULL; + char *target_cache_tmppath = NULL; + char *key_and_value = NULL; int rc = 0, r; int vfd = client->verbose_fd; - - if(NULL == json_metadata){ + struct handle_data *data = NULL; + + json_object *json_metadata = json_object_new_array(); + if(NULL == json_metadata) { rc = -ENOMEM; goto out; } @@ -1777,6 +2131,98 @@ int debuginfod_find_metadata (debuginfod_client *client, goto out; } + /* set paths needed to perform the query + example format: + cache_path: $HOME/.cache + target_cache_dir: $HOME/.cache/metadata + target_cache_path: $HOME/.cache/metadata/key=ENCODED&value=ENCODED + target_cache_path: $HOME/.cache/metadata/key=ENCODED&value=ENCODED.XXXXXX + */ + + // libcurl > 7.62ish has curl_url_set()/etc. to construct these things more properly. + // curl_easy_escape() is older + { + CURL *c = curl_easy_init(); + if (!c) { + rc = -ENOMEM; + goto out; + } + char *key_escaped = curl_easy_escape(c, key, 0); + char *value_escaped = curl_easy_escape(c, value, 0); + + // fallback to unescaped values in unlikely case of error + xalloc_str (key_and_value, "key=%s&value=%s", key_escaped ?: key, value_escaped ?: value); + curl_free(value_escaped); + curl_free(key_escaped); + curl_easy_cleanup(c); + } + + /* Check if we have a recent result already in the cache. */ + cache_path = make_cache_path(); + if (! cache_path) + goto out; + xalloc_str (target_cache_dir, "%s/metadata", cache_path); + (void) mkdir (target_cache_dir, 0700); + xalloc_str (target_cache_path, "%s/%s", target_cache_dir, key_and_value); + xalloc_str (target_cache_tmppath, "%s/%s.XXXXXX", target_cache_dir, key_and_value); + + int fd = open(target_cache_path, O_RDONLY); + if (fd >= 0) + { + struct stat st; + int metadata_retention = 0; + time_t now = time(NULL); + char *metadata_retention_path = NULL; + + xalloc_str (metadata_retention_path, "%s/%s", cache_path, metadata_retention_filename); + if (metadata_retention_path) + { + rc = debuginfod_config_cache(metadata_retention_path, + metadata_retention_default_s, &st); + free (metadata_retention_path); + if (rc < 0) + rc = 0; + } + else + rc = 0; + metadata_retention = rc; + + if (fstat(fd, &st) != 0) + { + rc = -errno; + close (fd); + goto out; + } + + if (metadata_retention > 0 && (now - st.st_mtime <= metadata_retention)) + { + if (client && client->verbose_fd >= 0) + dprintf (client->verbose_fd, "cached metadata %s", key_and_value); + + if (path != NULL) + { + *path = target_cache_path; // pass over the pointer + target_cache_path = NULL; // prevent free() in our own cleanup + } + + /* Success!!!! */ + rc = fd; + goto out; + } + + /* We don't have to clear the likely-expired cached object here + by unlinking. We will shortly make a new request and save + results right on top. Erasing here could trigger a TOCTOU + race with another thread just finishing a query and passing + its results back. + */ + // (void) unlink (target_cache_path); + + close (fd); + } + + /* No valid cached metadata found: time to make the queries. */ + /* Clear the client of previous urls*/ free (client->url); client->url = NULL; @@ -1805,11 +2251,12 @@ int debuginfod_find_metadata (debuginfod_client *client, rc = -ENOMEM; goto out; } + /* thereafter, goto out1 on error*/ char **server_url_list = NULL; char *server_url; - int num_urls; + int num_urls = 0; r = init_server_urls("metadata", server_urls, &server_url_list, &num_urls, vfd); if(0 != r){ rc = r; @@ -1820,7 +2267,7 @@ int debuginfod_find_metadata (debuginfod_client *client, assert (curlm != NULL); CURL *target_handle = NULL; - struct handle_data *data = malloc(sizeof(struct handle_data) * num_urls); + data = malloc(sizeof(struct handle_data) * num_urls); if (data == NULL) { rc = -ENOMEM; @@ -1829,7 +2276,6 @@ int debuginfod_find_metadata (debuginfod_client *client, /* thereafter, goto out2 on error. */ - /* Initialize handle_data */ for (int i = 0; i < num_urls; i++) { @@ -1844,19 +2290,7 @@ int debuginfod_find_metadata (debuginfod_client *client, data[i].metadata = NULL; data[i].metadata_size = 0; - // libcurl > 7.62ish has curl_url_set()/etc. to construct these things more properly. - // curl_easy_escape() is older - CURL *c = curl_easy_init(); - if (c) { - char *key_escaped = curl_easy_escape(c, key, 0); - char *value_escaped = curl_easy_escape(c, value, 0); - snprintf(data[i].url, PATH_MAX, "%s?key=%s&value=%s", server_url, - // fallback to unescaped values in unlikely case of error - key_escaped ?: key, value_escaped ?: value); - curl_free(value_escaped); - curl_free(key_escaped); - curl_easy_cleanup(c); - } + snprintf(data[i].url, PATH_MAX, "%s?%s", server_url, key_and_value); r = init_handle(client, metadata_callback, header_callback, &data[i], i, timeout, vfd); if(0 != r){ @@ -1870,7 +2304,7 @@ int debuginfod_find_metadata (debuginfod_client *client, if (vfd >= 0) dprintf (vfd, "Starting %d queries\n",num_urls); r = perform_queries(curlm, NULL, data, client, num_urls, maxtime, 0, false, vfd); - if(0 != r){ + if (0 != r) { rc = r; goto out2; } @@ -1880,19 +2314,23 @@ int debuginfod_find_metadata (debuginfod_client *client, available from servers which can be connected with no issues. If running with additional verbosity, the failure will be noted in stderr */ - /* Building the new json array from all the upstream data - and cleanup while at it - */ + /* Building the new json array from all the upstream data and + cleanup while at it. + */ for (int i = 0; i < num_urls; i++) { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ - if(NULL == data[i].metadata) + curl_easy_cleanup (data[i].handle); + free (data[i].response_data); + + if (NULL == data[i].metadata) { if (vfd >= 0) dprintf (vfd, "Query to %s failed with error message:\n\t\"%s\"\n", data[i].url, data[i].errbuf); continue; } + json_object *upstream_metadata = json_tokener_parse(data[i].metadata); if(NULL == upstream_metadata) continue; // Combine the upstream metadata into the json array @@ -1903,14 +2341,61 @@ int debuginfod_find_metadata (debuginfod_client *client, } json_object_put(upstream_metadata); - curl_easy_cleanup (data[i].handle); - free (data[i].response_data); free (data[i].metadata); } - *metadata = strdup(json_object_to_json_string_ext(json_metadata, JSON_C_TO_STRING_PRETTY)); + /* Because of race with cache cleanup / rmdir, try to mkdir/mkstemp up to twice. */ + for (int i=0; i<2; i++) { + /* (re)create target directory in cache */ + (void) mkdir(target_cache_dir, 0700); /* files will be 0400 later */ - free (data); + /* NB: write to a temporary file first, to avoid race condition of + multiple clients checking the cache, while a partially-written or empty + file is in there, being written from libcurl. */ + fd = mkstemp (target_cache_tmppath); + if (fd >= 0) break; + } + if (fd < 0) /* Still failed after two iterations. */ + { + rc = -errno; + goto out1; + } + + + /* Plop the complete json_metadata object into the cache. */ + const char* json_string = json_object_to_json_string_ext(json_metadata, JSON_C_TO_STRING_PRETTY); + if (json_string == NULL) + { + rc = -ENOMEM; + goto out1; + } + ssize_t res = write_retry (fd, json_string, strlen(json_string)); + (void) lseek(fd, 0, SEEK_SET); // rewind file so client can read it from the top + + /* NB: json_string is auto deleted when json_metadata object is nuked */ + if (res < 0 || (size_t) res != strlen(json_string)) + { + rc = -EIO; + goto out1; + } + /* PR27571: make cache files casually unwriteable; dirs are already 0700 */ + (void) fchmod(fd, 0400); + + /* rename tmp->real */ + rc = rename (target_cache_tmppath, target_cache_path); + if (rc < 0) + { + rc = -errno; + goto out1; + /* Perhaps we need not give up right away; could retry or something ... */ + } + + /* don't close fd - we're returning it */ + /* don't unlink the tmppath; it's already been renamed. */ + if (path != NULL) + *path = strdup(target_cache_path); + + rc = fd; goto out1; /* error exits */ @@ -1926,22 +2411,28 @@ out2: free (data[i].metadata); } } - free(data); out1: + free(data); + for (int i = 0; i < num_urls; ++i) free(server_url_list[i]); free(server_url_list); - free (server_urls); -/* general purpose exit */ out: + free (server_urls); json_object_put(json_metadata); /* Reset sent headers */ curl_slist_free_all (client->headers); client->headers = NULL; client->user_agent_set_p = 0; + free (target_cache_dir); + free (target_cache_path); + free (target_cache_tmppath); + free (key_and_value); + free (cache_path); + return rc; #else /* ! HAVE_JSON_C */ @@ -1949,6 +2440,7 @@ out: #endif } + /* Add an outgoing HTTP header. */ int debuginfod_add_http_header (debuginfod_client *client, const char* header) { diff --git a/debuginfod/debuginfod-find.c b/debuginfod/debuginfod-find.c index 4136c567..2df6436d 100644 --- a/debuginfod/debuginfod-find.c +++ b/debuginfod/debuginfod-find.c @@ -1,6 +1,6 @@ /* Command-line frontend for retrieving ELF / DWARF / source files from the debuginfod. - Copyright (C) 2019-2020 Red Hat, Inc. + Copyright (C) 2019-2023 Red Hat, Inc. This file is part of elfutils. This file is free software; you can redistribute it and/or modify @@ -52,8 +52,10 @@ static const char args_doc[] = N_("debuginfo BUILDID\n" "executable PATH\n" "source BUILDID /FILENAME\n" "source PATH /FILENAME\n" + "section BUILDID SECTION-NAME\n" + "section PATH SECTION-NAME\n" #ifdef HAVE_JSON_C - "metadata KEY VALUE" + "metadata KEY VALUE\n" #endif ); @@ -147,71 +149,54 @@ main(int argc, char** argv) return 1; } -#ifdef HAVE_JSON_C - if(strcmp(argv[remaining], "metadata") == 0){ - if (remaining+2 == argc) - { - fprintf(stderr, "Require KEY and VALUE for \"metadata\"\n"); - return 1; - } - - char* metadata; - int rc = debuginfod_find_metadata (client, argv[remaining+1], argv[remaining+2], - &metadata); - - if (rc < 0) - { - fprintf(stderr, "Server query failed: %s\n", strerror(-rc)); - return 1; - } - // Output the metadata to stdout - printf("%s\n", metadata); - free(metadata); - return 0; - } -#endif - /* If we were passed an ELF file name in the BUILDID slot, look in there. */ unsigned char* build_id = (unsigned char*) argv[remaining+1]; int build_id_len = 0; /* assume text */ - - int any_non_hex = 0; - int i; - for (i = 0; build_id[i] != '\0'; i++) - if ((build_id[i] >= '0' && build_id[i] <= '9') || - (build_id[i] >= 'a' && build_id[i] <= 'f')) - ; - else - any_non_hex = 1; - - int fd = -1; Elf* elf = NULL; - if (any_non_hex) /* raw build-id */ - { - fd = open ((char*) build_id, O_RDONLY); - if (fd < 0) - fprintf (stderr, "Cannot open %s: %s\n", build_id, strerror(errno)); - } - if (fd >= 0) - { - elf = dwelf_elf_begin (fd); - if (elf == NULL) - fprintf (stderr, "Cannot open as ELF file %s: %s\n", build_id, - elf_errmsg (-1)); - } - if (elf != NULL) + + /* Process optional buildid given via ELF file name, for some query types only. */ + if (strcmp(argv[remaining], "debuginfo") == 0 + || strcmp(argv[remaining], "executable") == 0 + || strcmp(argv[remaining], "source") == 0 + || strcmp(argv[remaining], "section") == 0) { - const void *extracted_build_id; - ssize_t s = dwelf_elf_gnu_build_id(elf, &extracted_build_id); - if (s > 0) + int any_non_hex = 0; + int i; + for (i = 0; build_id[i] != '\0'; i++) + if ((build_id[i] >= '0' && build_id[i] <= '9') || + (build_id[i] >= 'a' && build_id[i] <= 'f')) + ; + else + any_non_hex = 1; + + int fd = -1; + if (any_non_hex) /* raw build-id */ + { + fd = open ((char*) build_id, O_RDONLY); + if (fd < 0) + fprintf (stderr, "Cannot open %s: %s\n", build_id, strerror(errno)); + } + if (fd >= 0) { - /* Success: replace the build_id pointer/len with the binary blob - that elfutils is keeping for us. It'll remain valid until elf_end(). */ - build_id = (unsigned char*) extracted_build_id; - build_id_len = s; + elf = dwelf_elf_begin (fd); + if (elf == NULL) + fprintf (stderr, "Cannot open as ELF file %s: %s\n", build_id, + elf_errmsg (-1)); + } + if (elf != NULL) + { + const void *extracted_build_id; + ssize_t s = dwelf_elf_gnu_build_id(elf, &extracted_build_id); + if (s > 0) + { + /* Success: replace the build_id pointer/len with the binary blob + that elfutils is keeping for us. It'll remain valid until elf_end(). */ + build_id = (unsigned char*) extracted_build_id; + build_id_len = s; + } + else + fprintf (stderr, "Cannot extract build-id from %s: %s\n", build_id, elf_errmsg(-1)); } - else - fprintf (stderr, "Cannot extract build-id from %s: %s\n", build_id, elf_errmsg(-1)); } char *cache_name; @@ -239,6 +224,30 @@ main(int argc, char** argv) build_id, build_id_len, argv[remaining+2], &cache_name); } + else if (strcmp(argv[remaining], "section") == 0) + { + if (remaining+2 >= argc) + { + fprintf(stderr, + "If FILETYPE is \"section\" then a section name must be given\n"); + return 1; + } + rc = debuginfod_find_section(client, build_id, build_id_len, + argv[remaining+2], &cache_name); + } +#ifdef HAVE_JSON_C + else if (strcmp(argv[remaining], "metadata") == 0) /* no buildid! */ + { + if (remaining+2 == argc) + { + fprintf(stderr, "Require KEY and VALUE for \"metadata\"\n"); + return 1; + } + + rc = debuginfod_find_metadata (client, argv[remaining+1], argv[remaining+2], + &cache_name); + } +#endif else { argp_help (&argp, stderr, ARGP_HELP_USAGE, argv[0]); @@ -258,8 +267,6 @@ main(int argc, char** argv) debuginfod_end (client); if (elf) elf_end(elf); - if (fd >= 0) - close (fd); if (rc < 0) { diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx index 105c3908..0be01fba 100644 --- a/debuginfod/debuginfod.cxx +++ b/debuginfod/debuginfod.cxx @@ -1,5 +1,5 @@ /* Debuginfo-over-http server. - Copyright (C) 2019-2021 Red Hat, Inc. + Copyright (C) 2019-2023 Red Hat, Inc. Copyright (C) 2021, 2022 Mark J. Wielaard <mark@klomp.org> This file is part of elfutils. @@ -56,6 +56,7 @@ extern "C" { #include <unistd.h> #include <fcntl.h> #include <netdb.h> +#include <fnmatch.h> /* If fts.h is included before config.h, its indirect inclusions may not @@ -1149,66 +1150,6 @@ add_mhd_last_modified (struct MHD_Response *resp, time_t mtime) add_mhd_response_header (resp, "Cache-Control", "public"); } - - -static struct MHD_Response* -handle_buildid_f_match (bool internal_req_t, - int64_t b_mtime, - const string& b_source0, - int *result_fd) -{ - (void) internal_req_t; // ignored - int fd = open(b_source0.c_str(), O_RDONLY); - if (fd < 0) - throw libc_exception (errno, string("open ") + b_source0); - - // NB: use manual close(2) in error case instead of defer_dtor, because - // in the normal case, we want to hand the fd over to libmicrohttpd for - // file transfer. - - struct stat s; - int rc = fstat(fd, &s); - if (rc < 0) - { - close(fd); - throw libc_exception (errno, string("fstat ") + b_source0); - } - - if ((int64_t) s.st_mtime != b_mtime) - { - if (verbose) - obatched(clog) << "mtime mismatch for " << b_source0 << endl; - close(fd); - return 0; - } - - inc_metric ("http_responses_total","result","file"); - struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd); - if (r == 0) - { - if (verbose) - obatched(clog) << "cannot create fd-response for " << b_source0 << endl; - close(fd); - } - else - { - std::string file = b_source0.substr(b_source0.find_last_of("/")+1, b_source0.length()); - add_mhd_response_header (r, "Content-Type", "application/octet-stream"); - add_mhd_response_header (r, "X-DEBUGINFOD-SIZE", - to_string(s.st_size).c_str()); - add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str()); - add_mhd_last_modified (r, s.st_mtime); - if (verbose > 1) - obatched(clog) << "serving file " << b_source0 << endl; - /* libmicrohttpd will close it. */ - if (result_fd) - *result_fd = fd; - } - - return r; -} - - // quote all questionable characters of str for safe passage through a sh -c expansion. static string shell_escape(const string& str) @@ -1602,6 +1543,205 @@ public: }; static libarchive_fdcache fdcache; +/* Search ELF_FD for an ELF/DWARF section with name SECTION. + If found copy the section to a temporary file and return + its file descriptor, otherwise return -1. + + The temporary file's mtime will be set to PARENT_MTIME. + B_SOURCE should be a description of the parent file suitable + for printing to the log. */ + +static int +extract_section (int elf_fd, int64_t parent_mtime, + const string& b_source, const string& section) +{ + /* Search the fdcache. */ + struct stat fs; + int fd = fdcache.lookup (b_source, section); + if (fd >= 0) + { + if (fstat (fd, &fs) != 0) + { + if (verbose) + obatched (clog) << "cannot fstate fdcache " + << b_source << " " << section << endl; + close (fd); + return -1; + } + if ((int64_t) fs.st_mtime != parent_mtime) + { + if (verbose) + obatched(clog) << "mtime mismatch for " + << b_source << " " << section << endl; + close (fd); + return -1; + } + /* Success. */ + return fd; + } + + Elf *elf = elf_begin (elf_fd, ELF_C_READ_MMAP_PRIVATE, NULL); + if (elf == NULL) + return -1; + + /* Try to find the section and copy the contents into a separate file. */ + try + { + size_t shstrndx; + int rc = elf_getshdrstrndx (elf, &shstrndx); + if (rc < 0) + throw elfutils_exception (rc, "getshdrstrndx"); + + Elf_Scn *scn = NULL; + while (true) + { + scn = elf_nextscn (elf, scn); + if (scn == NULL) + break; + GElf_Shdr shdr_storage; + GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage); + if (shdr == NULL) + break; + + const char *scn_name = elf_strptr (elf, shstrndx, shdr->sh_name); + if (scn_name == NULL) + break; + if (scn_name == section) + { + Elf_Data *data = NULL; + + /* We found the desired section. */ + data = elf_rawdata (scn, NULL); + if (data == NULL) + throw elfutils_exception (elf_errno (), "elfraw_data"); + if (data->d_buf == NULL) + { + obatched(clog) << "section " << section + << " is empty" << endl; + break; + } + + /* Create temporary file containing the section. */ + char *tmppath = NULL; + rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str()); + if (rc < 0) + throw libc_exception (ENOMEM, "cannot allocate tmppath"); + defer_dtor<void*,void> tmmpath_freer (tmppath, free); + fd = mkstemp (tmppath); + if (fd < 0) + throw libc_exception (errno, "cannot create temporary file"); + ssize_t res = write_retry (fd, data->d_buf, data->d_size); + if (res < 0 || (size_t) res != data->d_size) + throw libc_exception (errno, "cannot write to temporary file"); + + /* Set mtime to be the same as the parent file's mtime. */ + struct timeval tvs[2]; + if (fstat (elf_fd, &fs) != 0) + throw libc_exception (errno, "cannot fstat file"); + + tvs[0].tv_sec = tvs[1].tv_sec = fs.st_mtime; + tvs[0].tv_usec = tvs[1].tv_usec = 0; + (void) futimes (fd, tvs); + + /* Add to fdcache. */ + fdcache.intern (b_source, section, tmppath, data->d_size, true); + break; + } + } + } + catch (const reportable_exception &e) + { + e.report (clog); + close (fd); + fd = -1; + } + + elf_end (elf); + return fd; +} + +static struct MHD_Response* +handle_buildid_f_match (bool internal_req_t, + int64_t b_mtime, + const string& b_source0, + const string& section, + int *result_fd) +{ + (void) internal_req_t; // ignored + int fd = open(b_source0.c_str(), O_RDONLY); + if (fd < 0) + throw libc_exception (errno, string("open ") + b_source0); + + // NB: use manual close(2) in error case instead of defer_dtor, because + // in the normal case, we want to hand the fd over to libmicrohttpd for + // file transfer. + + struct stat s; + int rc = fstat(fd, &s); + if (rc < 0) + { + close(fd); + throw libc_exception (errno, string("fstat ") + b_source0); + } + + if ((int64_t) s.st_mtime != b_mtime) + { + if (verbose) + obatched(clog) << "mtime mismatch for " << b_source0 << endl; + close(fd); + return 0; + } + + if (!section.empty ()) + { + int scn_fd = extract_section (fd, s.st_mtime, b_source0, section); + close (fd); + + if (scn_fd >= 0) + fd = scn_fd; + else + { + if (verbose) + obatched (clog) << "cannot find section " << section + << " for " << b_source0 << endl; + return 0; + } + + rc = fstat(fd, &s); + if (rc < 0) + { + close (fd); + throw libc_exception (errno, string ("fstat ") + b_source0 + + string (" ") + section); + } + } + + struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd); + inc_metric ("http_responses_total","result","file"); + if (r == 0) + { + if (verbose) + obatched(clog) << "cannot create fd-response for " << b_source0 + << " section=" << section << endl; + close(fd); + } + else + { + std::string file = b_source0.substr(b_source0.find_last_of("/")+1, b_source0.length()); + add_mhd_response_header (r, "Content-Type", "application/octet-stream"); + add_mhd_response_header (r, "X-DEBUGINFOD-SIZE", + to_string(s.st_size).c_str()); + add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str()); + add_mhd_last_modified (r, s.st_mtime); + if (verbose > 1) + obatched(clog) << "serving file " << b_source0 << " section=" << section << endl; + /* libmicrohttpd will close it. */ + if (result_fd) + *result_fd = fd; + } + + return r; +} // For security/portability reasons, many distro-package archives have // a "./" in front of path names; others have nothing, others have @@ -1627,6 +1767,7 @@ handle_buildid_r_match (bool internal_req_p, int64_t b_mtime, const string& b_source0, const string& b_source1, + const string& section, int *result_fd) { struct stat fs; @@ -1655,6 +1796,33 @@ handle_buildid_r_match (bool internal_req_p, break; // branch out of if "loop", to try new libarchive fetch attempt } + if (!section.empty ()) + { + int scn_fd = extract_section (fd, fs.st_mtime, + b_source0 + ":" + b_source1, + section); + close (fd); + if (scn_fd >= 0) + fd = scn_fd; + else + { + if (verbose) + obatched (clog) << "cannot find section " << section + << " for archive " << b_source0 + << " file " << b_source1 << endl; + return 0; + } + + rc = fstat(fd, &fs); + if (rc < 0) + { + close (fd); + throw libc_exception (errno, + string ("fstat archive ") + b_source0 + string (" file ") + b_source1 + + string (" section ") + section); + } + } + struct MHD_Response* r = MHD_create_response_from_fd (fs.st_size, fd); if (r == 0) { @@ -1673,7 +1841,9 @@ handle_buildid_r_match (bool internal_req_p, add_mhd_response_header (r, "X-DEBUGINFOD-FILE", b_source1.c_str()); add_mhd_last_modified (r, fs.st_mtime); if (verbose > 1) - obatched(clog) << "serving fdcache archive " << b_source0 << " file " << b_source1 << endl; + obatched(clog) << "serving fdcache archive " << b_source0 + << " file " << b_source1 + << " section=" << section << endl; /* libmicrohttpd will close it. */ if (result_fd) *result_fd = fd; @@ -1804,8 +1974,36 @@ handle_buildid_r_match (bool internal_req_p, tmppath, archive_entry_size(e), true); // requested ones go to the front of lru + if (!section.empty ()) + { + int scn_fd = extract_section (fd, b_mtime, + b_source0 + ":" + b_source1, + section); + close (fd); + if (scn_fd >= 0) + fd = scn_fd; + else + { + if (verbose) + obatched (clog) << "cannot find section " << section + << " for archive " << b_source0 + << " file " << b_source1 << endl; + return 0; + } + + rc = fstat(fd, &fs); + if (rc < 0) + { + close (fd); + throw libc_exception (errno, + string ("fstat ") + b_source0 + string (" ") + section); + } + r = MHD_create_response_from_fd (fs.st_size, fd); + } + else + r = MHD_create_response_from_fd (archive_entry_size(e), fd); + inc_metric ("http_responses_total","result",archive_extension + " archive"); - r = MHD_create_response_from_fd (archive_entry_size(e), fd); if (r == 0) { if (verbose) @@ -1825,7 +2023,9 @@ handle_buildid_r_match (bool internal_req_p, add_mhd_response_header (r, "X-DEBUGINFOD-FILE", file.c_str()); add_mhd_last_modified (r, archive_entry_mtime(e)); if (verbose > 1) - obatched(clog) << "serving archive " << b_source0 << " file " << b_source1 << endl; + obatched(clog) << "serving archive " << b_source0 + << " file " << b_source1 + << " section=" << section << endl; /* libmicrohttpd will close it. */ if (result_fd) *result_fd = fd; @@ -1896,14 +2096,17 @@ handle_buildid_match (bool internal_req_p, const string& b_stype, const string& b_source0, const string& b_source1, + const string& section, int *result_fd) { try { if (b_stype == "F") - return handle_buildid_f_match(internal_req_p, b_mtime, b_source0, result_fd); + return handle_buildid_f_match(internal_req_p, b_mtime, b_source0, + section, result_fd); else if (b_stype == "R") - return handle_buildid_r_match(internal_req_p, b_mtime, b_source0, b_source1, result_fd); + return handle_buildid_r_match(internal_req_p, b_mtime, b_source0, + b_source1, section, result_fd); } catch (const reportable_exception &e) { @@ -1978,6 +2181,7 @@ handle_buildid (MHD_Connection* conn, if (artifacttype == "debuginfo") atype_code = "D"; else if (artifacttype == "executable") atype_code = "E"; else if (artifacttype == "source") atype_code = "S"; + else if (artifacttype == "section") atype_code = "I"; else { artifacttype = "invalid"; // PR28242 ensure http_resposes metrics don't propagate unclean user data throw reportable_exception("invalid artifacttype"); @@ -1985,7 +2189,17 @@ handle_buildid (MHD_Connection* conn, if (conn != 0) inc_metric("http_requests_total", "type", artifacttype); - + + string section; + if (atype_code == "I") + { + if (suffix.size () < 2) + throw reportable_exception ("invalid section suffix"); + + // Remove leading '/' + section = suffix.substr(1); + } + if (atype_code == "S" && suffix == "") throw reportable_exception("invalid source suffix"); @@ -2037,8 +2251,21 @@ handle_buildid (MHD_Connection* conn, pp->bind(2, suffix); pp->bind(3, canon_pathname(suffix)); } + else if (atype_code == "I") + { + pp = new sqlite_ps (thisdb, "mhd-query-i", + "select mtime, sourcetype, source0, source1, 1 as debug_p from " BUILDIDS "_query_d where buildid = ? " + "union all " + "select mtime, sourcetype, source0, source1, 0 as debug_p from " BUILDIDS "_query_e where buildid = ? " + "order by debug_p desc, mtime desc"); + pp->reset(); + pp->bind(1, buildid); + pp->bind(2, buildid); + } unique_ptr<sqlite_ps> ps_closer(pp); // release pp if exception or return + bool do_upstream_section_query = true; + // consume all the rows while (1) { @@ -2059,12 +2286,30 @@ handle_buildid (MHD_Connection* conn, // Try accessing the located match. // XXX: in case of multiple matches, attempt them in parallel? auto r = handle_buildid_match (conn ? false : true, - b_mtime, b_stype, b_source0, b_source1, result_fd); + b_mtime, b_stype, b_source0, b_source1, + section, result_fd); if (r) return r; + + // If a debuginfo file matching BUILDID was found but didn't contain + // the desired section, then the section should not exist. Don't + // bother querying upstream servers. + if (!section.empty () && (sqlite3_column_int (*pp, 4) == 1)) + { + struct stat st; + + // For "F" sourcetype, check if the debuginfo exists. For "R" + // sourcetype, check if the debuginfo was interned into the fdcache. + if ((b_stype == "F" && (stat (b_source0.c_str (), &st) == 0)) + || (b_stype == "R" && fdcache.probe (b_source0, b_source1))) + do_upstream_section_query = false; + } } pp->reset(); + if (!do_upstream_section_query) + throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found"); + // We couldn't find it in the database. Last ditch effort // is to defer to other debuginfo servers. @@ -2089,6 +2334,11 @@ handle_buildid (MHD_Connection* conn, fd = debuginfod_find_source (client, (const unsigned char*) buildid.c_str(), 0, suffix.c_str(), NULL); + else if (artifacttype == "section") + fd = debuginfod_find_section (client, + (const unsigned char*) buildid.c_str(), + 0, section.c_str(), NULL); + } else fd = -errno; /* Set by debuginfod_begin. */ @@ -2296,8 +2546,7 @@ handle_metadata (MHD_Connection* conn, MHD_Response* r; sqlite3 *thisdb = dbq; - // Query locally for matching e, d and s files - + // Query locally for matching e, d files string op; if (key == "glob") op = "glob"; @@ -2317,7 +2566,7 @@ handle_metadata (MHD_Connection* conn, "from " BUILDIDS "_f_de d2, " BUILDIDS "_files f2, " BUILDIDS "_buildids b2 " "where f2.id = d2.file and d2.buildid = b2.id and f2.name " + op + " ? " "union all \n" - // delegate to query_s for this one + // delegate to query_s for this one // XXX: tolerable sqlite effort? "select 0, 0, 1, q.buildid, q.artifactsrc " "from " BUILDIDS "_query_s as q " "where q.artifactsrc " + op + " ? "); @@ -2355,6 +2604,11 @@ handle_metadata (MHD_Connection* conn, string m_buildid = (const char*) sqlite3_column_text (*pp, 3) ?: ""; // should always be non-null string m_file = (const char*) sqlite3_column_text (*pp, 4) ?: ""; + // Confirm that m_file matches in the fnmatch(FNM_PATHNAME) + // sense, since sqlite's GLOB operator is a looser filter. + if (key == "glob" && fnmatch(value.c_str(), m_file.c_str(), FNM_PATHNAME) != 0) + continue; + auto add_metadata = [metadata, m_buildid, m_file](const string& type) { json_object* entry = json_object_new_object(); if (NULL == entry) throw libc_exception (ENOMEM, "cannot allocate json"); @@ -2372,39 +2626,62 @@ handle_metadata (MHD_Connection* conn, add_entry_metadata("type", type.c_str()); add_entry_metadata("buildid", m_buildid); add_entry_metadata("file", m_file); - json_object_array_add(metadata, json_object_get(entry)); // Increase ref count to switch its ownership + if (verbose > 3) + obatched(clog) << "metadata found local " + << json_object_to_json_string_ext(entry, + JSON_C_TO_STRING_PRETTY) + << endl; + + // Increase ref count to switch its ownership + json_object_array_add(metadata, json_object_get(entry)); }; if (m_executable_p) add_metadata("executable"); if (m_debuginfo_p) add_metadata("debuginfo"); if (m_source_p) add_metadata("source"); - } pp->reset(); + unsigned num_local_results = json_object_array_length(metadata); + // Query upstream as well debuginfod_client *client = debuginfod_pool_begin(); if (metadata && client != NULL) { add_client_federation_headers(client, conn); - char * upstream_metadata; - if (0 == debuginfod_find_metadata(client, key.c_str(), value.c_str(), &upstream_metadata)) { - json_object *upstream_metadata_json = json_tokener_parse(upstream_metadata); - if(NULL != upstream_metadata_json) + int upstream_metadata_fd; + upstream_metadata_fd = debuginfod_find_metadata(client, key.c_str(), value.c_str(), NULL); + if (upstream_metadata_fd >= 0) { + json_object *upstream_metadata_json = json_object_from_fd(upstream_metadata_fd); + if (NULL != upstream_metadata_json) { for (int i = 0, n = json_object_array_length(upstream_metadata_json); i < n; i++) { json_object *entry = json_object_array_get_idx(upstream_metadata_json, i); + if (verbose > 3) + obatched(clog) << "metadata found remote " + << json_object_to_json_string_ext(entry, + JSON_C_TO_STRING_PRETTY) + << endl; + json_object_get(entry); // increment reference count json_object_array_add(metadata, entry); } json_object_put(upstream_metadata_json); } - free(upstream_metadata); + close(upstream_metadata_fd); } debuginfod_pool_end (client); } + unsigned num_total_results = json_object_array_length(metadata); + + if (verbose > 2) + obatched(clog) << "metadata found local=" << num_local_results + << " remote=" << (num_total_results-num_local_results) + << " total=" << num_total_results + << endl; + const char* metadata_str = (metadata != NULL) ? json_object_to_json_string(metadata) : "[ ]" ; if (! metadata_str) @@ -2487,6 +2764,7 @@ handler_cb (void * /*cls*/, clock_gettime (CLOCK_MONOTONIC, &ts_start); double afteryou = 0.0; string artifacttype, suffix; + string urlargs; // for logging try { @@ -2564,6 +2842,7 @@ handler_cb (void * /*cls*/, if (NULL == value || NULL == key) throw reportable_exception("/metadata webapi error, need key and value"); + urlargs = string("?key=") + string(key) + string("&value=") + string(value); // apprx., for logging artifacttype = "metadata"; inc_metric("http_requests_total", "type", artifacttype); r = handle_metadata(connection, key, value, &http_size); @@ -2605,7 +2884,7 @@ handler_cb (void * /*cls*/, // afteryou: delay waiting for other client's identical query to complete // deltas: total latency, including afteryou waiting obatched(clog) << conninfo(connection) - << ' ' << method << ' ' << url + << ' ' << method << ' ' << url << urlargs << ' ' << http_code << ' ' << http_size << ' ' << (int)(afteryou*1000) << '+' << (int)((deltas-afteryou)*1000) << "ms" << endl; diff --git a/debuginfod/debuginfod.h.in b/debuginfod/debuginfod.h.in index 4aa38abb..cb2ffce9 100644 --- a/debuginfod/debuginfod.h.in +++ b/debuginfod/debuginfod.h.in @@ -59,9 +59,9 @@ debuginfod_client *debuginfod_begin (void); it is a binary blob of given length. If successful, return a file descriptor to the target, otherwise - return a posix error code. If successful, set *path to a - strdup'd copy of the name of the same file in the cache. - Caller must free() it later. */ + return a negative POSIX error code. If successful, set *path to a + strdup'd copy of the name of the same file in the cache. Caller + must free() it later. */ int debuginfod_find_debuginfo (debuginfod_client *client, const unsigned char *build_id, @@ -79,15 +79,23 @@ int debuginfod_find_source (debuginfod_client *client, const char *filename, char **path); -/* Query the urls contained in $DEBUGINFOD_URLS for metadata - with given query key/value. - - If successful, return 0, otherwise return a posix error code. - If successful, set *metadata to a malloc'd json array - with each entry being a json object of metadata for 1 file. - Caller must free() it later. metadata MUST be non-NULL. */ +int debuginfod_find_section (debuginfod_client *client, + const unsigned char *build_id, + int build_id_len, + const char *section, + char **path); + +/* Query the urls contained in $DEBUGINFOD_URLS for metadata + with given query key/value. + + If successful, return a file descriptor to the JSON document + describing matches, otherwise return a negative POSIX error code. If + successful, set *path to a strdup'd copy of the name of the same + file in the cache. Caller must free() it later. */ int debuginfod_find_metadata (debuginfod_client *client, - const char *key, const char* value, char** metadata); + const char *key, + const char* value, + char **path); typedef int (*debuginfod_progressfn_t)(debuginfod_client *c, long a, long b); void debuginfod_set_progressfn(debuginfod_client *c, diff --git a/debuginfod/libdebuginfod.map b/debuginfod/libdebuginfod.map index 6e4fe4b5..355a89fd 100644 --- a/debuginfod/libdebuginfod.map +++ b/debuginfod/libdebuginfod.map @@ -20,5 +20,8 @@ ELFUTILS_0.183 { } ELFUTILS_0.179; ELFUTILS_0.188 { debuginfod_get_headers; - debuginfod_find_metadata; + debuginfod_find_section; } ELFUTILS_0.183; +ELFUTILS_0.190 { + debuginfod_find_metadata; +} ELFUTILS_0.188; |