diff options
Diffstat (limited to 'src/journal/sd-journal.c')
-rw-r--r-- | src/journal/sd-journal.c | 447 |
1 files changed, 311 insertions, 136 deletions
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c index 6da7bf8e81..11dbd83f2d 100644 --- a/src/journal/sd-journal.c +++ b/src/journal/sd-journal.c @@ -40,6 +40,7 @@ #include "fs-util.h" #include "hashmap.h" #include "hostname-util.h" +#include "id128-util.h" #include "io-util.h" #include "journal-def.h" #include "journal-file.h" @@ -51,6 +52,7 @@ #include "process-util.h" #include "replace-var.h" #include "stat-util.h" +#include "stat-util.h" #include "stdio-util.h" #include "string-util.h" #include "strv.h" @@ -820,15 +822,21 @@ static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direc } static int real_journal_next(sd_journal *j, direction_t direction) { - JournalFile *f, *new_file = NULL; - Iterator i; + JournalFile *new_file = NULL; + unsigned i, n_files; + const void **files; Object *o; int r; assert_return(j, -EINVAL); assert_return(!journal_pid_changed(j), -ECHILD); - ORDERED_HASHMAP_FOREACH(f, j->files, i) { + r = iterated_cache_get(j->files_cache, NULL, &files, &n_files); + if (r < 0) + return r; + + for (i = 0; i < n_files; i++) { + JournalFile *f = (JournalFile *)files[i]; bool found; r = next_beyond_location(j, f, direction); @@ -1133,7 +1141,6 @@ _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) { return 1; } - _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) { assert_return(j, -EINVAL); assert_return(!journal_pid_changed(j), -ECHILD); @@ -1180,22 +1187,12 @@ _public_ int sd_journal_seek_tail(sd_journal *j) { } static void check_network(sd_journal *j, int fd) { - struct statfs sfs; - assert(j); if (j->on_network) return; - if (fstatfs(fd, &sfs) < 0) - return; - - j->on_network = - F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) || - F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) || - F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) || - F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) || - F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC); + j->on_network = fd_is_network_fs(fd); } static bool file_has_type_prefix(const char *prefix, const char *filename) { @@ -1246,6 +1243,16 @@ static bool path_has_prefix(sd_journal *j, const char *path, const char *prefix) return path_startswith(path, prefix); } +static void track_file_disposition(sd_journal *j, JournalFile *f) { + assert(j); + assert(f); + + if (!j->has_runtime_files && path_has_prefix(j, f->path, "/run")) + j->has_runtime_files = true; + else if (!j->has_persistent_files && path_has_prefix(j, f->path, "/var")) + j->has_persistent_files = true; +} + static const char *skip_slash(const char *p) { if (!p) @@ -1257,76 +1264,125 @@ static const char *skip_slash(const char *p) { return p; } -static int add_any_file(sd_journal *j, int fd, const char *path) { - JournalFile *f = NULL; +static int add_any_file( + sd_journal *j, + int fd, + const char *path) { + bool close_fd = false; + JournalFile *f; + struct stat st; int r, k; assert(j); assert(fd >= 0 || path); - if (path && ordered_hashmap_get(j->files, path)) - return 0; + if (fd < 0) { + if (j->toplevel_fd >= 0) + /* If there's a top-level fd defined make the path relative, explicitly, since otherwise + * openat() ignores the first argument. */ - if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) { - log_debug("Too many open journal files, not adding %s.", path); - r = -ETOOMANYREFS; - goto fail; + fd = openat(j->toplevel_fd, skip_slash(path), O_RDONLY|O_CLOEXEC|O_NONBLOCK); + else + fd = open(path, O_RDONLY|O_CLOEXEC|O_NONBLOCK); + if (fd < 0) { + r = log_debug_errno(errno, "Failed to open journal file %s: %m", path); + goto finish; + } + + close_fd = true; + + r = fd_nonblock(fd, false); + if (r < 0) { + r = log_debug_errno(errno, "Failed to turn off O_NONBLOCK for %s: %m", path); + goto finish; + } } - if (fd < 0 && j->toplevel_fd >= 0) { + if (fstat(fd, &st) < 0) { + r = log_debug_errno(errno, "Failed to fstat file '%s': %m", path); + goto finish; + } - /* If there's a top-level fd defined, open the file relative to this now. (Make the path relative, - * explicitly, since otherwise openat() ignores the first argument.) */ + r = stat_verify_regular(&st); + if (r < 0) { + log_debug_errno(r, "Refusing to open '%s', as it is not a regular file.", path); + goto finish; + } - fd = openat(j->toplevel_fd, skip_slash(path), O_RDONLY|O_CLOEXEC); - if (fd < 0) { - r = log_debug_errno(errno, "Failed to open journal file %s: %m", path); - goto fail; + f = ordered_hashmap_get(j->files, path); + if (f) { + if (f->last_stat.st_dev == st.st_dev && + f->last_stat.st_ino == st.st_ino) { + + /* We already track this file, under the same path and with the same device/inode numbers, it's + * hence really the same. Mark this file as seen in this generation. This is used to GC old + * files in process_q_overflow() to detect journal files that are still there and discern them + * from those which are gone. */ + + f->last_seen_generation = j->generation; + r = 0; + goto finish; } - close_fd = true; + /* So we tracked a file under this name, but it has a different inode/device. In that case, it got + * replaced (probably due to rotation?), let's drop it hence from our list. */ + remove_file_real(j, f); + f = NULL; + } + + if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) { + log_debug("Too many open journal files, not adding %s.", path); + r = -ETOOMANYREFS; + goto finish; } r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f); if (r < 0) { - if (close_fd) - safe_close(fd); log_debug_errno(r, "Failed to open journal file %s: %m", path); - goto fail; + goto finish; } /* journal_file_dump(f); */ r = ordered_hashmap_put(j->files, f->path, f); if (r < 0) { - f->close_fd = close_fd; + f->close_fd = false; /* make sure journal_file_close() doesn't close the caller's fd (or our own). We'll let the caller do that, or ourselves */ (void) journal_file_close(f); - goto fail; + goto finish; } - if (!j->has_runtime_files && path_has_prefix(j, f->path, "/run")) - j->has_runtime_files = true; - else if (!j->has_persistent_files && path_has_prefix(j, f->path, "/var")) - j->has_persistent_files = true; + close_fd = false; /* the fd is now owned by the JournalFile object */ - log_debug("File %s added.", f->path); + f->last_seen_generation = j->generation; + track_file_disposition(j, f); check_network(j, f->fd); j->current_invalidate_counter++; - return 0; + log_debug("File %s added.", f->path); -fail: - k = journal_put_error(j, r, path); - if (k < 0) - return k; + r = 0; + +finish: + if (close_fd) + safe_close(fd); + + if (r < 0) { + k = journal_put_error(j, r, path); + if (k < 0) + return k; + } return r; } -static int add_file(sd_journal *j, const char *prefix, const char *filename) { +static int add_file_by_name( + sd_journal *j, + const char *prefix, + const char *filename) { + const char *path; assert(j); @@ -1343,7 +1399,11 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) { return add_any_file(j, -1, path); } -static void remove_file(sd_journal *j, const char *prefix, const char *filename) { +static void remove_file_by_name( + sd_journal *j, + const char *prefix, + const char *filename) { + const char *path; JournalFile *f; @@ -1363,7 +1423,7 @@ static void remove_file_real(sd_journal *j, JournalFile *f) { assert(j); assert(f); - ordered_hashmap_remove(j->files, f->path); + (void) ordered_hashmap_remove(j->files, f->path); log_debug("File %s removed.", f->path); @@ -1407,10 +1467,102 @@ static int dirname_is_machine_id(const char *fn) { return sd_id128_equal(id, machine); } +static bool dirent_is_journal_file(const struct dirent *de) { + assert(de); + + if (!IN_SET(de->d_type, DT_REG, DT_LNK, DT_UNKNOWN)) + return false; + + return endswith(de->d_name, ".journal") || + endswith(de->d_name, ".journal~"); +} + +static bool dirent_is_id128_subdir(const struct dirent *de) { + assert(de); + + if (!IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN)) + return false; + + return id128_is_valid(de->d_name); +} + +static int directory_open(sd_journal *j, const char *path, DIR **ret) { + DIR *d; + + assert(j); + assert(path); + assert(ret); + + if (j->toplevel_fd < 0) + d = opendir(path); + else + /* Open the specified directory relative to the toplevel fd. Enforce that the path specified is + * relative, by dropping the initial slash */ + d = xopendirat(j->toplevel_fd, skip_slash(path), 0); + if (!d) + return -errno; + + *ret = d; + return 0; +} + +static int add_directory(sd_journal *j, const char *prefix, const char *dirname); + +static void directory_enumerate(sd_journal *j, Directory *m, DIR *d) { + struct dirent *de; + + assert(j); + assert(m); + assert(d); + + FOREACH_DIRENT_ALL(de, d, goto fail) { + + if (dirent_is_journal_file(de)) + (void) add_file_by_name(j, m->path, de->d_name); + + if (m->is_root && dirent_is_id128_subdir(de)) + (void) add_directory(j, m->path, de->d_name); + } + + return; + +fail: + log_debug_errno(errno, "Failed to enumerate directory %s, ignoring: %m", m->path); +} + +static void directory_watch(sd_journal *j, Directory *m, int fd, uint32_t mask) { + int r; + + assert(j); + assert(m); + assert(fd >= 0); + + /* Watch this directory if that's enabled and if it not being watched yet. */ + + if (m->wd > 0) /* Already have a watch? */ + return; + if (j->inotify_fd < 0) /* Not watching at all? */ + return; + + m->wd = inotify_add_watch_fd(j->inotify_fd, fd, mask); + if (m->wd < 0) { + log_debug_errno(errno, "Failed to watch journal directory '%s', ignoring: %m", m->path); + return; + } + + r = hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m); + if (r == -EEXIST) + log_debug_errno(r, "Directory '%s' already being watched under a different path, ignoring: %m", m->path); + if (r < 0) { + log_debug_errno(r, "Failed to add watch for journal directory '%s' to hashmap, ignoring: %m", m->path); + (void) inotify_rm_watch(j->inotify_fd, m->wd); + m->wd = -1; + } +} + static int add_directory(sd_journal *j, const char *prefix, const char *dirname) { _cleanup_free_ char *path = NULL; _cleanup_closedir_ DIR *d = NULL; - struct dirent *de = NULL; Directory *m; int r, k; @@ -1429,22 +1581,16 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname) goto fail; } - log_debug("Considering directory %s.", path); + log_debug("Considering directory '%s'.", path); /* We consider everything local that is in a directory for the local machine ID, or that is stored in /run */ if ((j->flags & SD_JOURNAL_LOCAL_ONLY) && !((dirname && dirname_is_machine_id(dirname) > 0) || path_has_prefix(j, path, "/run"))) - return 0; - + return 0; - if (j->toplevel_fd < 0) - d = opendir(path); - else - /* Open the specified directory relative to the toplevel fd. Enforce that the path specified is - * relative, by dropping the initial slash */ - d = xopendirat(j->toplevel_fd, skip_slash(path), 0); - if (!d) { - r = log_debug_errno(errno, "Failed to open directory %s: %m", path); + r = directory_open(j, path, &d); + if (r < 0) { + log_debug_errno(r, "Failed to open directory '%s': %m", path); goto fail; } @@ -1471,26 +1617,17 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname) log_debug("Directory %s added.", m->path); } else if (m->is_root) - return 0; + return 0; /* Don't 'downgrade' from root directory */ - if (m->wd <= 0 && j->inotify_fd >= 0) { - /* Watch this directory, if it not being watched yet. */ + m->last_seen_generation = j->generation; - m->wd = inotify_add_watch_fd(j->inotify_fd, dirfd(d), - IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE| - IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM| - IN_ONLYDIR); + directory_watch(j, m, dirfd(d), + IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE| + IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM| + IN_ONLYDIR); - if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0) - inotify_rm_watch(j->inotify_fd, m->wd); - } - - FOREACH_DIRENT_ALL(de, d, r = log_debug_errno(errno, "Failed to read directory %s: %m", m->path); goto fail) { - - if (dirent_is_file_with_suffix(de, ".journal") || - dirent_is_file_with_suffix(de, ".journal~")) - (void) add_file(j, m->path, de->d_name); - } + if (!j->no_new_files) + directory_enumerate(j, m, d); check_network(j, dirfd(d)); @@ -1507,7 +1644,6 @@ fail: static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) { _cleanup_closedir_ DIR *d = NULL; - struct dirent *de; Directory *m; int r, k; @@ -1520,6 +1656,8 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) { if (p) { /* If there's a path specified, use it. */ + log_debug("Considering root directory '%s'.", p); + if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) && !path_has_prefix(j, p, "/run")) return -EINVAL; @@ -1527,16 +1665,11 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) { if (j->prefix) p = strjoina(j->prefix, p); - if (j->toplevel_fd < 0) - d = opendir(p); - else - d = xopendirat(j->toplevel_fd, skip_slash(p), 0); - - if (!d) { - if (errno == ENOENT && missing_ok) - return 0; - - r = log_debug_errno(errno, "Failed to open root directory %s: %m", p); + r = directory_open(j, p, &d); + if (r == -ENOENT && missing_ok) + return 0; + if (r < 0) { + log_debug_errno(r, "Failed to open root directory %s: %m", p); goto fail; } } else { @@ -1594,29 +1727,12 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) { } else if (!m->is_root) return 0; - if (m->wd <= 0 && j->inotify_fd >= 0) { - - m->wd = inotify_add_watch_fd(j->inotify_fd, dirfd(d), - IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE| - IN_ONLYDIR); - - if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0) - inotify_rm_watch(j->inotify_fd, m->wd); - } - - if (j->no_new_files) - return 0; - - FOREACH_DIRENT_ALL(de, d, r = log_debug_errno(errno, "Failed to read directory %s: %m", m->path); goto fail) { - sd_id128_t id; + directory_watch(j, m, dirfd(d), + IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE| + IN_ONLYDIR); - if (dirent_is_file_with_suffix(de, ".journal") || - dirent_is_file_with_suffix(de, ".journal~")) - (void) add_file(j, m->path, de->d_name); - else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) && - sd_id128_from_string(de->d_name, &id) >= 0) - (void) add_directory(j, m->path, de->d_name); - } + if (!j->no_new_files) + directory_enumerate(j, m, d); check_network(j, dirfd(d)); @@ -1736,10 +1852,14 @@ static sd_journal *journal_new(int flags, const char *path) { j->path = t; } - j->files = ordered_hashmap_new(&string_hash_ops); - j->directories_by_path = hashmap_new(&string_hash_ops); + j->files = ordered_hashmap_new(&path_hash_ops); + if (!j->files) + goto fail; + + j->files_cache = ordered_hashmap_iterated_cache_new(j->files); + j->directories_by_path = hashmap_new(&path_hash_ops); j->mmap = mmap_cache_new(); - if (!j->files || !j->directories_by_path || !j->mmap) + if (!j->files_cache || !j->directories_by_path || !j->mmap) goto fail; return j; @@ -1950,10 +2070,9 @@ _public_ int sd_journal_open_files_fd(sd_journal **ret, int fds[], unsigned n_fd goto fail; } - if (!S_ISREG(st.st_mode)) { - r = -EBADFD; + r = stat_verify_regular(&st); + if (r < 0) goto fail; - } r = add_any_file(j, fds[i], NULL); if (r < 0) @@ -1985,6 +2104,7 @@ _public_ void sd_journal_close(sd_journal *j) { sd_journal_flush_matches(j); ordered_hashmap_free_with_destructor(j->files, journal_file_close); + iterated_cache_free(j->files_cache); while ((d = hashmap_first(j->directories_by_path))) remove_directory(j, d); @@ -2286,6 +2406,24 @@ _public_ void sd_journal_restart_data(sd_journal *j) { j->current_field = 0; } +static int reiterate_all_paths(sd_journal *j) { + assert(j); + + if (j->no_new_files) + return add_current_paths(j); + + if (j->flags & SD_JOURNAL_OS_ROOT) + return add_search_paths(j); + + if (j->toplevel_fd >= 0) + return add_root_directory(j, NULL, false); + + if (j->path) + return add_root_directory(j, j->path, true); + + return add_search_paths(j); +} + _public_ int sd_journal_get_fd(sd_journal *j) { int r; @@ -2302,20 +2440,10 @@ _public_ int sd_journal_get_fd(sd_journal *j) { if (r < 0) return r; - log_debug("Reiterating files to get inotify watches established"); + log_debug("Reiterating files to get inotify watches established."); - /* Iterate through all dirs again, to add them to the - * inotify */ - if (j->no_new_files) - r = add_current_paths(j); - else if (j->flags & SD_JOURNAL_OS_ROOT) - r = add_search_paths(j); - else if (j->toplevel_fd >= 0) - r = add_root_directory(j, NULL, false); - else if (j->path) - r = add_root_directory(j, j->path, true); - else - r = add_search_paths(j); + /* Iterate through all dirs again, to add them to the inotify */ + r = reiterate_all_paths(j); if (r < 0) return r; @@ -2358,17 +2486,61 @@ _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) { return 1; } +static void process_q_overflow(sd_journal *j) { + JournalFile *f; + Directory *m; + Iterator i; + + assert(j); + + /* When the inotify queue overruns we need to enumerate and re-validate all journal files to bring our list + * back in sync with what's on disk. For this we pick a new generation counter value. It'll be assigned to all + * journal files we encounter. All journal files and all directories that don't carry it after reenumeration + * are subject for unloading. */ + + log_debug("Inotify queue overrun, reiterating everything."); + + j->generation++; + (void) reiterate_all_paths(j); + + ORDERED_HASHMAP_FOREACH(f, j->files, i) { + + if (f->last_seen_generation == j->generation) + continue; + + log_debug("File '%s' hasn't been seen in this enumeration, removing.", f->path); + remove_file_real(j, f); + } + + HASHMAP_FOREACH(m, j->directories_by_path, i) { + + if (m->last_seen_generation == j->generation) + continue; + + if (m->is_root) /* Never GC root directories */ + continue; + + log_debug("Directory '%s' hasn't been seen in this enumeration, removing.", f->path); + remove_directory(j, m); + } + + log_debug("Reiteration complete."); +} + static void process_inotify_event(sd_journal *j, struct inotify_event *e) { Directory *d; assert(j); assert(e); + if (e->mask & IN_Q_OVERFLOW) { + process_q_overflow(j); + return; + } + /* Is this a subdirectory we watch? */ d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd)); if (d) { - sd_id128_t id; - if (!(e->mask & IN_ISDIR) && e->len > 0 && (endswith(e->name, ".journal") || endswith(e->name, ".journal~"))) { @@ -2376,9 +2548,9 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { /* Event for a journal file */ if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) - (void) add_file(j, d->path, e->name); + (void) add_file_by_name(j, d->path, e->name); else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) - remove_file(j, d->path, e->name); + remove_file_by_name(j, d->path, e->name); } else if (!d->is_root && e->len == 0) { @@ -2387,7 +2559,7 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) remove_directory(j, d); - } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) { + } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && id128_is_valid(e->name)) { /* Event for root directory */ @@ -2401,7 +2573,7 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) { if (e->mask & IN_IGNORED) return; - log_debug("Unknown inotify event."); + log_debug("Unexpected inotify event."); } static int determine_change(sd_journal *j) { @@ -2421,6 +2593,9 @@ _public_ int sd_journal_process(sd_journal *j) { assert_return(j, -EINVAL); assert_return(!journal_pid_changed(j), -ECHILD); + if (j->inotify_fd < 0) /* We have no inotify fd yet? Then there's noting to process. */ + return 0; + j->last_process_usec = now(CLOCK_MONOTONIC); j->last_invalidate_counter = j->current_invalidate_counter; |