diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/import/import-common.c | 87 | ||||
| -rw-r--r-- | src/import/import-common.h | 21 | ||||
| -rw-r--r-- | src/import/import-fs.c | 262 | ||||
| -rw-r--r-- | src/import/import-raw.c | 217 | ||||
| -rw-r--r-- | src/import/import-raw.h | 2 | ||||
| -rw-r--r-- | src/import/import-tar.c | 91 | ||||
| -rw-r--r-- | src/import/import.c | 379 | ||||
| -rw-r--r-- | src/import/pull-common.c | 220 | ||||
| -rw-r--r-- | src/import/pull-common.h | 32 | ||||
| -rw-r--r-- | src/import/pull-job.c | 177 | ||||
| -rw-r--r-- | src/import/pull-job.h | 11 | ||||
| -rw-r--r-- | src/import/pull-raw.c | 469 | ||||
| -rw-r--r-- | src/import/pull-raw.h | 2 | ||||
| -rw-r--r-- | src/import/pull-tar.c | 310 | ||||
| -rw-r--r-- | src/import/pull-tar.h | 2 | ||||
| -rw-r--r-- | src/import/pull.c | 404 | ||||
| -rw-r--r-- | src/shared/install-file.c | 269 | ||||
| -rw-r--r-- | src/shared/install-file.h | 14 | ||||
| -rw-r--r-- | src/shared/meson.build | 2 | ||||
| -rw-r--r-- | src/test/meson.build | 2 | ||||
| -rw-r--r-- | src/test/test-install-file.c | 72 |
21 files changed, 2178 insertions, 867 deletions
diff --git a/src/import/import-common.c b/src/import/import-common.c index f77564c41d..247f49a855 100644 --- a/src/import/import-common.c +++ b/src/import/import-common.c @@ -13,6 +13,7 @@ #include "fd-util.h" #include "fileio.h" #include "fs-util.h" +#include "hostname-util.h" #include "import-common.h" #include "os-util.h" #include "process-util.h" @@ -22,57 +23,6 @@ #include "tmpfile-util.h" #include "util.h" -int import_make_read_only_fd(int fd) { - struct stat st; - int r; - - assert(fd >= 0); - - /* First, let's make this a read-only subvolume if it refers - * to a subvolume */ - r = btrfs_subvol_set_read_only_fd(fd, true); - if (r >= 0) - return 0; - - if (!ERRNO_IS_NOT_SUPPORTED(r) && !IN_SET(r, -ENOTDIR, -EINVAL)) - return log_error_errno(r, "Failed to make subvolume read-only: %m"); - - /* This doesn't refer to a subvolume, or the file system isn't even btrfs. In that, case fall back to - * chmod()ing */ - - r = fstat(fd, &st); - if (r < 0) - return log_error_errno(errno, "Failed to stat image: %m"); - - if (S_ISDIR(st.st_mode)) { - /* For directories set the immutable flag on the dir itself */ - - r = chattr_fd(fd, FS_IMMUTABLE_FL, FS_IMMUTABLE_FL, NULL); - if (r < 0) - return log_error_errno(r, "Failed to set +i attribute on directory image: %m"); - - } else if (S_ISREG(st.st_mode)) { - /* For regular files drop "w" flags */ - - if ((st.st_mode & 0222) != 0) - if (fchmod(fd, st.st_mode & 07555) < 0) - return log_error_errno(errno, "Failed to chmod() image: %m"); - } else - return log_error_errno(SYNTHETIC_ERRNO(EBADFD), "Image of unexpected type"); - - return 0; -} - -int import_make_read_only(const char *path) { - _cleanup_close_ int fd = 1; - - fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC); - if (fd < 0) - return log_error_errno(errno, "Failed to open %s: %m", path); - - return import_make_read_only_fd(fd); -} - int import_fork_tar_x(const char *path, pid_t *ret) { _cleanup_close_pair_ int pipefd[2] = { -1, -1 }; bool use_selinux; @@ -327,3 +277,38 @@ int import_mangle_os_tree(const char *path) { return 0; } + +bool import_validate_local(const char *name, ImportFlags flags) { + + /* By default we insist on a valid hostname for naming images. But optionally we relax that, in which + * case it can be any path name */ + + if (FLAGS_SET(flags, IMPORT_DIRECT)) + return path_is_valid(name); + + return hostname_is_valid(name, 0); +} + +static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + log_notice("Transfer aborted."); + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +int import_allocate_event_with_signals(sd_event **ret) { + _cleanup_(sd_event_unrefp) sd_event *event = NULL; + int r; + + assert(ret); + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to allocate event loop: %m"); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + + *ret = TAKE_PTR(event); + return 0; +} diff --git a/src/import/import-common.h b/src/import/import-common.h index d7e8fc485f..97fc16d1b9 100644 --- a/src/import/import-common.h +++ b/src/import/import-common.h @@ -3,17 +3,26 @@ #include <sys/types.h> +#include "sd-event.h" + typedef enum ImportFlags { - IMPORT_FORCE = 1 << 0, /* replace existing image */ - IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */ + IMPORT_FORCE = 1 << 0, /* replace existing image */ + IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */ + IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */ + IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ + IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */ + IMPORT_DIRECT = 1 << 5, /* import without rename games */ + IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */ - IMPORT_FLAGS_MASK = IMPORT_FORCE|IMPORT_READ_ONLY, + IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC, + IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC, } ImportFlags; -int import_make_read_only_fd(int fd); -int import_make_read_only(const char *path); - int import_fork_tar_c(const char *path, pid_t *ret); int import_fork_tar_x(const char *path, pid_t *ret); int import_mangle_os_tree(const char *path); + +bool import_validate_local(const char *name, ImportFlags flags); + +int import_allocate_event_with_signals(sd_event **ret); diff --git a/src/import/import-fs.c b/src/import/import-fs.c index 88bfe29970..0c0d470563 100644 --- a/src/import/import-fs.c +++ b/src/import/import-fs.c @@ -12,15 +12,24 @@ #include "hostname-util.h" #include "import-common.h" #include "import-util.h" +#include "install-file.h" +#include "main-func.h" #include "mkdir.h" +#include "parse-argument.h" #include "ratelimit.h" #include "rm-rf.h" +#include "signal-util.h" #include "string-util.h" +#include "terminal-util.h" #include "tmpfile-util.h" #include "verbs.h" static bool arg_force = false; static bool arg_read_only = false; +static bool arg_btrfs_subvol = true; +static bool arg_btrfs_quota = true; +static bool arg_sync = true; +static bool arg_direct = false; static const char *arg_image_root = "/var/lib/machines"; typedef struct ProgressInfo { @@ -31,12 +40,6 @@ typedef struct ProgressInfo { bool logged_incomplete; } ProgressInfo; -static volatile sig_atomic_t cancelled = false; - -static void sigterm_sigint(int sig) { - cancelled = true; -} - static void progress_info_free(ProgressInfo *p) { free(p->path); } @@ -56,7 +59,7 @@ static void progress_show(ProgressInfo *p) { /* Mention the list is incomplete before showing first output. */ if (!p->logged_incomplete) { - log_notice("(Note, file list shown below is incomplete, and is intended as sporadic progress report only.)"); + log_notice("(Note: file list shown below is incomplete, and is intended as sporadic progress report only.)"); p->logged_incomplete = true; } @@ -72,9 +75,6 @@ static int progress_path(const char *path, const struct stat *st, void *userdata assert(p); - if (cancelled) - return -EOWNERDEAD; - r = free_and_strdup(&p->path, path); if (r < 0) return r; @@ -91,9 +91,6 @@ static int progress_bytes(uint64_t nbytes, void *userdata) { assert(p); assert(p->size != UINT64_MAX); - if (cancelled) - return -EOWNERDEAD; - p->size += nbytes; progress_show(p); @@ -103,44 +100,60 @@ static int progress_bytes(uint64_t nbytes, void *userdata) { static int import_fs(int argc, char *argv[], void *userdata) { _cleanup_(rm_rf_subvolume_and_freep) char *temp_path = NULL; _cleanup_(progress_info_free) ProgressInfo progress = {}; - const char *path = NULL, *local = NULL, *final_path; + _cleanup_free_ char *l = NULL, *final_path = NULL; + const char *path = NULL, *local = NULL, *dest = NULL; _cleanup_close_ int open_fd = -1; - struct sigaction old_sigint_sa, old_sigterm_sa; - static const struct sigaction sa = { - .sa_handler = sigterm_sigint, - .sa_flags = SA_RESTART, - }; int r, fd; if (argc >= 2) - path = argv[1]; - path = empty_or_dash_to_null(path); + path = empty_or_dash_to_null(argv[1]); if (argc >= 3) - local = argv[2]; - else if (path) - local = basename(path); - local = empty_or_dash_to_null(local); + local = empty_or_dash_to_null(argv[2]); + else if (path) { + r = path_extract_filename(path, &l); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from path '%s': %m", path); + + local = l; + } + + if (arg_direct) { + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified."); + + if (path_is_absolute(local)) + final_path = strdup(local); + else + final_path = path_join(arg_image_root, local); + if (!final_path) + return log_oom(); - if (local) { - if (!hostname_is_valid(local, 0)) + if (!path_is_valid(final_path)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local image name '%s' is not valid.", - local); + "Local path name '%s' is not valid.", final_path); + } else { + if (local) { + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local image name '%s' is not valid.", local); + } else + local = "imported"; + + final_path = path_join(arg_image_root, local); + if (!final_path) + return log_oom(); if (!arg_force) { r = image_find(IMAGE_MACHINE, local, NULL, NULL); if (r < 0) { if (r != -ENOENT) return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); - } else { + } else return log_error_errno(SYNTHETIC_ERRNO(EEXIST), - "Image '%s' already exists.", - local); - } + "Image '%s' already exists.", local); } - } else - local = "imported"; + } if (path) { open_fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC); @@ -159,84 +172,107 @@ static int import_fs(int argc, char *argv[], void *userdata) { log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); } - final_path = prefix_roota(arg_image_root, local); + if (!arg_sync) + log_info("File system synchronization on completion is off."); - r = tempfn_random(final_path, NULL, &temp_path); - if (r < 0) - return log_oom(); + if (arg_direct) { + if (arg_force) + (void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + dest = final_path; + } else { + r = tempfn_random(final_path, NULL, &temp_path); + if (r < 0) + return log_oom(); - (void) mkdir_parents_label(temp_path, 0700); + (void) mkdir_parents_label(temp_path, 0700); + dest = temp_path; + } progress.limit = (RateLimit) { 200*USEC_PER_MSEC, 1 }; - /* Hook into SIGINT/SIGTERM, so that we can cancel things then */ - assert_se(sigaction(SIGINT, &sa, &old_sigint_sa) >= 0); - assert_se(sigaction(SIGTERM, &sa, &old_sigterm_sa) >= 0); - - r = btrfs_subvol_snapshot_fd_full( - fd, - temp_path, - BTRFS_SNAPSHOT_FALLBACK_COPY|BTRFS_SNAPSHOT_RECURSIVE|BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|BTRFS_SNAPSHOT_QUOTA, - progress_path, - progress_bytes, - &progress); - if (r == -EOWNERDEAD) { /* SIGINT + SIGTERM cause this, see signal handler above */ - log_error("Copy cancelled."); - goto finish; - } - if (r < 0) { - log_error_errno(r, "Failed to copy directory: %m"); - goto finish; + { + BLOCK_SIGNALS(SIGINT, SIGTERM); + + if (arg_btrfs_subvol) + r = btrfs_subvol_snapshot_fd_full( + fd, + dest, + BTRFS_SNAPSHOT_FALLBACK_COPY| + BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| + BTRFS_SNAPSHOT_RECURSIVE| + BTRFS_SNAPSHOT_SIGINT| + BTRFS_SNAPSHOT_SIGTERM, + progress_path, + progress_bytes, + &progress); + else + r = copy_directory_fd_full( + fd, + dest, + COPY_REFLINK| + COPY_SAME_MOUNT| + COPY_HARDLINKS| + COPY_SIGINT| + COPY_SIGTERM| + (arg_direct ? COPY_MERGE_EMPTY : 0), + progress_path, + progress_bytes, + &progress); + if (r == -EINTR) /* SIGINT/SIGTERM hit */ + return log_error_errno(r, "Copy cancelled."); + if (r < 0) + return log_error_errno(r, "Failed to copy directory: %m"); } - r = import_mangle_os_tree(temp_path); + r = import_mangle_os_tree(dest); if (r < 0) - goto finish; - - (void) import_assign_pool_quota_and_warn(arg_image_root); - (void) import_assign_pool_quota_and_warn(temp_path); + return r; - if (arg_read_only) { - r = import_make_read_only(temp_path); - if (r < 0) { - log_error_errno(r, "Failed to make directory read-only: %m"); - goto finish; - } + if (arg_btrfs_quota) { + if (!arg_direct) + (void) import_assign_pool_quota_and_warn(arg_image_root); + (void) import_assign_pool_quota_and_warn(dest); } - if (arg_force) - (void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); - - r = rename_noreplace(AT_FDCWD, temp_path, AT_FDCWD, final_path); - if (r < 0) { - log_error_errno(r, "Failed to move image into place: %m"); - goto finish; - } + r = install_file(AT_FDCWD, dest, + AT_FDCWD, arg_direct ? NULL : final_path, /* pass NULL as target in case of direct + * mode since file is already in place */ + (arg_force ? INSTALL_REPLACE : 0) | + (arg_read_only ? INSTALL_READ_ONLY : 0) | + (arg_sync ? INSTALL_SYNCFS : 0)); + if (r < 0) + return log_error_errno(r, "Failed install directory as '%s': %m", final_path); temp_path = mfree(temp_path); - log_info("Exiting."); - -finish: - /* Put old signal handlers into place */ - assert_se(sigaction(SIGINT, &old_sigint_sa, NULL) >= 0); - assert_se(sigaction(SIGTERM, &old_sigterm_sa, NULL) >= 0); - + log_info("Directory '%s successfully installed. Exiting.", final_path); return 0; } static int help(int argc, char *argv[], void *userdata) { - printf("%s [OPTIONS...] {COMMAND} ...\n\n" - "Import container images from a file system.\n\n" + printf("%1$s [OPTIONS...] {COMMAND} ...\n" + "\n%4$sImport container images from a file system directories.%5$s\n" + "\n%2$sCommands:%3$s\n" + " run DIRECTORY [NAME] Import a directory\n" + "\n%2$sOptions:%3$s\n" " -h --help Show this help\n" " --version Show package version\n" " --force Force creation of image\n" " --image-root=PATH Image root directory\n" - " --read-only Create a read-only image\n\n" - "Commands:\n" - " run DIRECTORY [NAME] Import a directory\n", - program_invocation_short_name); + " --read-only Create a read-only image\n" + " --direct Import directly to specified directory\n" + " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" + " instead of a directory\n" + " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" + " subvolume\n" + " --sync=BOOL Controls whether to sync() before completing\n", + program_invocation_short_name, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); return 0; } @@ -248,6 +284,10 @@ static int parse_argv(int argc, char *argv[]) { ARG_FORCE, ARG_IMAGE_ROOT, ARG_READ_ONLY, + ARG_DIRECT, + ARG_BTRFS_SUBVOL, + ARG_BTRFS_QUOTA, + ARG_SYNC, }; static const struct option options[] = { @@ -256,10 +296,14 @@ static int parse_argv(int argc, char *argv[]) { { "force", no_argument, NULL, ARG_FORCE }, { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "direct", no_argument, NULL, ARG_DIRECT }, + { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, + { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, + { "sync", required_argument, NULL, ARG_SYNC }, {} }; - int c; + int c, r; assert(argc >= 0); assert(argv); @@ -286,6 +330,31 @@ static int parse_argv(int argc, char *argv[]) { arg_read_only = true; break; + case ARG_DIRECT: + arg_direct = true; + break; + + case ARG_BTRFS_SUBVOL: + r = parse_boolean_argument("--btrfs-subvol=", optarg, &arg_btrfs_subvol); + if (r < 0) + return r; + + break; + + case ARG_BTRFS_QUOTA: + r = parse_boolean_argument("--btrfs-quota=", optarg, &arg_btrfs_quota); + if (r < 0) + return r; + + break; + + case ARG_SYNC: + r = parse_boolean_argument("--sync=", optarg, &arg_sync); + if (r < 0) + return r; + + break; + case '?': return -EINVAL; @@ -307,7 +376,7 @@ static int import_fs_main(int argc, char *argv[]) { return dispatch_verb(argc, argv, verbs, NULL); } -int main(int argc, char *argv[]) { +static int run(int argc, char *argv[]) { int r; setlocale(LC_ALL, ""); @@ -316,10 +385,9 @@ int main(int argc, char *argv[]) { r = parse_argv(argc, argv); if (r <= 0) - goto finish; - - r = import_fs_main(argc, argv); + return r; -finish: - return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; + return import_fs_main(argc, argv); } + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/import-raw.c b/src/import/import-raw.c index 649cffb55f..9a87dd6f10 100644 --- a/src/import/import-raw.c +++ b/src/import/import-raw.c @@ -14,6 +14,7 @@ #include "import-common.h" #include "import-compress.h" #include "import-raw.h" +#include "install-file.h" #include "io-util.h" #include "machine-pool.h" #include "mkdir.h" @@ -52,23 +53,27 @@ struct RawImport { uint64_t written_compressed; uint64_t written_uncompressed; - struct stat st; + struct stat input_stat; + struct stat output_stat; unsigned last_percent; RateLimit progress_ratelimit; + + uint64_t offset; + uint64_t size_max; }; RawImport* raw_import_unref(RawImport *i) { if (!i) return NULL; - sd_event_unref(i->event); + sd_event_source_unref(i->input_event_source); unlink_and_free(i->temp_path); import_compress_free(&i->compress); - sd_event_source_unref(i->input_event_source); + sd_event_unref(i->event); safe_close(i->output_fd); @@ -107,6 +112,8 @@ int raw_import_new( .last_percent = UINT_MAX, .image_root = TAKE_PTR(root), .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 }, + .offset = UINT64_MAX, + .size_max = UINT64_MAX, }; if (event) @@ -118,7 +125,6 @@ int raw_import_new( } *ret = TAKE_PTR(i); - return 0; } @@ -127,13 +133,13 @@ static void raw_import_report_progress(RawImport *i) { assert(i); /* We have no size information, unless the source is a regular file */ - if (!S_ISREG(i->st.st_mode)) + if (!S_ISREG(i->input_stat.st_mode)) return; - if (i->written_compressed >= (uint64_t) i->st.st_size) + if (i->written_compressed >= (uint64_t) i->input_stat.st_size) percent = 100; else - percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size); if (percent == i->last_percent) return; @@ -149,11 +155,18 @@ static void raw_import_report_progress(RawImport *i) { static int raw_import_maybe_convert_qcow2(RawImport *i) { _cleanup_close_ int converted_fd = -1; - _cleanup_free_ char *t = NULL; + _cleanup_(unlink_and_freep) char *t = NULL; + _cleanup_free_ char *f = NULL; int r; assert(i); + /* Do QCOW2 conversion if enabled and not in direct mode */ + if ((i->flags & (IMPORT_CONVERT_QCOW2|IMPORT_DIRECT)) != IMPORT_CONVERT_QCOW2) + return 0; + + assert(i->final_path); + r = qcow2_detect(i->output_fd); if (r < 0) return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); @@ -161,26 +174,26 @@ static int raw_import_maybe_convert_qcow2(RawImport *i) { return 0; /* This is a QCOW2 image, let's convert it */ - r = tempfn_random(i->final_path, NULL, &t); + r = tempfn_random(i->final_path, NULL, &f); if (r < 0) return log_oom(); - converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (converted_fd < 0) - return log_error_errno(errno, "Failed to create %s: %m", t); + return log_error_errno(errno, "Failed to create %s: %m", f); + + t = TAKE_PTR(f); (void) import_set_nocow_and_log(converted_fd, t); log_info("Unpacking QCOW2 file."); r = qcow2_convert(i->output_fd, converted_fd); - if (r < 0) { - (void) unlink(t); + if (r < 0) return log_error_errno(r, "Failed to convert qcow2 image: %m"); - } - (void) unlink(i->temp_path); - free_and_replace(i->temp_path, t); + unlink_and_free(i->temp_path); + i->temp_path = TAKE_PTR(t); CLOSE_AND_REPLACE(i->output_fd, converted_fd); return 1; @@ -191,34 +204,45 @@ static int raw_import_finish(RawImport *i) { assert(i); assert(i->output_fd >= 0); - assert(i->temp_path); - assert(i->final_path); - /* In case this was a sparse file, make sure the file system is right */ - if (i->written_uncompressed > 0) { - if (ftruncate(i->output_fd, i->written_uncompressed) < 0) - return log_error_errno(errno, "Failed to truncate file: %m"); - } + /* Nothing of what is below applies to block devices */ + if (S_ISBLK(i->output_stat.st_mode)) { - r = raw_import_maybe_convert_qcow2(i); - if (r < 0) - return r; + if (i->flags & IMPORT_SYNC) { + if (fsync(i->output_fd) < 0) + return log_error_errno(errno, "Failed to synchronize block device: %m"); + } - if (S_ISREG(i->st.st_mode)) { - (void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME); - (void) copy_xattr(i->input_fd, i->output_fd, 0); + return 0; } - if (i->flags & IMPORT_READ_ONLY) { - r = import_make_read_only_fd(i->output_fd); + assert(S_ISREG(i->output_stat.st_mode)); + + /* If an offset is specified we only are supposed to affect part of an existing output file or block + * device, thus don't manipulate file properties in that case */ + + if (i->offset == UINT64_MAX) { + /* In case this was a sparse file, make sure the file size is right */ + if (i->written_uncompressed > 0) { + if (ftruncate(i->output_fd, i->written_uncompressed) < 0) + return log_error_errno(errno, "Failed to truncate file: %m"); + } + + r = raw_import_maybe_convert_qcow2(i); if (r < 0) return r; - } - if (i->flags & IMPORT_FORCE) - (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + if (S_ISREG(i->input_stat.st_mode)) { + (void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME); + (void) copy_xattr(i->input_fd, i->output_fd, 0); + } + } - r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + r = install_file(AT_FDCWD, i->temp_path ?: i->local, + AT_FDCWD, i->final_path, + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) return log_error_errno(r, "Failed to move image into place: %m"); @@ -231,26 +255,56 @@ static int raw_import_open_disk(RawImport *i) { int r; assert(i); - + assert(i->local); assert(!i->final_path); assert(!i->temp_path); assert(i->output_fd < 0); - i->final_path = strjoin(i->image_root, "/", i->local, ".raw"); - if (!i->final_path) - return log_oom(); + if (i->flags & IMPORT_DIRECT) { + (void) mkdir_parents_label(i->local, 0700); - r = tempfn_random(i->final_path, NULL, &i->temp_path); - if (r < 0) - return log_oom(); + /* In direct mode we just open/create the local path and truncate it (like shell > + * redirection would do it) — except if an offset was passed, in which case we are supposed + * to operate on a section of the file only, in which case we apparently work on an some + * existing thing (i.e. are not the sole thing stored in the file), in which case we will + * neither truncate nor create. */ + + i->output_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664); + if (i->output_fd < 0) + return log_error_errno(errno, "Failed to open destination '%s': %m", i->local); + + if (i->offset == UINT64_MAX) + (void) import_set_nocow_and_log(i->output_fd, i->local); + } else { + i->final_path = strjoin(i->image_root, "/", i->local, ".raw"); + if (!i->final_path) + return log_oom(); + + r = tempfn_random(i->final_path, NULL, &i->temp_path); + if (r < 0) + return log_oom(); - (void) mkdir_parents_label(i->temp_path, 0700); + (void) mkdir_parents_label(i->temp_path, 0700); - i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); - if (i->output_fd < 0) - return log_error_errno(errno, "Failed to open destination %s: %m", i->temp_path); + i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (i->output_fd < 0) + return log_error_errno(errno, "Failed to open destination '%s': %m", i->temp_path); + + (void) import_set_nocow_and_log(i->output_fd, i->temp_path); + } + + if (fstat(i->output_fd, &i->output_stat) < 0) + return log_error_errno(errno, "Failed to stat() output file: %m"); + + if (!S_ISREG(i->output_stat.st_mode) && !S_ISBLK(i->output_stat.st_mode)) + return log_error_errno(SYNTHETIC_ERRNO(EBADFD), + "Target file is not a regular file or block device"); + + if (i->offset != UINT64_MAX) { + if (lseek(i->output_fd, i->offset, SEEK_SET) == (off_t) -1) + return log_error_errno(errno, "Failed to seek to offset: %m"); + } - (void) import_set_nocow_and_log(i->output_fd, i->temp_path); return 0; } @@ -265,7 +319,7 @@ static int raw_import_try_reflink(RawImport *i) { if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED) return 0; - if (!S_ISREG(i->st.st_mode)) + if (!S_ISREG(i->input_stat.st_mode) || !S_ISREG(i->output_stat.st_mode)) return 0; p = lseek(i->input_fd, 0, SEEK_CUR); @@ -280,21 +334,57 @@ static int raw_import_try_reflink(RawImport *i) { if (r >= 0) return 1; + log_debug_errno(r, "Couldn't establish reflink, using copy: %m"); return 0; } static int raw_import_write(const void *p, size_t sz, void *userdata) { RawImport *i = userdata; - ssize_t n; + bool too_much = false; + int r; - n = sparse_write(i->output_fd, p, sz, 64); - if (n < 0) - return (int) n; - if ((size_t) n < sz) - return -EIO; + assert(i); + assert(p); + assert(sz > 0); + + if (i->written_uncompressed >= UINT64_MAX - sz) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (i->size_max != UINT64_MAX) { + if (i->written_uncompressed >= i->size_max) { + too_much = true; + goto finish; + } + + if (i->written_uncompressed + sz > i->size_max) { + too_much = true; + sz = i->size_max - i->written_uncompressed; /* since we have the data in memory + * already, we might as well write it to + * disk to the max */ + } + } + + /* Generate sparse file if we created/truncated the file */ + if (S_ISREG(i->output_stat.st_mode)) { + ssize_t n; + + n = sparse_write(i->output_fd, p, sz, 64); + if (n < 0) + return log_error_errno((int) n, "Failed to write file: %m"); + if ((size_t) n < sz) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); + } else { + r = loop_write(i->output_fd, p, sz, false); + if (r < 0) + return log_error_errno(r, "Failed to write file: %m"); + } i->written_uncompressed += sz; +finish: + if (too_much) + return log_error_errno(SYNTHETIC_ERRNO(E2BIG), "File too large"); + return 0; } @@ -382,15 +472,22 @@ static int raw_import_on_defer(sd_event_source *s, void *userdata) { return raw_import_process(i); } -int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags) { +int raw_import_start( + RawImport *i, + int fd, + const char *local, + uint64_t offset, + uint64_t size_max, + ImportFlags flags) { int r; assert(i); assert(fd >= 0); assert(local); - assert(!(flags & ~IMPORT_FLAGS_MASK)); + assert(!(flags & ~IMPORT_FLAGS_MASK_RAW)); + assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT)); - if (!hostname_is_valid(local, 0)) + if (!import_validate_local(local, flags)) return -EINVAL; if (i->input_fd >= 0) @@ -405,8 +502,10 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags) return r; i->flags = flags; + i->offset = offset; + i->size_max = size_max; - if (fstat(fd, &i->st) < 0) + if (fstat(fd, &i->input_stat) < 0) return -errno; r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i); @@ -422,5 +521,5 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags) return r; i->input_fd = fd; - return r; + return 0; } diff --git a/src/import/import-raw.h b/src/import/import-raw.h index e99703c155..63384eb171 100644 --- a/src/import/import-raw.h +++ b/src/import/import-raw.h @@ -16,4 +16,4 @@ RawImport* raw_import_unref(RawImport *import); DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref); -int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags); +int raw_import_start(RawImport *i, int fd, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags); diff --git a/src/import/import-tar.c b/src/import/import-tar.c index 7c4e5127a9..bb67862d62 100644 --- a/src/import/import-tar.c +++ b/src/import/import-tar.c @@ -15,6 +15,7 @@ #include "import-common.h" #include "import-compress.h" #include "import-tar.h" +#include "install-file.h" #include "io-util.h" #include "machine-pool.h" #include "mkdir.h" @@ -54,7 +55,7 @@ struct TarImport { uint64_t written_compressed; uint64_t written_uncompressed; - struct stat st; + struct stat input_stat; pid_t tar_pid; @@ -136,13 +137,13 @@ static void tar_import_report_progress(TarImport *i) { assert(i); /* We have no size information, unless the source is a regular file */ - if (!S_ISREG(i->st.st_mode)) + if (!S_ISREG(i->input_stat.st_mode)) return; - if (i->written_compressed >= (uint64_t) i->st.st_size) + if (i->written_compressed >= (uint64_t) i->input_stat.st_size) percent = 100; else - percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); + percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size); if (percent == i->last_percent) return; @@ -157,12 +158,11 @@ static void tar_import_report_progress(TarImport *i) { } static int tar_import_finish(TarImport *i) { + const char *d; int r; assert(i); assert(i->tar_fd >= 0); - assert(i->temp_path); - assert(i->final_path); i->tar_fd = safe_close(i->tar_fd); @@ -175,22 +175,20 @@ static int tar_import_finish(TarImport *i) { return -EPROTO; } - r = import_mangle_os_tree(i->temp_path); + assert_se(d = i->temp_path ?: i->local); + + r = import_mangle_os_tree(d); if (r < 0) return r; - if (i->flags & IMPORT_READ_ONLY) { - r = import_make_read_only(i->temp_path); - if (r < 0) - return r; - } - - if (i->flags & IMPORT_FORCE) - (void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); - - r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + r = install_file( + AT_FDCWD, d, + AT_FDCWD, i->final_path, + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) - return log_error_errno(r, "Failed to move image into place: %m"); + return log_error_errno(r, "Failed to move '%s' into place: %m", i->final_path ?: i->local); i->temp_path = mfree(i->temp_path); @@ -198,33 +196,54 @@ static int tar_import_finish(TarImport *i) { } static int tar_import_fork_tar(TarImport *i) { + const char *d, *root; int r; assert(i); - + assert(i->local); assert(!i->final_path); assert(!i->temp_path); assert(i->tar_fd < 0); - i->final_path = path_join(i->image_root, i->local); - if (!i->final_path) - return log_oom(); + if (i->flags & IMPORT_DIRECT) { + d = i->local; + root = NULL; + } else { + i->final_path = path_join(i->image_root, i->local); + if (!i->final_path) + return log_oom(); - r = tempfn_random(i->final_path, NULL, &i->temp_path); - if (r < 0) - return log_oom(); + r = tempfn_random(i->final_path, NULL, &i->temp_path); + if (r < 0) + return log_oom(); + + d = i->temp_path; + root = i->image_root; + } + + assert(d); - (void) mkdir_parents_label(i->temp_path, 0700); + (void) mkdir_parents_label(d, 0700); - r = btrfs_subvol_make_fallback(i->temp_path, 0755); + if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE)) + (void) rm_rf(d, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); + + if (i->flags & IMPORT_BTRFS_SUBVOL) + r = btrfs_subvol_make_fallback(d, 0755); + else + r = mkdir(d, 0755) < 0 ? -errno : 0; + if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise, + * because in that case our temporary path collided */ + r = 0; if (r < 0) - return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path); - if (r > 0) { /* actually btrfs subvol */ - (void) import_assign_pool_quota_and_warn(i->image_root); - (void) import_assign_pool_quota_and_warn(i->temp_path); + return log_error_errno(r, "Failed to create directory/subvolume %s: %m", d); + if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */ + if (!(i->flags & IMPORT_DIRECT)) + (void) import_assign_pool_quota_and_warn(root); + (void) import_assign_pool_quota_and_warn(d); } - i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); + i->tar_fd = import_fork_tar_x(d, &i->tar_pid); if (i->tar_fd < 0) return i->tar_fd; @@ -327,9 +346,9 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags) assert(i); assert(fd >= 0); assert(local); - assert(!(flags & ~IMPORT_FLAGS_MASK)); + assert(!(flags & ~IMPORT_FLAGS_MASK_TAR)); - if (!hostname_is_valid(local, 0)) + if (!import_validate_local(local, flags)) return -EINVAL; if (i->input_fd >= 0) @@ -345,7 +364,7 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags) i->flags = flags; - if (fstat(fd, &i->st) < 0) + if (fstat(fd, &i->input_stat) < 0) return -errno; r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i); @@ -361,5 +380,5 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags) return r; i->input_fd = fd; - return r; + return 0; } diff --git a/src/import/import.c b/src/import/import.c index cc90732d71..87ed8767e5 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -8,26 +8,114 @@ #include "alloc-util.h" #include "discover-image.h" +#include "env-util.h" #include "fd-util.h" #include "fs-util.h" #include "hostname-util.h" #include "import-raw.h" #include "import-tar.h" #include "import-util.h" +#include "io-util.h" #include "main-func.h" +#include "parse-argument.h" +#include "parse-util.h" #include "signal-util.h" #include "string-util.h" +#include "terminal-util.h" #include "verbs.h" static const char *arg_image_root = "/var/lib/machines"; -static ImportFlags arg_import_flags = 0; +static ImportFlags arg_import_flags = IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; +static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; -static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { - log_notice("Transfer aborted."); - sd_event_exit(sd_event_source_get_event(s), EINTR); +static int normalize_local(const char *local, char **ret) { + _cleanup_free_ char *ll = NULL; + int r; + + assert(ret); + + if (arg_import_flags & IMPORT_DIRECT) { + + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified."); + + if (!path_is_absolute(local)) { + ll = path_join(arg_image_root, local); + if (!ll) + return log_oom(); + + local = ll; + } + + if (!path_is_valid(local)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local path name '%s' is not valid.", local); + } else { + if (local) { + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local image name '%s' is not valid.", + local); + } else + local = "imported"; + + if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { + r = image_find(IMAGE_MACHINE, local, NULL, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else + return log_error_errno(SYNTHETIC_ERRNO(EEXIST), + "Image '%s' already exists.", + local); + } + } + + if (!ll) { + ll = strdup(local); + if (!ll) + return log_oom(); + } + + *ret = TAKE_PTR(ll); return 0; } +static int open_source(const char *path, const char *local, int *ret_open_fd) { + _cleanup_close_ int open_fd = -1; + int retval; + + assert(local); + assert(ret_open_fd); + + if (path) { + open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (open_fd < 0) + return log_error_errno(errno, "Failed to open raw image to import: %m"); + + retval = open_fd; + + if (arg_offset != UINT64_MAX) + log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", path, arg_offset, local); + else + log_info("Importing '%s', saving as '%s'.", path, local); + } else { + _cleanup_free_ char *pretty = NULL; + + retval = STDIN_FILENO; + + (void) fd_get_path(STDIN_FILENO, &pretty); + + if (arg_offset != UINT64_MAX) + log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", strempty(pretty), arg_offset, local); + else + log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); + } + + *ret_open_fd = TAKE_FD(open_fd); + return retval; +} + static void on_tar_finished(TarImport *import, int error, void *userdata) { sd_event *event = userdata; assert(import); @@ -40,9 +128,9 @@ static void on_tar_finished(TarImport *import, int error, void *userdata) { static int import_tar(int argc, char *argv[], void *userdata) { _cleanup_(tar_import_unrefp) TarImport *import = NULL; + _cleanup_free_ char *ll = NULL, *normalized = NULL; _cleanup_(sd_event_unrefp) sd_event *event = NULL; const char *path = NULL, *local = NULL; - _cleanup_free_ char *ll = NULL; _cleanup_close_ int open_fd = -1; int r, fd; @@ -51,64 +139,44 @@ static int import_tar(int argc, char *argv[], void *userdata) { if (argc >= 3) local = empty_or_dash_to_null(argv[2]); - else if (path) - local = basename(path); + else if (path) { + _cleanup_free_ char *l = NULL; - if (local) { - r = tar_strip_suffixes(local, &ll); + r = path_extract_filename(path, &l); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from path '%s': %m", path); + + r = tar_strip_suffixes(l, &ll); if (r < 0) return log_oom(); local = ll; + } - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local image name '%s' is not valid.", - local); - - if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); - if (r < 0) { - if (r != -ENOENT) - return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); - } else - return log_error_errno(SYNTHETIC_ERRNO(EEXIST), - "Image '%s' already exists.", - local); - } - } else - local = "imported"; - - if (path) { - open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); - if (open_fd < 0) - return log_error_errno(errno, "Failed to open tar image to import: %m"); - - fd = open_fd; - - log_info("Importing '%s', saving as '%s'.", path, local); - } else { - _cleanup_free_ char *pretty = NULL; + r = normalize_local(local, &normalized); + if (r < 0) + return r; - fd = STDIN_FILENO; + fd = open_source(path, normalized, &open_fd); + if (fd < 0) + return r; - (void) fd_get_path(fd, &pretty); - log_info("Importing '%s', saving as '%s'.", strna(pretty), local); - } - - r = sd_event_default(&event); + r = import_allocate_event_with_signals(&event); if (r < 0) - return log_error_errno(r, "Failed to allocate event loop: %m"); + return r; - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) + log_info("File system synchronization on completion is off."); r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate importer: %m"); - r = tar_import_start(import, fd, local, arg_import_flags); + r = tar_import_start( + import, + fd, + normalized, + arg_import_flags & IMPORT_FLAGS_MASK_TAR); if (r < 0) return log_error_errno(r, "Failed to import image: %m"); @@ -132,9 +200,9 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) { static int import_raw(int argc, char *argv[], void *userdata) { _cleanup_(raw_import_unrefp) RawImport *import = NULL; + _cleanup_free_ char *ll = NULL, *normalized = NULL; _cleanup_(sd_event_unrefp) sd_event *event = NULL; const char *path = NULL, *local = NULL; - _cleanup_free_ char *ll = NULL; _cleanup_close_ int open_fd = -1; int r, fd; @@ -143,64 +211,46 @@ static int import_raw(int argc, char *argv[], void *userdata) { if (argc >= 3) local = empty_or_dash_to_null(argv[2]); - else if (path) - local = basename(path); + else if (path) { + _cleanup_free_ char *l = NULL; + + r = path_extract_filename(path, &l); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from path '%s': %m", path); - if (local) { - r = raw_strip_suffixes(local, &ll); + r = raw_strip_suffixes(l, &ll); if (r < 0) return log_oom(); local = ll; + } - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local image name '%s' is not valid.", - local); - - if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); - if (r < 0) { - if (r != -ENOENT) - return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); - } else - return log_error_errno(SYNTHETIC_ERRNO(EEXIST), - "Image '%s' already exists.", - local); - } - } else - local = "imported"; - - if (path) { - open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); - if (open_fd < 0) - return log_error_errno(errno, "Failed to open raw image to import: %m"); - - fd = open_fd; - - log_info("Importing '%s', saving as '%s'.", path, local); - } else { - _cleanup_free_ char *pretty = NULL; - - fd = STDIN_FILENO; + r = normalize_local(local, &normalized); + if (r < 0) + return r; - (void) fd_get_path(fd, &pretty); - log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); - } + fd = open_source(path, normalized, &open_fd); + if (fd < 0) + return fd; - r = sd_event_default(&event); + r = import_allocate_event_with_signals(&event); if (r < 0) - return log_error_errno(r, "Failed to allocate event loop: %m"); + return r; - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) + log_info("File system synchronization on completion is off."); r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate importer: %m"); - r = raw_import_start(import, fd, local, arg_import_flags); + r = raw_import_start( + import, + fd, + normalized, + arg_offset, + arg_size_max, + arg_import_flags & IMPORT_FLAGS_MASK_RAW); if (r < 0) return log_error_errno(r, "Failed to import image: %m"); @@ -214,17 +264,32 @@ static int import_raw(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { - printf("%s [OPTIONS...] {COMMAND} ...\n\n" - "Import container or virtual machine images.\n\n" + printf("%1$s [OPTIONS...] {COMMAND} ...\n" + "\n%4$sImport container or virtual machine images.%5$s\n" + "\n%2$sCommands:%3$s\n" + " tar FILE [NAME] Import a TAR image\n" + " raw FILE [NAME] Import a RAW image\n" + "\n%2$sOptions:%3$s\n" " -h --help Show this help\n" " --version Show package version\n" " --force Force creation of image\n" " --image-root=PATH Image root directory\n" - " --read-only Create a read-only image\n\n" - "Commands:\n" - " tar FILE [NAME] Import a TAR image\n" - " raw FILE [NAME] Import a RAW image\n", - program_invocation_short_name); + " --read-only Create a read-only image\n" + " --direct Import directly to specified file\n" + " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" + " instead of a directory\n" + " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" + " subvolume\n" + " --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n" + " regular disk images\n" + " --sync=BOOL Controls whether to sync() before completing\n" + " --offset=BYTES Offset to seek to in destination\n" + " --size-max=BYTES Maximum number of bytes to write to destination\n", + program_invocation_short_name, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); return 0; } @@ -236,6 +301,13 @@ static int parse_argv(int argc, char *argv[]) { ARG_FORCE, ARG_IMAGE_ROOT, ARG_READ_ONLY, + ARG_DIRECT, + ARG_BTRFS_SUBVOL, + ARG_BTRFS_QUOTA, + ARG_CONVERT_QCOW2, + ARG_SYNC, + ARG_OFFSET, + ARG_SIZE_MAX, }; static const struct option options[] = { @@ -244,10 +316,17 @@ static int parse_argv(int argc, char *argv[]) { { "force", no_argument, NULL, ARG_FORCE }, { "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "direct", no_argument, NULL, ARG_DIRECT }, + { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, + { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, + { "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 }, + { "sync", required_argument, NULL, ARG_SYNC }, + { "offset", required_argument, NULL, ARG_OFFSET }, + { "size-max", required_argument, NULL, ARG_SIZE_MAX }, {} }; - int c; + int r, c; assert(argc >= 0); assert(argv); @@ -274,6 +353,68 @@ static int parse_argv(int argc, char *argv[]) { arg_import_flags |= IMPORT_READ_ONLY; break; + case ARG_DIRECT: + arg_import_flags |= IMPORT_DIRECT; + break; + + case ARG_BTRFS_SUBVOL: + r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r); + break; + + case ARG_BTRFS_QUOTA: + r = parse_boolean_argument("--btrfs-quota=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r); + break; + + case ARG_CONVERT_QCOW2: + r = parse_boolean_argument("--convert-qcow2=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_CONVERT_QCOW2, r); + break; + + case ARG_SYNC: + r = parse_boolean_argument("--sync=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_import_flags, IMPORT_SYNC, r); + break; + + case ARG_OFFSET: { + uint64_t u; + + r = safe_atou64(optarg, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg); + + arg_offset = u; + break; + } + + case ARG_SIZE_MAX: { + uint64_t u; + + r = parse_size(optarg, 1024, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg); + + arg_size_max = u; + break; + } + case '?': return -EINVAL; @@ -281,6 +422,15 @@ static int parse_argv(int argc, char *argv[]) { assert_not_reached(); } + /* Make sure offset+size is still in the valid range if both set */ + if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX && + ((arg_size_max > (UINT64_MAX - arg_offset)) || + !FILE_SIZE_VALID(arg_offset + arg_size_max))) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range."); + + if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); + return 1; } @@ -295,6 +445,31 @@ static int import_main(int argc, char *argv[]) { return dispatch_verb(argc, argv, verbs, NULL); } +static void parse_env(void) { + int r; + + /* Let's make these relatively low-level settings also controllable via env vars. User can then set + * them to systemd-import if they like to tweak behaviour */ + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL"); + if (r >= 0) + SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA"); + if (r >= 0) + SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_SYNC"); + if (r >= 0) + SET_FLAG(arg_import_flags, IMPORT_SYNC, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m"); +} + static int run(int argc, char *argv[]) { int r; @@ -302,9 +477,11 @@ static int run(int argc, char *argv[]) { log_parse_environment(); log_open(); + parse_env(); + r = parse_argv(argc, argv); if (r <= 0) - return 0; + return r; (void) ignore_signals(SIGPIPE); diff --git a/src/import/pull-common.c b/src/import/pull-common.c index e1cd495102..39067933b5 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -9,6 +9,7 @@ #include "dirent-util.h" #include "escape.h" #include "fd-util.h" +#include "hostname-util.h" #include "io-util.h" #include "memory-util.h" #include "path-util.h" @@ -112,34 +113,6 @@ int pull_find_old_etags( return 0; } -int pull_make_local_copy(const char *final, const char *image_root, const char *local, PullFlags flags) { - const char *p; - int r; - - assert(final); - assert(local); - - if (!image_root) - image_root = "/var/lib/machines"; - - p = prefix_roota(image_root, local); - - if (FLAGS_SET(flags, PULL_FORCE)) - (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); - - r = btrfs_subvol_snapshot(final, p, - BTRFS_SNAPSHOT_QUOTA| - BTRFS_SNAPSHOT_FALLBACK_COPY| - BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| - BTRFS_SNAPSHOT_RECURSIVE); - if (r < 0) - return log_error_errno(r, "Failed to create local image: %m"); - - log_info("Created new local image '%s'.", local); - - return 0; -} - static int hash_url(const char *url, char **ret) { uint64_t h; static const sd_id128_t k = SD_ID128_ARRAY(df,89,16,87,01,cc,42,30,98,ab,4a,19,a6,a5,63,4f); @@ -206,7 +179,9 @@ int pull_make_auxiliary_job( const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, + ImportVerify verify, CurlGlue *glue, + PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata) { @@ -238,45 +213,82 @@ int pull_make_auxiliary_job( if (r < 0) return r; + job->on_open_disk = on_open_disk; job->on_finished = on_finished; job->compressed_max = job->uncompressed_max = 1ULL * 1024ULL * 1024ULL; + job->calc_checksum = IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE); *ret = TAKE_PTR(job); - return 0; } +static bool is_checksum_file(const char *fn) { + /* Returns true if the specified filename refers to a checksum file we grok */ + + if (!fn) + return false; + + return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256"); +} + +static bool is_signature_file(const char *fn) { + /* Returns true if the specified filename refers to a signature file we grok (reminder: + * suse-style .sha256 files are inline signed) */ + + if (!fn) + return false; + + return streq(fn, "SHA256SUMS.gpg") || endswith(fn, ".sha256"); +} + int pull_make_verification_jobs( PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, + const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */ const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata) { _cleanup_(pull_job_unrefp) PullJob *checksum_job = NULL, *signature_job = NULL; + _cleanup_free_ char *fn = NULL; int r; assert(ret_checksum_job); assert(ret_signature_job); - assert(verify >= 0); - assert(verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); assert(url); assert(glue); - if (verify != IMPORT_VERIFY_NO) { - _cleanup_free_ char *checksum_url = NULL, *fn = NULL; - const char *chksums = NULL; + /* If verification is turned off, or if the checksum to validate is already specified we don't need + * to download a checksum file or signature, hence shortcut things */ + if (verify == IMPORT_VERIFY_NO || checksum) { + *ret_checksum_job = *ret_signature_job = NULL; + return 0; + } + + r = import_url_last_component(url, &fn); + if (r < 0 && r != -EADDRNOTAVAIL) /* EADDRNOTAVAIL means there was no last component, which is OK for + * us, we'll just assume it's not a checksum/signature file */ + return r; + + /* Acquire the checksum file if verification or signature verification is requested and the main file + * to acquire isn't a checksum or signature file anyway */ + if (verify != IMPORT_VERIFY_NO && !is_checksum_file(fn) && !is_signature_file(fn)) { + _cleanup_free_ char *checksum_url = NULL; + const char *suffixed = NULL; /* Queue jobs for the checksum file for the image. */ - r = import_url_last_component(url, &fn); - if (r < 0) - return r; - chksums = strjoina(fn, ".sha256"); + if (fn) + suffixed = strjoina(fn, ".sha256"); /* Start with the suse-style checksum (if there's a base filename) */ + else + suffixed = "SHA256SUMS"; - r = import_url_change_last_component(url, chksums, &checksum_url); + r = import_url_change_last_component(url, suffixed, &checksum_url); if (r < 0) return r; @@ -286,9 +298,10 @@ int pull_make_verification_jobs( checksum_job->on_finished = on_finished; checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL; + checksum_job->on_not_found = pull_job_restart_with_sha256sum; /* if this fails, look for ubuntu-style checksum */ } - if (verify == IMPORT_VERIFY_SIGNATURE) { + if (verify == IMPORT_VERIFY_SIGNATURE && !is_signature_file(fn)) { _cleanup_free_ char *signature_url = NULL; /* Queue job for the SHA256SUMS.gpg file for the image. */ @@ -306,7 +319,6 @@ int pull_make_verification_jobs( *ret_checksum_job = TAKE_PTR(checksum_job); *ret_signature_job = TAKE_PTR(signature_job); - return 0; } @@ -335,31 +347,33 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { r = import_url_last_component(job->url, &fn); if (r < 0) - return log_oom(); + return log_error_errno(r, "Failed to extract filename from URL '%s': %m", job->url); if (!filename_is_valid(fn)) return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "Cannot verify checksum, could not determine server-side file name."); - line = strjoina(job->checksum, " *", fn, "\n"); + if (is_checksum_file(fn) || is_signature_file(fn)) /* We cannot verify checksum files or signature files with a checksum file */ + return log_error_errno(SYNTHETIC_ERRNO(ELOOP), + "Cannot verify checksum/signature files via themselves."); + line = strjoina(job->checksum, " *", fn, "\n"); /* string for binary mode */ p = memmem_safe(checksum_job->payload, checksum_job->payload_size, line, strlen(line)); - if (!p) { - line = strjoina(job->checksum, " ", fn, "\n"); - + line = strjoina(job->checksum, " ", fn, "\n"); /* string for text mode */ p = memmem_safe(checksum_job->payload, checksum_job->payload_size, line, strlen(line)); } + /* Only counts if found at beginning of a line */ if (!p || (p != (char*) checksum_job->payload && p[-1] != '\n')) return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), - "DOWNLOAD INVALID: Checksum of %s file did not checkout, file has been tampered with.", fn); + "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", fn); log_info("SHA256 checksum of %s is valid.", job->url); return 1; @@ -472,10 +486,10 @@ static int verify_gpg( pid = 0; if (r < 0) goto finish; - if (r != EXIT_SUCCESS) { - log_error("DOWNLOAD INVALID: Signature verification failed."); - r = -EBADMSG; - } else { + if (r != EXIT_SUCCESS) + r = log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "DOWNLOAD INVALID: Signature verification failed."); + else { log_info("Signature verification succeeded."); r = 0; } @@ -491,6 +505,7 @@ finish: } int pull_verify(ImportVerify verify, + const char *checksum, /* Verify with literal checksum */ PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, @@ -499,37 +514,79 @@ int pull_verify(ImportVerify verify, PullJob *roothash_signature_job, PullJob *verity_job) { + _cleanup_free_ char *fn = NULL; VerificationStyle style; - PullJob *j; + PullJob *verify_job; int r; + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); assert(main_job); assert(main_job->state == PULL_JOB_DONE); - if (verify == IMPORT_VERIFY_NO) + if (verify == IMPORT_VERIFY_NO) /* verification turned off */ return 0; - assert(main_job->calc_checksum); - assert(main_job->checksum); - assert(checksum_job); - assert(checksum_job->state == PULL_JOB_DONE); + if (checksum) { + /* Verification by literal checksum */ + assert(!checksum_job); + assert(!signature_job); + assert(!settings_job); + assert(!roothash_job); + assert(!roothash_signature_job); + assert(!verity_job); - if (!checksum_job->payload || checksum_job->payload_size <= 0) - return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), - "Checksum is empty, cannot verify."); + assert(main_job->calc_checksum); + assert(main_job->checksum); - FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) { - r = verify_one(checksum_job, j); - if (r < 0) - return r; + if (!strcaseeq(checksum, main_job->checksum)) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", + main_job->url); + + return 0; } - if (verify == IMPORT_VERIFY_CHECKSUM) + r = import_url_last_component(main_job->url, &fn); + if (r < 0) + return log_error_errno(r, "Failed to extract filename from URL '%s': %m", main_job->url); + + if (is_signature_file(fn)) + return log_error_errno(SYNTHETIC_ERRNO(ELOOP), + "Main download is a signature file, can't verify it."); + + if (is_checksum_file(fn)) { + log_debug("Main download is a checksum file, can't validate its checksum with itself, skipping."); + verify_job = main_job; + } else { + PullJob *j; + assert(main_job->calc_checksum); + assert(main_job->checksum); + assert(checksum_job); + assert(checksum_job->state == PULL_JOB_DONE); + + if (!checksum_job->payload || checksum_job->payload_size <= 0) + return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), + "Checksum is empty, cannot verify."); + + FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) { + r = verify_one(checksum_job, j); + if (r < 0) + return r; + } + + verify_job = checksum_job; + } + + if (verify != IMPORT_VERIFY_SIGNATURE) return 0; - r = verification_style_from_url(checksum_job->url, &style); + assert(verify_job); + + r = verification_style_from_url(verify_job->url, &style); if (r < 0) - return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", checksum_job->url); + return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", verify_job->url); if (style == VERIFICATION_PER_DIRECTORY) { assert(signature_job); @@ -539,9 +596,9 @@ int pull_verify(ImportVerify verify, return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "Signature is empty, cannot verify."); - return verify_gpg(checksum_job->payload, checksum_job->payload_size, signature_job->payload, signature_job->payload_size); + return verify_gpg(verify_job->payload, verify_job->payload_size, signature_job->payload, signature_job->payload_size); } else - return verify_gpg(checksum_job->payload, checksum_job->payload_size, NULL, 0); + return verify_gpg(verify_job->payload, verify_job->payload_size, NULL, 0); } int verification_style_from_url(const char *url, VerificationStyle *ret) { @@ -595,3 +652,28 @@ int pull_job_restart_with_sha256sum(PullJob *j, char **ret) { return 1; } + +bool pull_validate_local(const char *name, PullFlags flags) { + + if (FLAGS_SET(flags, PULL_DIRECT)) + return path_is_valid(name); + + return hostname_is_valid(name, 0); +} + +int pull_url_needs_checksum(const char *url) { + _cleanup_free_ char *fn = NULL; + int r; + + /* Returns true if we need to validate this resource via a hash value. This returns true for all + * files — except for gpg signature files and SHA256SUMS files and the like, which are validated with + * a validation tool like gpg. */ + + r = import_url_last_component(url, &fn); + if (r == -EADDRNOTAVAIL) /* no last component? then let's assume it's not a signature/checksum file */ + return false; + if (r < 0) + return r; + + return !is_checksum_file(fn) && !is_signature_file(fn); +} diff --git a/src/import/pull-common.h b/src/import/pull-common.h index 3902e29f2b..2347db3bac 100644 --- a/src/import/pull-common.h +++ b/src/import/pull-common.h @@ -7,27 +7,31 @@ #include "pull-job.h" typedef enum PullFlags { - PULL_FORCE = 1 << 0, /* replace existing image */ - PULL_SETTINGS = 1 << 1, /* .nspawn settings file */ - PULL_ROOTHASH = 1 << 2, /* only for raw: .roothash file for verity */ - PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: .roothash.p7s file for verity */ - PULL_VERITY = 1 << 4, /* only for raw: .verity file for verity */ + PULL_FORCE = 1 << 0, /* replace existing image */ + PULL_READ_ONLY = 1 << 1, /* make generated image read-only */ + PULL_SETTINGS = 1 << 1, /* download .nspawn settings file */ + PULL_ROOTHASH = 1 << 2, /* only for raw: download .roothash file for verity */ + PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: download .roothash.p7s file for verity */ + PULL_VERITY = 1 << 4, /* only for raw: download .verity file for verity */ + PULL_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */ + PULL_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ + PULL_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */ + PULL_DIRECT = 1 << 5, /* download without rename games */ + PULL_SYNC = 1 << 6, /* fsync() right before we are done */ /* The supported flags for the tar and the raw pulling */ - PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_SETTINGS, - PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY, + PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_BTRFS_SUBVOL|PULL_BTRFS_QUOTA|PULL_DIRECT|PULL_SYNC, + PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY|PULL_CONVERT_QCOW2|PULL_DIRECT|PULL_SYNC, } PullFlags; -int pull_make_local_copy(const char *final, const char *root, const char *local, PullFlags flags); - int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags); int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); -int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, CurlGlue *glue, PullJobFinished on_finished, void *userdata); -int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); +int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata); +int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); -int pull_verify(ImportVerify verify, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); +int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); typedef enum VerificationStyle { VERIFICATION_PER_FILE, /* SuSE-style ".sha256" files with inline gpg signature */ @@ -39,3 +43,7 @@ typedef enum VerificationStyle { int verification_style_from_url(const char *url, VerificationStyle *style); int pull_job_restart_with_sha256sum(PullJob *job, char **ret); + +bool pull_validate_local(const char *name, PullFlags flags); + +int pull_url_needs_checksum(const char *url); diff --git a/src/import/pull-job.c b/src/import/pull-job.c index e751d3af05..8764207960 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -7,6 +7,7 @@ #include "alloc-util.h" #include "fd-util.h" #include "format-util.h" +#include "fs-util.h" #include "gcrypt-util.h" #include "hexdecoct.h" #include "import-util.h" @@ -19,15 +20,25 @@ #include "strv.h" #include "xattr-util.h" +void pull_job_close_disk_fd(PullJob *j) { + if (!j) + return; + + if (j->close_disk_fd) + safe_close(j->disk_fd); + + j->disk_fd = -1; +} + PullJob* pull_job_unref(PullJob *j) { if (!j) return NULL; + pull_job_close_disk_fd(j); + curl_glue_remove_and_free(j->glue, j->curl); curl_slist_free_all(j->request_header); - safe_close(j->disk_fd); - import_compress_free(&j->compress); if (j->checksum_context) @@ -116,15 +127,13 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { return; if (result != CURLE_OK) { - log_error("Transfer failed: %s", curl_easy_strerror(result)); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Transfer failed: %s", curl_easy_strerror(result)); goto finish; } code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); if (code != CURLE_OK) { - log_error("Failed to retrieve response code: %s", curl_easy_strerror(code)); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code)); goto finish; } else if (status == 304) { log_info("Image already downloaded. Skipping download."); @@ -150,8 +159,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status); if (code != CURLE_OK) { - log_error("Failed to retrieve response code: %s", curl_easy_strerror(code)); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code)); goto finish; } @@ -160,25 +168,23 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { } } - log_error("HTTP request to %s failed with code %li.", j->url, status); - r = -EIO; + r = log_error_errno( + status == 404 ? SYNTHETIC_ERRNO(ENOMEDIUM) : SYNTHETIC_ERRNO(EIO), /* Make the most common error recognizable */ + "HTTP request to %s failed with code %li.", j->url, status); goto finish; } else if (status < 200) { - log_error("HTTP request to %s finished with unexpected code %li.", j->url, status); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status); goto finish; } if (j->state != PULL_JOB_RUNNING) { - log_error("Premature connection termination."); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination."); goto finish; } if (j->content_length != UINT64_MAX && j->content_length != j->written_compressed) { - log_error("Download truncated."); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated."); goto finish; } @@ -187,8 +193,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256); if (!k) { - log_error("Failed to get checksum."); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); goto finish; } @@ -201,28 +206,57 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { log_debug("SHA256 of %s is %s.", j->url, j->checksum); } - if (j->disk_fd >= 0 && j->allow_sparse) { - /* Make sure the file size is right, in case the file was - * sparse and we just seeked for the last part */ + /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no + * offset is specified, which indicates we only own the file partially) */ - if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) { - r = log_error_errno(errno, "Failed to truncate file: %m"); - goto finish; - } + if (j->disk_fd >= 0) { - if (j->etag) - (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0); - if (j->url) - (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0); + if (S_ISREG(j->disk_stat.st_mode)) { - if (j->mtime != 0) { - struct timespec ut[2]; + if (j->offset == UINT64_MAX) { + + if (j->written_compressed > 0) { + /* Make sure the file size is right, in case the file was sparse and we just seeked + * for the last part */ + if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) { + r = log_error_errno(errno, "Failed to truncate file: %m"); + goto finish; + } + } + + if (j->etag) + (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0); + if (j->url) + (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0); + + if (j->mtime != 0) { + struct timespec ut; + + timespec_store(&ut, j->mtime); + + if (futimens(j->disk_fd, (struct timespec[]) { ut, ut }) < 0) + log_debug_errno(errno, "Failed to adjust atime/mtime of created image, ignoring: %m"); + + r = fd_setcrtime(j->disk_fd, j->mtime); + if (r < 0) + log_debug_errno(r, "Failed to adjust crtime of created image, ignoring: %m"); + } + } + + if (j->sync) { + r = fsync_full(j->disk_fd); + if (r < 0) { + log_error_errno(r, "Failed to synchronize file to disk: %m"); + goto finish; + } + } - timespec_store(&ut[0], j->mtime); - ut[1] = ut[0]; - (void) futimens(j->disk_fd, ut); + } else if (S_ISBLK(j->disk_stat.st_mode) && j->sync) { - (void) fd_setcrtime(j->disk_fd, j->mtime); + if (fsync(j->disk_fd) < 0) { + r = log_error_errno(errno, "Failed to synchronize block device: %m"); + goto finish; + } } } @@ -234,37 +268,46 @@ finish: static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) { PullJob *j = userdata; - ssize_t n; + bool too_much = false; + int r; assert(j); assert(p); + assert(sz > 0); - if (sz <= 0) - return 0; + if (j->written_uncompressed > UINT64_MAX - sz) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); - if (j->written_uncompressed + sz < j->written_uncompressed) - return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), - "File too large, overflow"); + if (j->written_uncompressed >= j->uncompressed_max) { + too_much = true; + goto finish; + } - if (j->written_uncompressed + sz > j->uncompressed_max) - return log_error_errno(SYNTHETIC_ERRNO(EFBIG), - "File overly large, refusing"); + if (j->written_uncompressed + sz > j->uncompressed_max) { + too_much = true; + sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory + * already, we might as well write it to + * disk to the max */ + } if (j->disk_fd >= 0) { - if (j->allow_sparse) + if (S_ISREG(j->disk_stat.st_mode)) { + ssize_t n; + n = sparse_write(j->disk_fd, p, sz, 64); - else { - n = write(j->disk_fd, p, sz); if (n < 0) - n = -errno; + return log_error_errno((int) n, "Failed to write file: %m"); + if ((size_t) n < sz) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); + } else { + r = loop_write(j->disk_fd, p, sz, false); + if (r < 0) + return log_error_errno(r, "Failed to write file: %m"); } - if (n < 0) - return log_error_errno((int) n, "Failed to write file: %m"); - if ((size_t) n < sz) - return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); - } else { + } + if (j->disk_fd < 0 || j->force_memory) { if (!GREEDY_REALLOC(j->payload, j->payload_size + sz)) return log_oom(); @@ -274,6 +317,10 @@ static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) j->written_uncompressed += sz; +finish: + if (too_much) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); + return 0; } @@ -321,15 +368,12 @@ static int pull_job_open_disk(PullJob *j) { } if (j->disk_fd >= 0) { - /* Check if we can do sparse files */ + if (fstat(j->disk_fd, &j->disk_stat) < 0) + return log_error_errno(errno, "Failed to stat disk file: %m"); - if (lseek(j->disk_fd, SEEK_SET, 0) == 0) - j->allow_sparse = true; - else { - if (errno != ESPIPE) + if (j->offset != UINT64_MAX) { + if (lseek(j->disk_fd, j->offset, SEEK_SET) == (off_t) -1) return log_error_errno(errno, "Failed to seek on file descriptor: %m"); - - j->allow_sparse = false; } } @@ -461,8 +505,7 @@ static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status); if (code != CURLE_OK) { - log_error("Failed to retrieve response code: %s", curl_easy_strerror(code)); - r = -EIO; + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code)); goto fail; } @@ -576,7 +619,12 @@ static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_o return 0; } -int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) { +int pull_job_new( + PullJob **ret, + const char *url, + CurlGlue *glue, + void *userdata) { + _cleanup_(pull_job_unrefp) PullJob *j = NULL; _cleanup_free_ char *u = NULL; @@ -595,6 +643,7 @@ int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) *j = (PullJob) { .state = PULL_JOB_INIT, .disk_fd = -1, + .close_disk_fd = true, .userdata = userdata, .glue = glue, .content_length = UINT64_MAX, @@ -602,6 +651,8 @@ int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) .compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */ .uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */ .url = TAKE_PTR(u), + .offset = UINT64_MAX, + .sync = true, }; *ret = TAKE_PTR(j); diff --git a/src/import/pull-job.h b/src/import/pull-job.h index 023ccc8302..bc5258a693 100644 --- a/src/import/pull-job.h +++ b/src/import/pull-job.h @@ -2,10 +2,12 @@ #pragma once #include <gcrypt.h> +#include <sys/stat.h> #include "curl-util.h" #include "import-compress.h" #include "macro.h" +#include "pull-common.h" typedef struct PullJob PullJob; @@ -51,6 +53,7 @@ struct PullJob { uint64_t content_length; uint64_t written_compressed; uint64_t written_uncompressed; + uint64_t offset; uint64_t uncompressed_max; uint64_t compressed_max; @@ -59,6 +62,8 @@ struct PullJob { size_t payload_size; int disk_fd; + bool close_disk_fd; + struct stat disk_stat; usec_t mtime; @@ -68,12 +73,12 @@ struct PullJob { usec_t start_usec; usec_t last_status_usec; - bool allow_sparse; - bool calc_checksum; gcry_md_hd_t checksum_context; char *checksum; + bool sync; + bool force_memory; }; int pull_job_new(PullJob **job, const char *url, CurlGlue *glue, void *userdata); @@ -83,4 +88,6 @@ int pull_job_begin(PullJob *j); void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result); +void pull_job_close_disk_fd(PullJob *j); + DEFINE_TRIVIAL_CLEANUP_FUNC(PullJob*, pull_job_unref); diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index 56c99a690e..c071e134f2 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -15,6 +15,7 @@ #include "hostname-util.h" #include "import-common.h" #include "import-util.h" +#include "install-file.h" #include "macro.h" #include "mkdir.h" #include "path-util.h" @@ -46,6 +47,8 @@ struct RawPull { ImportVerify verify; char *image_root; + uint64_t offset; + PullJob *raw_job; PullJob *checksum_job; PullJob *signature_job; @@ -57,7 +60,8 @@ struct RawPull { RawPullFinished on_finished; void *userdata; - char *local; + char *local; /* In PULL_DIRECT mode the path we are supposed to place things in, otherwise the + * machine name of the final copy we make */ char *final_path; char *temp_path; @@ -73,6 +77,8 @@ struct RawPull { char *verity_path; char *verity_temp_path; + + char *checksum; }; RawPull* raw_pull_unref(RawPull *i) { @@ -103,6 +109,7 @@ RawPull* raw_pull_unref(RawPull *i) { free(i->verity_path); free(i->image_root); free(i->local); + free(i->checksum); return mfree(i); } @@ -148,6 +155,7 @@ int raw_pull_new( .image_root = TAKE_PTR(root), .event = TAKE_PTR(e), .glue = TAKE_PTR(g), + .offset = UINT64_MAX, }; i->glue->on_finished = pull_job_curl_on_finished; @@ -230,12 +238,20 @@ static void raw_pull_report_progress(RawPull *i, RawProgress p) { } static int raw_pull_maybe_convert_qcow2(RawPull *i) { + _cleanup_(unlink_and_freep) char *t = NULL; _cleanup_close_ int converted_fd = -1; - _cleanup_free_ char *t = NULL; + _cleanup_free_ char *f = NULL; int r; assert(i); assert(i->raw_job); + assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + + if (!FLAGS_SET(i->flags, PULL_CONVERT_QCOW2)) + return 0; + + assert(i->final_path); + assert(i->raw_job->close_disk_fd); r = qcow2_detect(i->raw_job->disk_fd); if (r < 0) @@ -244,32 +260,35 @@ static int raw_pull_maybe_convert_qcow2(RawPull *i) { return 0; /* This is a QCOW2 image, let's convert it */ - r = tempfn_random(i->final_path, NULL, &t); + r = tempfn_random(i->final_path, NULL, &f); if (r < 0) return log_oom(); - converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (converted_fd < 0) - return log_error_errno(errno, "Failed to create %s: %m", t); + return log_error_errno(errno, "Failed to create %s: %m", f); + + t = TAKE_PTR(f); (void) import_set_nocow_and_log(converted_fd, t); log_info("Unpacking QCOW2 file."); r = qcow2_convert(i->raw_job->disk_fd, converted_fd); - if (r < 0) { - (void) unlink(t); + if (r < 0) return log_error_errno(r, "Failed to convert qcow2 image: %m"); - } - (void) unlink(i->temp_path); - free_and_replace(i->temp_path, t); + unlink_and_free(i->temp_path); + i->temp_path = TAKE_PTR(t); CLOSE_AND_REPLACE(i->raw_job->disk_fd, converted_fd); return 1; } -static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) { +static int raw_pull_determine_path( + RawPull *i, + const char *suffix, + char **field /* input + output (!) */) { int r; assert(i); @@ -290,7 +309,7 @@ static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) static int raw_pull_copy_auxiliary_file( RawPull *i, const char *suffix, - char **path) { + char **path /* input + output (!) */) { const char *local; int r; @@ -305,7 +324,14 @@ static int raw_pull_copy_auxiliary_file( local = strjoina(i->image_root, "/", i->local, suffix); - r = copy_file_atomic(*path, local, 0644, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0)); + r = copy_file_atomic( + *path, + local, + 0644, + 0, 0, + COPY_REFLINK | + (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0)); if (r == -EEXIST) log_warning_errno(r, "File %s already exists, not replacing.", local); else if (r == -ENOENT) @@ -319,13 +345,15 @@ static int raw_pull_copy_auxiliary_file( } static int raw_pull_make_local_copy(RawPull *i) { - _cleanup_free_ char *tp = NULL; + _cleanup_(unlink_and_freep) char *tp = NULL; + _cleanup_free_ char *f = NULL; _cleanup_close_ int dfd = -1; const char *p; int r; assert(i); assert(i->raw_job); + assert(!FLAGS_SET(i->flags, PULL_DIRECT)); if (!i->local) return 0; @@ -342,6 +370,7 @@ static int raw_pull_make_local_copy(RawPull *i) { /* We freshly downloaded the image, use it */ assert(i->raw_job->disk_fd >= 0); + assert(i->offset == UINT64_MAX); if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1) return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m"); @@ -349,38 +378,38 @@ static int raw_pull_make_local_copy(RawPull *i) { p = strjoina(i->image_root, "/", i->local, ".raw"); - if (FLAGS_SET(i->flags, PULL_FORCE)) - (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); - - r = tempfn_random(p, NULL, &tp); + r = tempfn_random(p, NULL, &f); if (r < 0) return log_oom(); - dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (dfd < 0) return log_error_errno(errno, "Failed to create writable copy of image: %m"); + tp = TAKE_PTR(f); + /* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs, * since it reduces fragmentation caused by not allowing in-place writes. */ (void) import_set_nocow_and_log(dfd, tp); r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK); - if (r < 0) { - (void) unlink(tp); + if (r < 0) return log_error_errno(r, "Failed to make writable copy of image: %m"); - } (void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME); (void) copy_xattr(i->raw_job->disk_fd, dfd, 0); dfd = safe_close(dfd); - r = rename(tp, p); - if (r < 0) { - r = log_error_errno(errno, "Failed to move writable image into place: %m"); - (void) unlink(tp); - return r; - } + r = install_file(AT_FDCWD, tp, + AT_FDCWD, p, + (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) + return log_error_errno(errno, "Failed to move local image into place '%s': %m", p); + + tp = mfree(tp); log_info("Created new local image '%s'.", i->local); @@ -442,9 +471,10 @@ static int raw_pull_rename_auxiliary_file( int r; assert(i); + assert(path); assert(temp_path); + assert(*temp_path); assert(suffix); - assert(path); /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should * incorporate it in the file name if we can */ @@ -453,61 +483,78 @@ static int raw_pull_rename_auxiliary_file( if (r < 0) return r; - r = import_make_read_only(*temp_path); + r = install_file( + AT_FDCWD, *temp_path, + AT_FDCWD, *path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) - return r; - - r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path); - if (r < 0) - return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path); + return log_error_errno(r, "Failed to move '%s' into place: %m", *path); *temp_path = mfree(*temp_path); - return 1; } static void raw_pull_job_on_finished(PullJob *j) { RawPull *i; + PullJob *jj; int r; assert(j); assert(j->userdata); i = j->userdata; - if (j == i->settings_job) { - if (j->error != 0) + + if (j->error != 0) { + /* Only the main job and the checksum job are fatal if they fail. The other fails are just + * "decoration", that we'll download if we can. The signature job isn't fatal here because we + * might not actually need it in case Suse style signatures are used, that are inline in the + * checksum file. */ + + if (j == i->raw_job) { + if (j->error == ENOMEDIUM) /* HTTP 404 */ + r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); + else + r = log_error_errno(j->error, "Failed to retrieve image file."); + goto finish; + } else if (j == i->checksum_job) { + r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + goto finish; + } else if (j == i->signature_job) + log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url); + else if (j == i->settings_job) log_info_errno(j->error, "Settings file could not be retrieved, proceeding without."); - } else if (j == i->roothash_job) { - if (j->error != 0) + else if (j == i->roothash_job) log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without."); - } else if (j == i->roothash_signature_job) { - if (j->error != 0) + else if (j == i->roothash_signature_job) log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without."); - } else if (j == i->verity_job) { - if (j->error != 0) - log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without. %s", j->url); - } else if (j->error != 0 && j != i->signature_job) { - if (j == i->checksum_job) - log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + else if (j == i->verity_job) + log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without."); else - log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); - - r = j->error; - goto finish; + assert_not_reached(); } /* This is invoked if either the download completed successfully, or the download was skipped because * we already have the etag. In this case ->etag_exists is true. * - * We only do something when we got all three files */ + * We only do something when we got all files */ if (!raw_pull_is_done(i)) return; if (i->signature_job && i->signature_job->error != 0) { VerificationStyle style; + PullJob *verify_job; + + /* The signature job failed. Let's see if we actually need it */ + + verify_job = i->checksum_job ?: i->raw_job; /* if the checksum job doesn't exist this must be + * because the main job is the checksum file + * itself */ + + assert(verify_job); - r = verification_style_from_url(i->checksum_job->url, &style); + r = verification_style_from_url(verify_job->url, &style); if (r < 0) { log_error_errno(r, "Failed to determine verification style from checksum URL: %m"); goto finish; @@ -517,32 +564,21 @@ static void raw_pull_job_on_finished(PullJob *j) { * in per-directory verification mode, since only * then the signature is detached, and thus a file * of its own. */ - log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); - r = i->signature_job->error; + r = log_error_errno(i->signature_job->error, + "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); goto finish; } } - if (i->settings_job) - i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd); - if (i->roothash_job) - i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd); - if (i->roothash_signature_job) - i->roothash_signature_job->disk_fd = safe_close(i->roothash_signature_job->disk_fd); - if (i->verity_job) - i->verity_job->disk_fd = safe_close(i->verity_job->disk_fd); - - r = raw_pull_determine_path(i, ".raw", &i->final_path); - if (r < 0) - goto finish; + /* Let's close these auxiliary files now, we don't need access to them anymore. */ + FOREACH_POINTER(jj, i->settings_job, i->roothash_job, i->roothash_signature_job, i->verity_job) + pull_job_close_disk_fd(jj); if (!i->raw_job->etag_exists) { - /* This is a new download, verify it, and move it into place */ - assert(i->raw_job->disk_fd >= 0); - raw_pull_report_progress(i, RAW_VERIFYING); r = pull_verify(i->verify, + i->checksum, i->raw_job, i->checksum_job, i->signature_job, @@ -552,50 +588,91 @@ static void raw_pull_job_on_finished(PullJob *j) { i->verity_job); if (r < 0) goto finish; + } - raw_pull_report_progress(i, RAW_UNPACKING); + if (i->flags & PULL_DIRECT) { + assert(!i->settings_job); + assert(!i->roothash_job); + assert(!i->roothash_signature_job); + assert(!i->verity_job); + + raw_pull_report_progress(i, RAW_FINALIZING); - r = raw_pull_maybe_convert_qcow2(i); + if (i->local) { + r = install_file(AT_FDCWD, i->local, + AT_FDCWD, NULL, + ((i->flags & PULL_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) | + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) { + log_error_errno(r, "Failed to finalize raw file to '%s': %m", i->local); + goto finish; + } + } + } else { + r = raw_pull_determine_path(i, ".raw", &i->final_path); if (r < 0) goto finish; - raw_pull_report_progress(i, RAW_FINALIZING); + if (!i->raw_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ + + assert(i->temp_path); + assert(i->final_path); - if (i->raw_job->etag) { - /* Only make a read-only copy if ETag header is set. */ - r = import_make_read_only_fd(i->raw_job->disk_fd); + raw_pull_report_progress(i, RAW_UNPACKING); + + r = raw_pull_maybe_convert_qcow2(i); if (r < 0) goto finish; - r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + raw_pull_report_progress(i, RAW_FINALIZING); + + r = install_file(AT_FDCWD, i->temp_path, + AT_FDCWD, i->final_path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) { - log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path); + log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path); goto finish; } - } - i->temp_path = mfree(i->temp_path); + i->temp_path = mfree(i->temp_path); - if (i->roothash_job && - i->roothash_job->error == 0) { - r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path); - if (r < 0) - goto finish; - } + if (i->settings_job && + i->settings_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path); + if (r < 0) + goto finish; + } - if (i->settings_job && - i->settings_job->error == 0) { - r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path); - if (r < 0) - goto finish; + if (i->roothash_job && + i->roothash_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path); + if (r < 0) + goto finish; + } + + if (i->roothash_signature_job && + i->roothash_signature_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_temp_path, &i->roothash_signature_path); + if (r < 0) + goto finish; + } + + if (i->verity_job && + i->verity_job->error == 0) { + r = raw_pull_rename_auxiliary_file(i, ".verity", &i->verity_temp_path, &i->verity_path); + if (r < 0) + goto finish; + } } - } - raw_pull_report_progress(i, RAW_COPYING); + raw_pull_report_progress(i, RAW_COPYING); - r = raw_pull_make_local_copy(i); - if (r < 0) - goto finish; + r = raw_pull_make_local_copy(i); + if (r < 0) + goto finish; + } r = 0; @@ -610,7 +687,7 @@ static int raw_pull_job_on_open_disk_generic( RawPull *i, PullJob *j, const char *extra, - char **temp_path) { + char **temp_path /* input + output */) { int r; @@ -619,6 +696,8 @@ static int raw_pull_job_on_open_disk_generic( assert(extra); assert(temp_path); + assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + if (!*temp_path) { r = tempfn_random_child(i->image_root, extra, temp_path); if (r < 0) @@ -643,12 +722,34 @@ static int raw_pull_job_on_open_disk_raw(PullJob *j) { i = j->userdata; assert(i->raw_job == j); + assert(j->disk_fd < 0); - r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path); - if (r < 0) - return r; + if (i->flags & PULL_DIRECT) { + + if (!i->local) { /* If no local name specified, the pull job will write its data to stdout */ + j->disk_fd = STDOUT_FILENO; + j->close_disk_fd = false; + return 0; + } + + (void) mkdir_parents_label(i->local, 0700); + + j->disk_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664); + if (j->disk_fd < 0) + return log_error_errno(errno, "Failed to open destination '%s': %m", i->local); + + if (i->offset == UINT64_MAX) + (void) import_set_nocow_and_log(j->disk_fd, i->local); + + } else { + r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path); + if (r < 0) + return r; + + assert(i->offset == UINT64_MAX); + (void) import_set_nocow_and_log(j->disk_fd, i->temp_path); + } - (void) import_set_nocow_and_log(j->disk_fd, i->temp_path); return 0; } @@ -715,20 +816,29 @@ int raw_pull_start( RawPull *i, const char *url, const char *local, + uint64_t offset, + uint64_t size_max, PullFlags flags, - ImportVerify verify) { + ImportVerify verify, + const char *checksum) { + PullJob *j; int r; assert(i); - assert(verify < _IMPORT_VERIFY_MAX); - assert(verify >= 0); + assert(url); + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); assert(!(flags & ~PULL_FLAGS_MASK_RAW)); + assert(offset == UINT64_MAX || FLAGS_SET(flags, PULL_DIRECT)); + assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !(flags & PULL_DIRECT)); + assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !checksum); if (!http_url_is_valid(url)) return -EINVAL; - if (local && !hostname_is_valid(local, 0)) + if (local && !pull_validate_local(local, flags)) return -EINVAL; if (i->raw_job) @@ -738,6 +848,10 @@ int raw_pull_start( if (r < 0) return r; + r = free_and_strdup(&i->checksum, checksum); + if (r < 0) + return r; + i->flags = flags; i->verify = verify; @@ -748,98 +862,121 @@ int raw_pull_start( i->raw_job->on_finished = raw_pull_job_on_finished; i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw; - i->raw_job->on_progress = raw_pull_job_on_progress; - i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO; - - r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags); - if (r < 0) - return r; - - r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i); - if (r < 0) - return r; - - if (FLAGS_SET(flags, PULL_SETTINGS)) { - r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i); - if (r < 0) - return r; - i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings; - i->settings_job->on_progress = raw_pull_job_on_progress; - i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO; - } + if (checksum) + i->raw_job->calc_checksum = true; + else if (verify != IMPORT_VERIFY_NO) { + /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its + * signature, which we let gpg verify instead. */ - if (FLAGS_SET(flags, PULL_ROOTHASH)) { - r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i); + r = pull_url_needs_checksum(url); if (r < 0) return r; - i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash; - i->roothash_job->on_progress = raw_pull_job_on_progress; - i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO; + i->raw_job->calc_checksum = r; + i->raw_job->force_memory = true; /* make sure this is both written to disk if that's + * requested and into memory, since we need to verify it */ } - if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) { - r = pull_make_auxiliary_job(&i->roothash_signature_job, url, raw_strip_suffixes, ".roothash.p7s", i->glue, raw_pull_job_on_finished, i); - if (r < 0) - return r; - - i->roothash_signature_job->on_open_disk = raw_pull_job_on_open_disk_roothash_signature; - i->roothash_signature_job->on_progress = raw_pull_job_on_progress; - i->roothash_signature_job->calc_checksum = verify != IMPORT_VERIFY_NO; - } + if (size_max != UINT64_MAX) + i->raw_job->uncompressed_max = size_max; + if (offset != UINT64_MAX) + i->raw_job->offset = i->offset = offset; - if (FLAGS_SET(flags, PULL_VERITY)) { - r = pull_make_auxiliary_job(&i->verity_job, url, raw_strip_suffixes, ".verity", i->glue, raw_pull_job_on_finished, i); + if (!FLAGS_SET(flags, PULL_DIRECT)) { + r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags); if (r < 0) return r; - - i->verity_job->on_open_disk = raw_pull_job_on_open_disk_verity; - i->verity_job->on_progress = raw_pull_job_on_progress; - i->verity_job->calc_checksum = verify != IMPORT_VERIFY_NO; } - r = pull_job_begin(i->raw_job); + r = pull_make_verification_jobs( + &i->checksum_job, + &i->signature_job, + verify, + i->checksum, + url, + i->glue, + raw_pull_job_on_finished, + i); if (r < 0) return r; - if (i->checksum_job) { - i->checksum_job->on_progress = raw_pull_job_on_progress; - i->checksum_job->on_not_found = pull_job_restart_with_sha256sum; - - r = pull_job_begin(i->checksum_job); + if (FLAGS_SET(flags, PULL_SETTINGS)) { + r = pull_make_auxiliary_job( + &i->settings_job, + url, + raw_strip_suffixes, + ".nspawn", + verify, + i->glue, + raw_pull_job_on_open_disk_settings, + raw_pull_job_on_finished, + i); if (r < 0) return r; } - if (i->signature_job) { - i->signature_job->on_progress = raw_pull_job_on_progress; - - r = pull_job_begin(i->signature_job); + if (FLAGS_SET(flags, PULL_ROOTHASH)) { + r = pull_make_auxiliary_job( + &i->roothash_job, + url, + raw_strip_suffixes, + ".roothash", + verify, + i->glue, + raw_pull_job_on_open_disk_roothash, + raw_pull_job_on_finished, + i); if (r < 0) return r; } - if (i->settings_job) { - r = pull_job_begin(i->settings_job); + if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) { + r = pull_make_auxiliary_job( + &i->roothash_signature_job, + url, + raw_strip_suffixes, + ".roothash.p7s", + verify, + i->glue, + raw_pull_job_on_open_disk_roothash_signature, + raw_pull_job_on_finished, + i); if (r < 0) return r; } - if (i->roothash_job) { - r = pull_job_begin(i->roothash_job); + if (FLAGS_SET(flags, PULL_VERITY)) { + r = pull_make_auxiliary_job( + &i->verity_job, + url, + raw_strip_suffixes, + ".verity", + verify, + i->glue, + raw_pull_job_on_open_disk_verity, + raw_pull_job_on_finished, + i); if (r < 0) return r; } - if (i->roothash_signature_job) { - r = pull_job_begin(i->roothash_signature_job); - if (r < 0) - return r; - } + FOREACH_POINTER(j, + i->raw_job, + i->checksum_job, + i->signature_job, + i->settings_job, + i->roothash_job, + i->roothash_signature_job, + i->verity_job) { + + if (!j) + continue; + + j->on_progress = raw_pull_job_on_progress; + j->sync = FLAGS_SET(flags, PULL_SYNC); - if (i->verity_job) { - r = pull_job_begin(i->verity_job); + r = pull_job_begin(j); if (r < 0) return r; } diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h index 985bda4762..b39e4e257c 100644 --- a/src/import/pull-raw.h +++ b/src/import/pull-raw.h @@ -16,4 +16,4 @@ RawPull* raw_pull_unref(RawPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref); -int raw_pull_start(RawPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify); +int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, PullFlags flags, ImportVerify verify, const char *checksum); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index 1227c89ffb..6aca3c9979 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -14,6 +14,7 @@ #include "hostname-util.h" #include "import-common.h" #include "import-util.h" +#include "install-file.h" #include "macro.h" #include "mkdir.h" #include "path-util.h" @@ -25,6 +26,7 @@ #include "string-util.h" #include "strv.h" #include "tmpfile-util.h" +#include "user-util.h" #include "utf8.h" #include "util.h" #include "web-util.h" @@ -61,6 +63,8 @@ struct TarPull { char *settings_path; char *settings_temp_path; + + char *checksum; }; TarPull* tar_pull_unref(TarPull *i) { @@ -87,6 +91,7 @@ TarPull* tar_pull_unref(TarPull *i) { free(i->settings_path); free(i->image_root); free(i->local); + free(i->checksum); return mfree(i); } @@ -194,7 +199,10 @@ static void tar_pull_report_progress(TarPull *i, TarProgress p) { log_debug("Combined progress %u%%", percent); } -static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) { +static int tar_pull_determine_path( + TarPull *i, + const char *suffix, + char **field /* input + output (!) */) { int r; assert(i); @@ -213,6 +221,8 @@ static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) } static int tar_pull_make_local_copy(TarPull *i) { + _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL; + const char *p; int r; assert(i); @@ -221,9 +231,38 @@ static int tar_pull_make_local_copy(TarPull *i) { if (!i->local) return 0; - r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->flags); + assert(i->final_path); + + p = prefix_roota(i->image_root, i->local); + + r = tempfn_random(p, NULL, &t); if (r < 0) - return r; + return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p); + + if (i->flags & PULL_BTRFS_SUBVOL) + r = btrfs_subvol_snapshot( + i->final_path, + t, + (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)| + BTRFS_SNAPSHOT_FALLBACK_COPY| + BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| + BTRFS_SNAPSHOT_RECURSIVE); + else + r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS); + if (r < 0) + return log_error_errno(r, "Failed to create local image: %m"); + + r = install_file(AT_FDCWD, t, + AT_FDCWD, p, + (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + if (r < 0) + return log_error_errno(r, "Failed to install local image '%s': %m", p); + + t = mfree(t); + + log_info("Created new local image '%s'.", i->local); if (FLAGS_SET(i->flags, PULL_SETTINGS)) { const char *local_settings; @@ -235,7 +274,14 @@ static int tar_pull_make_local_copy(TarPull *i) { local_settings = strjoina(i->image_root, "/", i->local, ".nspawn"); - r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0)); + r = copy_file_atomic( + i->settings_path, + local_settings, + 0664, + 0, 0, + COPY_REFLINK | + (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0)); if (r == -EEXIST) log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings); else if (r == -ENOENT) @@ -274,17 +320,22 @@ static void tar_pull_job_on_finished(PullJob *j) { i = j->userdata; - if (j == i->settings_job) { - if (j->error != 0) + if (j->error != 0) { + if (j == i->tar_job) { + if (j->error == ENOMEDIUM) /* HTTP 404 */ + r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); + else + r = log_error_errno(j->error, "Failed to retrieve image file."); + goto finish; + } else if (j == i->checksum_job) { + r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); + goto finish; + } else if (j == i->signature_job) + log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url); + else if (j == i->settings_job) log_info_errno(j->error, "Settings file could not be retrieved, proceeding without."); - } else if (j->error != 0 && j != i->signature_job) { - if (j == i->checksum_job) - log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); else - log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); - - r = j->error; - goto finish; + assert("unexpected job"); } /* This is invoked if either the download completed successfully, or the download was skipped because @@ -296,6 +347,8 @@ static void tar_pull_job_on_finished(PullJob *j) { if (i->signature_job && i->signature_job->error != 0) { VerificationStyle style; + assert(i->checksum_job); + r = verification_style_from_url(i->checksum_job->url, &style); if (r < 0) { log_error_errno(r, "Failed to determine verification style from checksum URL: %m"); @@ -306,19 +359,14 @@ static void tar_pull_job_on_finished(PullJob *j) { * in per-directory verification mode, since only * then the signature is detached, and thus a file * of its own. */ - log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); - r = i->signature_job->error; + r = log_error_errno(i->signature_job->error, + "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); goto finish; } } - i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd); - if (i->settings_job) - i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd); - - r = tar_pull_determine_path(i, NULL, &i->final_path); - if (r < 0) - goto finish; + pull_job_close_disk_fd(i->tar_job); + pull_job_close_disk_fd(i->settings_job); if (i->tar_pid > 0) { r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG); @@ -337,6 +385,7 @@ static void tar_pull_job_on_finished(PullJob *j) { tar_pull_report_progress(i, TAR_VERIFYING); r = pull_verify(i->verify, + i->checksum, i->tar_job, i->checksum_job, i->signature_job, @@ -346,59 +395,92 @@ static void tar_pull_job_on_finished(PullJob *j) { /* verity_job = */ NULL); if (r < 0) goto finish; + } - tar_pull_report_progress(i, TAR_FINALIZING); + if (i->flags & PULL_DIRECT) { + assert(!i->settings_job); + assert(i->local); + assert(!i->temp_path); - r = import_mangle_os_tree(i->temp_path); - if (r < 0) - goto finish; + tar_pull_report_progress(i, TAR_FINALIZING); - r = import_make_read_only(i->temp_path); + r = import_mangle_os_tree(i->local); if (r < 0) goto finish; - r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); + r = install_file( + AT_FDCWD, i->local, + AT_FDCWD, NULL, + (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 | + (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) { - log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path); + log_error_errno(r, "Failed to finalize '%s': %m", i->local); goto finish; } + } else { + r = tar_pull_determine_path(i, NULL, &i->final_path); + if (r < 0) + goto finish; - i->temp_path = mfree(i->temp_path); - - if (i->settings_job && - i->settings_job->error == 0) { + if (!i->tar_job->etag_exists) { + /* This is a new download, verify it, and move it into place */ - /* Also move the settings file into place, if it exists. Note that we do so only if we also - * moved the tar file in place, to keep things strictly in sync. */ - assert(i->settings_temp_path); + assert(i->temp_path); + assert(i->final_path); - /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and - * we should incorporate it in the file name if we can */ - i->settings_path = mfree(i->settings_path); + tar_pull_report_progress(i, TAR_FINALIZING); - r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); + r = import_mangle_os_tree(i->temp_path); if (r < 0) goto finish; - r = import_make_read_only(i->settings_temp_path); - if (r < 0) - goto finish; - - r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path); + r = install_file( + AT_FDCWD, i->temp_path, + AT_FDCWD, i->final_path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) { - log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path); + log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path); goto finish; } - i->settings_temp_path = mfree(i->settings_temp_path); + i->temp_path = mfree(i->temp_path); + + if (i->settings_job && + i->settings_job->error == 0) { + + /* Also move the settings file into place, if it exists. Note that we do so only if we also + * moved the tar file in place, to keep things strictly in sync. */ + assert(i->settings_temp_path); + + /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and + * we should incorporate it in the file name if we can */ + i->settings_path = mfree(i->settings_path); + + r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); + if (r < 0) + goto finish; + + r = install_file( + AT_FDCWD, i->settings_temp_path, + AT_FDCWD, i->settings_path, + INSTALL_READ_ONLY| + (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + if (r < 0) { + log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path); + goto finish; + } + + i->settings_temp_path = mfree(i->settings_temp_path); + } } - } - tar_pull_report_progress(i, TAR_COPYING); + tar_pull_report_progress(i, TAR_COPYING); - r = tar_pull_make_local_copy(i); - if (r < 0) - goto finish; + r = tar_pull_make_local_copy(i); + if (r < 0) + goto finish; + } r = 0; @@ -410,6 +492,7 @@ finish: } static int tar_pull_job_on_open_disk_tar(PullJob *j) { + const char *where; TarPull *i; int r; @@ -420,23 +503,39 @@ static int tar_pull_job_on_open_disk_tar(PullJob *j) { assert(i->tar_job == j); assert(i->tar_pid <= 0); - if (!i->temp_path) { - r = tempfn_random_child(i->image_root, "tar", &i->temp_path); - if (r < 0) - return log_oom(); + if (i->flags & PULL_DIRECT) + where = i->local; + else { + if (!i->temp_path) { + r = tempfn_random_child(i->image_root, "tar", &i->temp_path); + if (r < 0) + return log_oom(); + } + + where = i->temp_path; } - mkdir_parents_label(i->temp_path, 0700); + (void) mkdir_parents_label(where, 0700); + + if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE)) + (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); - r = btrfs_subvol_make_fallback(i->temp_path, 0755); + if (i->flags & PULL_BTRFS_SUBVOL) + r = btrfs_subvol_make_fallback(where, 0755); + else + r = mkdir(where, 0755) < 0 ? -errno : 0; + if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise, + * because in that case our temporary path collided */ + r = 0; if (r < 0) - return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path); - if (r > 0) { /* actually btrfs subvol */ - (void) import_assign_pool_quota_and_warn(i->image_root); - (void) import_assign_pool_quota_and_warn(i->temp_path); + return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where); + if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */ + if (!(i->flags & PULL_DIRECT)) + (void) import_assign_pool_quota_and_warn(i->image_root); + (void) import_assign_pool_quota_and_warn(where); } - j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); + j->disk_fd = import_fork_tar_x(where, &i->tar_pid); if (j->disk_fd < 0) return j->disk_fd; @@ -459,7 +558,7 @@ static int tar_pull_job_on_open_disk_settings(PullJob *j) { return log_oom(); } - mkdir_parents_label(i->settings_temp_path, 0700); + (void) mkdir_parents_label(i->settings_temp_path, 0700); j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (j->disk_fd < 0) @@ -484,19 +583,24 @@ int tar_pull_start( const char *url, const char *local, PullFlags flags, - ImportVerify verify) { + ImportVerify verify, + const char *checksum) { + PullJob *j; int r; assert(i); - assert(verify < _IMPORT_VERIFY_MAX); - assert(verify >= 0); + assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); + assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); + assert((verify < 0) || !checksum); assert(!(flags & ~PULL_FLAGS_MASK_TAR)); + assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT)); + assert(!(flags & PULL_SETTINGS) || !checksum); if (!http_url_is_valid(url)) return -EINVAL; - if (local && !hostname_is_valid(local, 0)) + if (local && !pull_validate_local(local, flags)) return -EINVAL; if (i->tar_job) @@ -506,6 +610,10 @@ int tar_pull_start( if (r < 0) return r; + r = free_and_strdup(&i->checksum, checksum); + if (r < 0) + return r; + i->flags = flags; i->verify = verify; @@ -516,52 +624,56 @@ int tar_pull_start( i->tar_job->on_finished = tar_pull_job_on_finished; i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar; - i->tar_job->on_progress = tar_pull_job_on_progress; - i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO; + i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE); - r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags); - if (r < 0) - return r; + if (!FLAGS_SET(flags, PULL_DIRECT)) { + r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags); + if (r < 0) + return r; + } /* Set up download of checksum/signature files */ - r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i); + r = pull_make_verification_jobs( + &i->checksum_job, + &i->signature_job, + verify, + checksum, + url, + i->glue, + tar_pull_job_on_finished, + i); if (r < 0) return r; /* Set up download job for the settings file (.nspawn) */ if (FLAGS_SET(flags, PULL_SETTINGS)) { - r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i); + r = pull_make_auxiliary_job( + &i->settings_job, + url, + tar_strip_suffixes, + ".nspawn", + verify, + i->glue, + tar_pull_job_on_open_disk_settings, + tar_pull_job_on_finished, + i); if (r < 0) return r; - - i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings; - i->settings_job->on_progress = tar_pull_job_on_progress; - i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO; } - r = pull_job_begin(i->tar_job); - if (r < 0) - return r; - - if (i->checksum_job) { - i->checksum_job->on_progress = tar_pull_job_on_progress; - i->checksum_job->on_not_found = pull_job_restart_with_sha256sum; - - r = pull_job_begin(i->checksum_job); - if (r < 0) - return r; - } + FOREACH_POINTER(j, + i->tar_job, + i->checksum_job, + i->signature_job, + i->settings_job) { - if (i->signature_job) { - i->signature_job->on_progress = tar_pull_job_on_progress; + if (!j) + continue; - r = pull_job_begin(i->signature_job); - if (r < 0) - return r; - } + j->on_progress = tar_pull_job_on_progress; + j->sync = FLAGS_SET(flags, PULL_SYNC); - if (i->settings_job) { - r = pull_job_begin(i->settings_job); + r = pull_job_begin(j); if (r < 0) return r; } diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h index 414077549f..e54c01c302 100644 --- a/src/import/pull-tar.h +++ b/src/import/pull-tar.h @@ -16,4 +16,4 @@ TarPull* tar_pull_unref(TarPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref); -int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify); +int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify, const char *checksum); diff --git a/src/import/pull.c b/src/import/pull.c index 17ade56c2a..f6783f030c 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -8,24 +8,87 @@ #include "alloc-util.h" #include "discover-image.h" +#include "env-util.h" +#include "hexdecoct.h" #include "hostname-util.h" +#include "import-common.h" #include "import-util.h" +#include "io-util.h" #include "main-func.h" +#include "parse-argument.h" #include "parse-util.h" #include "pull-raw.h" #include "pull-tar.h" #include "signal-util.h" #include "string-util.h" +#include "terminal-util.h" #include "verbs.h" #include "web-util.h" static const char *arg_image_root = "/var/lib/machines"; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; -static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY; +static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY | PULL_BTRFS_SUBVOL | PULL_BTRFS_QUOTA | PULL_CONVERT_QCOW2 | PULL_SYNC; +static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; +static char *arg_checksum = NULL; -static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { - log_notice("Transfer aborted."); - sd_event_exit(sd_event_source_get_event(s), EINTR); +STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep); + +static int normalize_local(const char *local, const char *url, char **ret) { + _cleanup_free_ char *ll = NULL; + int r; + + if (arg_pull_flags & PULL_DIRECT) { + + if (!local) + log_debug("Writing downloaded data to STDOUT."); + else { + if (!path_is_absolute(local)) { + ll = path_join(arg_image_root, local); + if (!ll) + return log_oom(); + + local = ll; + } + + if (!path_is_valid(local)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local path name '%s' is not valid.", local); + } + + } else if (local) { + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local image name '%s' is not valid.", + local); + + if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) { + r = image_find(IMAGE_MACHINE, local, NULL, NULL); + if (r < 0) { + if (r != -ENOENT) + return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); + } else + return log_error_errno(SYNTHETIC_ERRNO(EEXIST), + "Image '%s' already exists.", + local); + } + } + + if (!ll && local) { + ll = strdup(local); + if (!ll) + return log_oom(); + } + + if (ll) { + if (arg_offset != UINT64_MAX) + log_info("Pulling '%s', saving at offset %" PRIu64 " in '%s'.", url, arg_offset, ll); + else + log_info("Pulling '%s', saving as '%s'.", url, ll); + } else + log_info("Pulling '%s'.", url); + + *ret = TAKE_PTR(ll); return 0; } @@ -40,70 +103,57 @@ static void on_tar_finished(TarPull *pull, int error, void *userdata) { } static int pull_tar(int argc, char *argv[], void *userdata) { - _cleanup_(tar_pull_unrefp) TarPull *pull = NULL; + _cleanup_free_ char *ll = NULL, *normalized = NULL; _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(tar_pull_unrefp) TarPull *pull = NULL; const char *url, *local; - _cleanup_free_ char *l = NULL, *ll = NULL; int r; url = argv[1]; if (!http_url_is_valid(url)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "URL '%s' is not valid.", url); + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url); if (argc >= 3) - local = argv[2]; + local = empty_or_dash_to_null(argv[2]); else { + _cleanup_free_ char *l = NULL; + r = import_url_last_component(url, &l); if (r < 0) - return log_error_errno(r, "Failed get final component of URL: %m"); - - local = l; - } + return log_error_errno(r, "Failed to get final component of URL: %m"); - local = empty_or_dash_to_null(local); - - if (local) { - r = tar_strip_suffixes(local, &ll); + r = tar_strip_suffixes(l, &ll); if (r < 0) return log_oom(); local = ll; + } - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local image name '%s' is not valid.", - local); - - if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); - if (r < 0) { - if (r != -ENOENT) - return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); - } else { - return log_error_errno(SYNTHETIC_ERRNO(EEXIST), - "Image '%s' already exists.", - local); - } - } + if (!local && FLAGS_SET(arg_pull_flags, PULL_DIRECT)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Pulling tar images to STDOUT is not supported."); - log_info("Pulling '%s', saving as '%s'.", url, local); - } else - log_info("Pulling '%s'.", url); + r = normalize_local(local, url, &normalized); + if (r < 0) + return r; - r = sd_event_default(&event); + r = import_allocate_event_with_signals(&event); if (r < 0) - return log_error_errno(r, "Failed to allocate event loop: %m"); + return r; - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) + log_info("File system synchronization on completion is off."); r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate puller: %m"); - r = tar_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_TAR, arg_verify); + r = tar_pull_start( + pull, + url, + normalized, + arg_pull_flags & PULL_FLAGS_MASK_TAR, + arg_verify, + arg_checksum); if (r < 0) return log_error_errno(r, "Failed to pull image: %m"); @@ -126,70 +176,55 @@ static void on_raw_finished(RawPull *pull, int error, void *userdata) { } static int pull_raw(int argc, char *argv[], void *userdata) { - _cleanup_(raw_pull_unrefp) RawPull *pull = NULL; + _cleanup_free_ char *ll = NULL, *normalized = NULL; _cleanup_(sd_event_unrefp) sd_event *event = NULL; + _cleanup_(raw_pull_unrefp) RawPull *pull = NULL; const char *url, *local; - _cleanup_free_ char *l = NULL, *ll = NULL; int r; url = argv[1]; if (!http_url_is_valid(url)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "URL '%s' is not valid.", url); + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url); if (argc >= 3) - local = argv[2]; + local = empty_or_dash_to_null(argv[2]); else { + _cleanup_free_ char *l = NULL; + r = import_url_last_component(url, &l); if (r < 0) - return log_error_errno(r, "Failed get final component of URL: %m"); - - local = l; - } + return log_error_errno(r, "Failed to get final component of URL: %m"); - local = empty_or_dash_to_null(local); - - if (local) { - r = raw_strip_suffixes(local, &ll); + r = raw_strip_suffixes(l, &ll); if (r < 0) return log_oom(); local = ll; + } - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local image name '%s' is not valid.", - local); - - if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); - if (r < 0) { - if (r != -ENOENT) - return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); - } else { - return log_error_errno(SYNTHETIC_ERRNO(EEXIST), - "Image '%s' already exists.", - local); - } - } - - log_info("Pulling '%s', saving as '%s'.", url, local); - } else - log_info("Pulling '%s'.", url); - - r = sd_event_default(&event); + r = normalize_local(local, url, &normalized); if (r < 0) - return log_error_errno(r, "Failed to allocate event loop: %m"); + return r; - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + r = import_allocate_event_with_signals(&event); + if (r < 0) + return r; - r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); + if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) + log_info("File system synchronization on completion is off."); + r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate puller: %m"); - r = raw_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_RAW, arg_verify); + r = raw_pull_start( + pull, + url, + normalized, + arg_offset, + arg_size_max, + arg_pull_flags & PULL_FLAGS_MASK_RAW, + arg_verify, + arg_checksum); if (r < 0) return log_error_errno(r, "Failed to pull image: %m"); @@ -203,23 +238,39 @@ static int pull_raw(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { - printf("%s [OPTIONS...] {COMMAND} ...\n\n" - "Download container or virtual machine images.\n\n" + printf("%1$s [OPTIONS...] {COMMAND} ...\n" + "\n%4$sDownload container or virtual machine images.%5$s\n" + "\n%2$sCommands:%3$s\n" + " tar URL [NAME] Download a TAR image\n" + " raw URL [NAME] Download a RAW image\n" + "\n%2$sOptions:%3$s\n" " -h --help Show this help\n" " --version Show package version\n" " --force Force creation of image\n" " --verify=MODE Verify downloaded image, one of: 'no',\n" - " 'checksum', 'signature'\n" + " 'checksum', 'signature' or literal SHA256 hash\n" " --settings=BOOL Download settings file with image\n" " --roothash=BOOL Download root hash file with image\n" " --roothash-signature=BOOL\n" " Download root hash signature file with image\n" " --verity=BOOL Download verity file with image\n" " --image-root=PATH Image root directory\n\n" - "Commands:\n" - " tar URL [NAME] Download a TAR image\n" - " raw URL [NAME] Download a RAW image\n", - program_invocation_short_name); + " --read-only Create a read-only image\n" + " --direct Download directly to specified file\n" + " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" + " instead of a directory\n" + " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" + " subvolume\n" + " --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n" + " regular disk images\n" + " --sync=BOOL Controls whether to sync() before completing\n" + " --offset=BYTES Offset to seek to in destination\n" + " --size-max=BYTES Maximum number of bytes to write to destination\n", + program_invocation_short_name, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); return 0; } @@ -235,6 +286,14 @@ static int parse_argv(int argc, char *argv[]) { ARG_ROOTHASH, ARG_ROOTHASH_SIGNATURE, ARG_VERITY, + ARG_READ_ONLY, + ARG_DIRECT, + ARG_BTRFS_SUBVOL, + ARG_BTRFS_QUOTA, + ARG_CONVERT_QCOW2, + ARG_SYNC, + ARG_OFFSET, + ARG_SIZE_MAX, }; static const struct option options[] = { @@ -247,6 +306,14 @@ static int parse_argv(int argc, char *argv[]) { { "roothash", required_argument, NULL, ARG_ROOTHASH }, { "roothash-signature", required_argument, NULL, ARG_ROOTHASH_SIGNATURE }, { "verity", required_argument, NULL, ARG_VERITY }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "direct", no_argument, NULL, ARG_DIRECT }, + { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, + { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, + { "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 }, + { "sync", required_argument, NULL, ARG_SYNC }, + { "offset", required_argument, NULL, ARG_OFFSET }, + { "size-max", required_argument, NULL, ARG_SIZE_MAX }, {} }; @@ -273,26 +340,51 @@ static int parse_argv(int argc, char *argv[]) { arg_image_root = optarg; break; - case ARG_VERIFY: - arg_verify = import_verify_from_string(optarg); - if (arg_verify < 0) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Invalid verification setting '%s'", optarg); + case ARG_VERIFY: { + ImportVerify v; + + v = import_verify_from_string(optarg); + if (v < 0) { + _cleanup_free_ void *h = NULL; + char *hh; + size_t n; + + /* If this is not a vaid verification mode, maybe it's a literally specified + * SHA256 hash? We can handle that too... */ + + r = unhexmem(optarg, (size_t) -1, &h, &n); + if (r < 0 || n == 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Invalid verification setting: %s", optarg); + if (n != 32) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2); + + hh = hexmem(h, n); /* bring into canonical (lowercase) form */ + if (!hh) + return log_oom(); + + free_and_replace(arg_checksum, hh); + arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY); + arg_verify = _IMPORT_VERIFY_INVALID; + } else + arg_verify = v; break; + } case ARG_SETTINGS: - r = parse_boolean(optarg); + r = parse_boolean_argument("--settings=", optarg, NULL); if (r < 0) - return log_error_errno(r, "Failed to parse --settings= parameter '%s': %m", optarg); + return r; SET_FLAG(arg_pull_flags, PULL_SETTINGS, r); break; case ARG_ROOTHASH: - r = parse_boolean(optarg); + r = parse_boolean_argument("--roothash=", optarg, NULL); if (r < 0) - return log_error_errno(r, "Failed to parse --roothash= parameter '%s': %m", optarg); + return r; SET_FLAG(arg_pull_flags, PULL_ROOTHASH, r); @@ -302,21 +394,88 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_ROOTHASH_SIGNATURE: - r = parse_boolean(optarg); + r = parse_boolean_argument("--roothash-signature=", optarg, NULL); if (r < 0) - return log_error_errno(r, "Failed to parse --roothash-signature= parameter '%s': %m", optarg); + return r; SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, r); break; case ARG_VERITY: - r = parse_boolean(optarg); + r = parse_boolean_argument("--verity=", optarg, NULL); if (r < 0) - return log_error_errno(r, "Failed to parse --verity= parameter '%s': %m", optarg); + return r; SET_FLAG(arg_pull_flags, PULL_VERITY, r); break; + case ARG_READ_ONLY: + arg_pull_flags |= PULL_READ_ONLY; + break; + + case ARG_DIRECT: + arg_pull_flags |= PULL_DIRECT; + arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY); + break; + + case ARG_BTRFS_SUBVOL: + r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r); + break; + + case ARG_BTRFS_QUOTA: + r = parse_boolean_argument("--btrfs-quota=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r); + break; + + case ARG_CONVERT_QCOW2: + r = parse_boolean_argument("--convert-qcow2=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_CONVERT_QCOW2, r); + break; + + case ARG_SYNC: + r = parse_boolean_argument("--sync=", optarg, NULL); + if (r < 0) + return r; + + SET_FLAG(arg_pull_flags, PULL_SYNC, r); + break; + + case ARG_OFFSET: { + uint64_t u; + + r = safe_atou64(optarg, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg); + + arg_offset = u; + break; + } + + case ARG_SIZE_MAX: { + uint64_t u; + + r = parse_size(optarg, 1024, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg); + if (!FILE_SIZE_VALID(u)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg); + + arg_size_max = u; + break; + } + case '?': return -EINVAL; @@ -324,9 +483,46 @@ static int parse_argv(int argc, char *argv[]) { assert_not_reached(); } + /* Make sure offset+size is still in the valid range if both set */ + if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX && + ((arg_size_max > (UINT64_MAX - arg_offset)) || + !FILE_SIZE_VALID(arg_offset + arg_size_max))) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range."); + + if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_pull_flags, PULL_DIRECT)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); + + if (arg_checksum && (arg_pull_flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) != 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded."); + return 1; } +static void parse_env(void) { + int r; + + /* Let's make these relatively low-level settings also controllable via env vars. User can then set + * them for systemd-importd.service if they like to tweak behaviour */ + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL"); + if (r >= 0) + SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA"); + if (r >= 0) + SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m"); + + r = getenv_bool("SYSTEMD_IMPORT_SYNC"); + if (r >= 0) + SET_FLAG(arg_pull_flags, PULL_SYNC, r); + else if (r != -ENXIO) + log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m"); +} + static int pull_main(int argc, char *argv[]) { static const Verb verbs[] = { { "help", VERB_ANY, VERB_ANY, 0, help }, @@ -345,6 +541,8 @@ static int run(int argc, char *argv[]) { log_parse_environment(); log_open(); + parse_env(); + r = parse_argv(argc, argv); if (r <= 0) return r; diff --git a/src/shared/install-file.c b/src/shared/install-file.c new file mode 100644 index 0000000000..664f028ad1 --- /dev/null +++ b/src/shared/install-file.c @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include <sys/ioctl.h> + +#include "btrfs-util.h" +#include "chattr-util.h" +#include "errno-util.h" +#include "fd-util.h" +#include "fs-util.h" +#include "install-file.h" +#include "missing_syscall.h" +#include "rm-rf.h" + +int fs_make_very_read_only(int fd) { + struct stat st; + int r; + + assert(fd >= 0); + + /* Tries to make the specified fd "comprehensively" read-only. Primary usecase for this is OS images, + * i.e. either loopback files or larger directory hierarchies. Depending on the inode type and + * backing file system this means something different: + * + * 1. If the fd refers to a btrfs subvolume we'll mark it read-only as a whole + * 2. If the fd refers to any other directory we'll set the FS_IMMUTABLE_FL flag on it + * 3. If the fd refers to a regular file we'll drop the w bits. + * 4. If the fd refers to a block device, use BLKROSET to set read-only state + * + * You might wonder why not drop the x bits for directories. That's because we want to guarantee that + * everything "inside" the image remains largely the way it is, in case you mount it. And since the + * mode of the root dir of the image is pretty visible we don't want to modify it. btrfs subvol flags + * and the FS_IMMUTABLE_FL otoh are much less visible. Changing the mode of regular files should be + * OK though, since after all this is supposed to be used for disk images, i.e. the fs in the disk + * image doesn't make the mode of the loopback file it is stored in visible. */ + + if (fstat(fd, &st) < 0) + return -errno; + + switch (st.st_mode & S_IFMT) { + + case S_IFDIR: + if (btrfs_might_be_subvol(&st)) { + r = btrfs_subvol_set_read_only_fd(fd, true); + if (r >= 0) + return 0; + + if (!ERRNO_IS_NOT_SUPPORTED(r) && r != -EINVAL) + return r; + } + + r = chattr_fd(fd, FS_IMMUTABLE_FL, FS_IMMUTABLE_FL, NULL); + if (r < 0) + return r; + + break; + + case S_IFREG: + if ((st.st_mode & 0222) != 0) + if (fchmod(fd, st.st_mode & 07555) < 0) + return -errno; + + break; + + case S_IFBLK: { + int ro = 1; + + if (ioctl(fd, BLKROSET, &ro) < 0) + return -errno; + + break; + } + + default: + return -EBADFD; + } + + return 0; +} + +static int unlinkat_maybe_dir(int dirfd, const char *pathname) { + + /* Invokes unlinkat() for regular files first, and if this fails with EISDIR tries again with + * AT_REMOVEDIR */ + + if (unlinkat(dirfd, pathname, 0) < 0) { + if (errno != EISDIR) + return -errno; + + if (unlinkat(dirfd, pathname, AT_REMOVEDIR) < 0) + return -errno; + } + + return 0; +} + +int install_file(int source_atfd, const char *source_name, + int target_atfd, const char *target_name, + InstallFileFlags flags) { + + _cleanup_close_ int rofd = -1; + int r; + + /* Moves a file or directory tree into place, with some bells and whistles: + * + * 1. Optionally syncs before/after to ensure file installation can be used as barrier + * 2. Optionally marks the file/directory read-only using fs_make_very_read_only() + * 3. Optionally operates in replacing or in non-replacing mode. + * 4. If it replaces will remove the old tree if needed. + */ + + assert(source_atfd >= 0 || source_atfd == AT_FDCWD); + assert(source_name); + assert(target_atfd >= 0 || target_atfd == AT_FDCWD); + + /* If target_name is specified as NULL no renaming takes place. Instead it is assumed the file is + * already in place, and only the syncing/read-only marking shall be applied. Note that with + * target_name=NULL and flags=0 this call is a NOP */ + + if ((flags & (INSTALL_FSYNC|INSTALL_FSYNC_FULL|INSTALL_SYNCFS|INSTALL_READ_ONLY)) != 0) { + _cleanup_close_ int pfd = -1; + struct stat st; + + /* Open an O_PATH fd for the source if we need to sync things or mark things read only. */ + + pfd = openat(source_atfd, source_name, O_PATH|O_CLOEXEC|O_NOFOLLOW); + if (pfd < 0) + return -errno; + + if (fstat(pfd, &st) < 0) + return -errno; + + switch (st.st_mode & S_IFMT) { + + case S_IFREG: { + _cleanup_close_ int regfd = -1; + + regfd = fd_reopen(pfd, O_RDONLY|O_CLOEXEC); + if (regfd < 0) + return regfd; + + if ((flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) { + /* If this is just a regular file (as oppose to a fully populated directory) + * let's downgrade INSTALL_SYNCFS to INSTALL_FSYNC_FULL, after all this is + * going to be a single inode we install */ + r = fsync_full(regfd); + if (r < 0) + return r; + } else if (flags & INSTALL_FSYNC) { + if (fsync(regfd) < 0) + return -errno; + } + + if (flags & INSTALL_READ_ONLY) + rofd = TAKE_FD(regfd); + + break; + } + + case S_IFDIR: { + _cleanup_close_ int dfd = -1; + + dfd = fd_reopen(pfd, O_RDONLY|O_DIRECTORY|O_CLOEXEC); + if (dfd < 0) + return dfd; + + if (flags & INSTALL_SYNCFS) { + if (syncfs(dfd) < 0) + return -errno; + } else if (flags & INSTALL_FSYNC_FULL) { + r = fsync_full(dfd); + if (r < 0) + return r; + } else if (flags & INSTALL_FSYNC) { + if (fsync(dfd) < 0) + return -errno; + } + + if (flags & INSTALL_READ_ONLY) + rofd = TAKE_FD(dfd); + + break; + } + + default: + /* Other inodes: char/block device inodes, fifos, symlinks, sockets don't need + * syncing themselves, as they only exist in the directory, and have no contents on + * disk */ + + if (target_name && (flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) { + r = fsync_directory_of_file(pfd); + if (r < 0) + return r; + } + + break; + } + } + + if (target_name) { + /* Rename the file */ + + if (flags & INSTALL_REPLACE) { + /* First, try a simple renamat(), maybe that's enough */ + if (renameat(source_atfd, source_name, target_atfd, target_name) < 0) { + _cleanup_close_ int dfd = -1; + + if (!IN_SET(errno, EEXIST, ENOTDIR, ENOTEMPTY, EISDIR, EBUSY)) + return -errno; + + /* Hmm, the target apparently existed already. Let's try to use + * RENAME_EXCHANGE. But let's first open the inode if it's a directory, so + * that we can later remove its contents if it's a directory. Why do this + * before the rename()? Mostly because if we have trouble opening the thing + * we want to know before we start actually modifying the file system. */ + + dfd = openat(target_atfd, target_name, O_RDONLY|O_DIRECTORY|O_CLOEXEC, 0); + if (dfd < 0 && errno != ENOTDIR) + return -errno; + + if (renameat2(source_atfd, source_name, target_atfd, target_name, RENAME_EXCHANGE) < 0) { + + if (!ERRNO_IS_NOT_SUPPORTED(errno) && errno != EINVAL) + return -errno; + + /* The exchange didn't work, let's remove the target first, and try again */ + + if (dfd >= 0) + (void) rm_rf_children(TAKE_FD(dfd), REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD, NULL); + + r = unlinkat_maybe_dir(target_atfd, target_name); + if (r < 0) + return log_debug_errno(r, "Failed to remove target directory: %m"); + + if (renameat(source_atfd, source_name, target_atfd, target_name) < 0) + return -errno; + } else { + /* The exchange worked, hence let's remove the source (i.e. the old target) */ + if (dfd >= 0) + (void) rm_rf_children(TAKE_FD(dfd), REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD, NULL); + + r = unlinkat_maybe_dir(source_atfd, source_name); + if (r < 0) + return log_debug_errno(r, "Failed to remove replaced target directory: %m"); + } + } + } else { + r = rename_noreplace(source_atfd, source_name, target_atfd, target_name); + if (r < 0) + return r; + } + } + + if (rofd >= 0) { + r = fs_make_very_read_only(rofd); + if (r < 0) + return r; + } + + if ((flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) { + if (target_name) + r = fsync_parent_at(target_atfd, target_name); + else + r = fsync_parent_at(source_atfd, source_name); + if (r < 0) + return r; + } + + return 0; +} diff --git a/src/shared/install-file.h b/src/shared/install-file.h new file mode 100644 index 0000000000..c37254fa72 --- /dev/null +++ b/src/shared/install-file.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +int fs_make_very_read_only(int fd); + +typedef enum InstallFileFlags { + INSTALL_REPLACE = 1 << 0, /* Replace an existing inode */ + INSTALL_READ_ONLY = 1 << 1, /* Call fs_make_very_read_only() to make the inode comprehensively read-only */ + INSTALL_FSYNC = 1 << 2, /* fsync() file contents before moving file in */ + INSTALL_FSYNC_FULL = 1 << 3, /* like INSTALL_FSYNC, but also fsync() parent dir before+after moving file in */ + INSTALL_SYNCFS = 1 << 4, /* syncfs() before moving file in, fsync() parent dir after moving file in */ +} InstallFileFlags; + +int install_file(int source_atfd, const char *source_name, int target_atfd, const char *target_name, InstallFileFlags flags); diff --git a/src/shared/meson.build b/src/shared/meson.build index 1f741f7417..7eb71b287c 100644 --- a/src/shared/meson.build +++ b/src/shared/meson.build @@ -152,6 +152,8 @@ shared_sources = files(''' import-util.c import-util.h initreq.h + install-file.c + install-file.h install-printf.c install-printf.h install.c diff --git a/src/test/meson.build b/src/test/meson.build index e9204b150d..ad226418e2 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -215,6 +215,8 @@ tests += [ [['src/test/test-fs-util.c']], + [['src/test/test-install-file.c']], + [['src/test/test-umask-util.c']], [['src/test/test-proc-cmdline.c']], diff --git a/src/test/test-install-file.c b/src/test/test-install-file.c new file mode 100644 index 0000000000..55d77ac236 --- /dev/null +++ b/src/test/test-install-file.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "fileio.h" +#include "install-file.h" +#include "path-util.h" +#include "rm-rf.h" +#include "tests.h" +#include "tmpfile-util.h" +#include "umask-util.h" + +static void test_install_file(void) { + _cleanup_(rm_rf_physical_and_freep) char *p = NULL; + _cleanup_free_ char *a = NULL, *b = NULL, *c = NULL; + struct stat stat1, stat2; + + log_info("/* %s */", __func__); + + assert_se(mkdtemp_malloc(NULL, &p) >= 0); + assert_se(a = path_join(p, "foo")); + assert_se(b = path_join(p, "bar")); + + RUN_WITH_UMASK(0077) + assert_se(write_string_file(a, "wups", WRITE_STRING_FILE_CREATE) >= 0); + + assert_se(lstat(a, &stat1) >= 0); + assert_se(S_ISREG(stat1.st_mode)); + + assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, 0) >= 0); + assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC) >= 0); + + assert_se(write_string_file(b, "ttss", WRITE_STRING_FILE_CREATE) >= 0); + assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, INSTALL_FSYNC_FULL) == -EEXIST); + assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, INSTALL_FSYNC_FULL|INSTALL_REPLACE) >= 0); + + assert_se(stat(b, &stat2) >= 0); + assert_se(stat1.st_dev == stat2.st_dev); + assert_se(stat1.st_ino == stat2.st_ino); + assert_se((stat2.st_mode & 0222) != 0); /* writable */ + + assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE|INSTALL_READ_ONLY) >= 0); + + assert_se(stat(a, &stat2) >= 0); + assert_se(stat1.st_dev == stat2.st_dev); + assert_se(stat1.st_ino == stat2.st_ino); + assert_se((stat2.st_mode & 0222) == 0); /* read-only */ + + assert_se(mkdir(b, 0755) >= 0); + assert_se(c = path_join(b, "dir")); + assert_se(mkdir(c, 0755) >= 0); + free(c); + assert_se(c = path_join(b, "reg")); + assert_se(mknod(c, S_IFREG|0755, 0) >= 0); + free(c); + assert_se(c = path_join(b, "fifo")); + assert_se(mknod(c, S_IFIFO|0755, 0) >= 0); + + assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL) == -EEXIST); + assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE) == 0); + + assert_se(write_string_file(b, "ttss", WRITE_STRING_FILE_CREATE) >= 0); + + assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL) == -EEXIST); + assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE) == 0); +} + +int main(int argc, char *argv[]) { + test_setup_logging(LOG_INFO); + + test_install_file(); + + return 0; +} |