/* * Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define is_hook_enabled(hook) ({ (hook) && *(hook); }) /* * Protects against payload_list operations and also allows only one * caller in schedule_work. */ static DEFINE_SPINLOCK(payload_lock); static LIST_HEAD(payload_list); /* * Patches which have been applied. Need RCU in case we crash (and then * traps code would iterate via applied_list) when adding entries on the list. */ static DEFINE_RCU_READ_LOCK(rcu_applied_lock); static LIST_HEAD(applied_list); static unsigned int payload_cnt; static unsigned int payload_version = 1; /* Defines an outstanding patching action. */ struct livepatch_work { atomic_t semaphore; /* Used to rendezvous CPUs in check_for_livepatch_work. */ uint32_t timeout; /* Timeout to do the operation. */ struct payload *data; /* The payload on which to act. */ volatile bool_t do_work; /* Signals work to do. */ volatile bool_t ready; /* Signals all CPUs synchronized. */ unsigned int cmd; /* Action request: LIVEPATCH_ACTION_* */ }; /* There can be only one outstanding patching action. */ static struct livepatch_work livepatch_work; /* * Indicate whether the CPU needs to consult livepatch_work structure. * We want an per-cpu data structure otherwise the check_for_livepatch_work * would hammer a global livepatch_work structure on every guest VMEXIT. * Having an per-cpu lessens the load. */ static DEFINE_PER_CPU(bool_t, work_to_do); static DEFINE_PER_CPU(struct tasklet, livepatch_tasklet); static int get_name(const struct xen_livepatch_name *name, char *n) { if ( !name->size || name->size > XEN_LIVEPATCH_NAME_SIZE ) return -EINVAL; if ( name->pad[0] || name->pad[1] || name->pad[2] ) return -EINVAL; if ( copy_from_guest(n, name->name, name->size) ) return -EFAULT; if ( n[name->size - 1] ) return -EINVAL; return 0; } static int verify_payload(const struct xen_sysctl_livepatch_upload *upload, char *n) { if ( get_name(&upload->name, n) ) return -EINVAL; if ( !upload->size ) return -EINVAL; if ( upload->size > LIVEPATCH_MAX_SIZE ) return -EINVAL; if ( !guest_handle_okay(upload->payload, upload->size) ) return -EFAULT; return 0; } bool_t is_patch(const void *ptr) { const struct payload *data; bool_t r = 0; /* * Only RCU locking since this list is only ever changed during apply * or revert context. And in case it dies there we need an safe list. */ rcu_read_lock(&rcu_applied_lock); list_for_each_entry_rcu ( data, &applied_list, applied_list ) { if ( (ptr >= data->rw_addr && ptr < (data->rw_addr + data->rw_size)) || (ptr >= data->ro_addr && ptr < (data->ro_addr + data->ro_size)) || (ptr >= data->text_addr && ptr < (data->text_addr + data->text_size)) ) { r = 1; break; } } rcu_read_unlock(&rcu_applied_lock); return r; } unsigned long livepatch_symbols_lookup_by_name(const char *symname) { const struct payload *data; ASSERT(spin_is_locked(&payload_lock)); list_for_each_entry ( data, &payload_list, list ) { unsigned int i; for ( i = 0; i < data->nsyms; i++ ) { if ( !data->symtab[i].new_symbol ) continue; if ( !strcmp(data->symtab[i].name, symname) ) return data->symtab[i].value; } } return 0; } static const char *cf_check livepatch_symbols_lookup( unsigned long addr, unsigned long *symbolsize, unsigned long *offset, char *namebuf) { const struct payload *data; unsigned int i, best; const void *va = (const void *)addr; const char *n = NULL; /* * Only RCU locking since this list is only ever changed during apply * or revert context. And in case it dies there we need an safe list. */ rcu_read_lock(&rcu_applied_lock); list_for_each_entry_rcu ( data, &applied_list, applied_list ) { if ( va < data->text_addr || va >= (data->text_addr + data->text_size) ) continue; best = UINT_MAX; for ( i = 0; i < data->nsyms; i++ ) { if ( data->symtab[i].value <= addr && (best == UINT_MAX || data->symtab[best].value < data->symtab[i].value) ) best = i; } if ( best == UINT_MAX ) break; if ( symbolsize ) *symbolsize = data->symtab[best].size; if ( offset ) *offset = addr - data->symtab[best].value; if ( namebuf ) strlcpy(namebuf, data->name, KSYM_NAME_LEN); n = data->symtab[best].name; break; } rcu_read_unlock(&rcu_applied_lock); return n; } /* Lookup function's old address if not already resolved. */ static int resolve_old_address(struct livepatch_func *f, const struct livepatch_elf *elf) { if ( f->old_addr ) return 0; f->old_addr = (void *)symbols_lookup_by_name(f->name); if ( !f->old_addr ) { f->old_addr = (void *)livepatch_symbols_lookup_by_name(f->name); if ( !f->old_addr ) { printk(XENLOG_ERR LIVEPATCH "%s: Could not resolve old address of %s\n", elf->name, f->name); return -ENOENT; } } dprintk(XENLOG_DEBUG, LIVEPATCH "%s: Resolved old address %s => %p\n", elf->name, f->name, f->old_addr); return 0; } static struct payload *find_payload(const char *name) { struct payload *data, *found = NULL; ASSERT(spin_is_locked(&payload_lock)); list_for_each_entry ( data, &payload_list, list ) { if ( !strcmp(data->name, name) ) { found = data; break; } } return found; } /* * Functions related to XEN_SYSCTL_LIVEPATCH_UPLOAD (see livepatch_upload), and * freeing payload (XEN_SYSCTL_LIVEPATCH_ACTION:LIVEPATCH_ACTION_UNLOAD). */ static void free_payload_data(struct payload *payload) { /* Set to zero until "move_payload". */ if ( !payload->pages ) return; vfree((void *)payload->text_addr); payload->pages = 0; } /* * calc_section computes the size (taking into account section alignment). * * Furthermore the offset is set with the offset from the start of the virtual * address space for the payload (using passed in size). This is used in * move_payload to figure out the destination location (load_addr). */ static void calc_section(const struct livepatch_elf_sec *sec, size_t *size, unsigned int *offset) { const Elf_Shdr *s = sec->sec; size_t align_size; align_size = ROUNDUP(*size, s->sh_addralign); *offset = align_size; *size = s->sh_size + align_size; } static int move_payload(struct payload *payload, struct livepatch_elf *elf) { void *text_buf, *ro_buf, *rw_buf; unsigned int i, rw_buf_sec, rw_buf_cnt = 0; size_t size = 0; unsigned int *offset; int rc = 0; offset = xmalloc_array(unsigned int, elf->hdr->e_shnum); if ( !offset ) return -ENOMEM; /* Compute size of different regions. */ for ( i = 1; i < elf->hdr->e_shnum; i++ ) { /* * Do nothing. These are .rel.text, rel.*, .symtab, .strtab, * and .shstrtab. For the non-relocate we allocate and copy these * via other means - and the .rel we can ignore as we only use it * once during loading. */ if ( livepatch_elf_ignore_section(elf->sec[i].sec) ) offset[i] = UINT_MAX; else if ( (elf->sec[i].sec->sh_flags & SHF_EXECINSTR) && !(elf->sec[i].sec->sh_flags & SHF_WRITE) ) calc_section(&elf->sec[i], &payload->text_size, &offset[i]); else if ( !(elf->sec[i].sec->sh_flags & SHF_EXECINSTR) && (elf->sec[i].sec->sh_flags & SHF_WRITE) ) calc_section(&elf->sec[i], &payload->rw_size, &offset[i]); else if ( !(elf->sec[i].sec->sh_flags & SHF_EXECINSTR) && !(elf->sec[i].sec->sh_flags & SHF_WRITE) ) calc_section(&elf->sec[i], &payload->ro_size, &offset[i]); else { dprintk(XENLOG_DEBUG, LIVEPATCH "%s: Not supporting %s section!\n", elf->name, elf->sec[i].name); rc = -EOPNOTSUPP; goto out; } } /* * Total of all three regions - RX, RW, and RO. We have to have * keep them in separate pages so we PAGE_ALIGN the RX and RW to have * them on separate pages. The last one will by default fall on its * own page. */ size = PAGE_ALIGN(payload->text_size) + PAGE_ALIGN(payload->rw_size) + payload->ro_size; size = PFN_UP(size); /* Nr of pages. */ text_buf = vmalloc_xen(size * PAGE_SIZE); if ( !text_buf ) { printk(XENLOG_ERR LIVEPATCH "%s: Could not allocate memory for payload\n", elf->name); rc = -ENOMEM; goto out; } rw_buf = text_buf + PAGE_ALIGN(payload->text_size); ro_buf = rw_buf + PAGE_ALIGN(payload->rw_size); payload->pages = size; payload->text_addr = text_buf; payload->rw_addr = rw_buf; payload->ro_addr = ro_buf; for ( i = 1; i < elf->hdr->e_shnum; i++ ) { if ( !livepatch_elf_ignore_section(elf->sec[i].sec) ) { void *buf; if ( elf->sec[i].sec->sh_flags & SHF_EXECINSTR ) buf = text_buf; else if ( elf->sec[i].sec->sh_flags & SHF_WRITE ) { buf = rw_buf; if ( elf->sec[i].sec->sh_size ) { /* * Special handling of RW empty regions: do not account for * them in order to decide whether a patch can safely be * re-applied, but assign them a load address so symbol * resolution and relocations work. */ rw_buf_sec = i; rw_buf_cnt++; } } else buf = ro_buf; ASSERT(offset[i] != UINT_MAX); elf->sec[i].load_addr = buf + offset[i]; /* Don't copy NOBITS - such as BSS. */ if ( elf->sec[i].sec->sh_type != SHT_NOBITS ) { memcpy(elf->sec[i].load_addr, elf->sec[i].data, elf->sec[i].sec->sh_size); dprintk(XENLOG_DEBUG, LIVEPATCH "%s: Loaded %s at %p\n", elf->name, elf->sec[i].name, elf->sec[i].load_addr); } else memset(elf->sec[i].load_addr, 0, elf->sec[i].sec->sh_size); } } /* * Only one RW section with non-zero size: .livepatch.funcs, * or only RO sections. */ if ( !rw_buf_cnt || (rw_buf_cnt == 1 && !strcmp(elf->sec[rw_buf_sec].name, ELF_LIVEPATCH_FUNC)) ) payload->safe_to_reapply = true; out: xfree(offset); return rc; } static int secure_payload(struct payload *payload, struct livepatch_elf *elf) { int rc = 0; unsigned int text_pages, rw_pages, ro_pages; text_pages = PFN_UP(payload->text_size); if ( text_pages ) { rc = arch_livepatch_secure(payload->text_addr, text_pages, LIVEPATCH_VA_RX); if ( rc ) return rc; } rw_pages = PFN_UP(payload->rw_size); if ( rw_pages ) { rc = arch_livepatch_secure(payload->rw_addr, rw_pages, LIVEPATCH_VA_RW); if ( rc ) return rc; } ro_pages = PFN_UP(payload->ro_size); if ( ro_pages ) rc = arch_livepatch_secure(payload->ro_addr, ro_pages, LIVEPATCH_VA_RO); ASSERT(ro_pages + rw_pages + text_pages == payload->pages); return rc; } static bool section_ok(const struct livepatch_elf *elf, const struct livepatch_elf_sec *sec, size_t sz) { if ( !elf || !sec ) return false; if ( sec->sec->sh_size % sz ) { printk(XENLOG_ERR LIVEPATCH "%s: Wrong size %"PRIuElfWord" of %s (must be multiple of %zu)\n", elf->name, sec->sec->sh_size, sec->name, sz); return false; } return true; } static int xen_build_id_dep(const struct payload *payload) { const void *id = NULL; unsigned int len = 0; int rc; ASSERT(payload->xen_dep.len); ASSERT(payload->xen_dep.p); rc = xen_build_id(&id, &len); if ( rc ) return rc; if ( payload->xen_dep.len != len || memcmp(id, payload->xen_dep.p, len) ) { printk(XENLOG_ERR LIVEPATCH "%s: check against hypervisor build-id failed\n", payload->name); return -EINVAL; } return 0; } static int check_special_sections(const struct livepatch_elf *elf) { unsigned int i; static const char *const names[] = { ELF_LIVEPATCH_DEPENDS, ELF_LIVEPATCH_XEN_DEPENDS, ELF_BUILD_ID_NOTE}; DECLARE_BITMAP(found, ARRAY_SIZE(names)) = { 0 }; for ( i = 0; i < ARRAY_SIZE(names); i++ ) { const struct livepatch_elf_sec *sec; sec = livepatch_elf_sec_by_name(elf, names[i]); if ( !sec ) { printk(XENLOG_ERR LIVEPATCH "%s: %s is missing\n", elf->name, names[i]); return -EINVAL; } if ( !sec->sec->sh_size ) { printk(XENLOG_ERR LIVEPATCH "%s: %s is empty\n", elf->name, names[i]); return -EINVAL; } if ( test_and_set_bit(i, found) ) { printk(XENLOG_ERR LIVEPATCH "%s: %s was seen more than once\n", elf->name, names[i]); return -EINVAL; } } return 0; } static int check_patching_sections(const struct livepatch_elf *elf) { unsigned int i; static const char *const names[] = { ELF_LIVEPATCH_FUNC, ELF_LIVEPATCH_LOAD_HOOKS, ELF_LIVEPATCH_UNLOAD_HOOKS, ELF_LIVEPATCH_PREAPPLY_HOOK, ELF_LIVEPATCH_APPLY_HOOK, ELF_LIVEPATCH_POSTAPPLY_HOOK, ELF_LIVEPATCH_PREREVERT_HOOK, ELF_LIVEPATCH_REVERT_HOOK, ELF_LIVEPATCH_POSTREVERT_HOOK}; DECLARE_BITMAP(found, ARRAY_SIZE(names)) = { 0 }; /* * The patching sections are optional, but at least one * must be present. Otherwise, there is nothing to do. * All the existing sections must not be empty and must * be present at most once. */ for ( i = 0; i < ARRAY_SIZE(names); i++ ) { const struct livepatch_elf_sec *sec; sec = livepatch_elf_sec_by_name(elf, names[i]); if ( !sec ) { dprintk(XENLOG_DEBUG, LIVEPATCH "%s: %s is missing\n", elf->name, names[i]); continue; /* This section is optional */ } if ( !sec->sec->sh_size ) { printk(XENLOG_ERR LIVEPATCH "%s: %s is empty\n", elf->name, names[i]); return -EINVAL; } if ( test_and_set_bit(i, found) ) { printk(XENLOG_ERR LIVEPATCH "%s: %s was seen more than once\n", elf->name, names[i]); return -EINVAL; } } /* Checking if at least one section is present. */ if ( bitmap_empty(found, ARRAY_SIZE(names)) ) { printk(XENLOG_ERR LIVEPATCH "%s: Nothing to patch. Aborting...\n", elf->name); return -EINVAL; } return 0; } static inline int livepatch_verify_expectation_fn(const struct livepatch_func *func) { const livepatch_expectation_t *exp = &func->expect; /* Ignore disabled expectations. */ if ( !exp->enabled ) return 0; /* There is nothing to expect */ if ( !func->old_addr ) return -EFAULT; if ( exp->len > sizeof(exp->data)) return -EOVERFLOW; if ( exp->rsv ) return -EINVAL; /* Incorrect expectation */ if ( func->old_size < exp->len ) return -ERANGE; if ( memcmp(func->old_addr, exp->data, exp->len) ) { printk(XENLOG_ERR LIVEPATCH "%s: expectation failed: expected:%*phN, actual:%*phN\n", func->name, exp->len, exp->data, exp->len, func->old_addr); return -EINVAL; } return 0; } static inline int livepatch_check_expectations(const struct payload *payload) { int i, rc; printk(XENLOG_INFO LIVEPATCH "%s: Verifying enabled expectations for all functions\n", payload->name); for ( i = 0; i < payload->nfuncs; i++ ) { const struct livepatch_func *func = &(payload->funcs[i]); rc = livepatch_verify_expectation_fn(func); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: expectations of %s failed (rc=%d), aborting!\n", payload->name, func->name ?: "unknown", rc); return rc; } } return 0; } /* * Lookup specified section and when exists assign its address to a specified hook. * Perform section pointer and size validation: single hook sections must contain a * single pointer only. */ #define LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, hook, section_name) do { \ const struct livepatch_elf_sec *__sec = livepatch_elf_sec_by_name(elf, section_name); \ if ( !__sec ) \ break; \ if ( !section_ok(elf, __sec, sizeof(*hook)) || __sec->sec->sh_size != sizeof(*hook) ) \ return -EINVAL; \ hook = __sec->load_addr; \ } while (0) /* * Lookup specified section and when exists assign its address to a specified hook. * Perform section pointer and size validation: multi hook sections must contain an * array whose size must be a multiple of the array's items size. */ #define LIVEPATCH_ASSIGN_MULTI_HOOK(elf, hook, nhooks, section_name) do { \ const struct livepatch_elf_sec *__sec = livepatch_elf_sec_by_name(elf, section_name); \ if ( !__sec ) \ break; \ if ( !section_ok(elf, __sec, sizeof(*hook)) ) \ return -EINVAL; \ hook = __sec->load_addr; \ nhooks = __sec->sec->sh_size / sizeof(*hook); \ } while (0) static int prepare_payload(struct payload *payload, struct livepatch_elf *elf) { const struct livepatch_elf_sec *sec; unsigned int i; struct livepatch_func *f; struct virtual_region *region; const Elf_Note *n; sec = livepatch_elf_sec_by_name(elf, ELF_LIVEPATCH_FUNC); if ( sec ) { if ( !section_ok(elf, sec, sizeof(*payload->funcs)) ) return -EINVAL; payload->funcs = sec->load_addr; payload->nfuncs = sec->sec->sh_size / sizeof(*payload->funcs); for ( i = 0; i < payload->nfuncs; i++ ) { int rc; f = &(payload->funcs[i]); if ( f->version != LIVEPATCH_PAYLOAD_VERSION ) { printk(XENLOG_ERR LIVEPATCH "%s: Wrong version (%u). Expected %d\n", elf->name, f->version, LIVEPATCH_PAYLOAD_VERSION); return -EOPNOTSUPP; } /* 'old_addr', 'new_addr', 'new_size' can all be zero. */ if ( !f->old_size ) { printk(XENLOG_ERR LIVEPATCH "%s: Address or size fields are zero\n", elf->name); return -EINVAL; } rc = resolve_old_address(f, elf); if ( rc ) return rc; rc = arch_livepatch_verify_func(f); if ( rc ) return rc; rc = livepatch_verify_distance(f); if ( rc ) return rc; } } LIVEPATCH_ASSIGN_MULTI_HOOK(elf, payload->load_funcs, payload->n_load_funcs, ELF_LIVEPATCH_LOAD_HOOKS); LIVEPATCH_ASSIGN_MULTI_HOOK(elf, payload->unload_funcs, payload->n_unload_funcs, ELF_LIVEPATCH_UNLOAD_HOOKS); LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, payload->hooks.apply.pre, ELF_LIVEPATCH_PREAPPLY_HOOK); LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, payload->hooks.apply.action, ELF_LIVEPATCH_APPLY_HOOK); LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, payload->hooks.apply.post, ELF_LIVEPATCH_POSTAPPLY_HOOK); LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, payload->hooks.revert.pre, ELF_LIVEPATCH_PREREVERT_HOOK); LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, payload->hooks.revert.action, ELF_LIVEPATCH_REVERT_HOOK); LIVEPATCH_ASSIGN_SINGLE_HOOK(elf, payload->hooks.revert.post, ELF_LIVEPATCH_POSTREVERT_HOOK); sec = livepatch_elf_sec_by_name(elf, ELF_BUILD_ID_NOTE); if ( sec ) { const struct payload *data; n = sec->load_addr; if ( sec->sec->sh_size <= sizeof(*n) ) return -EINVAL; if ( xen_build_id_check(n, sec->sec->sh_size, &payload->id.p, &payload->id.len) ) return -EINVAL; if ( !payload->id.len || !payload->id.p ) return -EINVAL; /* Make sure it is not a duplicate. */ list_for_each_entry ( data, &payload_list, list ) { /* No way _this_ payload is on the list. */ ASSERT(data != payload); if ( data->id.len == payload->id.len && !memcmp(data->id.p, payload->id.p, data->id.len) ) { dprintk(XENLOG_DEBUG, LIVEPATCH "%s: Already loaded as %s!\n", elf->name, data->name); return -EEXIST; } } } sec = livepatch_elf_sec_by_name(elf, ELF_LIVEPATCH_DEPENDS); if ( sec ) { n = sec->load_addr; if ( sec->sec->sh_size <= sizeof(*n) ) return -EINVAL; if ( xen_build_id_check(n, sec->sec->sh_size, &payload->dep.p, &payload->dep.len) ) return -EINVAL; if ( !payload->dep.len || !payload->dep.p ) return -EINVAL; } sec = livepatch_elf_sec_by_name(elf, ELF_LIVEPATCH_XEN_DEPENDS); if ( sec ) { n = sec->load_addr; if ( sec->sec->sh_size <= sizeof(*n) ) return -EINVAL; if ( xen_build_id_check(n, sec->sec->sh_size, &payload->xen_dep.p, &payload->xen_dep.len) ) return -EINVAL; if ( !payload->xen_dep.len || !payload->xen_dep.p ) return -EINVAL; } /* Setup the virtual region with proper data. */ region = &payload->region; region->symbols_lookup = livepatch_symbols_lookup; region->start = payload->text_addr; region->end = payload->text_addr + payload->text_size; /* Optional sections. */ for ( i = 0; i < BUGFRAME_NR; i++ ) { char str[14]; snprintf(str, sizeof(str), ".bug_frames.%u", i); sec = livepatch_elf_sec_by_name(elf, str); if ( !sec ) continue; if ( !section_ok(elf, sec, sizeof(*region->frame[i].bugs)) ) return -EINVAL; region->frame[i].bugs = sec->load_addr; region->frame[i].n_bugs = sec->sec->sh_size / sizeof(*region->frame[i].bugs); } sec = livepatch_elf_sec_by_name(elf, ".altinstructions"); if ( sec ) { #ifdef CONFIG_HAS_ALTERNATIVE /* * (As of April 2023), Alternatives are formed of: * - An .altinstructions section with an array of struct alt_instr's. * - An .altinstr_replacement section containing instructions. * * An individual alt_instr contains: * - An orig reference, pointing into .text with a nonzero length * - A repl reference, pointing into .altinstr_replacement * * It is legal to have zero-length replacements, meaning it is legal * for the .altinstr_replacement section to be empty too. An * implementation detail means that a zero-length replacement's repl * reference will still be in the .altinstr_replacement section. */ const struct livepatch_elf_sec *repl_sec; struct alt_instr *a, *start, *end; if ( !section_ok(elf, sec, sizeof(*a)) ) return -EINVAL; /* Tolerate an empty .altinstructions section... */ if ( sec->sec->sh_size == 0 ) goto alt_done; /* ... but otherwise, there needs to be something to alter... */ if ( payload->text_size == 0 ) { printk(XENLOG_ERR LIVEPATCH "%s Alternatives provided, but no .text\n", elf->name); return -EINVAL; } /* ... and something to be altered to. */ repl_sec = livepatch_elf_sec_by_name(elf, ".altinstr_replacement"); if ( !repl_sec ) { printk(XENLOG_ERR LIVEPATCH "%s .altinstructions provided, but no .altinstr_replacement\n", elf->name); return -EINVAL; } start = sec->load_addr; end = sec->load_addr + sec->sec->sh_size; for ( a = start; a < end; a++ ) { const void *orig = ALT_ORIG_PTR(a); const void *repl = ALT_REPL_PTR(a); /* orig must be fully within .text. */ if ( orig < payload->text_addr || a->orig_len > payload->text_size || orig + a->orig_len > payload->text_addr + payload->text_size ) { printk(XENLOG_ERR LIVEPATCH "%s Alternative orig %p+%#x outside payload text %p+%#zx\n", elf->name, orig, a->orig_len, payload->text_addr, payload->text_size); return -EINVAL; } /* * repl must be fully within .altinstr_replacement, even if the * replacement and the section happen to both have zero length. */ if ( repl < repl_sec->load_addr || a->repl_len > repl_sec->sec->sh_size || repl + a->repl_len > repl_sec->load_addr + repl_sec->sec->sh_size ) { printk(XENLOG_ERR LIVEPATCH "%s Alternative repl %p+%#x outside .altinstr_replacement %p+%#"PRIxElfWord"\n", elf->name, repl, a->repl_len, repl_sec->load_addr, repl_sec->sec->sh_size); return -EINVAL; } } apply_alternatives(start, end); alt_done:; #else printk(XENLOG_ERR LIVEPATCH "%s: We don't support alternative patching\n", elf->name); return -EOPNOTSUPP; #endif } sec = livepatch_elf_sec_by_name(elf, ".ex_table"); if ( sec ) { #ifdef CONFIG_HAS_EX_TABLE struct exception_table_entry *s, *e; if ( !section_ok(elf, sec, sizeof(*region->ex)) ) return -EINVAL; s = sec->load_addr; e = sec->load_addr + sec->sec->sh_size; sort_exception_table(s ,e); region->ex = s; region->ex_end = e; #else printk(XENLOG_ERR LIVEPATCH "%s: We don't support .ex_table\n", elf->name); return -EOPNOTSUPP; #endif } sec = livepatch_elf_sec_by_name(elf, ".modinfo"); if ( sec ) { if ( !section_ok(elf, sec, sizeof(*payload->metadata.data)) ) return -EINVAL; payload->metadata.data = sec->load_addr; payload->metadata.len = sec->sec->sh_size; /* The metadata is required to consists of null terminated strings. */ if ( payload->metadata.data[payload->metadata.len - 1] != '\0' ) { printk(XENLOG_ERR LIVEPATCH "%s: Incorrect metadata format detected\n", payload->name); return -EINVAL; } } return 0; } static bool_t is_payload_symbol(const struct livepatch_elf *elf, const struct livepatch_elf_sym *sym) { if ( sym->sym->st_shndx == SHN_UNDEF || sym->sym->st_shndx >= elf->hdr->e_shnum ) return 0; /* * The payload is not a final image as we dynamically link against it. * As such the linker has left symbols we don't care about and which * binutils would have removed had it be a final image. Hence we: * - For SHF_ALLOC - ignore symbols referring to sections that are not * loaded. */ if ( !(elf->sec[sym->sym->st_shndx].sec->sh_flags & SHF_ALLOC) ) return 0; /* - And ignore empty symbols (\0). */ if ( *sym->name == '\0' ) return 0; /* * - For SHF_MERGE - ignore local symbols referring to mergeable sections. * (ld squashes them all in one section and discards the symbols) when * those symbols start with '.L' (like .LCx). Those are intermediate * artifacts of assembly. * * See elf_link_input_bfd and _bfd_elf_is_local_label_name in binutils. */ if ( (elf->sec[sym->sym->st_shndx].sec->sh_flags & SHF_MERGE) && !strncmp(sym->name, ".L", 2) ) return 0; return arch_livepatch_symbol_ok(elf, sym); } static int build_symbol_table(struct payload *payload, const struct livepatch_elf *elf) { unsigned int i, j, nsyms = 0; size_t strtab_len = 0; struct livepatch_symbol *symtab; char *strtab; /* Recall that section @0 is always NULL. */ for ( i = 1; i < elf->nsym; i++ ) { if ( is_payload_symbol(elf, elf->sym + i) ) { nsyms++; strtab_len += strlen(elf->sym[i].name) + 1; } } symtab = xzalloc_array(struct livepatch_symbol, nsyms); strtab = xzalloc_array(char, strtab_len); if ( !strtab || !symtab ) { xfree(strtab); xfree(symtab); return -ENOMEM; } nsyms = 0; strtab_len = 0; for ( i = 1; i < elf->nsym; i++ ) { if ( is_payload_symbol(elf, elf->sym + i) ) { symtab[nsyms].name = strtab + strtab_len; symtab[nsyms].size = elf->sym[i].sym->st_size; symtab[nsyms].value = elf->sym[i].sym->st_value; symtab[nsyms].new_symbol = 0; /* May be overwritten below. */ strtab_len += strlcpy(strtab + strtab_len, elf->sym[i].name, KSYM_NAME_LEN) + 1; nsyms++; } } for ( i = 0; i < nsyms; i++ ) { bool_t found = 0; for ( j = 0; j < payload->nfuncs; j++ ) { if ( symtab[i].value == (unsigned long)payload->funcs[j].new_addr ) { found = 1; break; } } if ( !found ) { if ( symbols_lookup_by_name(symtab[i].name) || livepatch_symbols_lookup_by_name(symtab[i].name) ) { printk(XENLOG_ERR LIVEPATCH "%s: duplicate new symbol: %s\n", elf->name, symtab[i].name); xfree(symtab); xfree(strtab); return -EEXIST; } symtab[i].new_symbol = 1; dprintk(XENLOG_DEBUG, LIVEPATCH "%s: new symbol %s\n", elf->name, symtab[i].name); } else { /* new_symbol is not set. */ dprintk(XENLOG_DEBUG, LIVEPATCH "%s: overriding symbol %s\n", elf->name, symtab[i].name); } } payload->symtab = symtab; payload->strtab = strtab; payload->nsyms = nsyms; return 0; } static void free_payload(struct payload *data) { ASSERT(spin_is_locked(&payload_lock)); list_del(&data->list); payload_cnt--; payload_version++; free_payload_data(data); xfree((void *)data->symtab); xfree((void *)data->strtab); xfree(data); } static int load_payload_data(struct payload *payload, void *raw, size_t len) { struct livepatch_elf elf = { .name = payload->name, .len = len }; int rc = 0; rc = livepatch_elf_load(&elf, raw); if ( rc ) goto out; rc = move_payload(payload, &elf); if ( rc ) goto out; rc = livepatch_elf_resolve_symbols(&elf); if ( rc ) goto out; rc = livepatch_elf_perform_relocs(&elf); if ( rc ) goto out; rc = check_special_sections(&elf); if ( rc ) goto out; rc = check_patching_sections(&elf); if ( rc ) goto out; rc = prepare_payload(payload, &elf); if ( rc ) goto out; rc = xen_build_id_dep(payload); if ( rc ) goto out; rc = build_symbol_table(payload, &elf); if ( rc ) goto out; rc = secure_payload(payload, &elf); out: if ( rc ) free_payload_data(payload); /* Free our temporary data structure. */ livepatch_elf_free(&elf); return rc; } static int livepatch_upload(struct xen_sysctl_livepatch_upload *upload) { struct payload *data, *found; char n[XEN_LIVEPATCH_NAME_SIZE]; void *raw_data; int rc; rc = verify_payload(upload, n); if ( rc ) return rc; data = xzalloc(struct payload); raw_data = vmalloc(upload->size); spin_lock(&payload_lock); found = find_payload(n); if ( IS_ERR(found) ) rc = PTR_ERR(found); else if ( found ) rc = -EEXIST; else if ( !data || !raw_data ) rc = -ENOMEM; else if ( __copy_from_guest(raw_data, upload->payload, upload->size) ) rc = -EFAULT; else { memcpy(data->name, n, strlen(n)); rc = load_payload_data(data, raw_data, upload->size); if ( rc ) goto out; data->state = LIVEPATCH_STATE_CHECKED; INIT_LIST_HEAD(&data->list); INIT_LIST_HEAD(&data->applied_list); list_add_tail(&data->list, &payload_list); payload_cnt++; payload_version++; } out: spin_unlock(&payload_lock); vfree(raw_data); if ( rc && data ) { xfree((void *)data->symtab); xfree((void *)data->strtab); xfree(data); } return rc; } static int livepatch_get(struct xen_sysctl_livepatch_get *get) { struct payload *data; int rc; char n[XEN_LIVEPATCH_NAME_SIZE]; rc = get_name(&get->name, n); if ( rc ) return rc; spin_lock(&payload_lock); data = find_payload(n); if ( IS_ERR_OR_NULL(data) ) { spin_unlock(&payload_lock); if ( !data ) return -ENOENT; return PTR_ERR(data); } get->status.state = data->state; get->status.rc = data->rc; spin_unlock(&payload_lock); return 0; } static int livepatch_list(struct xen_sysctl_livepatch_list *list) { struct xen_livepatch_status status; struct payload *data; unsigned int idx = 0, i = 0; int rc = 0; if ( list->nr > 1024 ) return -E2BIG; if ( list->pad ) return -EINVAL; if ( list->nr && (!guest_handle_okay(list->status, list->nr) || !guest_handle_okay(list->len, list->nr) || !guest_handle_okay(list->metadata_len, list->nr)) ) return -EINVAL; spin_lock(&payload_lock); if ( list->idx >= payload_cnt && payload_cnt ) { spin_unlock(&payload_lock); return -EINVAL; } list->name_total_size = 0; list->metadata_total_size = 0; if ( list->nr ) { uint64_t name_offset = 0, metadata_offset = 0; list_for_each_entry( data, &payload_list, list ) { uint32_t name_len, metadata_len; if ( list->idx > i++ ) continue; status.state = data->state; status.rc = data->rc; name_len = strlen(data->name) + 1; list->name_total_size += name_len; metadata_len = data->metadata.len; list->metadata_total_size += metadata_len; if ( !guest_handle_subrange_okay(list->name, name_offset, name_offset + name_len - 1) || !guest_handle_subrange_okay(list->metadata, metadata_offset, metadata_offset + metadata_len - 1) ) { rc = -EINVAL; break; } /* N.B. 'idx' != 'i'. */ if ( __copy_to_guest_offset(list->name, name_offset, data->name, name_len) || __copy_to_guest_offset(list->len, idx, &name_len, 1) || __copy_to_guest_offset(list->status, idx, &status, 1) || __copy_to_guest_offset(list->metadata, metadata_offset, data->metadata.data, metadata_len) || __copy_to_guest_offset(list->metadata_len, idx, &metadata_len, 1) ) { rc = -EFAULT; break; } idx++; name_offset += name_len; metadata_offset += metadata_len; if ( (idx >= list->nr) || hypercall_preempt_check() ) break; } } else { list_for_each_entry( data, &payload_list, list ) { list->name_total_size += strlen(data->name) + 1; list->metadata_total_size += data->metadata.len; } } list->nr = payload_cnt - i; /* Remaining amount. */ list->version = payload_version; spin_unlock(&payload_lock); /* And how many we have processed. */ return rc ? : idx; } /* * The following functions get the CPUs into an appropriate state and * apply (or revert) each of the payload's functions. This is needed * for XEN_SYSCTL_LIVEPATCH_ACTION operation (see livepatch_action). */ static inline void livepatch_display_metadata(const struct livepatch_metadata *metadata) { const char *str; if ( metadata && metadata->data && metadata->len > 0 ) { printk(XENLOG_INFO LIVEPATCH "module metadata:\n"); for ( str = metadata->data; str < (metadata->data + metadata->len); str += (strlen(str) + 1) ) printk(XENLOG_INFO LIVEPATCH " %s\n", str); } } static int apply_payload(struct payload *data) { unsigned int i; int rc; rc = arch_livepatch_safety_check(); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: Safety checks failed: %d\n", data->name, rc); return rc; } printk(XENLOG_INFO LIVEPATCH "%s: Applying %u functions\n", data->name, data->nfuncs); rc = arch_livepatch_quiesce(); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: unable to quiesce!\n", data->name); return rc; } /* * Since we are running with IRQs disabled and the hooks may call common * code - which expects certain spinlocks to run with IRQs enabled - we * temporarily disable the spin locks IRQ state checks. */ spin_debug_disable(); for ( i = 0; i < data->n_load_funcs; i++ ) data->load_funcs[i](); spin_debug_enable(); ASSERT(!local_irq_is_enabled()); for ( i = 0; i < data->nfuncs; i++ ) common_livepatch_apply(&data->funcs[i]); arch_livepatch_revive(); livepatch_display_metadata(&data->metadata); return 0; } static inline void apply_payload_tail(struct payload *data) { /* * We need RCU variant (which has barriers) in case we crash here. * The applied_list is iterated by the trap code. */ list_add_tail_rcu(&data->applied_list, &applied_list); register_virtual_region(&data->region); data->state = LIVEPATCH_STATE_APPLIED; } static int revert_payload(struct payload *data) { unsigned int i; int rc; printk(XENLOG_INFO LIVEPATCH "%s: Reverting\n", data->name); rc = arch_livepatch_quiesce(); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: unable to quiesce!\n", data->name); return rc; } for ( i = 0; i < data->nfuncs; i++ ) common_livepatch_revert(&data->funcs[i]); /* * Since we are running with IRQs disabled and the hooks may call common * code - which expects certain spinlocks to run with IRQs enabled - we * temporarily disable the spin locks IRQ state checks. */ spin_debug_disable(); for ( i = 0; i < data->n_unload_funcs; i++ ) data->unload_funcs[i](); spin_debug_enable(); ASSERT(!local_irq_is_enabled()); arch_livepatch_revive(); return 0; } static inline void revert_payload_tail(struct payload *data) { /* * We need RCU variant (which has barriers) in case we crash here. * The applied_list is iterated by the trap code. */ list_del_rcu(&data->applied_list); unregister_virtual_region(&data->region); data->reverted = true; data->state = LIVEPATCH_STATE_CHECKED; } /* * Check if an action has applied the same state to all payload's functions consistently. */ static inline bool was_action_consistent(const struct payload *data, livepatch_func_state_t expected_state) { int i; for ( i = 0; i < data->nfuncs; i++ ) { struct livepatch_func *f = &(data->funcs[i]); if ( f->applied != expected_state ) { printk(XENLOG_ERR LIVEPATCH "%s: Payload has a function: '%s' with inconsistent applied state.\n", data->name, f->name ?: "noname"); return false; } } return true; } /* * This function is executed having all other CPUs with no deep stack (when * idle) and IRQs disabled. */ static void livepatch_do_action(void) { int rc; struct payload *data, *other, *tmp; data = livepatch_work.data; /* * This function and the transition from asm to C code should be the only * one on any stack. No need to lock the payload list or applied list. */ switch ( livepatch_work.cmd ) { case LIVEPATCH_ACTION_APPLY: if ( is_hook_enabled(data->hooks.apply.action) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling apply action hook function\n", data->name); rc = (*data->hooks.apply.action)(data); } else rc = apply_payload(data); if ( !was_action_consistent(data, rc ? LIVEPATCH_FUNC_NOT_APPLIED : LIVEPATCH_FUNC_APPLIED) ) panic("livepatch: partially applied payload '%s'!\n", data->name); if ( rc == 0 ) apply_payload_tail(data); break; case LIVEPATCH_ACTION_REVERT: if ( is_hook_enabled(data->hooks.revert.action) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling revert action hook function\n", data->name); rc = (*data->hooks.revert.action)(data); } else rc = revert_payload(data); if ( !was_action_consistent(data, rc ? LIVEPATCH_FUNC_APPLIED : LIVEPATCH_FUNC_NOT_APPLIED) ) panic("livepatch: partially reverted payload '%s'!\n", data->name); if ( rc == 0 ) revert_payload_tail(data); break; case LIVEPATCH_ACTION_REPLACE: rc = 0; /* * N.B: Use 'applied_list' member, not 'list'. We also abuse the * the 'normal' list iterator as the list is an RCU one. */ list_for_each_entry_safe_reverse ( other, tmp, &applied_list, applied_list ) { if ( is_hook_enabled(other->hooks.revert.action) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling revert action hook function\n", other->name); other->rc = (*other->hooks.revert.action)(other); } else other->rc = revert_payload(other); if ( !was_action_consistent(other, other->rc ? LIVEPATCH_FUNC_APPLIED : LIVEPATCH_FUNC_NOT_APPLIED) ) panic("livepatch: partially reverted payload '%s'!\n", other->name); if ( other->rc == 0 ) revert_payload_tail(other); else { rc = -EINVAL; break; } } if ( rc == 0 ) { /* * Make sure all expectation requirements are met. * Beware all the payloads are reverted at this point. * If expectations are not met the system is left in a * completely UNPATCHED state! */ rc = livepatch_check_expectations(data); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: SYSTEM MIGHT BE INSECURE: " "Replace action has been aborted after reverting ALL payloads!\n", data->name); break; } if ( is_hook_enabled(data->hooks.apply.action) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling apply action hook function\n", data->name); rc = (*data->hooks.apply.action)(data); } else rc = apply_payload(data); if ( !was_action_consistent(data, rc ? LIVEPATCH_FUNC_NOT_APPLIED : LIVEPATCH_FUNC_APPLIED) ) panic("livepatch: partially applied payload '%s'!\n", data->name); if ( rc == 0 ) apply_payload_tail(data); } break; default: rc = -EINVAL; /* Make GCC5 happy. */ ASSERT_UNREACHABLE(); break; } /* We must set rc as livepatch_action sets it to -EAGAIN when kicking of. */ data->rc = rc; } static bool_t is_work_scheduled(const struct payload *data) { ASSERT(spin_is_locked(&payload_lock)); return livepatch_work.do_work && livepatch_work.data == data; } /* * Check if payload has any of the vetoing, non-atomic hooks assigned. * A vetoing, non-atomic hook may perform an operation that changes the * hypervisor state and may not be guaranteed to succeed. Result of * such operation may be returned and may change the livepatch workflow. * Such hooks may require additional cleanup actions performed by other * hooks. Thus they are not suitable for replace action. */ static inline bool has_payload_any_vetoing_hooks(const struct payload *payload) { return is_hook_enabled(payload->hooks.apply.pre) || is_hook_enabled(payload->hooks.apply.post) || is_hook_enabled(payload->hooks.revert.pre) || is_hook_enabled(payload->hooks.revert.post); } /* * Checks if any of the already applied livepatches has any vetoing, * non-atomic hooks assigned. */ static inline bool livepatch_applied_have_vetoing_hooks(void) { struct payload *p; list_for_each_entry ( p, &applied_list, applied_list ) { if ( has_payload_any_vetoing_hooks(p) ) return true; } return false; } static int schedule_work(struct payload *data, uint32_t cmd, uint32_t timeout) { ASSERT(spin_is_locked(&payload_lock)); /* Fail if an operation is already scheduled. */ if ( livepatch_work.do_work ) return -EBUSY; if ( !get_cpu_maps() ) { printk(XENLOG_ERR LIVEPATCH "%s: unable to get cpu_maps lock!\n", data->name); return -EBUSY; } livepatch_work.cmd = cmd; livepatch_work.data = data; livepatch_work.timeout = timeout ?: MILLISECS(30); dprintk(XENLOG_DEBUG, LIVEPATCH "%s: timeout is %"PRIu32"ns\n", data->name, livepatch_work.timeout); atomic_set(&livepatch_work.semaphore, -1); livepatch_work.ready = 0; smp_wmb(); livepatch_work.do_work = 1; tasklet_schedule_on_cpu(&this_cpu(livepatch_tasklet), smp_processor_id()); put_cpu_maps(); return 0; } static void cf_check tasklet_fn(void *unused) { this_cpu(work_to_do) = 1; } static int livepatch_spin(atomic_t *counter, s_time_t timeout, unsigned int cpus, const char *s) { int rc = 0; while ( atomic_read(counter) != cpus && NOW() < timeout ) cpu_relax(); /* Log & abort. */ if ( atomic_read(counter) != cpus ) { printk(XENLOG_ERR LIVEPATCH "%s: Timed out on semaphore in %s quiesce phase %u/%u\n", livepatch_work.data->name, s, atomic_read(counter), cpus); rc = -EBUSY; livepatch_work.data->rc = rc; smp_wmb(); livepatch_work.do_work = 0; } return rc; } /* * The main function which manages the work of quiescing the system and * patching code. */ void check_for_livepatch_work(void) { #define ACTION(x) [LIVEPATCH_ACTION_##x] = #x static const char *const names[] = { ACTION(APPLY), ACTION(REVERT), ACTION(REPLACE), }; #undef ACTION unsigned int cpu = smp_processor_id(); s_time_t timeout; unsigned long flags; /* Only do any work when invoked in truly idle state. */ if ( system_state != SYS_STATE_active || !is_idle_domain(current->sched_unit->domain) ) return; /* Fast path: no work to do. */ if ( !per_cpu(work_to_do, cpu ) ) return; smp_rmb(); /* In case we aborted, other CPUs can skip right away. */ if ( !livepatch_work.do_work ) { per_cpu(work_to_do, cpu) = 0; return; } ASSERT(local_irq_is_enabled()); /* Set at -1, so will go up to num_online_cpus - 1. */ if ( atomic_inc_and_test(&livepatch_work.semaphore) ) { struct payload *p; unsigned int cpus, i; bool action_done = false; p = livepatch_work.data; if ( !get_cpu_maps() ) { printk(XENLOG_ERR LIVEPATCH "%s: CPU%u - unable to get cpu_maps lock!\n", p->name, cpu); per_cpu(work_to_do, cpu) = 0; livepatch_work.data->rc = -EBUSY; smp_wmb(); livepatch_work.do_work = 0; /* * Do NOT decrement livepatch_work.semaphore down - as that may cause * the other CPU (which may be at this point ready to increment it) * to assume the role of master and then needlessly time out * out (as do_work is zero). */ return; } /* "Mask" NMIs. */ arch_livepatch_mask(); barrier(); /* MUST do it after get_cpu_maps. */ cpus = num_online_cpus() - 1; if ( cpus ) { dprintk(XENLOG_DEBUG, LIVEPATCH "%s: CPU%u - IPIing the other %u CPUs\n", p->name, cpu, cpus); for_each_online_cpu ( i ) if ( i != cpu ) tasklet_schedule_on_cpu(&per_cpu(livepatch_tasklet, i), i); } timeout = livepatch_work.timeout + NOW(); if ( livepatch_spin(&livepatch_work.semaphore, timeout, cpus, "CPU") ) goto abort; /* All CPUs are waiting, now signal to disable IRQs. */ atomic_set(&livepatch_work.semaphore, 0); /* * MUST have a barrier after semaphore so that the other CPUs don't * leak out of the 'Wait for all CPUs to rendezvous' loop and increment * 'semaphore' before we set it to zero. */ smp_wmb(); livepatch_work.ready = 1; if ( !livepatch_spin(&livepatch_work.semaphore, timeout, cpus, "IRQ") ) { local_irq_save(flags); /* Do the patching. */ livepatch_do_action(); /* Serialize and flush out the CPU via CPUID instruction (on x86). */ arch_livepatch_post_action(); action_done = true; local_irq_restore(flags); } abort: arch_livepatch_unmask(); per_cpu(work_to_do, cpu) = 0; livepatch_work.do_work = 0; /* put_cpu_maps has an barrier(). */ put_cpu_maps(); if ( action_done ) { switch ( livepatch_work.cmd ) { case LIVEPATCH_ACTION_REVERT: if ( is_hook_enabled(p->hooks.revert.post) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling post-revert hook function with rc=%d\n", p->name, p->rc); (*p->hooks.revert.post)(p); } break; case LIVEPATCH_ACTION_APPLY: if ( is_hook_enabled(p->hooks.apply.post) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling post-apply hook function with rc=%d\n", p->name, p->rc); (*p->hooks.apply.post)(p); } break; case LIVEPATCH_ACTION_REPLACE: if ( has_payload_any_vetoing_hooks(p) ) { /* It should be impossible to get here since livepatch_action() guards against that. */ panic(LIVEPATCH "%s: REPLACE action is not supported on livepatches with vetoing hooks!\n", p->name); ASSERT_UNREACHABLE(); } default: break; } } printk(XENLOG_INFO LIVEPATCH "%s finished %s with rc=%d\n", p->name, names[livepatch_work.cmd], p->rc); } else { /* Wait for all CPUs to rendezvous. */ while ( livepatch_work.do_work && !livepatch_work.ready ) cpu_relax(); /* Disable IRQs and signal. */ local_irq_save(flags); /* * We re-use the sempahore, so MUST have it reset by master before * we exit the loop above. */ atomic_inc(&livepatch_work.semaphore); /* Wait for patching to complete. */ while ( livepatch_work.do_work ) cpu_relax(); /* To flush out pipeline. */ arch_livepatch_post_action(); local_irq_restore(flags); per_cpu(work_to_do, cpu) = 0; } } /* * Only allow dependent payload is applied on top of the correct * build-id. * * This enforces an stacking order - the first payload MUST be against the * hypervisor. The second against the first payload, and so on. * * Unless the 'internal' parameter is used - in which case we only * check against the hypervisor. */ static int build_id_dep(struct payload *payload, bool_t internal) { const void *id = NULL; unsigned int len = 0; int rc; const char *name = "hypervisor"; ASSERT(payload->dep.len && payload->dep.p); /* First time user is against hypervisor. */ if ( internal ) { rc = xen_build_id(&id, &len); if ( rc ) return rc; } else { /* We should be against the last applied one. */ const struct payload *data; data = list_last_entry(&applied_list, struct payload, applied_list); id = data->id.p; len = data->id.len; name = data->name; } if ( payload->dep.len != len || memcmp(id, payload->dep.p, len) ) { printk(XENLOG_ERR LIVEPATCH "%s: check against %s build-id failed\n", payload->name, name); return -EINVAL; } return 0; } static int livepatch_action(struct xen_sysctl_livepatch_action *action) { struct payload *data; char n[XEN_LIVEPATCH_NAME_SIZE]; int rc; if ( action->pad ) return -EINVAL; rc = get_name(&action->name, n); if ( rc ) return rc; spin_lock(&payload_lock); data = find_payload(n); if ( IS_ERR_OR_NULL(data) ) { spin_unlock(&payload_lock); if ( !data ) return -ENOENT; return PTR_ERR(data); } if ( is_work_scheduled(data) ) { rc = -EBUSY; goto out; } switch ( action->cmd ) { case LIVEPATCH_ACTION_UNLOAD: if ( data->state == LIVEPATCH_STATE_CHECKED ) { free_payload(data); /* No touching 'data' from here on! */ data = NULL; } else rc = -EINVAL; break; case LIVEPATCH_ACTION_REVERT: if ( data->state == LIVEPATCH_STATE_APPLIED ) { const struct payload *p; p = list_last_entry(&applied_list, struct payload, applied_list); ASSERT(p); /* We should be the last applied one. */ if ( p != data ) { printk(XENLOG_ERR LIVEPATCH "%s: can't unload. Top is %s\n", data->name, p->name); rc = -EBUSY; break; } if ( is_hook_enabled(data->hooks.revert.pre) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling pre-revert hook function\n", data->name); rc = (*data->hooks.revert.pre)(data); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: pre-revert hook failed (rc=%d), aborting!\n", data->name, rc); data->rc = rc; break; } } data->rc = -EAGAIN; rc = schedule_work(data, action->cmd, action->timeout); } break; case LIVEPATCH_ACTION_APPLY: if ( data->state == LIVEPATCH_STATE_CHECKED ) { /* * It is unsafe to apply an reverted payload as the .data (or .bss) * may not be in in pristine condition. Hence MUST unload and then * apply patch again. Unless the payload has only one * RW section (.livepatch.funcs). */ if ( data->reverted && !data->safe_to_reapply ) { printk(XENLOG_ERR LIVEPATCH "%s: can't revert as payload has .data. Please unload\n", data->name); data->rc = -EINVAL; break; } /* * Check if action is issued with nodeps flags to ignore module * stack dependencies. */ if ( !(action->flags & LIVEPATCH_ACTION_APPLY_NODEPS) ) { rc = build_id_dep(data, !!list_empty(&applied_list)); if ( rc ) break; } /* Make sure all expectation requirements are met. */ rc = livepatch_check_expectations(data); if ( rc ) break; if ( is_hook_enabled(data->hooks.apply.pre) ) { printk(XENLOG_INFO LIVEPATCH "%s: Calling pre-apply hook function\n", data->name); rc = (*data->hooks.apply.pre)(data); if ( rc ) { printk(XENLOG_ERR LIVEPATCH "%s: pre-apply hook failed (rc=%d), aborting!\n", data->name, rc); data->rc = rc; break; } } data->rc = -EAGAIN; rc = schedule_work(data, action->cmd, action->timeout); } break; case LIVEPATCH_ACTION_REPLACE: if ( data->state == LIVEPATCH_STATE_CHECKED ) { rc = build_id_dep(data, 1 /* against hypervisor. */); if ( rc ) break; /* * REPLACE action is not supported on livepatches with vetoing hooks. * Vetoing hooks usually perform mutating actions on the system and * typically exist in pairs (pre- hook doing an action and post- hook * undoing the action). Coalescing all hooks from all applied modules * cannot be performed without inspecting potential dependencies between * the mutating hooks and hence cannot be performed automatically by * the replace action. Also, the replace action cannot safely assume a * successful revert of all the module with vetoing hooks. When one * of the hooks fails due to not meeting certain conditions the whole * replace operation must have been reverted with all previous pre- and * post- hooks re-executed (which cannot be guaranteed to succeed). * The simplest response to this complication is disallow replace * action on modules with vetoing hooks. */ if ( has_payload_any_vetoing_hooks(data) || livepatch_applied_have_vetoing_hooks() ) { printk(XENLOG_ERR LIVEPATCH "%s: REPLACE action is not supported on livepatches with vetoing hooks!\n", data->name); rc = -EOPNOTSUPP; break; } data->rc = -EAGAIN; rc = schedule_work(data, action->cmd, action->timeout); } break; default: rc = -EOPNOTSUPP; break; } out: spin_unlock(&payload_lock); return rc; } int livepatch_op(struct xen_sysctl_livepatch_op *livepatch) { int rc; if ( livepatch->pad ) return -EINVAL; switch ( livepatch->cmd ) { case XEN_SYSCTL_LIVEPATCH_UPLOAD: rc = livepatch_upload(&livepatch->u.upload); break; case XEN_SYSCTL_LIVEPATCH_GET: rc = livepatch_get(&livepatch->u.get); break; case XEN_SYSCTL_LIVEPATCH_LIST: rc = livepatch_list(&livepatch->u.list); break; case XEN_SYSCTL_LIVEPATCH_ACTION: rc = livepatch_action(&livepatch->u.action); break; default: rc = -EOPNOTSUPP; break; } return rc; } static const char *state2str(unsigned int state) { #define STATE(x) [LIVEPATCH_STATE_##x] = #x static const char *const names[] = { STATE(CHECKED), STATE(APPLIED), }; #undef STATE if ( state >= ARRAY_SIZE(names) || !names[state] ) return "unknown"; return names[state]; } static void cf_check livepatch_printall(unsigned char key) { struct payload *data; const void *binary_id = NULL; unsigned int len = 0; unsigned int i; printk("'%c' pressed - Dumping all livepatch patches\n", key); if ( !xen_build_id(&binary_id, &len) ) printk("build-id: %*phN\n", len, binary_id); if ( !spin_trylock(&payload_lock) ) { printk("Lock held. Try again.\n"); return; } list_for_each_entry ( data, &payload_list, list ) { printk(" name=%s state=%s(%d) %p (.data=%p, .rodata=%p) using %u pages.\n", data->name, state2str(data->state), data->state, data->text_addr, data->rw_addr, data->ro_addr, data->pages); livepatch_display_metadata(&data->metadata); for ( i = 0; i < data->nfuncs; i++ ) { struct livepatch_func *f = &(data->funcs[i]); printk(" %s patch %p(%u) with %p (%u)\n", f->name, f->old_addr, f->old_size, f->new_addr, f->new_size); if ( i && !(i % 64) ) { spin_unlock(&payload_lock); process_pending_softirqs(); if ( !spin_trylock(&payload_lock) ) { printk("Couldn't reacquire lock. Try again.\n"); return; } } } if ( data->id.len ) printk("build-id=%*phN\n", data->id.len, data->id.p); if ( data->dep.len ) printk("depend-on=%*phN\n", data->dep.len, data->dep.p); if ( data->xen_dep.len ) printk("depend-on-xen=%*phN\n", data->xen_dep.len, data->xen_dep.p); } spin_unlock(&payload_lock); } static int cf_check cpu_callback( struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; if ( action == CPU_UP_PREPARE ) tasklet_init(&per_cpu(livepatch_tasklet, cpu), tasklet_fn, NULL); return NOTIFY_DONE; } static struct notifier_block cpu_nfb = { .notifier_call = cpu_callback }; static int __init cf_check livepatch_init(void) { unsigned int cpu; for_each_online_cpu ( cpu ) { void *hcpu = (void *)(long)cpu; cpu_callback(&cpu_nfb, CPU_UP_PREPARE, hcpu); } register_cpu_notifier(&cpu_nfb); register_keyhandler('x', livepatch_printall, "print livepatch info", 1); arch_livepatch_init(); return 0; } __initcall(livepatch_init); /* * Local variables: * mode: C * c-file-style: "BSD" * c-basic-offset: 4 * tab-width: 4 * indent-tabs-mode: nil * End: */