/* * Copyright (C) 2014 Seth Jennings * Copyright (C) 2013-2014 Josh Poimboeuf * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, see . */ /* * kpatch core module * * Patch modules register with this module to redirect old functions to new * functions. * * For each function patched by the module we must: * - Call stop_machine * - Ensure that no task has the old function in its call stack * - Add the new function address to kpatch_func_hash * * After that, each call to the old function calls into kpatch_ftrace_handler() * which finds the new function in kpatch_func_hash table and updates the * return instruction pointer so that ftrace will return to the new function. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "kpatch.h" #ifndef UTS_UBUNTU_RELEASE_ABI #define UTS_UBUNTU_RELEASE_ABI 0 #endif #if !defined(CONFIG_FUNCTION_TRACER) || \ !defined(CONFIG_HAVE_FENTRY) || \ !defined(CONFIG_MODULES) || \ !defined(CONFIG_SYSFS) || \ !defined(CONFIG_STACKTRACE) || \ !defined(CONFIG_KALLSYMS_ALL) #error "CONFIG_FUNCTION_TRACER, CONFIG_HAVE_FENTRY, CONFIG_MODULES, CONFIG_SYSFS, CONFIG_KALLSYMS_ALL kernel config options are required" #endif #define KPATCH_HASH_BITS 8 static DEFINE_HASHTABLE(kpatch_func_hash, KPATCH_HASH_BITS); static DEFINE_SEMAPHORE(kpatch_mutex); static LIST_HEAD(kpmod_list); static int kpatch_num_patched; struct kobject *kpatch_root_kobj; EXPORT_SYMBOL_GPL(kpatch_root_kobj); struct kpatch_kallsyms_args { const char *objname; const char *name; unsigned long addr; unsigned long count; unsigned long pos; }; struct kpatch_apply_patch_args { struct kpatch_module *kpmod; bool replace; }; /* this is a double loop, use goto instead of break */ #define do_for_each_linked_func(kpmod, func) { \ struct kpatch_object *_object; \ list_for_each_entry(_object, &kpmod->objects, list) { \ if (!kpatch_object_linked(_object)) \ continue; \ list_for_each_entry(func, &_object->funcs, list) { #define while_for_each_linked_func() \ } \ } \ } /* * The kpatch core module has a state machine which allows for proper * synchronization with kpatch_ftrace_handler() when it runs in NMI context. * * +-----------------------------------------------------+ * | | * | + * v +---> KPATCH_STATE_SUCCESS * KPATCH_STATE_IDLE +---> KPATCH_STATE_UPDATING | * ^ +---> KPATCH_STATE_FAILURE * | + * | | * +-----------------------------------------------------+ * * KPATCH_STATE_IDLE: No updates are pending. The func hash is valid, and the * reader doesn't need to check func->op. * * KPATCH_STATE_UPDATING: An update is in progress. The reader must call * kpatch_state_finish(KPATCH_STATE_FAILURE) before accessing the func hash. * * KPATCH_STATE_FAILURE: An update failed, and the func hash might be * inconsistent (pending patched funcs might not have been removed yet). If * func->op is KPATCH_OP_PATCH, then rollback to the previous version of the * func. * * KPATCH_STATE_SUCCESS: An update succeeded, but the func hash might be * inconsistent (pending unpatched funcs might not have been removed yet). If * func->op is KPATCH_OP_UNPATCH, then rollback to the previous version of the * func. */ enum { KPATCH_STATE_IDLE, KPATCH_STATE_UPDATING, KPATCH_STATE_SUCCESS, KPATCH_STATE_FAILURE, }; static atomic_t kpatch_state; static int (*kpatch_set_memory_rw)(unsigned long addr, int numpages); static int (*kpatch_set_memory_ro)(unsigned long addr, int numpages); #define MAX_STACK_TRACE_DEPTH 64 static unsigned long stack_entries[MAX_STACK_TRACE_DEPTH]; static struct stack_trace trace = { .max_entries = ARRAY_SIZE(stack_entries), .entries = &stack_entries[0], }; static inline void kpatch_state_idle(void) { int state = atomic_read(&kpatch_state); WARN_ON(state != KPATCH_STATE_SUCCESS && state != KPATCH_STATE_FAILURE); atomic_set(&kpatch_state, KPATCH_STATE_IDLE); } static inline void kpatch_state_updating(void) { WARN_ON(atomic_read(&kpatch_state) != KPATCH_STATE_IDLE); atomic_set(&kpatch_state, KPATCH_STATE_UPDATING); } /* If state is updating, change it to success or failure and return new state */ static inline int kpatch_state_finish(int state) { int result; WARN_ON(state != KPATCH_STATE_SUCCESS && state != KPATCH_STATE_FAILURE); result = atomic_cmpxchg(&kpatch_state, KPATCH_STATE_UPDATING, state); return result == KPATCH_STATE_UPDATING ? state : result; } static struct kpatch_func *kpatch_get_func(unsigned long ip) { struct kpatch_func *f; /* Here, we have to use rcu safe hlist because of NMI concurrency */ hash_for_each_possible_rcu(kpatch_func_hash, f, node, ip) if (f->old_addr == ip) return f; return NULL; } static struct kpatch_func *kpatch_get_prev_func(struct kpatch_func *f, unsigned long ip) { hlist_for_each_entry_continue_rcu(f, node) if (f->old_addr == ip) return f; return NULL; } static inline bool kpatch_object_linked(struct kpatch_object *object) { return object->mod || !strcmp(object->name, "vmlinux"); } static inline int kpatch_compare_addresses(unsigned long stack_addr, unsigned long func_addr, unsigned long func_size, const char *func_name) { if (stack_addr >= func_addr && stack_addr < func_addr + func_size) { pr_err("activeness safety check failed for %s\n", func_name); return -EBUSY; } return 0; } static int kpatch_backtrace_address_verify(struct kpatch_module *kpmod, unsigned long address, bool replace) { struct kpatch_func *func; int i; int ret; /* check kpmod funcs */ do_for_each_linked_func(kpmod, func) { unsigned long func_addr, func_size; const char *func_name; struct kpatch_func *active_func; if (func->force) continue; active_func = kpatch_get_func(func->old_addr); if (!active_func) { /* patching an unpatched func */ func_addr = func->old_addr; func_size = func->old_size; func_name = func->name; } else { /* repatching or unpatching */ func_addr = active_func->new_addr; func_size = active_func->new_size; func_name = active_func->name; } ret = kpatch_compare_addresses(address, func_addr, func_size, func_name); if (ret) return ret; } while_for_each_linked_func(); /* in the replace case, need to check the func hash as well */ if (replace) { hash_for_each_rcu(kpatch_func_hash, i, func, node) { if (func->op != KPATCH_OP_UNPATCH || func->force) continue; ret = kpatch_compare_addresses(address, func->new_addr, func->new_size, func->name); if (ret) return ret; } } return ret; } /* * Verify activeness safety, i.e. that none of the to-be-patched functions are * on the stack of any task. * * This function is called from stop_machine() context. */ static int kpatch_verify_activeness_safety(struct kpatch_module *kpmod, bool replace) { struct task_struct *g, *t; int i; int ret = 0; /* Check the stacks of all tasks. */ do_each_thread(g, t) { trace.nr_entries = 0; save_stack_trace_tsk(t, &trace); if (trace.nr_entries >= trace.max_entries) { ret = -EBUSY; pr_err("more than %u trace entries!\n", trace.max_entries); goto out; } for (i = 0; i < trace.nr_entries; i++) { if (trace.entries[i] == ULONG_MAX) break; ret = kpatch_backtrace_address_verify(kpmod, trace.entries[i], replace); if (ret) goto out; } } while_each_thread(g, t); out: if (ret) { pr_err("PID: %d Comm: %.20s\n", t->pid, t->comm); for (i = 0; i < trace.nr_entries; i++) { if (trace.entries[i] == ULONG_MAX) break; pr_err(" [<%pK>] %pB\n", (void *)trace.entries[i], (void *)trace.entries[i]); } } return ret; } static inline int pre_patch_callback(struct kpatch_object *object) { int ret; if (kpatch_object_linked(object) && object->pre_patch_callback) { ret = (*object->pre_patch_callback)(object); if (ret) { object->callbacks_enabled = false; return ret; } } object->callbacks_enabled = true; return 0; } static inline void post_patch_callback(struct kpatch_object *object) { if (kpatch_object_linked(object) && object->post_patch_callback && object->callbacks_enabled) (*object->post_patch_callback)(object); } static inline void pre_unpatch_callback(struct kpatch_object *object) { if (kpatch_object_linked(object) && object->pre_unpatch_callback && object->callbacks_enabled) (*object->pre_unpatch_callback)(object); } static inline void post_unpatch_callback(struct kpatch_object *object) { if (kpatch_object_linked(object) && object->post_unpatch_callback && object->callbacks_enabled) (*object->post_unpatch_callback)(object); } /* Called from stop_machine */ static int kpatch_apply_patch(void *data) { struct kpatch_apply_patch_args *args = data; struct kpatch_module *kpmod; struct kpatch_func *func; struct hlist_node *tmp; struct kpatch_object *object; int ret; int i; kpmod = args->kpmod; ret = kpatch_verify_activeness_safety(kpmod, args->replace); if (ret) { kpatch_state_finish(KPATCH_STATE_FAILURE); return ret; } /* tentatively add the new funcs to the global func hash */ do_for_each_linked_func(kpmod, func) { hash_add_rcu(kpatch_func_hash, &func->node, func->old_addr); } while_for_each_linked_func(); /* memory barrier between func hash add and state change */ smp_wmb(); /* * Check if any inconsistent NMI has happened while updating. If not, * move to success state. */ ret = kpatch_state_finish(KPATCH_STATE_SUCCESS); if (ret == KPATCH_STATE_FAILURE) { pr_err("NMI activeness safety check failed\n"); /* Failed, we have to rollback patching process */ do_for_each_linked_func(kpmod, func) { hash_del_rcu(&func->node); } while_for_each_linked_func(); return -EBUSY; } /* * The new patch has been applied successfully. Remove the functions * provided by the replaced patches (if any) from hash, to make sure * they will not be executed anymore. */ if (args->replace) { hash_for_each_safe(kpatch_func_hash, i, tmp, func, node) { if (func->op != KPATCH_OP_UNPATCH) continue; hash_del_rcu(&func->node); } } /* run any user-defined post-patch callbacks */ list_for_each_entry(object, &kpmod->objects, list) post_patch_callback(object); return 0; } /* Called from stop_machine */ static int kpatch_remove_patch(void *data) { struct kpatch_module *kpmod = data; struct kpatch_func *func; struct kpatch_object *object; int ret; ret = kpatch_verify_activeness_safety(kpmod, false); if (ret) { kpatch_state_finish(KPATCH_STATE_FAILURE); return ret; } /* run any user-defined pre-unpatch callbacks */ list_for_each_entry(object, &kpmod->objects, list) pre_unpatch_callback(object); /* Check if any inconsistent NMI has happened while updating */ ret = kpatch_state_finish(KPATCH_STATE_SUCCESS); if (ret == KPATCH_STATE_FAILURE) { ret = -EBUSY; goto err; } /* Succeeded, remove all updating funcs from hash table */ do_for_each_linked_func(kpmod, func) { hash_del_rcu(&func->node); } while_for_each_linked_func(); return 0; err: /* undo pre-unpatch callbacks by calling post-patch counterparts */ list_for_each_entry(object, &kpmod->objects, list) post_patch_callback(object); return ret; } /* * This is where the magic happens. Update regs->ip to tell ftrace to return * to the new function. * * If there are multiple patch modules that have registered to patch the same * function, the last one to register wins, as it'll be first in the hash * bucket. */ static void notrace kpatch_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *fops, struct pt_regs *regs) { struct kpatch_func *func; int state; preempt_disable_notrace(); if (likely(!in_nmi())) func = kpatch_get_func(ip); else { /* Checking for NMI inconsistency */ state = kpatch_state_finish(KPATCH_STATE_FAILURE); /* no memory reordering between state and func hash read */ smp_rmb(); func = kpatch_get_func(ip); if (likely(state == KPATCH_STATE_IDLE)) goto done; if (state == KPATCH_STATE_SUCCESS) { /* * Patching succeeded. If the function was being * unpatched, roll back to the previous version. */ if (func && func->op == KPATCH_OP_UNPATCH) func = kpatch_get_prev_func(func, ip); } else { /* * Patching failed. If the function was being patched, * roll back to the previous version. */ if (func && func->op == KPATCH_OP_PATCH) func = kpatch_get_prev_func(func, ip); } } done: if (func) regs->ip = func->new_addr + MCOUNT_INSN_SIZE; preempt_enable_notrace(); } #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) #define FTRACE_OPS_FL_IPMODIFY 0 #endif static struct ftrace_ops kpatch_ftrace_ops __read_mostly = { .func = kpatch_ftrace_handler, .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY, }; static int kpatch_ftrace_add_func(unsigned long ip) { int ret; /* check if any other patch modules have also patched this func */ if (kpatch_get_func(ip)) return 0; ret = ftrace_set_filter_ip(&kpatch_ftrace_ops, ip, 0, 0); if (ret) { pr_err("can't set ftrace filter at address 0x%lx\n", ip); return ret; } if (!kpatch_num_patched) { ret = register_ftrace_function(&kpatch_ftrace_ops); if (ret) { pr_err("can't register ftrace handler\n"); ftrace_set_filter_ip(&kpatch_ftrace_ops, ip, 1, 0); return ret; } } kpatch_num_patched++; return 0; } static int kpatch_ftrace_remove_func(unsigned long ip) { int ret; /* check if any other patch modules have also patched this func */ if (kpatch_get_func(ip)) return 0; if (kpatch_num_patched == 1) { ret = unregister_ftrace_function(&kpatch_ftrace_ops); if (ret) { pr_err("can't unregister ftrace handler\n"); return ret; } } kpatch_num_patched--; ret = ftrace_set_filter_ip(&kpatch_ftrace_ops, ip, 1, 0); if (ret) { pr_err("can't remove ftrace filter at address 0x%lx\n", ip); return ret; } return 0; } static int kpatch_kallsyms_callback(void *data, const char *name, struct module *mod, unsigned long addr) { struct kpatch_kallsyms_args *args = data; bool vmlinux = !strcmp(args->objname, "vmlinux"); if ((mod && vmlinux) || (!mod && !vmlinux)) return 0; if (strcmp(args->name, name)) return 0; if (!vmlinux && strcmp(args->objname, mod->name)) return 0; args->addr = addr; args->count++; /* * Finish the search when the symbol is found for the desired position * or the position is not defined for a non-unique symbol. */ if ((args->pos && (args->count == args->pos)) || (!args->pos && (args->count > 1))) { return 1; } return 0; } static int kpatch_find_object_symbol(const char *objname, const char *name, unsigned long sympos, unsigned long *addr) { struct kpatch_kallsyms_args args = { .objname = objname, .name = name, .addr = 0, .count = 0, .pos = sympos, }; mutex_lock(&module_mutex); kallsyms_on_each_symbol(kpatch_kallsyms_callback, &args); mutex_unlock(&module_mutex); /* * Ensure an address was found. If sympos is 0, ensure symbol is unique; * otherwise ensure the symbol position count matches sympos. */ if (args.addr == 0) pr_err("symbol '%s' not found in symbol table\n", name); else if (args.count > 1 && sympos == 0) { pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n", name, objname); } else if (sympos != args.count && sympos > 0) { pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n", sympos, name, objname); } else { *addr = args.addr; return 0; } *addr = 0; return -EINVAL; } /* * External symbols are located outside the parent object (where the parent * object is either vmlinux or the kmod being patched). */ static int kpatch_find_external_symbol(const char *objname, const char *name, unsigned long sympos, unsigned long *addr) { const struct kernel_symbol *sym; /* first, check if it's an exported symbol */ preempt_disable(); sym = find_symbol(name, NULL, NULL, true, true); preempt_enable(); if (sym) { #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS *addr = (unsigned long)offset_to_ptr(&sym->value_offset); #else *addr = sym->value; #endif return 0; } /* otherwise check if it's in another .o within the patch module */ return kpatch_find_object_symbol(objname, name, sympos, addr); } static int kpatch_write_relocations(struct kpatch_module *kpmod, struct kpatch_object *object) { int ret, size, readonly = 0, numpages; struct kpatch_dynrela *dynrela; u64 loc, val; #if (( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) ) || \ ( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && \ UTS_UBUNTU_RELEASE_ABI >= 7 ) \ ) unsigned long core = (unsigned long)kpmod->mod->core_layout.base; unsigned long core_size = kpmod->mod->core_layout.size; #else unsigned long core = (unsigned long)kpmod->mod->module_core; unsigned long core_size = kpmod->mod->core_size; #endif list_for_each_entry(dynrela, &object->dynrelas, list) { if (dynrela->external) ret = kpatch_find_external_symbol(kpmod->mod->name, dynrela->name, dynrela->sympos, &dynrela->src); else ret = kpatch_find_object_symbol(object->name, dynrela->name, dynrela->sympos, &dynrela->src); if (ret) { pr_err("unable to find symbol '%s'\n", dynrela->name); return ret; } switch (dynrela->type) { case R_X86_64_NONE: continue; case R_X86_64_PC32: case R_X86_64_PLT32: loc = dynrela->dest; val = (u32)(dynrela->src + dynrela->addend - dynrela->dest); size = 4; break; case R_X86_64_32S: loc = dynrela->dest; val = (s32)dynrela->src + dynrela->addend; size = 4; break; case R_X86_64_64: loc = dynrela->dest; val = dynrela->src + dynrela->addend; size = 8; break; default: pr_err("unsupported rela type %ld for source %s (0x%lx <- 0x%lx)\n", dynrela->type, dynrela->name, dynrela->dest, dynrela->src); return -EINVAL; } if (loc < core || loc >= core + core_size) { pr_err("bad dynrela location 0x%llx for symbol %s\n", loc, dynrela->name); return -EINVAL; } /* * Skip it if the instruction to be relocated has been * changed already (paravirt or alternatives may do this). */ if (memchr_inv((void *)loc, 0, size)) { pr_notice("Skipped dynrela for %s (0x%lx <- 0x%lx): the instruction has been changed already.\n", dynrela->name, dynrela->dest, dynrela->src); pr_notice_once( "This is not necessarily a bug but it may indicate in some cases " "that the binary patch does not handle paravirt operations, alternatives or the like properly.\n"); continue; } #if defined(CONFIG_DEBUG_SET_MODULE_RONX) || defined(CONFIG_ARCH_HAS_SET_MEMORY) #if (( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) ) || \ ( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && \ UTS_UBUNTU_RELEASE_ABI >= 7 ) \ ) readonly = (loc < core + kpmod->mod->core_layout.ro_size); #else readonly = (loc < core + kpmod->mod->core_ro_size); #endif #endif numpages = (PAGE_SIZE - (loc & ~PAGE_MASK) >= size) ? 1 : 2; if (readonly) kpatch_set_memory_rw(loc & PAGE_MASK, numpages); ret = probe_kernel_write((void *)loc, &val, size); if (readonly) kpatch_set_memory_ro(loc & PAGE_MASK, numpages); if (ret) { pr_err("write to 0x%llx failed for symbol %s\n", loc, dynrela->name); return ret; } } return 0; } static int kpatch_unlink_object(struct kpatch_object *object) { struct kpatch_func *func; int ret; list_for_each_entry(func, &object->funcs, list) { if (!func->old_addr) continue; ret = kpatch_ftrace_remove_func(func->old_addr); if (ret) { WARN(1, "can't unregister ftrace for address 0x%lx\n", func->old_addr); return ret; } } if (object->mod) { module_put(object->mod); object->mod = NULL; } return 0; } /* * Link to a to-be-patched object in preparation for patching it. * * - Find the object module * - Write patch module relocations which reference the object * - Calculate the patched functions' addresses * - Register them with ftrace */ static int kpatch_link_object(struct kpatch_module *kpmod, struct kpatch_object *object) { struct module *mod = NULL; struct kpatch_func *func, *func_err = NULL; int ret; bool vmlinux = !strcmp(object->name, "vmlinux"); if (!vmlinux) { mutex_lock(&module_mutex); mod = find_module(object->name); if (!mod) { /* * The module hasn't been loaded yet. We can patch it * later in kpatch_module_notify(). */ mutex_unlock(&module_mutex); return 0; } /* should never fail because we have the mutex */ WARN_ON(!try_module_get(mod)); mutex_unlock(&module_mutex); object->mod = mod; } ret = kpatch_write_relocations(kpmod, object); if (ret) goto err_put; list_for_each_entry(func, &object->funcs, list) { /* lookup the old location */ ret = kpatch_find_object_symbol(object->name, func->name, func->sympos, &func->old_addr); if (ret) { func_err = func; goto err_ftrace; } /* add to ftrace filter and register handler if needed */ ret = kpatch_ftrace_add_func(func->old_addr); if (ret) { func_err = func; goto err_ftrace; } } return 0; err_ftrace: list_for_each_entry(func, &object->funcs, list) { if (func == func_err) break; WARN_ON(kpatch_ftrace_remove_func(func->old_addr)); } err_put: if (!vmlinux) module_put(mod); return ret; } static int kpatch_module_notify_coming(struct notifier_block *nb, unsigned long action, void *data) { struct module *mod = data; struct kpatch_module *kpmod; struct kpatch_object *object; struct kpatch_func *func; int ret = 0; bool found = false; if (action != MODULE_STATE_COMING) return 0; down(&kpatch_mutex); list_for_each_entry(kpmod, &kpmod_list, list) { list_for_each_entry(object, &kpmod->objects, list) { if (kpatch_object_linked(object)) continue; if (!strcmp(object->name, mod->name)) { found = true; goto done; } } } done: if (!found) goto out; ret = kpatch_link_object(kpmod, object); if (ret) goto out; BUG_ON(!object->mod); pr_notice("patching newly loaded module '%s'\n", object->name); /* run user-defined pre-patch callback */ ret = pre_patch_callback(object); if (ret) { pr_err("pre-patch callback failed!\n"); goto out; /* and WARN */ } /* add to the global func hash */ list_for_each_entry(func, &object->funcs, list) hash_add_rcu(kpatch_func_hash, &func->node, func->old_addr); /* run user-defined post-patch callback */ post_patch_callback(object); out: up(&kpatch_mutex); /* no way to stop the module load on error */ WARN(ret, "error (%d) patching newly loaded module '%s'\n", ret, object->name); return 0; } static int kpatch_module_notify_going(struct notifier_block *nb, unsigned long action, void *data) { struct module *mod = data; struct kpatch_module *kpmod; struct kpatch_object *object; struct kpatch_func *func; bool found = false; if (action != MODULE_STATE_GOING) return 0; down(&kpatch_mutex); list_for_each_entry(kpmod, &kpmod_list, list) { list_for_each_entry(object, &kpmod->objects, list) { if (!kpatch_object_linked(object)) continue; if (!strcmp(object->name, mod->name)) { found = true; goto done; } } } done: if (!found) goto out; /* run user-defined pre-unpatch callback */ pre_unpatch_callback(object); /* remove from the global func hash */ list_for_each_entry(func, &object->funcs, list) hash_del_rcu(&func->node); /* run user-defined pre-unpatch callback */ post_unpatch_callback(object); kpatch_unlink_object(object); out: up(&kpatch_mutex); return 0; } /* * Remove the obsolete functions from the ftrace filter. * Return 1 if one or more of such functions have 'force' flag set, * 0 otherwise. */ static int kpatch_ftrace_remove_unpatched_funcs(void) { struct kpatch_module *kpmod; struct kpatch_func *func; int force = 0; list_for_each_entry(kpmod, &kpmod_list, list) { do_for_each_linked_func(kpmod, func) { if (func->op != KPATCH_OP_UNPATCH) continue; if (func->force) force = 1; WARN_ON(kpatch_ftrace_remove_func(func->old_addr)); } while_for_each_linked_func(); } return force; } int kpatch_register(struct kpatch_module *kpmod, bool replace) { int ret, i; struct kpatch_object *object, *object_err = NULL; struct kpatch_func *func; struct kpatch_apply_patch_args args = { .kpmod = kpmod, .replace = replace, }; if (!kpmod->mod || list_empty(&kpmod->objects)) return -EINVAL; down(&kpatch_mutex); if (kpmod->enabled) { ret = -EINVAL; goto err_up; } list_add_tail(&kpmod->list, &kpmod_list); if (!try_module_get(kpmod->mod)) { ret = -ENODEV; goto err_list; } list_for_each_entry(object, &kpmod->objects, list) { ret = kpatch_link_object(kpmod, object); if (ret) { object_err = object; goto err_unlink; } if (!kpatch_object_linked(object)) { pr_notice("delaying patch of unloaded module '%s'\n", object->name); continue; } if (strcmp(object->name, "vmlinux")) pr_notice("patching module '%s'\n", object->name); list_for_each_entry(func, &object->funcs, list) func->op = KPATCH_OP_PATCH; } if (replace) hash_for_each_rcu(kpatch_func_hash, i, func, node) func->op = KPATCH_OP_UNPATCH; /* memory barrier between func hash and state write */ smp_wmb(); kpatch_state_updating(); /* run any user-defined pre-patch callbacks */ list_for_each_entry(object, &kpmod->objects, list) { ret = pre_patch_callback(object); if(ret){ pr_err("pre-patch callback failed!\n"); kpatch_state_finish(KPATCH_STATE_FAILURE); break; } } /* if pre_patch_callback succeed. */ if (!ret) { /* * Idle the CPUs, verify activeness safety, and atomically make the new * functions visible to the ftrace handler. */ ret = stop_machine(kpatch_apply_patch, &args, NULL); } /* if pre_patch_callback or stop_machine failed */ if (ret) { list_for_each_entry(object, &kpmod->objects, list) post_unpatch_callback(object); } /* * For the replace case, remove any obsolete funcs from the ftrace * filter, and disable the owning patch module so that it can be * removed. */ if (!ret && replace) { struct kpatch_module *kpmod2, *safe; int force; force = kpatch_ftrace_remove_unpatched_funcs(); list_for_each_entry_safe(kpmod2, safe, &kpmod_list, list) { if (kpmod == kpmod2) continue; kpmod2->enabled = false; pr_notice("unloaded patch module '%s'\n", kpmod2->mod->name); /* * Don't allow modules with forced functions to be * removed because they might still be in use. */ if (!force) module_put(kpmod2->mod); list_del(&kpmod2->list); } } /* memory barrier between func hash and state write */ smp_wmb(); /* NMI handlers can return to normal now */ kpatch_state_idle(); /* * Wait for all existing NMI handlers to complete so that they don't * see any changes to funcs or funcs->op that might occur after this * point. * * Any NMI handlers starting after this point will see the IDLE state. */ synchronize_rcu(); if (ret) goto err_ops; do_for_each_linked_func(kpmod, func) { func->op = KPATCH_OP_NONE; } while_for_each_linked_func(); /* HAS_MODULE_TAINT - upstream 2992ef29ae01 "livepatch/module: make TAINT_LIVEPATCH module-specific" */ /* HAS_MODULE_TAINT_LONG - upstream 7fd8329ba502 "taint/module: Clean up global and module taint flags handling" */ #ifdef RHEL_RELEASE_CODE # if RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 4) # define HAS_MODULE_TAINT # endif #elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) # define HAS_MODULE_TAINT_LONG #elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) # define HAS_MODULE_TAINT #endif #ifdef TAINT_LIVEPATCH pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n"); add_taint(TAINT_LIVEPATCH, LOCKDEP_STILL_OK); # ifdef HAS_MODULE_TAINT_LONG set_bit(TAINT_LIVEPATCH, &kpmod->mod->taints); # elif defined(HAS_MODULE_TAINT) kpmod->mod->taints |= (1 << TAINT_LIVEPATCH); # endif #else pr_notice_once("tainting kernel with TAINT_USER\n"); add_taint(TAINT_USER, LOCKDEP_STILL_OK); #endif pr_notice("loaded patch module '%s'\n", kpmod->mod->name); kpmod->enabled = true; up(&kpatch_mutex); return 0; err_ops: if (replace) hash_for_each_rcu(kpatch_func_hash, i, func, node) func->op = KPATCH_OP_NONE; err_unlink: list_for_each_entry(object, &kpmod->objects, list) { if (object == object_err) break; if (!kpatch_object_linked(object)) continue; WARN_ON(kpatch_unlink_object(object)); } module_put(kpmod->mod); err_list: list_del(&kpmod->list); err_up: up(&kpatch_mutex); return ret; } EXPORT_SYMBOL(kpatch_register); int kpatch_unregister(struct kpatch_module *kpmod) { struct kpatch_object *object; struct kpatch_func *func; int ret, force = 0; down(&kpatch_mutex); if (!kpmod->enabled) { ret = -EINVAL; goto out; } do_for_each_linked_func(kpmod, func) { func->op = KPATCH_OP_UNPATCH; if (func->force) force = 1; } while_for_each_linked_func(); /* memory barrier between func hash and state write */ smp_wmb(); kpatch_state_updating(); ret = stop_machine(kpatch_remove_patch, kpmod, NULL); if (!ret) { /* run any user-defined post-unpatch callbacks */ list_for_each_entry(object, &kpmod->objects, list) post_unpatch_callback(object); } /* NMI handlers can return to normal now */ kpatch_state_idle(); /* * Wait for all existing NMI handlers to complete so that they don't * see any changes to funcs or funcs->op that might occur after this * point. * * Any NMI handlers starting after this point will see the IDLE state. */ synchronize_rcu(); if (ret) { do_for_each_linked_func(kpmod, func) { func->op = KPATCH_OP_NONE; } while_for_each_linked_func(); goto out; } list_for_each_entry(object, &kpmod->objects, list) { if (!kpatch_object_linked(object)) continue; ret = kpatch_unlink_object(object); if (ret) goto out; } pr_notice("unloaded patch module '%s'\n", kpmod->mod->name); kpmod->enabled = false; /* * Don't allow modules with forced functions to be removed because they * might still be in use. */ if (!force) module_put(kpmod->mod); list_del(&kpmod->list); out: up(&kpatch_mutex); return ret; } EXPORT_SYMBOL(kpatch_unregister); static struct notifier_block kpatch_module_nb_coming = { .notifier_call = kpatch_module_notify_coming, .priority = INT_MIN, /* called last */ }; static struct notifier_block kpatch_module_nb_going = { .notifier_call = kpatch_module_notify_going, .priority = INT_MAX, /* called first */ }; static int kpatch_init(void) { int ret; kpatch_set_memory_rw = (void *)kallsyms_lookup_name("set_memory_rw"); if (!kpatch_set_memory_rw) { pr_err("can't find set_memory_rw symbol\n"); return -ENXIO; } kpatch_set_memory_ro = (void *)kallsyms_lookup_name("set_memory_ro"); if (!kpatch_set_memory_ro) { pr_err("can't find set_memory_ro symbol\n"); return -ENXIO; } kpatch_root_kobj = kobject_create_and_add("kpatch", kernel_kobj); if (!kpatch_root_kobj) return -ENOMEM; ret = register_module_notifier(&kpatch_module_nb_coming); if (ret) goto err_root_kobj; ret = register_module_notifier(&kpatch_module_nb_going); if (ret) goto err_unregister_coming; return 0; err_unregister_coming: WARN_ON(unregister_module_notifier(&kpatch_module_nb_coming)); err_root_kobj: kobject_put(kpatch_root_kobj); return ret; } static void kpatch_exit(void) { rcu_barrier(); WARN_ON(kpatch_num_patched != 0); WARN_ON(unregister_module_notifier(&kpatch_module_nb_coming)); WARN_ON(unregister_module_notifier(&kpatch_module_nb_going)); kobject_put(kpatch_root_kobj); } module_init(kpatch_init); module_exit(kpatch_exit); MODULE_LICENSE("GPL");