/*

 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>

 * Copyright (C) 2013-2014 Josh Poimboeuf <jpoimboe@redhat.com>

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, see <http://www.gnu.org/licenses/>.

 */

/*

 * kpatch core module

 *

 * Patch modules register with this module to redirect old functions to new

 * functions.

 *

 * For each function patched by the module we must:

 * - Call stop_machine

 * - Ensure that no task has the old function in its call stack

 * - Add the new function address to kpatch_func_hash

 *

 * After that, each call to the old function calls into kpatch_ftrace_handler()

 * which finds the new function in kpatch_func_hash table and updates the

 * return instruction pointer so that ftrace will return to the new function.

 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>

#include <linux/slab.h>

#include <linux/stop_machine.h>

#include <linux/ftrace.h>

#include <linux/hashtable.h>

#include <linux/hardirq.h>

#include <linux/uaccess.h>

#include <linux/kallsyms.h>

#include <linux/version.h>

#include <linux/string.h>

#include <linux/stacktrace.h>

#include <asm/stacktrace.h>

#include <asm/cacheflush.h>

#include <generated/utsrelease.h>

#include "kpatch.h"

#ifndef UTS_UBUNTU_RELEASE_ABI

#define UTS_UBUNTU_RELEASE_ABI 0

#endif

#if !defined(CONFIG_FUNCTION_TRACER) || \

	!defined(CONFIG_HAVE_FENTRY) || \

	!defined(CONFIG_MODULES) || \

	!defined(CONFIG_SYSFS) || \

	!defined(CONFIG_STACKTRACE) || \

	!defined(CONFIG_KALLSYMS_ALL)

#error "CONFIG_FUNCTION_TRACER, CONFIG_HAVE_FENTRY, CONFIG_MODULES, CONFIG_SYSFS, CONFIG_KALLSYMS_ALL kernel config options are required"

#endif

#define KPATCH_HASH_BITS 8

static DEFINE_HASHTABLE(kpatch_func_hash, KPATCH_HASH_BITS);

static DEFINE_SEMAPHORE(kpatch_mutex);

static LIST_HEAD(kpmod_list);

static int kpatch_num_patched;

struct kobject *kpatch_root_kobj;

EXPORT_SYMBOL_GPL(kpatch_root_kobj);

struct kpatch_kallsyms_args {

	const char *objname;

	const char *name;

	unsigned long addr;

	unsigned long count;

	unsigned long pos;

};

struct kpatch_apply_patch_args {

	struct kpatch_module *kpmod;

	bool replace;

};

/* this is a double loop, use goto instead of break */

#define do_for_each_linked_func(kpmod, func) {				\

	struct kpatch_object *_object;					\

	list_for_each_entry(_object, &kpmod->objects, list) {		\

		if (!kpatch_object_linked(_object))			\

			continue;					\

		list_for_each_entry(func, &_object->funcs, list) {

#define while_for_each_linked_func()					\

		}							\

	}								\

}

/*

 * The kpatch core module has a state machine which allows for proper

 * synchronization with kpatch_ftrace_handler() when it runs in NMI context.

 *

 *         +-----------------------------------------------------+

 *         |                                                     |

 *         |                                                     +

 *         v                                     +---> KPATCH_STATE_SUCCESS

 * KPATCH_STATE_IDLE +---> KPATCH_STATE_UPDATING |

 *         ^                                     +---> KPATCH_STATE_FAILURE

 *         |                                                     +

 *         |                                                     |

 *         +-----------------------------------------------------+

 *

 * KPATCH_STATE_IDLE: No updates are pending.  The func hash is valid, and the

 * reader doesn't need to check func->op.

 *

 * KPATCH_STATE_UPDATING: An update is in progress.  The reader must call

 * kpatch_state_finish(KPATCH_STATE_FAILURE) before accessing the func hash.

 *

 * KPATCH_STATE_FAILURE: An update failed, and the func hash might be

 * inconsistent (pending patched funcs might not have been removed yet).  If

 * func->op is KPATCH_OP_PATCH, then rollback to the previous version of the

 * func.

 *

 * KPATCH_STATE_SUCCESS: An update succeeded, but the func hash might be

 * inconsistent (pending unpatched funcs might not have been removed yet).  If

 * func->op is KPATCH_OP_UNPATCH, then rollback to the previous version of the

 * func.

 */

enum {

	KPATCH_STATE_IDLE,

	KPATCH_STATE_UPDATING,

	KPATCH_STATE_SUCCESS,

	KPATCH_STATE_FAILURE,

};

static atomic_t kpatch_state;

static int (*kpatch_set_memory_rw)(unsigned long addr, int numpages);

static int (*kpatch_set_memory_ro)(unsigned long addr, int numpages);

#define MAX_STACK_TRACE_DEPTH   64

static unsigned long stack_entries[MAX_STACK_TRACE_DEPTH];

static struct stack_trace trace = {

	.max_entries	= ARRAY_SIZE(stack_entries),

	.entries	= &stack_entries[0],

};

static inline void kpatch_state_idle(void)

{

	int state = atomic_read(&kpatch_state);

	WARN_ON(state != KPATCH_STATE_SUCCESS && state != KPATCH_STATE_FAILURE);

	atomic_set(&kpatch_state, KPATCH_STATE_IDLE);

}

static inline void kpatch_state_updating(void)

{

	WARN_ON(atomic_read(&kpatch_state) != KPATCH_STATE_IDLE);

	atomic_set(&kpatch_state, KPATCH_STATE_UPDATING);

}

/* If state is updating, change it to success or failure and return new state */

static inline int kpatch_state_finish(int state)

{

	int result;

	WARN_ON(state != KPATCH_STATE_SUCCESS && state != KPATCH_STATE_FAILURE);

	result = atomic_cmpxchg(&kpatch_state, KPATCH_STATE_UPDATING, state);

	return result == KPATCH_STATE_UPDATING ? state : result;

}

static struct kpatch_func *kpatch_get_func(unsigned long ip)

{

	struct kpatch_func *f;

	/* Here, we have to use rcu safe hlist because of NMI concurrency */

	hash_for_each_possible_rcu(kpatch_func_hash, f, node, ip)

		if (f->old_addr == ip)

			return f;

	return NULL;

}

static struct kpatch_func *kpatch_get_prev_func(struct kpatch_func *f,

						unsigned long ip)

{

	hlist_for_each_entry_continue_rcu(f, node)

		if (f->old_addr == ip)

			return f;

	return NULL;

}

static inline bool kpatch_object_linked(struct kpatch_object *object)

{

	return object->mod || !strcmp(object->name, "vmlinux");

}

static inline int kpatch_compare_addresses(unsigned long stack_addr,

					   unsigned long func_addr,

					   unsigned long func_size,

					   const char *func_name)

{

	if (stack_addr >= func_addr && stack_addr < func_addr + func_size) {

		pr_err("activeness safety check failed for %s\n", func_name);

		return -EBUSY;

	}

	return 0;

}

static int kpatch_backtrace_address_verify(struct kpatch_module *kpmod,

					   unsigned long address,

					   bool replace)

{

	struct kpatch_func *func;

	int i;

	int ret;

	/* check kpmod funcs */

	do_for_each_linked_func(kpmod, func) {

		unsigned long func_addr, func_size;

		const char *func_name;

		struct kpatch_func *active_func;

		if (func->force)

			continue;

		active_func = kpatch_get_func(func->old_addr);

		if (!active_func) {

			/* patching an unpatched func */

			func_addr = func->old_addr;

			func_size = func->old_size;

			func_name = func->name;

		} else {

			/* repatching or unpatching */

			func_addr = active_func->new_addr;

			func_size = active_func->new_size;

			func_name = active_func->name;

		}

		ret = kpatch_compare_addresses(address, func_addr,

					       func_size, func_name);

		if (ret)

			return ret;

	} while_for_each_linked_func();

	/* in the replace case, need to check the func hash as well */

	if (replace) {

		hash_for_each_rcu(kpatch_func_hash, i, func, node) {

			if (func->op != KPATCH_OP_UNPATCH || func->force)

				continue;

			ret = kpatch_compare_addresses(address,

						       func->new_addr,

						       func->new_size,

						       func->name);

			if (ret)

				return ret;

		}

	}

	return ret;

}

/*

 * Verify activeness safety, i.e. that none of the to-be-patched functions are

 * on the stack of any task.

 *

 * This function is called from stop_machine() context.

 */

static int kpatch_verify_activeness_safety(struct kpatch_module *kpmod,

					   bool replace)

{

	struct task_struct *g, *t;

	int i;

	int ret = 0;

	/* Check the stacks of all tasks. */

	do_each_thread(g, t) {

		trace.nr_entries = 0;

		save_stack_trace_tsk(t, &trace);

		if (trace.nr_entries >= trace.max_entries) {

			ret = -EBUSY;

			pr_err("more than %u trace entries!\n",

			       trace.max_entries);

			goto out;

		}

                for (i = 0; i < trace.nr_entries; i++) {

			if (trace.entries[i] == ULONG_MAX)

				break;

			ret = kpatch_backtrace_address_verify(kpmod,

							      trace.entries[i],

							      replace);

			if (ret)

				goto out;

		}

	} while_each_thread(g, t);

out:

	if (ret) {

		pr_err("PID: %d Comm: %.20s\n", t->pid, t->comm);

		for (i = 0; i < trace.nr_entries; i++) {

			if (trace.entries[i] == ULONG_MAX)

				break;

			pr_err("  [<%pK>] %pB\n",

			       (void *)trace.entries[i],

			       (void *)trace.entries[i]);

		}

	}

	return ret;

}

static inline int pre_patch_callback(struct kpatch_object *object)

{

	int ret;

	if (kpatch_object_linked(object) &&

	    object->pre_patch_callback) {

		ret = (*object->pre_patch_callback)(object);

		if (ret) {

			object->callbacks_enabled = false;

			return ret;

		}

	}

	object->callbacks_enabled = true;

	return 0;

}

static inline void post_patch_callback(struct kpatch_object *object)

{

	if (kpatch_object_linked(object) &&

	    object->post_patch_callback &&

	    object->callbacks_enabled)

		(*object->post_patch_callback)(object);

}

static inline void pre_unpatch_callback(struct kpatch_object *object)

{

	if (kpatch_object_linked(object) &&

	    object->pre_unpatch_callback &&

	    object->callbacks_enabled)

		(*object->pre_unpatch_callback)(object);

}

static inline void post_unpatch_callback(struct kpatch_object *object)

{

	if (kpatch_object_linked(object) &&

	    object->post_unpatch_callback &&

	    object->callbacks_enabled)

		(*object->post_unpatch_callback)(object);

}

/* Called from stop_machine */

static int kpatch_apply_patch(void *data)

{

	struct kpatch_apply_patch_args *args = data;

	struct kpatch_module *kpmod;

	struct kpatch_func *func;

	struct hlist_node *tmp;

	struct kpatch_object *object;

	int ret;

	int i;

	kpmod = args->kpmod;

	ret = kpatch_verify_activeness_safety(kpmod, args->replace);

	if (ret) {

		kpatch_state_finish(KPATCH_STATE_FAILURE);

		return ret;

	}

	/* tentatively add the new funcs to the global func hash */

	do_for_each_linked_func(kpmod, func) {

		hash_add_rcu(kpatch_func_hash, &func->node, func->old_addr);

	} while_for_each_linked_func();

	/* memory barrier between func hash add and state change */

	smp_wmb();

	/*

	 * Check if any inconsistent NMI has happened while updating.  If not,

	 * move to success state.

	 */

	ret = kpatch_state_finish(KPATCH_STATE_SUCCESS);

	if (ret == KPATCH_STATE_FAILURE) {

		pr_err("NMI activeness safety check failed\n");

		/* Failed, we have to rollback patching process */

		do_for_each_linked_func(kpmod, func) {

			hash_del_rcu(&func->node);

		} while_for_each_linked_func();

		return -EBUSY;

	}

	/*

	 * The new patch has been applied successfully. Remove the functions

	 * provided by the replaced patches (if any) from hash, to make sure

	 * they will not be executed anymore.

	 */

	if (args->replace) {

		hash_for_each_safe(kpatch_func_hash, i, tmp, func, node) {

			if (func->op != KPATCH_OP_UNPATCH)

				continue;

			hash_del_rcu(&func->node);

		}

	}

	/* run any user-defined post-patch callbacks */

	list_for_each_entry(object, &kpmod->objects, list)

		post_patch_callback(object);

	return 0;

}

/* Called from stop_machine */

static int kpatch_remove_patch(void *data)

{

	struct kpatch_module *kpmod = data;

	struct kpatch_func *func;

	struct kpatch_object *object;

	int ret;

	ret = kpatch_verify_activeness_safety(kpmod, false);

	if (ret) {

		kpatch_state_finish(KPATCH_STATE_FAILURE);

		return ret;

	}

	/* run any user-defined pre-unpatch callbacks */

	list_for_each_entry(object, &kpmod->objects, list)

		pre_unpatch_callback(object);

	/* Check if any inconsistent NMI has happened while updating */

	ret = kpatch_state_finish(KPATCH_STATE_SUCCESS);

	if (ret == KPATCH_STATE_FAILURE) {

		ret = -EBUSY;

		goto err;

	}

	/* Succeeded, remove all updating funcs from hash table */

	do_for_each_linked_func(kpmod, func) {

		hash_del_rcu(&func->node);

	} while_for_each_linked_func();

	return 0;

err:

	/* undo pre-unpatch callbacks by calling post-patch counterparts */

	list_for_each_entry(object, &kpmod->objects, list)

		post_patch_callback(object);

	return ret;

}

/*

 * This is where the magic happens.  Update regs->ip to tell ftrace to return

 * to the new function.

 *

 * If there are multiple patch modules that have registered to patch the same

 * function, the last one to register wins, as it'll be first in the hash

 * bucket.

 */

static void notrace

kpatch_ftrace_handler(unsigned long ip, unsigned long parent_ip,

		      struct ftrace_ops *fops, struct pt_regs *regs)

{

	struct kpatch_func *func;

	int state;

	preempt_disable_notrace();

	if (likely(!in_nmi()))

		func = kpatch_get_func(ip);

	else {

		/* Checking for NMI inconsistency */

		state = kpatch_state_finish(KPATCH_STATE_FAILURE);

		/* no memory reordering between state and func hash read */

		smp_rmb();

		func = kpatch_get_func(ip);

		if (likely(state == KPATCH_STATE_IDLE))

			goto done;

		if (state == KPATCH_STATE_SUCCESS) {

			/*

			 * Patching succeeded.  If the function was being

			 * unpatched, roll back to the previous version.

			 */

			if (func && func->op == KPATCH_OP_UNPATCH)

				func = kpatch_get_prev_func(func, ip);

		} else {

			/*

			 * Patching failed.  If the function was being patched,

			 * roll back to the previous version.

			 */

			if (func && func->op == KPATCH_OP_PATCH)

				func = kpatch_get_prev_func(func, ip);

		}

	}

done:

	if (func)

		regs->ip = func->new_addr + MCOUNT_INSN_SIZE;

	preempt_enable_notrace();

}

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0)

#define FTRACE_OPS_FL_IPMODIFY 0

#endif

static struct ftrace_ops kpatch_ftrace_ops __read_mostly = {

	.func = kpatch_ftrace_handler,

	.flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,

};

static int kpatch_ftrace_add_func(unsigned long ip)

{

	int ret;

	/* check if any other patch modules have also patched this func */

	if (kpatch_get_func(ip))

		return 0;

	ret = ftrace_set_filter_ip(&kpatch_ftrace_ops, ip, 0, 0);

	if (ret) {

		pr_err("can't set ftrace filter at address 0x%lx\n", ip);

		return ret;

	}

	if (!kpatch_num_patched) {

		ret = register_ftrace_function(&kpatch_ftrace_ops);

		if (ret) {

			pr_err("can't register ftrace handler\n");

			ftrace_set_filter_ip(&kpatch_ftrace_ops, ip, 1, 0);

			return ret;

		}

	}

	kpatch_num_patched++;

	return 0;

}

static int kpatch_ftrace_remove_func(unsigned long ip)

{

	int ret;

	/* check if any other patch modules have also patched this func */

	if (kpatch_get_func(ip))

		return 0;

	if (kpatch_num_patched == 1) {

		ret = unregister_ftrace_function(&kpatch_ftrace_ops);

		if (ret) {

			pr_err("can't unregister ftrace handler\n");

			return ret;

		}

	}

	kpatch_num_patched--;

	ret = ftrace_set_filter_ip(&kpatch_ftrace_ops, ip, 1, 0);

	if (ret) {

		pr_err("can't remove ftrace filter at address 0x%lx\n", ip);

		return ret;

	}

	return 0;

}

static int kpatch_kallsyms_callback(void *data, const char *name,

					 struct module *mod,

					 unsigned long addr)

{

	struct kpatch_kallsyms_args *args = data;

	bool vmlinux = !strcmp(args->objname, "vmlinux");

	if ((mod && vmlinux) || (!mod && !vmlinux))

		return 0;

	if (strcmp(args->name, name))

		return 0;

	if (!vmlinux && strcmp(args->objname, mod->name))

		return 0;

	args->addr = addr;

	args->count++;

	/*

	 * Finish the search when the symbol is found for the desired position

	 * or the position is not defined for a non-unique symbol.

	 */

	if ((args->pos && (args->count == args->pos)) ||

	    (!args->pos && (args->count > 1))) {

		return 1;

	}

	return 0;

}

static int kpatch_find_object_symbol(const char *objname, const char *name,

				     unsigned long sympos, unsigned long *addr)

{

	struct kpatch_kallsyms_args args = {

		.objname = objname,

		.name = name,

		.addr = 0,

		.count = 0,

		.pos = sympos,

	};

	mutex_lock(&module_mutex);

	kallsyms_on_each_symbol(kpatch_kallsyms_callback, &args);

	mutex_unlock(&module_mutex);

	/*

	 * Ensure an address was found. If sympos is 0, ensure symbol is unique;

	 * otherwise ensure the symbol position count matches sympos.

	 */

	if (args.addr == 0)

		pr_err("symbol '%s' not found in symbol table\n", name);

	else if (args.count > 1 && sympos == 0) {

		pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",

		       name, objname);

	} else if (sympos != args.count && sympos > 0) {

		pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",

		       sympos, name, objname);

	} else {

		*addr = args.addr;

		return 0;

	}

	*addr = 0;

	return -EINVAL;

}

/*

 * External symbols are located outside the parent object (where the parent

 * object is either vmlinux or the kmod being patched).

 */

static int kpatch_find_external_symbol(const char *objname, const char *name,

				       unsigned long sympos, unsigned long *addr)

{

	const struct kernel_symbol *sym;

	/* first, check if it's an exported symbol */

	preempt_disable();

	sym = find_symbol(name, NULL, NULL, true, true);

	preempt_enable();

	if (sym) {

#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS

		*addr = (unsigned long)offset_to_ptr(&sym->value_offset);

#else

		*addr = sym->value;

#endif

		return 0;

	}

	/* otherwise check if it's in another .o within the patch module */

	return kpatch_find_object_symbol(objname, name, sympos, addr);

}

static int kpatch_write_relocations(struct kpatch_module *kpmod,

				    struct kpatch_object *object)

{

	int ret, size, readonly = 0, numpages;

	struct kpatch_dynrela *dynrela;

	u64 loc, val;

#if (( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) ) || \

     ( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && \

      UTS_UBUNTU_RELEASE_ABI >= 7 ) \

    )

	unsigned long core = (unsigned long)kpmod->mod->core_layout.base;

	unsigned long core_size = kpmod->mod->core_layout.size;

#else

	unsigned long core = (unsigned long)kpmod->mod->module_core;

	unsigned long core_size = kpmod->mod->core_size;

#endif

	list_for_each_entry(dynrela, &object->dynrelas, list) {

		if (dynrela->external)

			ret = kpatch_find_external_symbol(kpmod->mod->name,

							  dynrela->name,

							  dynrela->sympos,

							  &dynrela->src);

		else

			ret = kpatch_find_object_symbol(object->name,

							dynrela->name,

							dynrela->sympos,

							&dynrela->src);

		if (ret) {

			pr_err("unable to find symbol '%s'\n", dynrela->name);

			return ret;

		}

		switch (dynrela->type) {

		case R_X86_64_NONE:

			continue;

		case R_X86_64_PC32:

		case R_X86_64_PLT32:

			loc = dynrela->dest;

			val = (u32)(dynrela->src + dynrela->addend -

				    dynrela->dest);

			size = 4;

			break;

		case R_X86_64_32S:

			loc = dynrela->dest;

			val = (s32)dynrela->src + dynrela->addend;

			size = 4;

			break;

		case R_X86_64_64:

			loc = dynrela->dest;

			val = dynrela->src + dynrela->addend;

			size = 8;

			break;

		default:

			pr_err("unsupported rela type %ld for source %s (0x%lx <- 0x%lx)\n",

			       dynrela->type, dynrela->name, dynrela->dest,

			       dynrela->src);

			return -EINVAL;

		}

		if (loc < core || loc >= core + core_size) {

			pr_err("bad dynrela location 0x%llx for symbol %s\n",

			       loc, dynrela->name);

			return -EINVAL;

		}

		/*

		 * Skip it if the instruction to be relocated has been

		 * changed already (paravirt or alternatives may do this).

		 */

		if (memchr_inv((void *)loc, 0, size)) {

			pr_notice("Skipped dynrela for %s (0x%lx <- 0x%lx): the instruction has been changed already.\n",

				  dynrela->name, dynrela->dest, dynrela->src);

			pr_notice_once(

"This is not necessarily a bug but it may indicate in some cases "

"that the binary patch does not handle paravirt operations, alternatives or the like properly.\n");

			continue;

		}

#if defined(CONFIG_DEBUG_SET_MODULE_RONX) || defined(CONFIG_ARCH_HAS_SET_MEMORY)

#if (( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) ) || \

     ( LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && \

      UTS_UBUNTU_RELEASE_ABI >= 7 ) \

    )

               readonly = (loc < core + kpmod->mod->core_layout.ro_size);

#else

               readonly = (loc < core + kpmod->mod->core_ro_size);

#endif

#endif

		numpages = (PAGE_SIZE - (loc & ~PAGE_MASK) >= size) ? 1 : 2;

		if (readonly)

			kpatch_set_memory_rw(loc & PAGE_MASK, numpages);

		ret = probe_kernel_write((void *)loc, &val, size);

		if (readonly)

			kpatch_set_memory_ro(loc & PAGE_MASK, numpages);

		if (ret) {

			pr_err("write to 0x%llx failed for symbol %s\n",

			       loc, dynrela->name);

			return ret;

		}

	}

	return 0;

}

static int kpatch_unlink_object(struct kpatch_object *object)

{

	struct kpatch_func *func;

	int ret;

	list_for_each_entry(func, &object->funcs, list) {

		if (!func->old_addr)

			continue;

		ret = kpatch_ftrace_remove_func(func->old_addr);

		if (ret) {

			WARN(1, "can't unregister ftrace for address 0x%lx\n",

			     func->old_addr);

			return ret;

		}

	}

	if (object->mod) {

		module_put(object->mod);

		object->mod = NULL;

	}

	return 0;

}

/*

 * Link to a to-be-patched object in preparation for patching it.

 *

 * - Find the object module

 * - Write patch module relocations which reference the object

 * - Calculate the patched functions' addresses

 * - Register them with ftrace

 */

static int kpatch_link_object(struct kpatch_module *kpmod,

			      struct kpatch_object *object)

{

	struct module *mod = NULL;

	struct kpatch_func *func, *func_err = NULL;

	int ret;

	bool vmlinux = !strcmp(object->name, "vmlinux");

	if (!vmlinux) {

		mutex_lock(&module_mutex);

		mod = find_module(object->name);

		if (!mod) {

			/*

			 * The module hasn't been loaded yet.  We can patch it

			 * later in kpatch_module_notify().

			 */

			mutex_unlock(&module_mutex);

			return 0;

		}

		/* should never fail because we have the mutex */

		WARN_ON(!try_module_get(mod));

		mutex_unlock(&module_mutex);

		object->mod = mod;

	}

	ret = kpatch_write_relocations(kpmod, object);

	if (ret)

		goto err_put;

	list_for_each_entry(func, &object->funcs, list) {

		/* lookup the old location */

		ret = kpatch_find_object_symbol(object->name,

						func->name,

						func->sympos,

						&func->old_addr);

		if (ret) {

			func_err = func;

			goto err_ftrace;

		}

		/* add to ftrace filter and register handler if needed */

		ret = kpatch_ftrace_add_func(func->old_addr);

		if (ret) {

			func_err = func;

			goto err_ftrace;

		}

	}

	return 0;

err_ftrace:

	list_for_each_entry(func, &object->funcs, list) {

		if (func == func_err)

			break;

		WARN_ON(kpatch_ftrace_remove_func(func->old_addr));

	}

err_put:

	if (!vmlinux)

		module_put(mod);

	return ret;

}

static int kpatch_module_notify_coming(struct notifier_block *nb,