[BUG] utrace_attach_task() never returns when called from the report_clone callback

[Ananth N Mavinakayanahalli]

On Fri, Mar 06, 2009 at 12:52:34PM -0800, Roland McGrath wrote:
> > With the current utrace/master tree, I am seeing that utrace_attach_task()
> > never returns when invoked from the clone callback. The same module
> > works fine with prior utrace (rcu as well as with my embed version).
> 
> I changed the utrace_attach_delay() logic recently.  That is probably it.

Right, reverting dd30e86355e fixes the problem.

> Please post your test case.

Here it is -- does nothing much really :) I used this module in
conjunction with attachstop_mt with an engine attaching to it before the
pthread_create().

---
#include <linux/module.h>
#include <linux/utrace.h>
#include <linux/err.h>

MODULE_DESCRIPTION("Utrace tests");
MODULE_LICENSE("GPL");

static int target_pid;
module_param_named(pid, target_pid, int, 0);

/* Structure for all threads of a process having the same utrace ops */
struct proc_utrace {
	struct task_struct *tgid_task;

	/* list of task_utrace structs */
	struct list_head list;
	unsigned int num_threads;
};

struct task_utrace {
	struct list_head list;
	struct task_struct *task;
	/* TODO: Get rid of this and use MATCHING_OPS on task? */
	struct utrace_engine *engine;
};

static const struct utrace_engine_ops ut_ops;

static struct task_utrace *get_task_ut(struct task_struct *task,
		struct proc_utrace *proc_ut)
{
	struct task_utrace *task_ut, *temp;

	list_for_each_entry_safe(task_ut, temp, &proc_ut->list, list) {
		if (task_ut->task == task)
			return task_ut;
	}
	return NULL;
}

static int cleanup_proc_ut(struct proc_utrace *proc_ut)
{
	int ret = 0;
	struct task_utrace *task_ut, *temp;

	printk(KERN_INFO "Cleanup_proc_ut\n");
	if (proc_ut == NULL)
		return 0;

	if (list_empty(&proc_ut->list))
		goto out;

	/* walk proc_ut->list and free task_ut */
	list_for_each_entry_safe(task_ut, temp, &proc_ut->list, list) {
		if (task_ut->engine) {
			printk(KERN_INFO "Calling detach for %d\n",
					task_pid_nr(task_ut->task));
			ret = utrace_control(task_ut->task,
					task_ut->engine, UTRACE_DETACH);
			if (ret)
				printk(KERN_INFO "utrace_detach returned %d\n",
						ret);
			printk(KERN_INFO "Detached engine for %d\n",
					task_pid_nr(task_ut->task));
		}
		list_del(&task_ut->list);
		kfree(task_ut);
	}
out:
	kfree(proc_ut);
	return ret;
}

static int setup_task_ut(struct task_struct *t, struct proc_utrace *proc_ut)
{
	struct task_utrace *task_ut;
	int ret = 0;

	if (!t || !proc_ut)
		return -EINVAL;

	printk(KERN_INFO "setup_task_ut: attaching for task %d\n",
			task_pid_nr(t));
	task_ut = kzalloc(sizeof(*task_ut), GFP_KERNEL);
	if (!task_ut)
		return -ENOMEM;

	INIT_LIST_HEAD(&task_ut->list);
	task_ut->task = t;
	list_add_tail(&task_ut->list, &proc_ut->list);

	/*
	 * The utrace engine's *data will point to proc_ut.
	 */
	printk(KERN_INFO "Before utrace_attach_task: %d\n", task_pid_nr(t));
	task_ut->engine = utrace_attach_task(t, UTRACE_ATTACH_CREATE,
			&ut_ops, proc_ut);
	printk(KERN_INFO "After utrace_attach_task: %d, engine = %p\n",
			task_pid_nr(t), task_ut->engine);
	if (IS_ERR(task_ut->engine)) {
		printk(KERN_ERR "utrace_attach_task returned %d\n",
				(int)PTR_ERR(task_ut->engine));
		task_ut->engine = NULL;
		ret = -ESRCH;
		goto out;
	}
	printk(KERN_INFO "utrace_attach_task: SUCCESS! - engine = %p\n",
				task_ut->engine);
	if (utrace_set_events(t, task_ut->engine,
			UTRACE_EVENT(QUIESCE) | UTRACE_EVENT(CLONE) |
			UTRACE_EVENT(EXIT))) {
		ret = -ESRCH;
	}
	proc_ut->num_threads++;
out:
	return ret;
}

static u32 ut_quiesce(enum utrace_resume_action action,
		struct utrace_engine *engine,
		struct task_struct *task, unsigned long event)
{
	printk(KERN_INFO "In quiesce callback: tid = %d\n", task_pid_nr(task));
	return UTRACE_RESUME;
}

/* clone handler -- handle thread spawns and forks */
static u32 ut_clone(enum utrace_resume_action action,
		struct utrace_engine *engine,
		struct task_struct *parent, unsigned long clone_flags,
		struct task_struct *child)
{
	struct proc_utrace *proc_ut = (struct proc_utrace *)engine->data;

	printk(KERN_INFO "In clone callback: parent = %d, child = %d\n",
			task_pid_nr(parent), task_pid_nr(child));
	if (clone_flags & CLONE_THREAD) {
		/* New thread in the same process */
		printk(KERN_INFO "New thread - tid = %d\n", task_pid_nr(child));
		if (setup_task_ut(child, proc_ut)) {
			printk(KERN_INFO "ut_clone - calling cleanup_proc_ut\n");
			cleanup_proc_ut(proc_ut);
			goto out;
		}
	}
out:
	return UTRACE_RESUME;
}

static u32 ut_exit(enum utrace_resume_action action,
		struct utrace_engine *engine, struct task_struct *task,
		long orig_code, long *code)
{
	struct task_utrace *task_ut;
	struct proc_utrace *proc_ut = (struct proc_utrace *)engine->data;

	printk(KERN_INFO "In exit callback: tid = %d\n", task_pid_nr(task));
	/* One task dying */
	task_ut = get_task_ut(task, proc_ut);
	if (task_ut) {
		proc_ut->num_threads--;
		list_del(&task_ut->list);
		kfree(task_ut);

		/* If we are the last task, cleanup! */
		if (unlikely(list_empty(&proc_ut->list))) {
			printk(KERN_INFO "ut_exit - calling cleanup_proc_ut\n");
			cleanup_proc_ut(proc_ut);
		}
	}
	printk(KERN_INFO "Detaching %d\n", task_pid_nr(task));
	return UTRACE_DETACH;
}

static const struct utrace_engine_ops ut_ops =
{
	.report_clone = ut_clone,	/* new thread */
	.report_quiesce = ut_quiesce,
	.report_exit = ut_exit,	/* thread exit */
};

/* Engine attach -- for all threads of the process */
static struct proc_utrace *attach_utrace_engines(struct pid *pid)
{
	int ret = 0;
	struct task_struct *t;
	struct proc_utrace *proc_ut;
	struct task_utrace *task_ut;
	struct utrace_engine *engine;

	if (!pid) {
		ret = -EINVAL;
		goto out;
	}

	/*
	 * We already hold a ref to the pid here
	 */
	engine = utrace_attach_pid(pid, UTRACE_ATTACH_MATCH_OPS,
			&ut_ops, 0);
	if (IS_ERR(engine)) {
		if (PTR_ERR(engine) != -ENOENT) {
			printk(KERN_INFO "Engine already attached?\n");
			goto out;
		}
	}

	proc_ut = kzalloc(sizeof(*proc_ut), GFP_KERNEL);
	if (!proc_ut)
		return ERR_PTR(-ENOMEM);
	t = proc_ut->tgid_task = pid_task(pid, PIDTYPE_PID);
	INIT_LIST_HEAD(&proc_ut->list);

	rcu_read_lock();
	do {
		ret = setup_task_ut(t, proc_ut);
		printk(KERN_INFO "setup_task_ut returned %d\n", ret);
		if (ret)
			goto err_task_ut;

		task_ut = get_task_ut(t, proc_ut);
		ret = utrace_control(t, task_ut->engine, UTRACE_STOP);
		if (ret == 0)
			printk(KERN_INFO "Task %d is quiescent\n",
					task_pid_nr(t));
		else if (ret == -EINPROGRESS)
			printk(KERN_INFO "Task %d is on its way to quiesce\n",
				task_pid_nr(t));
		else {
			printk(KERN_ERR "utrace_control returned %d\n", ret);
			goto err_task_ut;
		}

		ret = 0;
		t = next_thread(t);
	} while (t != proc_ut->tgid_task);

	rcu_read_unlock();
	return proc_ut;

err_task_ut:
	rcu_read_unlock();
	printk(KERN_INFO "attach_utrace_engines - calling cleanup_proc_ut\n");
	ret = cleanup_proc_ut(proc_ut);
out:
	return ERR_PTR(ret);
}

static int __init utrace_init(void)
{
	int ret = 0;
	struct proc_utrace *proc_ut = NULL;
	struct pid *pid;

	pid = find_get_pid(target_pid);
	if (pid == NULL) {
		printk(KERN_ERR "Cannot find PID %d\n", target_pid);
		ret = -ESRCH;
		goto out;
	}

	/* attach an engine for each thread */
	proc_ut = attach_utrace_engines(pid);
	if (IS_ERR(proc_ut)) {
		ret = (int)PTR_ERR(proc_ut);
		printk(KERN_ERR "utrace_attach_engines returned %d\n",
				ret);
		goto out;
	}

out:
	put_pid(pid);
	return ret;
}

static void __exit utrace_exit(void)
{
	int ret = 0;
	struct pid *pid;
	struct utrace_engine *engine;
	struct proc_utrace *proc_ut;

	pid = find_get_pid(target_pid);
	if (pid == NULL) {
		printk(KERN_ERR "Cannot find PID %d\n", target_pid);
		return;
	}

	printk(KERN_INFO "In module_exit for pid = %d\n", pid_vnr(pid));
	engine = utrace_attach_pid(pid, UTRACE_ATTACH_MATCH_OPS, &ut_ops, 0);
	if (IS_ERR(engine))
		printk(KERN_ERR "Can't find self: %ld\n", PTR_ERR(engine));
	else if (engine == NULL)
		printk(KERN_ERR "Can't find self: no match\n");
	else {
		printk(KERN_INFO "Trying to detach\n");
		proc_ut = (struct proc_utrace *)engine->data;
		ret = cleanup_proc_ut(proc_ut);
		if (ret)
			printk(KERN_ERR "cleanup_proc_ut returned %d\n", ret);
	}
	put_pid(pid);
}

module_init(utrace_init);
module_exit(utrace_exit);