oss-security September 2010 archive
Main Archive Page > Month Archives  > oss-security archives
oss-security: [oss-security] [PATCH 1/2] oom: don't ignore rss i

[oss-security] [PATCH 1/2] oom: don't ignore rss in nascent mm

From: KOSAKI Motohiro <kosaki.motohiro_at_nospam>
Date: Thu Sep 09 2010 - 05:03:55 GMT
To: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>

This patch was made on top "oom: remove totalpage normalization from oom_badness()" patch.

===============================
Execve() makes new mm struct and setup stack and push argv vector,
Unfortunately this nascent mm is not pointed any tasks, then
OOM-killer can't detect this memory usage. therefore OOM-killer
may kill incorrect task.

Thus, this patch added task->in_exec_mm member and track
nascent mm usage.

Cc: pageexec@freemail.hu
Cc: Roland McGrath <roland@redhat.com>
Cc: Solar Designer <solar@openwall.com>
Cc: Brad Spengler <spender@grsecurity.net>
Cc: Eugene Teo <eteo@redhat.com>
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
--- fs/compat.c | 4 +++- fs/exec.c | 14 +++++++++++++- include/linux/binfmts.h | 1 + include/linux/sched.h | 1 + mm/oom_kill.c | 37 +++++++++++++++++++++++++++++-------- 5 files changed, 47 insertions(+), 10 deletions(-) diff --git a/fs/compat.c b/fs/compat.c index 718c706..b631120 100644 --- a/fs/compat.c +++ b/fs/compat.c @@ -1567,8 +1567,10 @@ int compat_do_execve(char * filename, return retval; out: - if (bprm->mm) + if (bprm->mm) { + set_exec_mm(NULL); mmput(bprm->mm); + } out_file: if (bprm->file) { diff --git a/fs/exec.c b/fs/exec.c index 2d94552..b41834c 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -347,6 +347,8 @@ int bprm_mm_init(struct linux_binprm *bprm) if (err) goto err; + set_exec_mm(bprm->mm); + return 0; err: @@ -983,6 +985,7 @@ int flush_old_exec(struct linux_binprm * bprm) goto out; bprm->mm = NULL; /* We're using it now */ + set_exec_mm(NULL); current->flags &= ~PF_RANDOMIZE; flush_thread(); @@ -1314,6 +1317,13 @@ int search_binary_handler(struct linux_binprm *bprm,struct pt_regs *regs) EXPORT_SYMBOL(search_binary_handler); +void set_exec_mm(struct mm_struct *mm) +{ + task_lock(current); + current->in_exec_mm = mm; + task_unlock(current); +} + /* * sys_execve() executes a new program. */ @@ -1402,8 +1412,10 @@ int do_execve(const char * filename, return retval; out: - if (bprm->mm) + if (bprm->mm) { + set_exec_mm(NULL); mmput (bprm->mm); + } out_file: if (bprm->file) { diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index a065612..2fde1ba 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -133,6 +133,7 @@ extern void install_exec_creds(struct linux_binprm *bprm); extern void do_coredump(long signr, int exit_code, struct pt_regs *regs); extern void set_binfmt(struct linux_binfmt *new); extern void free_bprm(struct linux_binprm *); +extern void set_exec_mm(struct mm_struct *mm); #endif /* __KERNEL__ */ #endif /* _LINUX_BINFMTS_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 5e61d60..bb5bf3d 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1226,6 +1226,7 @@ struct task_struct { int pdeath_signal; /* The signal sent when the parent dies */ /* ??? */ unsigned int personality; + struct mm_struct *in_exec_mm; unsigned did_exec:1; unsigned in_execve:1; /* Tell the LSMs that the process is doing an * execve */ diff --git a/mm/oom_kill.c b/mm/oom_kill.c index c1beda0..7d38435 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -120,6 +120,33 @@ struct task_struct *find_lock_task_mm(struct task_struct *p) return NULL; } +/* + * The baseline for the badness score is the proportion of RAM that each + * task's rss and swap space use. + */ +static unsigned long oom_rss_swap_usage(struct task_struct *p) +{ + struct task_struct *t = p; + int mm_accounted = 0; + unsigned long points = 0; + + do { + task_lock(t); + if (!mm_accounted && t->mm) { + points += get_mm_rss(t->mm); + points += get_mm_counter(t->mm, MM_SWAPENTS); + mm_accounted = 1; + } + if (t->in_exec_mm) { + points += get_mm_rss(t->in_exec_mm); + points += get_mm_counter(t->in_exec_mm, MM_SWAPENTS); + } + task_unlock(t); + } while_each_thread(p, t); + + return points; +} + /* return true if the task is not adequate as candidate victim task. */ static bool oom_unkillable_task(struct task_struct *p, struct mem_cgroup *mem, const nodemask_t *nodemask) @@ -169,16 +196,10 @@ unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *mem, if (p->flags & PF_OOM_ORIGIN) return ULONG_MAX; - p = find_lock_task_mm(p); - if (!p) + points = oom_rss_swap_usage(p); + if (!points) return 0; - /* - * The baseline for the badness score is the proportion of RAM that each - * task's rss and swap space use. - */ - points = (get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS)); - task_unlock(p); /* * Root processes get 3% bonus, just like the __vm_enough_memory() -- 1.6.5.2