Skip to content

Commit

Permalink
memcg: zap memcg_slab_caches and memcg_slab_mutex
Browse files Browse the repository at this point in the history 
mem_cgroup->memcg_slab_caches is a list of kmem caches corresponding to
the given cgroup.  Currently, it is only used on css free in order to
destroy all caches corresponding to the memory cgroup being freed.  The
list is protected by memcg_slab_mutex.  The mutex is also used to protect
kmem_cache->memcg_params->memcg_caches arrays and synchronizes
kmem_cache_destroy vs memcg_unregister_all_caches.

However, we can perfectly get on without these two.  To destroy all caches
corresponding to a memory cgroup, we can walk over the global list of kmem
caches, slab_caches, and we can do all the synchronization stuff using the
slab_mutex instead of the memcg_slab_mutex.  This patch therefore gets rid
of the memcg_slab_caches and memcg_slab_mutex.

Apart from this nice cleanup, it also:

 - assures that rcu_barrier() is called once at max when a root cache is
   destroyed or a memory cgroup is freed, no matter how many caches have
   SLAB_DESTROY_BY_RCU flag set;

 - fixes the race between kmem_cache_destroy and kmem_cache_create that
   exists, because memcg_cleanup_cache_params, which is called from
   kmem_cache_destroy after checking that kmem_cache->refcount=0,
   releases the slab_mutex, which gives kmem_cache_create a chance to
   make an alias to a cache doomed to be destroyed.

Signed-off-by: Vladimir Davydov <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Michal Hocko <[email protected]>
Acked-by: Christoph Lameter <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@torvalds
Vladimir Davydov authored and torvalds committed Feb 10, 2015
1 parent 3e0350a commit d5b3cf7
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 186 deletions.
2 changes: 0 additions & 2 deletions include/linux/memcontrol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,8 +407,6 @@ int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp,
unsigned long nr_pages);
void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages);

int __memcg_cleanup_cache_params(struct kmem_cache *s);

/**
* memcg_kmem_newpage_charge: verify if a new kmem allocation is allowed.
* @gfp: the gfp allocation flags.
Expand Down
6 changes: 2 additions & 4 deletions include/linux/slab.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,8 @@ struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
unsigned long,
void (*)(void *));
#ifdef CONFIG_MEMCG_KMEM
struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *,
struct kmem_cache *);
void memcg_create_kmem_cache(struct mem_cgroup *, struct kmem_cache *);
void memcg_destroy_kmem_caches(struct mem_cgroup *);
#endif
void kmem_cache_destroy(struct kmem_cache *);
int kmem_cache_shrink(struct kmem_cache *);
Expand Down Expand Up @@ -490,7 +490,6 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
* Child caches will hold extra metadata needed for its operation. Fields are:
*
* @memcg: pointer to the memcg this cache belongs to
* @list: list_head for the list of all caches in this memcg
* @root_cache: pointer to the global, root cache, this cache was derived from
*/
struct memcg_cache_params {
Expand All @@ -502,7 +501,6 @@ struct memcg_cache_params {
};
struct {
struct mem_cgroup *memcg;
struct list_head list;
struct kmem_cache *root_cache;
};
};
Expand Down
156 changes: 15 additions & 141 deletions mm/memcontrol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,9 +343,6 @@ struct mem_cgroup {
struct cg_proto tcp_mem;
#endif
#if defined(CONFIG_MEMCG_KMEM)
/* analogous to slab_common's slab_caches list, but per-memcg;
* protected by memcg_slab_mutex */
struct list_head memcg_slab_caches;
/* Index in the kmem_cache->memcg_params->memcg_caches array */
int kmemcg_id;
#endif
Expand Down Expand Up @@ -2476,25 +2473,6 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg,
}

#ifdef CONFIG_MEMCG_KMEM
/*
* The memcg_slab_mutex is held whenever a per memcg kmem cache is created or
* destroyed. It protects memcg_caches arrays and memcg_slab_caches lists.
*/
static DEFINE_MUTEX(memcg_slab_mutex);

/*
* This is a bit cumbersome, but it is rarely used and avoids a backpointer
* in the memcg_cache_params struct.
*/
static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
{
struct kmem_cache *cachep;

VM_BUG_ON(p->is_root_cache);
cachep = p->root_cache;
return cache_from_memcg_idx(cachep, memcg_cache_id(p->memcg));
}

int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp,
unsigned long nr_pages)
{
Expand Down Expand Up @@ -2578,10 +2556,7 @@ static int memcg_alloc_cache_id(void)
else if (size > MEMCG_CACHES_MAX_SIZE)
size = MEMCG_CACHES_MAX_SIZE;

mutex_lock(&memcg_slab_mutex);
err = memcg_update_all_caches(size);
mutex_unlock(&memcg_slab_mutex);

if (err) {
ida_simple_remove(&kmem_limited_groups, id);
return err;
Expand All @@ -2604,120 +2579,20 @@ void memcg_update_array_size(int num)
memcg_limited_groups_array_size = num;
}

static void memcg_register_cache(struct mem_cgroup *memcg,
struct kmem_cache *root_cache)
{
struct kmem_cache *cachep;
int id;

lockdep_assert_held(&memcg_slab_mutex);

id = memcg_cache_id(memcg);

/*
* Since per-memcg caches are created asynchronously on first
* allocation (see memcg_kmem_get_cache()), several threads can try to
* create the same cache, but only one of them may succeed.
*/
if (cache_from_memcg_idx(root_cache, id))
return;

cachep = memcg_create_kmem_cache(memcg, root_cache);
/*
* If we could not create a memcg cache, do not complain, because
* that's not critical at all as we can always proceed with the root
* cache.
*/
if (!cachep)
return;

list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);

/*
* Since readers won't lock (see cache_from_memcg_idx()), we need a
* barrier here to ensure nobody will see the kmem_cache partially
* initialized.
*/
smp_wmb();

BUG_ON(root_cache->memcg_params->memcg_caches[id]);
root_cache->memcg_params->memcg_caches[id] = cachep;
}

static void memcg_unregister_cache(struct kmem_cache *cachep)
{
struct kmem_cache *root_cache;
struct mem_cgroup *memcg;
int id;

lockdep_assert_held(&memcg_slab_mutex);

BUG_ON(is_root_cache(cachep));

root_cache = cachep->memcg_params->root_cache;
memcg = cachep->memcg_params->memcg;
id = memcg_cache_id(memcg);

BUG_ON(root_cache->memcg_params->memcg_caches[id] != cachep);
root_cache->memcg_params->memcg_caches[id] = NULL;

list_del(&cachep->memcg_params->list);

kmem_cache_destroy(cachep);
}

int __memcg_cleanup_cache_params(struct kmem_cache *s)
{
struct kmem_cache *c;
int i, failed = 0;

mutex_lock(&memcg_slab_mutex);
for_each_memcg_cache_index(i) {
c = cache_from_memcg_idx(s, i);
if (!c)
continue;

memcg_unregister_cache(c);

if (cache_from_memcg_idx(s, i))
failed++;
}
mutex_unlock(&memcg_slab_mutex);
return failed;
}

static void memcg_unregister_all_caches(struct mem_cgroup *memcg)
{
struct kmem_cache *cachep;
struct memcg_cache_params *params, *tmp;

if (!memcg_kmem_is_active(memcg))
return;

mutex_lock(&memcg_slab_mutex);
list_for_each_entry_safe(params, tmp, &memcg->memcg_slab_caches, list) {
cachep = memcg_params_to_cache(params);
memcg_unregister_cache(cachep);
}
mutex_unlock(&memcg_slab_mutex);
}

struct memcg_register_cache_work {
struct memcg_kmem_cache_create_work {
struct mem_cgroup *memcg;
struct kmem_cache *cachep;
struct work_struct work;
};

static void memcg_register_cache_func(struct work_struct *w)
static void memcg_kmem_cache_create_func(struct work_struct *w)
{
struct memcg_register_cache_work *cw =
container_of(w, struct memcg_register_cache_work, work);
struct memcg_kmem_cache_create_work *cw =
container_of(w, struct memcg_kmem_cache_create_work, work);
struct mem_cgroup *memcg = cw->memcg;
struct kmem_cache *cachep = cw->cachep;

mutex_lock(&memcg_slab_mutex);
memcg_register_cache(memcg, cachep);
mutex_unlock(&memcg_slab_mutex);
memcg_create_kmem_cache(memcg, cachep);

css_put(&memcg->css);
kfree(cw);
Expand All @@ -2726,10 +2601,10 @@ static void memcg_register_cache_func(struct work_struct *w)
/*
* Enqueue the creation of a per-memcg kmem_cache.
*/
static void __memcg_schedule_register_cache(struct mem_cgroup *memcg,
struct kmem_cache *cachep)
static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
struct kmem_cache *cachep)
{
struct memcg_register_cache_work *cw;
struct memcg_kmem_cache_create_work *cw;

cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
if (!cw)
Expand All @@ -2739,18 +2614,18 @@ static void __memcg_schedule_register_cache(struct mem_cgroup *memcg,

cw->memcg = memcg;
cw->cachep = cachep;
INIT_WORK(&cw->work, memcg_kmem_cache_create_func);

INIT_WORK(&cw->work, memcg_register_cache_func);
schedule_work(&cw->work);
}

static void memcg_schedule_register_cache(struct mem_cgroup *memcg,
struct kmem_cache *cachep)
static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
struct kmem_cache *cachep)
{
/*
* We need to stop accounting when we kmalloc, because if the
* corresponding kmalloc cache is not yet created, the first allocation
* in __memcg_schedule_register_cache will recurse.
* in __memcg_schedule_kmem_cache_create will recurse.
*
* However, it is better to enclose the whole function. Depending on
* the debugging options enabled, INIT_WORK(), for instance, can
Expand All @@ -2759,7 +2634,7 @@ static void memcg_schedule_register_cache(struct mem_cgroup *memcg,
* the safest choice is to do it like this, wrapping the whole function.
*/
current->memcg_kmem_skip_account = 1;
__memcg_schedule_register_cache(memcg, cachep);
__memcg_schedule_kmem_cache_create(memcg, cachep);
current->memcg_kmem_skip_account = 0;
}

Expand Down Expand Up @@ -2807,7 +2682,7 @@ struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep)
* could happen with the slab_mutex held. So it's better to
* defer everything.
*/
memcg_schedule_register_cache(memcg, cachep);
memcg_schedule_kmem_cache_create(memcg, cachep);
out:
css_put(&memcg->css);
return cachep;
Expand Down Expand Up @@ -4136,7 +4011,7 @@ static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)

static void memcg_destroy_kmem(struct mem_cgroup *memcg)
{
memcg_unregister_all_caches(memcg);
memcg_destroy_kmem_caches(memcg);
mem_cgroup_sockets_destroy(memcg);
}
#else
Expand Down Expand Up @@ -4664,7 +4539,6 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
spin_lock_init(&memcg->event_list_lock);
#ifdef CONFIG_MEMCG_KMEM
memcg->kmemcg_id = -1;
INIT_LIST_HEAD(&memcg->memcg_slab_caches);
#endif

return &memcg->css;
Expand Down
Loading

0 comments on commit d5b3cf7

Please sign in to comment.