[Devel] [PATCH RHEL7 COMMIT] mm/memcg: improve mem_cgroup_dcache_is_low() performance.
Konstantin Khorenko
khorenko at virtuozzo.com
Tue Jul 25 18:09:09 MSK 2017
The commit is pushed to "branch-rh7-3.10.0-514.26.1.vz7.33.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh7-3.10.0-514.26.1.vz7.33.16
------>
commit 6c37dde54e88887d2c748777281d515868ceffb7
Author: Andrey Ryabinin <aryabinin at virtuozzo.com>
Date: Tue Jul 25 19:09:09 2017 +0400
mm/memcg: improve mem_cgroup_dcache_is_low() performance.
mem_cgroup_dcache_is_low() is called during memory reclaim for every
mem cgroup, but it's awfully slow. It iterates through every possible
cpu to collect anon, file and slab reclaimable counters.
Switch to percpu_counter anon,file and slab reclaimable counters.
This allows to read them by doing a single load instead of itearting
over all cpus.
https://jira.sw.ru/browse/PSBM-68644
Signed-off-by: Andrey Ryabinin <aryabinin at virtuozzo.com>
---
mm/memcontrol.c | 108 +++++++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 84 insertions(+), 24 deletions(-)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 2dfe420..7264c05 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -95,28 +95,38 @@ enum mem_cgroup_stat_index {
/*
* For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
*/
- MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
- MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
MEM_CGROUP_STAT_RSS_HUGE, /* # of pages charged as anon huge */
MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
MEM_CGROUP_STAT_SHMEM, /* # of charged shmem pages */
- MEM_CGROUP_STAT_SLAB_RECLAIMABLE, /* # of reclaimable slab pages */
MEM_CGROUP_STAT_SLAB_UNRECLAIMABLE, /* # of unreclaimable slab pages */
MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
MEM_CGROUP_STAT_NSTATS,
};
+enum mem_cgroup_stat2_index {
+ /*
+ * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
+ */
+ MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
+ MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
+ MEM_CGROUP_STAT_SLAB_RECLAIMABLE, /* # of reclaimable slab pages */
+ MEM_CGROUP_STAT2_NSTATS,
+};
+
static const char * const mem_cgroup_stat_names[] = {
- "cache",
- "rss",
"rss_huge",
"mapped_file",
"shmem",
- "slab_reclaimable",
"slab_unreclaimable",
"swap",
};
+static const char * const mem_cgroup_stat2_names[] = {
+ "cache",
+ "rss",
+ "slab_reclaimable",
+};
+
enum mem_cgroup_events_index {
MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
@@ -167,6 +177,10 @@ struct mem_cgroup_stat_cpu {
unsigned long targets[MEM_CGROUP_NTARGETS];
};
+struct mem_cgroup_stat2_cpu {
+ struct percpu_counter counters[MEM_CGROUP_STAT2_NSTATS];
+};
+
struct mem_cgroup_reclaim_iter {
/*
* last scanned hierarchy member. Valid only if last_dead_count
@@ -368,6 +382,7 @@ struct mem_cgroup {
* percpu counter.
*/
struct mem_cgroup_stat_cpu __percpu *stat;
+ struct mem_cgroup_stat2_cpu stat2;
spinlock_t pcp_counter_lock;
atomic_t dead_count;
@@ -956,6 +971,11 @@ mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)
val = 0;
return val;
}
+static inline unsigned long
+mem_cgroup_read_stat2(struct mem_cgroup *memcg, enum mem_cgroup_stat2_index idx)
+{
+ return percpu_counter_read_positive(&memcg->stat2.counters[idx]);
+}
static void mem_cgroup_update_swap_max(struct mem_cgroup *memcg)
{
@@ -1013,10 +1033,10 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
* counted as CACHE even if it's on ANON LRU.
*/
if (PageAnon(page))
- __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
+ percpu_counter_add(&memcg->stat2.counters[MEM_CGROUP_STAT_RSS],
nr_pages);
else {
- __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
+ percpu_counter_add(&memcg->stat2.counters[MEM_CGROUP_STAT_CACHE],
nr_pages);
if (PageSwapBacked(page))
__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SHMEM],
@@ -1589,9 +1609,9 @@ bool mem_cgroup_dcache_is_low(struct mem_cgroup *memcg, int vfs_cache_min_ratio)
{
unsigned long anon, file, dcache;
- anon = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_RSS);
- file = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_CACHE);
- dcache = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_SLAB_RECLAIMABLE);
+ anon = mem_cgroup_read_stat2(memcg, MEM_CGROUP_STAT_RSS);
+ file = mem_cgroup_read_stat2(memcg, MEM_CGROUP_STAT_CACHE);
+ dcache = mem_cgroup_read_stat2(memcg, MEM_CGROUP_STAT_SLAB_RECLAIMABLE);
return dcache / vfs_cache_min_ratio <
(anon + file + dcache) / 100;
@@ -1975,6 +1995,10 @@ void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
pr_cont(" %s:%luKB", mem_cgroup_stat_names[i],
K(mem_cgroup_read_stat(iter, i)));
}
+ for (i = 0; i < MEM_CGROUP_STAT2_NSTATS; i++) {
+ pr_cont(" %s:%luKB", mem_cgroup_stat2_names[i],
+ K(mem_cgroup_read_stat2(iter, i)));
+ }
for (i = 0; i < NR_LRU_LISTS; i++)
pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
@@ -3116,10 +3140,11 @@ int __memcg_charge_slab(struct kmem_cache *s, gfp_t gfp, unsigned int nr_pages)
if (s->flags & SLAB_RECLAIM_ACCOUNT) {
page_counter_charge(&memcg->dcache, nr_pages);
idx = MEM_CGROUP_STAT_SLAB_RECLAIMABLE;
- } else
+ percpu_counter_add(&memcg->stat2.counters[idx], nr_pages);
+ } else {
idx = MEM_CGROUP_STAT_SLAB_UNRECLAIMABLE;
-
- this_cpu_add(memcg->stat->count[idx], nr_pages);
+ this_cpu_add(memcg->stat->count[idx], nr_pages);
+ }
return 0;
}
@@ -3135,10 +3160,11 @@ void __memcg_uncharge_slab(struct kmem_cache *s, unsigned int nr_pages)
if (s->flags & SLAB_RECLAIM_ACCOUNT) {
page_counter_uncharge(&memcg->dcache, nr_pages);
idx = MEM_CGROUP_STAT_SLAB_RECLAIMABLE;
- } else
+ percpu_counter_sub(&memcg->stat2.counters[idx], nr_pages);
+ } else {
idx = MEM_CGROUP_STAT_SLAB_UNRECLAIMABLE;
-
- this_cpu_sub(memcg->stat->count[idx], nr_pages);
+ this_cpu_sub(memcg->stat->count[idx], nr_pages);
+ }
}
/*
@@ -4191,6 +4217,17 @@ static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
return val;
}
+static unsigned long mem_cgroup_recursive_stat2(struct mem_cgroup *memcg,
+ enum mem_cgroup_stat2_index idx)
+{
+ struct mem_cgroup *iter;
+ unsigned long val = 0;
+
+ for_each_mem_cgroup_tree(iter, memcg)
+ val += mem_cgroup_read_stat2(iter, idx);
+
+ return val;
+}
static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
{
@@ -4207,8 +4244,8 @@ static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
* Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
* as well as in MEM_CGROUP_STAT_RSS_HUGE.
*/
- val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
- val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
+ val = mem_cgroup_recursive_stat2(memcg, MEM_CGROUP_STAT_CACHE);
+ val += mem_cgroup_recursive_stat2(memcg, MEM_CGROUP_STAT_RSS);
if (swap)
val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
@@ -4224,11 +4261,11 @@ void mem_cgroup_fill_meminfo(struct mem_cgroup *memcg, struct meminfo *mi)
for_each_online_node(nid)
mem_cgroup_get_nr_pages(memcg, nid, mi->pages);
- mi->slab_reclaimable = mem_cgroup_recursive_stat(memcg,
+ mi->slab_reclaimable = mem_cgroup_recursive_stat2(memcg,
MEM_CGROUP_STAT_SLAB_RECLAIMABLE);
mi->slab_unreclaimable = mem_cgroup_recursive_stat(memcg,
MEM_CGROUP_STAT_SLAB_UNRECLAIMABLE);
- mi->cached = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
+ mi->cached = mem_cgroup_recursive_stat2(memcg, MEM_CGROUP_STAT_CACHE);
mi->shmem = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SHMEM);
}
@@ -4243,7 +4280,7 @@ int mem_cgroup_enough_memory(struct mem_cgroup *memcg, long pages)
free += page_counter_read(&memcg->dcache);
/* assume file cache is reclaimable */
- free += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
+ free += mem_cgroup_recursive_stat2(memcg, MEM_CGROUP_STAT_CACHE);
/* but do not count shmem pages as they can't be purged,
* only swapped out */
@@ -5090,6 +5127,10 @@ static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i],
mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
}
+ for (i = 0; i < MEM_CGROUP_STAT2_NSTATS; i++) {
+ seq_printf(m, "%s %lu\n", mem_cgroup_stat2_names[i],
+ mem_cgroup_read_stat2(memcg, i) * PAGE_SIZE);
+ }
for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
@@ -5120,6 +5161,13 @@ static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val);
}
+ for (i = 0; i < MEM_CGROUP_STAT2_NSTATS; i++) {
+ unsigned long long val = 0;
+
+ for_each_mem_cgroup_tree(mi, memcg)
+ val += mem_cgroup_read_stat2(mi, i) * PAGE_SIZE;
+ seq_printf(m, "total_%s %llu\n", mem_cgroup_stat2_names[i], val);
+ }
for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
unsigned long long val = 0;
@@ -5854,6 +5902,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void)
{
struct mem_cgroup *memcg;
size_t size;
+ int i, ret;
size = sizeof(struct mem_cgroup);
size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
@@ -5865,9 +5914,20 @@ static struct mem_cgroup *mem_cgroup_alloc(void)
memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
if (!memcg->stat)
goto out_free;
+
+ for (i = 0; i < MEM_CGROUP_STAT2_NSTATS; i++) {
+ ret = percpu_counter_init(&memcg->stat2.counters[i], 0, GFP_KERNEL);
+ if (ret)
+ goto out_pcpu_free;
+ }
spin_lock_init(&memcg->pcp_counter_lock);
return memcg;
+out_pcpu_free:
+ while (--i >= 0)
+ percpu_counter_destroy(&memcg->stat2.counters[i]);
+
+ free_percpu(memcg->stat);
out_free:
kfree(memcg);
return NULL;
@@ -7013,8 +7073,8 @@ static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
}
local_irq_save(flags);
- __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
- __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
+ percpu_counter_sub(&memcg->stat2.counters[MEM_CGROUP_STAT_RSS], nr_anon);
+ percpu_counter_sub(&memcg->stat2.counters[MEM_CGROUP_STAT_CACHE], nr_file);
__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_SHMEM], nr_shmem);
__this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
More information about the Devel
mailing list