[Devel] [PATCH RHEL8 COMMIT] mm/memcg: Use per-cpu stock charges for ->kmem and ->cache counters #PSBM-101300
Konstantin Khorenko
khorenko at virtuozzo.com
Wed Oct 14 15:45:51 MSK 2020
The commit is pushed to "branch-rh8-4.18.0-193.6.3.vz8.4.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh8-4.18.0-193.6.3.vz8.4.12
------>
commit 84476bf2ce81c2fb9454adc048fbc9e9a0704538
Author: Andrey Ryabinin <aryabinin at virtuozzo.com>
Date: Wed Oct 14 15:45:51 2020 +0300
mm/memcg: Use per-cpu stock charges for ->kmem and ->cache counters #PSBM-101300
Currently we use per-cpu stocks to do precharges of the ->memory and ->memsw
counters. Do this for the ->kmem and ->cache as well to decrease contention
on these counters as well.
https://jira.sw.ru/browse/PSBM-101300
Signed-off-by: Andrey Ryabinin <aryabinin at virtuozzo.com>
---
mm/memcontrol.c | 75 +++++++++++++++++++++++++++++++++++++++------------------
1 file changed, 51 insertions(+), 24 deletions(-)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 134cb27307f2..b3f97309ca39 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2023,6 +2023,8 @@ EXPORT_SYMBOL(unlock_page_memcg);
struct memcg_stock_pcp {
struct mem_cgroup *cached; /* this never be root cgroup */
unsigned int nr_pages;
+ unsigned int cache_nr_pages;
+ unsigned int kmem_nr_pages;
struct work_struct work;
unsigned long flags;
#define FLUSHING_CACHED_CHARGE 0
@@ -2041,7 +2043,8 @@ static DEFINE_MUTEX(percpu_charge_mutex);
*
* returns true if successful, false otherwise.
*/
-static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
+static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages,
+ bool cache, bool kmem)
{
struct memcg_stock_pcp *stock;
unsigned long flags;
@@ -2053,9 +2056,19 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
local_irq_save(flags);
stock = this_cpu_ptr(&memcg_stock);
- if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
- stock->nr_pages -= nr_pages;
- ret = true;
+ if (memcg == stock->cached) {
+ if (cache && stock->cache_nr_pages >= nr_pages) {
+ stock->cache_nr_pages -= nr_pages;
+ ret = true;
+ }
+ if (kmem && stock->kmem_nr_pages >= nr_pages) {
+ stock->kmem_nr_pages -= nr_pages;
+ ret = true;
+ }
+ if (!cache && !kmem && stock->nr_pages >= nr_pages) {
+ stock->nr_pages -= nr_pages;
+ ret = true;
+ }
}
local_irq_restore(flags);
@@ -2069,13 +2082,21 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
static void drain_stock(struct memcg_stock_pcp *stock)
{
struct mem_cgroup *old = stock->cached;
+ unsigned long nr_pages = stock->nr_pages + stock->cache_nr_pages + stock->kmem_nr_pages;
+
+ if (stock->cache_nr_pages)
+ page_counter_uncharge(&old->cache, stock->cache_nr_pages);
+ if (stock->kmem_nr_pages)
+ page_counter_uncharge(&old->kmem, stock->kmem_nr_pages);
- if (stock->nr_pages) {
- page_counter_uncharge(&old->memory, stock->nr_pages);
+ if (nr_pages) {
+ page_counter_uncharge(&old->memory, nr_pages);
if (do_memsw_account())
- page_counter_uncharge(&old->memsw, stock->nr_pages);
+ page_counter_uncharge(&old->memsw, nr_pages);
css_put_many(&old->css, stock->nr_pages);
stock->nr_pages = 0;
+ stock->kmem_nr_pages = 0;
+ stock->cache_nr_pages = 0;
}
stock->cached = NULL;
}
@@ -2102,10 +2123,12 @@ static void drain_local_stock(struct work_struct *dummy)
* Cache charges(val) to local per_cpu area.
* This will be consumed by consume_stock() function, later.
*/
-static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
+static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages,
+ bool cache, bool kmem)
{
struct memcg_stock_pcp *stock;
unsigned long flags;
+ unsigned long stock_nr_pages;
local_irq_save(flags);
@@ -2114,9 +2137,17 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
drain_stock(stock);
stock->cached = memcg;
}
- stock->nr_pages += nr_pages;
- if (stock->nr_pages > MEMCG_CHARGE_BATCH)
+ if (cache)
+ stock->cache_nr_pages += nr_pages;
+ else if (kmem)
+ stock->kmem_nr_pages += nr_pages;
+ else
+ stock->nr_pages += nr_pages;
+
+ stock_nr_pages = stock->nr_pages + stock->cache_nr_pages +
+ stock->kmem_nr_pages;
+ if (nr_pages > MEMCG_CHARGE_BATCH)
drain_stock(stock);
local_irq_restore(flags);
@@ -2143,9 +2174,11 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
for_each_online_cpu(cpu) {
struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
struct mem_cgroup *memcg;
+ unsigned long nr_pages = stock->nr_pages + stock->kmem_nr_pages +
+ stock->cache_nr_pages;
memcg = stock->cached;
- if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
+ if (!memcg || !nr_pages || !css_tryget(&memcg->css))
continue;
if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
css_put(&memcg->css);
@@ -2273,17 +2306,11 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge
retry:
may_swap = true;
kmem_limit = false;
- if (consume_stock(memcg, nr_pages)) {
- if (kmem_charge && !page_counter_try_charge(
- &memcg->kmem, nr_pages, &counter)) {
- refill_stock(memcg, nr_pages);
- goto charge;
- }
+ if (consume_stock(memcg, nr_pages, cache_charge, kmem_charge)) {
css_get_many(&memcg->css, batch);
goto done;
}
-charge:
mem_over_limit = NULL;
if (page_counter_try_charge(&memcg->memory, batch, &counter)) {
if (do_memsw_account() && !page_counter_try_charge(
@@ -2296,7 +2323,7 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge
mem_over_limit = mem_cgroup_from_counter(counter, memory);
if (!mem_over_limit && kmem_charge) {
- if (!page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
+ if (!page_counter_try_charge(&memcg->kmem, batch, &counter)) {
kmem_limit = true;
mem_over_limit = mem_cgroup_from_counter(counter, kmem);
page_counter_uncharge(&memcg->memory, batch);
@@ -2425,13 +2452,13 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge
return 0;
done_restock:
+ if (cache_charge)
+ page_counter_charge(&memcg->cache, batch);
+
css_get_many(&memcg->css, batch);
if (batch > nr_pages)
- refill_stock(memcg, batch - nr_pages);
+ refill_stock(memcg, batch - nr_pages, cache_charge, kmem_charge);
done:
- if (cache_charge)
- page_counter_charge(&memcg->cache, nr_pages);
-
/*
* If the hierarchy is above the normal consumption range, schedule
* reclaim on returning to userland. We can perform reclaim here
@@ -6972,7 +6999,7 @@ void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
- refill_stock(memcg, nr_pages);
+ refill_stock(memcg, nr_pages, false, false);
}
static int __init cgroup_memory(char *s)
More information about the Devel
mailing list