[Devel] [PATCH rh7 3/3] mm/vmscan: shrink tcache, tswap upfront everything else.
Andrey Ryabinin
aryabinin at virtuozzo.com
Thu Oct 18 12:02:31 MSK 2018
We don't want to evict page cache or anon to swap while
there are a lot of reclaimable pages in tcache/tswap.
Reclaim them first, and only after that go to traditional reclaim
https://jira.sw.ru/browse/PSBM-89403
Signed-off-by: Andrey Ryabinin <aryabinin at virtuozzo.com>
---
mm/internal.h | 14 ++++++++++++++
mm/tcache.c | 45 ++++++++++-----------------------------------
mm/tswap.c | 30 +++++-------------------------
mm/vmscan.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++--
4 files changed, 77 insertions(+), 62 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index 2072b9b04b6b..07690863ca95 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -384,6 +384,20 @@ unsigned long reclaim_clean_pages_from_list(struct zone *zone,
enum ttu_flags;
struct tlbflush_unmap_batch;
+#ifdef CONFIG_TCACHE
+unsigned long tswap_shrink_scan(struct shrink_control *sc);
+#else
+static inline tswap_shrink_scan(struct shrink_control *sc)
+{ return 0; }
+#endif
+
+#ifdef CONFIG_TSWAP
+unsigned long tcache_shrink_scan(struct shrink_control *sc);
+#else
+static inline unsigned long tcache_shrink_scan(struct shrink_control *sc)
+{ return 0; }
+#endif
+
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
void try_to_unmap_flush(void);
void try_to_unmap_flush_dirty(void);
diff --git a/mm/tcache.c b/mm/tcache.c
index eb9c9dea4e51..fe67eda127fb 100644
--- a/mm/tcache.c
+++ b/mm/tcache.c
@@ -1196,52 +1196,33 @@ static struct page *tcache_alloc_page(struct tcache_pool *pool)
return page;
}
-static unsigned long tcache_shrink_count(struct shrinker *shrink,
- struct shrink_control *sc)
-{
- atomic_long_t *nr_pages = &tcache_nodeinfo[sc->nid].nr_pages;
- long ret;
-
- ret = atomic_long_read(nr_pages);
- WARN_ON(ret < 0);
- return ret >= 0 ? ret : 0;
-}
-
#define TCACHE_SCAN_BATCH 128UL
static DEFINE_PER_CPU(struct page * [TCACHE_SCAN_BATCH], tcache_page_vec);
-static unsigned long tcache_shrink_scan(struct shrinker *shrink,
- struct shrink_control *sc)
+unsigned long tcache_shrink_scan(struct shrink_control *sc)
{
- long nr_isolated, nr_reclaimed;
+ long nr_isolated, nr_reclaimed = 0;
struct page **pages;
+ atomic_long_t *nr_pages = &tcache_nodeinfo[sc->nid].nr_pages;
+
+ if (atomic_long_read(nr_pages) <= 0)
+ return 0;
pages = get_cpu_var(tcache_page_vec); /* Implies rcu_read_lock_sched() */
- if (WARN_ON(sc->nr_to_scan > TCACHE_SCAN_BATCH))
+ if (sc->nr_to_scan > TCACHE_SCAN_BATCH)
sc->nr_to_scan = TCACHE_SCAN_BATCH;
nr_isolated = tcache_lru_isolate(sc->nid, pages, sc->nr_to_scan);
- if (!nr_isolated) {
- nr_reclaimed = SHRINK_STOP;
+ if (!nr_isolated)
goto out;
- }
+
nr_reclaimed = tcache_reclaim_pages(pages, nr_isolated);
- if (current->reclaim_state)
- current->reclaim_state->reclaimed_slab += nr_reclaimed;
out:
put_cpu_var(tcache_page_vec); /* Implies rcu_read_unlock_sched() */
return nr_reclaimed;
}
-struct shrinker tcache_shrinker = {
- .count_objects = tcache_shrink_count,
- .scan_objects = tcache_shrink_scan,
- .seeks = 8,
- .batch = TCACHE_SCAN_BATCH,
- .flags = SHRINKER_NUMA_AWARE,
-};
-
static int tcache_cleancache_init_fs(size_t pagesize)
{
return tcache_create_pool();
@@ -1448,23 +1429,17 @@ static int __init tcache_init(void)
if (err)
goto out_fail;
- err = register_shrinker(&tcache_shrinker);
- if (err)
- goto out_free_lru;
-
#ifdef CONFIG_SMP
num_node_trees = roundup_pow_of_two(2 * num_possible_cpus());
#endif
err = cleancache_register_ops(&tcache_cleancache_ops);
if (err)
- goto out_unregister_shrinker;
+ goto out_free_lru;
pr_info("tcache loaded\n");
return 0;
-out_unregister_shrinker:
- unregister_shrinker(&tcache_shrinker);
out_free_lru:
kfree(tcache_nodeinfo);
out_fail:
diff --git a/mm/tswap.c b/mm/tswap.c
index e6804dcba6e2..c2dacac513a6 100644
--- a/mm/tswap.c
+++ b/mm/tswap.c
@@ -126,12 +126,6 @@ static struct page *tswap_delete_page(swp_entry_t entry, struct page *expected)
return page;
}
-static unsigned long tswap_shrink_count(struct shrinker *shrink,
- struct shrink_control *sc)
-{
- return tswap_lru_node[sc->nid].nr_items;
-}
-
static int tswap_evict_page(struct page *page)
{
struct address_space *swapper_space;
@@ -236,12 +230,14 @@ static int tswap_evict_page(struct page *page)
return err;
}
-static unsigned long tswap_shrink_scan(struct shrinker *shrink,
- struct shrink_control *sc)
+unsigned long tswap_shrink_scan(struct shrink_control *sc)
{
struct tswap_lru *lru = &tswap_lru_node[sc->nid];
unsigned long nr_reclaimed = 0;
+ if (tswap_lru_node[sc->nid].nr_items <= 0)
+ return 0;
+
spin_lock(&tswap_lock);
while (sc->nr_to_scan-- > 0) {
struct page *page;
@@ -274,13 +270,6 @@ static unsigned long tswap_shrink_scan(struct shrinker *shrink,
return nr_reclaimed;
}
-static struct shrinker tswap_shrinker = {
- .count_objects = tswap_shrink_count,
- .scan_objects = tswap_shrink_scan,
- .seeks = 4,
- .flags = SHRINKER_NUMA_AWARE,
-};
-
static void tswap_frontswap_init(unsigned type)
{
/*
@@ -424,11 +413,7 @@ static int __init tswap_init(void)
err = tswap_lru_init();
if (err)
- goto out_fail;
-
- err = register_shrinker(&tswap_shrinker);
- if (err)
- goto out_free_lru;
+ return err;
frontswap_tmem_exclusive_gets(true);
@@ -438,11 +423,6 @@ static int __init tswap_init(void)
pr_warn("tswap: frontswap_ops %p overridden\n", old_ops);
return 0;
-
-out_free_lru:
- kfree(tswap_lru_node);
-out_fail:
- return err;
}
module_init(tswap_init);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 6be538ce81b6..4cc561c2edcc 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -584,8 +584,15 @@ void drop_slab_node(int nid)
do {
struct mem_cgroup *memcg;
+ struct shrink_control sc = {
+ .nid = nid,
+ .memcg = NULL,
+ .nr_to_scan = -1,
+ };
+
+ freed = tcache_shrink_scan(&sc);
+ freed += tswap_shrink_scan(&sc);
- freed = 0;
memcg = mem_cgroup_iter(NULL, NULL, NULL);
do {
freed += shrink_slab(GFP_KERNEL, nid, memcg,
@@ -2793,6 +2800,35 @@ static bool all_unreclaimable(struct zonelist *zonelist,
return true;
}
+unsigned long tswap_shrink_scan(struct shrink_control *sc);
+unsigned long tcache_shrink_scan(struct shrink_control *sc);
+
+static void shrink_tcrutches(struct scan_control *scan_ctrl)
+{
+ int nid;
+ unsigned long shrunk;
+ nodemask_t *nodemask = scan_ctrl->nodemask ? : &node_online_map;
+
+ do {
+ shrunk = 0;
+
+ for_each_node_mask(nid, *nodemask) {
+ struct shrink_control sc = {
+ .gfp_mask = scan_ctrl->gfp_mask,
+ .nid = nid,
+ .memcg = NULL,
+ .nr_to_scan = scan_ctrl->nr_to_reclaim - scan_ctrl->nr_reclaimed,
+ };
+ shrunk = tcache_shrink_scan(&sc);
+ scan_ctrl->nr_reclaimed += shrunk;
+ if (shrunk < scan_ctrl->nr_to_reclaim)
+ shrunk += tswap_shrink_scan(&sc);
+ else
+ break;
+ }
+ } while (shrunk && scan_ctrl->nr_reclaimed < scan_ctrl->nr_to_reclaim);
+}
+
/*
* This is the main entry point to direct page reclaim.
*
@@ -2823,8 +2859,12 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
{KSTAT_PERF_ENTER(ttfp);
delayacct_freepages_start();
- if (global_reclaim(sc))
+ if (global_reclaim(sc)) {
count_vm_event(ALLOCSTALL);
+ shrink_tcrutches(sc);
+ if (sc->nr_reclaimed >= sc->nr_to_reclaim)
+ goto out;
+ }
do {
vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
@@ -3466,6 +3506,12 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
if (sc.priority < DEF_PRIORITY - 2)
sc.may_writepage = 1;
+ shrink_tcrutches(&sc);
+ if (sc.nr_reclaimed >= sc.nr_to_reclaim &&
+ pgdat_balanced(pgdat, order, *classzone_idx))
+ goto out;
+
+
/*
* Now scan the zone in the dma->highmem direction, stopping
* at the last zone which needs scanning.
--
2.18.1
More information about the Devel
mailing list