[Devel] [PATCH rh7 7/9] tcache: Make tcache_nodeinfo::nr_pages atomic_long_t

[Kirill Tkhai]

This allows to do not avoid tcache_nodeinfo::lock
to change nr_pages.

Signed-off-by: Kirill Tkhai <ktkhai at virtuozzo.com>
---
 mm/tcache.c |   23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/mm/tcache.c b/mm/tcache.c
index 21f1347b517..53fc08de447 100644
--- a/mm/tcache.c
+++ b/mm/tcache.c
@@ -159,7 +159,7 @@ struct tcache_nodeinfo {
 	struct rb_root reclaim_tree;
 
 	/* total number of pages on all LRU lists corresponding to this node */
-	unsigned long nr_pages;
+	atomic_long_t nr_pages;
 } ____cacheline_aligned_in_smp;
 
 /*
@@ -276,8 +276,6 @@ static void tcache_lru_add(struct tcache_pool *pool, struct page *page)
 
 	spin_lock(&ni->lock);
 	spin_lock(&pni->lock);
-
-	ni->nr_pages++;
 	pni->nr_pages++;
 	list_add_tail(&page->lru, &pni->lru);
 
@@ -287,6 +285,8 @@ static void tcache_lru_add(struct tcache_pool *pool, struct page *page)
 		pni->recent_puts /= 2;
 	}
 
+	atomic_long_inc(&ni->nr_pages);
+
 	if (tcache_check_events(ni, pni) || RB_EMPTY_NODE(&pni->reclaim_node)) {
 		if (!RB_EMPTY_NODE(&pni->reclaim_node))
 			rb_erase(&pni->reclaim_node, &ni->reclaim_tree);
@@ -324,11 +324,12 @@ static void tcache_lru_del(struct tcache_pool *pool, struct page *page,
 		goto out;
 
 	__tcache_lru_del(pni, page);
-	ni->nr_pages--;
 
 	if (reused)
 		pni->recent_gets++;
 
+	atomic_long_dec(&ni->nr_pages);
+
 	if (tcache_check_events(ni, pni)) {
 		if (!RB_EMPTY_NODE(&pni->reclaim_node))
 			rb_erase(&pni->reclaim_node, &ni->reclaim_tree);
@@ -1079,7 +1080,7 @@ tcache_lru_isolate(int nid, struct page **pages, int nr_to_isolate)
 		goto out_put;
 	}
 
-	ni->nr_pages -= nr;
+	atomic_long_sub(nr, &ni->nr_pages);
 
 	if (!RB_EMPTY_NODE(rbn)) {
 		rb_erase(rbn, &ni->reclaim_tree);
@@ -1141,9 +1142,7 @@ tcache_try_to_reclaim_page(struct tcache_pool *pool, int nid)
 	if (!ret)
 		goto out;
 
-	spin_lock(&ni->lock);
-	ni->nr_pages -= ret;
-	spin_unlock(&ni->lock);
+	atomic_long_dec(&ni->nr_pages);
 
 	if (!__tcache_reclaim_page(page))
 		page = NULL;
@@ -1168,7 +1167,12 @@ static struct page *tcache_alloc_page(struct tcache_pool *pool)
 static unsigned long tcache_shrink_count(struct shrinker *shrink,
 					 struct shrink_control *sc)
 {
-	return tcache_nodeinfo[sc->nid].nr_pages;
+	atomic_long_t *nr_pages = &tcache_nodeinfo[sc->nid].nr_pages;
+	long ret;
+
+	ret = atomic_long_read(nr_pages);
+	WARN_ON(ret < 0);
+	return ret >= 0 ? ret : 0;
 }
 
 #define TCACHE_SCAN_BATCH 128UL
@@ -1385,6 +1389,7 @@ static int __init tcache_nodeinfo_init(void)
 	for (i = 0; i < nr_node_ids; i++) {
 		ni = &tcache_nodeinfo[i];
 		spin_lock_init(&ni->lock);
+		atomic_long_set(&ni->nr_pages, 0);
 		ni->reclaim_tree = RB_ROOT;
 	}
 	return 0;