[Devel] [PATCH rh7 v3 02/10] tcache: Add tcache_pool_nodeinfo::lock

[Kirill Tkhai]

Currently, for protection of all LRU lists is used tcache_nodeinfo::lock,
which is the only for the NUMA node, and it is used for all containers.
It's used when every container adds a page to LRU list. This makes it
"big tcache lock", which does not scale good.

The patch introduces a new lock for protection of struct tcache_pool_nodeinfo
fields, in particular, LRU list. LRU lists of filesystems (of containers)
are independent of each other, so different locks allows to scale better.

This patch only introduces the lock, and the lock order is:
tcache_nodeinfo::lock -> tcache_pool_nodeinfo::lock at the moment.
Next patches gradually will allow to change it vice versa.

Note, that now update of tcache_pool_nodeinfo::nr_pages and
tcache_nodeinfo::nr_pages happens under different locks.

v3: Add spin_lock_init() for lockdep

Signed-off-by: Kirill Tkhai <ktkhai at virtuozzo.com>
---
 mm/tcache.c |   17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/mm/tcache.c b/mm/tcache.c
index 9f296dc3bde..ab70af2c4f7 100644
--- a/mm/tcache.c
+++ b/mm/tcache.c
@@ -66,6 +66,7 @@ struct tcache_pool_nodeinfo {
 	/* increased on every LRU add/del, reset once it gets big enough;
 	 * used for rate limiting rebalancing of reclaim_tree */
 	unsigned long			events;
+	spinlock_t			lock;
 } ____cacheline_aligned_in_smp;
 
 /*
@@ -255,6 +256,7 @@ static void tcache_lru_add(struct tcache_pool *pool, struct page *page)
 	struct tcache_pool_nodeinfo *pni = &pool->nodeinfo[nid];
 
 	spin_lock(&ni->lock);
+	spin_lock(&pni->lock);
 
 	ni->nr_pages++;
 	pni->nr_pages++;
@@ -271,6 +273,7 @@ static void tcache_lru_add(struct tcache_pool *pool, struct page *page)
 	if (unlikely(RB_EMPTY_NODE(&pni->reclaim_node)))
 		__tcache_insert_reclaim_node(ni, pni);
 
+	spin_unlock(&pni->lock);
 	spin_unlock(&ni->lock);
 }
 
@@ -293,6 +296,7 @@ static void tcache_lru_del(struct tcache_pool *pool, struct page *page,
 	struct tcache_pool_nodeinfo *pni = &pool->nodeinfo[nid];
 
 	spin_lock(&ni->lock);
+	spin_lock(&pni->lock);
 
 	/* Raced with reclaimer? */
 	if (unlikely(list_empty(&page->lru)))
@@ -306,6 +310,7 @@ static void tcache_lru_del(struct tcache_pool *pool, struct page *page,
 
 	__tcache_check_events(ni, pni);
 out:
+	spin_unlock(&pni->lock);
 	spin_unlock(&ni->lock);
 }
 
@@ -342,6 +347,7 @@ static int tcache_create_pool(void)
 		pni->pool = pool;
 		RB_CLEAR_NODE(&pni->reclaim_node);
 		INIT_LIST_HEAD(&pni->lru);
+		spin_lock_init(&pni->lock);
 	}
 
 	idr_preload(GFP_KERNEL);
@@ -1039,6 +1045,7 @@ tcache_lru_isolate(int nid, struct page **pages, int nr_to_isolate)
 	if (!tcache_grab_pool(pni->pool))
 		goto again;
 
+	spin_lock(&pni->lock);
 	nr = __tcache_lru_isolate(pni, pages, nr_to_isolate);
 	ni->nr_pages -= nr;
 	nr_isolated += nr;
@@ -1047,6 +1054,7 @@ tcache_lru_isolate(int nid, struct page **pages, int nr_to_isolate)
 	if (!list_empty(&pni->lru))
 		__tcache_insert_reclaim_node(ni, pni);
 
+	spin_unlock(&pni->lock);
 	tcache_put_pool(pni->pool);
 out:
 	spin_unlock_irq(&ni->lock);
@@ -1091,14 +1099,17 @@ tcache_try_to_reclaim_page(struct tcache_pool *pool, int nid)
 
 	local_irq_save(flags);
 
-	spin_lock(&ni->lock);
+	spin_lock(&pni->lock);
 	ret = __tcache_lru_isolate(pni, &page, 1);
-	ni->nr_pages -= ret;
-	spin_unlock(&ni->lock);
+	spin_unlock(&pni->lock);
 
 	if (!ret)
 		goto out;
 
+	spin_lock(&ni->lock);
+	ni->nr_pages -= ret;
+	spin_unlock(&ni->lock);
+
 	if (!__tcache_reclaim_page(page))
 		page = NULL;
 	else