[Devel] [PATCH RHEL8 COMMIT] mm: Add and use batched version of __tlb_remove_table()

[Konstantin Khorenko]

The commit is pushed to "branch-rh8-4.18.0-240.1.1.vz8.5.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh8-4.18.0-240.1.1.vz8.5.43
------>
commit 24de49d8f54514751d3e4e2ac9b730a84fdf6f3b
Author: Andrey Ryabinin <aryabinin at virtuozzo.com>
Date:   Wed Jun 16 13:57:45 2021 +0300

    mm: Add and use batched version of __tlb_remove_table()
    
    tlb_remove_table_rcu() removes tables on by one using the
    __tlb_remove_table() -> free_page_and_swap_cache(table). Use batched
    free_pages_and_swap_cache_nodrain() instead to remove all tables in
    one go. This helps to remove contention on the memcgroups counters
    since we decrease them only once instead of decrementing one by one
    for each page individually.
    
    https://jira.sw.ru/browse/PSBM-101300
    Signed-off-by: Andrey Ryabinin <aryabinin at virtuozzo.com>
    
    (cherry-picked from vz7 commit 9a3ca2497cdb ("mm: Add and use batched version of
    __tlb_remove_table()"))
    
    https://jira.sw.ru/browse/PSBM-127854
    Signed-off-by: Valeriy Vdovin <valeriy.vdovin at virtuozzo.com>
---
 arch/x86/include/asm/tlb.h |  5 +++++
 include/linux/swap.h       |  1 +
 mm/memory.c                |  4 +---
 mm/swap_state.c            | 16 ++++++++++++++--
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index cb0a1f470980..84b07ca204b0 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -30,4 +30,9 @@ static inline void __tlb_remove_table(void *table)
 	free_page_and_swap_cache(table);
 }
 
+static inline void __tlb_remove_tables(void **tables, int nr)
+{
+	free_pages_and_swap_cache_nodrain((struct page **)tables, nr);
+}
+
 #endif /* _ASM_X86_TLB_H */
diff --git a/include/linux/swap.h b/include/linux/swap.h
index e27872d995ad..ab512a668601 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -414,6 +414,7 @@ extern void __delete_from_swap_cache(struct page *);
 extern void delete_from_swap_cache(struct page *);
 extern void free_page_and_swap_cache(struct page *);
 extern void free_pages_and_swap_cache(struct page **, int);
+extern void free_pages_and_swap_cache_nodrain(struct page **, int);
 extern struct page *lookup_swap_cache(swp_entry_t entry,
 				      struct vm_area_struct *vma,
 				      unsigned long addr);
diff --git a/mm/memory.c b/mm/memory.c
index f13349d74a06..5105a0e727fd 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -358,12 +358,10 @@ static void tlb_remove_table_one(void *table)
 static void tlb_remove_table_rcu(struct rcu_head *head)
 {
 	struct mmu_table_batch *batch;
-	int i;
 
 	batch = container_of(head, struct mmu_table_batch, rcu);
 
-	for (i = 0; i < batch->nr; i++)
-		__tlb_remove_table(batch->tables[i]);
+	__tlb_remove_tables(batch->tables, batch->nr);
 
 	free_page((unsigned long)batch);
 }
diff --git a/mm/swap_state.c b/mm/swap_state.c
index a626e0da0daf..9a8e684ef707 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -304,17 +304,29 @@ void free_page_and_swap_cache(struct page *page)
  * Passed an array of pages, drop them all from swapcache and then release
  * them.  They are removed from the LRU and freed if this is their last use.
  */
-void free_pages_and_swap_cache(struct page **pages, int nr)
+void __free_pages_and_swap_cache(struct page **pages, int nr, bool drain)
 {
 	struct page **pagep = pages;
 	int i;
 
-	lru_add_drain();
+	if (drain)
+		lru_add_drain();
+
 	for (i = 0; i < nr; i++)
 		free_swap_cache(pagep[i]);
 	release_pages(pagep, nr);
 }
 
+void free_pages_and_swap_cache(struct page **pages, int nr)
+{
+	__free_pages_and_swap_cache(pages, nr, true);
+}
+
+void free_pages_and_swap_cache_nodrain(struct page **pages, int nr)
+{
+	__free_pages_and_swap_cache(pages, nr, false);
+}
+
 static inline bool swap_use_vma_readahead(void)
 {
 	return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap);