[Devel] [PATCH hci-8.1 6/8] oracle/mm: zap the preserved ranges of the old mm after copying them

[Konstantin Khorenko]

A preserved page keeps PageAnonExclusive in both the old and the new mm
after the copy. exec_mmap() only drops a reference to the old mm, and
anything that holds mm_users keeps it mapped there, such as a
/proc/pid/mem reader stuck in copy_to_user() or process_vm_readv(). Code
that trusts PageAnonExclusive to mean a single mapping then goes wrong.
UFFDIO_MOVE in either mm calls folio_move_anon_rmap() on a folio the
other mm still maps, and once the destination anon_vma is freed the next
rmap walk reads freed memory. try_to_migrate_one() hits VM_BUG_ON_PAGE()
for a writable PTE of a page that the other mapping made shared, and a
writable set_pte_at() in the new mm trips PAGE_TABLE_CHECK. A write
through the old mm reuses the page and lands in the new image.

Zap the VM_EXEC_KEEP ranges of the old mm right after the copy, under
its write lock, so that the copies are the only mappings left. The
pages are not freed, the new mm holds them; a later access through the
old mm faults in fresh memory there. The unmap work moves from the
teardown of the old mm, which ran it with several threads, into the
zap, so zap_vma_mt() spreads an anonymous VMA over several threads as
well, each with an mmu_gather of its own as concurrent MADV_DONTNEED
calls have; the teardown finds these VMAs empty and no longer treats
VM_EXEC_KEEP specially.

Fixes: 37d9f55655ea ("oracle/mm: do not write protect COW mappings when preserving across exec")
https://virtuozzo.atlassian.net/browse/VSTOR-150231

Feature: oracle/mm: MADV_DOEXEC madvise() flag
Assisted-by: LLM
Signed-off-by: Konstantin Khorenko <khorenko at virtuozzo.com>
---
 fs/exec.c          | 10 +++++++-
 include/linux/mm.h |  1 +
 mm/memory.c        | 74 +++++++++++++++++++++++++++++++++++++++++-------------
 3 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/fs/exec.c b/fs/exec.c
index 8677347963e20..32567010a2f27 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -997,7 +997,15 @@ static int vma_dup_some(struct mm_struct *old_mm, struct mm_struct *new_mm)
 		}
 
 	mmap_write_unlock(new_mm);
-	return ret;
+	if (ret)
+		return ret;
+
+	/* Whoever keeps old_mm alive past the exec must not reach the pages through it. */
+	vma_iter_set(&vmi, 0);
+	for_each_vma(vmi, vma)
+		if (vma->vm_flags & VM_EXEC_KEEP)
+			zap_vma_mt(vma);
+	return 0;
 }
 
 /*
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 75b8bcd9f4428..9ebf3c6dcb978 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2462,6 +2462,7 @@ struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 
 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
 		  unsigned long size);
+void zap_vma_mt(struct vm_area_struct *vma);
 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
 			   unsigned long size, struct zap_details *details);
 static inline void zap_vma_pages(struct vm_area_struct *vma)
diff --git a/mm/memory.c b/mm/memory.c
index 1c128162f5cda..2566f02fb7d5e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2043,30 +2043,20 @@ static void unmap_page_range_mt(struct mmu_gather *tlb,
 
 	BUG_ON(addr >= end);
 
-	/*
-	 * Assumptions made below about VM_EXEC_KEEP VMAs only hold true
-	 * if the entire address space is being unmapped.
-	 */
 	if (!tlb->fullmm) {
 		unmap_page_range(tlb, vma, addr, end, details);
 		return;
 	}
 
 	/*
-	 * Pages in VMAs being preserved will have an additional reference
-	 * due to the mappings having been copied prior to unmapping the full
-	 * address space. Since lock contention due to freeing pages will not
-	 * be an issue, use additional threads to unmap them.
-	 * Otherwise for sufficiently large, private anon VMAs, using
-	 * multiple threads can result in severe lock contention the swap
-	 * subsystem if a signficant portion is swapped out. Since the number
-	 * of swapped pages is tracked per MM and not per VMA, err on the
-	 * side of caution and assume the number applies to the VMA and revert
-	 * to unmapping single threaded if necessary.
+	 * For sufficiently large, private anon VMAs, using multiple threads
+	 * can result in severe lock contention in the swap subsystem if a
+	 * significant portion is swapped out. Since the number of swapped
+	 * pages is tracked per MM and not per VMA, err on the side of caution
+	 * and assume the number applies to the VMA and revert to unmapping
+	 * single threaded if necessary.
 	 */
-	if (vma->vm_flags & VM_EXEC_KEEP) {
-		job.max_threads = 16;
-	} else if (!(vma->vm_flags & VM_SHARED) && total_sz >= min_chunk_sz * 3) {
+	if (!(vma->vm_flags & VM_SHARED) && total_sz >= min_chunk_sz * 3) {
 		unsigned long swapped_sz;
 
 		swapped_sz = get_mm_counter(tlb->mm, MM_SWAPENTS) << PAGE_SHIFT;
@@ -2079,6 +2069,17 @@ static void unmap_page_range_mt(struct mmu_gather *tlb,
 	padata_do_multithreaded(&job);
 }
 
+static int zap_vma_chunk(unsigned long addr, unsigned long end, void *arg)
+{
+	struct vm_area_struct *vma = arg;
+	struct mmu_gather tlb;
+
+	tlb_gather_mmu(&tlb, vma->vm_mm);
+	unmap_page_range(&tlb, vma, addr, end, NULL);
+	tlb_finish_mmu(&tlb);
+	return 0;
+}
+
 #endif /* CONFIG_PADATA */
 
 static void unmap_single_vma(struct mmu_gather *tlb,
@@ -2214,6 +2215,45 @@ void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
 	hugetlb_zap_end(vma, details);
 }
 
+/**
+ * zap_vma_mt - remove all ptes of a vma, with several threads if it is large
+ * @vma: vm_area_struct holding ptes to be zapped
+ *
+ * The caller holds the mmap lock for writing. An anonymous vma is zapped in
+ * PMD-aligned parts, each with an mmu_gather of its own, by several threads
+ * when it is large; other vmas, and all of them without CONFIG_PADATA, are
+ * zapped by the caller alone.
+ */
+void zap_vma_mt(struct vm_area_struct *vma)
+{
+#ifdef CONFIG_PADATA
+	struct padata_mt_job job = {
+		.thread_fn   = zap_vma_chunk,
+		.fn_arg      = vma,
+		.start       = vma->vm_start,
+		.size        = vma->vm_end - vma->vm_start,
+		.align       = PMD_SIZE,
+		.min_chunk   = max(1ul << 27, PMD_SIZE),
+		.max_threads = 16,
+	};
+	struct mmu_notifier_range range;
+
+	mmap_assert_write_locked(vma->vm_mm);
+	if (vma_is_anonymous(vma)) {
+		lru_add_drain();
+		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
+					vma->vm_start, vma->vm_end);
+		update_hiwater_rss(vma->vm_mm);
+		mmu_notifier_invalidate_range_start(&range);
+		padata_do_multithreaded(&job);
+		mmu_notifier_invalidate_range_end(&range);
+		return;
+	}
+#endif
+	zap_page_range_single(vma, vma->vm_start, vma->vm_end - vma->vm_start,
+			      NULL);
+}
+
 /**
  * zap_vma_ptes - remove ptes mapping the vma
  * @vma: vm_area_struct holding ptes to be zapped

-- 
2.43.0