[Devel] [PATCH RFC PATCH hci-8.1 3/4] mm: move MADV_DOEXEC memory instead of copying it
Andrey Drobyshev
andrey.drobyshev at virtuozzo.com
Fri Oct 9 17:00:06 MSK 2026
Fork-style duplication write-protects preserved guest RAM. With long-term
DMA pins, subsequent writes can COW to new frames while the device keeps
accessing the old frames. Move mappings to the new mm instead, preserving
physical pages, permissions and anonymous-page exclusivity.
Use the exec reservation and old-mm write lock, clone the anonymous VMA
lineage, unmerge KSM and drop mlock state. Transfer the commit reservation
and pin-aware fork state, and remove the obsolete parallel copying and
exec-specific pinned-page sharing exception. Partial-move failures remain
fatal and tear down both mms after workers finish.
https://virtuozzo.atlassian.net/browse/VSTOR-103860
Signed-off-by: Andrey Drobyshev <andrey.drobyshev at virtuozzo.com>
---
fs/exec.c | 16 ++++++++---
include/linux/mm.h | 4 ---
include/linux/rmap.h | 7 ++---
mm/internal.h | 2 ++
mm/madvise.c | 2 +-
mm/memory.c | 81 ++++------------------------------------------------
mm/mlock.c | 2 +-
mm/mmap.c | 64 +++++++++++++++++++++++++----------------
8 files changed, 63 insertions(+), 115 deletions(-)
diff --git a/fs/exec.c b/fs/exec.c
index 5b2f9c3b0d06..32532111244f 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1057,6 +1057,7 @@ static int vma_dup_some(struct mm_struct *old_mm, struct mm_struct *new_mm)
VMA_ITERATOR(vmi, old_mm, 0);
int ret = 0;
+ update_hiwater_rss(old_mm);
mmap_write_lock_nested(new_mm, SINGLE_DEPTH_NESTING);
for_each_vma(vmi, vma)
if (vma->vm_flags & VM_EXEC_KEEP) {
@@ -1079,6 +1080,7 @@ static int exec_mmap(struct linux_binprm *bprm)
struct task_struct *tsk;
struct mm_struct *old_mm, *active_mm;
struct mm_struct *mm = bprm->mm;
+ const bool preserve = bprm->preserving_mem;
int ret;
/* Notify parent that we're no longer interested in the old VM */
@@ -1096,15 +1098,18 @@ static int exec_mmap(struct linux_binprm *bprm)
* whose default action is to create a coredump get
* out and die instead of going through with the exec.
*/
- ret = mmap_read_lock_killable(old_mm);
+ if (preserve)
+ ret = mmap_write_lock_killable(old_mm);
+ else
+ ret = mmap_read_lock_killable(old_mm);
if (ret) {
up_write(&tsk->signal->exec_update_lock);
return ret;
}
- if (bprm->preserving_mem) {
+ if (preserve) {
ret = vma_dup_some(old_mm, mm);
if (ret) {
- mmap_read_unlock(old_mm);
+ mmap_write_unlock(old_mm);
up_write(&tsk->signal->exec_update_lock);
return ret;
}
@@ -1136,7 +1141,10 @@ static int exec_mmap(struct linux_binprm *bprm)
task_unlock(tsk);
lru_gen_use_mm(mm);
if (old_mm) {
- mmap_read_unlock(old_mm);
+ if (preserve)
+ mmap_write_unlock(old_mm);
+ else
+ mmap_read_unlock(old_mm);
BUG_ON(active_mm != old_mm);
setmax_mm_hiwater_rss(&tsk->signal->maxrss, old_mm);
mm_update_next_owner(old_mm);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 75b8bcd9f442..a4df541f8591 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2479,10 +2479,6 @@ void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
unsigned long end, unsigned long floor, unsigned long ceiling);
int
copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
-#ifdef CONFIG_PADATA
-int copy_page_range_mt(struct vm_area_struct *dst_vma,
- struct vm_area_struct *src_vma);
-#endif
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write);
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 5dc4cd6e4e2c..3645abd38d68 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -400,7 +400,6 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
const int orig_nr_pages = nr_pages;
bool maybe_pinned;
int i;
- bool is_exec_keep;
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
__folio_rmap_sanity_checks(folio, page, nr_pages, level);
@@ -415,8 +414,6 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
maybe_pinned = likely(!folio_is_device_private(folio)) &&
unlikely(folio_needs_cow_for_dma(src_vma, folio));
- is_exec_keep = dst_vma->vm_flags & VM_EXEC_KEEP ? true : false;
-
/*
* No need to check+clear for already shared PTEs/PMDs of the
* folio. But if any page is PageAnonExclusive, we must fallback to
@@ -424,7 +421,7 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
*/
switch (level) {
case RMAP_LEVEL_PTE:
- if (unlikely(maybe_pinned) && likely(!is_exec_keep)) {
+ if (unlikely(maybe_pinned)) {
for (i = 0; i < nr_pages; i++)
if (PageAnonExclusive(page + i))
return -EBUSY;
@@ -446,7 +443,7 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
break;
case RMAP_LEVEL_PMD:
if (PageAnonExclusive(page)) {
- if (unlikely(maybe_pinned) && likely(!is_exec_keep))
+ if (unlikely(maybe_pinned))
return -EBUSY;
ClearPageAnonExclusive(page);
}
diff --git a/mm/internal.h b/mm/internal.h
index b0fed4393315..caf6eabd22b0 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -928,6 +928,8 @@ static inline void mlock_vma_folio(struct folio *folio,
}
void munlock_folio(struct folio *folio);
+void mlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end, vm_flags_t newflags);
static inline void munlock_vma_folio(struct folio *folio,
struct vm_area_struct *vma)
{
diff --git a/mm/madvise.c b/mm/madvise.c
index b89d891e122c..109b5222f48e 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -1655,7 +1655,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
* triggering read faults if required
* MADV_POPULATE_WRITE - populate (prefault) page tables writable by
* triggering write faults if required
- * MADV_DOEXEC - On exec, preserve and duplicate this area in the new process
+ * MADV_DOEXEC - On exec, transfer this area's mappings to the new process
* if the new process allows it.
* MADV_DONTEXEC - Undo the effect of MADV_DOEXEC.
*
diff --git a/mm/memory.c b/mm/memory.c
index 5e414c68f3a9..73060460496c 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1436,66 +1436,6 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
return ret;
}
-#ifdef CONFIG_PADATA
-
-struct copy_page_range_args {
- struct vm_area_struct *dst_vma;
- struct vm_area_struct *src_vma;
-};
-
-static int copy_page_range_chunk(unsigned long addr,
- unsigned long end, void *arg)
-{
- struct copy_page_range_args *args = arg;
- struct vm_area_struct *dst_vma = args->dst_vma;
- struct vm_area_struct *src_vma = args->src_vma;
- struct mm_struct *dst_mm = dst_vma->vm_mm;
- struct mm_struct *src_mm = src_vma->vm_mm;
- pgd_t *src_pgd, *dst_pgd;
- unsigned long next;
- int ret = 0;
-
- dst_pgd = pgd_offset(dst_mm, addr);
- src_pgd = pgd_offset(src_mm, addr);
-
- do {
- next = pgd_addr_end(addr, end);
- if (pgd_none_or_clear_bad(src_pgd))
- continue;
- if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
- addr, next))) {
- ret = -ENOMEM;
- break;
- }
- } while (dst_pgd++, src_pgd++, addr = next, addr != end);
-
- return ret;
-}
-
-/*
- * A stripped down version of copy_page_range() used to copy a VMA as part
- * of preserving it across exec. Multithreading via padata is used to speed
- * up the copying of very large VMAs.
- */
-int copy_page_range_mt(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
-{
- struct copy_page_range_args args = { dst_vma, src_vma };
- struct padata_mt_job job = {
- .thread_fn = copy_page_range_chunk,
- .fn_arg = &args,
- .start = src_vma->vm_start,
- .size = src_vma->vm_end - src_vma->vm_start,
- .align = PMD_SIZE,
- .min_chunk = max(1ul << 27, PMD_SIZE),
- .max_threads = 16,
- };
-
- BUG_ON(!(src_vma->vm_flags & VM_EXEC_KEEP));
-
- return padata_do_multithreaded(&job);
-}
-#endif /* CONFIG_PADATA */
-
/* Whether we should zap all COWed (private) pages too */
static inline bool should_zap_cows(struct zap_details *details)
{
@@ -2033,8 +1973,7 @@ static void unmap_page_range_mt(struct mmu_gather *tlb,
BUG_ON(addr >= end);
/*
- * Assumptions made below about VM_EXEC_KEEP VMAs only hold true
- * if the entire address space is being unmapped.
+ * Parallel unmapping is only safe when tearing down the full mm.
*/
if (!tlb->fullmm) {
unmap_page_range(tlb, vma, addr, end, details);
@@ -2042,20 +1981,12 @@ static void unmap_page_range_mt(struct mmu_gather *tlb,
}
/*
- * Pages in VMAs being preserved will have an additional reference
- * due to the mappings having been copied prior to unmapping the full
- * address space. Since lock contention due to freeing pages will not
- * be an issue, use additional threads to unmap them.
- * Otherwise for sufficiently large, private anon VMAs, using
- * multiple threads can result in severe lock contention the swap
- * subsystem if a signficant portion is swapped out. Since the number
- * of swapped pages is tracked per MM and not per VMA, err on the
- * side of caution and assume the number applies to the VMA and revert
- * to unmapping single threaded if necessary.
+ * For sufficiently large, private anon VMAs, parallel unmapping can
+ * cause severe swap lock contention. The swap count is per-mm, so
+ * conservatively assume it applies to this VMA. Preserved mappings
+ * have already been moved out and need no special treatment here.
*/
- if (vma->vm_flags & VM_EXEC_KEEP) {
- job.max_threads = 16;
- } else if (!(vma->vm_flags & VM_SHARED) && total_sz >= min_chunk_sz * 3) {
+ if (!(vma->vm_flags & VM_SHARED) && total_sz >= min_chunk_sz * 3) {
unsigned long swapped_sz;
swapped_sz = get_mm_counter(tlb->mm, MM_SWAPENTS) << PAGE_SHIFT;
diff --git a/mm/mlock.c b/mm/mlock.c
index 3fc174451561..0d362033c299 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -422,7 +422,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
* Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
* called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
*/
-static void mlock_vma_pages_range(struct vm_area_struct *vma,
+void mlock_vma_pages_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end, vm_flags_t newflags)
{
static const struct mm_walk_ops mlock_walk_ops = {
diff --git a/mm/mmap.c b/mm/mmap.c
index 320f397816df..3db7c8316382 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1751,21 +1751,40 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
}
/*
- * Duplicate an anonymous VMA into a new mm as part of preserving
- * it across exec.
+ * Transfer an anonymous VMA's mappings to a new mm across exec. Both mmap
+ * locks must be held for writing; other tasks using the old mm must be quiesced.
*/
int vma_dup(struct vm_area_struct *old_vma, struct mm_struct *mm)
{
struct vm_area_struct *vma;
unsigned long npages;
+ vm_flags_t flags = old_vma->vm_flags;
int ret = -ENOMEM;
+ mmap_assert_write_locked(old_vma->vm_mm);
+ mmap_assert_write_locked(mm);
+ vma_start_write(old_vma);
+
if (WARN_ON(!vma_is_anonymous(old_vma)))
return -EINVAL;
if (find_vma_intersection(mm, old_vma->vm_start, old_vma->vm_end))
return -EEXIST;
+ /* KSM tracks mappings by mm and address; unmerge before moving them. */
+ ret = ksm_madvise(old_vma, old_vma->vm_start, old_vma->vm_end,
+ MADV_UNMERGEABLE, &flags);
+ if (ret)
+ return ret;
+
+ /* The new image does not inherit mlock, including the folio state. */
+ if (old_vma->vm_flags & VM_LOCKED) {
+ old_vma->vm_mm->locked_vm -= vma_pages(old_vma);
+ mlock_vma_pages_range(old_vma, old_vma->vm_start,
+ old_vma->vm_end,
+ old_vma->vm_flags & ~VM_LOCKED_MASK);
+ }
+
npages = vma_pages(old_vma);
vm_stat_account(mm, old_vma->vm_flags, npages);
@@ -1778,17 +1797,17 @@ int vma_dup(struct vm_area_struct *old_vma, struct mm_struct *mm)
goto fail_nomem_policy;
vma->vm_mm = mm;
- ret = anon_vma_fork(vma, old_vma);
+ /* Keep the same anon_vma: this is a move, not a fork. */
+ ret = anon_vma_clone(vma, old_vma);
if (ret)
- goto fail_nomem_anon_vma_fork;
+ goto fail_nomem_anon_vma_clone;
/*
* Clear functionality that should not carry over to the new
- * process. Note that VM_EXEC_KEEP is cleared later to allow
- * code called by copy_page_range to infer that the copying is
- * for preserving over exec and not for process forking.
+ * process. VM_EXEC_KEEP is consumed by this exec.
*/
- vm_flags_clear(vma, VM_LOCKED|VM_LOCKONFAULT|VM_UFFD_MISSING|VM_UFFD_WP|VM_UFFD_MINOR);
+ vm_flags_clear(vma, VM_LOCKED_MASK | VM_UFFD_MISSING | VM_UFFD_WP |
+ VM_UFFD_MINOR | VM_EXEC_KEEP);
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
ret = vma_link(mm, vma);
@@ -1796,29 +1815,24 @@ int vma_dup(struct vm_area_struct *old_vma, struct mm_struct *mm)
goto fail_nomem_vma_link;
/*
- * Now that the dup vma is inserted into the mm, clear VM_ACCOUNT
- * from old_vma. Since vma_dup() is only called during exec to
- * duplicate a vma from the outgoing mm into the mm of the new
- * process, this effectively transfers the accounting from the old
- * vma to new one.
- * exec holds the mmap read lock on the outgoing mm at this point
- * so __vm_flags_mod() is used here to avoid an assert that the
- * mmap write lock is held which is not needed in this instance..
+ * Transfer the reservation to the linked destination VMA. If moving
+ * page tables fails partway through, exec cannot resume the old image:
+ * it must terminate and tear down both mms. Each mm then frees its
+ * remaining mappings, and the destination releases the reservation.
*/
- __vm_flags_mod(old_vma, 0, VM_ACCOUNT);
+ vm_flags_clear(old_vma, VM_ACCOUNT);
-#ifdef CONFIG_PADATA
- ret = copy_page_range_mt(vma, old_vma);
-#else
- ret = copy_page_range(vma, old_vma);
-#endif
- vm_flags_clear(vma, VM_EXEC_KEEP);
+ /* Preserve the pin-aware fork checks in the new mm. */
+ if (test_bit(MMF_HAS_PINNED, &old_vma->vm_mm->flags))
+ set_bit(MMF_HAS_PINNED, &mm->flags);
- return ret;
+ if (!test_bit(MMF_VM_MERGEABLE, &mm->flags))
+ ksm_fork(mm, old_vma->vm_mm);
+ return move_page_range_exec(vma, old_vma);
fail_nomem_vma_link:
unlink_anon_vmas(vma);
-fail_nomem_anon_vma_fork:
+fail_nomem_anon_vma_clone:
mpol_put(vma_policy(vma));
fail_nomem_policy:
vm_area_free(vma);
--
2.47.1
More information about the Devel
mailing list