[Devel] [PATCH hci-8.1 1/3] fs/fuse kio: teach pcs_umem about hugepage-backed mappings
Liu Kui
kui.liu at virtuozzo.com
Wed Sep 30 10:34:31 MSK 2026
pcs_umem pins every mapping as a flat array of 4k pages and the krpc
bvec builder emits one bio_vec per page, so a 1M chunk costs 256 page
pointers and 256 bio_vecs even when it sits in a single hugepage.
Detect hugepage backing after pinning: if the range is 2M-aligned and
each 2M chunk is backed by one folio (hugetlbfs or fully-mapped THP),
keep only the folio head page per chunk and raise umem->page_shift to
PMD_SHIFT. The bvec builder then caps bv_len at the backing unit
instead of PAGE_SIZE, so a 1M chunk in a hugepage umem becomes a
single bio_vec. Mixed or misaligned mappings stay at 4k.
Also pin with pin_user_pages_fast(), which needs no mmap_lock, and
fix the pcs_umem_get() error path, which leaked chunk page arrays and
could deref a NULL page_chunk.
https://virtuozzo.atlassian.net/browse/VSTOR-142166
Signed-off-by: Liu Kui <kui.liu at virtuozzo.com>
---
fs/fuse/kio/pcs/pcs_krpc.c | 9 ++++-
fs/fuse/kio/pcs/pcs_mr.c | 84 +++++++++++++++++++++++++++++++++++++---------
fs/fuse/kio/pcs/pcs_mr.h | 45 +++++++++++++++++++++----
3 files changed, 115 insertions(+), 23 deletions(-)
diff --git a/fs/fuse/kio/pcs/pcs_krpc.c b/fs/fuse/kio/pcs/pcs_krpc.c
index 7d40e17ce8ae..d4baf28cef71 100644
--- a/fs/fuse/kio/pcs/pcs_krpc.c
+++ b/fs/fuse/kio/pcs/pcs_krpc.c
@@ -410,6 +410,7 @@ static void kreq_prepare_data_buff(struct krpc_req *kreq, struct krpc_chunk *chu
}
} else {
struct pcs_umem *umem;
+ unsigned int contig;
umem = (chunk->type == KRPC_CHUNK_TYPE_MR) ? chunk->mr->umem : chunk->umem;
@@ -426,7 +427,13 @@ static void kreq_prepare_data_buff(struct krpc_req *kreq, struct krpc_chunk *chu
bvec->bv_page = page;
bvec->bv_offset = offset;
- bvec->bv_len = len < (PAGE_SIZE - offset) ? len : (PAGE_SIZE - offset);
+ /*
+ * One bio_vec per physically contiguous extent: a
+ * whole 2M block when the umem is hugepage-backed,
+ * a single page otherwise.
+ */
+ contig = pcs_umem_contig(umem, pos);
+ bvec->bv_len = len < contig ? len : contig;
pos += bvec->bv_len;
len -= bvec->bv_len;
diff --git a/fs/fuse/kio/pcs/pcs_mr.c b/fs/fuse/kio/pcs/pcs_mr.c
index 1e3a74033510..58f1ff41c750 100644
--- a/fs/fuse/kio/pcs/pcs_mr.c
+++ b/fs/fuse/kio/pcs/pcs_mr.c
@@ -7,20 +7,30 @@
#include <linux/slab.h>
#include <linux/sched/mm.h>
#include <linux/resource.h>
+#include <linux/mm.h>
+#include <linux/pgtable.h>
#include "pcs_mr.h"
void pcs_umem_release(struct pcs_umem *umem)
{
struct mm_struct *mm_s = umem->mm;
- int i, npages = umem->npages;
-
- for (i = 0; npages; i++) {
- int to_free = min_t(int, PCS_PAGES_PER_CHUNK, npages);
-
- unpin_user_pages(umem->page_chunk[i].pages, to_free);
- kfree(umem->page_chunk[i].pages);
- npages -= to_free;
+ int npages = umem->npages;
+ int i;
+
+ for (i = 0; umem->page_chunk && i < umem->nchunks; i++) {
+ struct pcs_page_chunk *chunk = &umem->page_chunk[i];
+ int n = min_t(int, PCS_PAGES_PER_CHUNK, npages);
+
+ if (umem->page_shift == PMD_SHIFT) {
+ /* one full hugepage folio per chunk */
+ unpin_user_page_range_dirty_lock(chunk->head,
+ PCS_PAGES_PER_CHUNK, false);
+ } else if (chunk->pages) {
+ unpin_user_pages(chunk->pages, n);
+ kfree(chunk->pages);
+ }
+ npages -= n;
}
atomic64_sub(umem->npages, &mm_s->pinned_vm);
mmdrop(mm_s);
@@ -28,6 +38,43 @@ void pcs_umem_release(struct pcs_umem *umem)
kfree(umem);
}
+/*
+ * A page chunk covers exactly one hugepage. If the whole pinned range
+ * is 2M-aligned and every chunk is backed by one aligned hugepage folio
+ * (hugetlbfs or fully-mapped THP), drop the 4k page arrays and keep just
+ * the folio head page in each chunk, raising page_shift accordingly.
+ * Mixed or misaligned mappings simply stay at 4k granularity.
+ */
+static void pcs_umem_try_compact(struct pcs_umem *umem)
+{
+ int i, j;
+
+ if (umem->fp_addr & (PMD_SIZE - 1))
+ return;
+ if (!umem->npages || umem->npages % PCS_PAGES_PER_CHUNK)
+ return;
+
+ for (i = 0; i < umem->nchunks; i++) {
+ struct page *head = umem->page_chunk[i].pages[0];
+ struct folio *folio = page_folio(head);
+
+ if (folio_nr_pages(folio) - folio_page_idx(folio, head) <
+ PCS_PAGES_PER_CHUNK)
+ return;
+ for (j = 1; j < PCS_PAGES_PER_CHUNK; j++)
+ if (umem->page_chunk[i].pages[j] != nth_page(head, j))
+ return;
+ }
+
+ for (i = 0; i < umem->nchunks; i++) {
+ struct page *head = umem->page_chunk[i].pages[0];
+
+ kfree(umem->page_chunk[i].pages);
+ umem->page_chunk[i].head = head;
+ }
+ umem->page_shift = PMD_SHIFT;
+}
+
/*
* Pin pages from userspace memory
*/
@@ -36,11 +83,10 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
struct pcs_umem *umem = NULL;
struct mm_struct *mm_s;
u64 fp_va;
- int npages, nchunks, i, ret;
+ int npages, i, ret;
fp_va = start & PAGE_MASK;
npages = PAGE_ALIGN(start + len - fp_va) >> PAGE_SHIFT;
- nchunks = DIV_ROUND_UP(npages, PCS_PAGES_PER_CHUNK);
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
if (!umem)
@@ -49,12 +95,14 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
/* pin user pages */
mm_s = current->mm;
umem->mm = mm_s;
+ umem->page_shift = PAGE_SHIFT;
+ umem->nchunks = DIV_ROUND_UP(npages, PCS_PAGES_PER_CHUNK);
mmgrab(mm_s);
- mmap_read_lock(mm_s);
umem->fp_addr = fp_va;
- umem->page_chunk = kcalloc(nchunks, sizeof(struct pcs_page_chunk), GFP_KERNEL);
+ umem->page_chunk = kcalloc(umem->nchunks, sizeof(struct pcs_page_chunk),
+ GFP_KERNEL);
if (!umem->page_chunk) {
ret = -ENOMEM;
goto out_err;
@@ -71,9 +119,13 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
umem->page_chunk[i].pages = pages;
while (n) {
- ret = pin_user_pages(fp_va, n, FOLL_WRITE | FOLL_LONGTERM, pages);
- if (ret < 0)
+ ret = pin_user_pages_fast(fp_va, n,
+ FOLL_WRITE | FOLL_LONGTERM, pages);
+ if (ret <= 0) {
+ if (!ret)
+ ret = -EFAULT;
goto out_err;
+ }
atomic64_add(ret, &mm_s->pinned_vm);
umem->npages += ret;
@@ -83,12 +135,12 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
npages -= ret;
}
}
- mmap_read_unlock(mm_s);
+
+ pcs_umem_try_compact(umem);
return umem;
out_err:
- mmap_read_unlock(mm_s);
pcs_umem_release(umem);
return ERR_PTR(ret);
diff --git a/fs/fuse/kio/pcs/pcs_mr.h b/fs/fuse/kio/pcs/pcs_mr.h
index 64f237f57dec..ea02af1dd622 100644
--- a/fs/fuse/kio/pcs/pcs_mr.h
+++ b/fs/fuse/kio/pcs/pcs_mr.h
@@ -6,12 +6,14 @@
#define _PCS_MR_H_ 1
#include <linux/types.h>
+#include <linux/mm.h>
#include <linux/highmem-internal.h>
struct pcs_umem;
#define PCS_MAX_MR 0x10000
-#define PCS_PAGE_CHUNK_SHIFT 9
+/* A page chunk covers exactly one hugepage: 2M, PMD_SIZE */
+#define PCS_PAGE_CHUNK_SHIFT (PMD_SHIFT - PAGE_SHIFT)
#define PCS_PAGES_PER_CHUNK (1 << PCS_PAGE_CHUNK_SHIFT)
struct pcs_mr_set {
@@ -20,14 +22,30 @@ struct pcs_mr_set {
atomic_t mr_num; /* number of registered MRs*/
};
+/*
+ * A chunk covers exactly one 2M hugepage of VA: PCS_PAGES_PER_CHUNK 4k
+ * pages. Depending on umem->page_shift it stores either an array of
+ * the pinned 4k pages, or — when the chunk is backed by one aligned
+ * hugepage folio — just the folio head page, with no array at all.
+ */
struct pcs_page_chunk
{
- struct page **pages; /* array of pinned pages */
+ union {
+ struct page **pages; /* page_shift == PAGE_SHIFT */
+ struct page *head; /* page_shift == PMD_SHIFT */
+ };
};
struct pcs_umem {
struct pcs_page_chunk *page_chunk;
- int npages; /* number of pinned pages */
+ int nchunks; /* number of allocated page chunks */
+ int npages; /* number of pinned PAGE_SIZE pages */
+ /*
+ * Granularity of the stored page pointers: PAGE_SHIFT (chunks
+ * hold 4k page arrays) or PMD_SHIFT (chunks hold hugepage folio
+ * heads).
+ */
+ unsigned int page_shift;
u64 fp_addr; /* First page base address */
struct mm_struct *mm; /* mm the memory belongs to */
};
@@ -47,13 +65,28 @@ struct pcs_mr {
*/
static inline struct page *pcs_umem_page(struct pcs_umem *umem, u64 addr)
{
- unsigned int page_idx = (addr - umem->fp_addr) >> PAGE_SHIFT;
+ u64 off = addr - umem->fp_addr;
+ unsigned int page_idx = off >> PAGE_SHIFT;
unsigned int chunk_idx = page_idx >> PCS_PAGE_CHUNK_SHIFT;
unsigned int page_in_chunk = page_idx & (PCS_PAGES_PER_CHUNK - 1);
- if (likely(page_idx < umem->npages))
+ if (unlikely(page_idx >= umem->npages))
+ return NULL;
+
+ if (umem->page_shift == PAGE_SHIFT)
return umem->page_chunk[chunk_idx].pages[page_in_chunk];
- return NULL;
+ return nth_page(umem->page_chunk[chunk_idx].head, page_in_chunk);
+}
+
+/*
+ * Number of physically contiguous bytes mapped at the address: from addr
+ * to the end of its backing unit (PAGE_SIZE or a 2M hugepage folio).
+ * Lets callers build bio_vecs spanning a whole hugepage instead of
+ * one bio_vec per 4k page.
+ */
+static inline unsigned int pcs_umem_contig(struct pcs_umem *umem, u64 addr)
+{
+ return (1U << umem->page_shift) - (addr & ((1U << umem->page_shift) - 1));
}
struct pcs_umem *pcs_umem_get(u64 start, u64 len);
--
Git-155)
More information about the Devel
mailing list