[Devel] [PATCH RH9 1/3] mm: introduce page vz extension (using page_ext)
Alexander Mikhalitsyn
alexander.mikhalitsyn at virtuozzo.com
Tue Oct 19 13:50:54 MSK 2021
This module allows to manage per-page data.
We will use it in memcg page cache limiting feature
to store additional flag on page.
We may also use this in the future in many other
circumstances (for debugging and so on).
See also mm/page_owner.c
https://jira.sw.ru/browse/PSBM-131957
Signed-off-by: Alexander Mikhalitsyn <alexander.mikhalitsyn at virtuozzo.com>
v2: khorenko@:
- renamed *_page_owner() so any later call of such a function breaks the
compilation and requires attention
- renamed *_page_vzext() funtions for making the code similar
https://jira.sw.ru/browse/PSBM-134013
8fb156c9ee ("mm/page_owner: change split_page_owner to take a count")
(cherry picked from commit da21a0ebcdfcbbb4c126525c458d18e4c7cf0aed)
Signed-off-by: Alexander Mikhalitsyn <alexander.mikhalitsyn at virtuozzo.com>
---
include/linux/page_ext.h | 36 ++++++++++++++++++++
include/linux/page_owner.h | 14 ++++----
include/linux/page_vzext.h | 33 +++++++++++++++++++
mm/Makefile | 1 +
mm/huge_memory.c | 3 +-
mm/migrate.c | 3 +-
mm/page_alloc.c | 7 ++--
mm/page_ext.c | 2 ++
mm/page_vzext.c | 67 ++++++++++++++++++++++++++++++++++++++
9 files changed, 154 insertions(+), 12 deletions(-)
create mode 100644 include/linux/page_vzext.h
create mode 100644 mm/page_vzext.c
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index aff81ba31bd8..4a2b76ba86f5 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -5,6 +5,7 @@
#include <linux/types.h>
#include <linux/stacktrace.h>
#include <linux/stackdepot.h>
+#include <linux/page_owner.h>
struct pglist_data;
struct page_ext_operations {
@@ -64,6 +65,28 @@ static inline struct page_ext *page_ext_next(struct page_ext *curr)
return next;
}
+extern void _reset_page_vzext(struct page *page, unsigned int order);
+extern void _split_page_vzext(struct page *page, unsigned int nr);
+extern void _copy_page_vzext(struct page *oldpage, struct page *newpage);
+
+static inline void reset_page_ext(struct page *page, unsigned int order)
+{
+ _reset_page_owner(page, order);
+ _reset_page_vzext(page, order);
+}
+
+static inline void split_page_ext(struct page *page, unsigned int nr)
+{
+ _split_page_owner(page, nr);
+ _split_page_vzext(page, nr);
+}
+
+static inline void copy_page_ext(struct page *oldpage, struct page *newpage)
+{
+ _copy_page_owner(oldpage, newpage);
+ _copy_page_vzext(oldpage, newpage);
+}
+
#else /* !CONFIG_PAGE_EXTENSION */
struct page_ext;
@@ -87,5 +110,18 @@ static inline void page_ext_init_flatmem_late(void)
static inline void page_ext_init_flatmem(void)
{
}
+
+static inline void reset_page_ext(struct page *page, unsigned int order)
+{
+}
+
+static inline void split_page_ext(struct page *page, unsigned int order)
+{
+}
+
+static inline void copy_page_ext(struct page *oldpage, struct page *newpage)
+{
+}
+
#endif /* CONFIG_PAGE_EXTENSION */
#endif /* __LINUX_PAGE_EXT_H */
diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h
index 719bfe5108c5..257ae1b52fba 100644
--- a/include/linux/page_owner.h
+++ b/include/linux/page_owner.h
@@ -18,7 +18,7 @@ extern void __dump_page_owner(const struct page *page);
extern void pagetypeinfo_showmixedcount_print(struct seq_file *m,
pg_data_t *pgdat, struct zone *zone);
-static inline void reset_page_owner(struct page *page, unsigned int order)
+static inline void _reset_page_owner(struct page *page, unsigned int order)
{
if (static_branch_unlikely(&page_owner_inited))
__reset_page_owner(page, order);
@@ -31,12 +31,12 @@ static inline void set_page_owner(struct page *page,
__set_page_owner(page, order, gfp_mask);
}
-static inline void split_page_owner(struct page *page, unsigned int nr)
+static inline void _split_page_owner(struct page *page, unsigned int nr)
{
if (static_branch_unlikely(&page_owner_inited))
__split_page_owner(page, nr);
}
-static inline void copy_page_owner(struct page *oldpage, struct page *newpage)
+static inline void _copy_page_owner(struct page *oldpage, struct page *newpage)
{
if (static_branch_unlikely(&page_owner_inited))
__copy_page_owner(oldpage, newpage);
@@ -52,18 +52,18 @@ static inline void dump_page_owner(const struct page *page)
__dump_page_owner(page);
}
#else
-static inline void reset_page_owner(struct page *page, unsigned int order)
+static inline void _reset_page_owner(struct page *page, unsigned int order)
{
}
static inline void set_page_owner(struct page *page,
unsigned int order, gfp_t gfp_mask)
{
}
-static inline void split_page_owner(struct page *page,
- unsigned int order)
+static inline void _split_page_owner(struct page *page,
+ unsigned int nr)
{
}
-static inline void copy_page_owner(struct page *oldpage, struct page *newpage)
+static inline void _copy_page_owner(struct page *oldpage, struct page *newpage)
{
}
static inline void set_page_owner_migrate_reason(struct page *page, int reason)
diff --git a/include/linux/page_vzext.h b/include/linux/page_vzext.h
new file mode 100644
index 000000000000..48f1d1dd6972
--- /dev/null
+++ b/include/linux/page_vzext.h
@@ -0,0 +1,33 @@
+/*
+ * mm/page_vzext.c
+ *
+ * Copyright (c) 2021 Virtuozzo International GmbH. All rights reserved.
+ *
+ */
+
+#ifndef __LINUX_PAGE_VZEXT_H
+#define __LINUX_PAGE_VZEXT_H
+
+#include <linux/page_ext.h>
+
+extern struct page_ext_operations page_vzext_ops;
+
+extern void _reset_page_vzext(struct page *page, unsigned int order);
+extern void _split_page_vzext(struct page *page, unsigned int order);
+extern void _copy_page_vzext(struct page *oldpage, struct page *newpage);
+
+struct page_vzext {
+ unsigned long vzflags;
+};
+
+static inline struct page_vzext *get_page_vzext(struct page *page)
+{
+ struct page_ext *page_ext = lookup_page_ext(page);
+
+ if (unlikely(!page_ext))
+ return NULL;
+
+ return (void *)page_ext + page_vzext_ops.offset;
+}
+
+#endif /* __LINUX_PAGE_VZEXT_H */
diff --git a/mm/Makefile b/mm/Makefile
index 6a256c83c4e0..e70b5c8ae207 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -103,6 +103,7 @@ obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o
obj-$(CONFIG_DEBUG_RODATA_TEST) += rodata_test.o
obj-$(CONFIG_DEBUG_VM_PGTABLE) += debug_vm_pgtable.o
obj-$(CONFIG_PAGE_OWNER) += page_owner.o
+obj-$(CONFIG_PAGE_EXTENSION) += page_vzext.o
obj-$(CONFIG_CLEANCACHE) += cleancache.o
obj-$(CONFIG_MEMORY_ISOLATION) += page_isolation.o
obj-$(CONFIG_ZPOOL) += zpool.o
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index afff3ac87067..2cc4b3af4eb4 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -34,6 +34,7 @@
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/page_owner.h>
+#include <linux/page_vzext.h>
#include <asm/tlb.h>
#include <asm/pgalloc.h>
@@ -2474,7 +2475,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
unlock_page_lruvec(lruvec);
/* Caller disabled irqs, so they are still disabled here */
- split_page_owner(head, nr);
+ split_page_ext(head, nr);
/* See comment in __split_huge_page_tail() */
if (PageAnon(head)) {
diff --git a/mm/migrate.c b/mm/migrate.c
index 7e240437e7d9..0d18e8e6d1c9 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -46,6 +46,7 @@
#include <linux/mmu_notifier.h>
#include <linux/page_idle.h>
#include <linux/page_owner.h>
+#include <linux/page_vzext.h>
#include <linux/sched/mm.h>
#include <linux/ptrace.h>
#include <linux/oom.h>
@@ -605,7 +606,7 @@ void migrate_page_states(struct page *newpage, struct page *page)
if (PageReadahead(page))
SetPageReadahead(newpage);
- copy_page_owner(page, newpage);
+ copy_page_ext(page, newpage);
if (!PageHuge(page))
mem_cgroup_migrate(page, newpage);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 0e511e1e45c5..3c9afe924231 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -63,6 +63,7 @@
#include <linux/sched/rt.h>
#include <linux/sched/mm.h>
#include <linux/page_owner.h>
+#include <linux/page_vzext.h>
#include <linux/kthread.h>
#include <linux/memcontrol.h>
#include <linux/ftrace.h>
@@ -1326,7 +1327,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
*/
if (memcg_kmem_enabled() && PageMemcgKmem(page))
__memcg_kmem_uncharge_page(page, order);
- reset_page_owner(page, order);
+ reset_page_ext(page, order);
return false;
}
@@ -1369,7 +1370,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
page_cpupid_reset_last(page);
page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
- reset_page_owner(page, order);
+ reset_page_ext(page, order);
if (!PageHighMem(page)) {
debug_check_no_locks_freed(page_address(page),
@@ -3534,7 +3535,7 @@ void split_page(struct page *page, unsigned int order)
for (i = 1; i < (1 << order); i++)
set_page_refcounted(page + i);
- split_page_owner(page, 1 << order);
+ split_page_ext(page, 1 << order);
split_page_memcg(page, 1 << order);
}
EXPORT_SYMBOL_GPL(split_page);
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 293b2685fc48..a2ac8b51576f 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -7,6 +7,7 @@
#include <linux/vmalloc.h>
#include <linux/kmemleak.h>
#include <linux/page_owner.h>
+#include <linux/page_vzext.h>
#include <linux/page_idle.h>
/*
@@ -65,6 +66,7 @@ static struct page_ext_operations *page_ext_ops[] = {
#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
&page_idle_ops,
#endif
+ &page_vzext_ops,
};
unsigned long page_ext_size = sizeof(struct page_ext);
diff --git a/mm/page_vzext.c b/mm/page_vzext.c
new file mode 100644
index 000000000000..4314a14f6845
--- /dev/null
+++ b/mm/page_vzext.c
@@ -0,0 +1,67 @@
+/*
+ * mm/page_vzext.c
+ *
+ * Copyright (c) 2021 Virtuozzo International GmbH. All rights reserved.
+ *
+ */
+
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/memblock.h>
+#include <linux/stacktrace.h>
+#include <linux/page_vzext.h>
+#include <linux/jump_label.h>
+#include <linux/migrate.h>
+
+#include "internal.h"
+
+static bool need_page_vzext(void)
+{
+ return true;
+}
+
+struct page_ext_operations page_vzext_ops = {
+ .size = sizeof(struct page_vzext),
+ .need = need_page_vzext,
+};
+
+static inline struct page_vzext *get_page_ext_vzext(struct page_ext *page_ext)
+{
+ return (void *)page_ext + page_vzext_ops.offset;
+}
+
+void _reset_page_vzext(struct page *page, unsigned int order)
+{
+ /* TODO: write universal code for page deinitialization */
+}
+
+void _split_page_vzext(struct page *page, unsigned int nr)
+{
+ int i;
+ struct page_ext *page_ext = lookup_page_ext(page);
+ struct page_vzext *page_vzext;
+
+ if (unlikely(!page_ext))
+ return;
+
+ page_vzext = get_page_ext_vzext(page_ext);
+ for (i = 1; i < nr; i++)
+ _copy_page_vzext(page, page + i);
+}
+
+void _copy_page_vzext(struct page *oldpage, struct page *newpage)
+{
+ struct page_ext *old_ext = lookup_page_ext(oldpage);
+ struct page_ext *new_ext = lookup_page_ext(newpage);
+ struct page_vzext *old_page_vzext, *new_page_vzext;
+
+ if (unlikely(!old_ext || !new_ext))
+ return;
+
+ old_page_vzext = get_page_ext_vzext(old_ext);
+ new_page_vzext = get_page_ext_vzext(new_ext);
+
+ /* TODO: add callbacks to handle separate vzext in different helpers */
+ new_page_vzext->vzflags = old_page_vzext->vzflags;
+}
--
2.31.1
More information about the Devel
mailing list