[Devel] [PATCH rh7 4/5] slab: ignore memcg limits during marked SLAB allocations

[Konstantin Khorenko]

Pretend marked SLABs are always allocated by a PF_MEMALLOC process to
bypass memory cgroup limits.

Added per-SLAB "pf_memalloc" attribute to disable/enable the logic on
the fly.

https://virtuozzo.atlassian.net/browse/PSBM-155867

Signed-off-by: Konstantin Khorenko <khorenko at virtuozzo.com>
---
 include/linux/slab.h |  1 +
 mm/slub.c            | 59 ++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 8722dc8864be..263f10a436f6 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -24,6 +24,7 @@
 #define SLAB_CONSISTENCY_CHECKS	0x00000100UL	/* DEBUG: Perform (expensive) checks on alloc/free */
 #define SLAB_RED_ZONE		0x00000400UL	/* DEBUG: Red zone objs in a cache */
 #define SLAB_POISON		0x00000800UL	/* DEBUG: Poison objects */
+#define SLAB_PF_MEMALLOC	0x00001000UL	/* DEBUG: Ignore memcg limits */
 #define SLAB_HWCACHE_ALIGN	0x00002000UL	/* Align objs on cache lines */
 #define SLAB_CACHE_DMA		0x00004000UL	/* Use GFP_DMA memory */
 #define SLAB_STORE_USER		0x00010000UL	/* DEBUG: Store the last owner for bug hunting */
diff --git a/mm/slub.c b/mm/slub.c
index 630d39a0d63f..ea3e20e73fe6 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1296,6 +1296,9 @@ static int __init setup_slub_debug(char *str)
 		case 'a':
 			slub_debug |= SLAB_FAILSLAB;
 			break;
+		case 'm':
+			slub_debug |= SLAB_PF_MEMALLOC;
+			break;
 		default:
 			printk(KERN_ERR "slub_debug option '%c' "
 				"unknown. skipped\n", *str);
@@ -2482,8 +2485,8 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
  * Version of __slab_alloc to use when we know that interrupts are
  * already disabled (which is the case for bulk allocation).
  */
-static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
-			  unsigned long addr, struct kmem_cache_cpu *c)
+static void *___slab_alloc_limited(struct kmem_cache *s, gfp_t gfpflags, int node,
+				   unsigned long addr, struct kmem_cache_cpu *c)
 {
 	void *freelist;
 	struct page *page;
@@ -2584,6 +2587,41 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return freelist;
 }
 
+/*
+ * "unlimited" function honors SLAB_PF_MEMALLOC flag on the SLAB and if set,
+ * ignores the memcg limits during the SLAB object allocation.
+ */
+static void *___slab_alloc_unlimited(struct kmem_cache *s, gfp_t gfpflags, int node,
+				     unsigned long addr, struct kmem_cache_cpu *c)
+{
+	/*
+	 * A big crutch: we suspect some SLAB allocation failure leads
+	 * to a memory corruption, so for now we want to make marked SLABs
+	 * to always succeed allocation.
+	 */
+	if (s->flags & SLAB_PF_MEMALLOC) {
+		void *freelist;
+		unsigned long pflags = current->flags;
+
+		current->flags |= PF_MEMALLOC;
+
+		freelist = ___slab_alloc_limited(s, gfpflags, node, addr, c);
+
+		tsk_restore_flags(current, pflags, PF_MEMALLOC);
+		return freelist;
+	} else
+		return ___slab_alloc_limited(s, gfpflags, node, addr, c);
+}
+
+/*
+ * A wrapper to keep the original function name.
+ */
+static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
+			   unsigned long addr, struct kmem_cache_cpu *c)
+{
+	return ___slab_alloc_unlimited(s, gfpflags, node, addr, c);
+}
+
 /*
  * Another one that disabled interrupt and compensates for possible
  * cpu changes by refetching the per cpu area pointer.
@@ -5253,6 +5291,22 @@ static ssize_t poison_store(struct kmem_cache *s,
 }
 SLAB_ATTR(poison);
 
+static ssize_t pf_memalloc_show(struct kmem_cache *s, char *buf)
+{
+	return sprintf(buf, "%d\n", !!(s->flags & SLAB_PF_MEMALLOC));
+}
+
+static ssize_t pf_memalloc_store(struct kmem_cache *s,
+				 const char *buf, size_t length)
+{
+	s->flags &= ~SLAB_PF_MEMALLOC;
+	if (buf[0] == '1') {
+		s->flags |= SLAB_PF_MEMALLOC;
+	}
+	return length;
+}
+SLAB_ATTR(pf_memalloc);
+
 static ssize_t store_user_show(struct kmem_cache *s, char *buf)
 {
 	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
@@ -5475,6 +5529,7 @@ static struct attribute *slab_attrs[] = {
 	&trace_attr.attr,
 	&red_zone_attr.attr,
 	&poison_attr.attr,
+	&pf_memalloc_attr.attr,
 	&store_user_attr.attr,
 	&validate_attr.attr,
 	&alloc_calls_attr.attr,
-- 
2.24.3