[Devel] [PATCH vz7] net/sctp: Allocate SSN map on a per-page basis
Oleg Babin
obabin at virtuozzo.com
Tue Mar 27 20:55:56 MSK 2018
SCTP protocol allocates TCB on receiving INIT and COOKIE ECHO chunks
which specify input and output stream count. As the total count of
those streams can be up to (2^16 - 1) of each type, it is possible
to occupy a fifth order of memory for only one type of streams. Plus
allocation can happen in a softirq contex with GFP_ATOMIC flag set
and we actually do not need the memory to be physically contiguous,
so for performance reasons we allocate memory on a per-page basis.
https://jira.sw.ru/browse/PSBM-82552
Signed-off-by: Oleg Babin <obabin at virtuozzo.com>
---
include/net/sctp/structs.h | 44 +++++++++++---
net/sctp/ssnmap.c | 142 ++++++++++++++++++++++++++++-----------------
2 files changed, 127 insertions(+), 59 deletions(-)
diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
index c2f7064..0a09dbe 100644
--- a/include/net/sctp/structs.h
+++ b/include/net/sctp/structs.h
@@ -396,11 +396,29 @@ typedef struct sctp_sender_hb_info {
*
* This is the structure we use to track both our outbound and inbound
* SSN, or Stream Sequence Numbers.
+ *
+ * As the total count of input or output streams can be up to (2^16 - 1)
+ * of each type, it is possible to occupy a fifth order of memory for
+ * only one type of streams. Plus allocation can happen in a softirq
+ * contex with GFP_ATOMIC flag set and we actually do not need the memory
+ * to be physically contiguous, so for performance reasons we allocate
+ * memory on a per-page basis.
*/
+#define SCTP_MAX_STREAMS 65535
+#define SCTP_SSN_PAGE_SHIFT (PAGE_SHIFT - 1) /* One SSN takes 2 bytes */
+#define SCTP_SSN_PAGE_SIZE (1u << SCTP_SSN_PAGE_SHIFT)
+#define SCTP_SSN_INDEX_MASK (SCTP_SSN_PAGE_SIZE - 1)
+#define SCTP_SSN_MAX_PAGES ((SCTP_MAX_STREAMS + SCTP_SSN_PAGE_SIZE - 1) \
+ / SCTP_SSN_PAGE_SIZE)
+
+struct sctp_ssn_page {
+ __u16 ssn[SCTP_SSN_PAGE_SIZE];
+};
+
struct sctp_stream {
- __u16 *ssn;
unsigned int len;
+ struct sctp_ssn_page *pages[SCTP_SSN_MAX_PAGES];
};
struct sctp_ssnmap {
@@ -408,30 +426,42 @@ struct sctp_ssnmap {
struct sctp_stream out;
};
-struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
- gfp_t gfp);
+struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp);
void sctp_ssnmap_free(struct sctp_ssnmap *map);
void sctp_ssnmap_clear(struct sctp_ssnmap *map);
+/* Helper function to get a pointer to a particular SSN storage. */
+static inline __u16 *sctp_ssn_ptr(struct sctp_stream *stream, __u16 id)
+{
+ unsigned int page = id >> SCTP_SSN_PAGE_SHIFT;
+ unsigned int index = id & SCTP_SSN_INDEX_MASK;
+
+ return &stream->pages[page]->ssn[index];
+}
+
/* What is the current SSN number for this stream? */
static inline __u16 sctp_ssn_peek(struct sctp_stream *stream, __u16 id)
{
- return stream->ssn[id];
+ __u16 *ssnp = sctp_ssn_ptr(stream, id);
+ return *ssnp;
}
/* Return the next SSN number for this stream. */
static inline __u16 sctp_ssn_next(struct sctp_stream *stream, __u16 id)
{
- return stream->ssn[id]++;
+ __u16 *ssnp = sctp_ssn_ptr(stream, id);
+ return (*ssnp)++;
}
/* Skip over this ssn and all below. */
static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id,
__u16 ssn)
{
- stream->ssn[id] = ssn+1;
+ __u16 *ssnp = sctp_ssn_ptr(stream, id);
+ *ssnp = ssn + 1;
+
}
-
+
/*
* Pointers to address related SCTP functions.
* (i.e. things that depend on the address family.)
diff --git a/net/sctp/ssnmap.c b/net/sctp/ssnmap.c
index da86035..610ba6a 100644
--- a/net/sctp/ssnmap.c
+++ b/net/sctp/ssnmap.c
@@ -41,37 +41,100 @@
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>
-static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
- __u16 out);
+/* Allocate one memory page of SSN storage. */
+static inline int sctp_ssn_page_alloc(struct sctp_ssn_page **pagep, gfp_t gfp)
+{
+ BUILD_BUG_ON(sizeof(struct sctp_ssn_page) != PAGE_SIZE);
-/* Storage size needed for map includes 2 headers and then the
- * specific needs of in or out streams.
- */
-static inline size_t sctp_ssnmap_size(__u16 in, __u16 out)
+ *pagep = (struct sctp_ssn_page *)get_zeroed_page(gfp);
+ return *pagep != NULL;
+}
+
+/* Free one memory page of SSN storage. */
+static inline void sctp_ssn_page_free(struct sctp_ssn_page *page)
+{
+ if (page)
+ free_pages((unsigned long)page, 0);
+}
+
+/* Allocate memory for SSNs taking less than one memory page. */
+static inline int sctp_ssn_page_partial_alloc(struct sctp_ssn_page **pagep,
+ unsigned int len, gfp_t gfp)
+{
+ if (len == 0)
+ return 1;
+
+ *pagep = (struct sctp_ssn_page *)kzalloc(len * sizeof(__u16), gfp);
+ return *pagep != NULL;
+}
+
+/* Free memory that is smaller than one memory page. */
+static inline void sctp_ssn_page_partial_free(struct sctp_ssn_page *page)
{
- return sizeof(struct sctp_ssnmap) + (in + out) * sizeof(__u16);
+ if (page)
+ kfree(page);
+}
+
+/* Allocate memory pages for one type of stream (in or out). */
+static int sctp_stream_alloc(struct sctp_stream *stream, __u16 len, gfp_t gfp)
+{
+ unsigned int pages = len >> SCTP_SSN_PAGE_SHIFT;
+ unsigned int rest = len & SCTP_SSN_INDEX_MASK;
+ unsigned int pi;
+
+ stream->len = len;
+
+ for (pi = 0; pi < pages; ++pi)
+ if (!sctp_ssn_page_alloc(&stream->pages[pi], gfp))
+ return 0;
+
+ if (!sctp_ssn_page_partial_alloc(&stream->pages[pi], rest, gfp))
+ return 0;
+
+ return 1;
}
+/* Free memory pages for one type of stream (in or out). */
+static void sctp_stream_free(struct sctp_stream *stream)
+{
+ unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
+ unsigned int pi;
+
+ for (pi = 0; pi < pages; ++pi)
+ sctp_ssn_page_free(stream->pages[pi]);
+
+ sctp_ssn_page_partial_free(stream->pages[pi]);
+}
+
+/* Clear all SSNs for one type of stream (in or out). */
+static void sctp_stream_clear(struct sctp_stream *stream)
+{
+ unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
+ unsigned int rest = stream->len & SCTP_SSN_INDEX_MASK;
+ unsigned int pi;
+
+ for (pi = 0; pi < pages; ++pi)
+ memset(stream->pages[pi], 0, sizeof(struct sctp_ssn_page));
+
+ if (rest)
+ memset(stream->pages[pi], 0, rest * sizeof(__u16));
+}
/* Create a new sctp_ssnmap.
- * Allocate room to store at least 'len' contiguous TSNs.
+ * Allocate room to store at least 'in' + 'out' SSNs.
*/
-struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
- gfp_t gfp)
+struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp)
{
struct sctp_ssnmap *retval;
- int size;
-
- size = sctp_ssnmap_size(in, out);
- if (size <= KMALLOC_MAX_SIZE)
- retval = kmalloc(size, gfp);
- else
- retval = (struct sctp_ssnmap *)
- __get_free_pages(gfp, get_order(size));
+
+ retval = (struct sctp_ssnmap *)kzalloc(sizeof(struct sctp_ssnmap), gfp);
if (!retval)
goto fail;
- if (!sctp_ssnmap_init(retval, in, out))
+ if (!sctp_stream_alloc(&retval->in, in, gfp))
+ goto fail_map;
+
+ if (!sctp_stream_alloc(&retval->out, out, gfp))
goto fail_map;
SCTP_DBG_OBJCNT_INC(ssnmap);
@@ -79,54 +142,29 @@ struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
return retval;
fail_map:
- if (size <= KMALLOC_MAX_SIZE)
- kfree(retval);
- else
- free_pages((unsigned long)retval, get_order(size));
+ sctp_stream_free(&retval->in);
+ sctp_stream_free(&retval->out);
+ kfree(retval);
fail:
return NULL;
}
-
-/* Initialize a block of memory as a ssnmap. */
-static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
- __u16 out)
-{
- memset(map, 0x00, sctp_ssnmap_size(in, out));
-
- /* Start 'in' stream just after the map header. */
- map->in.ssn = (__u16 *)&map[1];
- map->in.len = in;
-
- /* Start 'out' stream just after 'in'. */
- map->out.ssn = &map->in.ssn[in];
- map->out.len = out;
-
- return map;
-}
-
/* Clear out the ssnmap streams. */
void sctp_ssnmap_clear(struct sctp_ssnmap *map)
{
- size_t size;
-
- size = (map->in.len + map->out.len) * sizeof(__u16);
- memset(map->in.ssn, 0x00, size);
+ sctp_stream_clear(&map->in);
+ sctp_stream_clear(&map->out);
}
/* Dispose of a ssnmap. */
void sctp_ssnmap_free(struct sctp_ssnmap *map)
{
- int size;
-
if (unlikely(!map))
return;
- size = sctp_ssnmap_size(map->in.len, map->out.len);
- if (size <= KMALLOC_MAX_SIZE)
- kfree(map);
- else
- free_pages((unsigned long)map, get_order(size));
+ sctp_stream_free(&map->in);
+ sctp_stream_free(&map->out);
+ kfree(map);
SCTP_DBG_OBJCNT_DEC(ssnmap);
}
--
1.8.3.1
More information about the Devel
mailing list