[Devel] [PATCH vz7] net/sctp: Allocate SSN map on a per-page basis
Denis V. Lunev
den at virtuozzo.com
Tue Mar 27 21:04:14 MSK 2018
we can allocate pages on demand right now,
i.e. when it will be really necessary.
Can be done in the next patch of the set.
Den
On 03/27/2018 08:55 PM, Oleg Babin wrote:
> SCTP protocol allocates TCB on receiving INIT and COOKIE ECHO chunks
> which specify input and output stream count. As the total count of
> those streams can be up to (2^16 - 1) of each type, it is possible
> to occupy a fifth order of memory for only one type of streams. Plus
> allocation can happen in a softirq contex with GFP_ATOMIC flag set
> and we actually do not need the memory to be physically contiguous,
> so for performance reasons we allocate memory on a per-page basis.
>
> https://jira.sw.ru/browse/PSBM-82552
>
> Signed-off-by: Oleg Babin <obabin at virtuozzo.com>
> ---
> include/net/sctp/structs.h | 44 +++++++++++---
> net/sctp/ssnmap.c | 142 ++++++++++++++++++++++++++++-----------------
> 2 files changed, 127 insertions(+), 59 deletions(-)
>
> diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
> index c2f7064..0a09dbe 100644
> --- a/include/net/sctp/structs.h
> +++ b/include/net/sctp/structs.h
> @@ -396,11 +396,29 @@ typedef struct sctp_sender_hb_info {
> *
> * This is the structure we use to track both our outbound and inbound
> * SSN, or Stream Sequence Numbers.
> + *
> + * As the total count of input or output streams can be up to (2^16 - 1)
> + * of each type, it is possible to occupy a fifth order of memory for
> + * only one type of streams. Plus allocation can happen in a softirq
> + * contex with GFP_ATOMIC flag set and we actually do not need the memory
> + * to be physically contiguous, so for performance reasons we allocate
> + * memory on a per-page basis.
> */
>
> +#define SCTP_MAX_STREAMS 65535
> +#define SCTP_SSN_PAGE_SHIFT (PAGE_SHIFT - 1) /* One SSN takes 2 bytes */
> +#define SCTP_SSN_PAGE_SIZE (1u << SCTP_SSN_PAGE_SHIFT)
> +#define SCTP_SSN_INDEX_MASK (SCTP_SSN_PAGE_SIZE - 1)
> +#define SCTP_SSN_MAX_PAGES ((SCTP_MAX_STREAMS + SCTP_SSN_PAGE_SIZE - 1) \
> + / SCTP_SSN_PAGE_SIZE)
> +
> +struct sctp_ssn_page {
> + __u16 ssn[SCTP_SSN_PAGE_SIZE];
> +};
> +
> struct sctp_stream {
> - __u16 *ssn;
> unsigned int len;
> + struct sctp_ssn_page *pages[SCTP_SSN_MAX_PAGES];
> };
>
> struct sctp_ssnmap {
> @@ -408,30 +426,42 @@ struct sctp_ssnmap {
> struct sctp_stream out;
> };
>
> -struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> - gfp_t gfp);
> +struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp);
> void sctp_ssnmap_free(struct sctp_ssnmap *map);
> void sctp_ssnmap_clear(struct sctp_ssnmap *map);
>
> +/* Helper function to get a pointer to a particular SSN storage. */
> +static inline __u16 *sctp_ssn_ptr(struct sctp_stream *stream, __u16 id)
> +{
> + unsigned int page = id >> SCTP_SSN_PAGE_SHIFT;
> + unsigned int index = id & SCTP_SSN_INDEX_MASK;
> +
> + return &stream->pages[page]->ssn[index];
> +}
> +
> /* What is the current SSN number for this stream? */
> static inline __u16 sctp_ssn_peek(struct sctp_stream *stream, __u16 id)
> {
> - return stream->ssn[id];
> + __u16 *ssnp = sctp_ssn_ptr(stream, id);
> + return *ssnp;
> }
>
> /* Return the next SSN number for this stream. */
> static inline __u16 sctp_ssn_next(struct sctp_stream *stream, __u16 id)
> {
> - return stream->ssn[id]++;
> + __u16 *ssnp = sctp_ssn_ptr(stream, id);
> + return (*ssnp)++;
> }
>
> /* Skip over this ssn and all below. */
> static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id,
> __u16 ssn)
> {
> - stream->ssn[id] = ssn+1;
> + __u16 *ssnp = sctp_ssn_ptr(stream, id);
> + *ssnp = ssn + 1;
> +
> }
> -
> +
> /*
> * Pointers to address related SCTP functions.
> * (i.e. things that depend on the address family.)
> diff --git a/net/sctp/ssnmap.c b/net/sctp/ssnmap.c
> index da86035..610ba6a 100644
> --- a/net/sctp/ssnmap.c
> +++ b/net/sctp/ssnmap.c
> @@ -41,37 +41,100 @@
> #include <net/sctp/sctp.h>
> #include <net/sctp/sm.h>
>
> -static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
> - __u16 out);
> +/* Allocate one memory page of SSN storage. */
> +static inline int sctp_ssn_page_alloc(struct sctp_ssn_page **pagep, gfp_t gfp)
> +{
> + BUILD_BUG_ON(sizeof(struct sctp_ssn_page) != PAGE_SIZE);
>
> -/* Storage size needed for map includes 2 headers and then the
> - * specific needs of in or out streams.
> - */
> -static inline size_t sctp_ssnmap_size(__u16 in, __u16 out)
> + *pagep = (struct sctp_ssn_page *)get_zeroed_page(gfp);
> + return *pagep != NULL;
> +}
> +
> +/* Free one memory page of SSN storage. */
> +static inline void sctp_ssn_page_free(struct sctp_ssn_page *page)
> +{
> + if (page)
> + free_pages((unsigned long)page, 0);
> +}
> +
> +/* Allocate memory for SSNs taking less than one memory page. */
> +static inline int sctp_ssn_page_partial_alloc(struct sctp_ssn_page **pagep,
> + unsigned int len, gfp_t gfp)
> +{
> + if (len == 0)
> + return 1;
> +
> + *pagep = (struct sctp_ssn_page *)kzalloc(len * sizeof(__u16), gfp);
> + return *pagep != NULL;
> +}
> +
> +/* Free memory that is smaller than one memory page. */
> +static inline void sctp_ssn_page_partial_free(struct sctp_ssn_page *page)
> {
> - return sizeof(struct sctp_ssnmap) + (in + out) * sizeof(__u16);
> + if (page)
> + kfree(page);
> +}
> +
> +/* Allocate memory pages for one type of stream (in or out). */
> +static int sctp_stream_alloc(struct sctp_stream *stream, __u16 len, gfp_t gfp)
> +{
> + unsigned int pages = len >> SCTP_SSN_PAGE_SHIFT;
> + unsigned int rest = len & SCTP_SSN_INDEX_MASK;
> + unsigned int pi;
> +
> + stream->len = len;
> +
> + for (pi = 0; pi < pages; ++pi)
> + if (!sctp_ssn_page_alloc(&stream->pages[pi], gfp))
> + return 0;
> +
> + if (!sctp_ssn_page_partial_alloc(&stream->pages[pi], rest, gfp))
> + return 0;
> +
> + return 1;
> }
>
> +/* Free memory pages for one type of stream (in or out). */
> +static void sctp_stream_free(struct sctp_stream *stream)
> +{
> + unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
> + unsigned int pi;
> +
> + for (pi = 0; pi < pages; ++pi)
> + sctp_ssn_page_free(stream->pages[pi]);
> +
> + sctp_ssn_page_partial_free(stream->pages[pi]);
> +}
> +
> +/* Clear all SSNs for one type of stream (in or out). */
> +static void sctp_stream_clear(struct sctp_stream *stream)
> +{
> + unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
> + unsigned int rest = stream->len & SCTP_SSN_INDEX_MASK;
> + unsigned int pi;
> +
> + for (pi = 0; pi < pages; ++pi)
> + memset(stream->pages[pi], 0, sizeof(struct sctp_ssn_page));
> +
> + if (rest)
> + memset(stream->pages[pi], 0, rest * sizeof(__u16));
> +}
>
> /* Create a new sctp_ssnmap.
> - * Allocate room to store at least 'len' contiguous TSNs.
> + * Allocate room to store at least 'in' + 'out' SSNs.
> */
> -struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> - gfp_t gfp)
> +struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp)
> {
> struct sctp_ssnmap *retval;
> - int size;
> -
> - size = sctp_ssnmap_size(in, out);
> - if (size <= KMALLOC_MAX_SIZE)
> - retval = kmalloc(size, gfp);
> - else
> - retval = (struct sctp_ssnmap *)
> - __get_free_pages(gfp, get_order(size));
> +
> + retval = (struct sctp_ssnmap *)kzalloc(sizeof(struct sctp_ssnmap), gfp);
> if (!retval)
> goto fail;
>
> - if (!sctp_ssnmap_init(retval, in, out))
> + if (!sctp_stream_alloc(&retval->in, in, gfp))
> + goto fail_map;
> +
> + if (!sctp_stream_alloc(&retval->out, out, gfp))
> goto fail_map;
>
> SCTP_DBG_OBJCNT_INC(ssnmap);
> @@ -79,54 +142,29 @@ struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> return retval;
>
> fail_map:
> - if (size <= KMALLOC_MAX_SIZE)
> - kfree(retval);
> - else
> - free_pages((unsigned long)retval, get_order(size));
> + sctp_stream_free(&retval->in);
> + sctp_stream_free(&retval->out);
> + kfree(retval);
> fail:
> return NULL;
> }
>
> -
> -/* Initialize a block of memory as a ssnmap. */
> -static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
> - __u16 out)
> -{
> - memset(map, 0x00, sctp_ssnmap_size(in, out));
> -
> - /* Start 'in' stream just after the map header. */
> - map->in.ssn = (__u16 *)&map[1];
> - map->in.len = in;
> -
> - /* Start 'out' stream just after 'in'. */
> - map->out.ssn = &map->in.ssn[in];
> - map->out.len = out;
> -
> - return map;
> -}
> -
> /* Clear out the ssnmap streams. */
> void sctp_ssnmap_clear(struct sctp_ssnmap *map)
> {
> - size_t size;
> -
> - size = (map->in.len + map->out.len) * sizeof(__u16);
> - memset(map->in.ssn, 0x00, size);
> + sctp_stream_clear(&map->in);
> + sctp_stream_clear(&map->out);
> }
>
> /* Dispose of a ssnmap. */
> void sctp_ssnmap_free(struct sctp_ssnmap *map)
> {
> - int size;
> -
> if (unlikely(!map))
> return;
>
> - size = sctp_ssnmap_size(map->in.len, map->out.len);
> - if (size <= KMALLOC_MAX_SIZE)
> - kfree(map);
> - else
> - free_pages((unsigned long)map, get_order(size));
> + sctp_stream_free(&map->in);
> + sctp_stream_free(&map->out);
> + kfree(map);
>
> SCTP_DBG_OBJCNT_DEC(ssnmap);
> }
More information about the Devel
mailing list