RepoMirrors/haproxy
1
0
Fork 0
mirror of http://git.haproxy.org/git/haproxy.git/ synced 2024-12-17 00:44:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

MEDIUM: memory: make local pools independent on lockless pools

Browse Source 
Till now the local pool caches were implemented only when lockless pools
were in use. This was mainly due to the difficulties to disentangle the
code parts. However the locked pools would further benefit from the local
cache, and having this would reduce the variants in the code.

This patch does just this. It adds a new debug macro DEBUG_NO_LOCAL_POOLS
to forcefully disable local pool caches, and makes sure that the high
level functions are now strictly the same between locked and lockless
(pool_alloc(), pool_alloc_dirty(), pool_free(), pool_get_first()). The
pool index calculation was moved inside the CONFIG_HAP_LOCAL_POOLS guards.
This allowed to move them out of the giant #ifdef and to significantly
reduce the code duplication.

A quick perf test shows that with locked pools the performance increases
by roughly 10% on 8 threads and gets closer to the lockless one.
This commit is contained in:
Willy Tarreau 2020-06-01 19:00:28 +02:00
parent f8c1b648c0
commit ed891fda52
4 changed files with 163 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/common/buffer.h
View File

@ -145,16 +145,18 @@ static inline void b_free(struct buffer *buf)
static inline struct buffer *b_alloc_margin(struct buffer *buf, int margin)
{
char *area;
ssize_t idx;
ssize_t idx __maybe_unused;
unsigned int cached;
if (buf->size)
return buf;
cached = 0;
#ifdef CONFIG_HAP_LOCAL_POOLS
idx = pool_get_index(pool_head_buffer);
if (idx >= 0)
cached = pool_cache[tid][idx].count;
#endif
*buf = BUF_WANTED;

268
include/common/memory.h
View File

@ -31,12 +31,14 @@
#include <haproxy/pool-t.h>
#include <haproxy/thread.h>
#ifdef CONFIG_HAP_LOCAL_POOLS
extern struct pool_head pool_base_start[MAX_BASE_POOLS];
extern unsigned int pool_base_count;
extern struct pool_cache_head pool_cache[][MAX_BASE_POOLS];
extern struct list pool_lru_head[MAX_THREADS];
extern THREAD_LOCAL size_t pool_cache_bytes; /* total cache size */
extern THREAD_LOCAL size_t pool_cache_count; /* #cache objects */
extern struct pool_head pool_base_start[MAX_BASE_POOLS];
extern unsigned int pool_base_count;
#endif
/* poison each newly allocated area with this byte if >= 0 */
extern int mem_poison_byte;
@ -106,12 +108,14 @@ void pool_destroy_all();
/* returns the pool index for pool <pool>, or -1 if this pool has no index */
static inline ssize_t pool_get_index(const struct pool_head *pool)
{
#ifdef CONFIG_HAP_LOCAL_POOLS
size_t idx;
idx = pool - pool_base_start;
if (idx >= MAX_BASE_POOLS)
return -1;
return idx;
if (idx < MAX_BASE_POOLS)
return idx;
#endif
return -1;
}
/* returns true if the pool is considered to have too many free objects */
@ -121,7 +125,8 @@ static inline int pool_is_crowded(const struct pool_head *pool)
(int)(pool->allocated - pool->used) >= pool->minavail;
}
#ifdef CONFIG_HAP_LOCKLESS_POOLS
#ifdef CONFIG_HAP_LOCAL_POOLS
void pool_evict_from_cache();
/* Tries to retrieve an object from the local pool cache corresponding to pool
* <pool>. Returns NULL if none is available.
@ -153,6 +158,26 @@ static inline void *__pool_get_from_cache(struct pool_head *pool)
return item;
}
/* Frees an object to the local cache, possibly pushing oldest objects to the
* global pool.
*/
static inline void pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
{
struct pool_cache_item *item = (struct pool_cache_item *)ptr;
struct pool_cache_head *ph = &pool_cache[tid][idx];
LIST_ADD(&ph->list, &item->by_pool);
LIST_ADD(&pool_lru_head[tid], &item->by_lru);
ph->count++;
pool_cache_count++;
pool_cache_bytes += ph->size;
if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE))
pool_evict_from_cache(pool, ptr, idx);
}
#endif // CONFIG_HAP_LOCAL_POOLS
#ifdef CONFIG_HAP_LOCKLESS_POOLS
/*
* Returns a pointer to type <type> taken from the pool <pool_type> if
* available, otherwise returns NULL. No malloc() is attempted, and poisonning
@ -183,51 +208,6 @@ static inline void *__pool_get_first(struct pool_head *pool)
return cmp.free_list;
}
static inline void *pool_get_first(struct pool_head *pool)
{
void *ret;
if (likely(ret = __pool_get_from_cache(pool)))
return ret;
ret = __pool_get_first(pool);
return ret;
}
/*
* Returns a pointer to type <type> taken from the pool <pool_type> or
* dynamically allocated. In the first case, <pool_type> is updated to point to
* the next element in the list. No memory poisonning is ever performed on the
* returned area.
*/
static inline void *pool_alloc_dirty(struct pool_head *pool)
{
void *p;
if (likely(p = __pool_get_from_cache(pool)))
return p;
if ((p = __pool_get_first(pool)) == NULL)
p = __pool_refill_alloc(pool, 0);
return p;
}
/*
* Returns a pointer to type <type> taken from the pool <pool_type> or
* dynamically allocated. In the first case, <pool_type> is updated to point to
* the next element in the list. Memory poisonning is performed if enabled.
*/
static inline void *pool_alloc(struct pool_head *pool)
{
void *p;
p = pool_alloc_dirty(pool);
if (p && mem_poison_byte >= 0) {
memset(p, mem_poison_byte, pool->size);
}
return p;
}
/* Locklessly add item <ptr> to pool <pool>, then update the pool used count.
* Both the pool and the pointer must be valid. Use pool_free() for normal
* operations.
@ -251,63 +231,6 @@ static inline void __pool_free(struct pool_head *pool, void *ptr)
swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
}
void pool_evict_from_cache();
/* Frees an object to the local cache, possibly pushing oldest objects to the
* global pool.
*/
static inline void pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx)
{
struct pool_cache_item *item = (struct pool_cache_item *)ptr;
struct pool_cache_head *ph = &pool_cache[tid][idx];
LIST_ADD(&ph->list, &item->by_pool);
LIST_ADD(&pool_lru_head[tid], &item->by_lru);
ph->count++;
pool_cache_count++;
pool_cache_bytes += ph->size;
if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE))
pool_evict_from_cache(pool, ptr, idx);
}
/*
* Puts a memory area back to the corresponding pool.
* Items are chained directly through a pointer that
* is written in the beginning of the memory area, so
* there's no need for any carrier cell. This implies
* that each memory area is at least as big as one
* pointer. Just like with the libc's free(), nothing
* is done if <ptr> is NULL.
*/
static inline void pool_free(struct pool_head *pool, void *ptr)
{
if (likely(ptr != NULL)) {
ssize_t idx __maybe_unused;
#ifdef DEBUG_MEMORY_POOLS
/* we'll get late corruption if we refill to the wrong pool or double-free */
if (*POOL_LINK(pool, ptr) != (void *)pool)
*DISGUISE((volatile int *)0) = 0;
#endif
if (mem_poison_byte >= 0)
memset(ptr, mem_poison_byte, pool->size);
/* put the object back into the cache only if there are not too
* many objects yet in this pool (no more than half of the cached
* is used or this pool uses no more than 1/8 of the cache size).
*/
idx = pool_get_index(pool);
if (idx >= 0 &&
(pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4 ||
pool_cache[tid][idx].count < 16 + pool_cache_count / 8)) {
pool_put_to_cache(pool, ptr, idx);
return;
}
__pool_free(pool, ptr);
}
}
#else /* CONFIG_HAP_LOCKLESS_POOLS */
/*
* Returns a pointer to type <type> taken from the pool <pool_type> if
@ -329,49 +252,6 @@ static inline void *__pool_get_first(struct pool_head *pool)
return p;
}
static inline void *pool_get_first(struct pool_head *pool)
{
void *ret;
HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
ret = __pool_get_first(pool);
HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
return ret;
}
/*
* Returns a pointer to type <type> taken from the pool <pool_type> or
* dynamically allocated. In the first case, <pool_type> is updated to point to
* the next element in the list. No memory poisonning is ever performed on the
* returned area.
*/
static inline void *pool_alloc_dirty(struct pool_head *pool)
{
void *p;
HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
if ((p = __pool_get_first(pool)) == NULL)
p = __pool_refill_alloc(pool, 0);
HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
return p;
}
/*
* Returns a pointer to type <type> taken from the pool <pool_type> or
* dynamically allocated. In the first case, <pool_type> is updated to point to
* the next element in the list. Memory poisonning is performed if enabled.
*/
static inline void *pool_alloc(struct pool_head *pool)
{
void *p;
p = pool_alloc_dirty(pool);
if (p && mem_poison_byte >= 0) {
memset(p, mem_poison_byte, pool->size);
}
return p;
}
/* unconditionally stores the object as-is into the global pool. The object
* must not be NULL. Use pool_free() instead.
*/
@ -401,6 +281,70 @@ static inline void __pool_free(struct pool_head *pool, void *ptr)
#endif /* DEBUG_UAF */
}
#endif /* CONFIG_HAP_LOCKLESS_POOLS */
static inline void *pool_get_first(struct pool_head *pool)
{
void *p;
#ifdef CONFIG_HAP_LOCAL_POOLS
if (likely(p = __pool_get_from_cache(pool)))
return p;
#endif
#ifndef CONFIG_HAP_LOCKLESS_POOLS
HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
#endif
p = __pool_get_first(pool);
#ifndef CONFIG_HAP_LOCKLESS_POOLS
HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
#endif
return p;
}
/*
* Returns a pointer to type <type> taken from the pool <pool_type> or
* dynamically allocated. In the first case, <pool_type> is updated to point to
* the next element in the list. No memory poisonning is ever performed on the
* returned area.
*/
static inline void *pool_alloc_dirty(struct pool_head *pool)
{
void *p;
#ifdef CONFIG_HAP_LOCAL_POOLS
if (likely(p = __pool_get_from_cache(pool)))
return p;
#endif
#ifndef CONFIG_HAP_LOCKLESS_POOLS
HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
#endif
if ((p = __pool_get_first(pool)) == NULL)
p = __pool_refill_alloc(pool, 0);
#ifndef CONFIG_HAP_LOCKLESS_POOLS
HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
#endif
return p;
}
/*
* Returns a pointer to type <type> taken from the pool <pool_type> or
* dynamically allocated. In the first case, <pool_type> is updated to point to
* the next element in the list. Memory poisonning is performed if enabled.
*/
static inline void *pool_alloc(struct pool_head *pool)
{
void *p;
p = pool_alloc_dirty(pool);
if (p && mem_poison_byte >= 0) {
memset(p, mem_poison_byte, pool->size);
}
return p;
}
/*
* Puts a memory area back to the corresponding pool.
* Items are chained directly through a pointer that
@ -413,18 +357,34 @@ static inline void __pool_free(struct pool_head *pool, void *ptr)
static inline void pool_free(struct pool_head *pool, void *ptr)
{
if (likely(ptr != NULL)) {
ssize_t idx __maybe_unused;
#ifdef DEBUG_MEMORY_POOLS
/* we'll get late corruption if we refill to the wrong pool or double-free */
if (*POOL_LINK(pool, ptr) != (void *)pool)
*DISGUISE((volatile int *)0) = 0;
ABORT_NOW();
#endif
if (mem_poison_byte >= 0)
if (unlikely(mem_poison_byte >= 0))
memset(ptr, mem_poison_byte, pool->size);
#ifdef CONFIG_HAP_LOCAL_POOLS
/* put the object back into the cache only if there are not too
* many objects yet in this pool (no more than half of the cached
* is used or this pool uses no more than 1/8 of the cache size).
*/
idx = pool_get_index(pool);
if (idx >= 0 &&
(pool_cache_bytes <= CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4 ||
pool_cache[tid][idx].count < 16 + pool_cache_count / 8)) {
pool_put_to_cache(pool, ptr, idx);
return;
}
#endif
__pool_free(pool, ptr);
}
}
#endif /* CONFIG_HAP_LOCKLESS_POOLS */
#endif /* _COMMON_MEMORY_H */
/*

7
include/haproxy/pool-t.h
View File

@ -33,6 +33,13 @@
#define CONFIG_HAP_LOCKLESS_POOLS
#endif
/* On architectures supporting threads we can amortize the locking cost using
* local pools.
*/
#if defined(USE_THREAD) && !defined(DEBUG_NO_LOCAL_POOLS) && !defined(DEBUG_UAF) && !defined(DEBUG_FAIL_ALLOC)
#define CONFIG_HAP_LOCAL_POOLS
#endif
/* Pools of very similar size are shared by default, unless macro
* DEBUG_DONT_SHARE_POOLS is set.
*/

63
src/memory.c
View File

@ -32,6 +32,7 @@
#include <proto/stream_interface.h>
#include <proto/stats.h>
#ifdef CONFIG_HAP_LOCAL_POOLS
/* These are the most common pools, expected to be initialized first. These
* ones are allocated from an array, allowing to map them to an index.
*/
@ -43,6 +44,7 @@ struct pool_cache_head pool_cache[MAX_THREADS][MAX_BASE_POOLS];
struct list pool_lru_head[MAX_THREADS]; /* oldest objects */
THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
#endif
static struct list pools = LIST_HEAD_INIT(pools);
int mem_poison_byte = -1;
@ -64,7 +66,7 @@ struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
struct pool_head *entry;
struct list *start;
unsigned int align;
int thr, idx;
int idx __maybe_unused;
/* We need to store a (void *) at the end of the chunks. Since we know
* that the malloc() function will never return such a small size,
@ -107,6 +109,7 @@ struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
}
if (!pool) {
#ifdef CONFIG_HAP_LOCAL_POOLS
if (pool_base_count < MAX_BASE_POOLS)
pool = &pool_base_start[pool_base_count++];
@ -119,6 +122,7 @@ struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
}
}
}
#endif
if (!pool)
pool = calloc(1, sizeof(*pool));
@ -131,18 +135,47 @@ struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
pool->flags = flags;
LIST_ADDQ(start, &pool->list);
#ifdef CONFIG_HAP_LOCAL_POOLS
/* update per-thread pool cache if necessary */
idx = pool_get_index(pool);
if (idx >= 0) {
int thr;
for (thr = 0; thr < MAX_THREADS; thr++)
pool_cache[thr][idx].size = size;
}
#endif
HA_SPIN_INIT(&pool->lock);
}
pool->users++;
return pool;
}
#ifdef CONFIG_HAP_LOCAL_POOLS
/* Evicts some of the oldest objects from the local cache, pushing them to the
* global pool.
*/
void pool_evict_from_cache()
{
struct pool_cache_item *item;
struct pool_cache_head *ph;
do {
item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
/* note: by definition we remove oldest objects so they also are the
* oldest in their own pools, thus their next is the pool's head.
*/
ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
LIST_DEL(&item->by_pool);
LIST_DEL(&item->by_lru);
ph->count--;
pool_cache_count--;
pool_cache_bytes -= ph->size;
__pool_free(pool_base_start + (ph - pool_cache[tid]), item);
} while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
}
#endif
#ifdef CONFIG_HAP_LOCKLESS_POOLS
/* Allocates new entries for pool <pool> until there are at least <avail> + 1
* available, then returns the last one for immediate use, so that at least
@ -278,29 +311,6 @@ void pool_gc(struct pool_head *pool_ctx)
thread_release();
}
/* Evicts some of the oldest objects from the local cache, pushing them to the
* global pool.
*/
void pool_evict_from_cache()
{
struct pool_cache_item *item;
struct pool_cache_head *ph;
do {
item = LIST_PREV(&pool_lru_head[tid], struct pool_cache_item *, by_lru);
/* note: by definition we remove oldest objects so they also are the
* oldest in their own pools, thus their next is the pool's head.
*/
ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
LIST_DEL(&item->by_pool);
LIST_DEL(&item->by_lru);
ph->count--;
pool_cache_count--;
pool_cache_bytes -= ph->size;
__pool_free(pool_base_start + (ph - pool_cache[tid]), item);
} while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
}
#else /* CONFIG_HAP_LOCKLESS_POOLS */
/* Allocates new entries for pool <pool> until there are at least <avail> + 1
@ -443,9 +453,12 @@ void *pool_destroy(struct pool_head *pool)
#ifndef CONFIG_HAP_LOCKLESS_POOLS
HA_SPIN_DESTROY(&pool->lock);
#endif
#ifdef CONFIG_HAP_LOCAL_POOLS
if ((pool - pool_base_start) < MAX_BASE_POOLS)
memset(pool, 0, sizeof(*pool));
else
#endif
free(pool);
}
}
@ -565,6 +578,7 @@ void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
/* Initializes all per-thread arrays on startup */
static void init_pools()
{
#ifdef CONFIG_HAP_LOCAL_POOLS
int thr, idx;
for (thr = 0; thr < MAX_THREADS; thr++) {
@ -574,6 +588,7 @@ static void init_pools()
}
LIST_INIT(&pool_lru_head[thr]);
}
#endif
}
INITCALL0(STG_PREPARE, init_pools);