diff --git a/include/haproxy/pool-t.h b/include/haproxy/pool-t.h index bbdc5365a..d49982165 100644 --- a/include/haproxy/pool-t.h +++ b/include/haproxy/pool-t.h @@ -36,7 +36,7 @@ /* On architectures supporting threads and double-word CAS, we can implement * lock-less memory pools. This isn't supported for debugging modes however. */ -#if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && !defined(DEBUG_NO_LOCKLESS_POOLS) && !defined(DEBUG_UAF) && !defined(DEBUG_FAIL_ALLOC) +#if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && defined(CONFIG_HAP_POOLS) && !defined(DEBUG_NO_LOCKLESS_POOLS) #define CONFIG_HAP_LOCKLESS_POOLS #endif diff --git a/include/haproxy/pool.h b/include/haproxy/pool.h index c1faa2793..df0199196 100644 --- a/include/haproxy/pool.h +++ b/include/haproxy/pool.h @@ -65,14 +65,6 @@ void *pool_destroy(struct pool_head *pool); void pool_destroy_all(); -/* returns true if the pool is considered to have too many free objects */ -static inline int pool_is_crowded(const struct pool_head *pool) -{ - return pool->allocated >= swrate_avg(pool->needed_avg + pool->needed_avg / 4, POOL_AVG_SAMPLES) && - (int)(pool->allocated - pool->used) >= pool->minavail; -} - - #ifdef CONFIG_HAP_POOLS /****************** Thread-local cache management ******************/ @@ -82,6 +74,13 @@ extern THREAD_LOCAL size_t pool_cache_count; /* #cache objects */ void pool_evict_from_local_caches(); +/* returns true if the pool is considered to have too many free objects */ +static inline int pool_is_crowded(const struct pool_head *pool) +{ + return pool->allocated >= swrate_avg(pool->needed_avg + pool->needed_avg / 4, POOL_AVG_SAMPLES) && + (int)(pool->allocated - pool->used) >= pool->minavail; +} + /* Tries to retrieve an object from the local pool cache corresponding to pool * . Returns NULL if none is available. */ @@ -125,8 +124,6 @@ static inline void pool_put_to_local_cache(struct pool_head *pool, void *ptr) pool_evict_from_local_caches(); } -#endif // CONFIG_HAP_POOLS - #if defined(CONFIG_HAP_NO_GLOBAL_POOLS) @@ -263,6 +260,8 @@ static inline void pool_put_to_shared_cache(struct pool_head *pool, void *ptr) #endif /* CONFIG_HAP_LOCKLESS_POOLS */ +#endif /* CONFIG_HAP_POOLS */ + /****************** Common high-level code ******************/ @@ -274,7 +273,7 @@ static inline void pool_put_to_shared_cache(struct pool_head *pool, void *ptr) */ static inline void *__pool_alloc(struct pool_head *pool, unsigned int flags) { - void *p; + void *p = NULL; #ifdef DEBUG_FAIL_ALLOC if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool)) @@ -282,14 +281,15 @@ static inline void *__pool_alloc(struct pool_head *pool, unsigned int flags) #endif #ifdef CONFIG_HAP_POOLS - if (likely(p = pool_get_from_local_cache(pool))) - goto ret; -#endif + if (!p) + p = pool_get_from_local_cache(pool); - p = pool_get_from_shared_cache(pool); + if (!p) + p = pool_get_from_shared_cache(pool); +#endif if (!p) p = pool_alloc_nocache(pool); - ret: + if (p) { if (flags & POOL_F_MUST_ZERO) memset(p, 0, pool->size); @@ -347,8 +347,10 @@ static inline void pool_free(struct pool_head *pool, void *ptr) pool_put_to_local_cache(pool, ptr); return; } -#endif pool_put_to_shared_cache(pool, ptr); +#else + pool_free_nocache(pool, ptr); +#endif } } diff --git a/src/pool.c b/src/pool.c index 0956976ee..487e0da28 100644 --- a/src/pool.c +++ b/src/pool.c @@ -148,33 +148,6 @@ void pool_put_to_os(struct pool_head *pool, void *ptr) _HA_ATOMIC_DEC(&pool->allocated); } -#ifdef CONFIG_HAP_POOLS -/* Evicts some of the oldest objects from the local cache, pushing them to the - * global pool. - */ -void pool_evict_from_local_caches() -{ - struct pool_cache_item *item; - struct pool_cache_head *ph; - struct pool_head *pool; - - do { - item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru); - /* note: by definition we remove oldest objects so they also are the - * oldest in their own pools, thus their next is the pool's head. - */ - ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list); - pool = container_of(ph - tid, struct pool_head, cache); - LIST_DEL(&item->by_pool); - LIST_DEL(&item->by_lru); - ph->count--; - pool_cache_count--; - pool_cache_bytes -= pool->size; - pool_put_to_shared_cache(pool, item); - } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8); -} -#endif - /* Tries to allocate an object for the pool using the system's allocator * and directly returns it. The pool's counters are updated but the object is * never cached, so this is usable with and without local or shared caches. @@ -211,6 +184,33 @@ void pool_free_nocache(struct pool_head *pool, void *ptr) } +#ifdef CONFIG_HAP_POOLS + +/* Evicts some of the oldest objects from the local cache, pushing them to the + * global pool. + */ +void pool_evict_from_local_caches() +{ + struct pool_cache_item *item; + struct pool_cache_head *ph; + struct pool_head *pool; + + do { + item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru); + /* note: by definition we remove oldest objects so they also are the + * oldest in their own pools, thus their next is the pool's head. + */ + ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list); + pool = container_of(ph - tid, struct pool_head, cache); + LIST_DEL(&item->by_pool); + LIST_DEL(&item->by_lru); + ph->count--; + pool_cache_count--; + pool_cache_bytes -= pool->size; + pool_put_to_shared_cache(pool, item); + } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8); +} + #if defined(CONFIG_HAP_NO_GLOBAL_POOLS) /* legacy stuff */ @@ -349,7 +349,24 @@ void pool_gc(struct pool_head *pool_ctx) if (!isolated) thread_release(); } +#endif /* CONFIG_HAP_LOCKLESS_POOLS */ + +#else /* CONFIG_HAP_POOLS */ + +/* legacy stuff */ +void pool_flush(struct pool_head *pool) +{ +} + +/* This function might ask the malloc library to trim its buffers. */ +void pool_gc(struct pool_head *pool_ctx) +{ +#if defined(HA_HAVE_MALLOC_TRIM) + malloc_trim(0); #endif +} + +#endif /* CONFIG_HAP_POOLS */ /* * This function destroys a pool by freeing it completely, unless it's still