mirror of https://github.com/schoebel/mars
infra: better optimization of memory allocations
This commit is contained in:
Thomas Schoebel-Theuer
parent
1202f2ae8e
commit
20166b6c01
|
|
@ -6,6 +6,8 @@ ifneq ($(KBUILD_EXTMOD),)
|
|||
CONFIG_MARS_BIGMODULE := m
|
||||
endif
|
||||
|
||||
KBUILD_CFLAGS += -fdelete-null-pointer-checks
|
||||
|
||||
mars-objs := \
|
||||
brick_say.o \
|
||||
brick_mem.o \
|
||||
|
|
|
|||
|
|
@ -155,46 +155,44 @@ EXPORT_SYMBOL_GPL(_brick_mem_alloc);
|
|||
|
||||
void _brick_mem_free(void *data, int cline)
|
||||
{
|
||||
if (data) {
|
||||
#ifdef BRICK_DEBUG_MEM
|
||||
void *test = data - 4 * sizeof(int);
|
||||
int magic1= INT_ACCESS(test, 0 * sizeof(int));
|
||||
int len = INT_ACCESS(test, 1 * sizeof(int));
|
||||
int line = INT_ACCESS(test, 2 * sizeof(int));
|
||||
int magic2= INT_ACCESS(test, 3 * sizeof(int));
|
||||
if (unlikely(magic1 != MAGIC_MEM1)) {
|
||||
BRICK_ERR("line %d memory corruption: magix1 %08x != %08x, len = %d\n", cline, magic1, MAGIC_MEM1, len);
|
||||
return;
|
||||
}
|
||||
if (unlikely(magic2 != MAGIC_MEM2)) {
|
||||
BRICK_ERR("line %d memory corruption: magix2 %08x != %08x, len = %d\n", cline, magic2, MAGIC_MEM2, len);
|
||||
return;
|
||||
}
|
||||
if (unlikely(line < 0 || line >= BRICK_DEBUG_MEM)) {
|
||||
BRICK_ERR("line %d memory corruption: alloc line = %d, len = %d\n", cline, line, len);
|
||||
return;
|
||||
}
|
||||
INT_ACCESS(test, 0) = 0xffffffff;
|
||||
magic1 = INT_ACCESS(data, len + 0 * sizeof(int));
|
||||
if (unlikely(magic1 != MAGIC_MEND1)) {
|
||||
BRICK_ERR("line %d memory corruption: magix1 %08x != %08x, len = %d\n", cline, magic1, MAGIC_MEND1, len);
|
||||
return;
|
||||
}
|
||||
magic2 = INT_ACCESS(data, len + 1 * sizeof(int));
|
||||
if (unlikely(magic2 != MAGIC_MEND2)) {
|
||||
BRICK_ERR("line %d memory corruption: magix2 %08x != %08x, len = %d\n", cline, magic2, MAGIC_MEND2, len);
|
||||
return;
|
||||
}
|
||||
INT_ACCESS(data, len) = 0xffffffff;
|
||||
atomic_dec(&mem_count[line]);
|
||||
atomic_inc(&mem_free[line]);
|
||||
#else
|
||||
void *test = data - PLUS_SIZE;
|
||||
int len = INT_ACCESS(test, 0 * sizeof(int));
|
||||
#endif
|
||||
data = test;
|
||||
__brick_mem_free(data, len + PLUS_SIZE);
|
||||
void *test = data - 4 * sizeof(int);
|
||||
int magic1= INT_ACCESS(test, 0 * sizeof(int));
|
||||
int len = INT_ACCESS(test, 1 * sizeof(int));
|
||||
int line = INT_ACCESS(test, 2 * sizeof(int));
|
||||
int magic2= INT_ACCESS(test, 3 * sizeof(int));
|
||||
if (unlikely(magic1 != MAGIC_MEM1)) {
|
||||
BRICK_ERR("line %d memory corruption: magix1 %08x != %08x, len = %d\n", cline, magic1, MAGIC_MEM1, len);
|
||||
return;
|
||||
}
|
||||
if (unlikely(magic2 != MAGIC_MEM2)) {
|
||||
BRICK_ERR("line %d memory corruption: magix2 %08x != %08x, len = %d\n", cline, magic2, MAGIC_MEM2, len);
|
||||
return;
|
||||
}
|
||||
if (unlikely(line < 0 || line >= BRICK_DEBUG_MEM)) {
|
||||
BRICK_ERR("line %d memory corruption: alloc line = %d, len = %d\n", cline, line, len);
|
||||
return;
|
||||
}
|
||||
INT_ACCESS(test, 0) = 0xffffffff;
|
||||
magic1 = INT_ACCESS(data, len + 0 * sizeof(int));
|
||||
if (unlikely(magic1 != MAGIC_MEND1)) {
|
||||
BRICK_ERR("line %d memory corruption: magix1 %08x != %08x, len = %d\n", cline, magic1, MAGIC_MEND1, len);
|
||||
return;
|
||||
}
|
||||
magic2 = INT_ACCESS(data, len + 1 * sizeof(int));
|
||||
if (unlikely(magic2 != MAGIC_MEND2)) {
|
||||
BRICK_ERR("line %d memory corruption: magix2 %08x != %08x, len = %d\n", cline, magic2, MAGIC_MEND2, len);
|
||||
return;
|
||||
}
|
||||
INT_ACCESS(data, len) = 0xffffffff;
|
||||
atomic_dec(&mem_count[line]);
|
||||
atomic_inc(&mem_free[line]);
|
||||
#else
|
||||
void *test = data - PLUS_SIZE;
|
||||
int len = INT_ACCESS(test, 0 * sizeof(int));
|
||||
#endif
|
||||
data = test;
|
||||
__brick_mem_free(data, len + PLUS_SIZE);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(_brick_mem_free);
|
||||
|
||||
|
|
@ -260,36 +258,34 @@ EXPORT_SYMBOL_GPL(_brick_string_alloc);
|
|||
|
||||
void _brick_string_free(const char *data, int cline)
|
||||
{
|
||||
if (data) {
|
||||
#ifdef BRICK_DEBUG_MEM
|
||||
int magic;
|
||||
int len;
|
||||
int line;
|
||||
|
||||
data -= sizeof(int) * 3;
|
||||
magic = INT_ACCESS(data, 0);
|
||||
if (unlikely(magic != MAGIC_STR)) {
|
||||
BRICK_ERR("cline %d stringmem corruption: magix %08x != %08x\n", cline, magic, MAGIC_STR);
|
||||
return;
|
||||
}
|
||||
len = INT_ACCESS(data, sizeof(int));
|
||||
line = INT_ACCESS(data, sizeof(int) * 2);
|
||||
if (unlikely(line < 0 || line >= BRICK_DEBUG_MEM)) {
|
||||
BRICK_ERR("cline %d stringmem corruption: line = %d (len = %d)\n", cline, line, len);
|
||||
return;
|
||||
}
|
||||
magic = INT_ACCESS(data, len - sizeof(int));
|
||||
if (unlikely(magic != MAGIC_SEND)) {
|
||||
BRICK_ERR("cline %d stringmem corruption: end_magix %08x != %08x, line = %d len = %d\n", cline, magic, MAGIC_SEND, len, line);
|
||||
return;
|
||||
}
|
||||
INT_ACCESS(data, len - sizeof(int)) = 0xffffffff;
|
||||
atomic_dec(&string_count[line]);
|
||||
atomic_inc(&string_free[line]);
|
||||
atomic_dec(&phys_string_alloc);
|
||||
#endif
|
||||
kfree(data);
|
||||
int magic;
|
||||
int len;
|
||||
int line;
|
||||
|
||||
data -= sizeof(int) * 3;
|
||||
magic = INT_ACCESS(data, 0);
|
||||
if (unlikely(magic != MAGIC_STR)) {
|
||||
BRICK_ERR("cline %d stringmem corruption: magix %08x != %08x\n", cline, magic, MAGIC_STR);
|
||||
return;
|
||||
}
|
||||
len = INT_ACCESS(data, sizeof(int));
|
||||
line = INT_ACCESS(data, sizeof(int) * 2);
|
||||
if (unlikely(line < 0 || line >= BRICK_DEBUG_MEM)) {
|
||||
BRICK_ERR("cline %d stringmem corruption: line = %d (len = %d)\n", cline, line, len);
|
||||
return;
|
||||
}
|
||||
magic = INT_ACCESS(data, len - sizeof(int));
|
||||
if (unlikely(magic != MAGIC_SEND)) {
|
||||
BRICK_ERR("cline %d stringmem corruption: end_magix %08x != %08x, line = %d len = %d\n", cline, magic, MAGIC_SEND, len, line);
|
||||
return;
|
||||
}
|
||||
INT_ACCESS(data, len - sizeof(int)) = 0xffffffff;
|
||||
atomic_dec(&string_count[line]);
|
||||
atomic_inc(&string_free[line]);
|
||||
atomic_dec(&phys_string_alloc);
|
||||
#endif
|
||||
kfree(data);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(_brick_string_free);
|
||||
|
||||
|
|
@ -581,9 +577,6 @@ void _brick_block_free(void *data, int len, int cline)
|
|||
const int plus = 0;
|
||||
#endif
|
||||
|
||||
if (!data) {
|
||||
return;
|
||||
}
|
||||
order = len2order(len + plus);
|
||||
#ifdef BRICK_DEBUG_MEM
|
||||
if (order > 1) {
|
||||
|
|
|
|||
|
|
@ -22,12 +22,90 @@ extern atomic64_t brick_global_block_used;
|
|||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// compiler tweaking
|
||||
|
||||
/* Some functions are known to return non-null pointer values,
|
||||
* at least under some Kconfig conditions.
|
||||
*
|
||||
* In code like...
|
||||
*
|
||||
* void *ptr = myfunction();
|
||||
* if (unlikely(!ptr)) {
|
||||
* printk("ERROR: this should not happen\n");
|
||||
* goto fail;
|
||||
* }
|
||||
*
|
||||
* ... the dead code elimination of gcc will not remove the if clause
|
||||
* because the function might return a NULL value, even if a human
|
||||
* would know that myfunction() does not return a NULL value.
|
||||
*
|
||||
* Unfortunately, the __attribute__((nonnull)) can only be applied
|
||||
* to input parameters, but not to the return value.
|
||||
*
|
||||
* More unfortunately, a small inline wrapper does not help,
|
||||
* because it seems that together with the elimination of the wrapper,
|
||||
* its nonnull attribute seems to be eliminated alltogether.
|
||||
* I don't know whether this is a bug or a feature (or just a weakness).
|
||||
*
|
||||
* Following is a small hack which solves the problem at least for gcc 4.7.
|
||||
*
|
||||
* In order to be useful, the -fdelete-null-pointer-checks must be set.
|
||||
* Since MARS is superuser-only anyway, enabling this for MARS should not
|
||||
* be a security risk
|
||||
* (c.f. upstream kernel commit a3ca86aea507904148870946d599e07a340b39bf)
|
||||
*/
|
||||
extern inline
|
||||
void *__mark_ptr_nonnull(void *_ptr)
|
||||
{
|
||||
char *ptr = _ptr;
|
||||
// fool gcc to believe that the pointer were dereferenced...
|
||||
asm("" : : "X" (*ptr));
|
||||
return ptr;
|
||||
}
|
||||
|
||||
/* All the brick memory allocations are guaranteed to succeed when
|
||||
* CONFIG_MARS_MEM_RETRY is set. In case of low memory, they will just
|
||||
* retry (forever).
|
||||
*
|
||||
* Allow checking code to be written which works for both cases:
|
||||
* CONFIG_MARS_MEM_RETRY is selected, or not.
|
||||
*/
|
||||
#ifdef CONFIG_MARS_MEM_RETRY
|
||||
#define brick_mark_nonnull __mark_ptr_nonnull
|
||||
#else
|
||||
#define brick_mark_nonnull(p) (p)
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// small memory allocation (use this only for len < PAGE_SIZE)
|
||||
|
||||
#define brick_mem_alloc(_len_) _brick_mem_alloc(_len_, __LINE__)
|
||||
#define brick_zmem_alloc(_len_) ({ void *_res_ = _brick_mem_alloc(_len_, __LINE__); if (_res_) { memset(_res_, 0, _len_); } _res_; })
|
||||
extern void *_brick_mem_alloc(int len, int line);
|
||||
#define brick_mem_free(_data_) _brick_mem_free(_data_, __LINE__)
|
||||
#define brick_mem_alloc(_len_) \
|
||||
({ \
|
||||
void *_res_ = _brick_mem_alloc(_len_, __LINE__); \
|
||||
brick_mark_nonnull(_res_); \
|
||||
})
|
||||
|
||||
#define brick_zmem_alloc(_len_) \
|
||||
({ \
|
||||
void *_res_ = _brick_mem_alloc(_len_, __LINE__); \
|
||||
_res_ = brick_mark_nonnull(_res_); \
|
||||
if (_res_) { \
|
||||
memset(_res_, 0, _len_); \
|
||||
} \
|
||||
_res_; \
|
||||
})
|
||||
|
||||
#define brick_mem_free(_data_) \
|
||||
do { \
|
||||
if (_data_) { \
|
||||
_brick_mem_free(_data_, __LINE__); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
|
||||
// don't use the following directly
|
||||
extern void *_brick_mem_alloc(int len, int line) __attribute__((malloc)) __attribute__((alloc_size(1)));
|
||||
extern void _brick_mem_free(void *data, int line);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
|
@ -36,25 +114,68 @@ extern void _brick_mem_free(void *data, int line);
|
|||
|
||||
#define BRICK_STRING_LEN 1024 /* default value when len == 0 */
|
||||
|
||||
#define brick_string_alloc(_len_) _brick_string_alloc(_len_, __LINE__)
|
||||
#define brick_strndup(_orig_,_len_) ({ char *_res_ = _brick_string_alloc((_len_) + 1, __LINE__); if (_res_) { strncpy(_res_, _orig_, (_len_)); _res_[_len_] = '\0';} _res_; })
|
||||
#define brick_strdup(_orig_) ({ int _len_ = strlen(_orig_); char *_res_ = _brick_string_alloc(_len_ + 1, __LINE__); if (_res_) { strncpy(_res_, _orig_, _len_ + 1); } _res_; })
|
||||
extern char *_brick_string_alloc(int len, int line);
|
||||
#define brick_string_free(_data_) _brick_string_free(_data_, __LINE__)
|
||||
extern void _brick_string_free(const char *data, int line);
|
||||
#define brick_string_alloc(_len_) \
|
||||
({ \
|
||||
char *_res_ = _brick_string_alloc((_len_), __LINE__); \
|
||||
(char*)brick_mark_nonnull(_res_); \
|
||||
})
|
||||
|
||||
extern void brick_mem_statistics(void);
|
||||
#define brick_strndup(_orig_,_len_) \
|
||||
({ \
|
||||
char *_res_ = _brick_string_alloc((_len_) + 1, __LINE__); \
|
||||
_res_ = brick_mark_nonnull(_res_); \
|
||||
if (_res_) { \
|
||||
strncpy(_res_, (_orig_), (_len_) + 1); \
|
||||
/* always null-terminate for safety */ \
|
||||
_res_[_len_] = '\0'; \
|
||||
} \
|
||||
(char*)brick_mark_nonnull(_res_); \
|
||||
})
|
||||
|
||||
#define brick_strdup(_orig_) \
|
||||
({ \
|
||||
int _len_ = strlen(_orig_); \
|
||||
char *_res_ = _brick_string_alloc((_len_) + 1, __LINE__); \
|
||||
_res_ = brick_mark_nonnull(_res_); \
|
||||
if (_res_) { \
|
||||
strncpy(_res_, (_orig_), (_len_) + 1); \
|
||||
} \
|
||||
(char*)brick_mark_nonnull(_res_); \
|
||||
})
|
||||
|
||||
#define brick_string_free(_data_) \
|
||||
do { \
|
||||
if (_data_) { \
|
||||
_brick_string_free(_data_, __LINE__); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
// don't use the following directly
|
||||
extern char *_brick_string_alloc(int len, int line) __attribute__((malloc));
|
||||
extern void _brick_string_free(const char *data, int line);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// block memory allocation (for aligned multiples of 512 resp PAGE_SIZE)
|
||||
|
||||
#define brick_block_alloc(_pos_,_len_) _brick_block_alloc(_pos_, _len_, __LINE__)
|
||||
extern void *_brick_block_alloc(loff_t pos, int len, int line);
|
||||
extern void _brick_block_free(void *data, int len, int cline);
|
||||
#define brick_block_free(_data_,_len_) _brick_block_free(_data_, _len_, __LINE__)
|
||||
#define brick_block_alloc(_pos_,_len_) \
|
||||
({ \
|
||||
void *_res_ = _brick_block_alloc((_pos_), (_len_), __LINE__); \
|
||||
brick_mark_nonnull(_res_); \
|
||||
})
|
||||
|
||||
#define brick_block_free(_data_,_len_)\
|
||||
do { \
|
||||
if (_data_) { \
|
||||
_brick_block_free((_data_), (_len_), __LINE__); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
extern struct page *brick_iomap(void *data, int *offset, int *len);
|
||||
|
||||
// don't use the following directly
|
||||
extern void *_brick_block_alloc(loff_t pos, int len, int line) __attribute__((malloc)) __attribute__((alloc_size(2)));
|
||||
extern void _brick_block_free(void *data, int len, int cline);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
|
@ -70,6 +191,8 @@ struct mem_reservation {
|
|||
|
||||
extern int brick_mem_reserve(struct mem_reservation *r);
|
||||
|
||||
extern void brick_mem_statistics(void);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// init
|
||||
|
|
|
|||
Loading…
Reference in New Issue