diff --git a/Makefile b/Makefile index 13340e48f..1ccd45d39 100644 --- a/Makefile +++ b/Makefile @@ -969,7 +969,7 @@ OBJS += src/mux_h2.o src/mux_h1.o src/mux_fcgi.o src/stream.o \ src/errors.o src/extcheck.o src/dns_ring.o src/stats-json.o \ src/http_conv.o src/frontend.o src/proto_sockpair.o \ src/compression.o src/ncbuf.o src/stats-file.o src/raw_sock.o \ - src/lb_fwrr.o src/action.o src/uri_normalizer.o \ + src/lb_fwrr.o src/action.o src/uri_normalizer.o src/buf.o \ src/proto_uxst.o src/ebmbtree.o src/xprt_handshake.o \ src/protocol.o src/proto_udp.o src/lb_fwlc.o src/sha1.o \ src/proto_uxdg.o src/mailers.o src/lb_fas.o src/cfgcond.o \ diff --git a/include/haproxy/buf.h b/include/haproxy/buf.h index ad22e6f1f..9e5a9e861 100644 --- a/include/haproxy/buf.h +++ b/include/haproxy/buf.h @@ -33,6 +33,28 @@ #include #include +size_t b_getblk_ofs(const struct buffer *buf, char *blk, size_t len, size_t offset); +size_t b_getblk(const struct buffer *buf, char *blk, size_t len, size_t offset); +size_t b_getblk_nc(const struct buffer *buf, const char **blk1, size_t *len1, + const char **blk2, size_t *len2, size_t ofs, size_t max); +size_t b_getdelim(const struct buffer *buf, size_t offset, size_t count, + char *str, size_t len, const char *delim, char escape); +size_t b_getline(const struct buffer *buf, size_t offset, size_t count, + char *str, size_t len); +void b_slow_realign(struct buffer *b, char *swap, size_t output); +void b_slow_realign_ofs(struct buffer *b, char *swap, size_t ofs); +size_t b_putblk_ofs(struct buffer *buf, char *blk, size_t len, size_t offset); +void __b_putblk(struct buffer *b, const char *blk, size_t len); +size_t b_xfer(struct buffer *dst, struct buffer *src, size_t count); +size_t b_ncat(struct buffer *dst, const struct buffer *src, size_t count); +void b_move(const struct buffer *b, size_t src, size_t len, ssize_t shift); +int b_rep_blk(struct buffer *b, char *pos, char *end, const char *blk, size_t len); +int b_insert_blk(struct buffer *b, size_t off, const char *blk, size_t len); +void __b_put_varint(struct buffer *b, uint64_t v); +int b_put_varint(struct buffer *b, uint64_t v); +int b_get_varint(struct buffer *b, uint64_t *vptr); +int b_peek_varint(struct buffer *b, size_t ofs, uint64_t *vptr); + /***************************************************************************/ /* Functions used to compute offsets and pointers. Most of them exist in */ /* both wrapping-safe and unchecked ("__" prefix) variants. Some returning */ @@ -335,221 +357,6 @@ static inline size_t b_contig_space(const struct buffer *b) return left; } -/* b_getblk_ofs() : gets one full block of data at once from a buffer, starting - * from offset after the buffer's area, and for exactly bytes. - * As a convenience to avoid complex checks in callers, the offset is allowed - * to exceed a valid one by no more than one buffer size, and will automatically - * be wrapped. The caller is responsible for ensuring that doesn't exceed - * the known length of the available data at this position, otherwise undefined - * data will be returned. This is meant to be used on concurrently accessed - * buffers, so that a reader can read a known area while the buffer is being fed - * and trimmed. The function guarantees never to use ->head nor ->data. The - * buffer is left unaffected. It always returns the number of bytes copied. - */ -static inline size_t b_getblk_ofs(const struct buffer *buf, char *blk, size_t len, size_t offset) -{ - size_t firstblock; - - if (offset >= buf->size) - offset -= buf->size; - - BUG_ON(offset >= buf->size); - - firstblock = buf->size - offset; - - if (firstblock >= len) - firstblock = len; - - memcpy(blk, b_orig(buf) + offset, firstblock); - - if (len > firstblock) - memcpy(blk + firstblock, b_orig(buf), len - firstblock); - return len; -} - -/* b_getblk() : gets one full block of data at once from a buffer, starting - * from offset after the buffer's head, and limited to no more than - * bytes. The caller is responsible for ensuring that neither - * nor + exceed the total number of bytes available in the buffer. - * Return values : - * >0 : number of bytes read, equal to requested size. - * =0 : not enough data available. is left undefined. - * The buffer is left unaffected. - */ -static inline size_t b_getblk(const struct buffer *buf, char *blk, size_t len, size_t offset) -{ - size_t firstblock; - - BUG_ON(buf->data > buf->size); - BUG_ON(offset > buf->data); - BUG_ON(offset + len > buf->data); - - if (len + offset > b_data(buf)) - return 0; - - firstblock = b_wrap(buf) - b_head(buf); - if (firstblock > offset) { - if (firstblock >= len + offset) { - memcpy(blk, b_head(buf) + offset, len); - return len; - } - - memcpy(blk, b_head(buf) + offset, firstblock - offset); - memcpy(blk + firstblock - offset, b_orig(buf), len - firstblock + offset); - return len; - } - - memcpy(blk, b_orig(buf) + offset - firstblock, len); - return len; -} - -/* b_getblk_nc() : gets one or two blocks of data at once from a buffer, - * starting from offset after the beginning of its output, and limited to - * no more than bytes. The caller is responsible for ensuring that - * neither nor + exceed the total number of bytes available in - * the buffer. Return values : - * >0 : number of blocks filled (1 or 2). blk1 is always filled before blk2. - * =0 : not enough data available. are left undefined. - * The buffer is left unaffected. Unused buffers are left in an undefined state. - */ -static inline size_t b_getblk_nc(const struct buffer *buf, const char **blk1, size_t *len1, const char **blk2, size_t *len2, size_t ofs, size_t max) -{ - size_t l1; - - BUG_ON_HOT(buf->data > buf->size); - BUG_ON_HOT(ofs > buf->data); - BUG_ON_HOT(ofs + max > buf->data); - - if (!max) - return 0; - - *blk1 = b_peek(buf, ofs); - l1 = b_wrap(buf) - *blk1; - if (l1 < max) { - *len1 = l1; - *len2 = max - l1; - *blk2 = b_orig(buf); - return 2; - } - *len1 = max; - return 1; -} - -/* Locates the longest part of the buffer that is composed exclusively of - * characters not in the set, and delimited by one of these characters, - * and returns the initial part and the first of such delimiters. A single - * escape character in may be specified so that when not 0 and found, - * the character that follows it is never taken as a delimiter. Note that - * cannot contain the zero byte, hence this function is not usable with - * byte zero as a delimiter. - * - * Return values : - * >0 : number of bytes read. Includes the sep if present before len or end. - * =0 : no sep before end found. is left undefined. - * - * The buffer is left unaffected. Unused buffers are left in an undefined state. - */ -static inline size_t b_getdelim(const struct buffer *buf, size_t offset, size_t count, - char *str, size_t len, const char *delim, char escape) -{ - uchar delim_map[256 / 8]; - int found, escaped; - uint pos, bit; - size_t ret, max; - uchar b; - char *p; - - ret = 0; - p = b_peek(buf, offset); - - max = len; - if (!count || offset+count > b_data(buf)) - goto out; - if (max > count) { - max = count; - str[max-1] = 0; - } - - /* create the byte map */ - memset(delim_map, 0, sizeof(delim_map)); - while ((b = *delim)) { - pos = b >> 3; - bit = b & 7; - delim_map[pos] |= 1 << bit; - delim++; - } - - found = escaped = 0; - while (max) { - *str++ = b = *p; - ret++; - max--; - - if (escape && (escaped || *p == escape)) { - escaped = !escaped; - goto skip; - } - - pos = b >> 3; - bit = b & 7; - if (delim_map[pos] & (1 << bit)) { - found = 1; - break; - } - skip: - p = b_next(buf, p); - } - - if (ret > 0 && !found) - ret = 0; - out: - if (max) - *str = 0; - return ret; -} - -/* Gets one text line out of aa buffer. - * Return values : - * >0 : number of bytes read. Includes the \n if present before len or end. - * =0 : no '\n' before end found. is left undefined. - * - * The buffer is left unaffected. Unused buffers are left in an undefined state. - */ -static inline size_t b_getline(const struct buffer *buf, size_t offset, size_t count, - char *str, size_t len) -{ - size_t ret, max; - char *p; - - ret = 0; - p = b_peek(buf, offset); - - max = len; - if (!count || offset+count > b_data(buf)) - goto out; - if (max > count) { - max = count; - str[max-1] = 0; - } - - while (max) { - *str++ = *p; - ret++; - max--; - - if (*p == '\n') - break; - p = b_next(buf, p); - } - - if (ret > 0 && *(str-1) != '\n') - ret = 0; - out: - if (max) - *str = 0; - return ret; -} - /*********************************************/ /* Functions used to modify the buffer state */ @@ -615,84 +422,6 @@ static inline void b_realign_if_empty(struct buffer *b) b->head = 0; } -/* b_slow_realign() : this function realigns a possibly wrapping buffer so that - * the part remaining to be parsed is contiguous and starts at the beginning of - * the buffer and the already parsed output part ends at the end of the buffer. - * This provides the best conditions since it allows the largest inputs to be - * processed at once and ensures that once the output data leaves, the whole - * buffer is available at once. The number of output bytes supposedly present - * at the beginning of the buffer and which need to be moved to the end must be - * passed in . A temporary swap area at least as large as b->size must - * be provided in . It's up to the caller to ensure is no larger - * than the difference between the whole buffer's length and its input. - */ -static inline void b_slow_realign(struct buffer *b, char *swap, size_t output) -{ - size_t block1 = output; - size_t block2 = 0; - - BUG_ON_HOT(b->data > b->size); - - /* process output data in two steps to cover wrapping */ - if (block1 > b_size(b) - b_head_ofs(b)) { - block2 = b_peek_ofs(b, block1); - block1 -= block2; - } - memcpy(swap + b_size(b) - output, b_head(b), block1); - memcpy(swap + b_size(b) - block2, b_orig(b), block2); - - /* process input data in two steps to cover wrapping */ - block1 = b_data(b) - output; - block2 = 0; - - if (block1 > b_tail_ofs(b)) { - block2 = b_tail_ofs(b); - block1 = block1 - block2; - } - memcpy(swap, b_peek(b, output), block1); - memcpy(swap + block1, b_orig(b), block2); - - /* reinject changes into the buffer */ - memcpy(b_orig(b), swap, b_data(b) - output); - memcpy(b_wrap(b) - output, swap + b_size(b) - output, output); - - b->head = (output ? b_size(b) - output : 0); -} - -/* b_slow_realign_ofs() : this function realigns a possibly wrapping buffer - * setting its new head at . Depending of the value, the resulting - * buffer may also wrap. A temporary swap area at least as large as b->size must - * be provided in . It's up to the caller to ensuze is not larger - * than b->size. - */ -static inline void b_slow_realign_ofs(struct buffer *b, char *swap, size_t ofs) -{ - size_t block1 = b_data(b); - size_t block2 = 0; - - BUG_ON_HOT(b->data > b->size); - BUG_ON_HOT(ofs > b->size); - - if (__b_tail_ofs(b) >= b_size(b)) { - block2 = b_tail_ofs(b); - block1 -= block2; - } - memcpy(swap, b_head(b), block1); - memcpy(swap + block1, b_orig(b), block2); - - block1 = b_data(b); - block2 = 0; - if (block1 > b_size(b) - ofs) { - block1 = b_size(b) - ofs; - block2 = b_data(b) - block1; - } - memcpy(b_orig(b) + ofs, swap, block1); - memcpy(b_orig(b), swap + block1, block2); - - b->head = ofs; -} - - /* b_putchar() : tries to append char at the end of buffer . Supports * wrapping. Data are truncated if buffer is full. */ @@ -704,60 +433,6 @@ static inline void b_putchr(struct buffer *b, char c) b->data++; } -/* b_putblk_ofs(): puts one full block of data of length from into - * the buffer, starting from absolute offset after the buffer's area. - * As a convenience to avoid complex checks in callers, the offset is allowed - * to exceed a valid one by no more than one buffer size, and will automatically - * be wrapped. The caller is responsible for ensuring that doesn't exceed - * the known length of the available room at this position, otherwise data may - * be overwritten. The buffer's length is *not* updated, so generally the caller - * will have updated it before calling this function. This is meant to be used - * on concurrently accessed buffers, so that a writer can append data while a - * reader is blocked by other means from reaching the current area The function - * guarantees never to use ->head nor ->data. It always returns the number of - * bytes copied. - */ -static inline size_t b_putblk_ofs(struct buffer *buf, char *blk, size_t len, size_t offset) -{ - size_t firstblock; - - if (offset >= buf->size) - offset -= buf->size; - - BUG_ON(offset >= buf->size); - - firstblock = buf->size - offset; - - if (firstblock >= len) - firstblock = len; - - memcpy(b_orig(buf) + offset, blk, firstblock); - - if (len > firstblock) - memcpy(b_orig(buf), blk + firstblock, len - firstblock); - return len; -} - -/* __b_putblk() : tries to append bytes from block to the end of - * buffer without checking for free space (it's up to the caller to do it). - * Supports wrapping. It must not be called with len == 0. - */ -static inline void __b_putblk(struct buffer *b, const char *blk, size_t len) -{ - size_t half = b_contig_space(b); - - BUG_ON(b_data(b) + len > b_size(b)); - - if (half > len) - half = len; - - memcpy(b_tail(b), blk, half); - - if (len > half) - memcpy(b_peek(b, b_data(b) + half), blk + half, len - half); - b->data += len; -} - /* b_putblk() : tries to append block at the end of buffer . Supports * wrapping. Data are truncated if buffer is too short. It returns the number * of bytes copied. @@ -771,85 +446,6 @@ static inline size_t b_putblk(struct buffer *b, const char *blk, size_t len) return len; } -/* b_xfer() : transfers at most bytes from buffer to buffer - * and returns the number of bytes copied. The bytes are removed from and - * added to . The caller is responsible for ensuring that is not - * larger than b_room(dst). Whenever possible (if the destination is empty and - * at least as much as the source was requested), the buffers are simply - * swapped instead of copied. - */ -static inline size_t b_xfer(struct buffer *dst, struct buffer *src, size_t count) -{ - size_t ret, block1, block2; - - ret = 0; - if (!count) - goto leave; - - ret = b_data(src); - if (!ret) - goto leave; - - if (ret > count) - ret = count; - else if (!b_data(dst)) { - /* zero copy is possible by just swapping buffers */ - struct buffer tmp = *dst; - *dst = *src; - *src = tmp; - goto leave; - } - - block1 = b_contig_data(src, 0); - if (block1 > ret) - block1 = ret; - block2 = ret - block1; - - if (block1) - __b_putblk(dst, b_head(src), block1); - - if (block2) - __b_putblk(dst, b_peek(src, block1), block2); - - b_del(src, ret); - leave: - return ret; -} - -/* b_ncat() : Copy from buffer at the end of buffer. - * The caller is responsible for ensuring that is not larger than - * b_room(dst). - * Returns the number of bytes copied. - */ -static inline size_t b_ncat(struct buffer *dst, const struct buffer *src, size_t count) -{ - size_t ret, block1, block2; - - ret = 0; - if (!count) - goto leave; - - ret = b_data(src); - if (!ret) - goto leave; - - if (ret > count) - ret = count; - block1 = b_contig_data(src, 0); - if (block1 > ret) - block1 = ret; - block2 = ret - block1; - - if (block1) - __b_putblk(dst, b_head(src), block1); - - if (block2) - __b_putblk(dst, b_peek(src, block1), block2); - - leave: - return ret; -} - /* b_force_xfer() : same as b_xfer() but without zero copy. * The caller is responsible for ensuring that is not * larger than b_room(dst). @@ -865,299 +461,6 @@ static inline size_t b_force_xfer(struct buffer *dst, struct buffer *src, size_t } -/* Moves bytes from absolute position of buffer by - * bytes, while supporting wrapping of both the source and the destination. - * The position is relative to the buffer's origin and may overlap with the - * target position. The 's absolute value must be strictly lower than - * the buffer's size. The main purpose is to aggregate data block during - * parsing while removing unused delimiters. The buffer's length is not - * modified, and the caller must take care of size adjustments and holes by - * itself. - */ -static inline void b_move(const struct buffer *b, size_t src, size_t len, ssize_t shift) -{ - char *orig = b_orig(b); - size_t size = b_size(b); - size_t dst = src + size + shift; - size_t cnt; - - BUG_ON(len > size); - - if (dst >= size) - dst -= size; - - if (shift < 0) { - BUG_ON(-shift >= size); - /* copy from left to right */ - for (; (cnt = len); len -= cnt) { - if (cnt > size - src) - cnt = size - src; - if (cnt > size - dst) - cnt = size - dst; - - memmove(orig + dst, orig + src, cnt); - dst += cnt; - src += cnt; - if (dst >= size) - dst -= size; - if (src >= size) - src -= size; - } - } - else if (shift > 0) { - BUG_ON(shift >= size); - /* copy from right to left */ - for (; (cnt = len); len -= cnt) { - size_t src_end = src + len; - size_t dst_end = dst + len; - - if (dst_end > size) - dst_end -= size; - if (src_end > size) - src_end -= size; - - if (cnt > dst_end) - cnt = dst_end; - if (cnt > src_end) - cnt = src_end; - - memmove(orig + dst_end - cnt, orig + src_end - cnt, cnt); - } - } -} - -/* b_rep_blk() : writes the block at position which must be in - * buffer , and moves the part between and the buffer's tail just - * after the end of the copy of . This effectively replaces the part - * located between and with a copy of of length . The - * buffer's length is automatically updated. This is used to replace a block - * with another one inside a buffer. The shift value (positive or negative) is - * returned. If there's no space left, the move is not done. If is null, - * the pointer is allowed to be null, in order to erase a block. - */ -static inline int b_rep_blk(struct buffer *b, char *pos, char *end, const char *blk, size_t len) -{ - int delta; - - BUG_ON(pos < b->area || pos >= b->area + b->size); - - delta = len - (end - pos); - - if (__b_tail(b) + delta > b_wrap(b)) - return 0; /* no space left */ - - if (b_data(b) && - b_tail(b) + delta > b_head(b) && - b_head(b) >= b_tail(b)) - return 0; /* no space left before wrapping data */ - - /* first, protect the end of the buffer */ - memmove(end + delta, end, b_tail(b) - end); - - /* now, copy blk over pos */ - if (len) - memcpy(pos, blk, len); - - b_add(b, delta); - b_realign_if_empty(b); - - return delta; -} - -/* b_insert_blk(): inserts the block at the absolute offset moving - * data between this offset and the buffer's tail just after the end of the copy - * of . The buffer's length is automatically updated. It Supports - * wrapping. If there are not enough space to perform the copy, 0 is - * returned. Otherwise, the number of bytes copied is returned -*/ -static inline int b_insert_blk(struct buffer *b, size_t off, const char *blk, size_t len) -{ - size_t pos; - - if (!len || len > b_room(b)) - return 0; /* nothing to copy or not enough space left */ - - pos = b_peek_ofs(b, off); - if (pos == b_tail_ofs(b)) - __b_putblk(b, blk, len); - else { - size_t delta = b_data(b) - off; - - /* first, protect the end of the buffer */ - b_move(b, pos, delta, len); - - /* change the amount of data in the buffer during the copy */ - b_sub(b, delta); - __b_putblk(b, blk, len); - b_add(b, delta); - } - return len; -} - -/* __b_put_varint(): encode 64-bit value as a varint into buffer . The - * caller must have checked that the encoded value fits in the buffer so that - * there are no length checks. Wrapping is supported. You don't want to use - * this function but b_put_varint() instead. - */ -static inline void __b_put_varint(struct buffer *b, uint64_t v) -{ - size_t data = b->data; - size_t size = b_size(b); - char *wrap = b_wrap(b); - char *tail = b_tail(b); - - BUG_ON_HOT(data >= size); - - if (v >= 0xF0) { - /* more than one byte, first write the 4 least significant - * bits, then follow with 7 bits per byte. - */ - *tail = v | 0xF0; - v = (v - 0xF0) >> 4; - - while (1) { - if (++tail == wrap) - tail -= size; - data++; - if (v < 0x80) - break; - *tail = v | 0x80; - v = (v - 0x80) >> 7; - } - } - - /* last byte */ - *tail = v; - BUG_ON_HOT(data >= size); - data++; - b->data = data; -} - -/* b_put_varint(): try to encode value as a varint into buffer . Returns - * the number of bytes written in case of success, or 0 if there is not enough - * room. Wrapping is supported. No partial writes will be performed. - */ -static inline int b_put_varint(struct buffer *b, uint64_t v) -{ - size_t data = b->data; - size_t size = b_size(b); - char *wrap = b_wrap(b); - char *tail = b_tail(b); - - if (data != size && v >= 0xF0) { - BUG_ON_HOT(data > size); - - /* more than one byte, first write the 4 least significant - * bits, then follow with 7 bits per byte. - */ - *tail = v | 0xF0; - v = (v - 0xF0) >> 4; - - while (1) { - if (++tail == wrap) - tail -= size; - data++; - if (data == size || v < 0x80) - break; - *tail = v | 0x80; - v = (v - 0x80) >> 7; - } - } - - /* last byte */ - if (data == size) - return 0; - - *tail = v; - data++; - - size = data - b->data; - b->data = data; - return size; -} - -/* b_get_varint(): try to decode a varint from buffer into value . - * Returns the number of bytes read in case of success, or 0 if there were not - * enough bytes. Wrapping is supported. No partial reads will be performed. - */ -static inline int b_get_varint(struct buffer *b, uint64_t *vptr) -{ - const uint8_t *head = (const uint8_t *)b_head(b); - const uint8_t *wrap = (const uint8_t *)b_wrap(b); - size_t data = b->data; - size_t size = b_size(b); - uint64_t v = 0; - int bits = 0; - - if (data != 0 && (*head >= 0xF0)) { - v = *head; - bits += 4; - while (1) { - if (++head == wrap) - head -= size; - data--; - if (!data || !(*head & 0x80)) - break; - v += (uint64_t)*head << bits; - bits += 7; - } - } - - /* last byte */ - if (!data) - return 0; - - v += (uint64_t)*head << bits; - *vptr = v; - data--; - size = b->data - data; - b_del(b, size); - return size; -} - -/* b_peek_varint(): try to decode a varint from buffer at offset - * relative to head, into value . Returns the number of bytes parsed in - * case of success, or 0 if there were not enough bytes, in which case the - * contents of are not updated. Wrapping is supported. The buffer's head - * will NOT be updated. It is illegal to call this function with greater - * than b->data. - */ -static inline int b_peek_varint(struct buffer *b, size_t ofs, uint64_t *vptr) -{ - const uint8_t *head = (const uint8_t *)b_peek(b, ofs); - const uint8_t *wrap = (const uint8_t *)b_wrap(b); - size_t data = b_data(b) - ofs; - size_t size = b_size(b); - uint64_t v = 0; - int bits = 0; - - BUG_ON_HOT(ofs > b_data(b)); - - if (data != 0 && (*head >= 0xF0)) { - v = *head; - bits += 4; - while (1) { - if (++head == wrap) - head -= size; - data--; - if (!data || !(*head & 0x80)) - break; - v += (uint64_t)*head << bits; - bits += 7; - } - } - - /* last byte */ - if (!data) - return 0; - - v += (uint64_t)*head << bits; - *vptr = v; - data--; - size = b->data - ofs - data; - return size; -} - /* * Buffer ring management. diff --git a/src/buf.c b/src/buf.c new file mode 100644 index 000000000..4f455dba5 --- /dev/null +++ b/src/buf.c @@ -0,0 +1,732 @@ +/* + * Simple buffer handling - heavy functions definitions + * + * Most of the low-level operations are in buf.h, but this file centralizes + * heavier functions that shouldn't be inlined. + * + * SPDX-License-Identifier: GPL-2.0-or-later. + * + */ + +#include +#include +#include +#include + +/* b_getblk_ofs() : gets one full block of data at once from a buffer, starting + * from offset after the buffer's area, and for exactly bytes. + * As a convenience to avoid complex checks in callers, the offset is allowed + * to exceed a valid one by no more than one buffer size, and will automatically + * be wrapped. The caller is responsible for ensuring that doesn't exceed + * the known length of the available data at this position, otherwise undefined + * data will be returned. This is meant to be used on concurrently accessed + * buffers, so that a reader can read a known area while the buffer is being fed + * and trimmed. The function guarantees never to use ->head nor ->data. The + * buffer is left unaffected. It always returns the number of bytes copied. + */ +size_t b_getblk_ofs(const struct buffer *buf, char *blk, size_t len, size_t offset) +{ + size_t firstblock; + + if (offset >= buf->size) + offset -= buf->size; + + BUG_ON(offset >= buf->size); + + firstblock = buf->size - offset; + + if (firstblock >= len) + firstblock = len; + + memcpy(blk, b_orig(buf) + offset, firstblock); + + if (len > firstblock) + memcpy(blk + firstblock, b_orig(buf), len - firstblock); + return len; +} + +/* b_getblk() : gets one full block of data at once from a buffer, starting + * from offset after the buffer's head, and limited to no more than + * bytes. The caller is responsible for ensuring that neither + * nor + exceed the total number of bytes available in the buffer. + * Return values : + * >0 : number of bytes read, equal to requested size. + * =0 : not enough data available. is left undefined. + * The buffer is left unaffected. + */ +size_t b_getblk(const struct buffer *buf, char *blk, size_t len, size_t offset) + { + size_t firstblock; + + BUG_ON(buf->data > buf->size); + BUG_ON(offset > buf->data); + BUG_ON(offset + len > buf->data); + + if (len + offset > b_data(buf)) + return 0; + + firstblock = b_wrap(buf) - b_head(buf); + if (firstblock > offset) { + if (firstblock >= len + offset) { + memcpy(blk, b_head(buf) + offset, len); + return len; + } + + memcpy(blk, b_head(buf) + offset, firstblock - offset); + memcpy(blk + firstblock - offset, b_orig(buf), len - firstblock + offset); + return len; + } + + memcpy(blk, b_orig(buf) + offset - firstblock, len); + return len; +} + +/* b_getblk_nc() : gets one or two blocks of data at once from a buffer, + * starting from offset after the beginning of its output, and limited to + * no more than bytes. The caller is responsible for ensuring that + * neither nor + exceed the total number of bytes available in + * the buffer. Return values : + * >0 : number of blocks filled (1 or 2). blk1 is always filled before blk2. + * =0 : not enough data available. are left undefined. + * The buffer is left unaffected. Unused buffers are left in an undefined state. + */ +size_t b_getblk_nc(const struct buffer *buf, const char **blk1, size_t *len1, const char **blk2, size_t *len2, size_t ofs, size_t max) +{ + size_t l1; + + BUG_ON_HOT(buf->data > buf->size); + BUG_ON_HOT(ofs > buf->data); + BUG_ON_HOT(ofs + max > buf->data); + + if (!max) + return 0; + + *blk1 = b_peek(buf, ofs); + l1 = b_wrap(buf) - *blk1; + if (l1 < max) { + *len1 = l1; + *len2 = max - l1; + *blk2 = b_orig(buf); + return 2; + } + *len1 = max; + return 1; +} + +/* Locates the longest part of the buffer that is composed exclusively of + * characters not in the set, and delimited by one of these characters, + * and returns the initial part and the first of such delimiters. A single + * escape character in may be specified so that when not 0 and found, + * the character that follows it is never taken as a delimiter. Note that + * cannot contain the zero byte, hence this function is not usable with + * byte zero as a delimiter. + * + * Return values : + * >0 : number of bytes read. Includes the sep if present before len or end. + * =0 : no sep before end found. is left undefined. + * + * The buffer is left unaffected. Unused buffers are left in an undefined state. + */ +size_t b_getdelim(const struct buffer *buf, size_t offset, size_t count, + char *str, size_t len, const char *delim, char escape) +{ + uchar delim_map[256 / 8]; + int found, escaped; + uint pos, bit; + size_t ret, max; + uchar b; + char *p; + + ret = 0; + p = b_peek(buf, offset); + + max = len; + if (!count || offset+count > b_data(buf)) + goto out; + if (max > count) { + max = count; + str[max-1] = 0; + } + + /* create the byte map */ + memset(delim_map, 0, sizeof(delim_map)); + while ((b = *delim)) { + pos = b >> 3; + bit = b & 7; + delim_map[pos] |= 1 << bit; + delim++; + } + + found = escaped = 0; + while (max) { + *str++ = b = *p; + ret++; + max--; + + if (escape && (escaped || *p == escape)) { + escaped = !escaped; + goto skip; + } + + pos = b >> 3; + bit = b & 7; + if (delim_map[pos] & (1 << bit)) { + found = 1; + break; + } + skip: + p = b_next(buf, p); + } + + if (ret > 0 && !found) + ret = 0; + out: + if (max) + *str = 0; + return ret; +} + +/* Gets one text line out of aa buffer. + * Return values : + * >0 : number of bytes read. Includes the \n if present before len or end. + * =0 : no '\n' before end found. is left undefined. + * + * The buffer is left unaffected. Unused buffers are left in an undefined state. + */ +size_t b_getline(const struct buffer *buf, size_t offset, size_t count, + char *str, size_t len) +{ + size_t ret, max; + char *p; + + ret = 0; + p = b_peek(buf, offset); + + max = len; + if (!count || offset+count > b_data(buf)) + goto out; + if (max > count) { + max = count; + str[max-1] = 0; + } + + while (max) { + *str++ = *p; + ret++; + max--; + + if (*p == '\n') + break; + p = b_next(buf, p); + } + + if (ret > 0 && *(str-1) != '\n') + ret = 0; + out: + if (max) + *str = 0; + return ret; +} + +/* b_slow_realign() : this function realigns a possibly wrapping buffer so that + * the part remaining to be parsed is contiguous and starts at the beginning of + * the buffer and the already parsed output part ends at the end of the buffer. + * This provides the best conditions since it allows the largest inputs to be + * processed at once and ensures that once the output data leaves, the whole + * buffer is available at once. The number of output bytes supposedly present + * at the beginning of the buffer and which need to be moved to the end must be + * passed in . A temporary swap area at least as large as b->size must + * be provided in . It's up to the caller to ensure is no larger + * than the difference between the whole buffer's length and its input. + */ +void b_slow_realign(struct buffer *b, char *swap, size_t output) +{ + size_t block1 = output; + size_t block2 = 0; + + BUG_ON_HOT(b->data > b->size); + + /* process output data in two steps to cover wrapping */ + if (block1 > b_size(b) - b_head_ofs(b)) { + block2 = b_peek_ofs(b, block1); + block1 -= block2; + } + memcpy(swap + b_size(b) - output, b_head(b), block1); + memcpy(swap + b_size(b) - block2, b_orig(b), block2); + + /* process input data in two steps to cover wrapping */ + block1 = b_data(b) - output; + block2 = 0; + + if (block1 > b_tail_ofs(b)) { + block2 = b_tail_ofs(b); + block1 = block1 - block2; + } + memcpy(swap, b_peek(b, output), block1); + memcpy(swap + block1, b_orig(b), block2); + + /* reinject changes into the buffer */ + memcpy(b_orig(b), swap, b_data(b) - output); + memcpy(b_wrap(b) - output, swap + b_size(b) - output, output); + + b->head = (output ? b_size(b) - output : 0); +} + +/* b_slow_realign_ofs() : this function realigns a possibly wrapping buffer + * setting its new head at . Depending of the value, the resulting + * buffer may also wrap. A temporary swap area at least as large as b->size must + * be provided in . It's up to the caller to ensuze is not larger + * than b->size. + */ +void b_slow_realign_ofs(struct buffer *b, char *swap, size_t ofs) +{ + size_t block1 = b_data(b); + size_t block2 = 0; + + BUG_ON_HOT(b->data > b->size); + BUG_ON_HOT(ofs > b->size); + + if (__b_tail_ofs(b) >= b_size(b)) { + block2 = b_tail_ofs(b); + block1 -= block2; + } + memcpy(swap, b_head(b), block1); + memcpy(swap + block1, b_orig(b), block2); + + block1 = b_data(b); + block2 = 0; + if (block1 > b_size(b) - ofs) { + block1 = b_size(b) - ofs; + block2 = b_data(b) - block1; + } + memcpy(b_orig(b) + ofs, swap, block1); + memcpy(b_orig(b), swap + block1, block2); + + b->head = ofs; +} + +/* b_putblk_ofs(): puts one full block of data of length from into + * the buffer, starting from absolute offset after the buffer's area. + * As a convenience to avoid complex checks in callers, the offset is allowed + * to exceed a valid one by no more than one buffer size, and will automatically + * be wrapped. The caller is responsible for ensuring that doesn't exceed + * the known length of the available room at this position, otherwise data may + * be overwritten. The buffer's length is *not* updated, so generally the caller + * will have updated it before calling this function. This is meant to be used + * on concurrently accessed buffers, so that a writer can append data while a + * reader is blocked by other means from reaching the current area The function + * guarantees never to use ->head nor ->data. It always returns the number of + * bytes copied. + */ +size_t b_putblk_ofs(struct buffer *buf, char *blk, size_t len, size_t offset) +{ + size_t firstblock; + + if (offset >= buf->size) + offset -= buf->size; + + BUG_ON(offset >= buf->size); + + firstblock = buf->size - offset; + + if (firstblock >= len) + firstblock = len; + + memcpy(b_orig(buf) + offset, blk, firstblock); + + if (len > firstblock) + memcpy(b_orig(buf), blk + firstblock, len - firstblock); + return len; +} + +/* __b_putblk() : tries to append bytes from block to the end of + * buffer without checking for free space (it's up to the caller to do it). + * Supports wrapping. It must not be called with len == 0. + */ +void __b_putblk(struct buffer *b, const char *blk, size_t len) +{ + size_t half = b_contig_space(b); + + BUG_ON(b_data(b) + len > b_size(b)); + + if (half > len) + half = len; + + memcpy(b_tail(b), blk, half); + + if (len > half) + memcpy(b_peek(b, b_data(b) + half), blk + half, len - half); + b->data += len; +} + +/* b_xfer() : transfers at most bytes from buffer to buffer + * and returns the number of bytes copied. The bytes are removed from and + * added to . The caller is responsible for ensuring that is not + * larger than b_room(dst). Whenever possible (if the destination is empty and + * at least as much as the source was requested), the buffers are simply + * swapped instead of copied. + */ +size_t b_xfer(struct buffer *dst, struct buffer *src, size_t count) +{ + size_t ret, block1, block2; + + ret = 0; + if (!count) + goto leave; + + ret = b_data(src); + if (!ret) + goto leave; + + if (ret > count) + ret = count; + else if (!b_data(dst)) { + /* zero copy is possible by just swapping buffers */ + struct buffer tmp = *dst; + *dst = *src; + *src = tmp; + goto leave; + } + + block1 = b_contig_data(src, 0); + if (block1 > ret) + block1 = ret; + block2 = ret - block1; + + if (block1) + __b_putblk(dst, b_head(src), block1); + + if (block2) + __b_putblk(dst, b_peek(src, block1), block2); + + b_del(src, ret); + leave: + return ret; +} + +/* b_ncat() : Copy from buffer at the end of buffer. + * The caller is responsible for ensuring that is not larger than + * b_room(dst). + * Returns the number of bytes copied. + */ +size_t b_ncat(struct buffer *dst, const struct buffer *src, size_t count) +{ + size_t ret, block1, block2; + + ret = 0; + if (!count) + goto leave; + + ret = b_data(src); + if (!ret) + goto leave; + + if (ret > count) + ret = count; + block1 = b_contig_data(src, 0); + if (block1 > ret) + block1 = ret; + block2 = ret - block1; + + if (block1) + __b_putblk(dst, b_head(src), block1); + + if (block2) + __b_putblk(dst, b_peek(src, block1), block2); + + leave: + return ret; +} + +/* Moves bytes from absolute position of buffer by + * bytes, while supporting wrapping of both the source and the destination. + * The position is relative to the buffer's origin and may overlap with the + * target position. The 's absolute value must be strictly lower than + * the buffer's size. The main purpose is to aggregate data block during + * parsing while removing unused delimiters. The buffer's length is not + * modified, and the caller must take care of size adjustments and holes by + * itself. + */ +void b_move(const struct buffer *b, size_t src, size_t len, ssize_t shift) +{ + char *orig = b_orig(b); + size_t size = b_size(b); + size_t dst = src + size + shift; + size_t cnt; + + BUG_ON(len > size); + + if (dst >= size) + dst -= size; + + if (shift < 0) { + BUG_ON(-shift >= size); + /* copy from left to right */ + for (; (cnt = len); len -= cnt) { + if (cnt > size - src) + cnt = size - src; + if (cnt > size - dst) + cnt = size - dst; + + memmove(orig + dst, orig + src, cnt); + dst += cnt; + src += cnt; + if (dst >= size) + dst -= size; + if (src >= size) + src -= size; + } + } + else if (shift > 0) { + BUG_ON(shift >= size); + /* copy from right to left */ + for (; (cnt = len); len -= cnt) { + size_t src_end = src + len; + size_t dst_end = dst + len; + + if (dst_end > size) + dst_end -= size; + if (src_end > size) + src_end -= size; + + if (cnt > dst_end) + cnt = dst_end; + if (cnt > src_end) + cnt = src_end; + + memmove(orig + dst_end - cnt, orig + src_end - cnt, cnt); + } + } +} + +/* b_rep_blk() : writes the block at position which must be in + * buffer , and moves the part between and the buffer's tail just + * after the end of the copy of . This effectively replaces the part + * located between and with a copy of of length . The + * buffer's length is automatically updated. This is used to replace a block + * with another one inside a buffer. The shift value (positive or negative) is + * returned. If there's no space left, the move is not done. If is null, + * the pointer is allowed to be null, in order to erase a block. + */ +int b_rep_blk(struct buffer *b, char *pos, char *end, const char *blk, size_t len) +{ + int delta; + + BUG_ON(pos < b->area || pos >= b->area + b->size); + + delta = len - (end - pos); + + if (__b_tail(b) + delta > b_wrap(b)) + return 0; /* no space left */ + + if (b_data(b) && + b_tail(b) + delta > b_head(b) && + b_head(b) >= b_tail(b)) + return 0; /* no space left before wrapping data */ + + /* first, protect the end of the buffer */ + memmove(end + delta, end, b_tail(b) - end); + + /* now, copy blk over pos */ + if (len) + memcpy(pos, blk, len); + + b_add(b, delta); + b_realign_if_empty(b); + + return delta; +} + +/* b_insert_blk(): inserts the block at the absolute offset moving + * data between this offset and the buffer's tail just after the end of the copy + * of . The buffer's length is automatically updated. It Supports + * wrapping. If there are not enough space to perform the copy, 0 is + * returned. Otherwise, the number of bytes copied is returned +*/ +int b_insert_blk(struct buffer *b, size_t off, const char *blk, size_t len) +{ + size_t pos; + + if (!len || len > b_room(b)) + return 0; /* nothing to copy or not enough space left */ + + pos = b_peek_ofs(b, off); + if (pos == b_tail_ofs(b)) + __b_putblk(b, blk, len); + else { + size_t delta = b_data(b) - off; + + /* first, protect the end of the buffer */ + b_move(b, pos, delta, len); + + /* change the amount of data in the buffer during the copy */ + b_sub(b, delta); + __b_putblk(b, blk, len); + b_add(b, delta); + } + return len; +} + +/* __b_put_varint(): encode 64-bit value as a varint into buffer . The + * caller must have checked that the encoded value fits in the buffer so that + * there are no length checks. Wrapping is supported. You don't want to use + * this function but b_put_varint() instead. + */ +void __b_put_varint(struct buffer *b, uint64_t v) +{ + size_t data = b->data; + size_t size = b_size(b); + char *wrap = b_wrap(b); + char *tail = b_tail(b); + + BUG_ON_HOT(data >= size); + + if (v >= 0xF0) { + /* more than one byte, first write the 4 least significant + * bits, then follow with 7 bits per byte. + */ + *tail = v | 0xF0; + v = (v - 0xF0) >> 4; + + while (1) { + if (++tail == wrap) + tail -= size; + data++; + if (v < 0x80) + break; + *tail = v | 0x80; + v = (v - 0x80) >> 7; + } + } + + /* last byte */ + *tail = v; + BUG_ON_HOT(data >= size); + data++; + b->data = data; +} + +/* b_put_varint(): try to encode value as a varint into buffer . Returns + * the number of bytes written in case of success, or 0 if there is not enough + * room. Wrapping is supported. No partial writes will be performed. + */ +int b_put_varint(struct buffer *b, uint64_t v) +{ + size_t data = b->data; + size_t size = b_size(b); + char *wrap = b_wrap(b); + char *tail = b_tail(b); + + if (data != size && v >= 0xF0) { + BUG_ON_HOT(data > size); + + /* more than one byte, first write the 4 least significant + * bits, then follow with 7 bits per byte. + */ + *tail = v | 0xF0; + v = (v - 0xF0) >> 4; + + while (1) { + if (++tail == wrap) + tail -= size; + data++; + if (data == size || v < 0x80) + break; + *tail = v | 0x80; + v = (v - 0x80) >> 7; + } + } + + /* last byte */ + if (data == size) + return 0; + + *tail = v; + data++; + + size = data - b->data; + b->data = data; + return size; +} + +/* b_get_varint(): try to decode a varint from buffer into value . + * Returns the number of bytes read in case of success, or 0 if there were not + * enough bytes. Wrapping is supported. No partial reads will be performed. + */ +int b_get_varint(struct buffer *b, uint64_t *vptr) +{ + const uint8_t *head = (const uint8_t *)b_head(b); + const uint8_t *wrap = (const uint8_t *)b_wrap(b); + size_t data = b->data; + size_t size = b_size(b); + uint64_t v = 0; + int bits = 0; + + if (data != 0 && (*head >= 0xF0)) { + v = *head; + bits += 4; + while (1) { + if (++head == wrap) + head -= size; + data--; + if (!data || !(*head & 0x80)) + break; + v += (uint64_t)*head << bits; + bits += 7; + } + } + + /* last byte */ + if (!data) + return 0; + + v += (uint64_t)*head << bits; + *vptr = v; + data--; + size = b->data - data; + b_del(b, size); + return size; +} + +/* b_peek_varint(): try to decode a varint from buffer at offset + * relative to head, into value . Returns the number of bytes parsed in + * case of success, or 0 if there were not enough bytes, in which case the + * contents of are not updated. Wrapping is supported. The buffer's head + * will NOT be updated. It is illegal to call this function with greater + * than b->data. + */ +int b_peek_varint(struct buffer *b, size_t ofs, uint64_t *vptr) +{ + const uint8_t *head = (const uint8_t *)b_peek(b, ofs); + const uint8_t *wrap = (const uint8_t *)b_wrap(b); + size_t data = b_data(b) - ofs; + size_t size = b_size(b); + uint64_t v = 0; + int bits = 0; + + BUG_ON_HOT(ofs > b_data(b)); + + if (data != 0 && (*head >= 0xF0)) { + v = *head; + bits += 4; + while (1) { + if (++head == wrap) + head -= size; + data--; + if (!data || !(*head & 0x80)) + break; + v += (uint64_t)*head << bits; + bits += 7; + } + } + + /* last byte */ + if (!data) + return 0; + + v += (uint64_t)*head << bits; + *vptr = v; + data--; + size = b->data - ofs - data; + return size; +}