From 77f97cfe0d803afc1454ce1fd61f24fddd460b18 Mon Sep 17 00:00:00 2001 From: Christopher Faulet Date: Fri, 27 Oct 2023 16:41:58 +0200 Subject: [PATCH] MAJOR: h3: Implement zero-copy support to send DATA frame When possible, we try send DATA frame without copying data. To do so, we swap the input buffer with QCS tx buffer. It is only possible iff: * There is only one HTX block of data at the beginning of the message * Amount of data to send is equal to the size of the HTX data block * The QCS tx buffer is empty In this case, both buffers are swapped. The frame metadata are written at the begining of the buffer, before data and where the HTX structure is stored. --- include/haproxy/mux_quic-t.h | 2 +- src/h3.c | 45 +++++++++++++++++++++++++++++++----- src/hq_interop.c | 8 ++++++- src/qmux_http.c | 14 ++++------- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/include/haproxy/mux_quic-t.h b/include/haproxy/mux_quic-t.h index f293d96d5..abfc20a50 100644 --- a/include/haproxy/mux_quic-t.h +++ b/include/haproxy/mux_quic-t.h @@ -188,7 +188,7 @@ struct qcc_app_ops { int (*init)(struct qcc *qcc); int (*attach)(struct qcs *qcs, void *conn_ctx); ssize_t (*decode_qcs)(struct qcs *qcs, struct buffer *b, int fin); - size_t (*snd_buf)(struct qcs *qcs, struct htx *htx, size_t count); + size_t (*snd_buf)(struct qcs *qcs, struct buffer *buf, size_t count); size_t (*nego_ff)(struct qcs *qcs, size_t count); size_t (*done_ff)(struct qcs *qcs); int (*close)(struct qcs *qcs, enum qcc_app_ops_close_side side); diff --git a/src/h3.c b/src/h3.c index 6ae6ee211..793388df0 100644 --- a/src/h3.c +++ b/src/h3.c @@ -1674,8 +1674,9 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) } /* Returns the total of bytes sent. */ -static int h3_resp_data_send(struct qcs *qcs, struct htx *htx, size_t count) +static int h3_resp_data_send(struct qcs *qcs, struct buffer *buf, size_t count) { + struct htx *htx; struct buffer outbuf; struct buffer *res; size_t total = 0; @@ -1685,6 +1686,8 @@ static int h3_resp_data_send(struct qcs *qcs, struct htx *htx, size_t count) TRACE_ENTER(H3_EV_TX_DATA, qcs->qcc->conn, qcs); + htx = htx_from_buf(buf); + new_frame: if (!count || htx_is_empty(htx)) goto end; @@ -1693,17 +1696,38 @@ static int h3_resp_data_send(struct qcs *qcs, struct htx *htx, size_t count) type = htx_get_blk_type(blk); fsize = bsize = htx_get_blksz(blk); + /* h3 DATA headers : 1-byte frame type + varint frame length */ + hsize = 1 + QUIC_VARINT_MAX_SIZE; + if (type != HTX_BLK_DATA) goto end; res = mux_get_buf(qcs); + if (unlikely(fsize == count && + !b_data(res) && + htx_nbblks(htx) == 1 && type == HTX_BLK_DATA)) { + void *old_area = res->area; + + /* map an H2 frame to the HTX block so that we can put the + * frame header there. + */ + *res = b_make(buf->area, buf->size, sizeof(struct htx) + blk->addr - hsize, fsize + hsize); + outbuf = b_make(b_head(res), hsize, 0, 0); + b_putchr(&outbuf, 0x00); /* h3 frame type = DATA */ + b_quic_enc_int(&outbuf, fsize, QUIC_VARINT_MAX_SIZE); /* h3 frame length */ + + /* and exchange with our old area */ + buf->area = old_area; + buf->data = buf->head = 0; + total += fsize; + fsize = 0; + goto end; + } + if (fsize > count) fsize = count; - /* h3 DATA headers : 1-byte frame type + varint frame length */ - hsize = 1 + QUIC_VARINT_MAX_SIZE; - while (1) { b_reset(&outbuf); outbuf = b_make(b_tail(res), b_contig_space(res), 0, 0); @@ -1746,10 +1770,11 @@ static int h3_resp_data_send(struct qcs *qcs, struct htx *htx, size_t count) return total; } -static size_t h3_snd_buf(struct qcs *qcs, struct htx *htx, size_t count) +static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) { size_t total = 0; enum htx_blk_type btype; + struct htx *htx; struct htx_blk *blk; uint32_t bsize; int32_t idx; @@ -1757,6 +1782,11 @@ static size_t h3_snd_buf(struct qcs *qcs, struct htx *htx, size_t count) h3_debug_printf(stderr, "%s\n", __func__); + htx = htx_from_buf(buf); + + if (htx->extra && htx->extra == HTX_UNKOWN_PAYLOAD_LENGTH) + qcs->flags |= QC_SF_UNKNOWN_PL_LENGTH; + while (count && !htx_is_empty(htx) && !(qcs->flags & QC_SF_BLK_MROOM)) { idx = htx_get_head(htx); blk = htx_get_blk(htx, idx); @@ -1779,8 +1809,9 @@ static size_t h3_snd_buf(struct qcs *qcs, struct htx *htx, size_t count) break; case HTX_BLK_DATA: - ret = h3_resp_data_send(qcs, htx, count); + ret = h3_resp_data_send(qcs, buf, count); if (ret > 0) { + htx = htx_from_buf(buf); total += ret; count -= ret; if (ret < bsize) @@ -1833,6 +1864,8 @@ static size_t h3_snd_buf(struct qcs *qcs, struct htx *htx, size_t count) } out: + htx_to_buf(htx, buf); + return total; } diff --git a/src/hq_interop.c b/src/hq_interop.c index 8126a9390..bff11101f 100644 --- a/src/hq_interop.c +++ b/src/hq_interop.c @@ -91,10 +91,11 @@ static struct buffer *mux_get_buf(struct qcs *qcs) return &qcs->tx.buf; } -static size_t hq_interop_snd_buf(struct qcs *qcs, struct htx *htx, +static size_t hq_interop_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) { enum htx_blk_type btype; + struct htx *htx; struct htx_blk *blk; int32_t idx; uint32_t bsize, fsize; @@ -104,6 +105,11 @@ static size_t hq_interop_snd_buf(struct qcs *qcs, struct htx *htx, res = mux_get_buf(qcs); outbuf = b_make(b_tail(res), b_contig_space(res), 0, 0); + htx = htx_from_buf(buf); + + if (htx->extra && htx->extra == HTX_UNKOWN_PAYLOAD_LENGTH) + qcs->flags |= QC_SF_UNKNOWN_PL_LENGTH; + while (count && !htx_is_empty(htx) && !(qcs->flags & QC_SF_BLK_MROOM)) { /* Not implemented : QUIC on backend side */ idx = htx_get_head(htx); diff --git a/src/qmux_http.c b/src/qmux_http.c index fc891dc53..edf26b141 100644 --- a/src/qmux_http.c +++ b/src/qmux_http.c @@ -73,18 +73,14 @@ size_t qcs_http_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count, { struct htx *htx; size_t ret; + int eom = 0; TRACE_ENTER(QMUX_EV_STRM_SEND, qcs->qcc->conn, qcs); - htx = htx_from_buf(buf); - - if (htx->extra && htx->extra == HTX_UNKOWN_PAYLOAD_LENGTH) - qcs->flags |= QC_SF_UNKNOWN_PL_LENGTH; - - ret = qcs->qcc->app_ops->snd_buf(qcs, htx, count); - *fin = (htx->flags & HTX_FL_EOM) && htx_is_empty(htx); - - htx_to_buf(htx, buf); + htx = htxbuf(buf); + eom = (htx->flags & HTX_FL_EOM); + ret = qcs->qcc->app_ops->snd_buf(qcs, buf, count); + *fin = (eom && !b_data(buf)); TRACE_LEAVE(QMUX_EV_STRM_SEND, qcs->qcc->conn, qcs);