diff --git a/Makefile b/Makefile index 2b9a6670de..4d22f2eaa8 100644 --- a/Makefile +++ b/Makefile @@ -654,7 +654,7 @@ OPTIONS_OBJS += src/quic_rx.o src/mux_quic.o src/h3.o src/quic_tx.o \ src/cfgparse-quic.o src/qmux_trace.o src/qpack-enc.o \ src/qpack-tbl.o src/h3_stats.o src/quic_stats.o \ src/quic_fctl.o src/cbuf.o src/quic_rules.o \ - src/quic_token.o src/quic_pacing.o + src/quic_token.o src/quic_pacing.o src/quic_cc_drs.o endif ifneq ($(USE_QUIC_OPENSSL_COMPAT:0=),) diff --git a/include/haproxy/quic_cc-t.h b/include/haproxy/quic_cc-t.h index e5ed9b3c3f..430a6167ce 100644 --- a/include/haproxy/quic_cc-t.h +++ b/include/haproxy/quic_cc-t.h @@ -117,6 +117,7 @@ struct quic_cc_path { uint64_t ifae_pkts; /* Burst size if pacing is used. Not used if congestion algo handle pacing itself. */ uint32_t pacing_burst; + uint64_t delivery_rate; /* bytes per second */ }; struct quic_cc_algo { diff --git a/include/haproxy/quic_cc.h b/include/haproxy/quic_cc.h index bb15b3f915..db4cc58fa2 100644 --- a/include/haproxy/quic_cc.h +++ b/include/haproxy/quic_cc.h @@ -100,6 +100,7 @@ static inline void quic_cc_path_init(struct quic_cc_path *path, int ipv4, unsign path->ifae_pkts = 0; path->pacing_burst = burst; quic_cc_init(&path->cc, algo, qc); + path->delivery_rate = 0; } /* Return the remaining available on QUIC path for prepared data diff --git a/include/haproxy/quic_cc_drs.h b/include/haproxy/quic_cc_drs.h new file mode 100644 index 0000000000..fd44078bb0 --- /dev/null +++ b/include/haproxy/quic_cc_drs.h @@ -0,0 +1,41 @@ +#include + +#include + +/* Per-ACK Rate Sample State */ +struct quic_cc_rs { + uint64_t delivered; + uint64_t prior_delivered; + uint64_t tx_in_flight; + uint64_t lost; + uint64_t prior_lost; + int64_t last_end_seq; + uint32_t interval; + uint32_t prior_time; + uint32_t send_elapsed; + uint32_t ack_elapsed; + uint32_t is_app_limited; +}; + +/* Delivery rate sampling */ +struct quic_cc_drs { + struct quic_cc_rs rs; + struct wf wf; + uint64_t round_count; + uint64_t next_round_delivered; + uint64_t delivered; + uint64_t lost; + int64_t last_seq; + uint32_t delivered_time; + uint32_t first_sent_time; + int is_cwnd_limited; /* boolean */ + int app_limited; /* boolean */ +}; + +void quic_cc_drs_init(struct quic_cc_drs *drs); +void quic_cc_drs_on_pkt_sent(struct quic_cc_path *path, + struct quic_tx_packet *pkt, struct quic_cc_drs *drs); +void quic_cc_drs_update_rate_sample(struct quic_cc_drs *drs, + struct quic_tx_packet *pkt); +void quic_cc_drs_on_ack_recv(struct quic_cc_drs *drs, struct quic_cc_path *path, + uint64_t pkt_delivered); diff --git a/include/haproxy/quic_tx-t.h b/include/haproxy/quic_tx-t.h index ef5617d47c..1fba016609 100644 --- a/include/haproxy/quic_tx-t.h +++ b/include/haproxy/quic_tx-t.h @@ -53,6 +53,16 @@ struct quic_tx_packet { struct quic_tx_packet *prev; /* Largest acknowledged packet number if this packet contains an ACK frame */ int64_t largest_acked_pn; + /* Delivery rate sampling information */ + struct { + uint64_t delivered; + uint64_t tx_in_flight; + uint64_t lost; + int64_t end_seq; + uint32_t delivered_time; + uint32_t first_sent_time; + int is_app_limited; + } rs; unsigned char type; }; diff --git a/src/quic_cc_drs.c b/src/quic_cc_drs.c new file mode 100644 index 0000000000..8187de50a7 --- /dev/null +++ b/src/quic_cc_drs.c @@ -0,0 +1,155 @@ +/* Delivery Rate Sampling */ + +#include +#include +#include +#include +#include +#include + +static void quic_cc_rs_init(struct quic_cc_rs *rs) +{ + rs->interval = UINT32_MAX; + rs->delivered = 0; + rs->prior_delivered = 0; + rs->prior_time = TICK_ETERNITY; + rs->tx_in_flight = 0; + rs->lost = 0; + rs->prior_lost = 0; + rs->send_elapsed = 0; + rs->ack_elapsed = 0; + rs->last_end_seq = -1; + rs->is_app_limited = 0; +} + +void quic_cc_drs_init(struct quic_cc_drs *drs) +{ + quic_cc_rs_init(&drs->rs); + wf_init(&drs->wf, 12, 0, ~0U); + drs->round_count = 0; + drs->next_round_delivered = 0; + drs->delivered = 0; + drs->lost = 0; + drs->last_seq = -1; + drs->delivered_time = TICK_ETERNITY; + drs->first_sent_time = TICK_ETERNITY; + drs->app_limited = 0; + drs->is_cwnd_limited = 0; +} + +/* Update TX packet rate sampling information. + * Must be called after has just been sent. + */ +void quic_cc_drs_on_pkt_sent(struct quic_cc_path *path, + struct quic_tx_packet *pkt, struct quic_cc_drs *drs) +{ + if (!path->in_flight) + drs->first_sent_time = drs->delivered_time = pkt->time_sent; + + pkt->rs.first_sent_time = drs->first_sent_time; + pkt->rs.delivered_time = drs->delivered_time; + pkt->rs.delivered = drs->delivered; + pkt->rs.is_app_limited = drs->app_limited != 0; + + pkt->rs.tx_in_flight = path->in_flight + pkt->len; + pkt->rs.lost = drs->lost; + pkt->rs.end_seq = ++drs->last_seq; +} + +/* Return 1 if TX packet is the most recently sent packet + * that has been delivered, 0 if not. + */ +static inline int quic_cc_drs_is_newest_packet(struct quic_cc_drs *drs, + struct quic_tx_packet *pkt) +{ + return tick_is_lt(drs->first_sent_time, pkt->time_sent) || + (pkt->time_sent == drs->first_sent_time && + pkt->rs.end_seq > drs->rs.last_end_seq); +} + +/* RFC https://datatracker.ietf.org/doc/draft-ietf-ccwg-bbr/ + * 4.5.2.3.3. Upon receiving an ACK + * + * When an ACK arrives, the sender invokes GenerateRateSample() to fill + * in a rate sample. For each packet that was newly SACKed or ACKed, + * UpdateRateSample() updates the rate sample based on a snapshot of + * connection delivery information from the time at which the packet was + * last transmitted. UpdateRateSample() is invoked multiple times when + * a stretched ACK acknowledges multiple data packets. In this case we + * use the information from the most recently sent packet, i.e., the + * packet with the highest "P.delivered" value. + * + * haproxy implementation: quic_cc_drs_update_rate_sample() matches with + * RFC UpdateRateSample() called from first part of GenerateRateSample(). + */ +void quic_cc_drs_update_rate_sample(struct quic_cc_drs *drs, + struct quic_tx_packet *pkt) +{ + struct quic_cc_rs *rs = &drs->rs; + + if (!tick_isset(pkt->rs.delivered_time)) + return; + + drs->delivered += pkt->len; + drs->delivered_time = now_ms; + /* Update info using the newest packet. */ + if (tick_isset(rs->prior_time) && !quic_cc_drs_is_newest_packet(drs, pkt)) + return; + + rs->prior_delivered = pkt->rs.delivered; + rs->prior_time = pkt->rs.delivered_time; + rs->is_app_limited = pkt->rs.is_app_limited; + rs->send_elapsed = pkt->time_sent - pkt->rs.first_sent_time; + rs->ack_elapsed = drs->delivered_time - pkt->rs.delivered_time; + rs->tx_in_flight = pkt->rs.tx_in_flight; + rs->prior_lost = pkt->rs.lost; + rs->last_end_seq = pkt->rs.end_seq; + drs->first_sent_time = pkt->time_sent; + /* Mark the packet as delivered once it's SACKed to + * avoid being used again when it's cumulatively acked. + */ + pkt->rs.delivered_time = TICK_ETERNITY; +} + +/* RFC https://datatracker.ietf.org/doc/draft-ietf-ccwg-bbr/ + * 4.5.2.3.3. Upon receiving an ACK + * + * haproxy implementation: second part of GenerateRateSample(). Follows the + * first one above. + */ +void quic_cc_drs_on_ack_recv(struct quic_cc_drs *drs, struct quic_cc_path *path, + uint64_t pkt_delivered) +{ + struct quic_cc_rs *rs = &drs->rs; + uint64_t rate; + + if (drs->app_limited && drs->delivered > drs->app_limited) + drs->app_limited = 0; + + if (pkt_delivered >= drs->next_round_delivered) { + drs->next_round_delivered = pkt_delivered; + ++drs->round_count; + } + + if (!tick_isset(rs->prior_time)) + return; + + rs->interval = MAX(rs->send_elapsed, rs->ack_elapsed); + + BUG_ON(drs->delivered <= rs->prior_delivered); + rs->delivered = drs->delivered - rs->prior_delivered; + BUG_ON(drs->lost < rs->prior_lost); + rs->lost = drs->lost - rs->prior_lost; + + if (rs->interval < path->loss.rtt_min) { + rs->interval = UINT32_MAX; + return; + } + + if (!rs->interval) + return; + + rate = rs->delivered * 1000 / rs->interval; + if (rate >= wf_get_max(&drs->wf) || !drs->app_limited) + path->delivery_rate = wf_max_update(&drs->wf, rate, drs->round_count); +}