diff --git a/include/proto/connection.h b/include/proto/connection.h index 4c9e610c9..0236d95e4 100644 --- a/include/proto/connection.h +++ b/include/proto/connection.h @@ -96,49 +96,51 @@ void conn_update_data_polling(struct connection *c); int conn_local_send_proxy(struct connection *conn, unsigned int flag); /* inspects c->flags and returns non-zero if DATA ENA changes from the CURR ENA - * or if the WAIT flags set new flags that were not in CURR POL. Additionally, + * or if the WAIT flags are set with their respective ENA flags. Additionally, * non-zero is also returned if an error was reported on the connection. This * function is used quite often and is inlined. In order to proceed optimally * with very little code and CPU cycles, the bits are arranged so that a change - * can be detected by a simple left shift, a xor, and a mask. This operation - * detects when POLL:DATA differs from WAIT:CURR. In order to detect the ERROR - * flag without additional work, we remove it from the copy of the original - * flags (unshifted) before doing the XOR. This operation is parallelized with - * the shift and does not induce additional cycles. This explains why we check - * the error bit shifted left in the mask. Last, the final operation is an AND - * which the compiler is able to replace with a TEST in boolean conditions. The - * result is that all these checks are done in 5-6 cycles only and less than 20 - * bytes. + * can be detected by a few left shifts, a xor, and a mask. These operations + * detect when W&D are both enabled for either direction, when C&D differ for + * either direction and when Error is set. The trick consists in first keeping + * only the bits we're interested in, since they don't collide when shifted, + * and to perform the AND at the end. In practice, the compiler is able to + * replace the last AND with a TEST in boolean conditions. This results in + * checks that are done in 4-6 cycles and less than 30 bytes. */ static inline unsigned int conn_data_polling_changes(const struct connection *c) { - unsigned int f = c->flags << 2; - return ((c->flags & ~(CO_FL_ERROR << 2)) ^ f) & - ((CO_FL_ERROR<<2)|CO_FL_WAIT_WR|CO_FL_CURR_WR_ENA|CO_FL_WAIT_RD|CO_FL_CURR_RD_ENA) & - ~(f & (CO_FL_WAIT_WR|CO_FL_WAIT_RD)); + unsigned int f = c->flags; + f &= CO_FL_DATA_WR_ENA | CO_FL_DATA_RD_ENA | CO_FL_CURR_WR_ENA | + CO_FL_CURR_RD_ENA | CO_FL_ERROR | CO_FL_WAIT_WR | CO_FL_WAIT_RD; + + f = (f & (f << 2)) | /* test W & D */ + ((f ^ (f << 1)) & (CO_FL_CURR_WR_ENA|CO_FL_CURR_RD_ENA)); /* test C ^ D */ + return f & (CO_FL_WAIT_WR | CO_FL_WAIT_RD | CO_FL_CURR_WR_ENA | CO_FL_CURR_RD_ENA | CO_FL_ERROR); } /* inspects c->flags and returns non-zero if SOCK ENA changes from the CURR ENA - * or if the WAIT flags set new flags that were not in CURR POL. Additionally, + * or if the WAIT flags are set with their respective ENA flags. Additionally, * non-zero is also returned if an error was reported on the connection. This * function is used quite often and is inlined. In order to proceed optimally * with very little code and CPU cycles, the bits are arranged so that a change - * can be detected by a simple left shift, a xor, and a mask. This operation - * detects when CURR:POLL differs from SOCK:WAIT. In order to detect the ERROR - * flag without additional work, we remove it from the copy of the original - * flags (unshifted) before doing the XOR. This operation is parallelized with - * the shift and does not induce additional cycles. This explains why we check - * the error bit shifted left in the mask. Last, the final operation is an AND - * which the compiler is able to replace with a TEST in boolean conditions. The - * result is that all these checks are done in 5-6 cycles only and less than 20 - * bytes. + * can be detected by a few left shifts, a xor, and a mask. These operations + * detect when W&S are both enabled for either direction, when C&S differ for + * either direction and when Error is set. The trick consists in first keeping + * only the bits we're interested in, since they don't collide when shifted, + * and to perform the AND at the end. In practice, the compiler is able to + * replace the last AND with a TEST in boolean conditions. This results in + * checks that are done in 4-6 cycles and less than 30 bytes. */ static inline unsigned int conn_sock_polling_changes(const struct connection *c) { - unsigned int f = c->flags << 2; - return ((c->flags & ~(CO_FL_ERROR << 2)) ^ f) & - ((CO_FL_ERROR<<2)|CO_FL_WAIT_WR|CO_FL_SOCK_WR_ENA|CO_FL_WAIT_RD|CO_FL_SOCK_RD_ENA) & - ~(f & (CO_FL_WAIT_WR|CO_FL_WAIT_RD)); + unsigned int f = c->flags; + f &= CO_FL_SOCK_WR_ENA | CO_FL_SOCK_RD_ENA | CO_FL_CURR_WR_ENA | + CO_FL_CURR_RD_ENA | CO_FL_ERROR | CO_FL_WAIT_WR | CO_FL_WAIT_RD; + + f = (f & (f << 3)) | /* test W & S */ + ((f ^ (f << 2)) & (CO_FL_CURR_WR_ENA|CO_FL_CURR_RD_ENA)); /* test C ^ S */ + return f & (CO_FL_WAIT_WR | CO_FL_WAIT_RD | CO_FL_CURR_WR_ENA | CO_FL_CURR_RD_ENA | CO_FL_ERROR); } /* Automatically updates polling on connection depending on the DATA flags diff --git a/include/types/connection.h b/include/types/connection.h index 299093bee..b828ffacc 100644 --- a/include/types/connection.h +++ b/include/types/connection.h @@ -55,29 +55,18 @@ struct task; * - Stopping an I/O event consists in ANDing with ~1. * - Polling for an I/O event consists in ORing with ~3. * - * The last computed state is remembered in CO_FL_CURR_* so that differential + * The last ENA state is remembered in CO_FL_CURR_* so that differential * changes can be applied. After bits are applied, the POLL status bits are * cleared so that it is possible to detect when an EAGAIN was encountered. For * pollers that do not support speculative I/O, POLLED is the same as ENABLED * and the POL flag can safely be ignored. However it makes a difference for * the connection handler. * - * The ENA flags are per-layer (one pair for SOCK, another one for DATA). - * The POL flags are only for the socket layer since they indicate that EAGAIN - * was encountered. Thus, the DATA layer uses its own ENA flag and the socket - * layer's POL flag. - * - * The bits are arranged so that it is possible to detect a change by performing - * only a left shift followed by a xor and applying a mask to the result. The - * principle is that depending on what we want to check (data polling changes or - * sock polling changes), we mask different bits. The bits are arranged this way : - * - * S(ock) - W(ait) - C(urr) - P(oll) - D(ata) - * - * SOCK changes are reported when (S != C) || (W != P) => (S:W) != (C:P) - * DATA changes are reported when (D != C) || (W != P) => (W:C) != (P:D) - * The R and W bits are split apart so that we never shift more than 2 bits at - * a time, allowing move+shift to be done as a single operation on x86. + * The ENA flags are per-layer (one pair for SOCK, another one for DATA). The + * POL flags are irrelevant to these layers and only reflect the fact that + * EAGAIN was encountered, they're materialised by the CO_FL_WAIT_* connection + * flags. POL flags always indicate a polling change because it is assumed that + * the poller uses a cache and does not always poll. */ /* flags for use in connection->flags */ @@ -85,16 +74,15 @@ enum { CO_FL_NONE = 0x00000000, /* Just for initialization purposes */ /* Do not change these values without updating conn_*_poll_changes() ! */ - CO_FL_DATA_RD_ENA = 0x00000001, /* receiving data is allowed */ - CO_FL_CURR_RD_POL = 0x00000002, /* receiving needs to poll first */ + CO_FL_SOCK_RD_ENA = 0x00000001, /* receiving handshakes is allowed */ + CO_FL_DATA_RD_ENA = 0x00000002, /* receiving data is allowed */ CO_FL_CURR_RD_ENA = 0x00000004, /* receiving is currently allowed */ CO_FL_WAIT_RD = 0x00000008, /* receiving needs to poll first */ - CO_FL_SOCK_RD_ENA = 0x00000010, /* receiving handshakes is allowed */ + + CO_FL_SOCK_WR_ENA = 0x00000010, /* sending handshakes is desired */ CO_FL_DATA_WR_ENA = 0x00000020, /* sending data is desired */ - CO_FL_CURR_WR_POL = 0x00000040, /* sending needs to poll first */ - CO_FL_CURR_WR_ENA = 0x00000080, /* sending is currently desired */ - CO_FL_WAIT_WR = 0x00000100, /* sending needs to poll first */ - CO_FL_SOCK_WR_ENA = 0x00000200, /* sending handshakes is desired */ + CO_FL_CURR_WR_ENA = 0x00000040, /* sending is currently desired */ + CO_FL_WAIT_WR = 0x00000080, /* sending needs to poll first */ /* These flags are used by data layers to indicate they had to stop * sending data because a buffer was empty (WAIT_DATA) or stop receiving diff --git a/src/connection.c b/src/connection.c index d698e69d5..3cdc58fca 100644 --- a/src/connection.c +++ b/src/connection.c @@ -175,33 +175,31 @@ void conn_update_data_polling(struct connection *c) } /* update read status if needed */ - if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_CURR_RD_ENA)) { - f &= ~(CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL); - fd_stop_recv(c->t.sock.fd); - } - else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL)) != (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL) && - (f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) { - f |= (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL); + if (unlikely((f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) { fd_poll_recv(c->t.sock.fd); + f |= CO_FL_CURR_RD_ENA; } else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_DATA_RD_ENA)) { - f |= CO_FL_CURR_RD_ENA; fd_want_recv(c->t.sock.fd); + f |= CO_FL_CURR_RD_ENA; + } + else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_CURR_RD_ENA)) { + fd_stop_recv(c->t.sock.fd); + f &= ~CO_FL_CURR_RD_ENA; } /* update write status if needed */ - if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_CURR_WR_ENA)) { - f &= ~(CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL); - fd_stop_send(c->t.sock.fd); - } - else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL)) != (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL) && - (f & (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR))) { - f |= (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL); + if (unlikely((f & (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR))) { fd_poll_send(c->t.sock.fd); + f |= CO_FL_CURR_WR_ENA; } else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_DATA_WR_ENA)) { - f |= CO_FL_CURR_WR_ENA; fd_want_send(c->t.sock.fd); + f |= CO_FL_CURR_WR_ENA; + } + else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_CURR_WR_ENA)) { + fd_stop_send(c->t.sock.fd); + f &= ~CO_FL_CURR_WR_ENA; } c->flags = f; } @@ -225,33 +223,31 @@ void conn_update_sock_polling(struct connection *c) } /* update read status if needed */ - if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_CURR_RD_ENA)) { - f &= ~(CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL); - fd_stop_recv(c->t.sock.fd); - } - else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL)) != (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL) && - (f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) { - f |= (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL); + if (unlikely((f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) { fd_poll_recv(c->t.sock.fd); + f |= CO_FL_CURR_RD_ENA; } else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_SOCK_RD_ENA)) { - f |= CO_FL_CURR_RD_ENA; fd_want_recv(c->t.sock.fd); + f |= CO_FL_CURR_RD_ENA; + } + else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_CURR_RD_ENA)) { + fd_stop_recv(c->t.sock.fd); + f &= ~CO_FL_CURR_RD_ENA; } /* update write status if needed */ - if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_CURR_WR_ENA)) { - f &= ~(CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL); - fd_stop_send(c->t.sock.fd); - } - else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL)) != (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL) && - (f & (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR))) { - f |= (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL); + if (unlikely((f & (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR))) { fd_poll_send(c->t.sock.fd); + f |= CO_FL_CURR_WR_ENA; } else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_SOCK_WR_ENA)) { - f |= CO_FL_CURR_WR_ENA; fd_want_send(c->t.sock.fd); + f |= CO_FL_CURR_WR_ENA; + } + else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_CURR_WR_ENA)) { + fd_stop_send(c->t.sock.fd); + f &= ~CO_FL_CURR_WR_ENA; } c->flags = f; }