MEDIUM: streams: Add a way to replay failed 0rtt requests.

Add a new keyword for retry-on, 0rtt-rejected. If set, we will try to
replay requests for which we sent early data that got rejected by the
server.
If that option is set, we will attempt to use 0rtt if "allow-0rtt" is set
on the server line even if the client didn't send early data.

doc/configuration.txt

@@ -8036,6 +8036,10 @@ retry-on [list of keywords]
                         heavy database processing (full scans, etc) as it may
                         amplify denial of service attacks.
 
+      0rtt-rejected     retry requests which were sent over early data and were
+                        rejected by the server. These requests are generally
+                        considered to be safe to retry.
+
       <status>          any HTTP status code among "404" (Not Found), "408"
                         (Request Timeout), "425" (Too Early), "500" (Server
                         Error), "501" (Not Implemented), "502" (Bad Gateway),

include/types/proxy.h

@@ -218,6 +218,10 @@ enum PR_SRV_STATE_FILE {
 #define PR_RE_STATUS_MASK         (PR_RE_404 | PR_RE_408 | PR_RE_425 | \
                                    PR_RE_425 | PR_RE_500 | PR_RE_501 | \
                                    PR_RE_502 | PR_RE_503 | PR_RE_504)
+/* 0x00000800, 0x00001000, 0x00002000, 0x00004000 and 0x00008000 unused,
+ * reserved for eventual future status codes
+ */
+#define PR_RE_EARLY_ERROR         0x00010000 /* Retry if we failed at sending early data */
 struct stream;
 
 struct http_snapshot {

src/backend.c

@@ -1585,9 +1585,16 @@ int connect_server(struct stream *s)
 #ifdef USE_OPENSSL
 	if (!reuse && cli_conn && srv &&
 	    (srv->ssl_ctx.options & SRV_SSL_O_EARLY_DATA) &&
-		    (cli_conn->flags & CO_FL_EARLY_DATA) &&
+	    /* Only attempt to use early data if either the client sent
-		    !channel_is_empty(si_oc(&s->si[1])) &&
+	     * early data, so that we know it can handle a 425, or if
-		    srv_conn->flags & CO_FL_SSL_WAIT_HS)
+	     * we are allwoed to retry requests on early data failure, and
+	     * it's our first try
+	     */
+	    ((cli_conn->flags & CO_FL_EARLY_DATA) ||
+	     ((s->be->retry_type & PR_RE_EARLY_ERROR) &&
+	      s->si[1].conn_retries == s->be->conn_retries)) &&
+	    !channel_is_empty(si_oc(&s->si[1])) &&
+	    srv_conn->flags & CO_FL_SSL_WAIT_HS)
 		srv_conn->flags &= ~(CO_FL_SSL_WAIT_HS | CO_FL_WAIT_L6_CONN);
 #endif
 

src/proto_htx.c

@@ -1490,10 +1490,16 @@ int htx_wait_for_response(struct stream *s, struct channel *rep, int an_bit)
 
 		/* 1: have we encountered a read error ? */
 		if (rep->flags & CF_READ_ERROR) {
+			struct connection *conn = NULL;
+
 			if (txn->flags & TX_NOT_FIRST)
 				goto abort_keep_alive;
 
-			if (si_b->flags & SI_FL_L7_RETRY) {
+			if (objt_cs(s->si[1].end))
+				conn = objt_cs(s->si[1].end)->conn;
+
+			if (si_b->flags & SI_FL_L7_RETRY &&
+			    (!conn || conn->err_code != CO_ER_SSL_EARLY_FAILED)) {
 				/* If we arrive here, then CF_READ_ERROR was
 				 * set by si_cs_recv() because we matched a
 				 * status, overwise it would have removed
@@ -1516,11 +1522,11 @@ int htx_wait_for_response(struct stream *s, struct channel *rep, int an_bit)
 			/* Check to see if the server refused the early data.
 			 * If so, just send a 425
 			 */
-			if (objt_cs(s->si[1].end)) {
+			if (conn->err_code == CO_ER_SSL_EARLY_FAILED) {
-				struct connection *conn = objt_cs(s->si[1].end)->conn;
+				if ((s->be->retry_type & PR_RE_EARLY_ERROR) &&
-
+				    do_l7_retry(s, si_b) == 0)
-				if (conn->err_code == CO_ER_SSL_EARLY_FAILED)
+					return 0;
-					txn->status = 425;
+				txn->status = 425;
 			}
 
 			s->si[1].flags |= SI_FL_NOLINGER;

src/proxy.c

@@ -541,6 +541,8 @@ proxy_parse_retry_on(char **args, int section, struct proxy *curpx,
 			curpx->retry_type |= PR_RE_503;
 		else if (!strcmp(args[i], "504"))
 			curpx->retry_type |= PR_RE_504;
+		else if (!strcmp(args[i], "0rtt-rejected"))
+			curpx->retry_type |= PR_RE_EARLY_ERROR;
 		else if (!strcmp(args[i], "none")) {
 			if (i != 1 || *args[i + 1]) {
 				memprintf(err, "'%s' 'none' keyworld only usable alone", args[0]);