MEDIUM: stick-tables: flush old entries upon soft-stop

When a process with large stick tables is replaced by a new one and remains
present until the last connection finishes, it keeps these data in memory
for nothing since they will never be used anymore by incoming connections,
except during syncing with the new process. This is especially problematic
when dealing with long session protocols such as WebSocket as it becomes
possible to stack many processes and eat a lot of memory.

So the idea here is to know if a table still needs to be synced or not,
and to purge all unused entries once the sync is complete. This means that
after a few hundred milliseconds when everything has been synchronized with
the new process, only a few entries will remain allocated (only the ones
held by sessions during the restart) and all the remaining memory will be
freed.

Note that we carefully do that only after the grace period is expired so as
not to impact a possible proxy that needs to accept a few more connections
before leaving.

Doing this required to add a sync counter to the stick tables, to know how
many peer sync sessions are still in progress in order not to flush the entries
until all synchronizations are completed.

include/proto/stick_table.h

@@ -52,6 +52,7 @@ struct stktable_key *stktable_fetch_key(struct stktable *t, struct proxy *px,
 int stktable_compatible_sample(struct sample_expr *expr, unsigned long table_type);
 int stktable_get_data_type(char *name);
 struct proxy *find_stktable(const char *name);
+int stktable_trash_oldest(struct stktable *t, int to_batch);
 
 /* return allocation size for standard data type <type> */
 static inline int stktable_type_size(int type)

include/types/stick_table.h

@@ -150,6 +150,7 @@ struct stktable {
 	struct task *sync_task;   /* sync task */
 	unsigned int update;
 	unsigned int localupdate;
+	unsigned int syncing;     /* number of sync tasks watching this table now */
 	union {
 		struct peers *p; /* sync peers */
 		char *name;

src/peers.c

@@ -1401,6 +1401,7 @@ static struct task *process_peer_sync(struct task * task)
 				/* add DO NOT STOP flag if not present */
 				jobs++;
 				st->flags |= SHTABLE_F_DONOTSTOP;
+				st->table->syncing++;
 			}
 
 			/* disconnect all connected peers */
@@ -1418,6 +1419,7 @@ static struct task *process_peer_sync(struct task * task)
 				/* resync of new process was complete, current process can die now */
 				jobs--;
 				st->flags &= ~SHTABLE_F_DONOTSTOP;
+				st->table->syncing--;
 			}
 		}
 		else if (!ps->session) {
@@ -1440,6 +1442,7 @@ static struct task *process_peer_sync(struct task * task)
 					/* unable to resync new process, current process can die now */
 					jobs--;
 					st->flags &= ~SHTABLE_F_DONOTSTOP;
+					st->table->syncing--;
 				}
 			}
 		}

src/proxy.c

@@ -560,6 +560,24 @@ struct task *manage_proxy(struct task *t)
 		}
 	}
 
+	/* If the proxy holds a stick table, we need to purge all unused
+	 * entries. These are all the ones in the table with ref_cnt == 0
+	 * and all the ones in the pool used to allocate new entries. Any
+	 * entry attached to an existing session waiting for a store will
+	 * be in neither list. Any entry being dumped will have ref_cnt > 0.
+	 * However we protect tables that are being synced to peers.
+	 */
+	if (unlikely(stopping && p->state == PR_STSTOPPED && p->table.current)) {
+		if (!p->table.syncing) {
+			stktable_trash_oldest(&p->table, p->table.current);
+			pool_gc2();
+		}
+		if (p->table.current) {
+			/* some entries still remain, let's recheck in one second */
+			next = tick_first(next, tick_add(now_ms, 1000));
+		}
+	}
+
 	/* the rest below is just for frontends */
 	if (!(p->cap & PR_CAP_FE))
 		goto out;

src/stick_table.c

@@ -90,7 +90,7 @@ static struct stksess *stksess_init(struct stktable *t, struct stksess * ts)
  * Trash oldest <to_batch> sticky sessions from table <t>
  * Returns number of trashed sticky sessions.
  */
-static int stktable_trash_oldest(struct stktable *t, int to_batch)
+int stktable_trash_oldest(struct stktable *t, int to_batch)
 {
 	struct stksess *ts;
 	struct eb32_node *eb;