CONTRIB: plug qdiscs: Plug queuing disciplines mini HOWTO.

Add plug_qdisc.c source file which may help in how to programatically
use plug queueing disciplines with its README file.
Such code may be useful to reproduce painful network application bugs.

contrib/plug_qdisc/README

@@ -0,0 +1,59 @@
+  ** Plug queueing disciplines **
+
+  The 'plug' qdisc type is not documented. It is even not supported
+  by traffic shaping tools like 'tc' from iproute2 package.
+
+  Such qdiscs have already been used by Yelp engineers but outside
+  of haproxy with libnl-utils tools (especially nl-qdisc-* tools)
+  to implement a workaround and make haproxy reloads work.
+
+  Indeed with such plug qdiscs coupled with iptables configurations
+  we are able to temporarily bufferize IP packets and to release them as
+  needed. So, they may be very useful to "synchronize" TCP sessions
+  or at higher level to put network applications in states approaching
+  the ones suspected to occur during bugs. Furthermore to be sure
+  to produce a correct bug fix, it may be useful to reproduce
+  as mush as needed such painful bugs. This is where plug qdiscs
+  may be useful.
+
+  To have an idea about how to use plug qdisc on the command line I highly recommend to
+  read Willy Tarreau blog here:
+
+  https://www.haproxy.com/blog/truly-seamless-reloads-with-haproxy-no-more-hacks/
+
+  which refers to this other one from Yelp:
+
+  https://engineeringblog.yelp.com/2015/04/true-zero-downtime-haproxy-reloads.html
+
+  The code found in plug_qdisc.c file already helped in fixing a painful bug hard to
+  fix because hard to reproduce. To use the API it exports this is quite easy:
+
+    - First your program must call plug_disc_attach() to create if not already created
+      a plug qdisc and use it (must be done during your application own already existing
+      initializations).
+      Note that this function calls plug_qdisc_release_indefinite_buffer() so that to
+      release already buffered packets before you start your application,
+
+    - then call plug_qdisc_plug_buffer() to start buffering packets incoming to your
+      plug qdisc. So they won't be delivered to your application,
+
+    - then call plug_qdisc_release_indefinite_buffer() to stop buffering the packets
+      incoming to your plug qdisc and release those already buffered.
+      So, that to be deliver them to your application.
+
+  This code is short and simple. But uses several libraries especially libnl-route module
+  part of libnl library. To compile haproxy and make it use the plug_qdisc.c code we had
+  to link it against several libnl3 library modules like that:
+
+     -lnl-genl-3 -lnl-route-3 -lnl-3 -lnl-cli-3
+
+
+  - Some references:
+    Libnl API documentation may be found here:
+    https://www.infradead.org/~tgr/libnl/doc/api/index.html
+
+    Kernel sources:
+    http://elixir.free-electrons.com/linux/latest/source/net/sched/sch_plug.c
+
+    Nice website about traffic shaping with queuing disciplines:
+    http://wiki.linuxwall.info/doku.php/en:ressources:dossiers:networking:traffic_control

contrib/plug_qdisc/plug_qdisc.c

@@ -0,0 +1,86 @@
+#include <stdint.h>
+#include <netlink/cache.h>
+#include <netlink/cli/utils.h>
+#include <netlink/cli/tc.h>
+#include <netlink/cli/qdisc.h>
+#include <netlink/cli/link.h>
+#include <netlink/route/qdisc/plug.h>
+
+/*
+ * XXX Please, first note that this code is not safe. XXX
+ * It was developed fast so that to reproduce a bug.
+ * You will certainly have to adapt it to your application.
+ * But at least it gives an idea about how to programatically use plug
+ * queueing disciplines.
+ */
+
+static struct nl_sock *nl_sock;
+static struct nl_cache *link_cache;
+static struct rtnl_qdisc *qdisc;
+static struct rtnl_tc *tc;
+
+static int qdisc_init(void)
+{
+	nl_sock = nl_cli_alloc_socket();
+	nl_cli_connect(nl_sock, NETLINK_ROUTE);
+	link_cache = nl_cli_link_alloc_cache(nl_sock);
+	qdisc = nl_cli_qdisc_alloc();
+	tc = (struct rtnl_tc *)qdisc;
+
+	return 0;
+}
+
+/* Stop buffering and release all buffered and incoming 'qdisc'
+ * queueing discipline traffic.
+ */
+int plug_qdisc_release_indefinite_buffer(void)
+{
+	rtnl_qdisc_plug_release_indefinite(qdisc);
+	return rtnl_qdisc_add(nl_sock, qdisc, 0);
+}
+
+/* Start buffering incoming 'qdisc' queueing discipline traffic. */
+int plug_qdisc_plug_buffer(void)
+{
+	rtnl_qdisc_plug_buffer(qdisc);
+	return rtnl_qdisc_add(nl_sock, qdisc, 0);
+}
+
+/* Create a plug qdisc attached to 'device' network device with 'parent'
+ * as parent, with 'id' as ID and 'limit' as buffer size.
+ * This is equivalent to use nl-qdisc-add tool like that:
+ *  $ nl-qdisc-add --dev=<device> --parent=<parent> --id=<id> plug --limit <limit>
+ *  $ nl-qdisc-add --dev=<device> --parent=<parent> --id=<id> --update plug --release-indefinite
+ */
+int plug_qdisc_attach(char *device, char *parent, char *id, uint32_t limit)
+{
+	int ret;
+
+	if (!tc && qdisc_init() == -1)
+		return -1;
+
+	nl_cli_tc_parse_dev(tc, link_cache, device);
+	nl_cli_tc_parse_parent(tc, parent);
+	if (!rtnl_tc_get_ifindex(tc))
+		return -1;
+
+	if (!rtnl_tc_get_parent(tc))
+		return -1;
+	if (id)
+		nl_cli_tc_parse_handle(tc, id, 1);
+
+	rtnl_tc_set_kind(tc, "plug");
+	if (limit)
+		rtnl_qdisc_plug_set_limit(qdisc, limit);
+
+	ret = rtnl_qdisc_add(nl_sock, qdisc, NLM_F_CREATE);
+	if (ret < 0) {
+		fprintf(stderr, "Could add attach qdisc: %s\n", nl_geterror(ret));
+		return -1;
+	}
+	/* Release buffer. */
+	plug_qdisc_release_indefinite_buffer();
+
+	return 0;
+}
+