From dabf2e2647de2825b0115b68575a55f978d257c4 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sun, 28 Oct 2007 21:59:24 +0100 Subject: [PATCH] [MAJOR] added a new state to listeners There was a missing state for listeners, when they are not listening but still attached to the protocol. The LI_ASSIGNED state was added for this purpose. This permitted to clean up the assignment/release workflow quite a bit. Generic enable/enable_all/disable/disable_all primitives were added, and a disable_all entry was added to the struct protocol. --- include/proto/proto_uxst.h | 1 + include/proto/protocols.h | 27 +++++ include/types/protocols.h | 33 +++++- src/proto_uxst.c | 208 +++++++++++++++++++++++-------------- src/protocols.c | 78 +++++++++++++- 5 files changed, 262 insertions(+), 85 deletions(-) diff --git a/include/proto/proto_uxst.h b/include/proto/proto_uxst.h index 642beb8ce..dd5a40c5e 100644 --- a/include/proto/proto_uxst.h +++ b/include/proto/proto_uxst.h @@ -28,6 +28,7 @@ int uxst_event_accept(int fd); void uxst_add_listener(struct listener *listener); +void uxst_del_listener(struct listener *listener); void process_uxst_stats(struct task *t, struct timeval *next); #endif /* _PROTO_PROTO_UXST_H */ diff --git a/include/proto/protocols.h b/include/proto/protocols.h index aec243804..7b9e1dd9a 100644 --- a/include/proto/protocols.h +++ b/include/proto/protocols.h @@ -24,6 +24,33 @@ #include +/* This function adds the specified listener's file descriptor to the polling + * lists if it is in the LI_LISTEN state. The listener enters LI_READY or + * LI_FULL state depending on its number of connections. + */ +void enable_listener(struct listener *listener); + +/* This function removes the specified listener's file descriptor from the + * polling lists if it is in the LI_READY or in the LI_FULL state. The listener + * enters LI_LISTEN. + */ +void disable_listener(struct listener *listener); + +/* This function adds all of the protocol's listener's file descriptors to the + * polling lists when they are in the LI_LISTEN state. It is intended to be + * used as a protocol's generic enable_all() primitive, for use after the + * fork(). It puts the listeners into LI_READY or LI_FULL states depending on + * their number of connections. It always returns ERR_NONE. + */ +int enable_all_listeners(struct protocol *proto); + +/* This function removes all of the protocol's listener's file descriptors from + * the polling lists when they are in the LI_READY or LI_FULL states. It is + * intended to be used as a protocol's generic disable_all() primitive. It puts + * the listeners into LI_LISTEN, and always returns ERR_NONE. + */ +int disable_all_listeners(struct protocol *proto); + /* Registers the protocol */ void protocol_register(struct protocol *proto); diff --git a/include/types/protocols.h b/include/types/protocols.h index aef780527..6c59ce10a 100644 --- a/include/types/protocols.h +++ b/include/types/protocols.h @@ -35,10 +35,32 @@ /* listener state */ #define LI_NEW 0 /* not initialized yet */ -#define LI_INIT 1 /* attached to the protocol, but not listening yet */ -#define LI_LISTEN 2 /* started, listening but not enabled */ -#define LI_READY 3 /* started, listening and enabled */ -#define LI_FULL 4 /* reached its connection limit */ +#define LI_INIT 1 /* all parameters filled in, but not assigned yet */ +#define LI_ASSIGNED 2 /* assigned to the protocol, but not listening yet */ +#define LI_LISTEN 3 /* started, listening but not enabled */ +#define LI_READY 4 /* started, listening and enabled */ +#define LI_FULL 5 /* reached its connection limit */ + +/* Listener transitions + * calloc() set() add_listener() bind() + * -------> NEW ----> INIT ----------> ASSIGNED -----> LISTEN + * <------- <---- <---------- <----- + * free() bzero() del_listener() unbind() + * + * The file descriptor is valid only during these three states : + * + * disable() + * LISTEN <------------ READY + * A| ------------> |A + * || !max & enable() || + * || || + * || max || + * || max & enable() V| !max + * |+---------------> FULL + * +----------------- + * disable() + * + */ /* listener socket options */ #define LI_O_NONE 0x0000 @@ -50,7 +72,7 @@ */ struct listener { int fd; /* the listen socket */ - int state; /* state: NEW, INIT, LISTEN, READY, FULL */ + int state; /* state: NEW, INIT, ASSIGNED, LISTEN, READY, FULL */ int options; /* socket options : LI_O_* */ struct sockaddr_storage addr; /* the address we listen to */ struct protocol *proto; /* protocol this listener belongs to */ @@ -90,6 +112,7 @@ struct protocol { int (*bind_all)(struct protocol *proto); /* bind all unbound listeners */ int (*unbind_all)(struct protocol *proto); /* unbind all bound listeners */ int (*enable_all)(struct protocol *proto); /* enable all bound listeners */ + int (*disable_all)(struct protocol *proto); /* disable all bound listeners */ struct list listeners; /* list of listeners using this protocol */ int nb_listeners; /* number of listeners */ struct list list; /* list of registered protocols */ diff --git a/src/proto_uxst.c b/src/proto_uxst.c index 1a093263c..433837356 100644 --- a/src/proto_uxst.c +++ b/src/proto_uxst.c @@ -61,6 +61,34 @@ #define MAXPATHLEN 128 #endif +static int uxst_bind_listeners(struct protocol *proto); +static int uxst_unbind_listeners(struct protocol *proto); + +/* Note: must not be declared as its list will be overwritten */ +static struct protocol proto_unix = { + .name = "unix_stream", + .sock_domain = PF_UNIX, + .sock_type = SOCK_STREAM, + .sock_prot = 0, + .sock_family = AF_UNIX, + .sock_addrlen = sizeof(struct sockaddr_un), + .l3_addrlen = sizeof(((struct sockaddr_un*)0)->sun_path),/* path len */ + .read = &stream_sock_read, + .write = &stream_sock_write, + .bind_all = uxst_bind_listeners, + .unbind_all = uxst_unbind_listeners, + .enable_all = enable_all_listeners, + .disable_all = disable_all_listeners, + .listeners = LIST_HEAD_INIT(proto_unix.listeners), + .nb_listeners = 0, +}; + + +/******************************** + * 1) low-level socket functions + ********************************/ + + /* This function creates a named PF_UNIX stream socket at address . Note * that the path cannot be NULL nor empty. and different of -1 will * be used to change the socket owner. If is not 0, it will be used to @@ -211,6 +239,97 @@ static void destroy_uxst_socket(const char *path) } +/******************************** + * 2) listener-oriented functions + ********************************/ + + +/* This function creates the UNIX socket associated to the listener. It changes + * the state from ASSIGNED to LISTEN. The socket is NOT enabled for polling. + * The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL. + */ +static int uxst_bind_listener(struct listener *listener) +{ + int fd; + + if (listener->state != LI_ASSIGNED) + return ERR_NONE; /* already bound */ + + fd = create_uxst_socket(((struct sockaddr_un *)&listener->addr)->sun_path, + listener->perm.ux.uid, + listener->perm.ux.gid, + listener->perm.ux.mode); + if (fd == -1) + return ERR_FATAL; + + /* the socket is now listening */ + listener->fd = fd; + listener->state = LI_LISTEN; + + /* the function for the accept() event */ + fd_insert(fd); + fdtab[fd].cb[DIR_RD].f = listener->accept; + fdtab[fd].cb[DIR_WR].f = NULL; /* never called */ + fdtab[fd].cb[DIR_RD].b = fdtab[fd].cb[DIR_WR].b = NULL; + fdtab[fd].owner = (struct task *)listener; /* reference the listener instead of a task */ + fdtab[fd].state = FD_STLISTEN; + fdtab[fd].peeraddr = NULL; + fdtab[fd].peerlen = 0; + fdtab[fd].listener = NULL; + fdtab[fd].ev = 0; + return ERR_NONE; +} + +/* This function closes the UNIX sockets for the specified listener. + * The listener enters the LI_ASSIGNED state. It always returns ERR_NONE. + */ +static int uxst_unbind_listener(struct listener *listener) +{ + if (listener->state == LI_READY) + EV_FD_CLR(listener->fd, DIR_RD); + + if (listener->state >= LI_LISTEN) { + close(listener->fd); + listener->state = LI_ASSIGNED; + destroy_uxst_socket(((struct sockaddr_un *)&listener->addr)->sun_path); + } + return ERR_NONE; +} + +/* Add a listener to the list of unix stream listeners. The listener's state + * is automatically updated from LI_INIT to LI_ASSIGNED. The number of + * listeners is updated. This is the function to use to add a new listener. + */ +void uxst_add_listener(struct listener *listener) +{ + if (listener->state != LI_INIT) + return; + listener->state = LI_ASSIGNED; + listener->proto = &proto_unix; + LIST_ADDQ(&proto_unix.listeners, &listener->proto_list); + proto_unix.nb_listeners++; +} + +/* Delete a listener from the list of unix stream listeners. The listener's + * state is automatically updated from LI_ASSIGNED to LI_INIT. The number of + * listeners is updated. Note that the listener must have previously been + * unbound. This is the function to use to remove a listener. + */ +void uxst_del_listener(struct listener *listener) +{ + if (listener->state != LI_ASSIGNED) + return; + listener->state = LI_INIT; + LIST_DEL(&listener->proto_list); + proto_unix.nb_listeners--; +} + + +/******************************** + * 3) protocol-oriented functions + ********************************/ + + /* This function creates all UNIX sockets bound to the protocol entry . * It is intended to be used as the protocol's bind_all() function. * The sockets will be registered but not added to any fd_set, in order not to @@ -223,58 +342,15 @@ static int uxst_bind_listeners(struct protocol *proto) { struct listener *listener; int err = ERR_NONE; - int fd; list_for_each_entry(listener, &proto->listeners, proto_list) { - if (listener->state != LI_INIT) - continue; /* already started */ - - fd = create_uxst_socket(((struct sockaddr_un *)&listener->addr)->sun_path, - listener->perm.ux.uid, - listener->perm.ux.gid, - listener->perm.ux.mode); - if (fd == -1) { - err |= ERR_FATAL; + err |= uxst_bind_listener(listener); + if (err != ERR_NONE) continue; - } - - /* the socket is listening */ - listener->fd = fd; - listener->state = LI_LISTEN; - - /* the function for the accept() event */ - fd_insert(fd); - fdtab[fd].cb[DIR_RD].f = listener->accept; - fdtab[fd].cb[DIR_WR].f = NULL; /* never called */ - fdtab[fd].cb[DIR_RD].b = fdtab[fd].cb[DIR_WR].b = NULL; - fdtab[fd].owner = (struct task *)listener; /* reference the listener instead of a task */ - fdtab[fd].state = FD_STLISTEN; - fdtab[fd].peeraddr = NULL; - fdtab[fd].peerlen = 0; - fdtab[fd].listener = NULL; - fdtab[fd].ev = 0; } - return err; } -/* This function adds the UNIX sockets file descriptors to the polling lists - * for all listeners in the LI_LISTEN state. It is intended to be used as the - * protocol's enable_all() primitive, after the fork(). It always returns - * ERR_NONE. - */ -static int uxst_enable_listeners(struct protocol *proto) -{ - struct listener *listener; - - list_for_each_entry(listener, &proto->listeners, proto_list) { - if (listener->state == LI_LISTEN) { - EV_FD_SET(listener->fd, DIR_RD); - listener->state = LI_READY; - } - } - return ERR_NONE; -} /* This function stops all listening UNIX sockets bound to the protocol * . It does not detaches them from the protocol. @@ -284,17 +360,17 @@ static int uxst_unbind_listeners(struct protocol *proto) { struct listener *listener; - list_for_each_entry(listener, &proto->listeners, proto_list) { - if (listener->state != LI_INIT) { - EV_FD_CLR(listener->fd, DIR_RD); - close(listener->fd); - listener->state = LI_INIT; - destroy_uxst_socket(((struct sockaddr_un *)&listener->addr)->sun_path); - } - } + list_for_each_entry(listener, &proto->listeners, proto_list) + uxst_unbind_listener(listener); return ERR_NONE; } + +/******************************** + * 4) high-level functions + ********************************/ + + /* * This function is called on a read event from a listen socket, corresponding * to an accept. It tries to accept as many connections as possible. @@ -1400,32 +1476,6 @@ void process_uxst_stats(struct task *t, struct timeval *next) tv_eternity(next); } -/* Note: must not be declared as its list will be overwritten */ -static struct protocol proto_unix = { - .name = "unix_stream", - .sock_domain = PF_UNIX, - .sock_type = SOCK_STREAM, - .sock_prot = 0, - .sock_family = AF_UNIX, - .sock_addrlen = sizeof(struct sockaddr_un), - .l3_addrlen = sizeof(((struct sockaddr_un*)0)->sun_path),/* path len */ - .read = &stream_sock_read, - .write = &stream_sock_write, - .bind_all = uxst_bind_listeners, - .unbind_all = uxst_unbind_listeners, - .enable_all = uxst_enable_listeners, - .listeners = LIST_HEAD_INIT(proto_unix.listeners), - .nb_listeners = 0, -}; - -/* Adds listener to the list of unix stream listeners */ -void uxst_add_listener(struct listener *listener) -{ - listener->proto = &proto_unix; - LIST_ADDQ(&proto_unix.listeners, &listener->proto_list); - proto_unix.nb_listeners++; -} - __attribute__((constructor)) static void __uxst_protocol_init(void) { diff --git a/src/protocols.c b/src/protocols.c index 49204b891..2412d0e40 100644 --- a/src/protocols.c +++ b/src/protocols.c @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -22,6 +23,64 @@ /* List head of all registered protocols */ static struct list protocols = LIST_HEAD_INIT(protocols); +/* This function adds the specified listener's file descriptor to the polling + * lists if it is in the LI_LISTEN state. The listener enters LI_READY or + * LI_FULL state depending on its number of connections. + */ +void enable_listener(struct listener *listener) +{ + if (listener->state == LI_LISTEN) { + if (listener->nbconn < listener->maxconn) { + EV_FD_SET(listener->fd, DIR_RD); + listener->state = LI_READY; + } else { + listener->state = LI_FULL; + } + } +} + +/* This function removes the specified listener's file descriptor from the + * polling lists if it is in the LI_READY or in the LI_FULL state. The listener + * enters LI_LISTEN. + */ +void disable_listener(struct listener *listener) +{ + if (listener->state < LI_READY) + return; + if (listener->state == LI_READY) + EV_FD_CLR(listener->fd, DIR_RD); + listener->state = LI_LISTEN; +} + +/* This function adds all of the protocol's listener's file descriptors to the + * polling lists when they are in the LI_LISTEN state. It is intended to be + * used as a protocol's generic enable_all() primitive, for use after the + * fork(). It puts the listeners into LI_READY or LI_FULL states depending on + * their number of connections. It always returns ERR_NONE. + */ +int enable_all_listeners(struct protocol *proto) +{ + struct listener *listener; + + list_for_each_entry(listener, &proto->listeners, proto_list) + enable_listener(listener); + return ERR_NONE; +} + +/* This function removes all of the protocol's listener's file descriptors from + * the polling lists when they are in the LI_READY or LI_FULL states. It is + * intended to be used as a protocol's generic disable_all() primitive. It puts + * the listeners into LI_LISTEN, and always returns ERR_NONE. + */ +int disable_all_listeners(struct protocol *proto) +{ + struct listener *listener; + + list_for_each_entry(listener, &proto->listeners, proto_list) + disable_listener(listener); + return ERR_NONE; +} + /* Registers the protocol */ void protocol_register(struct protocol *proto) { @@ -37,7 +96,7 @@ void protocol_unregister(struct protocol *proto) LIST_INIT(&proto->list); } -/* binds all listeneres of all registered protocols. Returns a composition +/* binds all listeners of all registered protocols. Returns a composition * of ERR_NONE, ERR_RETRYABLE, ERR_FATAL. */ int protocol_bind_all(void) @@ -88,3 +147,20 @@ int protocol_enable_all(void) return err; } +/* disables all listeners of all registered protocols. This may be used before + * a fork() to avoid duplicating poll lists. Returns a composition of ERR_NONE, + * ERR_RETRYABLE, ERR_FATAL. + */ +int protocol_disable_all(void) +{ + struct protocol *proto; + int err; + + err = 0; + list_for_each_entry(proto, &protocols, list) { + if (proto->disable_all) + err |= proto->disable_all(proto); + } + return err; +} +