2007-10-16 15:34:28 +00:00
|
|
|
/*
|
|
|
|
* UNIX SOCK_STREAM protocol layer (uxst)
|
|
|
|
*
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
* Copyright 2000-2008 Willy Tarreau <w@1wt.eu>
|
2007-10-16 15:34:28 +00:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <ctype.h>
|
|
|
|
#include <errno.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <syslog.h>
|
|
|
|
#include <time.h>
|
|
|
|
|
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/socket.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/un.h>
|
|
|
|
|
|
|
|
#include <common/compat.h>
|
|
|
|
#include <common/config.h>
|
|
|
|
#include <common/debug.h>
|
2007-10-28 10:14:07 +00:00
|
|
|
#include <common/errors.h>
|
2007-10-16 15:34:28 +00:00
|
|
|
#include <common/memory.h>
|
|
|
|
#include <common/mini-clist.h>
|
|
|
|
#include <common/standard.h>
|
2008-07-06 22:09:58 +00:00
|
|
|
#include <common/ticks.h>
|
2007-10-16 15:34:28 +00:00
|
|
|
#include <common/time.h>
|
|
|
|
#include <common/version.h>
|
|
|
|
|
|
|
|
#include <types/global.h>
|
|
|
|
|
|
|
|
#include <proto/acl.h>
|
|
|
|
#include <proto/backend.h>
|
|
|
|
#include <proto/buffers.h>
|
2007-10-17 16:57:38 +00:00
|
|
|
#include <proto/dumpstats.h>
|
2007-10-16 15:34:28 +00:00
|
|
|
#include <proto/fd.h>
|
|
|
|
#include <proto/log.h>
|
|
|
|
#include <proto/protocols.h>
|
|
|
|
#include <proto/proto_uxst.h>
|
|
|
|
#include <proto/queue.h>
|
2007-10-17 16:57:38 +00:00
|
|
|
#include <proto/senddata.h>
|
2007-10-16 15:34:28 +00:00
|
|
|
#include <proto/session.h>
|
|
|
|
#include <proto/stream_sock.h>
|
|
|
|
#include <proto/task.h>
|
|
|
|
|
|
|
|
#ifndef MAXPATHLEN
|
|
|
|
#define MAXPATHLEN 128
|
|
|
|
#endif
|
|
|
|
|
2007-10-28 20:59:24 +00:00
|
|
|
static int uxst_bind_listeners(struct protocol *proto);
|
|
|
|
static int uxst_unbind_listeners(struct protocol *proto);
|
|
|
|
|
|
|
|
/* Note: must not be declared <const> as its list will be overwritten */
|
|
|
|
static struct protocol proto_unix = {
|
|
|
|
.name = "unix_stream",
|
|
|
|
.sock_domain = PF_UNIX,
|
|
|
|
.sock_type = SOCK_STREAM,
|
|
|
|
.sock_prot = 0,
|
|
|
|
.sock_family = AF_UNIX,
|
|
|
|
.sock_addrlen = sizeof(struct sockaddr_un),
|
|
|
|
.l3_addrlen = sizeof(((struct sockaddr_un*)0)->sun_path),/* path len */
|
|
|
|
.read = &stream_sock_read,
|
|
|
|
.write = &stream_sock_write,
|
|
|
|
.bind_all = uxst_bind_listeners,
|
|
|
|
.unbind_all = uxst_unbind_listeners,
|
|
|
|
.enable_all = enable_all_listeners,
|
|
|
|
.disable_all = disable_all_listeners,
|
|
|
|
.listeners = LIST_HEAD_INIT(proto_unix.listeners),
|
|
|
|
.nb_listeners = 0,
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
/********************************
|
|
|
|
* 1) low-level socket functions
|
|
|
|
********************************/
|
|
|
|
|
|
|
|
|
2007-10-16 15:34:28 +00:00
|
|
|
/* This function creates a named PF_UNIX stream socket at address <path>. Note
|
2007-10-18 10:45:54 +00:00
|
|
|
* that the path cannot be NULL nor empty. <uid> and <gid> different of -1 will
|
|
|
|
* be used to change the socket owner. If <mode> is not 0, it will be used to
|
|
|
|
* restrict access to the socket. While it is known not to be portable on every
|
|
|
|
* OS, it's still useful where it works.
|
2007-10-16 15:34:28 +00:00
|
|
|
* It returns the assigned file descriptor, or -1 in the event of an error.
|
|
|
|
*/
|
2007-10-18 10:45:54 +00:00
|
|
|
static int create_uxst_socket(const char *path, uid_t uid, gid_t gid, mode_t mode)
|
2007-10-16 15:34:28 +00:00
|
|
|
{
|
|
|
|
char tempname[MAXPATHLEN];
|
|
|
|
char backname[MAXPATHLEN];
|
|
|
|
struct sockaddr_un addr;
|
|
|
|
|
|
|
|
int ret, sock;
|
|
|
|
|
|
|
|
/* 1. create socket names */
|
|
|
|
if (!path[0]) {
|
|
|
|
Alert("Invalid name for a UNIX socket. Aborting.\n");
|
|
|
|
goto err_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = snprintf(tempname, MAXPATHLEN, "%s.%d.tmp", path, pid);
|
|
|
|
if (ret < 0 || ret >= MAXPATHLEN) {
|
|
|
|
Alert("name too long for UNIX socket. Aborting.\n");
|
|
|
|
goto err_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = snprintf(backname, MAXPATHLEN, "%s.%d.bak", path, pid);
|
|
|
|
if (ret < 0 || ret >= MAXPATHLEN) {
|
|
|
|
Alert("name too long for UNIX socket. Aborting.\n");
|
|
|
|
goto err_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 2. clean existing orphaned entries */
|
|
|
|
if (unlink(tempname) < 0 && errno != ENOENT) {
|
|
|
|
Alert("error when trying to unlink previous UNIX socket. Aborting.\n");
|
|
|
|
goto err_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlink(backname) < 0 && errno != ENOENT) {
|
|
|
|
Alert("error when trying to unlink previous UNIX socket. Aborting.\n");
|
|
|
|
goto err_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 3. backup existing socket */
|
|
|
|
if (link(path, backname) < 0 && errno != ENOENT) {
|
|
|
|
Alert("error when trying to preserve previous UNIX socket. Aborting.\n");
|
|
|
|
goto err_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 4. prepare new socket */
|
|
|
|
addr.sun_family = AF_UNIX;
|
|
|
|
strncpy(addr.sun_path, tempname, sizeof(addr.sun_path));
|
|
|
|
addr.sun_path[sizeof(addr.sun_path) - 1] = 0;
|
|
|
|
|
|
|
|
sock = socket(PF_UNIX, SOCK_STREAM, 0);
|
|
|
|
if (sock < 0) {
|
|
|
|
Alert("cannot create socket for UNIX listener. Aborting.\n");
|
|
|
|
goto err_unlink_back;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sock >= global.maxsock) {
|
|
|
|
Alert("socket(): not enough free sockets for UNIX listener. Raise -n argument. Aborting.\n");
|
|
|
|
goto err_unlink_temp;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fcntl(sock, F_SETFL, O_NONBLOCK) == -1) {
|
|
|
|
Alert("cannot make UNIX socket non-blocking. Aborting.\n");
|
|
|
|
goto err_unlink_temp;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
|
|
|
|
/* note that bind() creates the socket <tempname> on the file system */
|
|
|
|
Alert("cannot bind socket for UNIX listener. Aborting.\n");
|
|
|
|
goto err_unlink_temp;
|
|
|
|
}
|
|
|
|
|
2007-10-18 10:45:54 +00:00
|
|
|
if (((uid != -1 || gid != -1) && (chown(tempname, uid, gid) == -1)) ||
|
|
|
|
(mode != 0 && chmod(tempname, mode) == -1)) {
|
|
|
|
Alert("cannot change UNIX socket ownership. Aborting.\n");
|
|
|
|
goto err_unlink_temp;
|
|
|
|
}
|
|
|
|
|
2007-10-16 15:34:28 +00:00
|
|
|
if (listen(sock, 0) < 0) {
|
|
|
|
Alert("cannot listen to socket for UNIX listener. Aborting.\n");
|
|
|
|
goto err_unlink_temp;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 5. install.
|
|
|
|
* Point of no return: we are ready, we'll switch the sockets. We don't
|
|
|
|
* fear loosing the socket <path> because we have a copy of it in
|
|
|
|
* backname.
|
|
|
|
*/
|
|
|
|
if (rename(tempname, path) < 0) {
|
|
|
|
Alert("cannot switch final and temporary sockets for UNIX listener. Aborting.\n");
|
|
|
|
goto err_rename;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 6. cleanup */
|
|
|
|
unlink(backname); /* no need to keep this one either */
|
|
|
|
|
|
|
|
return sock;
|
|
|
|
|
|
|
|
err_rename:
|
|
|
|
ret = rename(backname, path);
|
|
|
|
if (ret < 0 && errno == ENOENT)
|
|
|
|
unlink(path);
|
|
|
|
err_unlink_temp:
|
|
|
|
unlink(tempname);
|
|
|
|
close(sock);
|
|
|
|
err_unlink_back:
|
|
|
|
unlink(backname);
|
|
|
|
err_return:
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Tries to destroy the UNIX stream socket <path>. The socket must not be used
|
|
|
|
* anymore. It practises best effort, and no error is returned.
|
|
|
|
*/
|
|
|
|
static void destroy_uxst_socket(const char *path)
|
|
|
|
{
|
|
|
|
struct sockaddr_un addr;
|
|
|
|
int sock, ret;
|
|
|
|
|
|
|
|
/* We might have been chrooted, so we may not be able to access the
|
|
|
|
* socket. In order to avoid bothering the other end, we connect with a
|
|
|
|
* wrong protocol, namely SOCK_DGRAM. The return code from connect()
|
|
|
|
* is enough to know if the socket is still live or not. If it's live
|
|
|
|
* in mode SOCK_STREAM, we get EPROTOTYPE or anything else but not
|
|
|
|
* ECONNREFUSED. In this case, we do not touch it because it's used
|
|
|
|
* by some other process.
|
|
|
|
*/
|
|
|
|
sock = socket(PF_UNIX, SOCK_DGRAM, 0);
|
|
|
|
if (sock < 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
addr.sun_family = AF_UNIX;
|
|
|
|
strncpy(addr.sun_path, path, sizeof(addr.sun_path));
|
2007-10-18 14:15:52 +00:00
|
|
|
addr.sun_path[sizeof(addr.sun_path) - 1] = 0;
|
2007-10-16 15:34:28 +00:00
|
|
|
ret = connect(sock, (struct sockaddr *)&addr, sizeof(addr));
|
|
|
|
if (ret < 0 && errno == ECONNREFUSED) {
|
|
|
|
/* Connect failed: the socket still exists but is not used
|
|
|
|
* anymore. Let's remove this socket now.
|
|
|
|
*/
|
|
|
|
unlink(path);
|
|
|
|
}
|
|
|
|
close(sock);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-10-28 20:59:24 +00:00
|
|
|
/********************************
|
|
|
|
* 2) listener-oriented functions
|
|
|
|
********************************/
|
|
|
|
|
|
|
|
|
|
|
|
/* This function creates the UNIX socket associated to the listener. It changes
|
|
|
|
* the state from ASSIGNED to LISTEN. The socket is NOT enabled for polling.
|
|
|
|
* The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL.
|
|
|
|
*/
|
|
|
|
static int uxst_bind_listener(struct listener *listener)
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
|
|
|
|
if (listener->state != LI_ASSIGNED)
|
|
|
|
return ERR_NONE; /* already bound */
|
|
|
|
|
|
|
|
fd = create_uxst_socket(((struct sockaddr_un *)&listener->addr)->sun_path,
|
|
|
|
listener->perm.ux.uid,
|
|
|
|
listener->perm.ux.gid,
|
|
|
|
listener->perm.ux.mode);
|
|
|
|
if (fd == -1)
|
|
|
|
return ERR_FATAL;
|
|
|
|
|
|
|
|
/* the socket is now listening */
|
|
|
|
listener->fd = fd;
|
|
|
|
listener->state = LI_LISTEN;
|
|
|
|
|
|
|
|
/* the function for the accept() event */
|
|
|
|
fd_insert(fd);
|
|
|
|
fdtab[fd].cb[DIR_RD].f = listener->accept;
|
|
|
|
fdtab[fd].cb[DIR_WR].f = NULL; /* never called */
|
|
|
|
fdtab[fd].cb[DIR_RD].b = fdtab[fd].cb[DIR_WR].b = NULL;
|
2008-08-29 21:36:51 +00:00
|
|
|
fdtab[fd].owner = listener; /* reference the listener instead of a task */
|
2007-10-28 20:59:24 +00:00
|
|
|
fdtab[fd].state = FD_STLISTEN;
|
|
|
|
fdtab[fd].peeraddr = NULL;
|
|
|
|
fdtab[fd].peerlen = 0;
|
|
|
|
fdtab[fd].listener = NULL;
|
|
|
|
return ERR_NONE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* This function closes the UNIX sockets for the specified listener.
|
|
|
|
* The listener enters the LI_ASSIGNED state. It always returns ERR_NONE.
|
|
|
|
*/
|
|
|
|
static int uxst_unbind_listener(struct listener *listener)
|
|
|
|
{
|
|
|
|
if (listener->state == LI_READY)
|
|
|
|
EV_FD_CLR(listener->fd, DIR_RD);
|
|
|
|
|
|
|
|
if (listener->state >= LI_LISTEN) {
|
2007-10-28 21:07:08 +00:00
|
|
|
fd_delete(listener->fd);
|
2007-10-28 20:59:24 +00:00
|
|
|
listener->state = LI_ASSIGNED;
|
|
|
|
destroy_uxst_socket(((struct sockaddr_un *)&listener->addr)->sun_path);
|
|
|
|
}
|
|
|
|
return ERR_NONE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Add a listener to the list of unix stream listeners. The listener's state
|
|
|
|
* is automatically updated from LI_INIT to LI_ASSIGNED. The number of
|
|
|
|
* listeners is updated. This is the function to use to add a new listener.
|
|
|
|
*/
|
|
|
|
void uxst_add_listener(struct listener *listener)
|
|
|
|
{
|
|
|
|
if (listener->state != LI_INIT)
|
|
|
|
return;
|
|
|
|
listener->state = LI_ASSIGNED;
|
|
|
|
listener->proto = &proto_unix;
|
|
|
|
LIST_ADDQ(&proto_unix.listeners, &listener->proto_list);
|
|
|
|
proto_unix.nb_listeners++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/********************************
|
|
|
|
* 3) protocol-oriented functions
|
|
|
|
********************************/
|
|
|
|
|
|
|
|
|
2007-10-16 15:34:28 +00:00
|
|
|
/* This function creates all UNIX sockets bound to the protocol entry <proto>.
|
|
|
|
* It is intended to be used as the protocol's bind_all() function.
|
|
|
|
* The sockets will be registered but not added to any fd_set, in order not to
|
|
|
|
* loose them across the fork(). A call to uxst_enable_listeners() is needed
|
|
|
|
* to complete initialization.
|
|
|
|
*
|
|
|
|
* The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL.
|
|
|
|
*/
|
|
|
|
static int uxst_bind_listeners(struct protocol *proto)
|
|
|
|
{
|
|
|
|
struct listener *listener;
|
|
|
|
int err = ERR_NONE;
|
|
|
|
|
|
|
|
list_for_each_entry(listener, &proto->listeners, proto_list) {
|
2007-10-28 20:59:24 +00:00
|
|
|
err |= uxst_bind_listener(listener);
|
|
|
|
if (err != ERR_NONE)
|
2007-10-16 15:34:28 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* This function stops all listening UNIX sockets bound to the protocol
|
|
|
|
* <proto>. It does not detaches them from the protocol.
|
|
|
|
* It always returns ERR_NONE.
|
|
|
|
*/
|
|
|
|
static int uxst_unbind_listeners(struct protocol *proto)
|
|
|
|
{
|
|
|
|
struct listener *listener;
|
|
|
|
|
2007-10-28 20:59:24 +00:00
|
|
|
list_for_each_entry(listener, &proto->listeners, proto_list)
|
|
|
|
uxst_unbind_listener(listener);
|
2007-10-16 15:34:28 +00:00
|
|
|
return ERR_NONE;
|
|
|
|
}
|
|
|
|
|
2007-10-28 20:59:24 +00:00
|
|
|
|
|
|
|
/********************************
|
|
|
|
* 4) high-level functions
|
|
|
|
********************************/
|
|
|
|
|
|
|
|
|
2007-10-16 15:34:28 +00:00
|
|
|
/*
|
|
|
|
* This function is called on a read event from a listen socket, corresponding
|
|
|
|
* to an accept. It tries to accept as many connections as possible.
|
|
|
|
* It returns 0. Since we use UNIX sockets on the local system for monitoring
|
|
|
|
* purposes and other related things, we do not need to output as many messages
|
|
|
|
* as with TCP which can fall under attack.
|
|
|
|
*/
|
|
|
|
int uxst_event_accept(int fd) {
|
2008-08-29 21:36:51 +00:00
|
|
|
struct listener *l = fdtab[fd].owner;
|
2007-10-16 15:34:28 +00:00
|
|
|
struct session *s;
|
|
|
|
struct task *t;
|
|
|
|
int cfd;
|
|
|
|
int max_accept;
|
|
|
|
|
|
|
|
if (global.nbproc > 1)
|
|
|
|
max_accept = 8; /* let other processes catch some connections too */
|
|
|
|
else
|
|
|
|
max_accept = -1;
|
|
|
|
|
|
|
|
while (max_accept--) {
|
|
|
|
struct sockaddr_storage addr;
|
|
|
|
socklen_t laddr = sizeof(addr);
|
|
|
|
|
|
|
|
if ((cfd = accept(fd, (struct sockaddr *)&addr, &laddr)) == -1) {
|
|
|
|
switch (errno) {
|
|
|
|
case EAGAIN:
|
|
|
|
case EINTR:
|
|
|
|
case ECONNABORTED:
|
|
|
|
return 0; /* nothing more to accept */
|
|
|
|
case ENFILE:
|
|
|
|
/* Process reached system FD limit. Check system tunables. */
|
|
|
|
return 0;
|
|
|
|
case EMFILE:
|
|
|
|
/* Process reached process FD limit. Check 'ulimit-n'. */
|
|
|
|
return 0;
|
|
|
|
case ENOBUFS:
|
|
|
|
case ENOMEM:
|
|
|
|
/* Process reached system memory limit. Check system tunables. */
|
|
|
|
return 0;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (l->nbconn >= l->maxconn) {
|
|
|
|
/* too many connections, we shoot this one and return.
|
|
|
|
* FIXME: it would be better to simply switch the listener's
|
|
|
|
* state to LI_FULL and disable the FD. We could re-enable
|
|
|
|
* it upon fd_delete(), but this requires all protocols to
|
|
|
|
* be switched.
|
|
|
|
*/
|
|
|
|
close(cfd);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((s = pool_alloc2(pool2_session)) == NULL) {
|
|
|
|
Alert("out of memory in uxst_event_accept().\n");
|
|
|
|
close(cfd);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
s->flags = 0;
|
2008-08-16 12:55:08 +00:00
|
|
|
s->term_trace = 0;
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
|
2007-10-16 15:34:28 +00:00
|
|
|
if ((t = pool_alloc2(pool2_task)) == NULL) {
|
|
|
|
Alert("out of memory in uxst_event_accept().\n");
|
|
|
|
close(cfd);
|
|
|
|
pool_free2(pool2_session, s);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
s->cli_addr = addr;
|
|
|
|
|
|
|
|
/* FIXME: should be checked earlier */
|
|
|
|
if (cfd >= global.maxsock) {
|
|
|
|
Alert("accept(): not enough free sockets. Raise -n argument. Giving up.\n");
|
|
|
|
close(cfd);
|
|
|
|
pool_free2(pool2_task, t);
|
|
|
|
pool_free2(pool2_session, s);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fcntl(cfd, F_SETFL, O_NONBLOCK) == -1) {
|
|
|
|
Alert("accept(): cannot set the socket in non blocking mode. Giving up\n");
|
|
|
|
close(cfd);
|
|
|
|
pool_free2(pool2_task, t);
|
|
|
|
pool_free2(pool2_session, s);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-06-24 06:17:16 +00:00
|
|
|
task_init(t);
|
2007-10-16 15:34:28 +00:00
|
|
|
t->process = l->handler;
|
|
|
|
t->context = s;
|
2008-06-30 05:51:00 +00:00
|
|
|
t->nice = -64; /* we want to boost priority for local stats */
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
s->task = t;
|
|
|
|
s->fe = NULL;
|
|
|
|
s->be = NULL;
|
|
|
|
|
|
|
|
s->cli_state = CL_STDATA;
|
|
|
|
s->srv_state = SV_STIDLE;
|
|
|
|
s->req = s->rep = NULL; /* will be allocated later */
|
|
|
|
|
|
|
|
s->cli_fd = cfd;
|
|
|
|
s->srv = NULL;
|
|
|
|
s->pend_pos = NULL;
|
|
|
|
|
|
|
|
memset(&s->logs, 0, sizeof(s->logs));
|
|
|
|
memset(&s->txn, 0, sizeof(s->txn));
|
|
|
|
|
2007-10-17 16:57:38 +00:00
|
|
|
s->data_state = DATA_ST_INIT;
|
2007-10-16 15:34:28 +00:00
|
|
|
s->data_source = DATA_SRC_NONE;
|
|
|
|
s->uniq_id = totalconn;
|
|
|
|
|
|
|
|
if ((s->req = pool_alloc2(pool2_buffer)) == NULL) { /* no memory */
|
|
|
|
close(cfd); /* nothing can be done for this fd without memory */
|
|
|
|
pool_free2(pool2_task, t);
|
|
|
|
pool_free2(pool2_session, s);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((s->rep = pool_alloc2(pool2_buffer)) == NULL) { /* no memory */
|
|
|
|
pool_free2(pool2_buffer, s->req);
|
|
|
|
close(cfd); /* nothing can be done for this fd without memory */
|
|
|
|
pool_free2(pool2_task, t);
|
|
|
|
pool_free2(pool2_session, s);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
buffer_init(s->req);
|
|
|
|
buffer_init(s->rep);
|
|
|
|
|
|
|
|
fd_insert(cfd);
|
|
|
|
fdtab[cfd].owner = t;
|
|
|
|
fdtab[cfd].listener = l;
|
|
|
|
fdtab[cfd].state = FD_STREADY;
|
|
|
|
fdtab[cfd].cb[DIR_RD].f = l->proto->read;
|
|
|
|
fdtab[cfd].cb[DIR_RD].b = s->req;
|
|
|
|
fdtab[cfd].cb[DIR_WR].f = l->proto->write;
|
|
|
|
fdtab[cfd].cb[DIR_WR].b = s->rep;
|
|
|
|
fdtab[cfd].peeraddr = (struct sockaddr *)&s->cli_addr;
|
|
|
|
fdtab[cfd].peerlen = sizeof(s->cli_addr);
|
|
|
|
|
2008-07-06 22:09:58 +00:00
|
|
|
s->req->rex = TICK_ETERNITY;
|
|
|
|
s->req->wex = TICK_ETERNITY;
|
2008-08-17 16:03:28 +00:00
|
|
|
s->req->analyse_exp = TICK_ETERNITY;
|
2008-07-06 22:09:58 +00:00
|
|
|
s->rep->rex = TICK_ETERNITY;
|
|
|
|
s->rep->wex = TICK_ETERNITY;
|
2008-08-17 16:03:28 +00:00
|
|
|
s->rep->analyse_exp = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
|
2008-07-06 22:09:58 +00:00
|
|
|
s->req->wto = TICK_ETERNITY;
|
|
|
|
s->req->cto = TICK_ETERNITY;
|
|
|
|
s->req->rto = TICK_ETERNITY;
|
|
|
|
s->rep->rto = TICK_ETERNITY;
|
|
|
|
s->rep->cto = TICK_ETERNITY;
|
|
|
|
s->rep->wto = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
if (l->timeout)
|
|
|
|
s->req->rto = *l->timeout;
|
|
|
|
|
|
|
|
if (l->timeout)
|
|
|
|
s->rep->wto = *l->timeout;
|
|
|
|
|
2008-07-06 22:09:58 +00:00
|
|
|
t->expire = TICK_ETERNITY;
|
|
|
|
if (l->timeout && *l->timeout) {
|
2007-10-16 15:34:28 +00:00
|
|
|
EV_FD_SET(cfd, DIR_RD);
|
2008-07-06 22:09:58 +00:00
|
|
|
s->req->rex = tick_add(now_ms, s->req->rto);
|
2007-10-16 15:34:28 +00:00
|
|
|
t->expire = s->req->rex;
|
|
|
|
}
|
|
|
|
|
2008-08-29 16:19:04 +00:00
|
|
|
task_wakeup(t, TASK_WOKEN_INIT);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
l->nbconn++; /* warning! right now, it's up to the handler to decrease this */
|
|
|
|
if (l->nbconn >= l->maxconn) {
|
|
|
|
EV_FD_CLR(l->fd, DIR_RD);
|
|
|
|
l->state = LI_FULL;
|
|
|
|
}
|
|
|
|
actconn++;
|
|
|
|
totalconn++;
|
|
|
|
|
|
|
|
//fprintf(stderr, "accepting from %p => %d conn, %d total, task=%p, cfd=%d, maxfd=%d\n", p, actconn, totalconn, t, cfd, maxfd);
|
|
|
|
} /* end of while (p->feconn < p->maxconn) */
|
|
|
|
//fprintf(stderr,"fct %s:%d\n", __FUNCTION__, __LINE__);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* manages the client FSM and its socket. It returns 1 if a state has changed
|
|
|
|
* (and a resync may be needed), otherwise 0.
|
|
|
|
*/
|
|
|
|
static int process_uxst_cli(struct session *t)
|
|
|
|
{
|
|
|
|
int s = t->srv_state;
|
|
|
|
int c = t->cli_state;
|
|
|
|
struct buffer *req = t->req;
|
|
|
|
struct buffer *rep = t->rep;
|
|
|
|
//fprintf(stderr,"fct %s:%d\n", __FUNCTION__, __LINE__);
|
|
|
|
if (c == CL_STDATA) {
|
|
|
|
/* FIXME: this error handling is partly buggy because we always report
|
|
|
|
* a 'DATA' phase while we don't know if the server was in IDLE, CONN
|
|
|
|
* or HEADER phase. BTW, it's not logical to expire the client while
|
|
|
|
* we're waiting for the server to connect.
|
|
|
|
*/
|
|
|
|
/* read or write error */
|
|
|
|
if (rep->flags & BF_WRITE_ERROR || req->flags & BF_READ_ERROR) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(req);
|
|
|
|
buffer_shutw(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLICL;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* last read, or end of server write */
|
|
|
|
else if (req->flags & BF_READ_NULL || s == SV_STSHUTW || s == SV_STCLOSE) {
|
|
|
|
EV_FD_CLR(t->cli_fd, DIR_RD);
|
|
|
|
buffer_shutr(req);
|
|
|
|
t->cli_state = CL_STSHUTR;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* last server read and buffer empty */
|
2008-08-16 20:18:07 +00:00
|
|
|
else if ((s == SV_STSHUTR || s == SV_STCLOSE) && (rep->flags & BF_EMPTY)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
EV_FD_CLR(t->cli_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
shutdown(t->cli_fd, SHUT_WR);
|
|
|
|
/* We must ensure that the read part is still alive when switching
|
|
|
|
* to shutw */
|
|
|
|
EV_FD_SET(t->cli_fd, DIR_RD);
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = tick_add_ifset(now_ms, req->rto);
|
2007-10-16 15:34:28 +00:00
|
|
|
t->cli_state = CL_STSHUTW;
|
|
|
|
//fprintf(stderr,"%p:%s(%d), c=%d, s=%d\n", t, __FUNCTION__, __LINE__, t->cli_state, t->cli_state);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* read timeout */
|
2008-07-06 22:09:58 +00:00
|
|
|
else if (tick_is_expired(req->rex, now_ms)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
EV_FD_CLR(t->cli_fd, DIR_RD);
|
|
|
|
buffer_shutr(req);
|
|
|
|
t->cli_state = CL_STSHUTR;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLITO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* write timeout */
|
2008-07-06 22:09:58 +00:00
|
|
|
else if (tick_is_expired(rep->wex, now_ms)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
EV_FD_CLR(t->cli_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
shutdown(t->cli_fd, SHUT_WR);
|
|
|
|
/* We must ensure that the read part is still alive when switching
|
|
|
|
* to shutw */
|
|
|
|
EV_FD_SET(t->cli_fd, DIR_RD);
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = tick_add_ifset(now_ms, req->rto);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
t->cli_state = CL_STSHUTW;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLITO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2008-08-16 20:18:07 +00:00
|
|
|
if (req->flags & BF_FULL) {
|
2007-10-16 15:34:28 +00:00
|
|
|
/* no room to read more data */
|
|
|
|
if (EV_FD_COND_C(t->cli_fd, DIR_RD)) {
|
|
|
|
/* stop reading until we get some space */
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* there's still some space in the buffer */
|
|
|
|
if (EV_FD_COND_S(t->cli_fd, DIR_RD)) {
|
2008-07-06 22:09:58 +00:00
|
|
|
if (!req->rto ||
|
|
|
|
(t->srv_state < SV_STDATA && req->wto))
|
2007-10-16 15:34:28 +00:00
|
|
|
/* If the client has no timeout, or if the server not ready yet, and we
|
|
|
|
* know for sure that it can expire, then it's cleaner to disable the
|
|
|
|
* timeout on the client side so that too low values cannot make the
|
|
|
|
* sessions abort too early.
|
|
|
|
*/
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
else
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = tick_add(now_ms, req->rto);
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-08-16 20:18:07 +00:00
|
|
|
if ((rep->flags & BF_EMPTY) ||
|
2007-10-16 15:34:28 +00:00
|
|
|
((s < SV_STDATA) /* FIXME: this may be optimized && (rep->w == rep->h)*/)) {
|
|
|
|
if (EV_FD_COND_C(t->cli_fd, DIR_WR)) {
|
|
|
|
/* stop writing */
|
2008-07-06 22:09:58 +00:00
|
|
|
rep->wex = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* buffer not empty */
|
|
|
|
if (EV_FD_COND_S(t->cli_fd, DIR_WR)) {
|
|
|
|
/* restart writing */
|
2008-07-06 22:09:58 +00:00
|
|
|
rep->wex = tick_add_ifset(now_ms, rep->wto);
|
|
|
|
if (rep->wex) {
|
2007-10-16 15:34:28 +00:00
|
|
|
/* FIXME: to prevent the client from expiring read timeouts during writes,
|
|
|
|
* we refresh it. */
|
|
|
|
req->rex = rep->wex;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0; /* other cases change nothing */
|
|
|
|
}
|
|
|
|
else if (c == CL_STSHUTR) {
|
|
|
|
if (rep->flags & BF_WRITE_ERROR) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLICL;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
2008-08-16 20:18:07 +00:00
|
|
|
else if ((s == SV_STSHUTR || s == SV_STCLOSE) && (rep->flags & BF_EMPTY)) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
return 1;
|
|
|
|
}
|
2008-07-06 22:09:58 +00:00
|
|
|
else if (tick_is_expired(rep->wex, now_ms)) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLITO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2008-08-16 20:18:07 +00:00
|
|
|
if (rep->flags & BF_EMPTY) {
|
2007-10-16 15:34:28 +00:00
|
|
|
if (EV_FD_COND_C(t->cli_fd, DIR_WR)) {
|
|
|
|
/* stop writing */
|
2008-07-06 22:09:58 +00:00
|
|
|
rep->wex = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* buffer not empty */
|
|
|
|
if (EV_FD_COND_S(t->cli_fd, DIR_WR)) {
|
|
|
|
/* restart writing */
|
2008-07-06 22:09:58 +00:00
|
|
|
rep->wex = tick_add_ifset(now_ms, rep->wto);
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
else if (c == CL_STSHUTW) {
|
|
|
|
if (req->flags & BF_READ_ERROR) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLICL;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else if (req->flags & BF_READ_NULL || s == SV_STSHUTW || s == SV_STCLOSE) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
return 1;
|
|
|
|
}
|
2008-07-06 22:09:58 +00:00
|
|
|
else if (tick_is_expired(req->rex, now_ms)) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->cli_fd);
|
|
|
|
t->cli_state = CL_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_CLITO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK)) {
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->flags |= SN_FINST_Q;
|
|
|
|
else if (s == SV_STCONN)
|
|
|
|
t->flags |= SN_FINST_C;
|
|
|
|
else
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
2008-08-16 20:18:07 +00:00
|
|
|
else if (req->flags & BF_FULL) {
|
2007-10-16 15:34:28 +00:00
|
|
|
/* no room to read more data */
|
|
|
|
|
|
|
|
/* FIXME-20050705: is it possible for a client to maintain a session
|
|
|
|
* after the timeout by sending more data after it receives a close ?
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (EV_FD_COND_C(t->cli_fd, DIR_RD)) {
|
|
|
|
/* stop reading until we get some space */
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* there's still some space in the buffer */
|
|
|
|
if (EV_FD_COND_S(t->cli_fd, DIR_RD)) {
|
2008-07-06 22:09:58 +00:00
|
|
|
req->rex = tick_add_ifset(now_ms, req->rto);
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
else { /* CL_STCLOSE: nothing to do */
|
|
|
|
if ((global.mode & MODE_DEBUG) && (!(global.mode & MODE_QUIET) || (global.mode & MODE_VERBOSE))) {
|
|
|
|
int len;
|
|
|
|
len = sprintf(trash, "%08x:%s.clicls[%04x:%04x]\n", t->uniq_id, t->be?t->be->id:"",
|
[MAJOR] rework of the server FSM
srv_state has been removed from HTTP state machines, and states
have been split in either TCP states or analyzers. For instance,
the TARPIT state has just become a simple analyzer.
New flags have been added to the struct buffer to compensate this.
The high-level stream processors sometimes need to force a disconnection
without touching a file-descriptor (eg: report an error). But if
they touched BF_SHUTW or BF_SHUTR, the file descriptor would not
be closed. Thus, the two SHUT?_NOW flags have been added so that
an application can request a forced close which the stream interface
will be forced to obey.
During this change, a new BF_HIJACK flag was added. It will
be used for data generation, eg during a stats dump. It
prevents the producer on a buffer from sending data into it.
BF_SHUTR_NOW /* the producer must shut down for reads ASAP */
BF_SHUTW_NOW /* the consumer must shut down for writes ASAP */
BF_HIJACK /* the producer is temporarily replaced */
BF_SHUTW_NOW has precedence over BF_HIJACK. BF_HIJACK has
precedence over BF_MAY_FORWARD (so that it does not need it).
New functions buffer_shutr_now(), buffer_shutw_now(), buffer_abort()
are provided to manipulate BF_SHUT* flags.
A new type "stream_interface" has been added to describe both
sides of a buffer. A stream interface has states and error
reporting. The session now has two stream interfaces (one per
side). Each buffer has stream_interface pointers to both
consumer and producer sides.
The server-side file descriptor has moved to its stream interface,
so that even the buffer has access to it.
process_srv() has been split into three parts :
- tcp_get_connection() obtains a connection to the server
- tcp_connection_failed() tests if a previously attempted
connection has succeeded or not.
- process_srv_data() only manages the data phase, and in
this sense should be roughly equivalent to process_cli.
Little code has been removed, and a lot of old code has been
left in comments for now.
2008-10-19 05:30:41 +00:00
|
|
|
(unsigned short)t->cli_fd, (unsigned short)t->req->cons->fd);
|
2007-10-16 15:34:28 +00:00
|
|
|
write(1, trash, len);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if 0
|
|
|
|
/* FIXME! This part has not been completely converted yet, and it may
|
|
|
|
* still be very specific to TCPv4 ! Also, it relies on some parameters
|
|
|
|
* such as conn_retries which are not set upon accept().
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* Manages the server FSM and its socket. It returns 1 if a state has changed
|
|
|
|
* (and a resync may be needed), otherwise 0.
|
|
|
|
*/
|
|
|
|
static int process_uxst_srv(struct session *t)
|
|
|
|
{
|
|
|
|
int s = t->srv_state;
|
|
|
|
int c = t->cli_state;
|
|
|
|
struct buffer *req = t->req;
|
|
|
|
struct buffer *rep = t->rep;
|
|
|
|
int conn_err;
|
|
|
|
|
|
|
|
if (s == SV_STIDLE) {
|
|
|
|
if (c == CL_STCLOSE || c == CL_STSHUTW ||
|
|
|
|
(c == CL_STSHUTR &&
|
2008-08-16 20:18:07 +00:00
|
|
|
(t->req->flags & BF_EMPTY || t->be->options & PR_O_ABRT_CLOSE))) { /* give up */
|
2007-10-16 15:34:28 +00:00
|
|
|
tv_eternity(&req->cex);
|
|
|
|
if (t->pend_pos)
|
|
|
|
t->logs.t_queue = tv_ms_elapsed(&t->logs.tv_accept, &now);
|
|
|
|
srv_close_with_err(t, SN_ERR_CLICL, t->pend_pos ? SN_FINST_Q : SN_FINST_C);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
/* FIXME: reimplement the TARPIT check here */
|
|
|
|
|
|
|
|
/* Right now, we will need to create a connection to the server.
|
|
|
|
* We might already have tried, and got a connection pending, in
|
|
|
|
* which case we will not do anything till it's pending. It's up
|
|
|
|
* to any other session to release it and wake us up again.
|
|
|
|
*/
|
|
|
|
if (t->pend_pos) {
|
|
|
|
if (!tv_isle(&req->cex, &now))
|
|
|
|
return 0;
|
|
|
|
else {
|
|
|
|
/* we've been waiting too long here */
|
|
|
|
tv_eternity(&req->cex);
|
|
|
|
t->logs.t_queue = tv_ms_elapsed(&t->logs.tv_accept, &now);
|
|
|
|
srv_close_with_err(t, SN_ERR_SRVTO, SN_FINST_Q);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->failed_conns++;
|
|
|
|
if (t->fe)
|
|
|
|
t->fe->failed_conns++;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
do {
|
|
|
|
/* first, get a connection */
|
|
|
|
if (srv_redispatch_connect(t))
|
|
|
|
return t->srv_state != SV_STIDLE;
|
|
|
|
|
|
|
|
/* try to (re-)connect to the server, and fail if we expire the
|
|
|
|
* number of retries.
|
|
|
|
*/
|
|
|
|
if (srv_retryable_connect(t)) {
|
|
|
|
t->logs.t_queue = tv_ms_elapsed(&t->logs.tv_accept, &now);
|
|
|
|
return t->srv_state != SV_STIDLE;
|
|
|
|
}
|
|
|
|
} while (1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (s == SV_STCONN) { /* connection in progress */
|
|
|
|
if (c == CL_STCLOSE || c == CL_STSHUTW ||
|
|
|
|
(c == CL_STSHUTR &&
|
2008-08-29 07:58:42 +00:00
|
|
|
((t->req->flags & BF_EMPTY && !(req->flags & BF_WRITE_ACTIVITY)) ||
|
2007-10-16 15:34:28 +00:00
|
|
|
t->be->options & PR_O_ABRT_CLOSE))) { /* give up */
|
|
|
|
tv_eternity(&req->cex);
|
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
|
|
|
|
srv_close_with_err(t, SN_ERR_CLICL, SN_FINST_C);
|
|
|
|
return 1;
|
|
|
|
}
|
2008-08-29 07:58:42 +00:00
|
|
|
if (!(req->flags & BF_WRITE_ACTIVITY) && !tv_isle(&req->cex, &now)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
//fprintf(stderr,"1: c=%d, s=%d, now=%d.%06d, exp=%d.%06d\n", c, s, now.tv_sec, now.tv_usec, req->cex.tv_sec, req->cex.tv_usec);
|
|
|
|
return 0; /* nothing changed */
|
|
|
|
}
|
2008-08-29 07:58:42 +00:00
|
|
|
else if (!(req->flags & BF_WRITE_ACTIVITY) || (req->flags & BF_WRITE_ERROR)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
/* timeout, asynchronous connect error or first write error */
|
|
|
|
//fprintf(stderr,"2: c=%d, s=%d\n", c, s);
|
|
|
|
|
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
|
2008-08-29 07:58:42 +00:00
|
|
|
if (!(req->flags & BF_WRITE_ACTIVITY))
|
2007-10-16 15:34:28 +00:00
|
|
|
conn_err = SN_ERR_SRVTO; // it was a connect timeout.
|
|
|
|
else
|
|
|
|
conn_err = SN_ERR_SRVCL; // it was an asynchronous connect error.
|
|
|
|
|
|
|
|
/* ensure that we have enough retries left */
|
|
|
|
if (srv_count_retry_down(t, conn_err))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
if (t->srv && t->conn_retries == 0 && t->be->options & PR_O_REDISP) {
|
|
|
|
/* We're on our last chance, and the REDISP option was specified.
|
|
|
|
* We will ignore cookie and force to balance or use the dispatcher.
|
|
|
|
*/
|
|
|
|
/* let's try to offer this slot to anybody */
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->failed_conns++;
|
|
|
|
t->be->failed_conns++;
|
|
|
|
|
|
|
|
t->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
|
|
|
|
t->srv = NULL; /* it's left to the dispatcher to choose a server */
|
|
|
|
|
|
|
|
/* first, get a connection */
|
|
|
|
if (srv_redispatch_connect(t))
|
|
|
|
return t->srv_state != SV_STIDLE;
|
|
|
|
}
|
|
|
|
|
|
|
|
do {
|
|
|
|
/* Now we will try to either reconnect to the same server or
|
|
|
|
* connect to another server. If the connection gets queued
|
|
|
|
* because all servers are saturated, then we will go back to
|
|
|
|
* the SV_STIDLE state.
|
|
|
|
*/
|
|
|
|
if (srv_retryable_connect(t)) {
|
|
|
|
t->logs.t_queue = tv_ms_elapsed(&t->logs.tv_accept, &now);
|
|
|
|
return t->srv_state != SV_STCONN;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* we need to redispatch the connection to another server */
|
|
|
|
if (srv_redispatch_connect(t))
|
|
|
|
return t->srv_state != SV_STCONN;
|
|
|
|
} while (1);
|
|
|
|
}
|
|
|
|
else { /* no error or write 0 */
|
|
|
|
t->logs.t_connect = tv_ms_elapsed(&t->logs.tv_accept, &now);
|
|
|
|
|
|
|
|
//fprintf(stderr,"3: c=%d, s=%d\n", c, s);
|
2008-08-16 20:18:07 +00:00
|
|
|
if (req->flags & BF_EMPTY) /* nothing to write */ {
|
2007-10-16 15:34:28 +00:00
|
|
|
EV_FD_CLR(t->srv_fd, DIR_WR);
|
|
|
|
tv_eternity(&req->wex);
|
|
|
|
} else /* need the right to write */ {
|
|
|
|
EV_FD_SET(t->srv_fd, DIR_WR);
|
|
|
|
if (tv_add_ifset(&req->wex, &now, &req->wto)) {
|
|
|
|
/* FIXME: to prevent the server from expiring read timeouts during writes,
|
|
|
|
* we refresh it. */
|
|
|
|
rep->rex = req->wex;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
tv_eternity(&req->wex);
|
|
|
|
}
|
|
|
|
|
|
|
|
EV_FD_SET(t->srv_fd, DIR_RD);
|
|
|
|
if (!tv_add_ifset(&rep->rex, &now, &rep->rto))
|
|
|
|
tv_eternity(&rep->rex);
|
|
|
|
|
|
|
|
t->srv_state = SV_STDATA;
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cum_sess++;
|
2008-08-16 20:18:07 +00:00
|
|
|
buffer_set_rlim(rep, BUFSIZE); /* no rewrite needed */
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
/* if the user wants to log as soon as possible, without counting
|
|
|
|
bytes from the server, then this is the right moment. */
|
|
|
|
if (t->fe && t->fe->to_log && !(t->logs.logwait & LW_BYTES)) {
|
|
|
|
t->logs.t_close = t->logs.t_connect; /* to get a valid end date */
|
|
|
|
//uxst_sess_log(t);
|
|
|
|
}
|
|
|
|
tv_eternity(&req->cex);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (s == SV_STDATA) {
|
|
|
|
/* read or write error */
|
|
|
|
if (req->flags & BF_WRITE_ERROR || rep->flags & BF_READ_ERROR) {
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(rep);
|
|
|
|
buffer_shutw(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv) {
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
t->srv->failed_resp++;
|
|
|
|
}
|
|
|
|
t->be->failed_resp++;
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVCL;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* last read, or end of client write */
|
|
|
|
else if (rep->flags & BF_READ_NULL || c == CL_STSHUTW || c == CL_STCLOSE) {
|
|
|
|
EV_FD_CLR(t->srv_fd, DIR_RD);
|
|
|
|
buffer_shutr(rep);
|
|
|
|
t->srv_state = SV_STSHUTR;
|
|
|
|
//fprintf(stderr,"%p:%s(%d), c=%d, s=%d\n", t, __FUNCTION__, __LINE__, t->cli_state, t->cli_state);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* end of client read and no more data to send */
|
2008-08-16 20:18:07 +00:00
|
|
|
else if ((c == CL_STSHUTR || c == CL_STCLOSE) && (req->flags & BF_EMPTY)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
EV_FD_CLR(t->srv_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
shutdown(t->srv_fd, SHUT_WR);
|
|
|
|
/* We must ensure that the read part is still alive when switching
|
|
|
|
* to shutw */
|
|
|
|
EV_FD_SET(t->srv_fd, DIR_RD);
|
|
|
|
tv_add_ifset(&rep->rex, &now, &rep->rto);
|
|
|
|
|
|
|
|
t->srv_state = SV_STSHUTW;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* read timeout */
|
|
|
|
else if (tv_isle(&rep->rex, &now)) {
|
|
|
|
EV_FD_CLR(t->srv_fd, DIR_RD);
|
|
|
|
buffer_shutr(rep);
|
|
|
|
t->srv_state = SV_STSHUTR;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVTO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
/* write timeout */
|
|
|
|
else if (tv_isle(&req->wex, &now)) {
|
|
|
|
EV_FD_CLR(t->srv_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
shutdown(t->srv_fd, SHUT_WR);
|
|
|
|
/* We must ensure that the read part is still alive when switching
|
|
|
|
* to shutw */
|
|
|
|
EV_FD_SET(t->srv_fd, DIR_RD);
|
|
|
|
tv_add_ifset(&rep->rex, &now, &rep->rto);
|
|
|
|
t->srv_state = SV_STSHUTW;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVTO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* recompute request time-outs */
|
2008-08-16 20:18:07 +00:00
|
|
|
if (req->flags & BF_EMPTY) {
|
2007-10-16 15:34:28 +00:00
|
|
|
if (EV_FD_COND_C(t->srv_fd, DIR_WR)) {
|
|
|
|
/* stop writing */
|
|
|
|
tv_eternity(&req->wex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else { /* buffer not empty, there are still data to be transferred */
|
|
|
|
if (EV_FD_COND_S(t->srv_fd, DIR_WR)) {
|
|
|
|
/* restart writing */
|
|
|
|
if (tv_add_ifset(&req->wex, &now, &req->wto)) {
|
|
|
|
/* FIXME: to prevent the server from expiring read timeouts during writes,
|
|
|
|
* we refresh it. */
|
|
|
|
rep->rex = req->wex;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
tv_eternity(&req->wex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* recompute response time-outs */
|
|
|
|
if (rep->l == BUFSIZE) { /* no room to read more data */
|
|
|
|
if (EV_FD_COND_C(t->srv_fd, DIR_RD)) {
|
|
|
|
tv_eternity(&rep->rex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
if (EV_FD_COND_S(t->srv_fd, DIR_RD)) {
|
|
|
|
if (!tv_add_ifset(&rep->rex, &now, &rep->rto))
|
|
|
|
tv_eternity(&rep->rex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0; /* other cases change nothing */
|
|
|
|
}
|
|
|
|
else if (s == SV_STSHUTR) {
|
|
|
|
if (req->flags & BF_WRITE_ERROR) {
|
|
|
|
//EV_FD_CLR(t->srv_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv) {
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
t->srv->failed_resp++;
|
|
|
|
}
|
|
|
|
t->be->failed_resp++;
|
|
|
|
//close(t->srv_fd);
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVCL;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
2008-08-16 20:18:07 +00:00
|
|
|
else if ((c == CL_STSHUTR || c == CL_STCLOSE) && (req->flags & BF_EMPTY)) {
|
2007-10-16 15:34:28 +00:00
|
|
|
//EV_FD_CLR(t->srv_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
//close(t->srv_fd);
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else if (tv_isle(&req->wex, &now)) {
|
|
|
|
//EV_FD_CLR(t->srv_fd, DIR_WR);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutw(req);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
//close(t->srv_fd);
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVTO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
2008-08-16 20:18:07 +00:00
|
|
|
else if (req->flags & BF_EMPTY) {
|
2007-10-16 15:34:28 +00:00
|
|
|
if (EV_FD_COND_C(t->srv_fd, DIR_WR)) {
|
|
|
|
/* stop writing */
|
|
|
|
tv_eternity(&req->wex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else { /* buffer not empty */
|
|
|
|
if (EV_FD_COND_S(t->srv_fd, DIR_WR)) {
|
|
|
|
/* restart writing */
|
|
|
|
if (!tv_add_ifset(&req->wex, &now, &req->wto))
|
|
|
|
tv_eternity(&req->wex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
else if (s == SV_STSHUTW) {
|
|
|
|
if (rep->flags & BF_READ_ERROR) {
|
|
|
|
//EV_FD_CLR(t->srv_fd, DIR_RD);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv) {
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
t->srv->failed_resp++;
|
|
|
|
}
|
|
|
|
t->be->failed_resp++;
|
|
|
|
//close(t->srv_fd);
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVCL;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else if (rep->flags & BF_READ_NULL || c == CL_STSHUTW || c == CL_STCLOSE) {
|
|
|
|
//EV_FD_CLR(t->srv_fd, DIR_RD);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
//close(t->srv_fd);
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else if (tv_isle(&rep->rex, &now)) {
|
|
|
|
//EV_FD_CLR(t->srv_fd, DIR_RD);
|
2008-08-16 19:13:23 +00:00
|
|
|
buffer_shutr(rep);
|
2007-10-16 15:34:28 +00:00
|
|
|
fd_delete(t->srv_fd);
|
|
|
|
if (t->srv)
|
|
|
|
t->srv->cur_sess--;
|
|
|
|
//close(t->srv_fd);
|
|
|
|
t->srv_state = SV_STCLOSE;
|
|
|
|
if (!(t->flags & SN_ERR_MASK))
|
|
|
|
t->flags |= SN_ERR_SRVTO;
|
|
|
|
if (!(t->flags & SN_FINST_MASK))
|
|
|
|
t->flags |= SN_FINST_D;
|
|
|
|
/* We used to have a free connection slot. Since we'll never use it,
|
|
|
|
* we have to inform the server that it may be used by another session.
|
|
|
|
*/
|
|
|
|
if (may_dequeue_tasks(t->srv, t->be))
|
[BUG] fix the dequeuing logic to ensure that all requests get served
The dequeuing logic was completely wrong. First, a task was assigned
to all servers to process the queue, but this task was never scheduled
and was only woken up on session free. Second, there was no reservation
of server entries when a task was assigned a server. This means that
as long as the task was not connected to the server, its presence was
not accounted for. This was causing trouble when detecting whether or
not a server had reached maxconn. Third, during a redispatch, a session
could lose its place at the server's and get blocked because another
session at the same moment would have stolen the entry. Fourth, the
redispatch option did not work when maxqueue was reached for a server,
and it was not possible to do so without indefinitely hanging a session.
The root cause of all those problems was the lack of pre-reservation of
connections at the server's, and the lack of tracking of servers during
a redispatch. Everything relied on combinations of flags which could
appear similarly in quite distinct situations.
This patch is a major rework but there was no other solution, as the
internal logic was deeply flawed. The resulting code is cleaner, more
understandable, uses less magics and is overall more robust.
As an added bonus, "option redispatch" now works when maxqueue has
been reached on a server.
2008-06-20 13:04:11 +00:00
|
|
|
process_srv_queue(t->srv);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else if (rep->l == BUFSIZE) { /* no room to read more data */
|
|
|
|
if (EV_FD_COND_C(t->srv_fd, DIR_RD)) {
|
|
|
|
tv_eternity(&rep->rex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
if (EV_FD_COND_S(t->srv_fd, DIR_RD)) {
|
|
|
|
if (!tv_add_ifset(&rep->rex, &now, &rep->rto))
|
|
|
|
tv_eternity(&rep->rex);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
else { /* SV_STCLOSE : nothing to do */
|
|
|
|
if ((global.mode & MODE_DEBUG) && (!(global.mode & MODE_QUIET) || (global.mode & MODE_VERBOSE))) {
|
|
|
|
int len;
|
|
|
|
len = sprintf(trash, "%08x:%s.srvcls[%04x:%04x]\n",
|
|
|
|
t->uniq_id, t->be->id, (unsigned short)t->cli_fd, (unsigned short)t->srv_fd);
|
|
|
|
write(1, trash, len);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Processes the client and server jobs of a session task, then
|
|
|
|
* puts it back to the wait queue in a clean state, or
|
|
|
|
* cleans up its resources if it must be deleted. Returns
|
|
|
|
* the time the task accepts to wait, or TIME_ETERNITY for
|
|
|
|
* infinity.
|
|
|
|
*/
|
2008-07-06 22:09:58 +00:00
|
|
|
void process_uxst_session(struct task *t, int *next)
|
2007-10-16 15:34:28 +00:00
|
|
|
{
|
|
|
|
struct session *s = t->context;
|
|
|
|
int fsm_resync = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
fsm_resync = 0;
|
|
|
|
fsm_resync |= process_uxst_cli(s);
|
|
|
|
if (s->srv_state == SV_STIDLE) {
|
|
|
|
if (s->cli_state == CL_STCLOSE || s->cli_state == CL_STSHUTW) {
|
|
|
|
s->srv_state = SV_STCLOSE;
|
|
|
|
fsm_resync |= 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (s->cli_state == CL_STSHUTR ||
|
2008-08-16 20:18:07 +00:00
|
|
|
(s->req->flags & BF_FULL)) {
|
|
|
|
if (s->req->flags & BF_EMPTY) {
|
2007-10-16 15:34:28 +00:00
|
|
|
s->srv_state = SV_STCLOSE;
|
|
|
|
fsm_resync |= 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
/* OK we have some remaining data to process */
|
|
|
|
/* Just as an exercice, we copy the req into the resp,
|
|
|
|
* and flush the req.
|
|
|
|
*/
|
|
|
|
memcpy(s->rep->data, s->req->data, sizeof(s->rep->data));
|
|
|
|
s->rep->l = s->req->l;
|
2008-08-16 20:18:07 +00:00
|
|
|
buffer_set_rlim(s->rep, BUFSIZE);
|
2007-10-16 15:34:28 +00:00
|
|
|
s->rep->w = s->rep->data;
|
|
|
|
s->rep->lr = s->rep->r = s->rep->data + s->rep->l;
|
|
|
|
|
|
|
|
s->req->l = 0;
|
|
|
|
s->srv_state = SV_STCLOSE;
|
|
|
|
|
|
|
|
fsm_resync |= 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} while (fsm_resync);
|
|
|
|
|
|
|
|
if (likely(s->cli_state != CL_STCLOSE || s->srv_state != SV_STCLOSE)) {
|
2007-11-24 21:12:47 +00:00
|
|
|
|
|
|
|
if ((s->fe->options & PR_O_CONTSTATS) && (s->flags & SN_BE_ASSIGNED))
|
|
|
|
session_process_counters(s);
|
|
|
|
|
2007-10-16 15:34:28 +00:00
|
|
|
s->req->flags &= BF_CLEAR_READ & BF_CLEAR_WRITE;
|
|
|
|
s->rep->flags &= BF_CLEAR_READ & BF_CLEAR_WRITE;
|
|
|
|
|
|
|
|
t->expire = s->req->rex;
|
|
|
|
tv_min(&t->expire, &s->req->rex, &s->req->wex);
|
|
|
|
tv_bound(&t->expire, &s->req->cex);
|
|
|
|
tv_bound(&t->expire, &s->rep->rex);
|
|
|
|
tv_bound(&t->expire, &s->rep->wex);
|
|
|
|
|
|
|
|
/* restore t to its place in the task list */
|
|
|
|
task_queue(t);
|
|
|
|
|
|
|
|
*next = t->expire;
|
|
|
|
return; /* nothing more to do */
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s->fe)
|
|
|
|
s->fe->feconn--;
|
|
|
|
if (s->be && (s->flags & SN_BE_ASSIGNED))
|
|
|
|
s->be->beconn--;
|
|
|
|
actconn--;
|
|
|
|
|
|
|
|
if (unlikely((global.mode & MODE_DEBUG) &&
|
|
|
|
(!(global.mode & MODE_QUIET) || (global.mode & MODE_VERBOSE)))) {
|
|
|
|
int len;
|
|
|
|
len = sprintf(trash, "%08x:%s.closed[%04x:%04x]\n",
|
|
|
|
s->uniq_id, s->be->id,
|
|
|
|
(unsigned short)s->cli_fd, (unsigned short)s->srv_fd);
|
|
|
|
write(1, trash, len);
|
|
|
|
}
|
|
|
|
|
|
|
|
s->logs.t_close = tv_ms_elapsed(&s->logs.tv_accept, &now);
|
2007-11-24 21:12:47 +00:00
|
|
|
session_process_counters(s);
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
/* let's do a final log if we need it */
|
|
|
|
if (s->logs.logwait &&
|
|
|
|
!(s->flags & SN_MONITOR) &&
|
|
|
|
(s->req->total || !(s->fe && s->fe->options & PR_O_NULLNOLOG))) {
|
|
|
|
//uxst_sess_log(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* the task MUST not be in the run queue anymore */
|
|
|
|
task_delete(t);
|
|
|
|
session_free(s);
|
|
|
|
task_free(t);
|
|
|
|
tv_eternity(next);
|
|
|
|
}
|
|
|
|
#endif /* not converted */
|
|
|
|
|
|
|
|
|
|
|
|
/* Processes data exchanges on the statistics socket. The client processing
|
|
|
|
* is called and the task is put back in the wait queue or it is cleared.
|
|
|
|
* In order to ease the transition, we simply simulate the server status
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
* for now. It only knows states SV_STIDLE, SV_STCONN, SV_STDATA, and
|
|
|
|
* SV_STCLOSE. Returns in <next> the task's expiration date.
|
2007-10-16 15:34:28 +00:00
|
|
|
*/
|
2008-07-06 22:09:58 +00:00
|
|
|
void process_uxst_stats(struct task *t, int *next)
|
2007-10-16 15:34:28 +00:00
|
|
|
{
|
|
|
|
struct session *s = t->context;
|
|
|
|
struct listener *listener;
|
|
|
|
int fsm_resync = 0;
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
int last_rep_l;
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
do {
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
char *args[MAX_UXST_ARGS + 1];
|
|
|
|
char *line, *p;
|
|
|
|
int arg;
|
|
|
|
|
2007-10-17 16:57:38 +00:00
|
|
|
fsm_resync = process_uxst_cli(s);
|
|
|
|
|
|
|
|
if (s->cli_state == CL_STCLOSE || s->cli_state == CL_STSHUTW) {
|
|
|
|
s->srv_state = SV_STCLOSE;
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
break;
|
2007-10-17 16:57:38 +00:00
|
|
|
}
|
|
|
|
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
switch (s->srv_state) {
|
|
|
|
case SV_STIDLE:
|
|
|
|
/* stats output not initialized yet */
|
|
|
|
memset(&s->data_ctx.stats, 0, sizeof(s->data_ctx.stats));
|
|
|
|
s->data_source = DATA_SRC_STATS;
|
|
|
|
s->srv_state = SV_STCONN;
|
|
|
|
fsm_resync |= 1;
|
|
|
|
break;
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
|
2008-08-11 13:24:42 +00:00
|
|
|
case SV_STCONN: /* should be changed to SV_STHEADERS or something more obvious */
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
/* stats initialized, but waiting for the command */
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
line = s->req->data;
|
|
|
|
p = memchr(line, '\n', s->req->l);
|
|
|
|
|
|
|
|
if (!p)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
*p = '\0';
|
|
|
|
|
|
|
|
while (isspace((unsigned char)*line))
|
|
|
|
line++;
|
|
|
|
|
|
|
|
arg = 0;
|
|
|
|
args[arg] = line;
|
|
|
|
|
|
|
|
while (*line && arg < MAX_UXST_ARGS) {
|
|
|
|
if (isspace((unsigned char)*line)) {
|
|
|
|
*line++ = '\0';
|
|
|
|
|
|
|
|
while (isspace((unsigned char)*line))
|
|
|
|
line++;
|
|
|
|
|
|
|
|
args[++arg] = line;
|
2008-01-03 09:19:15 +00:00
|
|
|
continue;
|
|
|
|
}
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
|
|
|
|
line++;
|
2008-01-03 09:19:15 +00:00
|
|
|
}
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
|
|
|
|
while (++arg <= MAX_UXST_ARGS)
|
|
|
|
args[arg] = line;
|
|
|
|
|
|
|
|
if (!strcmp(args[0], "show")) {
|
|
|
|
if (!strcmp(args[1], "stat")) {
|
|
|
|
if (*args[2] && *args[3] && *args[4]) {
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
s->data_ctx.stats.flags |= STAT_BOUND;
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
s->data_ctx.stats.iid = atoi(args[2]);
|
|
|
|
s->data_ctx.stats.type = atoi(args[3]);
|
|
|
|
s->data_ctx.stats.sid = atoi(args[4]);
|
|
|
|
}
|
|
|
|
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
s->data_ctx.stats.flags |= STAT_SHOW_STAT;
|
|
|
|
s->data_ctx.stats.flags |= STAT_FMT_CSV;
|
|
|
|
s->srv_state = SV_STDATA;
|
|
|
|
fsm_resync |= 1;
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!strcmp(args[1], "info")) {
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
s->data_ctx.stats.flags |= STAT_SHOW_INFO;
|
|
|
|
s->data_ctx.stats.flags |= STAT_FMT_CSV;
|
|
|
|
s->srv_state = SV_STDATA;
|
|
|
|
fsm_resync |= 1;
|
2007-10-16 15:34:28 +00:00
|
|
|
continue;
|
|
|
|
}
|
2007-10-17 16:57:38 +00:00
|
|
|
}
|
|
|
|
|
[MAJOR] proto_uxst rework -> SNMP support
Currently there is a ~16KB limit for a data size passed via unix socket.
It is caused by a trivial bug ttat is going to fixed soon, however
in most cases there is no need to dump a full stats.
This patch makes possible to select a scope of dumped data by extending
current "show stat" to "show stat [<iid> <type> <sid>]":
- iid is a proxy id, -1 to dump all proxies
- type selects type of dumpable objects: 1 for frontend, 2 for backend, 4 for
server, -1 for all types. Values can be ORed, for example:
1+2=3 -> frontend+backend.
1+2+4=7 -> frontend+backend+server.
- sid is a service id, -1 to dump everything from the selected proxy.
To do this I implemented a new session flag (SN_STAT_BOUND), added three
variables in data_ctx.stats (iid, type, sid), modified dumpstats.c and
completely revorked the process_uxst_stats: now it waits for a "\n"
terminated string, splits args and uses them. BTW: It should be quite easy
to add new commands, for example to enable/disable servers, the only problem
I can see is a not very lucky config name (*stats* socket). :|
During the work I also fixed two bug:
- s->flags were not initialized for proto_uxst
- missing comma if throttling not enabled (caused by a stupid change in
"Implement persistent id for proxies and servers")
Other changes:
- No more magic type valuse, use STATS_TYPE_FE/STATS_TYPE_BE/STATS_TYPE_SV
- Don't memset full s->data_ctx (it was clearing s->data_ctx.stats.{iid/type/sid},
instead initialize stats.sv & stats.sv_st (stats.px and stats.px_st were already
initialized)
With all that changes it was extremely easy to write a short perl plugin
for a perl-enabled net-snmp (also included in this patch).
29385 is my PEN (Private Enterprise Number) and I'm willing to donate
the SNMPv2-SMI::enterprises.29385.106.* OIDs for HAProxy if there
is nothing assigned already.
2008-03-02 01:42:14 +00:00
|
|
|
s->srv_state = SV_STCLOSE;
|
|
|
|
fsm_resync |= 1;
|
2007-10-17 16:57:38 +00:00
|
|
|
continue;
|
|
|
|
|
[MEDIUM] fix stats socket limitation to 16 kB
Due to the way the stats socket work, it was not possible to
maintain the information related to the command entered, so
after filling a whole buffer, the request was lost and it was
considered that there was nothing to write anymore.
The major reason was that some flags were passed directly
during the first call to stats_dump_raw() instead of being
stored persistently in the session.
To definitely fix this problem, flags were added to the stats
member of the session structure.
A second problem appeared. When the stats were produced, a first
call to client_retnclose() was performed, then one or multiple
subsequent calls to buffer_write_chunks() were done. But once the
stats buffer was full and a reschedule operated, the buffer was
flushed, the write flag cleared from the buffer and nothing was
done to re-arm it.
For this reason, a check was added in the proto_uxst_stats()
function in order to re-call the client FSM when data were added
by stats_dump_raw(). Finally, the whole unix stats dump FSM was
rewritten to avoid all the magics it depended on. It is now
simpler and looks more like the HTTP one.
2008-03-17 20:38:24 +00:00
|
|
|
case SV_STDATA:
|
|
|
|
/* OK we have to process the request. Since it is possible
|
|
|
|
* that we get there with the client output paused, we
|
|
|
|
* will simply check that we have really sent some data
|
|
|
|
* and wake the client up if needed.
|
|
|
|
*/
|
|
|
|
last_rep_l = s->rep->l;
|
|
|
|
if (stats_dump_raw(s, NULL) != 0) {
|
|
|
|
s->srv_state = SV_STCLOSE;
|
|
|
|
fsm_resync |= 1;
|
|
|
|
}
|
|
|
|
if (s->rep->l != last_rep_l)
|
|
|
|
fsm_resync |= 1;
|
|
|
|
break;
|
2007-10-17 16:57:38 +00:00
|
|
|
}
|
2007-10-16 15:34:28 +00:00
|
|
|
} while (fsm_resync);
|
|
|
|
|
|
|
|
if (likely(s->cli_state != CL_STCLOSE || s->srv_state != SV_STCLOSE)) {
|
|
|
|
s->req->flags &= BF_CLEAR_READ & BF_CLEAR_WRITE;
|
|
|
|
s->rep->flags &= BF_CLEAR_READ & BF_CLEAR_WRITE;
|
|
|
|
|
2008-07-06 22:09:58 +00:00
|
|
|
t->expire = tick_first(tick_first(s->req->rex, s->req->wex),
|
|
|
|
tick_first(s->rep->rex, s->rep->wex));
|
2007-10-16 15:34:28 +00:00
|
|
|
|
|
|
|
/* restore t to its place in the task list */
|
|
|
|
task_queue(t);
|
|
|
|
|
|
|
|
*next = t->expire;
|
|
|
|
return; /* nothing more to do */
|
|
|
|
}
|
|
|
|
|
|
|
|
actconn--;
|
|
|
|
listener = fdtab[s->cli_fd].listener;
|
|
|
|
if (listener) {
|
|
|
|
listener->nbconn--;
|
|
|
|
if (listener->state == LI_FULL &&
|
|
|
|
listener->nbconn < listener->maxconn) {
|
|
|
|
/* we should reactivate the listener */
|
|
|
|
EV_FD_SET(listener->fd, DIR_RD);
|
|
|
|
listener->state = LI_READY;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* the task MUST not be in the run queue anymore */
|
|
|
|
task_delete(t);
|
|
|
|
session_free(s);
|
|
|
|
task_free(t);
|
2008-07-06 22:09:58 +00:00
|
|
|
*next = TICK_ETERNITY;
|
2007-10-16 15:34:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
__attribute__((constructor))
|
|
|
|
static void __uxst_protocol_init(void)
|
|
|
|
{
|
|
|
|
protocol_register(&proto_unix);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Local variables:
|
|
|
|
* c-indent-level: 8
|
|
|
|
* c-basic-offset: 8
|
|
|
|
* End:
|
|
|
|
*/
|