2006-06-26 00:48:02 +00:00
|
|
|
/*
|
2010-08-27 15:56:48 +00:00
|
|
|
* include/proto/task.h
|
|
|
|
* Functions for task management.
|
|
|
|
*
|
|
|
|
* Copyright (C) 2000-2010 Willy Tarreau - w@1wt.eu
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation, version 2.1
|
|
|
|
* exclusively.
|
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
*/
|
2006-06-26 00:48:02 +00:00
|
|
|
|
|
|
|
#ifndef _PROTO_TASK_H
|
|
|
|
#define _PROTO_TASK_H
|
|
|
|
|
|
|
|
|
|
|
|
#include <sys/time.h>
|
2006-06-29 16:54:54 +00:00
|
|
|
|
|
|
|
#include <common/config.h>
|
2006-06-29 15:53:05 +00:00
|
|
|
#include <common/memory.h>
|
2007-04-29 08:41:56 +00:00
|
|
|
#include <common/mini-clist.h>
|
|
|
|
#include <common/standard.h>
|
2009-03-08 14:53:06 +00:00
|
|
|
#include <common/ticks.h>
|
2017-09-27 12:59:38 +00:00
|
|
|
#include <common/hathreads.h>
|
|
|
|
|
MAJOR: task: make use of the scope-aware ebtree functions
Currently the task scheduler suffers from an O(n) lookup when
skipping tasks that are not for the current thread. The reason
is that eb32_lookup_ge() has no information about the current
thread so it always revisits many tasks for other threads before
finding its own tasks.
This is particularly visible with HTTP/2 since the number of
concurrent streams created at once causes long series of tasks
for the same stream in the scheduler. With only 10 connections
and 100 streams each, by running on two threads, the performance
drops from 640kreq/s to 11.2kreq/s! Lookup metrics show that for
only 200000 task lookups, 430 million skips had to be performed,
which means that on average, each lookup leads to 2150 nodes to
be visited.
This commit backports the principle of scope lookups for ebtrees
from the ebtree_v7 development tree. The idea is that each node
contains a mask indicating the union of the scopes for the nodes
below it, which is fed during insertion, and used during lookups.
Then during lookups, branches that do not contain any leaf matching
the requested scope are simply ignored. This perfectly matches a
thread mask, allowing a thread to only extract the tasks it cares
about from the run queue, and to always find them in O(log(n))
instead of O(n). Thus the scheduler uses tid_bit and
task->thread_mask as the ebtree scope here.
Doing this has recovered most of the performance, as can be seen on
the test below with two threads, 10 connections, 100 streams each,
and 1 million requests total :
Before After Gain
test duration : 89.6s 4.73s x19
HTTP requests/s (DEBUG) : 11200 211300 x19
HTTP requests/s (PROD) : 15900 447000 x28
spin_lock time : 85.2s 0.46s /185
time per lookup : 13us 40ns /325
Even when going to 6 threads (on 3 hyperthreaded CPU cores), the
performance stays around 284000 req/s, showing that the contention
is much lower.
A test showed that there's no benefit in using this for the wait queue
though.
2017-11-05 12:34:20 +00:00
|
|
|
#include <eb32sctree.h>
|
2009-10-26 20:10:04 +00:00
|
|
|
#include <eb32tree.h>
|
2007-04-29 08:41:56 +00:00
|
|
|
|
2014-11-13 15:57:19 +00:00
|
|
|
#include <types/global.h>
|
2006-06-29 16:54:54 +00:00
|
|
|
#include <types/task.h>
|
2006-06-26 00:48:02 +00:00
|
|
|
|
2019-09-24 12:55:28 +00:00
|
|
|
#include <proto/fd.h>
|
|
|
|
|
2009-03-08 14:53:06 +00:00
|
|
|
/* Principle of the wait queue.
|
|
|
|
*
|
|
|
|
* We want to be able to tell whether an expiration date is before of after the
|
|
|
|
* current time <now>. We KNOW that expiration dates are never too far apart,
|
|
|
|
* because they are measured in ticks (milliseconds). We also know that almost
|
|
|
|
* all dates will be in the future, and that a very small part of them will be
|
|
|
|
* in the past, they are the ones which have expired since last time we checked
|
|
|
|
* them. Using ticks, we know if a date is in the future or in the past, but we
|
|
|
|
* cannot use that to store sorted information because that reference changes
|
|
|
|
* all the time.
|
|
|
|
*
|
2009-03-21 09:01:42 +00:00
|
|
|
* We'll use the fact that the time wraps to sort timers. Timers above <now>
|
|
|
|
* are in the future, timers below <now> are in the past. Here, "above" and
|
|
|
|
* "below" are to be considered modulo 2^31.
|
2009-03-08 14:53:06 +00:00
|
|
|
*
|
2009-03-21 09:01:42 +00:00
|
|
|
* Timers are stored sorted in an ebtree. We use the new ability for ebtrees to
|
|
|
|
* lookup values starting from X to only expire tasks between <now> - 2^31 and
|
|
|
|
* <now>. If the end of the tree is reached while walking over it, we simply
|
|
|
|
* loop back to the beginning. That way, we have no problem keeping sorted
|
|
|
|
* wrapping timers in a tree, between (now - 24 days) and (now + 24 days). The
|
|
|
|
* keys in the tree always reflect their real position, none can be infinite.
|
|
|
|
* This reduces the number of checks to be performed.
|
2009-03-08 14:53:06 +00:00
|
|
|
*
|
|
|
|
* Another nice optimisation is to allow a timer to stay at an old place in the
|
|
|
|
* queue as long as it's not further than the real expiration date. That way,
|
|
|
|
* we use the tree as a place holder for a minorant of the real expiration
|
|
|
|
* date. Since we have a very low chance of hitting a timeout anyway, we can
|
|
|
|
* bounce the nodes to their right place when we scan the tree if we encounter
|
|
|
|
* a misplaced node once in a while. This even allows us not to remove the
|
|
|
|
* infinite timers from the wait queue.
|
|
|
|
*
|
|
|
|
* So, to summarize, we have :
|
|
|
|
* - node->key always defines current position in the wait queue
|
|
|
|
* - timer is the real expiration date (possibly infinite)
|
2009-03-21 09:01:42 +00:00
|
|
|
* - node->key is always before or equal to timer
|
2009-03-08 14:53:06 +00:00
|
|
|
*
|
|
|
|
* The run queue works similarly to the wait queue except that the current date
|
|
|
|
* is replaced by an insertion counter which can also wrap without any problem.
|
|
|
|
*/
|
|
|
|
|
2009-03-21 09:01:42 +00:00
|
|
|
/* The farthest we can look back in a timer tree */
|
|
|
|
#define TIMER_LOOK_BACK (1U << 31)
|
2009-03-08 14:53:06 +00:00
|
|
|
|
|
|
|
/* a few exported variables */
|
2009-03-21 17:13:21 +00:00
|
|
|
extern unsigned int nb_tasks; /* total number of tasks */
|
2019-05-16 15:37:27 +00:00
|
|
|
extern volatile unsigned long global_tasks_mask; /* Mask of threads with tasks in the global runqueue */
|
2016-12-06 08:15:30 +00:00
|
|
|
extern unsigned int tasks_run_queue; /* run queue size */
|
|
|
|
extern unsigned int tasks_run_queue_cur;
|
2009-03-21 17:33:52 +00:00
|
|
|
extern unsigned int nb_tasks_cur;
|
2008-06-30 05:51:00 +00:00
|
|
|
extern unsigned int niced_tasks; /* number of niced tasks in the run queue */
|
2017-11-24 16:34:44 +00:00
|
|
|
extern struct pool_head *pool_head_task;
|
2018-05-18 16:45:28 +00:00
|
|
|
extern struct pool_head *pool_head_tasklet;
|
2017-11-24 16:34:44 +00:00
|
|
|
extern struct pool_head *pool_head_notification;
|
2019-09-24 06:25:15 +00:00
|
|
|
extern THREAD_LOCAL struct task_per_thread *sched; /* current's thread scheduler context */
|
2018-06-06 12:22:03 +00:00
|
|
|
#ifdef USE_THREAD
|
2018-10-15 12:52:21 +00:00
|
|
|
extern struct eb_root timers; /* sorted timers tree, global */
|
2018-05-18 16:38:23 +00:00
|
|
|
extern struct eb_root rqueue; /* tree constituting the run queue */
|
2018-07-26 13:59:38 +00:00
|
|
|
extern int global_rqueue_size; /* Number of element sin the global runqueue */
|
2018-06-06 12:22:03 +00:00
|
|
|
#endif
|
2018-07-26 13:59:38 +00:00
|
|
|
|
2018-10-15 14:12:48 +00:00
|
|
|
extern struct task_per_thread task_per_thread[MAX_THREADS];
|
2017-11-13 09:34:01 +00:00
|
|
|
|
|
|
|
__decl_hathreads(extern HA_SPINLOCK_T rq_lock); /* spin lock related to run queue */
|
2019-05-28 16:48:07 +00:00
|
|
|
__decl_hathreads(extern HA_RWLOCK_T wq_lock); /* RW lock related to the wait queue */
|
2007-05-13 17:43:47 +00:00
|
|
|
|
2018-08-01 13:58:44 +00:00
|
|
|
|
2009-03-07 16:25:21 +00:00
|
|
|
/* return 0 if task is in run queue, otherwise non-zero */
|
|
|
|
static inline int task_in_rq(struct task *t)
|
|
|
|
{
|
2018-05-18 16:45:28 +00:00
|
|
|
/* Check if leaf_p is NULL, in case he's not in the runqueue, and if
|
|
|
|
* it's not 0x1, which would mean it's in the tasklet list.
|
|
|
|
*/
|
2019-04-17 17:13:07 +00:00
|
|
|
return t->rq.node.leaf_p != NULL;
|
2009-03-07 16:25:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* return 0 if task is in wait queue, otherwise non-zero */
|
|
|
|
static inline int task_in_wq(struct task *t)
|
|
|
|
{
|
|
|
|
return t->wq.node.leaf_p != NULL;
|
|
|
|
}
|
|
|
|
|
2008-08-29 16:19:04 +00:00
|
|
|
/* puts the task <t> in run queue with reason flags <f>, and returns <t> */
|
2018-05-18 16:38:23 +00:00
|
|
|
/* This will put the task in the local runqueue if the task is only runnable
|
|
|
|
* by the current thread, in the global runqueue otherwies.
|
|
|
|
*/
|
|
|
|
void __task_wakeup(struct task *t, struct eb_root *);
|
|
|
|
static inline void task_wakeup(struct task *t, unsigned int f)
|
2008-08-29 13:26:14 +00:00
|
|
|
{
|
2018-05-18 16:38:23 +00:00
|
|
|
unsigned short state;
|
|
|
|
|
|
|
|
#ifdef USE_THREAD
|
|
|
|
struct eb_root *root;
|
|
|
|
|
2018-05-18 16:45:28 +00:00
|
|
|
if (t->thread_mask == tid_bit || global.nbthread == 1)
|
2019-09-24 06:25:15 +00:00
|
|
|
root = &sched->rqueue;
|
2018-05-18 16:38:23 +00:00
|
|
|
else
|
|
|
|
root = &rqueue;
|
|
|
|
#else
|
2019-09-24 06:25:15 +00:00
|
|
|
struct eb_root *root = &sched->rqueue;
|
2018-05-18 16:38:23 +00:00
|
|
|
#endif
|
|
|
|
|
2019-04-17 09:47:18 +00:00
|
|
|
state = _HA_ATOMIC_OR(&t->state, f);
|
|
|
|
while (!(state & (TASK_RUNNING | TASK_QUEUED))) {
|
2019-04-17 18:52:51 +00:00
|
|
|
if (_HA_ATOMIC_CAS(&t->state, &state, state | TASK_QUEUED)) {
|
|
|
|
__task_wakeup(t, root);
|
2019-04-17 09:47:18 +00:00
|
|
|
break;
|
2019-04-17 18:52:51 +00:00
|
|
|
}
|
2019-04-17 09:47:18 +00:00
|
|
|
}
|
2008-08-29 13:26:14 +00:00
|
|
|
}
|
2006-06-26 00:48:02 +00:00
|
|
|
|
2017-10-31 15:06:06 +00:00
|
|
|
/* change the thread affinity of a task to <thread_mask> */
|
2017-09-27 12:59:38 +00:00
|
|
|
static inline void task_set_affinity(struct task *t, unsigned long thread_mask)
|
|
|
|
{
|
2017-10-31 15:06:06 +00:00
|
|
|
t->thread_mask = thread_mask;
|
2017-09-27 12:59:38 +00:00
|
|
|
}
|
2017-10-31 15:06:06 +00:00
|
|
|
|
2009-03-07 16:25:21 +00:00
|
|
|
/*
|
|
|
|
* Unlink the task from the wait queue, and possibly update the last_timer
|
|
|
|
* pointer. A pointer to the task itself is returned. The task *must* already
|
|
|
|
* be in the wait queue before calling this function. If unsure, use the safer
|
|
|
|
* task_unlink_wq() function.
|
2006-06-26 00:48:02 +00:00
|
|
|
*/
|
2009-03-07 16:25:21 +00:00
|
|
|
static inline struct task *__task_unlink_wq(struct task *t)
|
|
|
|
{
|
|
|
|
eb32_delete(&t->wq);
|
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
2018-10-15 12:52:21 +00:00
|
|
|
/* remove a task from its wait queue. It may either be the local wait queue if
|
|
|
|
* the task is bound to a single thread (in which case there's no locking
|
|
|
|
* involved) or the global queue, with locking.
|
|
|
|
*/
|
2009-03-07 16:25:21 +00:00
|
|
|
static inline struct task *task_unlink_wq(struct task *t)
|
2006-06-26 00:48:02 +00:00
|
|
|
{
|
BUG/MAJOR: fd/threads, task/threads: ensure all spin locks are unlocked
Calculate if the fd or task should be locked once, before locking, and
reuse the calculation when determing when to unlock.
Fixes a race condition added in 87d54a9a for fds, and b20aa9ee for tasks,
released in 1.9-dev4. When one thread modifies thread_mask to be a single
thread for a task or fd while a second thread has locked or is waiting on a
lock for that task or fd, the second thread will not unlock it. For FDs,
this is observable when a listener is polled by multiple threads, and is
closed while those threads have events pending. For tasks, this seems
possible, where task_set_affinity is called, but I did not observe it.
This must be backported to 1.9.
2019-02-20 20:43:45 +00:00
|
|
|
unsigned long locked;
|
|
|
|
|
2018-10-15 12:52:21 +00:00
|
|
|
if (likely(task_in_wq(t))) {
|
BUG/MAJOR: fd/threads, task/threads: ensure all spin locks are unlocked
Calculate if the fd or task should be locked once, before locking, and
reuse the calculation when determing when to unlock.
Fixes a race condition added in 87d54a9a for fds, and b20aa9ee for tasks,
released in 1.9-dev4. When one thread modifies thread_mask to be a single
thread for a task or fd while a second thread has locked or is waiting on a
lock for that task or fd, the second thread will not unlock it. For FDs,
this is observable when a listener is polled by multiple threads, and is
closed while those threads have events pending. For tasks, this seems
possible, where task_set_affinity is called, but I did not observe it.
This must be backported to 1.9.
2019-02-20 20:43:45 +00:00
|
|
|
locked = atleast2(t->thread_mask);
|
|
|
|
if (locked)
|
2019-05-28 16:48:07 +00:00
|
|
|
HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
|
2009-03-07 16:25:21 +00:00
|
|
|
__task_unlink_wq(t);
|
BUG/MAJOR: fd/threads, task/threads: ensure all spin locks are unlocked
Calculate if the fd or task should be locked once, before locking, and
reuse the calculation when determing when to unlock.
Fixes a race condition added in 87d54a9a for fds, and b20aa9ee for tasks,
released in 1.9-dev4. When one thread modifies thread_mask to be a single
thread for a task or fd while a second thread has locked or is waiting on a
lock for that task or fd, the second thread will not unlock it. For FDs,
this is observable when a listener is polled by multiple threads, and is
closed while those threads have events pending. For tasks, this seems
possible, where task_set_affinity is called, but I did not observe it.
This must be backported to 1.9.
2019-02-20 20:43:45 +00:00
|
|
|
if (locked)
|
2019-05-28 16:48:07 +00:00
|
|
|
HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
|
2018-10-15 12:52:21 +00:00
|
|
|
}
|
2007-04-29 08:41:56 +00:00
|
|
|
return t;
|
2006-06-26 00:48:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2016-12-06 08:15:30 +00:00
|
|
|
* Unlink the task from the run queue. The tasks_run_queue size and number of
|
|
|
|
* niced tasks are updated too. A pointer to the task itself is returned. The
|
|
|
|
* task *must* already be in the run queue before calling this function. If
|
|
|
|
* unsure, use the safer task_unlink_rq() function. Note that the pointer to the
|
|
|
|
* next run queue entry is neither checked nor updated.
|
2006-06-26 00:48:02 +00:00
|
|
|
*/
|
2009-03-07 16:25:21 +00:00
|
|
|
static inline struct task *__task_unlink_rq(struct task *t)
|
|
|
|
{
|
2019-03-08 17:48:47 +00:00
|
|
|
_HA_ATOMIC_SUB(&tasks_run_queue, 1);
|
2018-07-26 13:59:38 +00:00
|
|
|
#ifdef USE_THREAD
|
|
|
|
if (t->state & TASK_GLOBAL) {
|
2019-03-08 17:48:47 +00:00
|
|
|
_HA_ATOMIC_AND(&t->state, ~TASK_GLOBAL);
|
2018-07-26 13:59:38 +00:00
|
|
|
global_rqueue_size--;
|
|
|
|
} else
|
|
|
|
#endif
|
2019-09-24 06:25:15 +00:00
|
|
|
sched->rqueue_size--;
|
2018-07-26 13:59:38 +00:00
|
|
|
eb32sc_delete(&t->rq);
|
2009-03-07 16:25:21 +00:00
|
|
|
if (likely(t->nice))
|
2019-03-08 17:48:47 +00:00
|
|
|
_HA_ATOMIC_SUB(&niced_tasks, 1);
|
2009-03-07 16:25:21 +00:00
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
2015-02-23 15:07:01 +00:00
|
|
|
/* This function unlinks task <t> from the run queue if it is in it. It also
|
|
|
|
* takes care of updating the next run queue task if it was this task.
|
|
|
|
*/
|
2009-03-07 16:25:21 +00:00
|
|
|
static inline struct task *task_unlink_rq(struct task *t)
|
2006-06-26 00:48:02 +00:00
|
|
|
{
|
2019-03-14 15:14:04 +00:00
|
|
|
int is_global = t->state & TASK_GLOBAL;
|
|
|
|
|
|
|
|
if (is_global)
|
2018-05-18 16:38:23 +00:00
|
|
|
HA_SPIN_LOCK(TASK_RQ_LOCK, &rq_lock);
|
2019-04-12 14:10:55 +00:00
|
|
|
if (likely(task_in_rq(t)))
|
2009-03-07 16:25:21 +00:00
|
|
|
__task_unlink_rq(t);
|
2019-03-14 15:14:04 +00:00
|
|
|
if (is_global)
|
2018-05-18 16:38:23 +00:00
|
|
|
HA_SPIN_UNLOCK(TASK_RQ_LOCK, &rq_lock);
|
2008-07-05 16:16:19 +00:00
|
|
|
return t;
|
|
|
|
}
|
2008-06-24 06:17:16 +00:00
|
|
|
|
2018-05-18 16:45:28 +00:00
|
|
|
static inline void tasklet_wakeup(struct tasklet *tl)
|
|
|
|
{
|
2019-10-18 04:43:53 +00:00
|
|
|
if (likely(tl->tid < 0)) {
|
|
|
|
/* this tasklet runs on the caller thread */
|
2019-10-11 14:35:01 +00:00
|
|
|
if (LIST_ISEMPTY(&tl->list)) {
|
2019-10-18 04:43:53 +00:00
|
|
|
LIST_ADDQ(&task_per_thread[tid].task_list, &tl->list);
|
2019-10-11 14:35:01 +00:00
|
|
|
_HA_ATOMIC_ADD(&tasks_run_queue, 1);
|
|
|
|
}
|
|
|
|
} else {
|
2019-10-18 04:43:53 +00:00
|
|
|
/* this tasklet runs on a specific thread */
|
2019-10-11 14:35:01 +00:00
|
|
|
if (MT_LIST_ADDQ(&task_per_thread[tl->tid].shared_tasklet_list, (struct mt_list *)&tl->list) == 1) {
|
|
|
|
_HA_ATOMIC_ADD(&tasks_run_queue, 1);
|
2019-10-18 06:45:41 +00:00
|
|
|
if (sleeping_thread_mask & (1UL << tl->tid)) {
|
|
|
|
_HA_ATOMIC_AND(&sleeping_thread_mask, ~(1UL << tl->tid));
|
2019-10-11 14:35:01 +00:00
|
|
|
wake_thread(tl->tid);
|
|
|
|
}
|
2019-09-24 12:55:28 +00:00
|
|
|
}
|
|
|
|
}
|
2018-05-18 16:45:28 +00:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2019-06-14 16:05:54 +00:00
|
|
|
/* Insert a tasklet into the tasklet list. If used with a plain task instead,
|
|
|
|
* the caller must update the task_list_size.
|
|
|
|
*/
|
|
|
|
static inline void tasklet_insert_into_tasklet_list(struct tasklet *tl)
|
2018-05-18 16:45:28 +00:00
|
|
|
{
|
2019-10-11 14:35:01 +00:00
|
|
|
_HA_ATOMIC_ADD(&tasks_run_queue, 1);
|
|
|
|
LIST_ADDQ(&sched->task_list, &tl->list);
|
2018-05-18 16:45:28 +00:00
|
|
|
}
|
|
|
|
|
2019-06-14 16:05:54 +00:00
|
|
|
/* Remove the tasklet from the tasklet list. The tasklet MUST already be there.
|
|
|
|
* If unsure, use tasklet_remove_from_tasklet_list() instead. If used with a
|
|
|
|
* plain task, the caller must update the task_list_size.
|
2019-10-11 14:35:01 +00:00
|
|
|
* This should only be used by the thread that owns the tasklet, any other
|
|
|
|
* thread should use tasklet_cancel().
|
2019-03-25 17:02:54 +00:00
|
|
|
*/
|
2019-06-14 12:47:49 +00:00
|
|
|
static inline void __tasklet_remove_from_tasklet_list(struct tasklet *t)
|
2018-05-18 16:45:28 +00:00
|
|
|
{
|
2019-10-11 14:35:01 +00:00
|
|
|
LIST_DEL_INIT(&t->list);
|
|
|
|
_HA_ATOMIC_SUB(&tasks_run_queue, 1);
|
2018-05-18 16:45:28 +00:00
|
|
|
}
|
|
|
|
|
2019-06-14 12:47:49 +00:00
|
|
|
static inline void tasklet_remove_from_tasklet_list(struct tasklet *t)
|
2019-03-25 17:02:54 +00:00
|
|
|
{
|
2019-10-11 14:35:01 +00:00
|
|
|
if (likely(!LIST_ISEMPTY(&t->list)))
|
2019-06-14 12:47:49 +00:00
|
|
|
__tasklet_remove_from_tasklet_list(t);
|
2019-03-25 17:02:54 +00:00
|
|
|
}
|
|
|
|
|
2008-06-24 06:17:16 +00:00
|
|
|
/*
|
2009-03-21 17:13:21 +00:00
|
|
|
* Initialize a new task. The bare minimum is performed (queue pointers and
|
|
|
|
* state). The task is returned. This function should not be used outside of
|
|
|
|
* task_new().
|
2008-06-24 06:17:16 +00:00
|
|
|
*/
|
2017-09-27 12:59:38 +00:00
|
|
|
static inline struct task *task_init(struct task *t, unsigned long thread_mask)
|
2008-06-24 06:17:16 +00:00
|
|
|
{
|
2009-03-07 16:25:21 +00:00
|
|
|
t->wq.node.leaf_p = NULL;
|
|
|
|
t->rq.node.leaf_p = NULL;
|
2018-05-18 16:38:23 +00:00
|
|
|
t->state = TASK_SLEEPING;
|
2017-10-31 15:06:06 +00:00
|
|
|
t->thread_mask = thread_mask;
|
2008-06-30 05:51:00 +00:00
|
|
|
t->nice = 0;
|
2009-03-28 16:54:35 +00:00
|
|
|
t->calls = 0;
|
2018-05-31 12:48:54 +00:00
|
|
|
t->call_date = 0;
|
|
|
|
t->cpu_time = 0;
|
|
|
|
t->lat_time = 0;
|
2017-07-24 15:52:58 +00:00
|
|
|
t->expire = TICK_ETERNITY;
|
2006-06-26 00:48:02 +00:00
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
2019-10-18 04:43:53 +00:00
|
|
|
/* Initialize a new tasklet. It's identified as a tasklet by ->nice=-32768. It
|
|
|
|
* is expected to run on the calling thread by default, it's up to the caller
|
|
|
|
* to change ->tid if it wants to own it.
|
|
|
|
*/
|
2018-05-18 16:45:28 +00:00
|
|
|
static inline void tasklet_init(struct tasklet *t)
|
|
|
|
{
|
|
|
|
t->nice = -32768;
|
|
|
|
t->calls = 0;
|
|
|
|
t->state = 0;
|
2018-07-19 14:02:16 +00:00
|
|
|
t->process = NULL;
|
2019-10-18 04:43:53 +00:00
|
|
|
t->tid = -1;
|
2019-10-11 14:35:01 +00:00
|
|
|
LIST_INIT(&t->list);
|
2018-05-18 16:45:28 +00:00
|
|
|
}
|
|
|
|
|
2019-10-18 04:43:53 +00:00
|
|
|
/* Allocate and initialize a new tasklet, local to the thread by default. The
|
|
|
|
* caller may assing its tid if it wants to own the tasklet.
|
|
|
|
*/
|
2018-05-18 16:45:28 +00:00
|
|
|
static inline struct tasklet *tasklet_new(void)
|
|
|
|
{
|
|
|
|
struct tasklet *t = pool_alloc(pool_head_tasklet);
|
|
|
|
|
|
|
|
if (t) {
|
|
|
|
tasklet_init(t);
|
|
|
|
}
|
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
2006-06-26 00:48:02 +00:00
|
|
|
/*
|
2009-03-21 17:13:21 +00:00
|
|
|
* Allocate and initialise a new task. The new task is returned, or NULL in
|
|
|
|
* case of lack of memory. The task count is incremented. Tasks should only
|
|
|
|
* be allocated this way, and must be freed using task_free().
|
|
|
|
*/
|
2017-09-27 12:59:38 +00:00
|
|
|
static inline struct task *task_new(unsigned long thread_mask)
|
2009-03-21 17:13:21 +00:00
|
|
|
{
|
2017-11-24 16:34:44 +00:00
|
|
|
struct task *t = pool_alloc(pool_head_task);
|
2009-03-21 17:13:21 +00:00
|
|
|
if (t) {
|
2019-03-08 17:48:47 +00:00
|
|
|
_HA_ATOMIC_ADD(&nb_tasks, 1);
|
2017-09-27 12:59:38 +00:00
|
|
|
task_init(t, thread_mask);
|
2009-03-21 17:13:21 +00:00
|
|
|
}
|
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2019-05-17 12:16:51 +00:00
|
|
|
* Free a task. Its context must have been freed since it will be lost. The
|
|
|
|
* task count is decremented. It it is the current task, this one is reset.
|
2006-06-26 00:48:02 +00:00
|
|
|
*/
|
2018-05-04 13:46:16 +00:00
|
|
|
static inline void __task_free(struct task *t)
|
2006-06-26 00:48:02 +00:00
|
|
|
{
|
2019-09-24 06:25:15 +00:00
|
|
|
if (t == sched->current) {
|
|
|
|
sched->current = NULL;
|
2019-05-17 12:16:51 +00:00
|
|
|
__ha_barrier_store();
|
|
|
|
}
|
2017-11-24 16:34:44 +00:00
|
|
|
pool_free(pool_head_task, t);
|
2014-11-13 15:57:19 +00:00
|
|
|
if (unlikely(stopping))
|
2017-11-24 16:34:44 +00:00
|
|
|
pool_flush(pool_head_task);
|
2019-03-08 17:48:47 +00:00
|
|
|
_HA_ATOMIC_SUB(&nb_tasks, 1);
|
2006-06-26 00:48:02 +00:00
|
|
|
}
|
|
|
|
|
2019-05-07 17:05:35 +00:00
|
|
|
/* Destroys a task : it's unlinked from the wait queues and is freed if it's
|
|
|
|
* the current task or not queued otherwise it's marked to be freed by the
|
|
|
|
* scheduler. It does nothing if <t> is NULL.
|
|
|
|
*/
|
2019-04-17 20:51:06 +00:00
|
|
|
static inline void task_destroy(struct task *t)
|
2018-05-04 13:46:16 +00:00
|
|
|
{
|
2019-05-07 13:25:25 +00:00
|
|
|
if (!t)
|
|
|
|
return;
|
|
|
|
|
2019-04-17 20:51:06 +00:00
|
|
|
task_unlink_wq(t);
|
|
|
|
/* We don't have to explicitely remove from the run queue.
|
|
|
|
* If we are in the runqueue, the test below will set t->process
|
|
|
|
* to NULL, and the task will be free'd when it'll be its turn
|
|
|
|
* to run.
|
|
|
|
*/
|
|
|
|
|
2018-05-04 13:46:16 +00:00
|
|
|
/* There's no need to protect t->state with a lock, as the task
|
|
|
|
* has to run on the current thread.
|
|
|
|
*/
|
2019-09-24 06:25:15 +00:00
|
|
|
if (t == sched->current || !(t->state & (TASK_QUEUED | TASK_RUNNING)))
|
2018-05-04 13:46:16 +00:00
|
|
|
__task_free(t);
|
|
|
|
else
|
|
|
|
t->process = NULL;
|
|
|
|
}
|
|
|
|
|
2019-10-11 14:35:01 +00:00
|
|
|
/* Should only be called by the thread responsible for the tasklet */
|
2018-05-18 16:45:28 +00:00
|
|
|
static inline void tasklet_free(struct tasklet *tl)
|
|
|
|
{
|
2019-10-11 14:35:01 +00:00
|
|
|
if (!LIST_ISEMPTY(&tl->list)) {
|
|
|
|
LIST_DEL(&tl->list);
|
|
|
|
_HA_ATOMIC_SUB(&tasks_run_queue, 1);
|
2018-12-24 13:03:10 +00:00
|
|
|
}
|
2018-06-08 15:08:19 +00:00
|
|
|
|
2018-05-18 16:45:28 +00:00
|
|
|
pool_free(pool_head_tasklet, tl);
|
|
|
|
if (unlikely(stopping))
|
|
|
|
pool_flush(pool_head_tasklet);
|
|
|
|
}
|
|
|
|
|
2019-09-20 15:18:35 +00:00
|
|
|
static inline void tasklet_set_tid(struct tasklet *tl, int tid)
|
|
|
|
{
|
|
|
|
tl->tid = tid;
|
|
|
|
}
|
|
|
|
|
2018-10-15 12:52:21 +00:00
|
|
|
void __task_queue(struct task *task, struct eb_root *wq);
|
|
|
|
|
2009-03-07 16:25:21 +00:00
|
|
|
/* Place <task> into the wait queue, where it may already be. If the expiration
|
2009-03-08 15:35:27 +00:00
|
|
|
* timer is infinite, do nothing and rely on wake_expired_task to clean up.
|
2018-10-15 12:52:21 +00:00
|
|
|
* If the task is bound to a single thread, it's assumed to be bound to the
|
|
|
|
* current thread's queue and is queued without locking. Otherwise it's queued
|
|
|
|
* into the global wait queue, protected by locks.
|
2006-06-26 00:48:02 +00:00
|
|
|
*/
|
2009-03-08 15:35:27 +00:00
|
|
|
static inline void task_queue(struct task *task)
|
|
|
|
{
|
|
|
|
/* If we already have a place in the wait queue no later than the
|
|
|
|
* timeout we're trying to set, we'll stay there, because it is very
|
|
|
|
* unlikely that we will reach the timeout anyway. If the timeout
|
|
|
|
* has been disabled, it's useless to leave the queue as well. We'll
|
|
|
|
* rely on wake_expired_tasks() to catch the node and move it to the
|
|
|
|
* proper place should it ever happen. Finally we only add the task
|
|
|
|
* to the queue if it was not there or if it was further than what
|
|
|
|
* we want.
|
|
|
|
*/
|
|
|
|
if (!tick_isset(task->expire))
|
|
|
|
return;
|
|
|
|
|
2018-10-15 12:52:21 +00:00
|
|
|
#ifdef USE_THREAD
|
|
|
|
if (atleast2(task->thread_mask)) {
|
2019-05-28 16:48:07 +00:00
|
|
|
HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
|
2018-10-15 12:52:21 +00:00
|
|
|
if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
|
|
|
|
__task_queue(task, &timers);
|
2019-05-28 16:48:07 +00:00
|
|
|
HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
|
2018-10-15 12:52:21 +00:00
|
|
|
} else
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
|
2019-09-24 06:25:15 +00:00
|
|
|
__task_queue(task, &sched->timers);
|
2018-10-15 12:52:21 +00:00
|
|
|
}
|
2009-03-08 15:35:27 +00:00
|
|
|
}
|
2006-06-26 00:48:02 +00:00
|
|
|
|
2011-07-25 12:30:42 +00:00
|
|
|
/* Ensure <task> will be woken up at most at <when>. If the task is already in
|
|
|
|
* the run queue (but not running), nothing is done. It may be used that way
|
|
|
|
* with a delay : task_schedule(task, tick_add(now_ms, delay));
|
|
|
|
*/
|
|
|
|
static inline void task_schedule(struct task *task, int when)
|
|
|
|
{
|
2017-09-27 12:59:38 +00:00
|
|
|
/* TODO: mthread, check if there is no tisk with this test */
|
2011-07-25 12:30:42 +00:00
|
|
|
if (task_in_rq(task))
|
|
|
|
return;
|
|
|
|
|
2018-10-15 12:52:21 +00:00
|
|
|
#ifdef USE_THREAD
|
|
|
|
if (atleast2(task->thread_mask)) {
|
2019-05-28 16:48:07 +00:00
|
|
|
/* FIXME: is it really needed to lock the WQ during the check ? */
|
|
|
|
HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
|
2018-10-15 12:52:21 +00:00
|
|
|
if (task_in_wq(task))
|
|
|
|
when = tick_first(when, task->expire);
|
|
|
|
|
|
|
|
task->expire = when;
|
|
|
|
if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
|
|
|
|
__task_queue(task, &timers);
|
2019-05-28 16:48:07 +00:00
|
|
|
HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
|
2018-10-15 12:52:21 +00:00
|
|
|
} else
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
if (task_in_wq(task))
|
|
|
|
when = tick_first(when, task->expire);
|
2011-07-25 12:30:42 +00:00
|
|
|
|
2018-10-15 12:52:21 +00:00
|
|
|
task->expire = when;
|
|
|
|
if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
|
2019-09-24 06:25:15 +00:00
|
|
|
__task_queue(task, &sched->timers);
|
2018-10-15 12:52:21 +00:00
|
|
|
}
|
2011-07-25 12:30:42 +00:00
|
|
|
}
|
|
|
|
|
2017-07-12 12:31:10 +00:00
|
|
|
/* This function register a new signal. "lua" is the current lua
|
|
|
|
* execution context. It contains a pointer to the associated task.
|
|
|
|
* "link" is a list head attached to an other task that must be wake
|
|
|
|
* the lua task if an event occurs. This is useful with external
|
|
|
|
* events like TCP I/O or sleep functions. This funcion allocate
|
|
|
|
* memory for the signal.
|
|
|
|
*/
|
|
|
|
static inline struct notification *notification_new(struct list *purge, struct list *event, struct task *wakeup)
|
|
|
|
{
|
2017-11-24 16:34:44 +00:00
|
|
|
struct notification *com = pool_alloc(pool_head_notification);
|
2017-07-12 12:31:10 +00:00
|
|
|
if (!com)
|
|
|
|
return NULL;
|
|
|
|
LIST_ADDQ(purge, &com->purge_me);
|
|
|
|
LIST_ADDQ(event, &com->wake_me);
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_INIT(&com->lock);
|
2017-07-12 12:31:10 +00:00
|
|
|
com->task = wakeup;
|
|
|
|
return com;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* This function purge all the pending signals when the LUA execution
|
|
|
|
* is finished. This prevent than a coprocess try to wake a deleted
|
|
|
|
* task. This function remove the memory associated to the signal.
|
2017-12-10 16:14:07 +00:00
|
|
|
* The purge list is not locked because it is owned by only one
|
|
|
|
* process. before browsing this list, the caller must ensure to be
|
|
|
|
* the only one browser.
|
2017-07-12 12:31:10 +00:00
|
|
|
*/
|
|
|
|
static inline void notification_purge(struct list *purge)
|
|
|
|
{
|
|
|
|
struct notification *com, *back;
|
|
|
|
|
|
|
|
/* Delete all pending communication signals. */
|
|
|
|
list_for_each_entry_safe(com, back, purge, purge_me) {
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_LOCK(NOTIF_LOCK, &com->lock);
|
2017-07-12 12:31:10 +00:00
|
|
|
LIST_DEL(&com->purge_me);
|
2017-07-16 22:14:07 +00:00
|
|
|
if (!com->task) {
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
|
2017-11-24 16:34:44 +00:00
|
|
|
pool_free(pool_head_notification, com);
|
2017-07-16 22:14:07 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
com->task = NULL;
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
|
2017-07-12 12:31:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-10 16:10:57 +00:00
|
|
|
/* In some cases, the disconnected notifications must be cleared.
|
|
|
|
* This function just release memory blocs. The purge list is not
|
|
|
|
* locked because it is owned by only one process. Before browsing
|
|
|
|
* this list, the caller must ensure to be the only one browser.
|
|
|
|
* The "com" is not locked because when com->task is NULL, the
|
|
|
|
* notification is no longer used.
|
|
|
|
*/
|
|
|
|
static inline void notification_gc(struct list *purge)
|
|
|
|
{
|
|
|
|
struct notification *com, *back;
|
|
|
|
|
|
|
|
/* Delete all pending communication signals. */
|
|
|
|
list_for_each_entry_safe (com, back, purge, purge_me) {
|
|
|
|
if (com->task)
|
|
|
|
continue;
|
|
|
|
LIST_DEL(&com->purge_me);
|
|
|
|
pool_free(pool_head_notification, com);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-12 12:31:10 +00:00
|
|
|
/* This function sends signals. It wakes all the tasks attached
|
|
|
|
* to a list head, and remove the signal, and free the used
|
2017-12-10 16:14:07 +00:00
|
|
|
* memory. The wake list is not locked because it is owned by
|
|
|
|
* only one process. before browsing this list, the caller must
|
|
|
|
* ensure to be the only one browser.
|
2017-07-12 12:31:10 +00:00
|
|
|
*/
|
|
|
|
static inline void notification_wake(struct list *wake)
|
|
|
|
{
|
|
|
|
struct notification *com, *back;
|
|
|
|
|
|
|
|
/* Wake task and delete all pending communication signals. */
|
|
|
|
list_for_each_entry_safe(com, back, wake, wake_me) {
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_LOCK(NOTIF_LOCK, &com->lock);
|
2017-07-12 12:31:10 +00:00
|
|
|
LIST_DEL(&com->wake_me);
|
2017-07-16 22:14:07 +00:00
|
|
|
if (!com->task) {
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
|
2017-11-24 16:34:44 +00:00
|
|
|
pool_free(pool_head_notification, com);
|
2017-07-16 22:14:07 +00:00
|
|
|
continue;
|
|
|
|
}
|
2017-07-12 12:31:10 +00:00
|
|
|
task_wakeup(com->task, TASK_WOKEN_MSG);
|
2017-07-16 22:14:07 +00:00
|
|
|
com->task = NULL;
|
2017-11-07 09:42:54 +00:00
|
|
|
HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
|
2017-07-12 12:31:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-30 09:40:08 +00:00
|
|
|
/* This function returns true is some notification are pending
|
|
|
|
*/
|
|
|
|
static inline int notification_registered(struct list *wake)
|
|
|
|
{
|
|
|
|
return !LIST_ISEMPTY(wake);
|
|
|
|
}
|
|
|
|
|
2019-05-29 17:22:43 +00:00
|
|
|
static inline int thread_has_tasks(void)
|
|
|
|
{
|
|
|
|
return (!!(global_tasks_mask & tid_bit) |
|
2019-09-24 06:25:15 +00:00
|
|
|
(sched->rqueue_size > 0) |
|
2019-10-11 14:35:01 +00:00
|
|
|
!LIST_ISEMPTY(&sched->task_list) | !MT_LIST_ISEMPTY(&sched->shared_tasklet_list));
|
2019-05-29 17:22:43 +00:00
|
|
|
}
|
|
|
|
|
2019-07-12 06:31:17 +00:00
|
|
|
/* adds list item <item> to work list <work> and wake up the associated task */
|
2019-08-08 13:47:21 +00:00
|
|
|
static inline void work_list_add(struct work_list *work, struct mt_list *item)
|
2019-07-12 06:31:17 +00:00
|
|
|
{
|
2019-08-08 13:47:21 +00:00
|
|
|
MT_LIST_ADDQ(&work->head, item);
|
2019-07-12 06:31:17 +00:00
|
|
|
task_wakeup(work->task, TASK_WOKEN_OTHER);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct work_list *work_list_create(int nbthread,
|
|
|
|
struct task *(*fct)(struct task *, void *, unsigned short),
|
|
|
|
void *arg);
|
|
|
|
|
|
|
|
void work_list_destroy(struct work_list *work, int nbthread);
|
|
|
|
|
2006-06-26 00:48:02 +00:00
|
|
|
/*
|
2011-07-25 14:33:49 +00:00
|
|
|
* This does 3 things :
|
2006-06-26 00:48:02 +00:00
|
|
|
* - wake up all expired tasks
|
|
|
|
* - call all runnable tasks
|
2007-05-12 20:35:00 +00:00
|
|
|
* - return the date of next event in <next> or eternity.
|
2006-06-26 00:48:02 +00:00
|
|
|
*/
|
|
|
|
|
2014-12-15 12:26:01 +00:00
|
|
|
void process_runnable_tasks();
|
2006-06-26 00:48:02 +00:00
|
|
|
|
2008-06-29 20:40:23 +00:00
|
|
|
/*
|
|
|
|
* Extract all expired timers from the timer queue, and wakes up all
|
|
|
|
* associated tasks. Returns the date of next event (or eternity).
|
|
|
|
*/
|
2014-12-15 12:26:01 +00:00
|
|
|
int wake_expired_tasks();
|
2008-06-29 20:40:23 +00:00
|
|
|
|
2018-12-06 13:05:20 +00:00
|
|
|
/*
|
|
|
|
* Delete every tasks before running the master polling loop
|
|
|
|
*/
|
|
|
|
void mworker_cleantasks();
|
|
|
|
|
2006-06-26 00:48:02 +00:00
|
|
|
#endif /* _PROTO_TASK_H */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Local variables:
|
|
|
|
* c-indent-level: 8
|
|
|
|
* c-basic-offset: 8
|
|
|
|
* End:
|
|
|
|
*/
|