diff --git a/include/proto/freq_ctr.h b/include/proto/freq_ctr.h index 0d0411041a..c6ea5c24c7 100644 --- a/include/proto/freq_ctr.h +++ b/include/proto/freq_ctr.h @@ -24,22 +24,9 @@ #include #include +#include #include -/* Rotate a frequency counter when current period is over. Must not be called - * during a valid period. It is important that it correctly initializes a null - * area. - */ -static inline void rotate_freq_ctr(struct freq_ctr *ctr) -{ - ctr->prev_ctr = ctr->curr_ctr; - if (likely(now.tv_sec - ctr->curr_sec != 1)) { - /* we missed more than one second */ - ctr->prev_ctr = 0; - } - ctr->curr_sec = now.tv_sec; - ctr->curr_ctr = 0; /* leave it at the end to help gcc optimize it away */ -} /* Update a frequency counter by incremental units. It is automatically * rotated if the period is over. It is important that it correctly initializes @@ -47,32 +34,33 @@ static inline void rotate_freq_ctr(struct freq_ctr *ctr) */ static inline unsigned int update_freq_ctr(struct freq_ctr *ctr, unsigned int inc) { - if (likely(ctr->curr_sec == now.tv_sec)) { - ctr->curr_ctr += inc; - return ctr->curr_ctr; - } - rotate_freq_ctr(ctr); - ctr->curr_ctr = inc; - return ctr->curr_ctr; - /* Note: later we may want to propagate the update to other counters */ -} + unsigned int elapsed; + unsigned int tot_inc; + unsigned int curr_sec; -/* Rotate a frequency counter when current period is over. Must not be called - * during a valid period. It is important that it correctly initializes a null - * area. This one works on frequency counters which have a period different - * from one second. - */ -static inline void rotate_freq_ctr_period(struct freq_ctr_period *ctr, - unsigned int period) -{ - ctr->prev_ctr = ctr->curr_ctr; - ctr->curr_tick += period; - if (likely(now_ms - ctr->curr_tick >= period)) { - /* we missed at least two periods */ - ctr->prev_ctr = 0; - ctr->curr_tick = now_ms; + do { + /* remove the bit, used for the lock */ + curr_sec = ctr->curr_sec & 0x7fffffff; } - ctr->curr_ctr = 0; /* leave it at the end to help gcc optimize it away */ + while (!HA_ATOMIC_CAS(&ctr->curr_sec, &curr_sec, curr_sec | 0x8000000)); + + elapsed = (now.tv_sec & 0x7fffffff)- curr_sec; + if (unlikely(elapsed)) { + ctr->prev_ctr = ctr->curr_ctr; + ctr->curr_ctr = 0; + if (likely(elapsed != 1)) { + /* we missed more than one second */ + ctr->prev_ctr = 0; + } + } + + ctr->curr_ctr += inc; + tot_inc = ctr->curr_ctr; + + /* release the lock and update the time in case of rotate. */ + HA_ATOMIC_STORE(&ctr->curr_sec, now.tv_sec & 0x7fffffff); + return tot_inc; + /* Note: later we may want to propagate the update to other counters */ } /* Update a frequency counter by incremental units. It is automatically @@ -83,13 +71,31 @@ static inline void rotate_freq_ctr_period(struct freq_ctr_period *ctr, static inline unsigned int update_freq_ctr_period(struct freq_ctr_period *ctr, unsigned int period, unsigned int inc) { - if (likely(now_ms - ctr->curr_tick < period)) { - ctr->curr_ctr += inc; - return ctr->curr_ctr; + unsigned int tot_inc; + unsigned int curr_tick; + + do { + /* remove the bit, used for the lock */ + curr_tick = (ctr->curr_tick >> 1) << 1; } - rotate_freq_ctr_period(ctr, period); - ctr->curr_ctr = inc; - return ctr->curr_ctr; + while (!HA_ATOMIC_CAS(&ctr->curr_tick, &curr_tick, curr_tick | 0x1)); + + if (now_ms - curr_tick >= period) { + ctr->prev_ctr = ctr->curr_ctr; + ctr->curr_ctr = 0; + curr_tick += period; + if (likely(now_ms - curr_tick >= period)) { + /* we missed at least two periods */ + ctr->prev_ctr = 0; + curr_tick = now_ms; + } + } + + ctr->curr_ctr += inc; + tot_inc = ctr->curr_ctr; + /* release the lock and update the time in case of rotate. */ + HA_ATOMIC_STORE(&ctr->curr_tick, (curr_tick >> 1) << 1); + return tot_inc; /* Note: later we may want to propagate the update to other counters */ } diff --git a/src/freq_ctr.c b/src/freq_ctr.c index 6dec970024..37fb1f385e 100644 --- a/src/freq_ctr.c +++ b/src/freq_ctr.c @@ -31,17 +31,25 @@ unsigned int read_freq_ctr(struct freq_ctr *ctr) { unsigned int curr, past; - unsigned int age; + unsigned int age, curr_sec; - age = now.tv_sec - ctr->curr_sec; + do { + curr = ctr->curr_ctr; + past = ctr->prev_ctr; + curr_sec = ctr->curr_sec; + + } while (curr != ctr->curr_ctr + || past != ctr->prev_ctr + || curr_sec != ctr->curr_sec + || (curr_sec & 0x80000000)); + + age = now.tv_sec - curr_sec; if (unlikely(age > 1)) return 0; - curr = 0; - past = ctr->curr_ctr; - if (likely(!age)) { - curr = past; - past = ctr->prev_ctr; + if (unlikely(age)) { + past = curr; + curr = 0; } if (past <= 1 && !curr) @@ -57,16 +65,25 @@ unsigned int read_freq_ctr(struct freq_ctr *ctr) unsigned int freq_ctr_remain(struct freq_ctr *ctr, unsigned int freq, unsigned int pend) { unsigned int curr, past; - unsigned int age; + unsigned int age, curr_sec; - curr = 0; - age = now.tv_sec - ctr->curr_sec; + do { + curr = ctr->curr_ctr; + past = ctr->prev_ctr; + curr_sec = ctr->curr_sec; - if (likely(age <= 1)) { - past = ctr->curr_ctr; - if (likely(!age)) { - curr = past; - past = ctr->prev_ctr; + } while (curr != ctr->curr_ctr + || past != ctr->prev_ctr + || curr_sec != ctr->curr_sec + || (curr_sec & 0x80000000)); + + age = now.tv_sec - curr_sec; + if (unlikely(age > 1)) + curr = 0; + else { + if (unlikely(age == 1)) { + past = curr; + curr = 0; } curr += mul32hi(past, ms_left_scaled); } @@ -86,17 +103,25 @@ unsigned int freq_ctr_remain(struct freq_ctr *ctr, unsigned int freq, unsigned i unsigned int next_event_delay(struct freq_ctr *ctr, unsigned int freq, unsigned int pend) { unsigned int curr, past; - unsigned int wait, age; + unsigned int wait, age, curr_sec; - past = 0; - curr = 0; - age = now.tv_sec - ctr->curr_sec; + do { + curr = ctr->curr_ctr; + past = ctr->prev_ctr; + curr_sec = ctr->curr_sec; - if (likely(age <= 1)) { - past = ctr->curr_ctr; - if (likely(!age)) { - curr = past; - past = ctr->prev_ctr; + } while (curr != ctr->curr_ctr + || past != ctr->prev_ctr + || curr_sec != ctr->curr_sec + || (curr_sec & 0x80000000)); + + age = now.tv_sec - curr_sec; + if (unlikely(age > 1)) + curr = 0; + else { + if (unlikely(age == 1)) { + past = curr; + curr = 0; } curr += mul32hi(past, ms_left_scaled); } @@ -128,12 +153,19 @@ unsigned int next_event_delay(struct freq_ctr *ctr, unsigned int freq, unsigned unsigned int read_freq_ctr_period(struct freq_ctr_period *ctr, unsigned int period) { unsigned int curr, past; - unsigned int remain; + unsigned int remain, curr_tick; - curr = ctr->curr_ctr; - past = ctr->prev_ctr; + do { + curr = ctr->curr_ctr; + past = ctr->prev_ctr; + curr_tick = ctr->curr_tick; - remain = ctr->curr_tick + period - now_ms; + } while (curr != ctr->curr_ctr + || past != ctr->prev_ctr + || curr_tick != ctr->curr_tick + || (curr_tick & 0x1)); + + remain = curr_tick + period - now_ms; if (unlikely((int)remain < 0)) { /* We're past the first period, check if we can still report a * part of last period or if we're too far away. @@ -159,12 +191,19 @@ unsigned int freq_ctr_remain_period(struct freq_ctr_period *ctr, unsigned int pe unsigned int freq, unsigned int pend) { unsigned int curr, past; - unsigned int remain; + unsigned int remain, curr_tick; - curr = ctr->curr_ctr; - past = ctr->prev_ctr; + do { + curr = ctr->curr_ctr; + past = ctr->prev_ctr; + curr_tick = ctr->curr_tick; - remain = ctr->curr_tick + period - now_ms; + } while (curr != ctr->curr_ctr + || past != ctr->prev_ctr + || curr_tick != ctr->curr_tick + || (curr_tick & 0x1)); + + remain = curr_tick + period - now_ms; if (likely((int)remain < 0)) { /* We're past the first period, check if we can still report a * part of last period or if we're too far away.