From 76f4d29d92be3f9f45767cb1ac6cc50da528ecec Mon Sep 17 00:00:00 2001 From: Jianpeng Ma Date: Mon, 8 Nov 2021 14:33:28 +0800 Subject: [PATCH] librbd/cache/pwl: fix reorder issue between func process_writeback_dirty_entries In fact, we not only make sure ops in order in func process_writeback_dirty_entries, but also make sure ops in order between func process_writeback_dirty_entries. Signed-off-by: Jianpeng Ma --- src/librbd/cache/pwl/AbstractWriteLog.cc | 6 ++++++ src/librbd/cache/pwl/AbstractWriteLog.h | 1 + src/librbd/cache/pwl/rwl/WriteLog.cc | 1 + src/librbd/cache/pwl/ssd/WriteLog.cc | 3 +++ 4 files changed, 11 insertions(+) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index 695b2f0041c..c63df19c400 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -1639,6 +1639,7 @@ Context* AbstractWriteLog::construct_flush_entry(std::shared_ptrram_entry.sync_gen_number; } m_flush_ops_in_flight += 1; + m_flush_ops_will_send += 1; /* For write same this is the bytes affected by the flush op, not the bytes transferred */ m_flush_bytes_in_flight += log_entry->ram_entry.write_bytes; @@ -1710,6 +1711,11 @@ void AbstractWriteLog::process_writeback_dirty_entries() { all_clean = !m_flush_ops_in_flight; break; } + + if (m_flush_ops_will_send) { + ldout(cct, 20) << "Previous flush-ops is still not sent" << dendl; + break; + } auto candidate = m_dirty_log_entries.front(); bool flushable = can_flush_entry(candidate); if (flushable) { diff --git a/src/librbd/cache/pwl/AbstractWriteLog.h b/src/librbd/cache/pwl/AbstractWriteLog.h index 88e6470f12f..887d77010bb 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.h +++ b/src/librbd/cache/pwl/AbstractWriteLog.h @@ -294,6 +294,7 @@ protected: std::atomic m_async_flush_ops = {0}; std::atomic m_async_append_ops = {0}; + std::atomic m_flush_ops_will_send = {0}; /* Acquire locks in order declared here */ mutable ceph::mutex m_log_retire_lock; diff --git a/src/librbd/cache/pwl/rwl/WriteLog.cc b/src/librbd/cache/pwl/rwl/WriteLog.cc index f635015329a..55a22407dd8 100644 --- a/src/librbd/cache/pwl/rwl/WriteLog.cc +++ b/src/librbd/cache/pwl/rwl/WriteLog.cc @@ -592,6 +592,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry << " " << *log_entry << dendl; log_entry->writeback(this->m_image_writeback, ctx); + this->m_flush_ops_will_send -= 1; }), 0); }); } diff --git a/src/librbd/cache/pwl/ssd/WriteLog.cc b/src/librbd/cache/pwl/ssd/WriteLog.cc index fcced9d2874..5c4be2fde0a 100644 --- a/src/librbd/cache/pwl/ssd/WriteLog.cc +++ b/src/librbd/cache/pwl/ssd/WriteLog.cc @@ -539,6 +539,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry << " " << *log_entry << dendl; log_entry->writeback(this->m_image_writeback, ctx); + this->m_flush_ops_will_send -= 1; }), 0); }); } @@ -589,6 +590,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus << " " << *log_entry << dendl; log_entry->writeback_bl(this->m_image_writeback, ctx, std::move(captured_entry_bl)); + this->m_flush_ops_will_send -= 1; }), 0); } else { m_image_ctx.op_work_queue->queue(new LambdaContext( @@ -596,6 +598,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry << " " << *log_entry << dendl; log_entry->writeback(this->m_image_writeback, ctx); + this->m_flush_ops_will_send -= 1; }), 0); } }