From b5db8bcbe91ddad4a891a657171c9c5681cc0860 Mon Sep 17 00:00:00 2001 From: tianqing Date: Wed, 7 Sep 2016 21:40:34 +0800 Subject: [PATCH] rbd-mirror: Add sparse read for sync image Currently, the image sync do full read, and we shall add sparse read to let the sync more efficiently. Feature: http://tracker.ceph.com/issues/16780 Signed-off-by: tianqing --- .../librados_test_stub/MockTestMemIoCtxImpl.h | 11 +++++++ .../image_sync/test_mock_ObjectCopyRequest.cc | 27 +++++++++-------- .../image_sync/ObjectCopyRequest.cc | 29 ++++++++++++------- .../rbd_mirror/image_sync/ObjectCopyRequest.h | 2 +- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/test/librados_test_stub/MockTestMemIoCtxImpl.h b/src/test/librados_test_stub/MockTestMemIoCtxImpl.h index b15b241db62..3547e453f21 100644 --- a/src/test/librados_test_stub/MockTestMemIoCtxImpl.h +++ b/src/test/librados_test_stub/MockTestMemIoCtxImpl.h @@ -79,6 +79,16 @@ public: return TestMemIoCtxImpl::notify(o, bl, timeout_ms, pbl); } + MOCK_METHOD5(sparse_read, int(const std::string& oid, + uint64_t off, + size_t len, + std::map *m, + bufferlist *bl)); + int do_sparse_read(const std::string& oid, uint64_t off, size_t len, + std::map *m, bufferlist *bl){ + return TestMemIoCtxImpl::sparse_read(oid, off, len, m, bl); + } + MOCK_METHOD4(read, int(const std::string& oid, size_t len, uint64_t off, @@ -142,6 +152,7 @@ public: ON_CALL(*this, list_watchers(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_list_watchers)); ON_CALL(*this, notify(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_notify)); ON_CALL(*this, read(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_read)); + ON_CALL(*this, sparse_read(_, _, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_sparse_read)); ON_CALL(*this, remove(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_remove)); ON_CALL(*this, selfmanaged_snap_create(_)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_selfmanaged_snap_create)); ON_CALL(*this, selfmanaged_snap_remove(_)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_selfmanaged_snap_remove)); diff --git a/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc index b018f16c21b..52d879cdc29 100644 --- a/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc +++ b/src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc @@ -4,6 +4,7 @@ #include "test/rbd_mirror/test_mock_fixture.h" #include "include/interval_set.h" #include "include/rbd/librbd.hpp" +#include "include/rbd/object_map_types.h" #include "librbd/AioImageRequestWQ.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" @@ -108,9 +109,10 @@ public: 0, on_finish); } - void expect_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, uint64_t offset, - uint64_t length, int r) { - auto &expect = EXPECT_CALL(mock_io_ctx, read(_, length, offset, _)); + void expect_sparse_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, uint64_t offset, + uint64_t length, int r) { + + auto &expect = EXPECT_CALL(mock_io_ctx, sparse_read(_, offset, length, _, _)); if (r < 0) { expect.WillOnce(Return(r)); } else { @@ -118,10 +120,10 @@ public: } } - void expect_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, + void expect_sparse_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, const interval_set &extents, int r) { for (auto extent : extents) { - expect_read(mock_io_ctx, extent.first, extent.second, r); + expect_sparse_read(mock_io_ctx, extent.first, extent.second, r); if (r < 0) { break; } @@ -212,6 +214,7 @@ public: } m_snap_map[remote_snap_id] = local_snap_ids; m_local_snap_ids.push_back(local_snap_id); + return 0; } @@ -348,7 +351,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Write) { InSequence seq; expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0); - expect_read(mock_remote_io_ctx, 0, one.range_end(), 0); + expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0); expect_write(mock_local_io_ctx, 0, one.range_end(), 0); expect_update_object_map(mock_local_image_ctx, mock_object_map, m_local_snap_ids[0], OBJECT_EXISTS, 0); @@ -381,7 +384,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, ReadError) { InSequence seq; expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0); - expect_read(mock_remote_io_ctx, 0, one.range_end(), -EINVAL); + expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), -EINVAL); request->send(); ASSERT_EQ(-EINVAL, ctx.wait()); @@ -412,7 +415,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, WriteError) { InSequence seq; expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0); - expect_read(mock_remote_io_ctx, 0, one.range_end(), 0); + expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0); expect_write(mock_local_io_ctx, 0, one.range_end(), -EINVAL); request->send(); @@ -455,9 +458,9 @@ TEST_F(TestMockImageSyncObjectCopyRequest, WriteSnaps) { InSequence seq; expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0); - expect_read(mock_remote_io_ctx, 0, one.range_end(), 0); + expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0); expect_write(mock_local_io_ctx, 0, one.range_end(), 0); - expect_read(mock_remote_io_ctx, two, 0); + expect_sparse_read(mock_remote_io_ctx, two, 0); expect_write(mock_local_io_ctx, two, 0); expect_update_object_map(mock_local_image_ctx, mock_object_map, m_local_snap_ids[0], OBJECT_EXISTS, 0); @@ -504,7 +507,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Trim) { InSequence seq; expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0); - expect_read(mock_remote_io_ctx, 0, one.range_end(), 0); + expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0); expect_write(mock_local_io_ctx, 0, one.range_end(), 0); expect_truncate(mock_local_io_ctx, trim_offset, 0); expect_update_object_map(mock_local_image_ctx, mock_object_map, @@ -546,7 +549,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Remove) { InSequence seq; expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0); - expect_read(mock_remote_io_ctx, 0, one.range_end(), 0); + expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0); expect_write(mock_local_io_ctx, 0, one.range_end(), 0); expect_remove(mock_local_io_ctx, 0); expect_update_object_map(mock_local_image_ctx, mock_object_map, diff --git a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc index 9975b5bf815..0d1c7040497 100644 --- a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc +++ b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc @@ -110,11 +110,10 @@ void ObjectCopyRequest::send_read_object() { dout(20) << ": remote_snap_seq=" << remote_snap_seq << dendl; read_required = true; } - dout(20) << ": read op: " << std::get<1>(sync_op) << "~" << std::get<2>(sync_op) << dendl; - op.read(std::get<1>(sync_op), std::get<2>(sync_op), - &std::get<3>(sync_op), nullptr); + op.sparse_read(std::get<1>(sync_op), std::get<2>(sync_op), &std::get<4>(sync_op), + &std::get<3>(sync_op), nullptr); break; default: break; @@ -173,14 +172,19 @@ void ObjectCopyRequest::send_write_object() { auto &sync_ops = m_snap_sync_ops.begin()->second; assert(!sync_ops.empty()); - + uint64_t buffer_offset; librados::ObjectWriteOperation op; for (auto &sync_op : sync_ops) { switch (std::get<0>(sync_op)) { case SYNC_OP_TYPE_WRITE: - dout(20) << ": write op: " << std::get<1>(sync_op) << "~" - << std::get<3>(sync_op).length() << dendl; - op.write(std::get<1>(sync_op), std::get<3>(sync_op)); + buffer_offset = 0; + for(auto it : std::get<4>(sync_op)){ + bufferlist tmpbl; + tmpbl.substr_of(std::get<3>(sync_op), buffer_offset, it.second); + op.write(it.first, tmpbl); + buffer_offset += it.second; + dout(20) << ": write op: " << it.first<< "~" << it.second << dendl; + } break; case SYNC_OP_TYPE_TRUNC: dout(20) << ": trunc op: " << std::get<1>(sync_op) << dendl; @@ -228,6 +232,7 @@ void ObjectCopyRequest::handle_write_object(int r) { template void ObjectCopyRequest::send_update_object_map() { + m_local_image_ctx->snap_lock.get_read(); if (!m_local_image_ctx->test_features(RBD_FEATURE_OBJECT_MAP, m_local_image_ctx->snap_lock) || @@ -258,6 +263,7 @@ void ObjectCopyRequest::send_update_object_map() { Context *ctx = create_context_callback< ObjectCopyRequest, &ObjectCopyRequest::handle_update_object_map>( this); + m_local_image_ctx->object_map->aio_update(snap_object_state.first, m_object_number, m_object_number + 1, @@ -333,20 +339,23 @@ void ObjectCopyRequest::compute_diffs() { m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_WRITE, it.get_start(), it.get_len(), - bufferlist()); + bufferlist(), + std::map()); } if (end_size < prev_end_size) { dout(20) << ": trunc op: " << end_size << dendl; m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_TRUNC, end_size, 0U, - bufferlist()); + bufferlist(), + std::map()); } } else { if (prev_exists) { // object remove dout(20) << ": remove op" << dendl; m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_REMOVE, - 0U, 0U, bufferlist()); + 0U, 0U, bufferlist(), + std::map()); } } diff --git a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h index 83fe16c52c9..2e353383afa 100644 --- a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h +++ b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h @@ -81,7 +81,7 @@ private: SYNC_OP_TYPE_REMOVE }; - typedef std::tuple SyncOp; + typedef std::tuple > SyncOp; typedef std::list SyncOps; typedef std::map SnapSyncOps; typedef std::map SnapObjectStates;