From f57789c04e323f78c0f0b65334f4a5f97eff385b Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Mon, 16 Jan 2017 14:47:31 +0000 Subject: [PATCH 1/2] test/objectstore: add bufferptr & blouestore_blob_t::extents to "sizeof" UT Signed-off-by: Igor Fedotvo --- src/test/objectstore/test_bluestore_types.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc index 38b0e8dacc8..3a959c2dc9c 100644 --- a/src/test/objectstore/test_bluestore_types.cc +++ b/src/test/objectstore/test_bluestore_types.cc @@ -29,6 +29,7 @@ TEST(bluestore, sizeof) { P(BlueStore::Buffer); P(bluestore_onode_t); P(bluestore_blob_t); + P(bluestore_blob_t::extents); P(bluestore_extent_ref_map_t); P(bluestore_extent_ref_map_t::record_t); P(std::atomic_int); @@ -36,6 +37,7 @@ TEST(bluestore, sizeof) { P(boost::intrusive::set_base_hook<>); P(boost::intrusive::unordered_set_base_hook<>); P(bufferlist); + P(bufferptr); cout << "map\t" << sizeof(map) << std::endl; cout << "map\t" << sizeof(map) << std::endl; } From 949df92a059c97ed9731096ed88c8c24f7904307 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Mon, 16 Jan 2017 15:12:23 +0000 Subject: [PATCH 2/2] os/bluestore: add pextent vector to mempool Signed-off-by: Igor Fedotov --- src/os/bluestore/BlueStore.cc | 24 +++++++++--------- src/os/bluestore/BlueStore.h | 8 +++--- src/os/bluestore/bluestore_types.cc | 2 +- src/os/bluestore/bluestore_types.h | 17 +++++++------ src/test/objectstore/test_bluestore_types.cc | 26 ++++++++++---------- 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 8f766a5d5ff..42f4dba8cb3 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -1461,9 +1461,9 @@ bool BlueStore::Blob::put_ref( Collection *coll, uint64_t offset, uint64_t length, - vector *r) + PExtentVector *r) { - vector logical; + PExtentVector logical; ref_map.put(offset, length, &logical); r->clear(); @@ -1521,7 +1521,7 @@ bool BlueStore::Blob::put_ref( // cut it out of extents struct vecbuilder { - vector v; + PExtentVector v; uint64_t invalid = 0; void add_invalid(uint64_t length) { @@ -3723,7 +3723,7 @@ int BlueStore::_reconcile_bluefs_freespace() return 0; } -int BlueStore::_balance_bluefs_freespace(vector *extents) +int BlueStore::_balance_bluefs_freespace(PExtentVector *extents) { int ret = 0; assert(bluefs); @@ -3848,7 +3848,7 @@ int BlueStore::_balance_bluefs_freespace(vector *extents) } void BlueStore::_commit_bluefs_freespace( - const vector& bluefs_gift_extents) + const PExtentVector& bluefs_gift_extents) { dout(10) << __func__ << dendl; for (auto& p : bluefs_gift_extents) { @@ -4339,7 +4339,7 @@ static void apply(uint64_t off, int BlueStore::_fsck_check_extents( const ghobject_t& oid, - const vector& extents, + const PExtentVector& extents, bool compressed, boost::dynamic_bitset<> &used_blocks, store_statfs_t& expected_statfs) @@ -4717,7 +4717,7 @@ int BlueStore::fsck(bool deep) << sbi.ref_map << dendl; ++errors; } - vector extents; + PExtentVector extents; for (auto &r : shared_blob.ref_map.ref_map) { extents.emplace_back(bluestore_pextent_t(r.first, r.second.length)); } @@ -5403,7 +5403,7 @@ int BlueStore::_verify_csum(OnodeRef& o, int r = blob->verify_csum(blob_xoffset, bl, &bad, &bad_csum); if (r < 0) { if (r == -1) { - vector pex; + PExtentVector pex; int r = blob->map( bad, blob->get_csum_chunk_size(), @@ -6891,7 +6891,7 @@ void BlueStore::_kv_sync_thread() } } - vector bluefs_gift_extents; + PExtentVector bluefs_gift_extents; if (bluefs) { int r = _balance_bluefs_freespace(&bluefs_gift_extents); assert(r >= 0); @@ -8156,7 +8156,7 @@ void BlueStore::_wctx_finish( dout(20) << __func__ << " lex_old " << lo << dendl; BlobRef b = lo.blob; const bluestore_blob_t& blob = b->get_blob(); - vector r; + PExtentVector r; if (b->put_ref(c.get(), lo.blob_offset, lo.length, &r)) { if (blob.is_compressed()) { txc->statfs_delta.compressed() -= blob.get_compressed_payload_length(); @@ -8169,12 +8169,12 @@ void BlueStore::_wctx_finish( if (!r.empty()) { dout(20) << __func__ << " blob release " << r << dendl; if (blob.is_shared()) { - vector final; + PExtentVector final; if (!b->shared_blob->loaded) { c->load_shared_blob(b->shared_blob); } for (auto e : r) { - vector cur; + PExtentVector cur; b->shared_blob->shared_blob.ref_map.put(e.offset, e.length, &cur); final.insert(final.end(), cur.begin(), cur.end()); } diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 41ebb912b73..c891fc67782 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -471,7 +471,7 @@ public: void get_ref(uint64_t offset, uint64_t length); /// put logical references, and get back any released extents bool put_ref(Collection *coll, uint64_t offset, uint64_t length, - vector *r); + PExtentVector *r); /// pass references for specific range to other blob void pass_ref(Blob* other, uint64_t src_offset, uint64_t length, uint64_t dest_offset); @@ -1640,8 +1640,8 @@ private: int _open_super_meta(); int _reconcile_bluefs_freespace(); - int _balance_bluefs_freespace(vector *extents); - void _commit_bluefs_freespace(const vector& extents); + int _balance_bluefs_freespace(PExtentVector *extents); + void _commit_bluefs_freespace(const PExtentVector& extents); CollectionRef _get_collection(const coll_t& cid); void _queue_reap_collection(CollectionRef& c); @@ -1696,7 +1696,7 @@ private: int _fsck_check_extents( const ghobject_t& oid, - const vector& extents, + const PExtentVector& extents, bool compressed, boost::dynamic_bitset<> &used_blocks, store_statfs_t& expected_statfs); diff --git a/src/os/bluestore/bluestore_types.cc b/src/os/bluestore/bluestore_types.cc index eb6442265c7..c2b3eb59e99 100644 --- a/src/os/bluestore/bluestore_types.cc +++ b/src/os/bluestore/bluestore_types.cc @@ -200,7 +200,7 @@ void bluestore_extent_ref_map_t::get(uint64_t offset, uint32_t length) void bluestore_extent_ref_map_t::put( uint64_t offset, uint32_t length, - vector *release) + PExtentVector *release) { auto p = ref_map.lower_bound(offset); if (p == ref_map.end() || p->first > offset) { diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h index 8d37b214d48..a3e05c68c8f 100644 --- a/src/os/bluestore/bluestore_types.h +++ b/src/os/bluestore/bluestore_types.h @@ -152,25 +152,27 @@ WRITE_CLASS_DENC(bluestore_pextent_t) ostream& operator<<(ostream& out, const bluestore_pextent_t& o); +typedef mempool::bluestore_meta_other::vector PExtentVector; + template<> -struct denc_traits> { +struct denc_traits { enum { supported = true }; enum { bounded = false }; enum { featured = false }; - static void bound_encode(const vector& v, size_t& p) { + static void bound_encode(const PExtentVector& v, size_t& p) { p += sizeof(uint32_t); size_t per = 0; denc(*(bluestore_pextent_t*)nullptr, per); p += per * v.size(); } - static void encode(const vector& v, + static void encode(const PExtentVector& v, bufferlist::contiguous_appender& p) { denc_varint(v.size(), p); for (auto& i : v) { denc(i, p); } } - static void decode(vector& v, bufferptr::iterator& p) { + static void decode(PExtentVector& v, bufferptr::iterator& p) { unsigned num; denc_varint(num, p); v.clear(); @@ -208,7 +210,7 @@ struct bluestore_extent_ref_map_t { } void get(uint64_t offset, uint32_t len); - void put(uint64_t offset, uint32_t len, vector *release); + void put(uint64_t offset, uint32_t len, PExtentVector *release); bool contains(uint64_t offset, uint32_t len) const; bool intersects(uint64_t offset, uint32_t len) const; @@ -283,7 +285,8 @@ struct bluestore_blob_t { }; static string get_flags_string(unsigned flags); - vector extents;///< raw data position on device + + PExtentVector extents; ///< raw data position on device uint32_t compressed_length_orig = 0;///< original length of compressed blob if any uint32_t compressed_length = 0; ///< compressed length if any uint32_t flags = 0; ///< FLAG_* @@ -742,7 +745,7 @@ struct bluestore_wal_op_t { } type_t; __u8 op = 0; - vector extents; + PExtentVector extents; bufferlist data; DENC(bluestore_wal_op_t, v, p) { diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc index 3a959c2dc9c..a6498fd0c48 100644 --- a/src/test/objectstore/test_bluestore_types.cc +++ b/src/test/objectstore/test_bluestore_types.cc @@ -117,7 +117,7 @@ TEST(bluestore_extent_ref_map_t, get) TEST(bluestore_extent_ref_map_t, put) { bluestore_extent_ref_map_t m; - vector r; + PExtentVector r; m.get(10, 30); m.put(10, 30, &r); cout << m << " " << r << std::endl; @@ -333,7 +333,7 @@ TEST(Blob, put_ref) b.get_ref(0, 0x1200); b.get_ref(0xae00, 0x4200); cout << b << std::endl; - vector r; + PExtentVector r; b.put_ref(&coll, 0, 0x1200, &r); cout << " r " << r << std::endl; @@ -356,7 +356,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(0, mas*2)); B.get_ref(0, mas*2); ASSERT_TRUE(b.is_allocated(0, mas*2)); @@ -376,7 +376,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(123, mas*2)); B.get_ref(0, mas*2); B.put_ref(&coll, 0, mas, &r); @@ -397,7 +397,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas)); b.extents.push_back(bluestore_pextent_t(2, mas)); b.extents.push_back(bluestore_pextent_t(3, mas)); @@ -432,7 +432,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas)); b.extents.push_back(bluestore_pextent_t(2, mas)); b.extents.push_back(bluestore_pextent_t(3, mas)); @@ -470,7 +470,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas * 6)); B.get_ref(0, mas*6); B.put_ref(&coll, mas, mas, &r); @@ -499,7 +499,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas * 4)); b.extents.push_back(bluestore_pextent_t(2, mas * 4)); b.extents.push_back(bluestore_pextent_t(3, mas * 4)); @@ -534,7 +534,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas * 4)); b.extents.push_back(bluestore_pextent_t(2, mas * 4)); b.extents.push_back(bluestore_pextent_t(3, mas * 4)); @@ -584,7 +584,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas * 4)); b.extents.push_back(bluestore_pextent_t(2, mas * 4)); b.extents.push_back(bluestore_pextent_t(3, mas * 4)); @@ -634,7 +634,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(1, mas * 8)); B.get_ref(0, mas*8); B.put_ref(&coll, 0, mas, &r); @@ -672,7 +672,7 @@ TEST(Blob, put_ref) B.shared_blob = new BlueStore::SharedBlob(nullptr); B.shared_blob->get(); // hack to avoid dtor from running bluestore_blob_t& b = B.dirty_blob(); - vector r; + PExtentVector r; b.extents.push_back(bluestore_pextent_t(0, mas*4)); b.init_csum(Checksummer::CSUM_CRC32C, 14, mas * 4); B.get_ref(0, mas*4); @@ -699,7 +699,7 @@ TEST(Blob, put_ref) b.init_csum(Checksummer::CSUM_CRC32C, 12, 0x1e000); cout << "before: " << B << std::endl; - vector r; + PExtentVector r; B.put_ref(&coll, 0x1800, 0x2000, &r); cout << "after: " << B << std::endl; cout << "r " << r << std::endl;