From 81e46e708a35873defe6deaaee4229024c213f7f Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Wed, 15 Apr 2020 18:17:26 +0300 Subject: [PATCH 1/5] test/store_test: refator spillover test. Fixes: https://tracker.ceph.com/issues/44880 Signed-off-by: Igor Fedotov --- src/test/objectstore/store_test.cc | 78 ++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 9226ce302e5..3f3ec8efec4 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -3768,6 +3768,7 @@ public: unsigned in_flight; map contents; set available_objects; + set::iterator next_available_object; set in_flight_objects; ObjectGenerator *object_gen; gen_type *rng; @@ -3892,8 +3893,9 @@ public: unsigned max_write, unsigned alignment) : cid(cid), write_alignment(alignment), max_object_len(max_size), - max_write_len(max_write), in_flight(0), object_gen(gen), - rng(rng), store(store) {} + max_write_len(max_write), in_flight(0), + next_available_object(available_objects.end()), + object_gen(gen), rng(rng), store(store) {} int init() { ObjectStore::Transaction t; @@ -3902,17 +3904,19 @@ public: return queue_transaction(store, ch, std::move(t)); } void shutdown() { + ghobject_t next; while (1) { vector objects; - int r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), - 10, &objects, 0); + int r = store->collection_list(ch, next, ghobject_t::get_max(), + 10, &objects, &next); ceph_assert(r >= 0); - if (objects.empty()) - break; + if (objects.size() == 0) + break; ObjectStore::Transaction t; + std::map attrset; for (vector::iterator p = objects.begin(); - p != objects.end(); ++p) { - t.remove(cid, *p); + p != objects.end(); ++p) { + t.remove(cid, *p); } queue_transaction(store, ch, std::move(t)); } @@ -3936,6 +3940,20 @@ public: return ret; } + ghobject_t get_next_object(std::unique_lock& locker) { + cond.wait(locker, [this] { + return in_flight < max_in_flight && !available_objects.empty(); + }); + + if (next_available_object == available_objects.end()) { + next_available_object = available_objects.begin(); + } + + ghobject_t ret = *next_available_object; + ++next_available_object; + return ret; + } + void wait_for_ready(std::unique_lock& locker) { cond.wait(locker, [this] { return in_flight < max_in_flight; }); } @@ -4392,6 +4410,35 @@ public: return status; } + int set_fixed_attrs(size_t entries, size_t key_size, size_t val_size) { + std::unique_lock locker{ lock }; + EnterExit ee("setattrs"); + if (!can_unlink()) + return -ENOENT; + wait_for_ready(locker); + + ghobject_t obj = get_next_object(locker); + available_objects.erase(obj); + ObjectStore::Transaction t; + + map attrs; + set keys; + + while (entries--) { + bufferlist name, value; + filled_byte_array(value, val_size); + filled_byte_array(name, key_size); + attrs[name.c_str()] = value; + contents[obj].attrs[name.c_str()] = value; + } + t.setattrs(cid, obj, attrs); + ++in_flight; + in_flight_objects.insert(obj); + t.register_on_applied(new C_SyntheticOnReadable(this, obj)); + int status = store->queue_transaction(ch, std::move(t)); + return status; + } + void getattrs() { EnterExit ee("getattrs"); ghobject_t obj; @@ -8709,22 +8756,23 @@ void doManySetAttr(ObjectStore* store, gen_type rng(time(NULL)); coll_t cid(spg_t(pg_t(0, 447), shard_id_t::NO_SHARD)); - SyntheticWorkloadState test_obj(store, &gen, &rng, cid, 40 * 1024, 4 * 1024, 0); + SyntheticWorkloadState test_obj(store, &gen, &rng, cid, 0, 0, 0); test_obj.init(); - for (int i = 0; i < 1500; ++i) { + size_t object_count = 256; + for (size_t i = 0; i < object_count; ++i) { if (!(i % 10)) cerr << "seeding object " << i << std::endl; test_obj.touch(); } - for (int i = 0; i < 10000; ++i) { + for (size_t i = 0; i < object_count; ++i) { if (!(i % 100)) { cerr << "Op " << i << std::endl; test_obj.print_internal_state(); } - boost::uniform_int<> true_false(0, 99); - test_obj.setattrs(); + test_obj.set_fixed_attrs(1024, 64, 4096); // 1024 attributes, 64 bytes name and 4K value } test_obj.wait_for_done(); + std::cout << "done" << std::endl; AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); ceph_assert(admin_socket); @@ -8761,7 +8809,9 @@ TEST_P(StoreTestSpecificAUSize, SpilloverTest) { const PerfCounters* logger = bstore->get_bluefs_perf_counters(); //experimentally it was discovered that this case results in 400+MB spillover //using lower 300MB threshold just to be safe enough - ASSERT_GE(logger->get(l_bluefs_slow_used_bytes), 300 * 1024 * 1024); + std::cout << "db_used:" << logger->get(l_bluefs_db_used_bytes) << std::endl; + std::cout << "slow_used:" << logger->get(l_bluefs_slow_used_bytes) << std::endl; + ASSERT_GE(logger->get(l_bluefs_slow_used_bytes), 16 * 1024 * 1024); } ); From 2f86e19d7f1b7c5c690bb2bbee939591ed269e3a Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Thu, 16 Apr 2020 14:23:44 +0300 Subject: [PATCH 2/5] os/bluestore: set bluestore_min_alloc_size to 4K for spinners. Adjusting dependant params as well. Reolving space amplification caused by small objects and/or EC overwrites. Relates to: https://tracker.ceph.com/issues/44213 Signed-off-by: Igor Fedotov --- src/common/options.cc | 10 +++++----- src/test/objectstore/store_test.cc | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/common/options.cc b/src/common/options.cc index fbdd1aecd4c..b20bf1137f0 100644 --- a/src/common/options.cc +++ b/src/common/options.cc @@ -4179,7 +4179,7 @@ std::vector