RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-03-11 02:39:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

crimson/onode-staged-tree: implement synthetic tests for value erase

Browse Source 
Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
This commit is contained in:
Yingxin Cheng 2021-04-23 10:20:50 +08:00
parent c6427de4db
commit 80868b8359
2 changed files with 179 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
src/crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h
View File

@ -132,6 +132,22 @@ class KVPool {
std::random_shuffle(random_p_kvs.begin(), random_p_kvs.end());
}
void erase_from_random(iterator_t begin, iterator_t end) {
random_p_kvs.erase(begin, end);
kv_vector_t new_kvs;
for (auto p_kv : random_p_kvs) {
new_kvs.emplace_back(*p_kv);
}
std::sort(new_kvs.begin(), new_kvs.end(), [](auto& l, auto& r) {
return l.key < r.key;
});
kvs.swap(new_kvs);
serial_p_kvs.resize(kvs.size());
random_p_kvs.resize(kvs.size());
init();
}
static KVPool create_raw_range(
const std::vector<size_t>& str_sizes,
const std::vector<size_t>& value_sizes,
@ -188,6 +204,10 @@ class KVPool {
private:
KVPool(kv_vector_t&& _kvs)
: kvs(std::move(_kvs)), serial_p_kvs(kvs.size()), random_p_kvs(kvs.size()) {
init();
}
void init() {
std::transform(kvs.begin(), kvs.end(), serial_p_kvs.begin(),
[] (kv_t& item) { return &item; });
std::transform(kvs.begin(), kvs.end(), random_p_kvs.begin(),
@ -278,12 +298,15 @@ class TreeBuilder {
auto cursors = seastar::make_lw_shared<std::vector<BtreeCursor>>();
logger().warn("start inserting {} kvs ...", kvs.size());
auto start_time = mono_clock::now();
return crimson::do_until([&t, this, cursors, ref_kv_iter]() -> future<bool> {
return crimson::do_until([&t, this, cursors, ref_kv_iter,
start_time]() -> future<bool> {
if (*ref_kv_iter == kvs.random_end()) {
std::chrono::duration<double> duration = mono_clock::now() - start_time;
logger().warn("Insert done! {}s", duration.count());
return ertr::template make_ready_future<bool>(true);
}
auto p_kv = **ref_kv_iter;
logger().debug("[{}] {} -> {}",
logger().debug("[{}] insert {} -> {}",
(*ref_kv_iter) - kvs.random_begin(),
key_hobj_t{p_kv->key},
p_kv->value);
@ -313,9 +336,7 @@ class TreeBuilder {
return ertr::template make_ready_future<bool>(false);
#endif
});
}).safe_then([&t, this, start_time, cursors, ref_kv_iter] {
std::chrono::duration<double> duration = mono_clock::now() - start_time;
logger().warn("Insert done! {}s", duration.count());
}).safe_then([&t, this, cursors, ref_kv_iter] {
if (!cursors->empty()) {
logger().info("Verifing tracked cursors ...");
*ref_kv_iter = kvs.random_begin();
@ -346,6 +367,94 @@ class TreeBuilder {
});
}
future<> erase(Transaction& t, std::size_t erase_size) {
assert(erase_size <= kvs.size());
kvs.shuffle();
auto begin = kvs.random_begin();
auto end = begin + erase_size;
auto ref_kv_iter = seastar::make_lw_shared<iterator_t>();
auto cursors = seastar::make_lw_shared<std::map<ghobject_t, BtreeCursor>>();
return ertr::now().safe_then([&t, this, cursors, ref_kv_iter] {
if constexpr (TRACK) {
logger().info("Tracking cursors before erase ...");
*ref_kv_iter = kvs.begin();
auto start_time = mono_clock::now();
return crimson::do_until([&t, this, cursors, ref_kv_iter, start_time] () -> future<bool> {
if (*ref_kv_iter == kvs.end()) {
std::chrono::duration<double> duration = mono_clock::now() - start_time;
logger().info("Track done! {}s", duration.count());
return ertr::template make_ready_future<bool>(true);
}
auto p_kv = **ref_kv_iter;
return tree->find(t, p_kv->key).safe_then([this, cursors, ref_kv_iter](auto cursor) {
auto p_kv = **ref_kv_iter;
validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
cursors->emplace(p_kv->key, cursor);
++(*ref_kv_iter);
return ertr::template make_ready_future<bool>(false);
});
});
} else {
return ertr::now();
}
}).safe_then([&t, this, ref_kv_iter, begin, end] {
*ref_kv_iter = begin;
logger().warn("start erasing {}/{} kvs ...", end - begin, kvs.size());
auto start_time = mono_clock::now();
return crimson::do_until([&t, this, ref_kv_iter,
start_time, begin, end] () -> future<bool> {
if (*ref_kv_iter == end) {
std::chrono::duration<double> duration = mono_clock::now() - start_time;
logger().warn("Erase done! {}s", duration.count());
return ertr::template make_ready_future<bool>(true);
}
auto p_kv = **ref_kv_iter;
logger().debug("[{}] erase {} -> {}",
(*ref_kv_iter) - begin,
key_hobj_t{p_kv->key},
p_kv->value);
return tree->erase(t, p_kv->key).safe_then([&t, this, ref_kv_iter] (auto size) {
ceph_assert(size == 1);
#ifndef NDEBUG
auto p_kv = **ref_kv_iter;
return tree->contains(t, p_kv->key).safe_then([ref_kv_iter] (bool ret) {
ceph_assert(ret == false);
++(*ref_kv_iter);
return ertr::template make_ready_future<bool>(false);
});
#else
++(*ref_kv_iter);
return ertr::template make_ready_future<bool>(false);
#endif
});
});
}).safe_then([this, cursors, ref_kv_iter, begin, end] {
if constexpr (TRACK) {
logger().info("Verifing tracked cursors ...");
*ref_kv_iter = begin;
while (*ref_kv_iter != end) {
auto p_kv = **ref_kv_iter;
auto c_it = cursors->find(p_kv->key);
ceph_assert(c_it != cursors->end());
ceph_assert(c_it->second.is_end());
cursors->erase(c_it);
++(*ref_kv_iter);
}
}
kvs.erase_from_random(begin, end);
if constexpr (TRACK) {
*ref_kv_iter = kvs.begin();
for (auto& [k, c] : *cursors) {
assert(*ref_kv_iter != kvs.end());
auto p_kv = **ref_kv_iter;
validate_cursor_from_item(p_kv->key, p_kv->value, c);
++(*ref_kv_iter);
}
logger().info("Verify done!");
}
});
}
future<> get_stats(Transaction& t) {
return tree->get_stats_slow(t
).safe_then([this](auto stats) {
@ -353,13 +462,17 @@ class TreeBuilder {
});
}
future<std::size_t> height(Transaction& t) {
return tree->height(t);
}
void reload(NodeExtentManagerURef&& nm) {
tree.emplace(std::move(nm));
}
future<> validate(Transaction& t) {
return seastar::async([this, &t] {
logger().info("Verifing insertion ...");
logger().info("Verifing inserted ...");
for (auto& p_kv : kvs) {
auto cursor = tree->find(t, p_kv->key).unsafe_get0();
validate_cursor_from_item(p_kv->key, p_kv->value, cursor);

68
src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
View File

@ -1514,7 +1514,7 @@ struct d_seastore_tm_test_t :
}
};
TEST_F(d_seastore_tm_test_t, 6_random_insert_leaf_node)
TEST_F(d_seastore_tm_test_t, 6_random_tree_insert_erase)
{
run_async([this] {
constexpr bool TEST_SEASTORE = true;
@ -1523,15 +1523,19 @@ TEST_F(d_seastore_tm_test_t, 6_random_insert_leaf_node)
{8, 11, 64, 256, 301, 320},
{8, 16, 128, 512, 576, 640},
{0, 32}, {0, 10}, {0, 4});
auto tree = std::make_unique<TreeBuilder<TRACK_CURSORS, test_item_t>>(kvs,
(TEST_SEASTORE ? NodeExtentManager::create_seastore(*tm)
: NodeExtentManager::create_dummy(IS_DUMMY_SYNC)));
auto moved_nm = (TEST_SEASTORE ? NodeExtentManager::create_seastore(*tm)
: NodeExtentManager::create_dummy(IS_DUMMY_SYNC));
auto p_nm = moved_nm.get();
auto tree = std::make_unique<TreeBuilder<TRACK_CURSORS, test_item_t>>(
kvs, std::move(moved_nm));
{
auto t = tm->create_transaction();
tree->bootstrap(*t).unsafe_get();
tm->submit_transaction(std::move(t)).unsafe_get();
segment_cleaner->run_until_halt().get0();
}
// test insert
{
auto t = tm->create_transaction();
tree->insert(*t).unsafe_get();
@ -1541,20 +1545,68 @@ TEST_F(d_seastore_tm_test_t, 6_random_insert_leaf_node)
{
auto t = tm->create_transaction();
tree->get_stats(*t).unsafe_get();
tm->submit_transaction(std::move(t)).unsafe_get();
segment_cleaner->run_until_halt().get0();
}
if constexpr (TEST_SEASTORE) {
logger().info("seastore replay begin");
logger().info("seastore replay insert begin");
restart();
tree->reload(NodeExtentManager::create_seastore(*tm));
logger().info("seastore replay end");
logger().info("seastore replay insert end");
}
{
// Note: tm->create_weak_transaction() can also work, but too slow.
auto t = tm->create_transaction();
tree->validate(*t).unsafe_get();
}
// test erase 3/4
{
auto t = tm->create_transaction();
tree->erase(*t, kvs.size() / 4 * 3).unsafe_get();
tm->submit_transaction(std::move(t)).unsafe_get();
segment_cleaner->run_until_halt().get0();
}
{
auto t = tm->create_transaction();
tree->get_stats(*t).unsafe_get();
}
if constexpr (TEST_SEASTORE) {
logger().info("seastore replay erase-1 begin");
restart();
tree->reload(NodeExtentManager::create_seastore(*tm));
logger().info("seastore replay erase-1 end");
}
{
auto t = tm->create_transaction();
tree->validate(*t).unsafe_get();
}
// test erase remaining
{
auto t = tm->create_transaction();
tree->erase(*t, kvs.size()).unsafe_get();
tm->submit_transaction(std::move(t)).unsafe_get();
segment_cleaner->run_until_halt().get0();
}
{
auto t = tm->create_transaction();
tree->get_stats(*t).unsafe_get();
}
if constexpr (TEST_SEASTORE) {
logger().info("seastore replay erase-2 begin");
restart();
tree->reload(NodeExtentManager::create_seastore(*tm));
logger().info("seastore replay erase-2 end");
}
{
auto t = tm->create_transaction();
tree->validate(*t).unsafe_get();
EXPECT_EQ(tree->height(*t).unsafe_get0(), 1);
}
if constexpr (!TEST_SEASTORE) {
auto p_dummy = static_cast<DummyManager*>(p_nm);
EXPECT_EQ(p_dummy->size(), 1);
}
tree.reset();
});
}