RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-29 22:43:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

mon, osd: Add objects trimmed to pg dump stats.

Browse Source 
Add a new column, OBJECTS_TRIMMED, to the pg dump stats that shows the
number of objects trimmed when a snap is removed.

When a pg splits, the stats from the parent pg is copied to the child
pg. In such a case, reset objects_trimmed to 0 for the child pg
(see PeeringState::split_into()). Otherwise, this will result in incorrect
stats to be shown for a child pg after the split operation.

Tests:
 - Librados C and C++ API tests to verify the number of objects trimmed
   during snaptrim operation. These tests use the self-managed snaps APIs.
 - Standalone tests to verify objects trimmed using rados pool snaps.

Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
This commit is contained in:
Sridhar Seshasayee 2022-02-17 17:08:36 +05:30
parent 7a90a85500
commit 00249dc0cc
10 changed files with 809 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
qa/standalone/misc/test-snaptrim-stats.sh Executable file
View File

@ -0,0 +1,108 @@
#!/usr/bin/env bash
#
# Copyright (C) 2022 Red Hat <contact@redhat.com>
#
# Author: Sridhar Seshasayee <sseshasa@redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Public License for more details.
#
source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
function run() {
local dir=$1
shift
export CEPH_MON="127.0.0.1:7124" # git grep '\<7124\>' : there must be only one
export CEPH_ARGS
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
CEPH_ARGS+="--mon-host=$CEPH_MON "
CEPH_ARGS+="--debug-bluestore 20 "
local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
for func in $funcs ; do
setup $dir || return 1
$func $dir || return 1
teardown $dir || return 1
done
}
function TEST_snaptrim_stats() {
local dir=$1
local poolname=test
local OSDS=3
local PGNUM=8
local PGPNUM=8
local objects=10
local WAIT_FOR_UPDATE=10
setup $dir || return 1
run_mon $dir a --osd_pool_default_size=$OSDS || return 1
run_mgr $dir x || return 1
for osd in $(seq 0 $(expr $OSDS - 1))
do
run_osd $dir $osd --osd_pool_default_pg_autoscale_mode=off || return 1
done
# disable scrubs
ceph osd set noscrub || return 1
ceph osd set nodeep-scrub || return 1
# Create a pool
create_pool $poolname $PGNUM $PGPNUM
wait_for_clean || return 1
poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
# write a few objects
TESTDATA="testdata.1"
dd if=/dev/urandom of=$TESTDATA bs=4096 count=1
for i in `seq 1 $objects`
do
rados -p $poolname put obj${i} $TESTDATA
done
rm -f $TESTDATA
# create a snapshot, clones
SNAP=1
rados -p $poolname mksnap snap${SNAP}
TESTDATA="testdata.2"
dd if=/dev/urandom of=$TESTDATA bs=4096 count=1
for i in `seq 1 $objects`
do
rados -p $poolname put obj${i} $TESTDATA
done
rm -f $TESTDATA
# remove the snapshot, should trigger snaptrim
rados -p $poolname rmsnap snap${SNAP}
# check for snaptrim stats
wait_for_clean || return 1
sleep $WAIT_FOR_UPDATE
local objects_trimmed=0
for i in $(seq 0 $(expr $PGNUM - 1))
do
local pgid="${poolid}.${i}"
objects_trimmed=$(expr $objects_trimmed + $(ceph pg $pgid query | \
jq '.info.stats.objects_trimmed'))
done
test $objects_trimmed -eq $objects || return 1
teardown $dir || return 1
}
main test-snaptrim-stats "$@"
# Local Variables:
# compile-command: "cd build ; make -j4 && \
# ../qa/run-standalone.sh test-snaptrim-stats.sh"
# End:

2
src/mon/PGMap.cc
View File

@ -1676,6 +1676,7 @@ void PGMap::dump_pg_stats_plain(
tab.define_column("LAST_SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT);
tab.define_column("SCRUB_SCHEDULING", TextTable::LEFT, TextTable::LEFT);
tab.define_column("OBJECTS_SCRUBBED", TextTable::LEFT, TextTable::RIGHT);
tab.define_column("OBJECTS_TRIMMED", TextTable::LEFT, TextTable::RIGHT);
}
for (const auto& [pg, st] : pg_stats) {
@ -1718,6 +1719,7 @@ void PGMap::dump_pg_stats_plain(
<< st.last_scrub_duration
<< st.dump_scrub_schedule()
<< st.objects_scrubbed
<< st.objects_trimmed
<< TextTable::endrow;
}
}

24
src/osd/PG.h
View File

@ -553,6 +553,30 @@ public:
uint64_t get_snap_trimq_size() const override {
return snap_trimq.size();
}
static void add_objects_trimmed_count(
int64_t count, pg_stat_t &stats) {
stats.objects_trimmed += count;
}
void add_objects_trimmed_count(int64_t count) {
recovery_state.update_stats_wo_resched(
[=](auto &history, auto &stats) {
add_objects_trimmed_count(count, stats);
});
}
static void reset_objects_trimmed(pg_stat_t &stats) {
stats.objects_trimmed = 0;
}
void reset_objects_trimmed() {
recovery_state.update_stats_wo_resched(
[=](auto &history, auto &stats) {
reset_objects_trimmed(stats);
});
}
unsigned get_target_pg_log_entries() const override;
void clear_publish_stats() override;

1
src/osd/PeeringState.cc
View File

@ -3238,6 +3238,7 @@ void PeeringState::split_into(
child->info.stats.parent_split_bits = split_bits;
info.stats.stats_invalid = true;
child->info.stats.stats_invalid = true;
child->info.stats.objects_trimmed = 0;
child->info.last_epoch_started = info.last_epoch_started;
child->info.last_interval_started = info.last_interval_started;

10
src/osd/PrimaryLogPG.cc
View File

@ -4692,6 +4692,8 @@ int PrimaryLogPG::trim_object(
PGTransaction *t = ctx->op_t.get();
int64_t num_objects_before_trim = ctx->delta_stats.num_objects;
if (new_snaps.empty()) {
// remove clone
dout(10) << coid << " snaps " << old_snaps << " -> "
@ -4879,6 +4881,13 @@ int PrimaryLogPG::trim_object(
t->setattrs(head_oid, attrs);
}
// Stats reporting - Set number of objects trimmed
if (num_objects_before_trim > ctx->delta_stats.num_objects) {
int64_t num_objects_trimmed =
num_objects_before_trim - ctx->delta_stats.num_objects;
add_objects_trimmed_count(num_objects_trimmed);
}
*ctxp = std::move(ctx);
return 0;
}
@ -4894,6 +4903,7 @@ void PrimaryLogPG::kick_snap_trim()
dout(10) << __func__ << ": nosnaptrim set, not kicking" << dendl;
} else {
dout(10) << __func__ << ": clean and snaps to trim, kicking" << dendl;
reset_objects_trimmed();
snap_trimmer_machine.process_event(KickTrim());
}
}

13
src/osd/osd_types.cc
View File

@ -2874,6 +2874,7 @@ void pg_stat_t::dump(Formatter *f) const
f->dump_int("last_scrub_duration", last_scrub_duration);
f->dump_string("scrub_schedule", dump_scrub_schedule());
f->dump_float("scrub_duration", scrub_duration);
f->dump_int("objects_trimmed", objects_trimmed);
stats.dump(f);
f->open_array_section("up");
for (auto p = up.cbegin(); p != up.cend(); ++p)
@ -2969,7 +2970,7 @@ bool operator==(const pg_scrubbing_status_t& l, const pg_scrubbing_status_t& r)
void pg_stat_t::encode(ceph::buffer::list &bl) const
{
ENCODE_START(27, 22, bl);
ENCODE_START(28, 22, bl);
encode(version, bl);
encode(reported_seq, bl);
encode(reported_epoch, bl);
@ -3026,6 +3027,7 @@ void pg_stat_t::encode(ceph::buffer::list &bl) const
encode(scrub_sched_status.m_is_periodic, bl);
encode(objects_scrubbed, bl);
encode(scrub_duration, bl);
encode(objects_trimmed, bl);
ENCODE_FINISH(bl);
}
@ -3034,7 +3036,7 @@ void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl)
{
bool tmp;
uint32_t old_state;
DECODE_START(27, bl);
DECODE_START(28, bl);
decode(version, bl);
decode(reported_seq, bl);
decode(reported_epoch, bl);
@ -3117,6 +3119,9 @@ void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl)
decode(objects_scrubbed, bl);
decode(scrub_duration, bl);
}
if (struct_v >= 28) {
decode(objects_trimmed, bl);
}
}
DECODE_FINISH(bl);
}
@ -3153,6 +3158,7 @@ void pg_stat_t::generate_test_instances(list<pg_stat_t*>& o)
a.scrub_duration = 0.003;
a.snaptrimq_len = 1048576;
a.objects_scrubbed = 0;
a.objects_trimmed = 0;
list<object_stat_collection_t*> l;
object_stat_collection_t::generate_test_instances(l);
a.stats = *l.back();
@ -3229,7 +3235,8 @@ bool operator==(const pg_stat_t& l, const pg_stat_t& r)
l.last_scrub_duration == r.last_scrub_duration &&
l.scrub_sched_status == r.scrub_sched_status &&
l.objects_scrubbed == r.objects_scrubbed &&
l.scrub_duration == r.scrub_duration;
l.scrub_duration == r.scrub_duration &&
l.objects_trimmed == r.objects_trimmed;
}
// -- store_statfs_t --

2
src/osd/osd_types.h
View File

@ -2273,6 +2273,7 @@ struct pg_stat_t {
// snaptrimq.size() is 64bit, but let's be serious - anything over 50k is
// absurd already, so cap it to 2^32 and save 4 bytes at the same time
uint32_t snaptrimq_len;
int64_t objects_trimmed;
pg_scrubbing_status_t scrub_sched_status;
@ -2299,6 +2300,7 @@ struct pg_stat_t {
up_primary(-1),
acting_primary(-1),
snaptrimq_len(0),
objects_trimmed(0),
stats_invalid(false),
dirty_stats_invalid(false),
omap_stats_invalid(false),

8
src/test/librados/CMakeLists.txt
View File

@ -140,6 +140,14 @@ add_executable(ceph_test_rados_api_snapshots_pp
snapshots_cxx.cc)
target_link_libraries(ceph_test_rados_api_snapshots_pp
librados ${UNITTEST_LIBS} radostest-cxx)
add_executable(ceph_test_rados_api_snapshots_stats
snapshots_stats.cc)
target_link_libraries(ceph_test_rados_api_snapshots_stats
librados ${UNITTEST_LIBS} radostest)
add_executable(ceph_test_rados_api_snapshots_stats_pp
snapshots_stats_cxx.cc)
target_link_libraries(ceph_test_rados_api_snapshots_stats_pp
librados ${UNITTEST_LIBS} radostest-cxx)
add_executable(ceph_test_rados_api_cls_remote_reads
cls_remote_reads.cc

326
src/test/librados/snapshots_stats.cc Normal file
View File

@ -0,0 +1,326 @@
#include "include/rados.h"
#include "json_spirit/json_spirit.h"
#include "test/librados/test.h"
#include "test/librados/TestCase.h"
#include <algorithm>
#include <errno.h>
#include "gtest/gtest.h"
#include <string>
#include <vector>
using std::string;
class LibRadosSnapshotStatsSelfManaged : public RadosTest {
public:
LibRadosSnapshotStatsSelfManaged() {};
~LibRadosSnapshotStatsSelfManaged() override {};
protected:
void SetUp() override {
// disable pg autoscaler for the tests
string c =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"off\""
"}";
char *cmd[1];
cmd[0] = (char *)c.c_str();
std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
0, NULL, 0));
// disable scrubs for the test
c = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
c = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
RadosTest::SetUp();
}
void TearDown() override {
// re-enable pg autoscaler
string c =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"on\""
"}";
char *cmd[1];
cmd[0] = (char *)c.c_str();
std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
0, NULL, 0));
// re-enable scrubs
c = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
c = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
RadosTest::TearDown();
}
};
class LibRadosSnapshotStatsSelfManagedEC : public RadosTestEC {
public:
LibRadosSnapshotStatsSelfManagedEC() {};
~LibRadosSnapshotStatsSelfManagedEC() override {};
protected:
void SetUp() override {
// disable pg autoscaler for the tests
string c =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"off\""
"}";
char *cmd[1];
cmd[0] = (char *)c.c_str();
std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
0, NULL, 0));
// disable scrubs for the test
c = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
c = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
RadosTestEC::SetUp();
}
void TearDown() override {
// re-enable pg autoscaler
string c =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"on\""
"}";
char *cmd[1];
cmd[0] = (char *)c.c_str();
std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
0, NULL, 0));
// re-enable scrubs
c = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
c = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
cmd[0] = (char *)c.c_str();
ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
RadosTestEC::TearDown();
}
};
int get_objects_trimmed(json_spirit::Object& pg_dump) {
int objs_trimmed = 0;
// pg_map
json_spirit::Object pgmap;
for (json_spirit::Object::size_type i = 0; i < pg_dump.size(); ++i) {
json_spirit::Pair& p = pg_dump[i];
if (p.name_ == "pg_map") {
pgmap = p.value_.get_obj();
break;
}
}
// pg_stats array
json_spirit::Array pgs;
for (json_spirit::Object::size_type i = 0; i < pgmap.size(); ++i) {
json_spirit::Pair& p = pgmap[i];
if (p.name_ == "pg_stats") {
pgs = p.value_.get_array();
break;
}
}
// snaptrim stats
for (json_spirit::Object::size_type j = 0; j < pgs.size(); ++j) {
json_spirit::Object& pg_stat = pgs[j].get_obj();
for(json_spirit::Object::size_type k = 0; k < pg_stat.size(); ++k) {
json_spirit::Pair& stats = pg_stat[k];
if (stats.name_ == "objects_trimmed") {
objs_trimmed += stats.value_.get_int();
break;
}
}
}
return objs_trimmed;
}
const int bufsize = 128;
TEST_F(LibRadosSnapshotStatsSelfManaged, SnaptrimStats) {
int num_objs = 10;
// create objects
char buf[bufsize];
memset(buf, 0xcc, sizeof(buf));
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf, sizeof(buf), 0));
}
std::vector<uint64_t> my_snaps;
for (int snap = 0; snap < 1; ++snap) {
// create a snapshot, clone
std::vector<uint64_t> ns(1);
ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
my_snaps.swap(ns);
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps[0]));
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
&my_snaps[0], my_snaps.size()));
char buf2[sizeof(buf)];
memset(buf2, 0xdd, sizeof(buf2));
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf2, sizeof(buf2), 0));
}
}
// wait for maps to settle
ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster));
// remove snaps - should trigger snaptrim
rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps[snap]));
}
// sleep for few secs for the trim stats to populate
std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
sleep(30);
// Dump pg stats and determine if snaptrim stats are getting set
int objects_trimmed = 0;
int tries = 0;
do {
char *buf, *st;
size_t buflen, stlen;
string c = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
const char *cmd = c.c_str();
ASSERT_EQ(0, rados_mon_command(cluster, (const char **)&cmd, 1, "", 0,
&buf, &buflen, &st, &stlen));
string outstr(buf, buflen);
json_spirit::Value v;
ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json."
<< '\n' << outstr;
// pg dump object
json_spirit::Object& obj = v.get_obj();
objects_trimmed = get_objects_trimmed(obj);
if (objects_trimmed < num_objs) {
tries++;
objects_trimmed = 0;
std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
sleep(30);
}
} while(objects_trimmed < num_objs && tries < 5);
// final check for objects trimmed
ASSERT_EQ(objects_trimmed, num_objs);
// clean-up remaining objects
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, rados_remove(ioctx, obj.c_str()));
}
}
// EC testing
TEST_F(LibRadosSnapshotStatsSelfManagedEC, SnaptrimStats) {
int num_objs = 10;
int bsize = alignment;
char *buf = (char *)new char[bsize];
memset(buf, 0xcc, bsize);
// create objects
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf, bsize, 0));
}
std::vector<uint64_t> my_snaps;
for (int snap = 0; snap < 1; ++snap) {
// create a snapshot, clone
std::vector<uint64_t> ns(1);
ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
my_snaps.swap(ns);
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps[0]));
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
&my_snaps[0], my_snaps.size()));
char *buf2 = (char *)new char[bsize];
memset(buf2, 0xdd, bsize);
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf2, bsize, bsize));
}
delete[] buf2;
}
// wait for maps to settle
ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster));
// remove snaps - should trigger snaptrim
rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps[snap]));
}
// sleep for few secs for the trim stats to populate
std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
sleep(30);
// Dump pg stats and determine if snaptrim stats are getting set
int objects_trimmed = 0;
int tries = 0;
do {
char *buf, *st;
size_t buflen, stlen;
string c = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
const char *cmd = c.c_str();
ASSERT_EQ(0, rados_mon_command(cluster, (const char **)&cmd, 1, 0, 0,
&buf, &buflen, &st, &stlen));
string outstr(buf, buflen);
json_spirit::Value v;
ASSERT_NE(0, json_spirit::read(outstr, v)) << "Unable tp parse json."
<< '\n' << outstr;
// pg dump object
json_spirit::Object& obj = v.get_obj();
objects_trimmed = get_objects_trimmed(obj);
if (objects_trimmed != num_objs) {
tries++;
objects_trimmed = 0;
std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
sleep(30);
}
} while (objects_trimmed != num_objs && tries < 5);
// final check for objects trimmed
ASSERT_EQ(objects_trimmed, num_objs);
// clean-up remaining objects
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, rados_remove(ioctx, obj.c_str()));
}
delete[] buf;
}

318
src/test/librados/snapshots_stats_cxx.cc Normal file
View File

@ -0,0 +1,318 @@
#include <algorithm>
#include <errno.h>
#include <string>
#include <vector>
#include "gtest/gtest.h"
#include "include/rados.h"
#include "include/rados/librados.hpp"
#include "json_spirit/json_spirit.h"
#include "test/librados/test_cxx.h"
#include "test/librados/testcase_cxx.h"
using namespace librados;
using std::string;
class LibRadosSnapshotStatsSelfManagedPP : public RadosTestPP {
public:
LibRadosSnapshotStatsSelfManagedPP() {};
~LibRadosSnapshotStatsSelfManagedPP() override {};
protected:
void SetUp() override {
// disable pg autoscaler for the tests
string cmd =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"off\""
"}";
std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
bufferlist inbl;
bufferlist outbl;
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
// disable scrubs for the test
cmd = "{\"prefix\": \"osd set\",\"key\":\"noscrub\"}";
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
cmd = "{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}";
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
RadosTestPP::SetUp();
}
void TearDown() override {
// re-enable pg autoscaler
string cmd =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"on\""
"}";
std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
bufferlist inbl;
bufferlist outbl;
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
// re-enable scrubs
cmd = "{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}";
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
cmd = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
RadosTestPP::TearDown();
}
};
class LibRadosSnapshotStatsSelfManagedECPP : public RadosTestECPP {
public:
LibRadosSnapshotStatsSelfManagedECPP() {};
~LibRadosSnapshotStatsSelfManagedECPP() override {};
protected:
void SetUp() override {
// disable pg autoscaler for the tests
string cmd =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"off\""
"}";
std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
bufferlist inbl;
bufferlist outbl;
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
// disable scrubs for the test
cmd = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}");
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
cmd = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}");
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
RadosTestECPP::SetUp();
}
void TearDown() override {
// re-enable pg autoscaler
string cmd =
"{"
"\"prefix\": \"config set\", "
"\"who\": \"global\", "
"\"name\": \"osd_pool_default_pg_autoscale_mode\", "
"\"value\": \"on\""
"}";
std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
bufferlist inbl;
bufferlist outbl;
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
// re-enable scrubs
cmd = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}");
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
cmd = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
RadosTestECPP::TearDown();
}
};
int get_objects_trimmed(json_spirit::Object& pg_dump) {
int objs_trimmed = 0;
// pg_map
json_spirit::Object pgmap;
for (json_spirit::Object::size_type i = 0; i < pg_dump.size(); ++i) {
json_spirit::Pair& p = pg_dump[i];
if (p.name_ == "pg_map") {
pgmap = p.value_.get_obj();
break;
}
}
// pg_stats array
json_spirit::Array pgs;
for (json_spirit::Object::size_type i = 0; i < pgmap.size(); ++i) {
json_spirit::Pair& p = pgmap[i];
if (p.name_ == "pg_stats") {
pgs = p.value_.get_array();
break;
}
}
// snaptrim stats
for (json_spirit::Object::size_type j = 0; j < pgs.size(); ++j) {
json_spirit::Object& pg_stat = pgs[j].get_obj();
for(json_spirit::Object::size_type k = 0; k < pg_stat.size(); ++k) {
json_spirit::Pair& stats = pg_stat[k];
if (stats.name_ == "objects_trimmed") {
objs_trimmed += stats.value_.get_int();
break;
}
}
}
return objs_trimmed;
}
const int bufsize = 128;
TEST_F(LibRadosSnapshotStatsSelfManagedPP, SnaptrimStatsPP) {
int num_objs = 10;
// create objects
char buf[bufsize];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf, sizeof(buf));
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, ioctx.write(obj, bl, sizeof(buf), 0));
}
std::vector<uint64_t> my_snaps;
char buf2[sizeof(buf)];
memset(buf2, 0xdd, sizeof(buf2));
bufferlist bl2;
bl2.append(buf2, sizeof(buf2));
for (int snap = 0; snap < 1; ++snap) {
// create a snapshot, clone
std::vector<uint64_t> ns(1);
ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
my_snaps.swap(ns);
ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, ioctx.write(obj, bl2, sizeof(buf2), 0));
}
}
// wait for maps to settle
cluster.wait_for_latest_osdmap();
// remove snaps - should trigger snaptrim
for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
ioctx.selfmanaged_snap_remove(my_snaps[snap]);
}
// sleep for few secs for the trim stats to populate
std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
sleep(30);
// Dump pg stats and determine if snaptrim stats are getting set
int objects_trimmed = 0;
int tries = 0;
do {
string cmd = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
bufferlist inbl;
bufferlist outbl;
ASSERT_EQ(0, cluster.mon_command(cmd, inbl, &outbl, NULL));
string outstr(outbl.c_str(), outbl.length());
json_spirit::Value v;
ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." << '\n' << outstr;
// pg_map
json_spirit::Object& obj = v.get_obj();
objects_trimmed = get_objects_trimmed(obj);
if (objects_trimmed < num_objs) {
tries++;
objects_trimmed = 0;
std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
sleep(30);
}
} while(objects_trimmed < num_objs && tries < 5);
// final check for objects trimmed
ASSERT_EQ(objects_trimmed, num_objs);
// clean-up remaining objects
ioctx.snap_set_read(librados::SNAP_HEAD);
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, ioctx.remove(obj));
}
}
// EC testing
TEST_F(LibRadosSnapshotStatsSelfManagedECPP, SnaptrimStatsECPP) {
int num_objs = 10;
int bsize = alignment;
// create objects
char *buf = (char *)new char[bsize];
memset(buf, 0xcc, bsize);
bufferlist bl;
bl.append(buf, bsize);
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, ioctx.write(obj, bl, bsize, 0));
}
std::vector<uint64_t> my_snaps;
char *buf2 = (char *)new char[bsize];
memset(buf2, 0xdd, bsize);
bufferlist bl2;
bl2.append(buf2, bsize);
for (int snap = 0; snap < 1; ++snap) {
// create a snapshot, clone
std::vector<uint64_t> ns(1);
ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
my_snaps.swap(ns);
ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, ioctx.write(obj, bl2, bsize, bsize));
}
}
// wait for maps to settle
cluster.wait_for_latest_osdmap();
// remove snaps - should trigger snaptrim
for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
ioctx.selfmanaged_snap_remove(my_snaps[snap]);
}
// sleep for few secs for the trim stats to populate
std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
sleep(30);
// Dump pg stats and determine if snaptrim stats are getting set
int objects_trimmed = 0;
int tries = 0;
do {
string cmd = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
bufferlist inbl;
bufferlist outbl;
ASSERT_EQ(0, cluster.mon_command(cmd, inbl, &outbl, NULL));
string outstr(outbl.c_str(), outbl.length());
json_spirit::Value v;
ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." << '\n' << outstr;
// pg_map
json_spirit::Object& obj = v.get_obj();
objects_trimmed = get_objects_trimmed(obj);
if (objects_trimmed < num_objs) {
tries++;
objects_trimmed = 0;
std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
sleep(30);
}
} while(objects_trimmed < num_objs && tries < 5);
// final check for objects trimmed
ASSERT_EQ(objects_trimmed, num_objs);
// clean-up remaining objects
ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
for (int i = 0; i < num_objs; ++i) {
string obj = string("foo") + std::to_string(i);
ASSERT_EQ(0, ioctx.remove(obj));
}
delete[] buf;
delete[] buf2;
}