From 25a9bd3251ceb805c1cdcd7b470b939ab4dd2514 Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Mon, 3 Mar 2014 14:40:07 +0000 Subject: [PATCH 1/2] osd: OSD: limit the value of 'size' and 'count' on 'osd bench' Otherwise, a high enough 'count' value will trigger all sorts of timeouts on the OSD; a low enough 'size' value will have the same effect for a high enough value of 'count' (even the default value may have ill effects on the osd's behaviour). Limiting these values do not fix how 'osd bench' should behave, but avoid someone from inadvertently bork an OSD. Four options have been added and the user may adjust them if he so desires to play with the OSD's fate: - 'osd_bench_small_size_max_iops' [default: 100] defines the amount of expected IOPS for a small block size (i.e., <1MB). - 'osd_bench_large_size_max_throughput' [default: 100<<20] defines the expected throughput in B/s. We assume 100MB/s. - 'osd_bench_max_block_size' [default: 64 << 20] caps the block size allowed. We have defined 64 MB. - 'osd_bench_duration' [default: 30] caps the expected duration. This values is used when calculating the maximum allowed 'count', and is not enforced as the maximum duration of the operation. If other IO is undergoing, or 'osd bench' is somehow slowed down, 'osd bench' may go over this duration. Adjusting this option does however allow the user to specify higher 'count' values for (e.g.) a small block size, as the operation is assumed to perform the operation over a longer time span. These options attempt to avoid combinations of dangerous parameters. For instance, we limit the block size to 64 MB (by default) so that there is no temptation to specify a large enough block size, along with a very small 'count', such that the end result is similar to specifying a big count with a sane block size. Fixes: 7248 Signed-off-by: Joao Eduardo Luis --- src/common/config_opts.h | 5 ++++ src/osd/OSD.cc | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 3549e5465ec..1f7645d7bee 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -573,6 +573,11 @@ OPTION(osd_objectstore, OPT_STR, "filestore") // ObjectStore backend type // Set to true for testing. Users should NOT set this. OPTION(osd_debug_override_acting_compat, OPT_BOOL, false) +OPTION(osd_bench_small_size_max_iops, OPT_U32, 100) // 100 IOPS +OPTION(osd_bench_large_size_max_throughput, OPT_U64, 100 << 20) // 100 MB/s +OPTION(osd_bench_max_block_size, OPT_U64, 64 << 20) // cap the block size at 64MB +OPTION(osd_bench_duration, OPT_U32, 30) // duration of 'osd bench', capped at 30s to avoid triggering timeouts + OPTION(filestore_debug_disable_sharded_check, OPT_BOOL, false) /// filestore wb throttle limits diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index ac610290304..53e33ca6797 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -4263,6 +4263,59 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist cmd_getval(cct, cmdmap, "count", count, (int64_t)1 << 30); cmd_getval(cct, cmdmap, "size", bsize, (int64_t)4 << 20); + uint32_t duration = g_conf->osd_bench_duration; + + if (bsize > (int64_t) g_conf->osd_bench_max_block_size) { + // let us limit the block size because the next checks rely on it + // having a sane value. If we allow any block size to be set things + // can still go sideways. + ss << "block 'size' values are capped at " + << prettybyte_t(g_conf->osd_bench_max_block_size) << ". If you wish to use" + << " a higher value, please adjust 'osd_bench_max_block_size'"; + r = -EINVAL; + goto out; + } else if (bsize < (int64_t) (1 << 20)) { + // entering the realm of small block sizes. + // limit the count to a sane value, assuming a configurable amount of + // IOPS and duration, so that the OSD doesn't get hung up on this, + // preventing timeouts from going off + int64_t max_count = + bsize * duration * g_conf->osd_bench_small_size_max_iops; + if (count > max_count) { + ss << "'count' values greater than " << max_count + << " for a block size of " << prettybyte_t(bsize) << ", assuming " + << g_conf->osd_bench_small_size_max_iops << " IOPS," + << " for " << duration << " seconds," + << " can cause ill effects on osd. " + << " Please adjust 'osd_bench_small_size_max_iops' with a higher" + << " value if you wish to use a higher 'count'."; + r = -EINVAL; + goto out; + } + } else { + // 1MB block sizes are big enough so that we get more stuff done. + // However, to avoid the osd from getting hung on this and having + // timers being triggered, we are going to limit the count assuming + // a configurable throughput and duration. + int64_t total_throughput = + g_conf->osd_bench_large_size_max_throughput * duration; + int64_t max_count = (int64_t) (total_throughput / bsize); + if (count > max_count) { + ss << "'count' values greater than " << max_count + << " for a block size of " << prettybyte_t(bsize) << ", assuming " + << prettybyte_t(g_conf->osd_bench_large_size_max_throughput) << "/s," + << " for " << duration << " seconds," + << " can cause ill effects on osd. " + << " Please adjust 'osd_bench_large_size_max_throughput'" + << " with a higher value if you wish to use a higher 'count'."; + r = -EINVAL; + goto out; + } + } + + dout(1) << " bench count " << count + << " bsize " << prettybyte_t(bsize) << dendl; + bufferlist bl; bufferptr bp(bsize); bp.zero(); From aca6ac343e39e7fed3c2f5e0f6e3898461537567 Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Mon, 3 Mar 2014 15:28:04 +0000 Subject: [PATCH 2/2] qa: workunits: cephtool: test 'osd bench' limits Signed-off-by: Joao Eduardo Luis --- qa/workunits/cephtool/test.sh | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 4da5cd58aa1..61219f79d89 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -421,6 +421,8 @@ ceph osd pool get rbd crush_ruleset | grep 'crush_ruleset: 0' ceph osd thrash 10 + + set +e # expect error about missing 'pool' argument @@ -442,4 +444,43 @@ ceph heap dump ceph heap stop_profiler ceph heap release + +# test osd bench limits +# As we should not rely on defaults (as they may change over time), +# lets inject some values and perform some simple tests +# max iops: 10 # 100 IOPS +# max throughput: 10485760 # 10MB/s +# max block size: 2097152 # 2MB +# duration: 10 # 10 seconds + +ceph tell osd.0 injectargs "\ + --osd-bench-duration 10 \ + --osd-bench-max-block-size 2097152 \ + --osd-bench-large-size-max-throughput 10485760 \ + --osd-bench-small-size-max-iops 10" + +# anything with a bs larger than 2097152 must fail +expect_false ceph tell osd.0 bench 1 2097153 +# but using 'osd_bench_max_bs' must succeed +ceph tell osd.0 bench 1 2097152 + +# we assume 1MB as a large bs; anything lower is a small bs +# for a 4096 bytes bs, for 10 seconds, we are limited by IOPS +# max count: 409600 + +# more than max count must not be allowed +expect_false ceph tell osd.0 bench 409601 4096 +# but 409600 must be succeed +ceph tell osd.0 bench 409600 4096 + +# for a large bs, we are limited by throughput. +# for a 2MB block size for 10 seconds, out max count is 50 +# max count: 50 + +# more than max count must not be allowed +expect_false ceph tell osd.0 bench 51 2097152 +# but 50 must succeed +ceph tell osd.0 bench 50 2097152 + + echo OK