RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-02 00:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

mon,crush: 'osd crush rule create-replicated ...'

Browse Source 
New command to create crush rule that specifies a class of device.  Plus
all of the fallout in other callers to the CrushWrapper helpers, the
crushtool cli change, and cli test.

Signed-off-by: Sage Weil <sage@redhat.com>
This commit is contained in:
Sage Weil 2017-06-29 16:33:15 -04:00
parent 676e1c9b7c
commit 5a4ba443c4
18 changed files with 361 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/release-notes.rst
View File

@ -224,9 +224,14 @@ Major Changes from Kraken
- ``ceph osd reweightn`` will specify the `reweight` values for
multiple OSDs in a single command. This is equivalent to a series of
``ceph osd reweight`` commands.
- ``ceph crush class {create,rm,ls,rename}`` manage the new CRUSH *device
class* feature. ``ceph crush set-device-class <osd> <class>``
will set the class for a particular device.
- ``ceph osd crush class {create,rm,ls,rename}`` manage the new
CRUSH *device class* feature. ``ceph crush set-device-class
<osd> <class>`` will set the class for a particular device.
- ``ceph osd crush rule create-replicated`` replaces the old
``ceph osd crush rule create-simple`` command to create a CRUSH
rule for a replicated pool. Notably it takes a `class` argument
for the *device class* the rule should target (e.g., `ssd` or
`hdd`).
- ``ceph mon feature ls`` will list monitor features recorded in the
MonMap. ``ceph mon feature set`` will set an optional feature (none of
these exist yet).

7
qa/workunits/mon/crush_ops.sh
View File

@ -19,6 +19,13 @@ ceph osd crush rule create-simple foo default host
ceph osd crush rule create-simple foo default host
ceph osd crush rule create-simple bar default host
ceph osd crush class create ssd
ceph osd crush class create hdd
ceph osd crush set-device-class osd.0 ssd
ceph osd crush set-device-class osd.1 hdd
ceph osd crush rule create-replicated foo-ssd default host ssd
ceph osd crush rule create-replicated foo-hdd default host hdd
ceph osd crush rule ls | grep foo
ceph osd crush rule rm foo

24
src/crush/CrushWrapper.cc
View File

@ -1300,8 +1300,10 @@ void CrushWrapper::reweight(CephContext *cct)
int CrushWrapper::add_simple_rule_at(
string name, string root_name,
string failure_domain_name,
string device_class,
string mode, int rule_type,
int rno, ostream *err)
int rno,
ostream *err)
{
if (rule_exists(name)) {
if (err)
@ -1340,6 +1342,22 @@ int CrushWrapper::add_simple_rule_at(
return -EINVAL;
}
}
if (device_class.size()) {
if (!class_exists(device_class)) {
if (err)
*err << "device class " << device_class << " does not exist";
return -EINVAL;
}
int c = get_class_id(device_class);
if (class_bucket.count(root) == 0 ||
class_bucket[root].count(c) == 0) {
if (err)
*err << "root " << root_name << " has no devices with class "
<< device_class;
return -EINVAL;
}
root = class_bucket[root][c];
}
if (mode != "firstn" && mode != "indep") {
if (err)
*err << "unknown mode " << mode;
@ -1387,10 +1405,12 @@ int CrushWrapper::add_simple_rule_at(
int CrushWrapper::add_simple_rule(
string name, string root_name,
string failure_domain_name,
string device_class,
string mode, int rule_type,
ostream *err)
{
return add_simple_rule_at(name, root_name, failure_domain_name, mode,
return add_simple_rule_at(name, root_name, failure_domain_name, device_class,
mode,
rule_type, -1, err);
}

3
src/crush/CrushWrapper.h
View File

@ -1015,6 +1015,7 @@ public:
int add_simple_rule(
string name, string root_name, string failure_domain_type,
string device_class,
string mode, int rule_type, ostream *err = 0);
/**
@ -1022,7 +1023,7 @@ public:
*/
int add_simple_rule_at(
string name, string root_name,
string failure_domain_type, string mode,
string failure_domain_type, string device_class, string mode,
int rule_type, int rno, ostream *err = 0);
int remove_rule(int ruleno);

1
src/erasure-code/isa/ErasureCodeIsa.cc
View File

@ -55,6 +55,7 @@ ErasureCodeIsa::create_ruleset(const string &name,
name,
ruleset_root,
ruleset_failure_domain,
"",
"indep",
pg_pool_t::TYPE_ERASURE,
ss);

2
src/erasure-code/jerasure/ErasureCodeJerasure.cc
View File

@ -47,7 +47,7 @@ int ErasureCodeJerasure::create_ruleset(const string &name,
ostream *ss) const
{
int ruleid = crush.add_simple_rule(
name, ruleset_root, ruleset_failure_domain,
name, ruleset_root, ruleset_failure_domain, "",
"indep", pg_pool_t::TYPE_ERASURE, ss);
if (ruleid < 0)
return ruleid;

2
src/erasure-code/shec/ErasureCodeShec.cc
View File

@ -52,7 +52,7 @@ int ErasureCodeShec::create_ruleset(const string &name,
ostream *ss) const
{
int ruleid = crush.add_simple_rule(
name, ruleset_root, ruleset_failure_domain,
name, ruleset_root, ruleset_failure_domain, "",
"indep", pg_pool_t::TYPE_ERASURE, ss);
if (ruleid < 0) {
return ruleid;

7
src/mon/MonCommands.h
View File

@ -602,6 +602,13 @@ COMMAND("osd crush rule create-simple " \
"name=mode,type=CephChoices,strings=firstn|indep,req=false",
"create crush rule <name> to start from <root>, replicate across buckets of type <type>, using a choose mode of <firstn|indep> (default firstn; indep best for erasure pools)", \
"osd", "rw", "cli,rest")
COMMAND("osd crush rule create-replicated " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=root,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=type,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=class,type=CephString,goodchars=[A-Za-z0-9-_.],req=false",
"create crush rule <name> for replicated pool to start from <root>, replicate across buckets of type <type>, using a choose mode of <firstn|indep> (default firstn; indep best for erasure pools)", \
"osd", "rw", "cli,rest")
COMMAND("osd crush rule create-erasure " \
"name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \
"name=profile,type=CephString,req=false,goodchars=[A-Za-z0-9-_.=]", \

42
src/mon/OSDMonitor.cc
View File

@ -7794,7 +7794,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
ss << "rule " << name << " already exists";
err = 0;
} else {
int ruleno = newcrush.add_simple_rule(name, root, type, mode,
int ruleno = newcrush.add_simple_rule(name, root, type, "", mode,
pg_pool_t::TYPE_REPLICATED, &ss);
if (ruleno < 0) {
err = ruleno;
@ -7809,6 +7809,46 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
get_last_committed() + 1));
return true;
} else if (prefix == "osd crush rule create-replicated") {
string name, root, type, device_class;
cmd_getval(g_ceph_context, cmdmap, "name", name);
cmd_getval(g_ceph_context, cmdmap, "root", root);
cmd_getval(g_ceph_context, cmdmap, "type", type);
cmd_getval(g_ceph_context, cmdmap, "class", device_class);
if (osdmap.crush->rule_exists(name)) {
// The name is uniquely associated to a ruleid and the rule it contains
// From the user point of view, the rule is more meaningfull.
ss << "rule " << name << " already exists";
err = 0;
goto reply;
}
CrushWrapper newcrush;
_get_pending_crush(newcrush);
if (newcrush.rule_exists(name)) {
// The name is uniquely associated to a ruleid and the rule it contains
// From the user point of view, the rule is more meaningfull.
ss << "rule " << name << " already exists";
err = 0;
} else {
int ruleno = newcrush.add_simple_rule(
name, root, type, device_class,
"firstn", pg_pool_t::TYPE_REPLICATED, &ss);
if (ruleno < 0) {
err = ruleno;
goto reply;
}
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush, mon->get_quorum_con_features());
}
getline(ss, rs);
wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
get_last_committed() + 1));
return true;
} else if (prefix == "osd erasure-code-profile rm") {
string name;
cmd_getval(g_ceph_context, cmdmap, "name", name);

2
src/osd/OSDMap.cc
View File

@ -3471,7 +3471,7 @@ int OSDMap::build_simple_crush_rules(
int r;
r = crush.add_simple_rule_at(
"replicated_rule", root, failure_domain,
"replicated_rule", root, failure_domain, "",
"firstn", pg_pool_t::TYPE_REPLICATED,
crush_rule, ss);
if (r < 0)

5
src/test/cli/crushtool/help.t
View File

@ -62,6 +62,11 @@
create crush rule <name> to start from <root>,
replicate across buckets of type <type>, using
a choose mode of <firstn|indep>
-i mapfn --create-replicated-rule name root type
create crush rule <name> to start from <root>,
replicate across buckets of type <type>
--device-class <class>
use device class <class> for new rule
-i mapfn --remove-rule name
remove the specified crush rule

149
src/test/cli/crushtool/rules.t Normal file
View File

@ -0,0 +1,149 @@
$ crushtool -c $TESTDIR/rules.txt --create-replicated-rule foo default host -o one > /dev/null
$ crushtool -d one
# begin crush map
# devices
device 0 osd.0 class ssd
device 1 osd.1 class ssd
device 2 osd.2 class ssd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
# types
type 0 osd
type 1 host
type 2 root
# buckets
host foo {
\tid -3\t\t# do not change unnecessarily (esc)
\tid -4 class ssd\t\t# do not change unnecessarily (esc)
\tid -7 class hdd\t\t# do not change unnecessarily (esc)
\t# weight 3.000 (esc)
\talg straw2 (esc)
\thash 0\t# rjenkins1 (esc)
\titem osd.0 weight 1.000 (esc)
\titem osd.1 weight 1.000 (esc)
\titem osd.2 weight 1.000 (esc)
}
host bar {
\tid -2\t\t# do not change unnecessarily (esc)
\tid -5 class ssd\t\t# do not change unnecessarily (esc)
\tid -8 class hdd\t\t# do not change unnecessarily (esc)
\t# weight 3.000 (esc)
\talg straw2 (esc)
\thash 0\t# rjenkins1 (esc)
\titem osd.3 weight 1.000 (esc)
\titem osd.4 weight 1.000 (esc)
\titem osd.5 weight 1.000 (esc)
}
root default {
\tid -1\t\t# do not change unnecessarily (esc)
\tid -6 class ssd\t\t# do not change unnecessarily (esc)
\tid -9 class hdd\t\t# do not change unnecessarily (esc)
\t# weight 6.000 (esc)
\talg straw2 (esc)
\thash 0\t# rjenkins1 (esc)
\titem foo weight 3.000 (esc)
\titem bar weight 3.000 (esc)
}
# rules
rule data {
\truleset 0 (esc)
\ttype replicated (esc)
\tmin_size 1 (esc)
\tmax_size 10 (esc)
\tstep take default (esc)
\tstep chooseleaf firstn 0 type host (esc)
\tstep emit (esc)
}
rule foo {
\truleset 1 (esc)
\ttype replicated (esc)
\tmin_size 1 (esc)
\tmax_size 10 (esc)
\tstep take default (esc)
\tstep chooseleaf firstn 0 type host (esc)
\tstep emit (esc)
}
# end crush map
$ crushtool -c $TESTDIR/rules.txt --create-replicated-rule foo-ssd default host -o two --device-class ssd > /dev/null
$ crushtool -d two
# begin crush map
# devices
device 0 osd.0 class ssd
device 1 osd.1 class ssd
device 2 osd.2 class ssd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
# types
type 0 osd
type 1 host
type 2 root
# buckets
host foo {
\tid -3\t\t# do not change unnecessarily (esc)
\tid -4 class ssd\t\t# do not change unnecessarily (esc)
\tid -7 class hdd\t\t# do not change unnecessarily (esc)
\t# weight 3.000 (esc)
\talg straw2 (esc)
\thash 0\t# rjenkins1 (esc)
\titem osd.0 weight 1.000 (esc)
\titem osd.1 weight 1.000 (esc)
\titem osd.2 weight 1.000 (esc)
}
host bar {
\tid -2\t\t# do not change unnecessarily (esc)
\tid -5 class ssd\t\t# do not change unnecessarily (esc)
\tid -8 class hdd\t\t# do not change unnecessarily (esc)
\t# weight 3.000 (esc)
\talg straw2 (esc)
\thash 0\t# rjenkins1 (esc)
\titem osd.3 weight 1.000 (esc)
\titem osd.4 weight 1.000 (esc)
\titem osd.5 weight 1.000 (esc)
}
root default {
\tid -1\t\t# do not change unnecessarily (esc)
\tid -6 class ssd\t\t# do not change unnecessarily (esc)
\tid -9 class hdd\t\t# do not change unnecessarily (esc)
\t# weight 6.000 (esc)
\talg straw2 (esc)
\thash 0\t# rjenkins1 (esc)
\titem foo weight 3.000 (esc)
\titem bar weight 3.000 (esc)
}
# rules
rule data {
\truleset 0 (esc)
\ttype replicated (esc)
\tmin_size 1 (esc)
\tmax_size 10 (esc)
\tstep take default (esc)
\tstep chooseleaf firstn 0 type host (esc)
\tstep emit (esc)
}
rule foo-ssd {
\truleset 1 (esc)
\ttype replicated (esc)
\tmin_size 1 (esc)
\tmax_size 10 (esc)
\tstep take default class ssd (esc)
\tstep chooseleaf firstn 0 type host (esc)
\tstep emit (esc)
}
# end crush map

54
src/test/cli/crushtool/rules.txt Normal file
View File

@ -0,0 +1,54 @@
# begin crush map
# devices
device 0 osd.0 class ssd
device 1 osd.1 class ssd
device 2 osd.2 class ssd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
# types
type 0 osd
type 1 host
type 2 root
# buckets
host foo {
id -3
alg straw2
hash 0
item osd.0 weight 1.0
item osd.1 weight 1.0
item osd.2 weight 1.0
}
host bar {
id -2
alg straw2
hash 0
item osd.3 weight 1.0
item osd.4 weight 1.0
item osd.5 weight 1.0
}
root default {
id -1
alg straw2
hash 0
item foo weight 3.0
item bar weight 3.0
}
# rules
rule data {
ruleset 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
# end crush map

14
src/test/crush/CrushWrapper.cc
View File

@ -909,7 +909,7 @@ TEST(CrushWrapper, dump_rules) {
}
string name("NAME");
int rule = c->add_simple_rule(name, root_name, failure_domain_type,
int rule = c->add_simple_rule(name, root_name, failure_domain_type, "",
"firstn", pg_pool_t::TYPE_ERASURE);
EXPECT_EQ(0, rule);
@ -1032,7 +1032,8 @@ TEST(CrushWrapper, choose_args_compat) {
item = 2;
c.insert_item(g_ceph_context, item, weight, "osd.2", loc);
assert(c.add_simple_rule("rule1", "r11", "host", "firstn", pg_pool_t::TYPE_ERASURE) >= 0);
assert(c.add_simple_rule("rule1", "r11", "host", "",
"firstn", pg_pool_t::TYPE_ERASURE) >= 0);
int id = c.get_item_id("b1");
@ -1102,7 +1103,8 @@ TEST(CrushWrapper, remove_unused_root) {
loc["root"] = "default";
c.insert_item(g_ceph_context, item, weight, "osd.2", loc);
assert(c.add_simple_rule("rule1", "r11", "host", "firstn", pg_pool_t::TYPE_ERASURE) >= 0);
assert(c.add_simple_rule("rule1", "r11", "host", "",
"firstn", pg_pool_t::TYPE_ERASURE) >= 0);
ASSERT_TRUE(c.name_exists("default"));
ASSERT_TRUE(c.name_exists("r11"));
ASSERT_TRUE(c.name_exists("r12"));
@ -1346,7 +1348,8 @@ TEST(CrushWrapper, try_remap_rule) {
{
cout << "take + choose + emit" << std::endl;
ostringstream err;
int rule = c.add_simple_rule("one", "default", "osd", "firstn", 0, &err);
int rule = c.add_simple_rule("one", "default", "osd", "",
"firstn", 0, &err);
ASSERT_EQ(rule, 0);
vector<int> orig = { 0, 3, 9 };
@ -1382,7 +1385,8 @@ TEST(CrushWrapper, try_remap_rule) {
{
cout << "take + chooseleaf + emit" << std::endl;
ostringstream err;
int rule = c.add_simple_rule("two", "default", "host", "firstn", 0, &err);
int rule = c.add_simple_rule("two", "default", "host", "",
"firstn", 0, &err);
ASSERT_EQ(rule, 1);
vector<int> orig = { 0, 3, 9 };

14
src/test/crush/crush.cc
View File

@ -271,7 +271,7 @@ TEST(CRUSH, straw_zero) {
EXPECT_EQ(0, c->set_item_name(root0, root_name0));
string name0("rule0");
int rule0 = c->add_simple_rule(name0, root_name0, "osd",
int rule0 = c->add_simple_rule(name0, root_name0, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
EXPECT_EQ(0, rule0);
@ -282,7 +282,7 @@ TEST(CRUSH, straw_zero) {
EXPECT_EQ(0, c->set_item_name(root1, root_name1));
string name1("rule1");
int rule1 = c->add_simple_rule(name1, root_name1, "osd",
int rule1 = c->add_simple_rule(name1, root_name1, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
EXPECT_EQ(1, rule1);
@ -336,7 +336,7 @@ TEST(CRUSH, straw_same) {
EXPECT_EQ(0, c->set_item_name(root0, root_name0));
string name0("rule0");
int rule0 = c->add_simple_rule(name0, root_name0, "osd",
int rule0 = c->add_simple_rule(name0, root_name0, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
EXPECT_EQ(0, rule0);
@ -352,7 +352,7 @@ TEST(CRUSH, straw_same) {
EXPECT_EQ(0, c->set_item_name(root1, root_name1));
string name1("rule1");
int rule1 = c->add_simple_rule(name1, root_name1, "osd",
int rule1 = c->add_simple_rule(name1, root_name1, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
EXPECT_EQ(1, rule1);
@ -436,7 +436,7 @@ double calc_straw2_stddev(int *weights, int n, bool verbose)
c->set_item_name(root0, root_name0);
string name0("rule0");
int rule0 = c->add_simple_rule(name0, root_name0, "osd",
int rule0 = c->add_simple_rule(name0, root_name0, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
int sum[n];
@ -561,7 +561,7 @@ TEST(CRUSH, straw2_reweight) {
EXPECT_EQ(0, c->set_item_name(root0, root_name0));
string name0("rule0");
int rule0 = c->add_simple_rule(name0, root_name0, "osd",
int rule0 = c->add_simple_rule(name0, root_name0, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
EXPECT_EQ(0, rule0);
@ -577,7 +577,7 @@ TEST(CRUSH, straw2_reweight) {
EXPECT_EQ(0, c->set_item_name(root1, root_name1));
string name1("rule1");
int rule1 = c->add_simple_rule(name1, root_name1, "osd",
int rule1 = c->add_simple_rule(name1, root_name1, "osd", "",
"firstn", pg_pool_t::TYPE_REPLICATED);
EXPECT_EQ(1, rule1);

2
src/test/erasure-code/ErasureCodeExample.h
View File

@ -42,7 +42,7 @@ public:
int create_ruleset(const string &name,
CrushWrapper &crush,
ostream *ss) const override {
return crush.add_simple_rule(name, "default", "host",
return crush.add_simple_rule(name, "default", "host", "",
"indep", pg_pool_t::TYPE_ERASURE, ss);
}

2
src/test/osd/TestOSDMap.cc
View File

@ -60,7 +60,7 @@ public:
// Create an EC ruleset and a pool using it
int r = osdmap.crush->add_simple_rule(
"erasure", "default", "osd",
"erasure", "default", "osd", "",
"indep", pg_pool_t::TYPE_ERASURE,
&cerr);

47
src/tools/crushtool.cc
View File

@ -170,6 +170,11 @@ void usage()
<< " create crush rule <name> to start from <root>,\n"
<< " replicate across buckets of type <type>, using\n"
<< " a choose mode of <firstn|indep>\n";
cout << " -i mapfn --create-replicated-rule name root type\n"
<< " create crush rule <name> to start from <root>,\n"
<< " replicate across buckets of type <type>\n";
cout << " --device-class <class>\n";
cout << " use device class <class> for new rule\n";
cout << " -i mapfn --remove-rule name\n"
<< " remove the specified crush rule\n";
cout << "\n";
@ -259,7 +264,7 @@ int main(int argc, const char **argv)
int add_item = -1;
bool update_item = false;
bool add_rule = false;
std::string rule_name, rule_root, rule_type, rule_mode;
std::string rule_name, rule_root, rule_type, rule_mode, rule_device_class;
bool del_rule = false;
float add_weight = 0;
map<string,string> add_loc;
@ -450,6 +455,41 @@ int main(int argc, const char **argv)
<< " mode=" << rule_mode
<< std::endl;
add_rule = true;
} else if (ceph_argparse_witharg(args, i, &val, err, "--create-replicated-rule", (char*)NULL)) {
rule_name.assign(val);
if (!err.str().empty()) {
cerr << err.str() << std::endl;
return EXIT_FAILURE;
}
if (i == args.end()) {
cerr << "expecting additional argument to --create-replicated-rule" << std::endl;
return EXIT_FAILURE;
}
rule_root.assign(*i);
i = args.erase(i);
if (i == args.end()) {
cerr << "expecting additional argument to --create-replicated-rule" << std::endl;
return EXIT_FAILURE;
}
rule_type.assign(*i);
i = args.erase(i);
rule_mode = "firstn";
cout << "--create-replicated-rule:"
<< " name=" << rule_name
<< " root=" << rule_root
<< " type=" << rule_type
<< std::endl;
add_rule = true;
} else if (ceph_argparse_witharg(args, i, &val, "--device-class", (char*)NULL)) {
rule_device_class.assign(val);
if (!err.str().empty()) {
cerr << err.str() << std::endl;
return EXIT_FAILURE;
}
} else if (ceph_argparse_witharg(args, i, &val, "--remove-rule", (char*)NULL)) {
rule_name.assign(val);
if (!err.str().empty()) {
@ -896,8 +936,9 @@ int main(int argc, const char **argv)
cerr << "rule " << rule_name << " already exists" << std::endl;
return EXIT_FAILURE;
}
int r = crush.add_simple_rule(rule_name, rule_root, rule_type, rule_mode,
pg_pool_t::TYPE_REPLICATED, &err);
int r = crush.add_simple_rule(rule_name, rule_root, rule_type,
rule_device_class,
rule_mode, pg_pool_t::TYPE_REPLICATED, &err);
if (r < 0) {
cerr << err.str() << std::endl;
return EXIT_FAILURE;