Merge pull request #1046 from dachary/wip-7103

common: recursive implementation of config::expand_meta …

Reviewed-by: Sage Weil <sage@inktank.com>
This commit is contained in:
Sage Weil 2014-01-06 21:30:18 -08:00
commit ffc9d72d4f
7 changed files with 320 additions and 58 deletions

View File

@ -48,6 +48,7 @@
#undef dendl
using std::map;
using std::list;
using std::multimap;
using std::ostringstream;
using std::pair;
@ -229,7 +230,7 @@ int md_config_t::parse_config_files_impl(const std::list<std::string> &conf_file
for (c = conf_files.begin(); c != conf_files.end(); ++c) {
cf.clear();
string fn = *c;
expand_meta(fn);
expand_meta(fn, warnings);
int ret = cf.parse_file(fn.c_str(), parse_errors, warnings);
if (ret == 0)
break;
@ -440,7 +441,7 @@ int md_config_t::parse_argv(std::vector<const char*>& args)
_exit(1);
}
string s = buf;
expand_meta(s);
expand_meta(s, &std::cerr);
std::cout << s << std::endl;
_exit(0);
}
@ -667,7 +668,7 @@ int md_config_t::set_val(const char *key, const char *val, bool meta)
std::string v(val);
if (meta)
expand_meta(v);
expand_meta(v, &std::cerr);
string k(ConfFile::normalize_key_name(key));
@ -850,7 +851,7 @@ int md_config_t::_get_val_from_conf_file(const std::vector <std::string> &sectio
int ret = cf.read(s->c_str(), key, out);
if (ret == 0) {
if (emeta)
expand_meta(out);
expand_meta(out, &std::cerr);
return 0;
}
else if (ret != -ENOENT)
@ -952,28 +953,62 @@ static const int NUM_CONF_METAVARIABLES =
void md_config_t::expand_all_meta()
{
// Expand all metavariables
ostringstream oss;
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
config_option *opt = config_optionsp + i;
if (opt->type == OPT_STR) {
std::string *str = (std::string *)opt->conf_ptr(this);
expand_meta(*str);
list<config_option *> stack;
expand_meta(*str, opt, stack, &oss);
}
}
cerr << oss.str();
}
bool md_config_t::expand_meta(std::string &origval) const
bool md_config_t::expand_meta(std::string &origval,
std::ostream *oss) const
{
list<config_option *> stack;
return expand_meta(origval, NULL, stack, oss);
}
bool md_config_t::expand_meta(std::string &origval,
config_option *opt,
std::list<config_option *> stack,
std::ostream *oss) const
{
assert(lock.is_locked());
// no $ means no variable expansion is necessary
if (origval.find("$") == string::npos)
return false;
// ignore an expansion loop and create a human readable
// message about it
if (opt) {
for (list<config_option *>::iterator i = stack.begin();
i != stack.end();
++i) {
if (strcmp(opt->name, (*i)->name) == 0) {
*oss << "variable expansion loop at "
<< opt->name << "=" << origval << std::endl;
*oss << "expansion stack: " << std::endl;
for (list<config_option *>::iterator j = stack.begin();
j != stack.end();
j++) {
*oss << (*j)->name << "=" << *(string *)(*j)->conf_ptr(this) << std::endl;
}
return false;
}
}
}
if (opt)
stack.push_front(opt);
bool found_meta = false;
set<string> resolved;
string val = origval;
restart:
string out;
out.reserve(val.size());
string val = origval;
for (string::size_type s = 0; s < val.size(); ) {
if (val[s] != '$') {
out += val[s++];
@ -981,7 +1016,7 @@ bool md_config_t::expand_meta(std::string &origval) const
}
// try to parse the variable name into var, either \$\{(.+)\} or
// \$([a-z\_]+)
// \$[a-z\_]+
const char *valid_chars = "abcdefghijklmnopqrstuvwxyz_";
string var;
size_t endpos = 0;
@ -1001,8 +1036,8 @@ bool md_config_t::expand_meta(std::string &origval) const
else
var = val.substr(s+1);
}
//cout << "var='" << var << "'" << std::endl;
bool expanded = false;
if (var.length()) {
// special metavariable?
for (int i = 0; i < NUM_CONF_METAVARIABLES; ++i) {
@ -1025,42 +1060,39 @@ bool md_config_t::expand_meta(std::string &origval) const
out += stringify(getpid());
else
assert(0); // unreachable
found_meta = true;
if (endpos != std::string::npos)
out += val.substr(endpos);
//cout << "val '" << val << "' s " << s << " out '" << out << "'" << std::endl;
val = out;
goto restart;
expanded = true;
}
// config option?
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
config_option *opt = &config_optionsp[i];
if (var != opt->name)
continue;
// avoid loops
if (resolved.count(opt->name))
continue; // loop; skip
resolved.insert(opt->name);
found_meta = true;
char *vv = NULL;
_get_val(opt->name, &vv, -1);
out += vv;
if (endpos != std::string::npos)
out += val.substr(endpos);
//cout << "val '" << val << "' s " << s << " out '" << out << "' after sub " << opt->name << " -> " << vv << std::endl;
val = out;
free(vv);
goto restart;
if (!expanded) {
// config option?
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
config_option *opt = &config_optionsp[i];
if (var == opt->name) {
if (opt->type == OPT_STR) {
string *origval = (string *)opt->conf_ptr(this);
expand_meta(*origval, opt, stack, oss);
out += *origval;
} else {
char *vv = NULL;
_get_val(opt->name, &vv, -1);
out += vv;
free(vv);
}
expanded = true;
break;
}
}
}
}
// pass it thru
out += val[s++];
if (expanded) {
found_meta = true;
s = endpos;
} else {
out += val[s++];
}
}
//cout << "done '" << origval << "' -> '" << out << "'" << std::endl;
// override the original value with the expanded value
origval = out;
return found_meta;
}

View File

@ -173,9 +173,13 @@ private:
void init_subsys();
// Expand metavariables in the provided string.
// Returns true if any metavariables were found and expanded.
bool expand_meta(std::string &val) const;
bool expand_meta(std::string &val,
std::ostream *oss) const;
bool expand_meta(std::string &val,
config_option *opt,
std::list<config_option *> stack,
std::ostream *oss) const;
/// expand all metavariables in config structure.
void expand_all_meta();
@ -233,6 +237,8 @@ public:
* It is best if this lock comes first in the lock hierarchy. We will
* hold this lock when calling configuration observers. */
mutable Mutex lock;
friend class test_md_config_t;
};
typedef enum {

View File

@ -535,6 +535,11 @@ unittest_confutils_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_confutils_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_confutils
unittest_config_SOURCES = test/common/test_config.cc
unittest_config_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_config_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_config
unittest_heartbeatmap_SOURCES = test/heartbeat_map.cc
unittest_heartbeatmap_LDADD = $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_heartbeatmap_CXXFLAGS = $(UNITTEST_CXXFLAGS)

View File

@ -8,3 +8,13 @@
$ ceph-conf -n osd.0 --show-config-value INVALID -c /dev/null
failed to get config option 'INVALID': option not found
[1]
$ echo '[global]' > $TESTDIR/ceph.conf
$ echo 'mon_host=$public_network' >> $TESTDIR/ceph.conf
$ echo 'public_network=$mon_host' >> $TESTDIR/ceph.conf
$ ceph-conf --show-config-value mon_host -c $TESTDIR/ceph.conf
variable expansion loop at public_network=$mon_host
expansion stack:
mon_host=$public_network
public_network=$mon_host
$mon_host
$ rm $TESTDIR/ceph.conf

View File

@ -0,0 +1,187 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
*
* Author: Loic Dachary <loic@dachary.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Library Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Library Public License for more details.
*
*
*/
#include "common/config.h"
#include "common/errno.h"
#include "gtest/gtest.h"
#define _STR(x) #x
#define STRINGIFY(x) _STR(x)
static struct config_option config_optionsp[] = {
#define OPTION(name, type, def_val) \
{ STRINGIFY(name), type, offsetof(struct md_config_t, name) },
#define SUBSYS(name, log, gather)
#define DEFAULT_SUBSYS(log, gather)
#include "common/config_opts.h"
#undef OPTION
#undef SUBSYS
#undef DEFAULT_SUBSYS
};
static const int NUM_CONFIG_OPTIONS = sizeof(config_optionsp) / sizeof(config_option);
class test_md_config_t : public md_config_t, public ::testing::Test {
public:
void test_expand_meta() {
Mutex::Locker l(lock);
// successfull meta expansion $run_dir and ${run_dir}
{
ostringstream oss;
std::string before = " BEFORE ";
std::string after = " AFTER ";
std::string val(before + "$run_dir${run_dir}" + after);
EXPECT_TRUE(expand_meta(val, &oss));
EXPECT_EQ(before + "/var/run/ceph/var/run/ceph" + after, val);
EXPECT_EQ("", oss.str());
}
// no meta expansion if variables are unknown
{
ostringstream oss;
std::string expected = "expect $foo and ${bar} to not expand";
std::string val = expected;
EXPECT_FALSE(expand_meta(val, &oss));
EXPECT_EQ(expected, val);
EXPECT_EQ("", oss.str());
}
// recursive variable expansion
{
std::string mon_host = "$cluster_network";
EXPECT_EQ(0, set_val("mon_host", mon_host.c_str(), false));
std::string lockdep = "true";
EXPECT_EQ(0, set_val("lockdep", lockdep.c_str(), false));
std::string cluster_network = "$public_network $public_network $lockdep $host";
EXPECT_EQ(0, set_val("cluster_network", cluster_network.c_str(), false));
std::string public_network = "NETWORK";
EXPECT_EQ(0, set_val("public_network", public_network.c_str(), false));
ostringstream oss;
std::string val = "$mon_host";
EXPECT_TRUE(expand_meta(val, &oss));
EXPECT_EQ(public_network + " " +
public_network + " " +
lockdep + " " +
"localhost", val);
EXPECT_EQ("", oss.str());
}
// variable expansion loops are non fatal
{
std::string mon_host = "$cluster_network";
EXPECT_EQ(0, set_val("mon_host", mon_host.c_str(), false));
std::string cluster_network = "$public_network";
EXPECT_EQ(0, set_val("cluster_network", cluster_network.c_str(), false));
std::string public_network = "$mon_host";
EXPECT_EQ(0, set_val("public_network", public_network.c_str(), false));
ostringstream oss;
std::string val = "$mon_host";
EXPECT_TRUE(expand_meta(val, &oss));
EXPECT_EQ("$cluster_network", val);
const char *expected_oss =
"variable expansion loop at mon_host=$cluster_network\n"
"expansion stack: \n"
"public_network=$mon_host\n"
"cluster_network=$public_network\n"
"mon_host=$cluster_network\n";
EXPECT_EQ(expected_oss, oss.str());
}
}
void test_expand_all_meta() {
Mutex::Locker l(lock);
int before_count = 0;
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
config_option *opt = config_optionsp + i;
if (opt->type == OPT_STR) {
std::string *str = (std::string *)opt->conf_ptr(this);
if (str->find("$") != string::npos)
before_count++;
}
}
// if there are no meta variables in the default configuration,
// something must be done to check the expected side effect
// of expand_all_meta
ASSERT_LT(0, before_count);
expand_all_meta();
int after_count = 0;
for (int i = 0; i < NUM_CONFIG_OPTIONS; i++) {
config_option *opt = config_optionsp + i;
if (opt->type == OPT_STR) {
std::string *str = (std::string *)opt->conf_ptr(this);
if (str->find("$") != string::npos)
after_count++;
}
}
ASSERT_EQ(0, after_count);
}
};
TEST_F(test_md_config_t, expand_meta)
{
test_expand_meta();
}
TEST_F(test_md_config_t, expand_all_meta)
{
test_expand_all_meta();
}
TEST(md_config_t, set_val)
{
int buf_size = 1024;
md_config_t conf;
// disable meta variable expansion
{
char *buf = (char*)malloc(buf_size);
std::string expected = "$host";
EXPECT_EQ(0, conf.set_val("mon_host", expected.c_str(), false));
EXPECT_EQ(0, conf.get_val("mon_host", &buf, buf_size));
EXPECT_EQ(expected, buf);
free(buf);
}
// meta variable expansion is enabled by default
{
char *run_dir = (char*)malloc(buf_size);
EXPECT_EQ(0, conf.get_val("run_dir", &run_dir, buf_size));
EXPECT_EQ(0, conf.set_val("admin_socket", "$run_dir"));
char *admin_socket = (char*)malloc(buf_size);
EXPECT_EQ(0, conf.get_val("admin_socket", &admin_socket, buf_size));
EXPECT_EQ(std::string(run_dir), std::string(admin_socket));
free(run_dir);
free(admin_socket);
}
}
/*
* Local Variables:
* compile-command: "cd ../.. ;
* make unittest_config &&
* valgrind \
* --max-stackframe=20000000 --tool=memcheck \
* ./unittest_config # --gtest_filter=md_config_t.set_val
* "
* End:
*/

View File

@ -263,7 +263,7 @@ const char dup_key_config_1[] = "\
log_file = 3\n\
";
TEST(Whitespace, ConfUtils) {
TEST(ConfUtils, Whitespace) {
std::string test0("");
ConfFile::trim_whitespace(test0, false);
ASSERT_EQ(test0, "");
@ -301,7 +301,7 @@ TEST(Whitespace, ConfUtils) {
ASSERT_EQ(test5, "abcd");
}
TEST(ParseFiles0, ConfUtils) {
TEST(ConfUtils, ParseFiles0) {
std::deque<std::string> err;
std::string val;
std::ostringstream warn;
@ -332,7 +332,7 @@ TEST(ParseFiles0, ConfUtils) {
ASSERT_EQ(val, "barbaz");
}
TEST(ParseFiles1, ConfUtils) {
TEST(ConfUtils, ParseFiles1) {
std::deque<std::string> err;
std::ostringstream warn;
std::string simple_conf_1_f(next_tempfile(simple_conf_1));
@ -358,7 +358,7 @@ TEST(ParseFiles1, ConfUtils) {
ASSERT_EQ(err.size(), 0U);
}
TEST(ReadFiles1, ConfUtils) {
TEST(ConfUtils, ReadFiles1) {
std::deque<std::string> err;
std::ostringstream warn;
std::string simple_conf_1_f(next_tempfile(simple_conf_1));
@ -391,7 +391,7 @@ TEST(ReadFiles1, ConfUtils) {
ASSERT_EQ(cf2.read("nonesuch", "keyring", val), -ENOENT);
}
TEST(ReadFiles2, ConfUtils) {
TEST(ConfUtils, ReadFiles2) {
std::deque<std::string> err;
std::ostringstream warn;
std::string conf3_f(next_tempfile(conf3));
@ -412,7 +412,7 @@ TEST(ReadFiles2, ConfUtils) {
ASSERT_EQ(val, "\x66\xd1\x86\xd1\x9d\xd3\xad\xd3\xae");
}
TEST(IllegalFiles, ConfUtils) {
TEST(ConfUtils, IllegalFiles) {
std::deque<std::string> err;
std::ostringstream warn;
std::string illegal_conf1_f(next_tempfile(illegal_conf1));
@ -443,7 +443,7 @@ TEST(IllegalFiles, ConfUtils) {
ASSERT_EQ(err.size(), 1U);
}
TEST(EscapingFiles, ConfUtils) {
TEST(ConfUtils, EscapingFiles) {
std::deque<std::string> err;
std::ostringstream warn;
std::string escaping_conf_1_f(next_tempfile(escaping_conf_1));
@ -470,7 +470,7 @@ TEST(EscapingFiles, ConfUtils) {
ASSERT_EQ(val, "backslash\\");
}
TEST(Overrides, ConfUtils) {
TEST(ConfUtils, Overrides) {
md_config_t conf;
std::deque<std::string> err;
std::ostringstream warn;
@ -492,7 +492,7 @@ TEST(Overrides, ConfUtils) {
ASSERT_EQ(conf.log_file, "osd0_log");
}
TEST(DupKey, ConfUtils) {
TEST(ConfUtils, DupKey) {
md_config_t conf;
std::deque<std::string> err;
std::ostringstream warn;

View File

@ -121,6 +121,23 @@ TEST(DaemonConfig, SubstitutionLoop) {
ASSERT_TRUE(strchr(buf, '$') || strchr(buf2, '$'));
}
// config: variable substitution happen only once http://tracker.ceph.com/issues/7103
TEST(DaemonConfig, SubstitutionMultiple) {
int ret;
ret = g_ceph_context->_conf->set_val("mon_host", "localhost", false);
ASSERT_EQ(ret, 0);
ret = g_ceph_context->_conf->set_val("keyring", "$mon_host/$cluster.keyring,$mon_host/$cluster.mon.keyring", false);
ASSERT_EQ(ret, 0);
g_ceph_context->_conf->apply_changes(NULL);
char buf[512];
memset(buf, 0, sizeof(buf));
char *tmp = buf;
ret = g_ceph_context->_conf->get_val("keyring", &tmp, sizeof(buf));
ASSERT_EQ(ret, 0);
ASSERT_EQ(string("localhost/ceph.keyring,localhost/ceph.mon.keyring"), tmp);
ASSERT_TRUE(strchr(buf, '$') == NULL);
}
TEST(DaemonConfig, ArgV) {
ASSERT_EQ(0, g_ceph_context->_conf->set_val("internal_safe_to_start_threads",
"false"));
@ -332,3 +349,8 @@ TEST(DaemonConfig, ThreadSafety1) {
"false"));
ASSERT_EQ(ret, 0);
}
/*
* Local Variables:
* compile-command: "cd .. ; make unittest_daemon_config && ./unittest_daemon_config"
* End:
*/