RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-03 01:22:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #56735 from samarahu/wip-rgw-d4n-long-descriptions

Browse Source 
d4n: Add more in-depth config descriptions and improve naming

Reviewed-by: Casey Bodley <cbodley@redhat.com>
Reviewed-by: Samarah Uriarte <samarah.uriarte@ibm.com>
This commit is contained in:
Samarah Uriarte 2024-05-10 13:35:35 -04:00 committed by GitHub
commit 668d84068b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 73 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
doc/radosgw/config-ref.rst
View File

@ -312,8 +312,15 @@ Currently, the architecture can only function on one Redis instance at a time.
The address is configurable and can be changed by accessing the parameters
below.
.. confval:: rgw_d4n_host
.. confval:: rgw_d4n_port
.. confval:: rgw_d4n_address
.. confval:: rgw_d4n_l1_datacache_persistent_path
.. confval:: rgw_d4n_l1_datacache_size
.. confval:: rgw_d4n_l1_evict_cache_on_start
.. confval:: rgw_d4n_l1_fadvise
.. confval:: rgw_d4n_libaio_aio_threads
.. confval:: rgw_d4n_libaio_aio_num
.. confval:: rgw_lfuda_sync_frequency
.. confval:: rgw_d4n_l1_datacache_address
Topic persistency settings
==========================
@ -335,4 +342,4 @@ and retried as frequently as possible).
.. confval:: rgw_topic_persistency_max_retries
.. confval:: rgw_topic_persistency_sleep_duration
.. _Bucket Notifications: ../notifications
.. _Bucket Notifications: ../notifications

58
src/common/options/rgw.yaml.in
View File

@ -3649,7 +3649,12 @@ options:
- name: rgw_d4n_l1_datacache_persistent_path
type: str
level: advanced
desc: path for the directory for storing the local cache objects data for d4n
desc: path used for storing locally cached object data
long_desc: One cache backend option for D4N is the local SSD, which uses this path to
write and read object data. This is the default cache backend chosen by the D4N filter.
Only the SSD cache backend uses this path for object data storage since the RedisDriver
uses a Redis server instead and there are no additional cache backend implementations
available at the moment.
default: /tmp/rgw_d4n_datacache/
services:
- rgw
@ -3657,7 +3662,9 @@ options:
- name: rgw_d4n_l1_datacache_size
type: size
level: advanced
desc: datacache maximum size on disk in bytes
desc: maximum size on disk for datacache
long_desc: The local SSD cache uses this option to configure its size in bytes. This
option is not used by the Redis cache backend.
default: 1_G
services:
- rgw
@ -3665,7 +3672,10 @@ options:
- name: rgw_d4n_l1_evict_cache_on_start
type: bool
level: advanced
desc: clear the content of the persistent data cache directory on start
desc: clear the contents of the persistent datacache on start
long_desc: The local SSD cache uses this option to clear the contents of the path supplied
by the rgw_d4n_l1_datacache_persistent_path config option on start. If false, the path's
contents will be retained.
default: true
services:
- rgw
@ -3674,7 +3684,7 @@ options:
type: int
level: advanced
desc: posix_fadvise() flag for access pattern of cache files
long_desc: for example to bypass the page-cache -
long_desc: For example, to bypass the page-cache -
POSIX_FADV_DONTNEED=4
default: 4
services:
@ -3684,6 +3694,8 @@ options:
type: int
level: advanced
desc: specifies the maximum number of worker threads that may be used by libaio
long_desc: This option is used by the SSD cache backend during initialization to set the maximum
number of worker threads libaio may use. It does not apply to the Redis cache backend.
default: 20
services:
- rgw
@ -3694,6 +3706,9 @@ options:
type: int
level: advanced
desc: specifies the maximum number of simultaneous I/O requests that libaio expects to enqueue
long_desc: This option is used by the SSD cache backend during initialization to set the maximum
number of simultaneous I/O requests that libaio can expect to enqueue. It
does not apply to the Redis cache backend.
default: 64
services:
- rgw
@ -3704,6 +3719,12 @@ options:
type: int
level: advanced
desc: LFUDA variables' sync frequency in seconds
long_desc: By default, the D4N cache uses the Least Frequently Used with Dynamic Aging (LFUDA)
cache replacement policy. This class globally stores values that are used by the policy's
algorithm. However, strong consistency for these values is not necessary and adds additional
overhead to support. As a result, a thread periodically retrieves these global values and posts
updates when certain conditions are satisfied. This Redis thread completes this logic in a loop
that is called once every interval, with the interval being set by this option.
default: 60
services:
- rgw
@ -3957,21 +3978,15 @@ options:
default: true
services:
- rgw
- name: rgw_d4n_host
- name: rgw_d4n_address
type: str
level: advanced
desc: The rgw directory host
default: 127.0.0.1
services:
- rgw
flags:
- startup
with_legacy: true
- name: rgw_d4n_port
type: int
level: advanced
desc: The rgw directory port
default: 6379
desc: address for the D4N Redis connection
long_desc: The current D4N implementation supports one Redis node
which the D4N directory, policy, and overall filter communicate
with. This default value is also the address that a Redis server
with no additional configuration will use.
default: 127.0.0.1:6379
services:
- rgw
flags:
@ -4114,12 +4129,15 @@ options:
services:
- rgw
with_legacy: true
- name: rgw_local_cache_address
- name: rgw_d4n_l1_datacache_address
type: str
level: advanced
desc: Local CacheDriver address
desc: local Redis cache address
long_desc: This is the address used to configure the Redis cache backend connection. The default
value is the same address used by Redis without any additional configuration. The SSD cache
does not use this option.
default: 127.0.0.1:6379
services:
services:
- rgw
flags:
- startup

12
src/rgw/driver/d4n/d4n_policy.cc
View File

@ -58,7 +58,7 @@ int LFUDAPolicy::init(CephContext *cct, const DoutPrefixProvider* dpp, asio::io_
req.push("HEXISTS", "lfuda", "age");
req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum)); /* New cache node will always have the minimum average weight */
req.push("HSET", "lfuda", "minLocalWeights_size", std::to_string(entries_map.size()));
req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_local_cache_address);
req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_d4n_l1_datacache_address);
redis_exec(conn, ec, req, resp, y);
@ -169,7 +169,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield
response<int, int, int> value;
req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum));
req.push("HSET", "lfuda", "minLocalWeights_size", std::to_string(entries_map.size()));
req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_local_cache_address);
req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_d4n_l1_datacache_address);
redis_exec(conn, ec, req, resp, y);
if (ec) {
@ -191,8 +191,8 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield
boost::system::error_code ec;
request req;
response<int, int> resp;
req.push("HSET", dpp->get_cct()->_conf->rgw_local_cache_address, "avgLocalWeight_sum", std::to_string(weightSum));
req.push("HSET", dpp->get_cct()->_conf->rgw_local_cache_address, "avgLocalWeight_size", std::to_string(entries_map.size()));
req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_sum", std::to_string(weightSum));
req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_size", std::to_string(entries_map.size()));
redis_exec(conn, ec, req, resp, y);
if (ec) {
@ -291,7 +291,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional
int avgWeight = weightSum / entries_map.size();
if (victim->hostsList.size() == 1 && victim->hostsList[0] == dir->cct->_conf->rgw_local_cache_address) { /* Last copy */
if (victim->hostsList.size() == 1 && victim->hostsList[0] == dir->cct->_conf->rgw_d4n_l1_datacache_address) { /* Last copy */
if (victim->globalWeight) {
it->second->localWeight += victim->globalWeight;
(*it->second->handle)->localWeight = it->second->localWeight;
@ -321,7 +321,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional
return ret;
}
if (int ret = dir->remove_host(victim, dir->cct->_conf->rgw_local_cache_address, y) < 0) {
if (int ret = dir->remove_host(victim, dir->cct->_conf->rgw_d4n_l1_datacache_address, y) < 0) {
delete victim;
return ret;
}

12
src/rgw/driver/d4n/rgw_sal_d4n.cc
View File

@ -66,7 +66,7 @@ int D4NFilterDriver::initialize(CephContext *cct, const DoutPrefixProvider *dpp)
namespace net = boost::asio;
using boost::redis::config;
std::string address = cct->_conf->rgw_local_cache_address;
std::string address = cct->_conf->rgw_d4n_address;
config cfg;
cfg.addr.host = address.substr(0, address.find(":"));
cfg.addr.port = address.substr(address.find(":") + 1, address.length());
@ -699,7 +699,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
if (write_to_cache) {
rgw::d4n::CacheBlock block, existing_block;
rgw::d4n::BlockDirectory* blockDir = source->driver->get_block_dir();
block.hostsList.push_back(blockDir->cct->_conf->rgw_local_cache_address);
block.hostsList.push_back(blockDir->cct->_conf->rgw_d4n_l1_datacache_address);
block.cacheObj.objName = source->get_key().get_oid();
block.cacheObj.bucketName = source->get_bucket()->get_name();
std::stringstream s;
@ -747,7 +747,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
if (blockDir->set(&block, *y) < 0) //new versioned block will have new version, hostsList etc, how about globalWeight?
ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl;
} else {
if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0)
if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0)
ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed for hostsList." << dendl;
}
}
@ -787,7 +787,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
if (blockDir->set(&block, *y) < 0)
ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl;
} else {
if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0)
if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0)
ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed for blockHosts." << dendl;
}
}
@ -834,7 +834,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
if (blockDir->set(&block, *y) < 0)
ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl;
} else {
if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0)
if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0)
ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed." << dendl;
}
}
@ -926,7 +926,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
.bucketName = obj->get_bucket()->get_name(),
.creationTime = to_iso_8601(*mtime),
.dirty = false,
.hostsList = { /*driver->get_block_dir()->cct->_conf->rgw_local_cache_address*/ } //TODO: Object is not currently being cached
.hostsList = { /*driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address*/ } //TODO: Object is not currently being cached
};
if (driver->get_obj_dir()->set(&object, y) < 0)

2
src/rgw/rgw_redis_driver.cc
View File

@ -68,7 +68,7 @@ int RedisDriver::initialize(const DoutPrefixProvider* dpp)
partition_info.location += "/";
}
std::string address = dpp->get_cct()->_conf->rgw_local_cache_address;
std::string address = dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address;
config cfg;
cfg.addr.host = address.substr(0, address.find(":"));

10
src/test/rgw/test_d4n_directory.cc
View File

@ -34,7 +34,7 @@ class Environment : public ::testing::Environment {
cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {});
dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: ");
redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port);
redisHost = cct->_conf->rgw_d4n_address;
}
void TearDown() override {
@ -67,8 +67,8 @@ class ObjectDirectoryFixture: public ::testing::Test {
/* Run fixture's connection */
config cfg;
cfg.addr.host = env->cct->_conf->rgw_d4n_host;
cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length());
conn->async_run(cfg, {}, net::detached);
}
@ -115,8 +115,8 @@ class BlockDirectoryFixture: public ::testing::Test {
/* Run fixture's connection */
config cfg;
cfg.addr.host = env->cct->_conf->rgw_d4n_host;
cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length());
conn->async_run(cfg, {}, net::detached);
}

8
src/test/rgw/test_d4n_policy.cc
View File

@ -36,7 +36,7 @@ class Environment : public ::testing::Environment {
cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {});
dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: ");
redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port);
redisHost = cct->_conf->rgw_d4n_address;
}
std::string redisHost;
@ -82,8 +82,8 @@ class LFUDAPolicyFixture : public ::testing::Test {
/* Run fixture's connection */
config cfg;
cfg.addr.host = env->cct->_conf->rgw_d4n_host;
cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length());
conn->async_run(cfg, {}, net::detached);
}
@ -130,7 +130,7 @@ class LFUDAPolicyFixture : public ::testing::Test {
}
}
} else if (!exists) { /* No remote copy */
block->hostsList.push_back(dir->cct->_conf->rgw_local_cache_address);
block->hostsList.push_back(dir->cct->_conf->rgw_d4n_l1_datacache_address);
if (dir->set(block, y) < 0)
return -1;

6
src/test/rgw/test_redis_driver.cc
View File

@ -74,7 +74,7 @@ class Environment : public ::testing::Environment {
cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {});
dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: ");
redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port);
redisHost = cct->_conf->rgw_d4n_address;
}
std::string redisHost;
@ -102,8 +102,8 @@ class RedisDriverFixture: public ::testing::Test {
/* Run fixture's connection */
config cfg;
cfg.addr.host = env->cct->_conf->rgw_d4n_host;
cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port);
cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":"));
cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length());
conn->async_run(cfg, {}, net::detached);
}