diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst index 9732a2c830c..0c374d1ae4e 100644 --- a/doc/radosgw/config-ref.rst +++ b/doc/radosgw/config-ref.rst @@ -312,8 +312,15 @@ Currently, the architecture can only function on one Redis instance at a time. The address is configurable and can be changed by accessing the parameters below. -.. confval:: rgw_d4n_host -.. confval:: rgw_d4n_port +.. confval:: rgw_d4n_address +.. confval:: rgw_d4n_l1_datacache_persistent_path +.. confval:: rgw_d4n_l1_datacache_size +.. confval:: rgw_d4n_l1_evict_cache_on_start +.. confval:: rgw_d4n_l1_fadvise +.. confval:: rgw_d4n_libaio_aio_threads +.. confval:: rgw_d4n_libaio_aio_num +.. confval:: rgw_lfuda_sync_frequency +.. confval:: rgw_d4n_l1_datacache_address Topic persistency settings ========================== @@ -335,4 +342,4 @@ and retried as frequently as possible). .. confval:: rgw_topic_persistency_max_retries .. confval:: rgw_topic_persistency_sleep_duration -.. _Bucket Notifications: ../notifications \ No newline at end of file +.. _Bucket Notifications: ../notifications diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 9b2731f9f5a..3bb4942a9e3 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3649,7 +3649,12 @@ options: - name: rgw_d4n_l1_datacache_persistent_path type: str level: advanced - desc: path for the directory for storing the local cache objects data for d4n + desc: path used for storing locally cached object data + long_desc: One cache backend option for D4N is the local SSD, which uses this path to + write and read object data. This is the default cache backend chosen by the D4N filter. + Only the SSD cache backend uses this path for object data storage since the RedisDriver + uses a Redis server instead and there are no additional cache backend implementations + available at the moment. default: /tmp/rgw_d4n_datacache/ services: - rgw @@ -3657,7 +3662,9 @@ options: - name: rgw_d4n_l1_datacache_size type: size level: advanced - desc: datacache maximum size on disk in bytes + desc: maximum size on disk for datacache + long_desc: The local SSD cache uses this option to configure its size in bytes. This + option is not used by the Redis cache backend. default: 1_G services: - rgw @@ -3665,7 +3672,10 @@ options: - name: rgw_d4n_l1_evict_cache_on_start type: bool level: advanced - desc: clear the content of the persistent data cache directory on start + desc: clear the contents of the persistent datacache on start + long_desc: The local SSD cache uses this option to clear the contents of the path supplied + by the rgw_d4n_l1_datacache_persistent_path config option on start. If false, the path's + contents will be retained. default: true services: - rgw @@ -3674,7 +3684,7 @@ options: type: int level: advanced desc: posix_fadvise() flag for access pattern of cache files - long_desc: for example to bypass the page-cache - + long_desc: For example, to bypass the page-cache - POSIX_FADV_DONTNEED=4 default: 4 services: @@ -3684,6 +3694,8 @@ options: type: int level: advanced desc: specifies the maximum number of worker threads that may be used by libaio + long_desc: This option is used by the SSD cache backend during initialization to set the maximum + number of worker threads libaio may use. It does not apply to the Redis cache backend. default: 20 services: - rgw @@ -3694,6 +3706,9 @@ options: type: int level: advanced desc: specifies the maximum number of simultaneous I/O requests that libaio expects to enqueue + long_desc: This option is used by the SSD cache backend during initialization to set the maximum + number of simultaneous I/O requests that libaio can expect to enqueue. It + does not apply to the Redis cache backend. default: 64 services: - rgw @@ -3704,6 +3719,12 @@ options: type: int level: advanced desc: LFUDA variables' sync frequency in seconds + long_desc: By default, the D4N cache uses the Least Frequently Used with Dynamic Aging (LFUDA) + cache replacement policy. This class globally stores values that are used by the policy's + algorithm. However, strong consistency for these values is not necessary and adds additional + overhead to support. As a result, a thread periodically retrieves these global values and posts + updates when certain conditions are satisfied. This Redis thread completes this logic in a loop + that is called once every interval, with the interval being set by this option. default: 60 services: - rgw @@ -3957,21 +3978,15 @@ options: default: true services: - rgw -- name: rgw_d4n_host +- name: rgw_d4n_address type: str level: advanced - desc: The rgw directory host - default: 127.0.0.1 - services: - - rgw - flags: - - startup - with_legacy: true -- name: rgw_d4n_port - type: int - level: advanced - desc: The rgw directory port - default: 6379 + desc: address for the D4N Redis connection + long_desc: The current D4N implementation supports one Redis node + which the D4N directory, policy, and overall filter communicate + with. This default value is also the address that a Redis server + with no additional configuration will use. + default: 127.0.0.1:6379 services: - rgw flags: @@ -4114,12 +4129,15 @@ options: services: - rgw with_legacy: true -- name: rgw_local_cache_address +- name: rgw_d4n_l1_datacache_address type: str level: advanced - desc: Local CacheDriver address + desc: local Redis cache address + long_desc: This is the address used to configure the Redis cache backend connection. The default + value is the same address used by Redis without any additional configuration. The SSD cache + does not use this option. default: 127.0.0.1:6379 - services: + services: - rgw flags: - startup diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc index 08386cf419e..9e2fa358f81 100644 --- a/src/rgw/driver/d4n/d4n_policy.cc +++ b/src/rgw/driver/d4n/d4n_policy.cc @@ -58,7 +58,7 @@ int LFUDAPolicy::init(CephContext *cct, const DoutPrefixProvider* dpp, asio::io_ req.push("HEXISTS", "lfuda", "age"); req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum)); /* New cache node will always have the minimum average weight */ req.push("HSET", "lfuda", "minLocalWeights_size", std::to_string(entries_map.size())); - req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_local_cache_address); + req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_d4n_l1_datacache_address); redis_exec(conn, ec, req, resp, y); @@ -169,7 +169,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield response value; req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum)); req.push("HSET", "lfuda", "minLocalWeights_size", std::to_string(entries_map.size())); - req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_local_cache_address); + req.push("HSET", "lfuda", "minLocalWeights_address", dir->cct->_conf->rgw_d4n_l1_datacache_address); redis_exec(conn, ec, req, resp, y); if (ec) { @@ -191,8 +191,8 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield boost::system::error_code ec; request req; response resp; - req.push("HSET", dpp->get_cct()->_conf->rgw_local_cache_address, "avgLocalWeight_sum", std::to_string(weightSum)); - req.push("HSET", dpp->get_cct()->_conf->rgw_local_cache_address, "avgLocalWeight_size", std::to_string(entries_map.size())); + req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_sum", std::to_string(weightSum)); + req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_size", std::to_string(entries_map.size())); redis_exec(conn, ec, req, resp, y); if (ec) { @@ -291,7 +291,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional int avgWeight = weightSum / entries_map.size(); - if (victim->hostsList.size() == 1 && victim->hostsList[0] == dir->cct->_conf->rgw_local_cache_address) { /* Last copy */ + if (victim->hostsList.size() == 1 && victim->hostsList[0] == dir->cct->_conf->rgw_d4n_l1_datacache_address) { /* Last copy */ if (victim->globalWeight) { it->second->localWeight += victim->globalWeight; (*it->second->handle)->localWeight = it->second->localWeight; @@ -321,7 +321,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional return ret; } - if (int ret = dir->remove_host(victim, dir->cct->_conf->rgw_local_cache_address, y) < 0) { + if (int ret = dir->remove_host(victim, dir->cct->_conf->rgw_d4n_l1_datacache_address, y) < 0) { delete victim; return ret; } diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 1afd6822fe2..03d3295f3c1 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -66,7 +66,7 @@ int D4NFilterDriver::initialize(CephContext *cct, const DoutPrefixProvider *dpp) namespace net = boost::asio; using boost::redis::config; - std::string address = cct->_conf->rgw_local_cache_address; + std::string address = cct->_conf->rgw_d4n_address; config cfg; cfg.addr.host = address.substr(0, address.find(":")); cfg.addr.port = address.substr(address.find(":") + 1, address.length()); @@ -699,7 +699,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (write_to_cache) { rgw::d4n::CacheBlock block, existing_block; rgw::d4n::BlockDirectory* blockDir = source->driver->get_block_dir(); - block.hostsList.push_back(blockDir->cct->_conf->rgw_local_cache_address); + block.hostsList.push_back(blockDir->cct->_conf->rgw_d4n_l1_datacache_address); block.cacheObj.objName = source->get_key().get_oid(); block.cacheObj.bucketName = source->get_bucket()->get_name(); std::stringstream s; @@ -747,7 +747,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (blockDir->set(&block, *y) < 0) //new versioned block will have new version, hostsList etc, how about globalWeight? ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl; } else { - if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0) + if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0) ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed for hostsList." << dendl; } } @@ -787,7 +787,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (blockDir->set(&block, *y) < 0) ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl; } else { - if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0) + if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0) ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed for blockHosts." << dendl; } } @@ -834,7 +834,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (blockDir->set(&block, *y) < 0) ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set method failed." << dendl; } else { - if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_local_cache_address, *y) < 0) + if (blockDir->update_field(&block, "blockHosts", blockDir->cct->_conf->rgw_d4n_l1_datacache_address, *y) < 0) ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory update_field method failed." << dendl; } } @@ -926,7 +926,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, .bucketName = obj->get_bucket()->get_name(), .creationTime = to_iso_8601(*mtime), .dirty = false, - .hostsList = { /*driver->get_block_dir()->cct->_conf->rgw_local_cache_address*/ } //TODO: Object is not currently being cached + .hostsList = { /*driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address*/ } //TODO: Object is not currently being cached }; if (driver->get_obj_dir()->set(&object, y) < 0) diff --git a/src/rgw/rgw_redis_driver.cc b/src/rgw/rgw_redis_driver.cc index a07507b5454..183f5351e2f 100644 --- a/src/rgw/rgw_redis_driver.cc +++ b/src/rgw/rgw_redis_driver.cc @@ -68,7 +68,7 @@ int RedisDriver::initialize(const DoutPrefixProvider* dpp) partition_info.location += "/"; } - std::string address = dpp->get_cct()->_conf->rgw_local_cache_address; + std::string address = dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address; config cfg; cfg.addr.host = address.substr(0, address.find(":")); diff --git a/src/test/rgw/test_d4n_directory.cc b/src/test/rgw/test_d4n_directory.cc index 59aceabd2a0..c518eb50c7d 100644 --- a/src/test/rgw/test_d4n_directory.cc +++ b/src/test/rgw/test_d4n_directory.cc @@ -34,7 +34,7 @@ class Environment : public ::testing::Environment { cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {}); dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: "); - redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port); + redisHost = cct->_conf->rgw_d4n_address; } void TearDown() override { @@ -67,8 +67,8 @@ class ObjectDirectoryFixture: public ::testing::Test { /* Run fixture's connection */ config cfg; - cfg.addr.host = env->cct->_conf->rgw_d4n_host; - cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port); + cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":")); + cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); conn->async_run(cfg, {}, net::detached); } @@ -115,8 +115,8 @@ class BlockDirectoryFixture: public ::testing::Test { /* Run fixture's connection */ config cfg; - cfg.addr.host = env->cct->_conf->rgw_d4n_host; - cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port); + cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":")); + cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); conn->async_run(cfg, {}, net::detached); } diff --git a/src/test/rgw/test_d4n_policy.cc b/src/test/rgw/test_d4n_policy.cc index fbdbc148761..fe277536003 100644 --- a/src/test/rgw/test_d4n_policy.cc +++ b/src/test/rgw/test_d4n_policy.cc @@ -36,7 +36,7 @@ class Environment : public ::testing::Environment { cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {}); dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: "); - redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port); + redisHost = cct->_conf->rgw_d4n_address; } std::string redisHost; @@ -82,8 +82,8 @@ class LFUDAPolicyFixture : public ::testing::Test { /* Run fixture's connection */ config cfg; - cfg.addr.host = env->cct->_conf->rgw_d4n_host; - cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port); + cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":")); + cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); conn->async_run(cfg, {}, net::detached); } @@ -130,7 +130,7 @@ class LFUDAPolicyFixture : public ::testing::Test { } } } else if (!exists) { /* No remote copy */ - block->hostsList.push_back(dir->cct->_conf->rgw_local_cache_address); + block->hostsList.push_back(dir->cct->_conf->rgw_d4n_l1_datacache_address); if (dir->set(block, y) < 0) return -1; diff --git a/src/test/rgw/test_redis_driver.cc b/src/test/rgw/test_redis_driver.cc index 18d53ab1be4..e8ca1de6d55 100644 --- a/src/test/rgw/test_redis_driver.cc +++ b/src/test/rgw/test_redis_driver.cc @@ -74,7 +74,7 @@ class Environment : public ::testing::Environment { cct = common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, {}); dpp = new DoutPrefix(cct->get(), dout_subsys, "D4N Object Directory Test: "); - redisHost = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port); + redisHost = cct->_conf->rgw_d4n_address; } std::string redisHost; @@ -102,8 +102,8 @@ class RedisDriverFixture: public ::testing::Test { /* Run fixture's connection */ config cfg; - cfg.addr.host = env->cct->_conf->rgw_d4n_host; - cfg.addr.port = std::to_string(env->cct->_conf->rgw_d4n_port); + cfg.addr.host = env->redisHost.substr(0, env->redisHost.find(":")); + cfg.addr.port = env->redisHost.substr(env->redisHost.find(":") + 1, env->redisHost.length()); conn->async_run(cfg, {}, net::detached); }