From cb7f43cfacc11ae87e8b65ddce044be62144e791 Mon Sep 17 00:00:00 2001 From: xie xingguo Date: Sat, 27 May 2017 17:30:24 +0800 Subject: [PATCH] os/bluestore: move cache_trim into MempoolThread Signed-off-by: xie xingguo --- src/common/config_opts.h | 2 +- src/os/bluestore/BlueStore.cc | 67 +++++++++++------------------------ src/os/bluestore/BlueStore.h | 16 --------- 3 files changed, 21 insertions(+), 64 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index f693913bf99..4fcd3d7c854 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1119,7 +1119,7 @@ OPTION(bluestore_extent_map_shard_target_size, OPT_U32, 500) OPTION(bluestore_extent_map_shard_min_size, OPT_U32, 150) OPTION(bluestore_extent_map_shard_target_size_slop, OPT_DOUBLE, .2) OPTION(bluestore_extent_map_inline_shard_prealloc_size, OPT_U32, 256) -OPTION(bluestore_cache_trim_interval, OPT_DOUBLE, .1) +OPTION(bluestore_cache_trim_interval, OPT_DOUBLE, .2) OPTION(bluestore_cache_trim_max_skip_pinned, OPT_U32, 64) // skip this many onodes pinned in cache before we give up OPTION(bluestore_cache_type, OPT_STR, "2q") // lru, 2q OPTION(bluestore_2q_cache_kin_ratio, OPT_DOUBLE, .5) // kin page slot size / max page slot size diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index e5bccea2e94..8699a278099 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -3228,48 +3228,33 @@ void BlueStore::Collection::split_cache( } } -void BlueStore::Collection::trim_cache() -{ - // see if mempool stats have updated - uint64_t total_bytes; - uint64_t total_onodes; - size_t seq; - store->get_mempool_stats(&seq, &total_bytes, &total_onodes); - if (seq == cache->last_trim_seq) { - ldout(store->cct, 30) << __func__ << " no new mempool stats; nothing to do" - << dendl; - return; - } - cache->last_trim_seq = seq; - - // trim - if (total_onodes < 2) { - total_onodes = 2; - } - float bytes_per_onode = (float)total_bytes / (float)total_onodes; - size_t num_shards = store->cache_shards.size(); - uint64_t shard_target = store->cct->_conf->bluestore_cache_size / num_shards; - ldout(store->cct, 30) << __func__ - << " total meta bytes " << total_bytes - << ", total onodes " << total_onodes - << ", bytes_per_onode " << bytes_per_onode - << dendl; - cache->trim(shard_target, store->cct->_conf->bluestore_cache_meta_ratio, - bytes_per_onode); - - store->_update_cache_logger(); -} - // ======================================================= void *BlueStore::MempoolThread::entry() { Mutex::Locker l(lock); while (!stop) { - store->mempool_bytes = mempool::bluestore_cache_other::allocated_bytes() + + uint64_t meta_bytes = + mempool::bluestore_cache_other::allocated_bytes() + mempool::bluestore_cache_onode::allocated_bytes(); - store->mempool_onodes = mempool::bluestore_cache_onode::allocated_items(); - ++store->mempool_seq; + uint64_t onode_num = + mempool::bluestore_cache_onode::allocated_items(); + + if (onode_num < 2) { + onode_num = 2; + } + + float bytes_per_onode = (float)meta_bytes / (float)onode_num; + size_t num_shards = store->cache_shards.size(); + uint64_t shard_target = store->cct->_conf->bluestore_cache_size / num_shards; + + for (auto i : store->cache_shards) { + i->trim(shard_target, store->cct->_conf->bluestore_cache_meta_ratio, + bytes_per_onode); + } + + store->_update_cache_logger(); + utime_t wait; wait += store->cct->_conf->bluestore_cache_trim_interval; cond.WaitInterval(lock, wait); @@ -5552,7 +5537,6 @@ int BlueStore::fsck(bool deep) used_omap_head.insert(o->onode.nid); } } - c->trim_cache(); } } dout(1) << __func__ << " checking shared_blobs" << dendl; @@ -5892,7 +5876,6 @@ bool BlueStore::exists(CollectionHandle &c_, const ghobject_t& oid) r = false; } - c->trim_cache(); return r; } @@ -5930,7 +5913,6 @@ int BlueStore::stat( st->st_nlink = 1; } - c->trim_cache(); int r = 0; if (_debug_mdata_eio(oid)) { r = -EIO; @@ -6007,7 +5989,6 @@ int BlueStore::read( out: assert(allow_eio || r != -EIO); - c->trim_cache(); if (r == 0 && _debug_data_eio(oid)) { r = -EIO; derr << __func__ << " " << c->cid << " " << oid << " INJECT EIO" << dendl; @@ -6447,7 +6428,6 @@ int BlueStore::_fiemap( } out: - c->trim_cache(); dout(20) << __func__ << " 0x" << std::hex << offset << "~" << length << " size = 0x(" << destset << ")" << std::dec << dendl; return 0; @@ -6551,7 +6531,6 @@ int BlueStore::getattr( r = 0; } out: - c->trim_cache(); if (r == 0 && _debug_mdata_eio(oid)) { r = -EIO; derr << __func__ << " " << c->cid << " " << oid << " INJECT EIO" << dendl; @@ -6599,7 +6578,6 @@ int BlueStore::getattrs( } out: - c->trim_cache(); if (r == 0 && _debug_mdata_eio(oid)) { r = -EIO; derr << __func__ << " " << c->cid << " " << oid << " INJECT EIO" << dendl; @@ -6676,7 +6654,6 @@ int BlueStore::collection_list( r = _collection_list(c, start, end, max, ls, pnext); } - c->trim_cache(); dout(10) << __func__ << " " << c->cid << " start " << start << " end " << end << " max " << max << " = " << r << ", ls.size() = " << ls->size() @@ -7861,10 +7838,6 @@ void BlueStore::_txc_finish(TransContext *txc) delete txc; } - if (c) { - c->trim_cache(); - } - if (submit_deferred) { // we're pinning memory; flush! we could be more fine-grained here but // i'm not sure it's worth the bother. diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index f2f3ace5dce..90aa23ff392 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1036,8 +1036,6 @@ public: std::atomic num_extents = {0}; std::atomic num_blobs = {0}; - size_t last_trim_seq = 0; - static Cache *create(CephContext* cct, string type, PerfCounters *logger); Cache(CephContext* cct) : cct(cct), logger(nullptr) {} @@ -1363,7 +1361,6 @@ public: } void split_cache(Collection *dest); - void trim_cache(); Collection(BlueStore *ns, Cache *ca, coll_t c); }; @@ -1889,19 +1886,6 @@ private: // cache trim control - // note that these update in a racy way, but we don't *really* care if - // they're perfectly accurate. they are all word sized so they will - // individually update atomically, but may not be coherent with each other. - size_t mempool_seq = 0; - size_t mempool_bytes = 0; - size_t mempool_onodes = 0; - - void get_mempool_stats(size_t *seq, uint64_t *bytes, uint64_t *onodes) { - *seq = mempool_seq; - *bytes = mempool_bytes; - *onodes = mempool_onodes; - } - struct MempoolThread : public Thread { BlueStore *store; Cond cond;