diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index 6d513155681..581ae2d0d91 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -3958,6 +3958,12 @@ next: << "(" << path << "), rewriting it"; in->_mark_dirty_parent(in->mdcache->mds->mdlog->get_current_segment(), false); + // Flag that we repaired this BT so that it won't go into damagetable + results->backtrace.repaired = true; + + // Flag that we did some repair work so that our repair operation + // can be flushed at end of scrub + in->scrub_infop->header->set_repaired(); } // If the inode's number was free in the InoTable, fix that @@ -4096,6 +4102,7 @@ next: dir->scrub_infop->pending_scrub_error) { dir->scrub_infop->pending_scrub_error = false; if (dir->scrub_infop->header->get_repair()) { + results->raw_stats.repaired = true; results->raw_stats.error_str << "dirfrag(" << p->first << ") has bad stats (will be fixed); "; } else { @@ -4114,6 +4121,7 @@ next: results->raw_stats.error_str << "freshly-calculated rstats don't match existing ones (will be fixed)"; in->mdcache->repair_inode_stats(in); + results->raw_stats.repaired = true; } else { results->raw_stats.error_str << "freshly-calculated rstats don't match existing ones"; @@ -4188,6 +4196,18 @@ void CInode::validated_data::dump(Formatter *f) const f->close_section(); // results } +bool CInode::validated_data::all_damage_repaired() const +{ + bool unrepaired = + (raw_stats.checked && !raw_stats.passed && !raw_stats.repaired) + || + (backtrace.checked && !backtrace.passed && !backtrace.repaired) + || + (inode.checked && !inode.passed && !inode.repaired); + + return !unrepaired; +} + void CInode::dump(Formatter *f) const { InodeStoreBase::dump(f); @@ -4323,7 +4343,7 @@ void CInode::scrub_maybe_delete_info() } void CInode::scrub_initialize(CDentry *scrub_parent, - const ScrubHeaderRefConst& header, + ScrubHeaderRef& header, MDSInternalContextBase *f) { dout(20) << __func__ << " with scrub_version " << get_version() << dendl; @@ -4453,7 +4473,12 @@ void CInode::scrub_finished(MDSInternalContextBase **c) { if (scrub_infop->header->get_origin() == this) { // We are at the point that a tagging scrub was initiated LogChannelRef clog = mdcache->mds->clog; - clog->info() << "scrub complete with tag '" << scrub_infop->header->get_tag() << "'"; + if (scrub_infop->header->get_tag().empty()) { + clog->info() << "scrub complete"; + } else { + clog->info() << "scrub complete with tag '" + << scrub_infop->header->get_tag() << "'"; + } } } diff --git a/src/mds/CInode.h b/src/mds/CInode.h index d0a49d1feb5..01c942c9cf5 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -273,7 +273,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter dirfrag_stamps; - ScrubHeaderRefConst header; + ScrubHeaderRef header; scrub_info_t() : scrub_stamp_info_t(), scrub_parent(NULL), on_finish(NULL), @@ -287,6 +287,14 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counterheader; + } + } + bool scrub_is_in_progress() const { return (scrub_infop && scrub_infop->scrub_in_progress); } @@ -299,7 +307,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counterstruct member_status { - bool checked; - bool passed; - int ondisk_read_retval; + bool checked = false; + bool passed = false; + bool repaired = false; + int ondisk_read_retval = 0; T ondisk_value; T memory_value; std::stringstream error_str; - member_status() : checked(false), passed(false), - ondisk_read_retval(0) {} }; bool performed_validation; @@ -1134,6 +1141,8 @@ public: passed_validation(false) {} void dump(Formatter *f) const; + + bool all_damage_repaired() const; }; /** diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 7b10b5f7126..59ffa841ca0 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -12112,14 +12112,45 @@ void MDCache::enqueue_scrub_work(MDRequestRef& mdr) header->set_origin(in); - // only set completion context for non-recursive scrub, because we don't - // want to block asok caller on long running scrub + Context *fin = nullptr; + if (!header->get_recursive()) { + cs->take_finisher(); + } + + // If the scrub did some repair, then flush the journal at the end of + // the scrub. Otherwise in the case of e.g. rewriting a backtrace + // the on disk state will still look damaged. + auto expiry_fin = new FunctionContext([this, header, fin](int r){ + if (header->get_repaired()) { + dout(4) << "Flushing journal because scrub did some repairs" << dendl; + mds->mdlog->start_new_segment(); + mds->mdlog->trim_all(); + if (fin) { + MDSGatherBuilder expiry_gather(g_ceph_context); + const std::set &expiring_segments = mds->mdlog->get_expiring_segments(); + for (std::set::const_iterator i = expiring_segments.begin(); + i != expiring_segments.end(); ++i) { + (*i)->wait_for_expiry(expiry_gather.new_sub()); + } + expiry_gather.set_finisher(new MDSInternalContextWrapper(mds, fin)); + expiry_gather.activate(); + } + } else { + if (fin) { + fin->complete(r); + } + } + }); + if (!header->get_recursive()) { - Context *fin = cs->take_finisher(); mds->scrubstack->enqueue_inode_top(in, header, - new MDSInternalContextWrapper(mds, fin)); - } else - mds->scrubstack->enqueue_inode_bottom(in, header, NULL); + new MDSInternalContextWrapper(mds, + expiry_fin)); + } else { + mds->scrubstack->enqueue_inode_bottom(in, header, + new MDSInternalContextWrapper(mds, + expiry_fin)); + } mds->server->respond_to_request(mdr, 0); return; diff --git a/src/mds/ScrubHeader.h b/src/mds/ScrubHeader.h index 18c68ae8f9d..c22683d0280 100644 --- a/src/mds/ScrubHeader.h +++ b/src/mds/ScrubHeader.h @@ -43,6 +43,9 @@ public: const std::string &get_tag() const { return tag; } Formatter &get_formatter() const { return *formatter; } + bool get_repaired() const { return repaired; } + void set_repaired() { repaired = true; } + protected: const std::string tag; const bool force; @@ -50,6 +53,8 @@ protected: const bool repair; Formatter * const formatter; CInode *origin; + + bool repaired = false; // May be set during scrub if repairs happened }; typedef ceph::shared_ptr ScrubHeaderRef; diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc index 2a595acdd7a..cb31a93f0fd 100644 --- a/src/mds/ScrubStack.cc +++ b/src/mds/ScrubStack.cc @@ -59,7 +59,7 @@ void ScrubStack::pop_inode(CInode *in) } void ScrubStack::_enqueue_inode(CInode *in, CDentry *parent, - const ScrubHeaderRefConst& header, + ScrubHeaderRef& header, MDSInternalContextBase *on_finish, bool top) { dout(10) << __func__ << " with {" << *in << "}" @@ -72,7 +72,7 @@ void ScrubStack::_enqueue_inode(CInode *in, CDentry *parent, push_inode_bottom(in); } -void ScrubStack::enqueue_inode(CInode *in, const ScrubHeaderRefConst& header, +void ScrubStack::enqueue_inode(CInode *in, ScrubHeaderRef& header, MDSInternalContextBase *on_finish, bool top) { _enqueue_inode(in, NULL, header, on_finish, top); @@ -134,7 +134,8 @@ void ScrubStack::scrub_dir_inode(CInode *in, bool all_frags_terminal = true; bool all_frags_done = true; - const ScrubHeaderRefConst& header = in->scrub_info()->header; + ScrubHeaderRef header = in->get_scrub_header(); + assert(header != nullptr); if (header->get_recursive()) { list scrubbing_frags; @@ -289,7 +290,7 @@ void ScrubStack::scrub_dir_inode_final(CInode *in) } void ScrubStack::scrub_dirfrag(CDir *dir, - const ScrubHeaderRefConst& header, + ScrubHeaderRef& header, bool *added_children, bool *is_terminal, bool *done) { @@ -376,7 +377,9 @@ void ScrubStack::_validate_inode_done(CInode *in, int r, in->make_path_string(path, true); } - if (result.backtrace.checked && !result.backtrace.passed) { + if (result.backtrace.checked && !result.backtrace.passed + && !result.backtrace.repaired) + { // Record backtrace fails as remote linkage damage, as // we may not be able to resolve hard links to this inode mdcache->mds->damage_table.notify_remote_damaged(in->inode.ino, path); @@ -393,9 +396,14 @@ void ScrubStack::_validate_inode_done(CInode *in, int r, // Inform the cluster log if we found an error if (!result.passed_validation) { - clog->warn() << "Scrub error on inode " << in->ino() - << " (" << path << ") see " << g_conf->name - << " log and `damage ls` output for details"; + if (result.all_damage_repaired()) { + clog->info() << "Scrub repaired inode " << in->ino() + << " (" << path << ")"; + } else { + clog->warn() << "Scrub error on inode " << in->ino() + << " (" << path << ") see " << g_conf->name + << " log and `damage ls` output for details"; + } // Put the verbose JSON output into the MDS log for later inspection JSONFormatter f; diff --git a/src/mds/ScrubStack.h b/src/mds/ScrubStack.h index c35b1aa9c46..848efdc841f 100644 --- a/src/mds/ScrubStack.h +++ b/src/mds/ScrubStack.h @@ -73,14 +73,14 @@ public: * @param header The ScrubHeader propagated from whereever this scrub * was initiated */ - void enqueue_inode_top(CInode *in, const ScrubHeaderRefConst& header, + void enqueue_inode_top(CInode *in, ScrubHeaderRef& header, MDSInternalContextBase *on_finish) { enqueue_inode(in, header, on_finish, true); } /** Like enqueue_inode_top, but we wait for all pending scrubs before * starting this one. */ - void enqueue_inode_bottom(CInode *in, const ScrubHeaderRefConst& header, + void enqueue_inode_bottom(CInode *in, ScrubHeaderRef& header, MDSInternalContextBase *on_finish) { enqueue_inode(in, header, on_finish, false); } @@ -90,9 +90,9 @@ private: * Put the inode at either the top or bottom of the stack, with * the given scrub params, and then try and kick off more scrubbing. */ - void enqueue_inode(CInode *in, const ScrubHeaderRefConst& header, + void enqueue_inode(CInode *in, ScrubHeaderRef& header, MDSInternalContextBase *on_finish, bool top); - void _enqueue_inode(CInode *in, CDentry *parent, const ScrubHeaderRefConst& header, + void _enqueue_inode(CInode *in, CDentry *parent, ScrubHeaderRef& header, MDSInternalContextBase *on_finish, bool top); /** * Kick off as many scrubs as are appropriate, based on the current @@ -164,7 +164,7 @@ private: * progress. Try again later. * */ - void scrub_dirfrag(CDir *dir, const ScrubHeaderRefConst& header, + void scrub_dirfrag(CDir *dir, ScrubHeaderRef& header, bool *added_children, bool *is_terminal, bool *done); /** * Scrub a directory-representing dentry.