Merge branch 'wip-7701' of git://github.com/xinxinsh/ceph into hammer

Conflicts:
	src/osd/osd_types.h

qa/workunits/cephtool/test.sh

@@ -1063,6 +1063,24 @@ function test_mon_pg()
   ceph pg dump_stuck stale
   ceph pg dump_stuck undersized
   ceph pg dump_stuck degraded
+  ceph pg ls
+  ceph pg ls 0
+  ceph pg ls stale
+  ceph pg ls active stale
+  ceph pg ls 0 active
+  ceph pg ls 0 active stale
+  ceph pg ls-by-primary osd.0
+  ceph pg ls-by-primary osd.0 0
+  ceph pg ls-by-primary osd.0 active
+  ceph pg ls-by-primary osd.0 active stale
+  ceph pg ls-by-primary osd.0 0 active stale
+  ceph pg ls-by-osd osd.0
+  ceph pg ls-by-osd osd.0 0
+  ceph pg ls-by-osd osd.0 active
+  ceph pg ls-by-osd osd.0 active stale
+  ceph pg ls-by-osd osd.0 0 active stale
+  ceph pg ls-by-pool rbd
+  ceph pg ls-by-pool rbd active stale
   # can't test this...
   # ceph pg force_create_pg
   ceph pg getmap -o $TMPDIR/map.$$

src/mon/MonCommands.h

@@ -131,6 +131,24 @@ COMMAND("pg dump_stuck " \
 	"name=threshold,type=CephInt,req=false",
 	"show information about stuck pgs",\
 	"pg", "r", "cli,rest")
+COMMAND("pg ls-by-pool " \
+        "name=poolstr,type=CephString " \
+	"name=states,type=CephChoices,strings=active|clean|down|replay|splitting|scrubbing|scrubq|degraded|inconsistent|peering|repair|recovery|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized,n=N,req=false ", \
+	"list pg with pool = [poolname | poolid]", "pg", "r", "cli,rest")
+COMMAND("pg ls-by-primary " \
+        "name=osd,type=CephOsdName " \
+        "name=pool,type=CephInt,req=false " \
+	"name=states,type=CephChoices,strings=active|clean|down|replay|splitting|scrubbing|scrubq|degraded|inconsistent|peering|repair|recovery|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized,n=N,req=false ", \
+	"list pg with primary = [osd]", "pg", "r", "cli,rest")
+COMMAND("pg ls-by-osd " \
+        "name=osd,type=CephOsdName " \
+        "name=pool,type=CephInt,req=false " \
+	"name=states,type=CephChoices,strings=active|clean|down|replay|splitting|scrubbing|scrubq|degraded|inconsistent|peering|repair|recovery|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized,n=N,req=false ", \
+	"list pg on osd [osd]", "pg", "r", "cli,rest")
+COMMAND("pg ls " \
+        "name=pool,type=CephInt,req=false " \
+	"name=states,type=CephChoices,strings=active|clean|down|replay|splitting|scrubbing|scrubq|degraded|inconsistent|peering|repair|recovery|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized,n=N,req=false ", \
+	"list pg with specific pool, osd, state", "pg", "r", "cli,rest")
 COMMAND("pg map name=pgid,type=CephPgid", "show mapping of pg to osds", \
 	"pg", "r", "cli,rest")
 COMMAND("pg scrub name=pgid,type=CephPgid", "start scrub on <pgid>", \

src/mon/PGMap.cc

@@ -10,6 +10,7 @@
 #include "common/Formatter.h"
 #include "include/ceph_features.h"
 #include "mon/MonitorDBStore.h"
+#include "osd/osd_types.h"
 
 // --
 
@@ -1426,3 +1427,68 @@ void PGMap::generate_test_instances(list<PGMap*>& o)
     inc.pop_front();
   }
 }
+
+void PGMap::get_filtered_pg_stats(string& state, int64_t poolid, int64_t osdid,
+                                  bool primary, set<pg_t>& pgs)
+{
+  int type = 0;
+  if (state != "all") {
+    type = pg_string_state(state);
+    if (type == -1)
+      assert(0 == "invalid type");
+  }
+
+  for (ceph::unordered_map<pg_t, pg_stat_t>::const_iterator i = pg_stat.begin();
+       i != pg_stat.end();
+       ++i) {
+    if ((poolid >= 0) && (uint64_t(poolid) != i->first.pool()))
+      continue;
+    if ((osdid >= 0) && !(i->second.is_acting_osd(osdid,primary)))
+      continue;
+    if ((state != "all") && !(i->second.state & type))
+      continue;
+    pgs.insert(i->first);
+  }
+}
+
+void PGMap::dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs)
+{
+  f->open_array_section("pg_stats");
+  for (set<pg_t>::iterator i = pgs.begin(); i != pgs.end(); i++) {
+    pg_stat_t& st = pg_stat[*i];
+    f->open_object_section("pg_stat");
+    f->dump_stream("pgid") << *i;
+    st.dump(f);
+    f->close_section();
+  } 
+  f->close_section();
+}  
+void PGMap::dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs)
+{
+  ss << "pg_stat\tobjects\tmip\tdegr\tmisp\tunf\tbytes\tlog\tdisklog\tstate\t"
+    "state_stamp\tv\treported\tup\tup_primary\tacting\tacting_primary\t"
+    "last_scrub\tscrub_stamp\tlast_deep_scrub\tdeep_scrub_stamp" << std::endl;
+  for (set<pg_t>::iterator i = pgs.begin(); i != pgs.end(); i++) {
+    pg_stat_t& st = pg_stat[*i];
+    ss << *i
+       << "\t" << st.stats.sum.num_objects
+       << "\t" << st.stats.sum.num_objects_missing_on_primary
+       << "\t" << st.stats.sum.num_objects_degraded
+       << "\t" << st.stats.sum.num_objects_misplaced
+       << "\t" << st.stats.sum.num_objects_unfound
+       << "\t" << st.stats.sum.num_bytes
+       << "\t" << st.log_size
+       << "\t" << st.ondisk_log_size
+       << "\t" << pg_state_string(st.state)
+       << "\t" << st.last_change
+       << "\t" << st.version
+       << "\t" << st.reported_epoch << ":" << st.reported_seq
+       << "\t" << st.up
+       << "\t" << st.up_primary
+       << "\t" << st.acting
+       << "\t" << st.acting_primary
+       << "\t" << st.last_scrub << "\t" << st.last_scrub_stamp
+       << "\t" << st.last_deep_scrub << "\t" << st.last_deep_scrub_stamp
+       << std::endl;
+  }
+}

src/mon/PGMap.h

@@ -267,6 +267,7 @@ public:
   void dump_pool_stats(Formatter *f) const;
   void dump_osd_stats(Formatter *f) const;
   void dump_delta(Formatter *f) const;
+  void dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs);
 
   void dump_pg_stats_plain(ostream& ss,
 			   const ceph::unordered_map<pg_t, pg_stat_t>& pg_stats,
@@ -283,6 +284,7 @@ public:
   void dump_pool_stats(ostream& ss, bool header) const;
   void dump_osd_stats(ostream& ss) const;
   void dump_osd_sum_stats(ostream& ss) const;
+  void dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs);
 
   void dump_osd_perf_stats(Formatter *f) const;
   void print_osd_perf_stats(std::ostream *ss) const;
@@ -290,6 +292,8 @@ public:
   void dump_osd_blocked_by_stats(Formatter *f) const;
   void print_osd_blocked_by_stats(std::ostream *ss) const;
 
+  void get_filtered_pg_stats(string& state, int64_t poolid, int64_t osdid,
+                             bool primary, set<pg_t>& pgs);
   void recovery_summary(Formatter *f, list<string> *psl,
                         const pool_stat_t& delta_sum) const;
   void overall_recovery_summary(Formatter *f, list<string> *psl) const;

src/mon/PGMonitor.cc

@@ -1461,6 +1461,7 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
   int r = -1;
   bufferlist rdata;
   stringstream ss, ds;
+  bool primary = false;
 
   map<string, cmd_vartype> cmdmap;
   if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
@@ -1492,7 +1493,27 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
     cmd_putval(g_ceph_context, cmdmap, "format", string("json"));
     cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v);
     prefix = "pg dump";
+  } else if (prefix == "pg ls-by-primary") {
+    primary = true;
+    prefix = "pg ls";
+  } else if (prefix == "pg ls-by-osd") {
+    prefix = "pg ls";
+  } else if (prefix == "pg ls-by-pool") {
+    prefix = "pg ls";
+    string poolstr;
+    cmd_getval(g_ceph_context, cmdmap, "poolstr", poolstr);
+    int64_t pool = -2;
+    pool = mon->osdmon()->osdmap.lookup_pg_pool_name(poolstr.c_str());
+    if (pool < 0) {
+      r = -ENOENT;
+      ss << "pool " << poolstr << " does not exist";
+      string rs = ss.str();
+      mon->reply_command(m, r, rs, get_last_committed());
+      return true;
+    }
+    cmd_putval(g_ceph_context, cmdmap, "pool", pool);
   }
+   
 
   string format;
   cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
@@ -1591,6 +1612,41 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
     }
     ss << "dumped " << what << " in format " << format;
     r = 0;
+  } else if (prefix == "pg ls") {
+    int64_t osd = -1;
+    int64_t pool = -1;
+    vector<string>states;
+    set<pg_t> pgs;
+    set<string> what;
+    cmd_getval(g_ceph_context, cmdmap, "pool", pool);
+    cmd_getval(g_ceph_context, cmdmap, "osd", osd);
+    cmd_getval(g_ceph_context, cmdmap, "states", states);
+    if (pool >= 0 && !mon->osdmon()->osdmap.have_pg_pool(pool)) {
+      r = -ENOENT;
+      ss << "pool " << pool << " does not exist";
+      goto reply;
+    } 
+    if (osd >= 0 && !mon->osdmon()->osdmap.is_up(osd)) {
+      ss << "osd " << osd << " is not up";
+      r = -EAGAIN;
+      goto reply;
+    }
+    if (states.empty())
+      states.push_back("all");
+    while (!states.empty()) {
+      string state = states.back();
+      what.insert(state);
+      pg_map.get_filtered_pg_stats(state,pool,osd,primary,pgs);
+      states.pop_back();
+    }
+    if (f && !pgs.empty()){
+      pg_map.dump_filtered_pg_stats(f.get(),pgs);
+      f->flush(ds);
+    } else if (!pgs.empty()){
+      pg_map.dump_filtered_pg_stats(ds,pgs);
+    }
+    ss << "list states " << what <<  " pool " << pool << " osd osd." << osd;
+    r = 0;
   } else if (prefix == "pg dump_stuck") {
     vector<string> stuckop_vec;
     cmd_getval(g_ceph_context, cmdmap, "stuckops", stuckop_vec);

src/osd/osd_types.cc

@@ -770,6 +770,53 @@ std::string pg_state_string(int state)
   return ret;
 }
 
+int pg_string_state(std::string state)
+{
+  int type;
+  if (state == "active")
+    type = PG_STATE_ACTIVE;
+  else if (state == "clean")
+    type = PG_STATE_CLEAN;
+  else if (state == "down")
+    type = PG_STATE_DOWN;
+  else if (state == "replay")
+    type = PG_STATE_REPLAY;
+  else if (state == "splitting")
+    type = PG_STATE_SPLITTING;
+  else if (state == "scrubbing")
+    type = PG_STATE_SCRUBBING;
+  else if (state == "scrubq")
+    type = PG_STATE_SCRUBQ;
+  else if (state == "degraded")
+    type = PG_STATE_DEGRADED;
+  else if (state == "inconsistent")
+    type = PG_STATE_INCONSISTENT;
+  else if (state == "peering")
+    type = PG_STATE_PEERING;
+  else if (state == "recoverying")
+    type = PG_STATE_RECOVERING;
+  else if (state == "backfill_wait")
+    type = PG_STATE_BACKFILL_WAIT;
+  else if (state == "incomplete")
+    type = PG_STATE_INCOMPLETE;
+  else if (state == "remapped")
+    type = PG_STATE_REMAPPED;
+  else if (state == "stale")
+    type = PG_STATE_STALE;
+  else if (state == "deep_scrub")
+    type = PG_STATE_DEEP_SCRUB;
+  else if (state == "backfill")
+    type = PG_STATE_BACKFILL;
+  else if (state == "backfill_toofull")
+    type = PG_STATE_BACKFILL_TOOFULL;
+  else if (state == "recovery_wait")
+    type = PG_STATE_RECOVERY_WAIT;
+  else if (state == "undersized")
+    type = PG_STATE_UNDERSIZED;
+  else
+    type = -1;
+  return type;
+}
 
 // -- eversion_t --
 string eversion_t::get_key_name() const
@@ -1725,6 +1772,21 @@ void object_stat_collection_t::generate_test_instances(list<object_stat_collecti
 
 // -- pg_stat_t --
 
+bool pg_stat_t::is_acting_osd(int32_t osd, bool primary) const
+{
+  if (primary && osd == acting_primary) {
+    return true;
+  } else if (!primary) {
+    for(vector<int32_t>::const_iterator it = acting.begin();
+        it != acting.end(); it++)
+    {
+      if (*it == osd)
+        return true;
+    }
+  }
+  return false;
+}
+
 void pg_stat_t::dump(Formatter *f) const
 {
   f->dump_stream("version") << version;

src/osd/osd_types.h

@@ -772,6 +772,8 @@ inline ostream& operator<<(ostream& out, const osd_stat_t& s) {
 
 std::string pg_state_string(int state);
 std::string pg_vector_string(const vector<int32_t> &a);
+int pg_string_state(std::string state);
+
 
 /*
  * pool_snap_info_t
@@ -1499,6 +1501,7 @@ struct pg_stat_t {
     ondisk_log_size -= o.ondisk_log_size;
   }
 
+  bool is_acting_osd(int32_t osd, bool primary) const;
   void dump(Formatter *f) const;
   void dump_brief(Formatter *f) const;
   void encode(bufferlist &bl) const;

src/pybind/ceph_argparse.py

@@ -727,7 +727,10 @@ def matchnum(args, signature, partial=False):
             word = words.pop(0)
 
             try:
-                validate_one(word, desc, partial)
+                # only allow partial matching if we're on the last supplied
+                # word; avoid matching foo bar and foot bar just because
+                # partial is set
+                validate_one(word, desc, partial and (len(words) == 0))
                 valid = True
             except ArgumentError:
                 # matchnum doesn't care about type of error