Merge pull request #19368 from dzafman/wip-22086

ceph-objectstore-tool: Add option "dump-import" to examine an export

Reviewed-by: Sage Weil <sage@redhat.com>
This commit is contained in:
David Zafman 2017-12-08 19:37:45 -08:00 committed by GitHub
commit 0f764890d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 351 additions and 15 deletions

View File

@ -1027,7 +1027,7 @@ def main(argv):
# Specify a bad --op command
cmd = (CFSD_PREFIX + "--op oops").format(osd=ONEOSD)
ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)")
ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, dump-import)")
# Provide just the object param not a command
cmd = (CFSD_PREFIX + "object").format(osd=ONEOSD)
@ -1749,6 +1749,13 @@ def main(argv):
PGS = [f for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]
for pg in PGS:
file = os.path.join(dir, pg)
# Make sure this doesn't crash
cmd = (CFSD_PREFIX + "--op dump-import --file {file}").format(osd=osd, file=file)
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=nullfd)
if ret != 0:
logging.error("Dump-import failed from {file} with {ret}".format(file=file, ret=ret))
IMP_ERRORS += 1
# This should do nothing
cmd = (CFSD_PREFIX + "--op import --file {file} --dry-run").format(osd=osd, file=file)
logging.debug(cmd)

View File

@ -292,7 +292,6 @@ int file_fd = fd_none;
bool debug;
bool force = false;
super_header sh;
uint64_t testalign;
static int get_fd_data(int fd, bufferlist &bl)
{
@ -348,9 +347,6 @@ void dump_log(Formatter *formatter, ostream &out, pg_log_t &log,
formatter->open_object_section("pg_missing_t");
missing.dump(formatter);
formatter->close_section();
formatter->flush(out);
formatter->open_object_section("map");
formatter->close_section();
formatter->close_section();
formatter->flush(out);
}
@ -803,6 +799,21 @@ int ObjectStoreTool::do_export(ObjectStore *fs, coll_t coll, spg_t pgid,
return 0;
}
int dump_data(Formatter *formatter, bufferlist &bl)
{
bufferlist::iterator ebliter = bl.begin();
data_section ds;
ds.decode(ebliter);
formatter->open_object_section("data_block");
formatter->dump_unsigned("offset", ds.offset);
formatter->dump_unsigned("len", ds.len);
// XXX: Add option to dump data like od -cx ?
formatter->close_section();
formatter->flush(cout);
return 0;
}
int get_data(ObjectStore *store, coll_t coll, ghobject_t hoid,
ObjectStore::Transaction *t, bufferlist &bl)
{
@ -816,6 +827,63 @@ int get_data(ObjectStore *store, coll_t coll, ghobject_t hoid,
return 0;
}
int dump_attrs(
Formatter *formatter, ghobject_t hoid,
bufferlist &bl)
{
bufferlist::iterator ebliter = bl.begin();
attr_section as;
as.decode(ebliter);
// This could have been handled in the caller if we didn't need to
// support exports that didn't include object_info_t in object_begin.
if (hoid.generation == ghobject_t::NO_GEN &&
hoid.hobj.is_head()) {
map<string,bufferlist>::iterator mi = as.data.find(SS_ATTR);
if (mi != as.data.end()) {
SnapSet snapset;
auto p = mi->second.begin();
snapset.decode(p);
formatter->open_object_section("snapset");
snapset.dump(formatter);
formatter->close_section();
} else {
formatter->open_object_section("snapset");
formatter->dump_string("error", "missing SS_ATTR");
formatter->close_section();
}
}
formatter->open_object_section("attrs");
formatter->open_array_section("user");
for (auto kv : as.data) {
// Skip system attributes
if (('_' != kv.first.at(0)) || kv.first.size() == 1)
continue;
formatter->open_object_section("user_attr");
formatter->dump_string("name", kv.first.substr(1));
bool b64;
formatter->dump_string("value", cleanbin(kv.second, b64));
formatter->dump_bool("Base64", b64);
formatter->close_section();
}
formatter->close_section();
formatter->open_array_section("system");
for (auto kv : as.data) {
// Skip user attributes
if (('_' == kv.first.at(0)) && kv.first.size() != 1)
continue;
formatter->open_object_section("sys_attr");
formatter->dump_string("name", kv.first);
formatter->close_section();
}
formatter->close_section();
formatter->close_section();
formatter->flush(cout);
return 0;
}
int get_attrs(
ObjectStore *store, coll_t coll, ghobject_t hoid,
ObjectStore::Transaction *t, bufferlist &bl,
@ -866,6 +934,19 @@ int get_attrs(
return 0;
}
int dump_omap_hdr(Formatter *formatter, bufferlist &bl)
{
bufferlist::iterator ebliter = bl.begin();
omap_hdr_section oh;
oh.decode(ebliter);
formatter->open_object_section("omap_header");
formatter->dump_string("value", string(oh.hdr.c_str(), oh.hdr.length()));
formatter->close_section();
formatter->flush(cout);
return 0;
}
int get_omap_hdr(ObjectStore *store, coll_t coll, ghobject_t hoid,
ObjectStore::Transaction *t, bufferlist &bl)
{
@ -880,6 +961,29 @@ int get_omap_hdr(ObjectStore *store, coll_t coll, ghobject_t hoid,
return 0;
}
int dump_omap(Formatter *formatter, bufferlist &bl)
{
bufferlist::iterator ebliter = bl.begin();
omap_section os;
os.decode(ebliter);
formatter->open_object_section("omaps");
formatter->dump_unsigned("count", os.omap.size());
formatter->open_array_section("data");
for (auto o : os.omap) {
formatter->open_object_section("omap");
formatter->dump_string("name", o.first);
bool b64;
formatter->dump_string("value", cleanbin(o.second, b64));
formatter->dump_bool("Base64", b64);
formatter->close_section();
}
formatter->close_section();
formatter->close_section();
formatter->flush(cout);
return 0;
}
int get_omap(ObjectStore *store, coll_t coll, ghobject_t hoid,
ObjectStore::Transaction *t, bufferlist &bl)
{
@ -893,6 +997,73 @@ int get_omap(ObjectStore *store, coll_t coll, ghobject_t hoid,
return 0;
}
int ObjectStoreTool::dump_object(Formatter *formatter,
bufferlist &bl)
{
bufferlist::iterator ebliter = bl.begin();
object_begin ob;
ob.decode(ebliter);
if (ob.hoid.hobj.is_temp()) {
cerr << "ERROR: Export contains temporary object '" << ob.hoid << "'" << std::endl;
return -EFAULT;
}
formatter->open_object_section("object");
formatter->open_object_section("oid");
ob.hoid.dump(formatter);
formatter->close_section();
formatter->open_object_section("object_info");
ob.oi.dump(formatter);
formatter->close_section();
bufferlist ebl;
bool done = false;
while(!done) {
sectiontype_t type;
int ret = read_section(&type, &ebl);
if (ret)
return ret;
//cout << "\tdo_object: Section type " << hex << type << dec << std::endl;
//cout << "\t\tsection size " << ebl.length() << std::endl;
if (type >= END_OF_TYPES) {
cout << "Skipping unknown object section type" << std::endl;
continue;
}
switch(type) {
case TYPE_DATA:
if (dry_run) break;
ret = dump_data(formatter, ebl);
if (ret) return ret;
break;
case TYPE_ATTRS:
if (dry_run) break;
ret = dump_attrs(formatter, ob.hoid, ebl);
if (ret) return ret;
break;
case TYPE_OMAP_HDR:
if (dry_run) break;
ret = dump_omap_hdr(formatter, ebl);
if (ret) return ret;
break;
case TYPE_OMAP:
if (dry_run) break;
ret = dump_omap(formatter, ebl);
if (ret) return ret;
break;
case TYPE_OBJECT_END:
done = true;
break;
default:
cerr << "Unknown section type " << type << std::endl;
return -EFAULT;
}
}
formatter->close_section();
return 0;
}
int ObjectStoreTool::get_object(ObjectStore *store, coll_t coll,
bufferlist &bl, OSDMap &curmap,
bool *skipped_objects,
@ -994,6 +1165,49 @@ int ObjectStoreTool::get_object(ObjectStore *store, coll_t coll,
return 0;
}
int dump_pg_metadata(Formatter *formatter, bufferlist &bl, metadata_section &ms)
{
bufferlist::iterator ebliter = bl.begin();
ms.decode(ebliter);
formatter->open_object_section("metadata_section");
formatter->dump_unsigned("pg_disk_version", (int)ms.struct_ver);
formatter->dump_unsigned("map_epoch", ms.map_epoch);
formatter->open_object_section("OSDMap");
ms.osdmap.dump(formatter);
formatter->close_section();
formatter->flush(cout);
cout << std::endl;
formatter->open_object_section("info");
ms.info.dump(formatter);
formatter->close_section();
formatter->flush(cout);
formatter->open_object_section("log");
ms.log.dump(formatter);
formatter->close_section();
formatter->flush(cout);
formatter->open_object_section("pg_missing_t");
ms.missing.dump(formatter);
formatter->close_section();
// XXX: ms.past_intervals?
formatter->close_section();
formatter->flush(cout);
if (ms.osdmap.get_epoch() != 0 && ms.map_epoch != ms.osdmap.get_epoch()) {
cerr << "FATAL: Invalid OSDMap epoch in export data" << std::endl;
return -EFAULT;
}
return 0;
}
int get_pg_metadata(ObjectStore *store, bufferlist &bl, metadata_section &ms,
const OSDSuperblock& sb, OSDMap& curmap, spg_t pgid)
{
@ -1256,6 +1470,106 @@ void filter_divergent_priors(spg_t import_pgid, const OSDMap &curmap,
}
}
int ObjectStoreTool::dump_import(Formatter *formatter)
{
bufferlist ebl;
pg_info_t info;
PGLog::IndexedLog log;
//bool skipped_objects = false;
int ret = read_super();
if (ret)
return ret;
if (sh.magic != super_header::super_magic) {
cerr << "Invalid magic number" << std::endl;
return -EFAULT;
}
if (sh.version > super_header::super_ver) {
cerr << "Can't handle export format version=" << sh.version << std::endl;
return -EINVAL;
}
formatter->open_object_section("Export");
//First section must be TYPE_PG_BEGIN
sectiontype_t type;
ret = read_section(&type, &ebl);
if (ret)
return ret;
if (type == TYPE_POOL_BEGIN) {
cerr << "Dump of pool exports not supported" << std::endl;
return -EINVAL;
} else if (type != TYPE_PG_BEGIN) {
cerr << "Invalid first section type " << std::to_string(type) << std::endl;
return -EFAULT;
}
bufferlist::iterator ebliter = ebl.begin();
pg_begin pgb;
pgb.decode(ebliter);
spg_t pgid = pgb.pgid;
formatter->dump_string("pgid", stringify(pgid));
formatter->dump_string("cluster_fsid", stringify(pgb.superblock.cluster_fsid));
formatter->dump_string("features", stringify(pgb.superblock.compat_features));
bool done = false;
bool found_metadata = false;
metadata_section ms;
bool objects_started = false;
while(!done) {
ret = read_section(&type, &ebl);
if (ret)
return ret;
if (debug) {
cerr << "dump_import: Section type " << std::to_string(type) << std::endl;
}
if (type >= END_OF_TYPES) {
cerr << "Skipping unknown section type" << std::endl;
continue;
}
switch(type) {
case TYPE_OBJECT_BEGIN:
if (!objects_started) {
formatter->open_array_section("objects");
objects_started = true;
}
ret = dump_object(formatter, ebl);
if (ret) return ret;
break;
case TYPE_PG_METADATA:
if (objects_started)
cerr << "WARNING: metadata_section out of order" << std::endl;
ret = dump_pg_metadata(formatter, ebl, ms);
if (ret) return ret;
found_metadata = true;
break;
case TYPE_PG_END:
if (objects_started) {
formatter->close_section();
}
done = true;
break;
default:
cerr << "Unknown section type " << std::to_string(type) << std::endl;
return -EFAULT;
}
}
if (!found_metadata) {
cerr << "Missing metadata section" << std::endl;
return -EFAULT;
}
formatter->close_section();
formatter->flush(cout);
return 0;
}
int ObjectStoreTool::do_import(ObjectStore *store, OSDSuperblock& sb,
bool force, std::string pgidstr,
ObjectStore::Sequencer &osr)
@ -1291,7 +1605,7 @@ int ObjectStoreTool::do_import(ObjectStore *store, OSDSuperblock& sb,
cerr << "Pool exports cannot be imported into a PG" << std::endl;
return -EINVAL;
} else if (type != TYPE_PG_BEGIN) {
cerr << "Invalid first section type " << type << std::endl;
cerr << "Invalid first section type " << std::to_string(type) << std::endl;
return -EFAULT;
}
@ -1404,7 +1718,9 @@ int ObjectStoreTool::do_import(ObjectStore *store, OSDSuperblock& sb,
if (ret)
return ret;
//cout << "do_import: Section type " << hex << type << dec << std::endl;
if (debug) {
cout << __func__ << ": Section type " << std::to_string(type) << std::endl;
}
if (type >= END_OF_TYPES) {
cout << "Skipping unknown section type" << std::endl;
continue;
@ -1423,7 +1739,7 @@ int ObjectStoreTool::do_import(ObjectStore *store, OSDSuperblock& sb,
done = true;
break;
default:
cerr << "Unknown section type " << type << std::endl;
cerr << "Unknown section type " << std::to_string(type) << std::endl;
return -EFAULT;
}
}
@ -2609,7 +2925,7 @@ int main(int argc, char **argv)
"Pool name, mandatory for apply-layout-settings if --pgid is not specified")
("op", po::value<string>(&op),
"Arg is one of [info, log, remove, mkfs, fsck, repair, fuse, dup, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, "
"get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, apply-layout-settings, update-mon-db]")
"get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, apply-layout-settings, update-mon-db, dump-import]")
("epoch", po::value<unsigned>(&epoch),
"epoch# for get-osdmap and get-inc-osdmap, the current epoch in use if not specified")
("file", po::value<string>(&file),
@ -2640,10 +2956,9 @@ int main(int argc, char **argv)
("arg1", po::value<string>(&arg1), "arg1 based on cmd, "
"for apply-layout-settings: target hash level split to")
("arg2", po::value<string>(&arg2), "arg2 based on cmd")
("test-align", po::value<uint64_t>(&testalign)->default_value(0), "hidden align option for testing")
;
po::options_description all("All options");
po::options_description all;
all.add(desc).add(positional);
po::positional_options_description pd;
@ -2664,7 +2979,7 @@ int main(int argc, char **argv)
}
if (vm.count("help")) {
usage(all);
usage(desc);
return 1;
}
@ -2716,6 +3031,7 @@ int main(int argc, char **argv)
type = "bluestore";
}
if (!vm.count("data-path") &&
op != "dump-import" &&
!(op == "dump-journal" && type == "filestore")) {
cerr << "Must provide --data-path" << std::endl;
usage(desc);
@ -2764,7 +3080,7 @@ int main(int argc, char **argv)
} else {
file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
}
} else if (op == "import" || op == "set-osdmap" || op == "set-inc-osdmap") {
} else if (op == "import" || op == "dump-import" || op == "set-osdmap" || op == "set-inc-osdmap") {
if (!vm.count("file") || file == "-") {
if (isatty(STDIN_FILENO)) {
cerr << "stdin is a tty and no --file filename specified" << std::endl;
@ -2779,7 +3095,7 @@ int main(int argc, char **argv)
ObjectStoreTool tool = ObjectStoreTool(file_fd, dry_run);
if (vm.count("file") && file_fd == fd_none && !dry_run) {
cerr << "--file option only applies to import, export, export-remove, "
cerr << "--file option only applies to import, dump-import, export, export-remove, "
<< "get-osdmap, set-osdmap, get-inc-osdmap or set-inc-osdmap" << std::endl;
return 1;
}
@ -2828,6 +3144,16 @@ int main(int argc, char **argv)
return 0;
}
if (op == "dump-import") {
int ret = tool.dump_import(formatter);
if (ret < 0) {
cerr << "dump-import: "
<< cpp_strerror(ret) << std::endl;
return 1;
}
return 0;
}
//Verify that data-path really exists
struct stat st;
if (::stat(dpath.c_str(), &st) == -1) {
@ -3309,7 +3635,7 @@ int main(int argc, char **argv)
// before complaining about a bad pgid
if (!vm.count("objcmd") && op != "export" && op != "export-remove" && op != "info" && op != "log" && op != "mark-complete") {
cerr << "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, "
"get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)"
"get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, dump-import)"
<< std::endl;
usage(desc);
ret = 1;

View File

@ -24,6 +24,7 @@ class ObjectStoreTool : public RadosDump
: RadosDump(file_fd, dry_run)
{}
int dump_import(Formatter *formatter);
int do_import(ObjectStore *store, OSDSuperblock& sb, bool force,
std::string pgidstr,
ObjectStore::Sequencer &osr);
@ -31,6 +32,8 @@ class ObjectStoreTool : public RadosDump
pg_info_t &info, epoch_t map_epoch, __u8 struct_ver,
const OSDSuperblock& superblock,
PastIntervals &past_intervals);
int dump_object(Formatter *formatter,
bufferlist &bl);
int get_object(
ObjectStore *store, coll_t coll,
bufferlist &bl, OSDMap &curmap, bool *skipped_objects,