mirror of
https://github.com/ceph/ceph
synced 2024-12-17 00:46:05 +00:00
Merge pull request #2124 from ceph/wip-objectstore
simplify ObjectStore interface sage-2014-11-11_08:26:01-rados-wip-sage-testing-distro-basic-multi Reviewed-by: Haomai Wang <haomaiwang@gmail.com> Reviewed-by: Samuel Just <sjust@redhat.com>
This commit is contained in:
commit
22b6c1a531
@ -718,7 +718,6 @@ ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1
|
||||
%files -n ceph-test
|
||||
%defattr(-,root,root,-)
|
||||
%{_bindir}/ceph_bench_log
|
||||
%{_bindir}/ceph_dupstore
|
||||
%{_bindir}/ceph_kvstorebench
|
||||
%{_bindir}/ceph_multi_stress_watch
|
||||
%{_bindir}/ceph_erasure_code
|
||||
|
1
debian/ceph-test.install
vendored
1
debian/ceph-test.install
vendored
@ -1,6 +1,5 @@
|
||||
usr/bin/ceph-coverage
|
||||
usr/bin/ceph_bench_log
|
||||
usr/bin/ceph_dupstore
|
||||
usr/bin/ceph_objectstore_tool
|
||||
usr/bin/ceph_kvstorebench
|
||||
usr/bin/ceph_multi_stress_watch
|
||||
|
1
src/.gitignore
vendored
1
src/.gitignore
vendored
@ -24,7 +24,6 @@ Makefile
|
||||
/ceph-syn
|
||||
/ceph.conf
|
||||
/ceph_bench_log
|
||||
/ceph_dupstore
|
||||
/ceph_objectstore_tool
|
||||
/ceph_mon_store_converter
|
||||
/ceph_multi_stress_watch
|
||||
|
@ -294,7 +294,14 @@ int main(int argc, const char **argv)
|
||||
|
||||
|
||||
if (convertfilestore) {
|
||||
int err = OSD::do_convertfs(store);
|
||||
int err = store->mount();
|
||||
if (err < 0) {
|
||||
derr << TEXT_RED << " ** ERROR: error mounting store " << g_conf->osd_data
|
||||
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
|
||||
exit(1);
|
||||
}
|
||||
err = store->upgrade();
|
||||
store->umount();
|
||||
if (err < 0) {
|
||||
derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data
|
||||
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
|
||||
@ -468,15 +475,6 @@ int main(int argc, const char **argv)
|
||||
global_init_daemonize(g_ceph_context, 0);
|
||||
common_init_finish(g_ceph_context);
|
||||
|
||||
if (g_conf->filestore_update_to >= (int)store->get_target_version()) {
|
||||
int err = OSD::do_convertfs(store);
|
||||
if (err < 0) {
|
||||
derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data
|
||||
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
MonClient mc(g_ceph_context);
|
||||
if (mc.build_initial_monmap() < 0)
|
||||
return -1;
|
||||
|
@ -1164,6 +1164,19 @@ int FileStore::write_version_stamp()
|
||||
bl.c_str(), bl.length());
|
||||
}
|
||||
|
||||
int FileStore::upgrade()
|
||||
{
|
||||
uint32_t version;
|
||||
int r = version_stamp_is_valid(&version);
|
||||
if (r < 0)
|
||||
return r;
|
||||
if (r == 1)
|
||||
return 0;
|
||||
|
||||
derr << "ObjectStore is old at version " << version << ". Please upgrade to firefly v0.80.x, convert your store, and then upgrade." << dendl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
int FileStore::read_op_seq(uint64_t *seq)
|
||||
{
|
||||
int op_fd = ::open(current_op_seq_fn.c_str(), O_CREAT|O_RDWR, 0644);
|
||||
@ -1568,6 +1581,16 @@ int FileStore::mount()
|
||||
|
||||
timer.init();
|
||||
|
||||
// upgrade?
|
||||
if (g_conf->filestore_update_to >= (int)get_target_version()) {
|
||||
int err = upgrade();
|
||||
if (err < 0) {
|
||||
derr << "error converting store" << dendl;
|
||||
umount();
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
// all okay.
|
||||
return 0;
|
||||
|
||||
@ -2452,19 +2475,26 @@ unsigned FileStore::_do_transaction(
|
||||
coll_t ncid = i.decode_cid();
|
||||
coll_t ocid = i.decode_cid();
|
||||
ghobject_t oid = i.decode_oid();
|
||||
|
||||
// always followed by OP_COLL_REMOVE
|
||||
int op = i.decode_op();
|
||||
coll_t ocid2 = i.decode_cid();
|
||||
ghobject_t oid2 = i.decode_oid();
|
||||
assert(op == Transaction::OP_COLL_REMOVE);
|
||||
assert(ocid2 == ocid);
|
||||
assert(oid2 == oid);
|
||||
|
||||
tracepoint(objectstore, coll_add_enter);
|
||||
r = _collection_add(ncid, ocid, oid, spos);
|
||||
tracepoint(objectstore, coll_add_exit, r);
|
||||
}
|
||||
break;
|
||||
|
||||
case Transaction::OP_COLL_REMOVE:
|
||||
{
|
||||
coll_t cid = i.decode_cid();
|
||||
ghobject_t oid = i.decode_oid();
|
||||
if (r == -ENOENT && i.tolerate_collection_add_enoent())
|
||||
r = 0;
|
||||
spos.op++;
|
||||
if (r < 0)
|
||||
break;
|
||||
tracepoint(objectstore, coll_remove_enter, osr_name);
|
||||
if (_check_replay_guard(cid, oid, spos) > 0)
|
||||
r = _remove(cid, oid, spos);
|
||||
if (_check_replay_guard(ocid, oid, spos) > 0)
|
||||
r = _remove(ocid, oid, spos);
|
||||
tracepoint(objectstore, coll_remove_exit, r);
|
||||
}
|
||||
break;
|
||||
@ -2530,9 +2560,7 @@ unsigned FileStore::_do_transaction(
|
||||
{
|
||||
coll_t cid(i.decode_cid());
|
||||
coll_t ncid(i.decode_cid());
|
||||
tracepoint(objectstore, coll_rename_enter, osr_name);
|
||||
r = _collection_rename(cid, ncid, spos);
|
||||
tracepoint(objectstore, coll_rename_exit, r);
|
||||
r = -EOPNOTSUPP;
|
||||
}
|
||||
break;
|
||||
|
||||
@ -4292,56 +4320,6 @@ int FileStore::_collection_remove_recursive(const coll_t &cid,
|
||||
return _destroy_collection(cid);
|
||||
}
|
||||
|
||||
int FileStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
|
||||
const SequencerPosition& spos)
|
||||
{
|
||||
char new_coll[PATH_MAX], old_coll[PATH_MAX];
|
||||
get_cdir(cid, old_coll, sizeof(old_coll));
|
||||
get_cdir(ncid, new_coll, sizeof(new_coll));
|
||||
|
||||
if (_check_replay_guard(cid, spos) < 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (_check_replay_guard(ncid, spos) < 0) {
|
||||
return _collection_remove_recursive(cid, spos);
|
||||
}
|
||||
|
||||
if (!collection_exists(cid)) {
|
||||
if (replaying) {
|
||||
// already happened
|
||||
return 0;
|
||||
} else {
|
||||
return -ENOENT;
|
||||
}
|
||||
}
|
||||
_set_global_replay_guard(cid, spos);
|
||||
|
||||
int ret = 0;
|
||||
if (::rename(old_coll, new_coll)) {
|
||||
if (replaying && !backend->can_checkpoint() &&
|
||||
(errno == EEXIST || errno == ENOTEMPTY))
|
||||
ret = _collection_remove_recursive(cid, spos);
|
||||
else
|
||||
ret = -errno;
|
||||
|
||||
dout(10) << "collection_rename '" << cid << "' to '" << ncid << "'"
|
||||
<< ": ret = " << ret << dendl;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ret >= 0) {
|
||||
int fd = ::open(new_coll, O_RDONLY);
|
||||
assert(fd >= 0);
|
||||
_set_replay_guard(fd, spos);
|
||||
VOID_TEMP_FAILURE_RETRY(::close(fd));
|
||||
}
|
||||
|
||||
dout(10) << "collection_rename '" << cid << "' to '" << ncid << "'"
|
||||
<< ": ret = " << ret << dendl;
|
||||
return ret;
|
||||
}
|
||||
|
||||
// --------------------------
|
||||
// collections
|
||||
|
||||
|
@ -377,7 +377,6 @@ private:
|
||||
void op_queue_release_throttle(Op *o);
|
||||
void _journaled_ahead(OpSequencer *osr, Op *o, Context *ondisk);
|
||||
friend struct C_JournaledAhead;
|
||||
int write_version_stamp();
|
||||
|
||||
int open_journal();
|
||||
|
||||
@ -410,8 +409,6 @@ public:
|
||||
int _sanity_check_fs();
|
||||
|
||||
bool test_mount_in_use();
|
||||
int version_stamp_is_valid(uint32_t *version);
|
||||
int update_version_stamp();
|
||||
int read_op_seq(uint64_t *seq);
|
||||
int write_op_seq(int, uint64_t seq);
|
||||
int mount();
|
||||
@ -428,6 +425,11 @@ public:
|
||||
int mkfs();
|
||||
int mkjournal();
|
||||
|
||||
int write_version_stamp();
|
||||
int version_stamp_is_valid(uint32_t *version);
|
||||
int update_version_stamp();
|
||||
int upgrade();
|
||||
|
||||
/**
|
||||
* set_allow_sharded_objects()
|
||||
*
|
||||
@ -593,8 +595,6 @@ public:
|
||||
int _collection_setattrs(coll_t cid, map<string,bufferptr> &aset);
|
||||
int _collection_remove_recursive(const coll_t &cid,
|
||||
const SequencerPosition &spos);
|
||||
int _collection_rename(const coll_t &cid, const coll_t &ncid,
|
||||
const SequencerPosition& spos);
|
||||
|
||||
// collections
|
||||
int list_collections(vector<coll_t>& ls);
|
||||
|
@ -1447,7 +1447,7 @@ unsigned KeyValueStore::_do_transaction(Transaction& transaction,
|
||||
{
|
||||
coll_t cid(i.decode_cid());
|
||||
coll_t ncid(i.decode_cid());
|
||||
r = _collection_rename(cid, ncid, t);
|
||||
r = -EOPNOTSUPP;
|
||||
}
|
||||
break;
|
||||
|
||||
@ -2582,61 +2582,6 @@ int KeyValueStore::_collection_remove_recursive(const coll_t &cid,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KeyValueStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
|
||||
BufferTransaction &t)
|
||||
{
|
||||
dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
|
||||
<< dendl;
|
||||
|
||||
StripObjectMap::StripObjectHeaderRef header;
|
||||
|
||||
int r = t.lookup_cached_header(get_coll_for_coll(),
|
||||
make_ghobject_for_coll(ncid),
|
||||
&header, false);
|
||||
if (r == 0) {
|
||||
dout(2) << __func__ << ": " << ncid << " DNE" << dendl;
|
||||
return -EEXIST;
|
||||
}
|
||||
|
||||
r = t.lookup_cached_header(get_coll_for_coll(), make_ghobject_for_coll(cid),
|
||||
&header, false);
|
||||
if (r < 0) {
|
||||
dout(2) << __func__ << ": " << cid << " DNE" << dendl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
vector<ghobject_t> objects;
|
||||
ghobject_t next, current;
|
||||
int move_size = 0;
|
||||
while (1) {
|
||||
collection_list_partial(cid, current, get_ideal_list_min(),
|
||||
get_ideal_list_max(), 0, &objects, &next);
|
||||
|
||||
dout(20) << __func__ << cid << "objects size: " << objects.size()
|
||||
<< dendl;
|
||||
|
||||
if (objects.empty())
|
||||
break;
|
||||
|
||||
for (vector<ghobject_t>::iterator i = objects.begin();
|
||||
i != objects.end(); ++i) {
|
||||
if (_collection_move_rename(cid, *i, ncid, *i, t) < 0) {
|
||||
return -1;
|
||||
}
|
||||
move_size++;
|
||||
}
|
||||
|
||||
objects.clear();
|
||||
current = next;
|
||||
}
|
||||
|
||||
t.rename_buffer(header, get_coll_for_coll(), make_ghobject_for_coll(ncid));
|
||||
|
||||
dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
|
||||
<< dendl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KeyValueStore::list_collections(vector<coll_t>& ls)
|
||||
{
|
||||
dout(10) << __func__ << " " << dendl;
|
||||
|
@ -611,8 +611,6 @@ class KeyValueStore : public ObjectStore,
|
||||
BufferTransaction &t);
|
||||
int _collection_remove_recursive(const coll_t &cid,
|
||||
BufferTransaction &t);
|
||||
int _collection_rename(const coll_t &cid, const coll_t &ncid,
|
||||
BufferTransaction &t);
|
||||
int list_collections(vector<coll_t>& ls);
|
||||
bool collection_exists(coll_t c);
|
||||
bool collection_empty(coll_t c);
|
||||
|
@ -911,7 +911,7 @@ void MemStore::_do_transaction(Transaction& t)
|
||||
{
|
||||
coll_t cid(i.decode_cid());
|
||||
coll_t ncid(i.decode_cid());
|
||||
r = _collection_rename(cid, ncid);
|
||||
r = -EOPNOTSUPP;
|
||||
}
|
||||
break;
|
||||
|
||||
@ -1485,19 +1485,6 @@ int MemStore::_collection_rmattr(coll_t cid, const char *name)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MemStore::_collection_rename(const coll_t &cid, const coll_t &ncid)
|
||||
{
|
||||
dout(10) << __func__ << " " << cid << " -> " << ncid << dendl;
|
||||
RWLock::WLocker l(coll_lock);
|
||||
if (coll_map.count(cid) == 0)
|
||||
return -ENOENT;
|
||||
if (coll_map.count(ncid))
|
||||
return -EEXIST;
|
||||
coll_map[ncid] = coll_map[cid];
|
||||
coll_map.erase(cid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MemStore::_split_collection(coll_t cid, uint32_t bits, uint32_t match,
|
||||
coll_t dest)
|
||||
{
|
||||
|
@ -218,7 +218,6 @@ private:
|
||||
size_t size);
|
||||
int _collection_setattrs(coll_t cid, map<string,bufferptr> &aset);
|
||||
int _collection_rmattr(coll_t cid, const char *name);
|
||||
int _collection_rename(const coll_t &cid, const coll_t &ncid);
|
||||
int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest);
|
||||
|
||||
int _save();
|
||||
@ -236,13 +235,6 @@ public:
|
||||
sharded(false) { }
|
||||
~MemStore() { }
|
||||
|
||||
int update_version_stamp() {
|
||||
return 0;
|
||||
}
|
||||
uint32_t get_target_version() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool need_journal() { return false; };
|
||||
int peek_journal_fsid(uuid_d *fsid);
|
||||
|
||||
|
@ -113,7 +113,6 @@ int ObjectStore::queue_transactions(
|
||||
onreadable_sync, op);
|
||||
}
|
||||
|
||||
|
||||
int ObjectStore::collection_list(coll_t c, vector<hobject_t>& o)
|
||||
{
|
||||
vector<ghobject_t> go;
|
||||
|
@ -870,33 +870,20 @@ public:
|
||||
::encode(cid, tbl);
|
||||
ops++;
|
||||
}
|
||||
/**
|
||||
* Add object to another collection (DEPRECATED)
|
||||
*
|
||||
* The Object is added to the new collection. This is a virtual
|
||||
* add, we now have two names for the same object. This is only
|
||||
* used for conversion of old stores to new stores and is not
|
||||
* needed for new implementations unless they expect to make use
|
||||
* of the conversion infrastructure.
|
||||
*/
|
||||
void collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid) {
|
||||
void collection_move(coll_t cid, coll_t oldcid, const ghobject_t& oid) {
|
||||
// NOTE: we encode this as a fixed combo of ADD + REMOVE. they
|
||||
// always appear together, so this is effectively a single MOVE.
|
||||
__u32 op = OP_COLL_ADD;
|
||||
::encode(op, tbl);
|
||||
::encode(cid, tbl);
|
||||
::encode(ocid, tbl);
|
||||
::encode(oldcid, tbl);
|
||||
::encode(oid, tbl);
|
||||
ops++;
|
||||
}
|
||||
void collection_remove(coll_t cid, const ghobject_t& oid) {
|
||||
__u32 op = OP_COLL_REMOVE;
|
||||
op = OP_COLL_REMOVE;
|
||||
::encode(op, tbl);
|
||||
::encode(cid, tbl);
|
||||
::encode(oldcid, tbl);
|
||||
::encode(oid, tbl);
|
||||
ops++;
|
||||
}
|
||||
void collection_move(coll_t cid, coll_t oldcid, const ghobject_t& oid) {
|
||||
collection_add(cid, oldcid, oid);
|
||||
collection_remove(oldcid, oid);
|
||||
return;
|
||||
}
|
||||
void collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
|
||||
@ -954,14 +941,6 @@ public:
|
||||
::encode(aset, tbl);
|
||||
ops++;
|
||||
}
|
||||
/// Change the name of a collection
|
||||
void collection_rename(coll_t cid, coll_t ncid) {
|
||||
__u32 op = OP_COLL_RENAME;
|
||||
::encode(op, tbl);
|
||||
::encode(cid, tbl);
|
||||
::encode(ncid, tbl);
|
||||
ops++;
|
||||
}
|
||||
|
||||
/// Remove omap from oid
|
||||
void omap_clear(
|
||||
@ -1226,9 +1205,12 @@ public:
|
||||
ObjectStore(const ObjectStore& o);
|
||||
const ObjectStore& operator=(const ObjectStore& o);
|
||||
|
||||
// versioning
|
||||
virtual int upgrade() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// mgmt
|
||||
virtual int version_stamp_is_valid(uint32_t *version) { return 1; }
|
||||
virtual int update_version_stamp() = 0;
|
||||
virtual bool test_mount_in_use() = 0;
|
||||
virtual int mount() = 0;
|
||||
virtual int umount() = 0;
|
||||
@ -1243,11 +1225,6 @@ public:
|
||||
|
||||
virtual void collect_metadata(map<string,string> *pm) { }
|
||||
|
||||
/**
|
||||
* get the most recent "on-disk format version" supported
|
||||
*/
|
||||
virtual uint32_t get_target_version() = 0;
|
||||
|
||||
/**
|
||||
* check whether need journal device
|
||||
*
|
||||
|
@ -480,13 +480,10 @@ void ObjectStore::Transaction::generate_test_instances(list<ObjectStore::Transac
|
||||
t->clone_range(c, o1, o2, 1, 12, 99);
|
||||
|
||||
t->create_collection(c);
|
||||
t->collection_add(c, c2, o1);
|
||||
t->collection_add(c, c2, o2);
|
||||
t->collection_move(c, c2, o3);
|
||||
t->remove_collection(c);
|
||||
t->collection_setattr(c, "this", bl);
|
||||
t->collection_rmattr(c, "foo");
|
||||
t->collection_setattrs(c, m);
|
||||
t->collection_rename(c, c2);
|
||||
o.push_back(t);
|
||||
}
|
||||
|
132
src/osd/OSD.cc
132
src/osd/OSD.cc
@ -1284,132 +1284,6 @@ void OSDService::queue_for_peering(PG *pg)
|
||||
#undef dout_prefix
|
||||
#define dout_prefix *_dout
|
||||
|
||||
int OSD::convert_collection(ObjectStore *store, coll_t cid)
|
||||
{
|
||||
coll_t tmp0("convertfs_temp");
|
||||
coll_t tmp1("convertfs_temp1");
|
||||
vector<ghobject_t> objects;
|
||||
|
||||
map<string, bufferptr> aset;
|
||||
int r = store->collection_getattrs(cid, aset);
|
||||
if (r < 0)
|
||||
return r;
|
||||
|
||||
{
|
||||
ObjectStore::Transaction t;
|
||||
t.create_collection(tmp0);
|
||||
for (map<string, bufferptr>::iterator i = aset.begin();
|
||||
i != aset.end();
|
||||
++i) {
|
||||
bufferlist val;
|
||||
val.push_back(i->second);
|
||||
t.collection_setattr(tmp0, i->first, val);
|
||||
}
|
||||
store->apply_transaction(t);
|
||||
}
|
||||
|
||||
ghobject_t next;
|
||||
while (!next.is_max()) {
|
||||
objects.clear();
|
||||
ghobject_t start = next;
|
||||
r = store->collection_list_partial(cid, start,
|
||||
200, 300, 0,
|
||||
&objects, &next);
|
||||
if (r < 0)
|
||||
return r;
|
||||
|
||||
ObjectStore::Transaction t;
|
||||
for (vector<ghobject_t>::iterator i = objects.begin();
|
||||
i != objects.end();
|
||||
++i) {
|
||||
t.collection_add(tmp0, cid, *i);
|
||||
}
|
||||
store->apply_transaction(t);
|
||||
}
|
||||
|
||||
{
|
||||
ObjectStore::Transaction t;
|
||||
t.collection_rename(cid, tmp1);
|
||||
t.collection_rename(tmp0, cid);
|
||||
store->apply_transaction(t);
|
||||
}
|
||||
|
||||
recursive_remove_collection(store, tmp1);
|
||||
store->sync_and_flush();
|
||||
store->sync();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int OSD::do_convertfs(ObjectStore *store)
|
||||
{
|
||||
int r = store->mount();
|
||||
if (r < 0)
|
||||
return r;
|
||||
|
||||
uint32_t version;
|
||||
r = store->version_stamp_is_valid(&version);
|
||||
if (r < 0)
|
||||
return r;
|
||||
if (r == 1)
|
||||
return store->umount();
|
||||
|
||||
derr << "ObjectStore is old at version " << version << ". Updating..." << dendl;
|
||||
|
||||
derr << "Removing tmp pgs" << dendl;
|
||||
vector<coll_t> collections;
|
||||
r = store->list_collections(collections);
|
||||
if (r < 0)
|
||||
return r;
|
||||
for (vector<coll_t>::iterator i = collections.begin();
|
||||
i != collections.end();
|
||||
++i) {
|
||||
spg_t pgid;
|
||||
if (i->is_temp(pgid))
|
||||
recursive_remove_collection(store, *i);
|
||||
else if (i->to_str() == "convertfs_temp" ||
|
||||
i->to_str() == "convertfs_temp1")
|
||||
recursive_remove_collection(store, *i);
|
||||
}
|
||||
store->flush();
|
||||
|
||||
|
||||
derr << "Getting collections" << dendl;
|
||||
|
||||
derr << collections.size() << " to process." << dendl;
|
||||
collections.clear();
|
||||
r = store->list_collections(collections);
|
||||
if (r < 0)
|
||||
return r;
|
||||
int processed = 0;
|
||||
for (vector<coll_t>::iterator i = collections.begin();
|
||||
i != collections.end();
|
||||
++i, ++processed) {
|
||||
derr << processed << "/" << collections.size() << " processed" << dendl;
|
||||
uint32_t collection_version;
|
||||
r = store->collection_version_current(*i, &collection_version);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
} else if (r == 1) {
|
||||
derr << "Collection " << *i << " is up to date" << dendl;
|
||||
} else {
|
||||
derr << "Updating collection " << *i << " current version is "
|
||||
<< collection_version << dendl;
|
||||
r = convert_collection(store, *i);
|
||||
if (r < 0)
|
||||
return r;
|
||||
derr << "collection " << *i << " updated" << dendl;
|
||||
}
|
||||
}
|
||||
derr << "All collections up to date, updating version stamp..." << dendl;
|
||||
r = store->update_version_stamp();
|
||||
if (r < 0)
|
||||
return r;
|
||||
store->sync_and_flush();
|
||||
store->sync();
|
||||
derr << "Version stamp updated, done with upgrade!" << dendl;
|
||||
return store->umount();
|
||||
}
|
||||
|
||||
int OSD::mkfs(CephContext *cct, ObjectStore *store, const string &dev,
|
||||
uuid_d fsid, int whoami)
|
||||
{
|
||||
@ -2570,7 +2444,7 @@ void OSD::recursive_remove_collection(ObjectStore *store, coll_t tmp)
|
||||
int r = mapper.remove_oid(p->hobj, &_t);
|
||||
if (r != 0 && r != -ENOENT)
|
||||
assert(0);
|
||||
t.collection_remove(tmp, *p);
|
||||
t.remove(tmp, *p);
|
||||
if (removed > 300) {
|
||||
int r = store->apply_transaction(t);
|
||||
assert(r == 0);
|
||||
@ -2859,9 +2733,11 @@ void OSD::load_pgs()
|
||||
spg_t pgid;
|
||||
snapid_t snap;
|
||||
uint64_t seq;
|
||||
char val;
|
||||
|
||||
if (it->is_temp(pgid) ||
|
||||
it->is_removal(&seq, &pgid)) {
|
||||
it->is_removal(&seq, &pgid) ||
|
||||
store->collection_getattr(*it, "remove", &val, 1) > 0) {
|
||||
dout(10) << "load_pgs " << *it << " clearing temp" << dendl;
|
||||
recursive_remove_collection(store, *it);
|
||||
continue;
|
||||
|
@ -2323,8 +2323,6 @@ protected:
|
||||
|
||||
// static bits
|
||||
static int find_osd_dev(char *result, int whoami);
|
||||
static int do_convertfs(ObjectStore *store);
|
||||
static int convert_collection(ObjectStore *store, coll_t cid);
|
||||
static int mkfs(CephContext *cct, ObjectStore *store,
|
||||
const string& dev,
|
||||
uuid_d fsid, int whoami);
|
||||
|
@ -70,11 +70,8 @@ bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
|
||||
case DSOP_OBJ_REMOVE:
|
||||
ok = do_remove(gen);
|
||||
break;
|
||||
case DSOP_COLL_ADD:
|
||||
ok = do_coll_add(gen);
|
||||
break;
|
||||
case DSOP_COLL_RENAME:
|
||||
//do_coll_rename(gen);
|
||||
case DSOP_COLL_MOVE:
|
||||
ok = do_coll_move(gen);
|
||||
break;
|
||||
case DSOP_SET_ATTRS:
|
||||
ok = do_set_attrs(gen);
|
||||
@ -384,31 +381,7 @@ bool DeterministicOpSequence::_prepare_colls(rngen_t& gen,
|
||||
}
|
||||
|
||||
|
||||
bool DeterministicOpSequence::do_coll_rename(rngen_t& gen)
|
||||
{
|
||||
int coll_pos = _gen_coll_id(gen);
|
||||
dout(0) << "do_coll_rename coll pos #" << coll_pos << dendl;
|
||||
|
||||
coll_entry_t *coll_entry = get_coll_at(coll_pos);
|
||||
if (!coll_entry) {
|
||||
dout(0) << "do_coll_rename no collection at pos #" << coll_pos << dendl;
|
||||
return false;
|
||||
}
|
||||
|
||||
coll_t orig_coll = coll_entry->m_coll;
|
||||
char buf[100];
|
||||
memset(buf, 0, 100);
|
||||
snprintf(buf, 100, "0.%d_head", m_next_coll_nr++);
|
||||
coll_t new_coll(buf);
|
||||
coll_entry->m_coll = new_coll;
|
||||
|
||||
dout(0) << "do_coll_rename " << orig_coll.to_str()
|
||||
<< " => " << new_coll.to_str() << dendl;
|
||||
_do_coll_rename(orig_coll, new_coll);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
|
||||
bool DeterministicOpSequence::do_coll_move(rngen_t& gen)
|
||||
{
|
||||
coll_entry_t *orig_coll = NULL, *new_coll = NULL;
|
||||
if (!_prepare_colls(gen, orig_coll, new_coll))
|
||||
@ -421,22 +394,25 @@ bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
|
||||
int obj_key = -1;
|
||||
hobject_t *obj = orig_coll->get_obj_at(obj_pos, &obj_key);
|
||||
if (!obj) {
|
||||
dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
|
||||
dout(0) << "do_coll_move coll " << orig_coll->m_coll.to_str()
|
||||
<< " has no object as pos #" << obj_pos << " (key " << obj_key << ")"
|
||||
<< dendl;
|
||||
return false;
|
||||
}
|
||||
if (new_coll->check_for_obj(obj_key)) {
|
||||
dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
|
||||
dout(0) << "do_coll_move coll " << orig_coll->m_coll.to_str()
|
||||
<< " already has object as pos #" << obj_pos << " (key " << obj_key << ")"
|
||||
<< dendl;
|
||||
return false;
|
||||
}
|
||||
dout(0) << "do_coll_add " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
|
||||
dout(0) << "do_coll_move " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
|
||||
<< " => " << new_coll->m_coll.to_str() << "/" << obj->oid.name << dendl;
|
||||
new_coll->touch_obj(obj_key);
|
||||
|
||||
_do_coll_add(orig_coll->m_coll, new_coll->m_coll, *obj);
|
||||
orig_coll->remove_obj(obj_key);
|
||||
|
||||
_do_coll_move(orig_coll->m_coll, new_coll->m_coll, *obj);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -550,20 +526,13 @@ void DeterministicOpSequence::_do_write_and_clone_range(coll_t coll,
|
||||
m_store->apply_transaction(t);
|
||||
}
|
||||
|
||||
void DeterministicOpSequence::_do_coll_add(coll_t orig_coll, coll_t new_coll,
|
||||
hobject_t& obj)
|
||||
void DeterministicOpSequence::_do_coll_move(coll_t orig_coll, coll_t new_coll,
|
||||
hobject_t& obj)
|
||||
{
|
||||
ObjectStore::Transaction t;
|
||||
note_txn(&t);
|
||||
t.remove(new_coll, obj);
|
||||
t.collection_add(new_coll, orig_coll, obj);
|
||||
t.collection_move(new_coll, orig_coll, obj);
|
||||
m_store->apply_transaction(t);
|
||||
}
|
||||
|
||||
void DeterministicOpSequence::_do_coll_rename(coll_t orig_coll, coll_t new_coll)
|
||||
{
|
||||
ObjectStore::Transaction t;
|
||||
note_txn(&t);
|
||||
t.collection_rename(orig_coll, new_coll);
|
||||
m_store->apply_transaction(t);
|
||||
}
|
||||
|
@ -39,8 +39,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
|
||||
DSOP_CLONE = 2,
|
||||
DSOP_CLONE_RANGE = 3,
|
||||
DSOP_OBJ_REMOVE = 4,
|
||||
DSOP_COLL_RENAME = 5,
|
||||
DSOP_COLL_ADD = 6,
|
||||
DSOP_COLL_MOVE = 6,
|
||||
DSOP_SET_ATTRS = 7,
|
||||
DSOP_COLL_CREATE = 8,
|
||||
|
||||
@ -64,8 +63,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
|
||||
bool do_write(rngen_t& gen);
|
||||
bool do_clone(rngen_t& gen);
|
||||
bool do_clone_range(rngen_t& gen);
|
||||
bool do_coll_rename(rngen_t& gen);
|
||||
bool do_coll_add(rngen_t& gen);
|
||||
bool do_coll_move(rngen_t& gen);
|
||||
bool do_set_attrs(rngen_t& gen);
|
||||
bool do_coll_create(rngen_t& gen);
|
||||
|
||||
@ -82,8 +80,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
|
||||
virtual void _do_write_and_clone_range(coll_t coll, hobject_t& orig_obj,
|
||||
hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
|
||||
uint64_t dstoff, bufferlist& bl);
|
||||
virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
|
||||
virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll);
|
||||
virtual void _do_coll_move(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
|
||||
virtual void _do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs);
|
||||
|
||||
int _gen_coll_id(rngen_t& gen);
|
||||
|
@ -1060,7 +1060,7 @@ TEST_P(StoreTest, HashCollisionTest) {
|
||||
i != created.end();
|
||||
++i) {
|
||||
ObjectStore::Transaction t;
|
||||
t.collection_remove(cid, *i);
|
||||
t.remove(cid, *i);
|
||||
r = store->apply_transaction(t);
|
||||
ASSERT_EQ(r, 0);
|
||||
}
|
||||
@ -1134,7 +1134,7 @@ TEST_P(StoreTest, ScrubTest) {
|
||||
i != created.end();
|
||||
++i) {
|
||||
ObjectStore::Transaction t;
|
||||
t.collection_remove(cid, *i);
|
||||
t.remove(cid, *i);
|
||||
r = store->apply_transaction(t);
|
||||
ASSERT_EQ(r, 0);
|
||||
}
|
||||
|
@ -42,10 +42,6 @@ ceph_psim_SOURCES = tools/psim.cc
|
||||
ceph_psim_LDADD = $(CEPH_GLOBAL)
|
||||
bin_DEBUGPROGRAMS += ceph_psim
|
||||
|
||||
ceph_dupstore_SOURCES = tools/dupstore.cc
|
||||
ceph_dupstore_LDADD = $(LIBOS) $(CEPH_GLOBAL)
|
||||
bin_DEBUGPROGRAMS += ceph_dupstore
|
||||
|
||||
ceph_radosacl_SOURCES = tools/radosacl.cc
|
||||
ceph_radosacl_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
|
||||
bin_DEBUGPROGRAMS += ceph_radosacl
|
||||
|
@ -495,7 +495,7 @@ out:
|
||||
}
|
||||
|
||||
//Based on part of OSD::load_pgs()
|
||||
int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
|
||||
int finish_remove_pgs(ObjectStore *store)
|
||||
{
|
||||
vector<coll_t> ls;
|
||||
int r = store->list_collections(ls);
|
||||
@ -522,11 +522,12 @@ int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
|
||||
}
|
||||
|
||||
uint64_t seq;
|
||||
if (it->is_removal(&seq, &pgid)) {
|
||||
if (seq >= *next_removal_seq)
|
||||
*next_removal_seq = seq + 1;
|
||||
cout << "finish_remove_pgs removing " << *it << ", seq is "
|
||||
<< seq << " pgid is " << pgid << std::endl;
|
||||
coll_t coll(pgid);
|
||||
char val;
|
||||
if (it->is_removal(&seq, &pgid) ||
|
||||
store->collection_getattr(coll, "remove", &val, 1) == 1) {
|
||||
cout << "finish_remove_pgs removing " << *it
|
||||
<< " pgid is " << pgid << std::endl;
|
||||
remove_coll(store, *it);
|
||||
continue;
|
||||
}
|
||||
@ -536,17 +537,15 @@ int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid,
|
||||
uint64_t *next_removal_seq)
|
||||
int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid)
|
||||
{
|
||||
ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
|
||||
|
||||
if (store->collection_exists(coll_t(r_pgid))) {
|
||||
coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++, r_pgid);
|
||||
cout << "collection rename " << coll_t(r_pgid)
|
||||
<< " to " << to_remove
|
||||
<< std::endl;
|
||||
rmt->collection_rename(coll_t(r_pgid), to_remove);
|
||||
cout << " marking collection for removal" << std::endl;
|
||||
bufferlist one;
|
||||
one.append('1');
|
||||
rmt->collection_setattr(coll_t(r_pgid), "remove", one);
|
||||
} else {
|
||||
delete rmt;
|
||||
return ENOENT;
|
||||
@ -1199,9 +1198,6 @@ int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl)
|
||||
cout << std::endl;
|
||||
#endif
|
||||
|
||||
coll_t newcoll(ms.info.pgid);
|
||||
t->collection_rename(coll, newcoll);
|
||||
|
||||
int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver, ms.past_intervals);
|
||||
if (ret) return ret;
|
||||
|
||||
@ -1335,8 +1331,7 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
|
||||
pg_info_t info;
|
||||
PGLog::IndexedLog log;
|
||||
|
||||
uint64_t next_removal_seq = 0; //My local seq
|
||||
finish_remove_pgs(store, &next_removal_seq);
|
||||
finish_remove_pgs(store);
|
||||
|
||||
int ret = sh.read_super();
|
||||
if (ret)
|
||||
@ -1385,11 +1380,11 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
|
||||
return 1;
|
||||
}
|
||||
|
||||
//Switch to collection which will be removed automatically if
|
||||
//this program is interupted.
|
||||
coll_t rmcoll = coll_t::make_removal_coll(next_removal_seq, pgid);
|
||||
// mark this coll for removal until we are done
|
||||
bufferlist one;
|
||||
one.append('1');
|
||||
ObjectStore::Transaction *t = new ObjectStore::Transaction;
|
||||
t->create_collection(rmcoll);
|
||||
t->collection_setattr(coll, "remove", one);
|
||||
store->apply_transaction(*t);
|
||||
delete t;
|
||||
|
||||
@ -1409,11 +1404,11 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
|
||||
}
|
||||
switch(type) {
|
||||
case TYPE_OBJECT_BEGIN:
|
||||
ret = get_object(store, rmcoll, ebl);
|
||||
ret = get_object(store, coll, ebl);
|
||||
if (ret) return ret;
|
||||
break;
|
||||
case TYPE_PG_METADATA:
|
||||
ret = get_pg_metadata(store, rmcoll, ebl);
|
||||
ret = get_pg_metadata(store, coll, ebl);
|
||||
if (ret) return ret;
|
||||
found_metadata = true;
|
||||
break;
|
||||
@ -1430,6 +1425,13 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
|
||||
return EFAULT;
|
||||
}
|
||||
|
||||
// done, clear removal flag
|
||||
cout << "done, clearing remove flag" << std::endl;
|
||||
t = new ObjectStore::Transaction;
|
||||
t->collection_rmattr(coll, "remove");
|
||||
store->apply_transaction(*t);
|
||||
delete t;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -2138,15 +2140,14 @@ int main(int argc, char **argv)
|
||||
biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
|
||||
|
||||
if (op == "remove") {
|
||||
uint64_t next_removal_seq = 0; //My local seq
|
||||
finish_remove_pgs(fs, &next_removal_seq);
|
||||
int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq);
|
||||
finish_remove_pgs(fs);
|
||||
int r = initiate_new_remove_pg(fs, pgid);
|
||||
if (r) {
|
||||
cerr << "PG '" << pgid << "' not found" << std::endl;
|
||||
ret = 1;
|
||||
goto out;
|
||||
}
|
||||
finish_remove_pgs(fs, &next_removal_seq);
|
||||
finish_remove_pgs(fs);
|
||||
cout << "Remove successful" << std::endl;
|
||||
goto out;
|
||||
}
|
||||
|
@ -1,110 +0,0 @@
|
||||
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
|
||||
// vim: ts=8 sw=2 smarttab
|
||||
/*
|
||||
* Ceph - scalable distributed file system
|
||||
*
|
||||
* Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
|
||||
*
|
||||
* This is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License version 2.1, as published by the Free Software
|
||||
* Foundation. See file COPYING.
|
||||
*
|
||||
*/
|
||||
#include "common/ceph_argparse.h"
|
||||
#include "global/global_init.h"
|
||||
#include "include/unordered_map.h"
|
||||
#include "os/FileStore.h"
|
||||
|
||||
int dupstore(ObjectStore* src, ObjectStore* dst)
|
||||
{
|
||||
if (src->mount() < 0) return 1;
|
||||
if (dst->mkfs() < 0) return 1;
|
||||
if (dst->mount() < 0) return 1;
|
||||
|
||||
// objects
|
||||
ceph::unordered_map<ghobject_t, coll_t> did_object;
|
||||
|
||||
// collections
|
||||
vector<coll_t> collections;
|
||||
|
||||
int ret = src->list_collections(collections);
|
||||
if (ret < 0) {
|
||||
cerr << "Error " << ret << " while listing collections" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int num = collections.size();
|
||||
cout << num << " collections" << std::endl;
|
||||
int i = 1;
|
||||
for (vector<coll_t>::iterator p = collections.begin();
|
||||
p != collections.end();
|
||||
++p) {
|
||||
cout << "collection " << i++ << "/" << num << " " << hex << *p << dec << std::endl;
|
||||
{
|
||||
ObjectStore::Transaction t;
|
||||
t.create_collection(*p);
|
||||
map<string,bufferptr> attrs;
|
||||
src->collection_getattrs(*p, attrs);
|
||||
t.collection_setattrs(*p, attrs);
|
||||
dst->apply_transaction(t);
|
||||
}
|
||||
|
||||
vector<ghobject_t> o;
|
||||
src->collection_list(*p, o);
|
||||
int numo = o.size();
|
||||
int j = 1;
|
||||
for (vector<ghobject_t>::iterator q = o.begin(); q != o.end(); ++q) {
|
||||
ObjectStore::Transaction t;
|
||||
if (did_object.count(*q))
|
||||
t.collection_add(*p, did_object[*q], *q);
|
||||
else {
|
||||
bufferlist bl;
|
||||
src->read(*p, *q, 0, 0, bl);
|
||||
cout << "object " << j++ << "/" << numo << " " << *q << " = " << bl.length() << " bytes" << std::endl;
|
||||
t.write(*p, *q, 0, bl.length(), bl);
|
||||
map<string,bufferptr> attrs;
|
||||
src->getattrs(*p, *q, attrs);
|
||||
t.setattrs(*p, *q, attrs);
|
||||
did_object[*q] = *p;
|
||||
}
|
||||
dst->apply_transaction(t);
|
||||
}
|
||||
}
|
||||
|
||||
src->umount();
|
||||
dst->umount();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void usage()
|
||||
{
|
||||
cerr << "usage: ceph_dupstore filestore SRC filestore DST" << std::endl;
|
||||
exit(0);
|
||||
}
|
||||
|
||||
int main(int argc, const char **argv)
|
||||
{
|
||||
vector<const char*> args;
|
||||
argv_to_vec(argc, argv, args);
|
||||
env_to_vec(args);
|
||||
|
||||
global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
|
||||
common_init_finish(g_ceph_context);
|
||||
|
||||
// args
|
||||
if (args.size() != 4)
|
||||
usage();
|
||||
|
||||
ObjectStore *src = 0, *dst = 0;
|
||||
|
||||
if (strcmp(args[0], "filestore") == 0)
|
||||
src = new FileStore(args[1], NULL);
|
||||
else usage();
|
||||
|
||||
if (strcmp(args[2], "filestore") == 0)
|
||||
dst = new FileStore(args[3], NULL);
|
||||
else usage();
|
||||
|
||||
return dupstore(src, dst);
|
||||
}
|
@ -447,22 +447,6 @@ TRACEPOINT_EVENT(objectstore, omap_clear_exit,
|
||||
)
|
||||
)
|
||||
|
||||
TRACEPOINT_EVENT(objectstore, coll_rename_enter,
|
||||
TP_ARGS(
|
||||
const char *, osr_name),
|
||||
TP_FIELDS(
|
||||
ctf_string(osr_name, osr_name)
|
||||
)
|
||||
)
|
||||
|
||||
TRACEPOINT_EVENT(objectstore, coll_rename_exit,
|
||||
TP_ARGS(
|
||||
int, retval),
|
||||
TP_FIELDS(
|
||||
ctf_integer(int, retval, retval)
|
||||
)
|
||||
)
|
||||
|
||||
TRACEPOINT_EVENT(objectstore, startsync_enter,
|
||||
TP_ARGS(
|
||||
const char *, osr_name),
|
||||
|
Loading…
Reference in New Issue
Block a user