Merge pull request #2124 from ceph/wip-objectstore

simplify ObjectStore interface

sage-2014-11-11_08:26:01-rados-wip-sage-testing-distro-basic-multi

Reviewed-by: Haomai Wang <haomaiwang@gmail.com>
Reviewed-by: Samuel Just <sjust@redhat.com>
This commit is contained in:
Sage Weil 2014-11-12 10:02:52 -08:00
commit 22b6c1a531
22 changed files with 117 additions and 538 deletions

View File

@ -718,7 +718,6 @@ ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1
%files -n ceph-test
%defattr(-,root,root,-)
%{_bindir}/ceph_bench_log
%{_bindir}/ceph_dupstore
%{_bindir}/ceph_kvstorebench
%{_bindir}/ceph_multi_stress_watch
%{_bindir}/ceph_erasure_code

View File

@ -1,6 +1,5 @@
usr/bin/ceph-coverage
usr/bin/ceph_bench_log
usr/bin/ceph_dupstore
usr/bin/ceph_objectstore_tool
usr/bin/ceph_kvstorebench
usr/bin/ceph_multi_stress_watch

1
src/.gitignore vendored
View File

@ -24,7 +24,6 @@ Makefile
/ceph-syn
/ceph.conf
/ceph_bench_log
/ceph_dupstore
/ceph_objectstore_tool
/ceph_mon_store_converter
/ceph_multi_stress_watch

View File

@ -294,7 +294,14 @@ int main(int argc, const char **argv)
if (convertfilestore) {
int err = OSD::do_convertfs(store);
int err = store->mount();
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error mounting store " << g_conf->osd_data
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
exit(1);
}
err = store->upgrade();
store->umount();
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
@ -468,15 +475,6 @@ int main(int argc, const char **argv)
global_init_daemonize(g_ceph_context, 0);
common_init_finish(g_ceph_context);
if (g_conf->filestore_update_to >= (int)store->get_target_version()) {
int err = OSD::do_convertfs(store);
if (err < 0) {
derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data
<< ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl;
exit(1);
}
}
MonClient mc(g_ceph_context);
if (mc.build_initial_monmap() < 0)
return -1;

View File

@ -1164,6 +1164,19 @@ int FileStore::write_version_stamp()
bl.c_str(), bl.length());
}
int FileStore::upgrade()
{
uint32_t version;
int r = version_stamp_is_valid(&version);
if (r < 0)
return r;
if (r == 1)
return 0;
derr << "ObjectStore is old at version " << version << ". Please upgrade to firefly v0.80.x, convert your store, and then upgrade." << dendl;
return -EINVAL;
}
int FileStore::read_op_seq(uint64_t *seq)
{
int op_fd = ::open(current_op_seq_fn.c_str(), O_CREAT|O_RDWR, 0644);
@ -1568,6 +1581,16 @@ int FileStore::mount()
timer.init();
// upgrade?
if (g_conf->filestore_update_to >= (int)get_target_version()) {
int err = upgrade();
if (err < 0) {
derr << "error converting store" << dendl;
umount();
return err;
}
}
// all okay.
return 0;
@ -2452,19 +2475,26 @@ unsigned FileStore::_do_transaction(
coll_t ncid = i.decode_cid();
coll_t ocid = i.decode_cid();
ghobject_t oid = i.decode_oid();
// always followed by OP_COLL_REMOVE
int op = i.decode_op();
coll_t ocid2 = i.decode_cid();
ghobject_t oid2 = i.decode_oid();
assert(op == Transaction::OP_COLL_REMOVE);
assert(ocid2 == ocid);
assert(oid2 == oid);
tracepoint(objectstore, coll_add_enter);
r = _collection_add(ncid, ocid, oid, spos);
tracepoint(objectstore, coll_add_exit, r);
}
break;
case Transaction::OP_COLL_REMOVE:
{
coll_t cid = i.decode_cid();
ghobject_t oid = i.decode_oid();
if (r == -ENOENT && i.tolerate_collection_add_enoent())
r = 0;
spos.op++;
if (r < 0)
break;
tracepoint(objectstore, coll_remove_enter, osr_name);
if (_check_replay_guard(cid, oid, spos) > 0)
r = _remove(cid, oid, spos);
if (_check_replay_guard(ocid, oid, spos) > 0)
r = _remove(ocid, oid, spos);
tracepoint(objectstore, coll_remove_exit, r);
}
break;
@ -2530,9 +2560,7 @@ unsigned FileStore::_do_transaction(
{
coll_t cid(i.decode_cid());
coll_t ncid(i.decode_cid());
tracepoint(objectstore, coll_rename_enter, osr_name);
r = _collection_rename(cid, ncid, spos);
tracepoint(objectstore, coll_rename_exit, r);
r = -EOPNOTSUPP;
}
break;
@ -4292,56 +4320,6 @@ int FileStore::_collection_remove_recursive(const coll_t &cid,
return _destroy_collection(cid);
}
int FileStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
const SequencerPosition& spos)
{
char new_coll[PATH_MAX], old_coll[PATH_MAX];
get_cdir(cid, old_coll, sizeof(old_coll));
get_cdir(ncid, new_coll, sizeof(new_coll));
if (_check_replay_guard(cid, spos) < 0) {
return 0;
}
if (_check_replay_guard(ncid, spos) < 0) {
return _collection_remove_recursive(cid, spos);
}
if (!collection_exists(cid)) {
if (replaying) {
// already happened
return 0;
} else {
return -ENOENT;
}
}
_set_global_replay_guard(cid, spos);
int ret = 0;
if (::rename(old_coll, new_coll)) {
if (replaying && !backend->can_checkpoint() &&
(errno == EEXIST || errno == ENOTEMPTY))
ret = _collection_remove_recursive(cid, spos);
else
ret = -errno;
dout(10) << "collection_rename '" << cid << "' to '" << ncid << "'"
<< ": ret = " << ret << dendl;
return ret;
}
if (ret >= 0) {
int fd = ::open(new_coll, O_RDONLY);
assert(fd >= 0);
_set_replay_guard(fd, spos);
VOID_TEMP_FAILURE_RETRY(::close(fd));
}
dout(10) << "collection_rename '" << cid << "' to '" << ncid << "'"
<< ": ret = " << ret << dendl;
return ret;
}
// --------------------------
// collections

View File

@ -377,7 +377,6 @@ private:
void op_queue_release_throttle(Op *o);
void _journaled_ahead(OpSequencer *osr, Op *o, Context *ondisk);
friend struct C_JournaledAhead;
int write_version_stamp();
int open_journal();
@ -410,8 +409,6 @@ public:
int _sanity_check_fs();
bool test_mount_in_use();
int version_stamp_is_valid(uint32_t *version);
int update_version_stamp();
int read_op_seq(uint64_t *seq);
int write_op_seq(int, uint64_t seq);
int mount();
@ -428,6 +425,11 @@ public:
int mkfs();
int mkjournal();
int write_version_stamp();
int version_stamp_is_valid(uint32_t *version);
int update_version_stamp();
int upgrade();
/**
* set_allow_sharded_objects()
*
@ -593,8 +595,6 @@ public:
int _collection_setattrs(coll_t cid, map<string,bufferptr> &aset);
int _collection_remove_recursive(const coll_t &cid,
const SequencerPosition &spos);
int _collection_rename(const coll_t &cid, const coll_t &ncid,
const SequencerPosition& spos);
// collections
int list_collections(vector<coll_t>& ls);

View File

@ -1447,7 +1447,7 @@ unsigned KeyValueStore::_do_transaction(Transaction& transaction,
{
coll_t cid(i.decode_cid());
coll_t ncid(i.decode_cid());
r = _collection_rename(cid, ncid, t);
r = -EOPNOTSUPP;
}
break;
@ -2582,61 +2582,6 @@ int KeyValueStore::_collection_remove_recursive(const coll_t &cid,
return 0;
}
int KeyValueStore::_collection_rename(const coll_t &cid, const coll_t &ncid,
BufferTransaction &t)
{
dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
<< dendl;
StripObjectMap::StripObjectHeaderRef header;
int r = t.lookup_cached_header(get_coll_for_coll(),
make_ghobject_for_coll(ncid),
&header, false);
if (r == 0) {
dout(2) << __func__ << ": " << ncid << " DNE" << dendl;
return -EEXIST;
}
r = t.lookup_cached_header(get_coll_for_coll(), make_ghobject_for_coll(cid),
&header, false);
if (r < 0) {
dout(2) << __func__ << ": " << cid << " DNE" << dendl;
return 0;
}
vector<ghobject_t> objects;
ghobject_t next, current;
int move_size = 0;
while (1) {
collection_list_partial(cid, current, get_ideal_list_min(),
get_ideal_list_max(), 0, &objects, &next);
dout(20) << __func__ << cid << "objects size: " << objects.size()
<< dendl;
if (objects.empty())
break;
for (vector<ghobject_t>::iterator i = objects.begin();
i != objects.end(); ++i) {
if (_collection_move_rename(cid, *i, ncid, *i, t) < 0) {
return -1;
}
move_size++;
}
objects.clear();
current = next;
}
t.rename_buffer(header, get_coll_for_coll(), make_ghobject_for_coll(ncid));
dout(10) << __func__ << " origin cid " << cid << " new cid " << ncid
<< dendl;
return 0;
}
int KeyValueStore::list_collections(vector<coll_t>& ls)
{
dout(10) << __func__ << " " << dendl;

View File

@ -611,8 +611,6 @@ class KeyValueStore : public ObjectStore,
BufferTransaction &t);
int _collection_remove_recursive(const coll_t &cid,
BufferTransaction &t);
int _collection_rename(const coll_t &cid, const coll_t &ncid,
BufferTransaction &t);
int list_collections(vector<coll_t>& ls);
bool collection_exists(coll_t c);
bool collection_empty(coll_t c);

View File

@ -911,7 +911,7 @@ void MemStore::_do_transaction(Transaction& t)
{
coll_t cid(i.decode_cid());
coll_t ncid(i.decode_cid());
r = _collection_rename(cid, ncid);
r = -EOPNOTSUPP;
}
break;
@ -1485,19 +1485,6 @@ int MemStore::_collection_rmattr(coll_t cid, const char *name)
return 0;
}
int MemStore::_collection_rename(const coll_t &cid, const coll_t &ncid)
{
dout(10) << __func__ << " " << cid << " -> " << ncid << dendl;
RWLock::WLocker l(coll_lock);
if (coll_map.count(cid) == 0)
return -ENOENT;
if (coll_map.count(ncid))
return -EEXIST;
coll_map[ncid] = coll_map[cid];
coll_map.erase(cid);
return 0;
}
int MemStore::_split_collection(coll_t cid, uint32_t bits, uint32_t match,
coll_t dest)
{

View File

@ -218,7 +218,6 @@ private:
size_t size);
int _collection_setattrs(coll_t cid, map<string,bufferptr> &aset);
int _collection_rmattr(coll_t cid, const char *name);
int _collection_rename(const coll_t &cid, const coll_t &ncid);
int _split_collection(coll_t cid, uint32_t bits, uint32_t rem, coll_t dest);
int _save();
@ -236,13 +235,6 @@ public:
sharded(false) { }
~MemStore() { }
int update_version_stamp() {
return 0;
}
uint32_t get_target_version() {
return 1;
}
bool need_journal() { return false; };
int peek_journal_fsid(uuid_d *fsid);

View File

@ -113,7 +113,6 @@ int ObjectStore::queue_transactions(
onreadable_sync, op);
}
int ObjectStore::collection_list(coll_t c, vector<hobject_t>& o)
{
vector<ghobject_t> go;

View File

@ -870,33 +870,20 @@ public:
::encode(cid, tbl);
ops++;
}
/**
* Add object to another collection (DEPRECATED)
*
* The Object is added to the new collection. This is a virtual
* add, we now have two names for the same object. This is only
* used for conversion of old stores to new stores and is not
* needed for new implementations unless they expect to make use
* of the conversion infrastructure.
*/
void collection_add(coll_t cid, coll_t ocid, const ghobject_t& oid) {
void collection_move(coll_t cid, coll_t oldcid, const ghobject_t& oid) {
// NOTE: we encode this as a fixed combo of ADD + REMOVE. they
// always appear together, so this is effectively a single MOVE.
__u32 op = OP_COLL_ADD;
::encode(op, tbl);
::encode(cid, tbl);
::encode(ocid, tbl);
::encode(oldcid, tbl);
::encode(oid, tbl);
ops++;
}
void collection_remove(coll_t cid, const ghobject_t& oid) {
__u32 op = OP_COLL_REMOVE;
op = OP_COLL_REMOVE;
::encode(op, tbl);
::encode(cid, tbl);
::encode(oldcid, tbl);
::encode(oid, tbl);
ops++;
}
void collection_move(coll_t cid, coll_t oldcid, const ghobject_t& oid) {
collection_add(cid, oldcid, oid);
collection_remove(oldcid, oid);
return;
}
void collection_move_rename(coll_t oldcid, const ghobject_t& oldoid,
@ -954,14 +941,6 @@ public:
::encode(aset, tbl);
ops++;
}
/// Change the name of a collection
void collection_rename(coll_t cid, coll_t ncid) {
__u32 op = OP_COLL_RENAME;
::encode(op, tbl);
::encode(cid, tbl);
::encode(ncid, tbl);
ops++;
}
/// Remove omap from oid
void omap_clear(
@ -1226,9 +1205,12 @@ public:
ObjectStore(const ObjectStore& o);
const ObjectStore& operator=(const ObjectStore& o);
// versioning
virtual int upgrade() {
return 0;
}
// mgmt
virtual int version_stamp_is_valid(uint32_t *version) { return 1; }
virtual int update_version_stamp() = 0;
virtual bool test_mount_in_use() = 0;
virtual int mount() = 0;
virtual int umount() = 0;
@ -1243,11 +1225,6 @@ public:
virtual void collect_metadata(map<string,string> *pm) { }
/**
* get the most recent "on-disk format version" supported
*/
virtual uint32_t get_target_version() = 0;
/**
* check whether need journal device
*

View File

@ -480,13 +480,10 @@ void ObjectStore::Transaction::generate_test_instances(list<ObjectStore::Transac
t->clone_range(c, o1, o2, 1, 12, 99);
t->create_collection(c);
t->collection_add(c, c2, o1);
t->collection_add(c, c2, o2);
t->collection_move(c, c2, o3);
t->remove_collection(c);
t->collection_setattr(c, "this", bl);
t->collection_rmattr(c, "foo");
t->collection_setattrs(c, m);
t->collection_rename(c, c2);
o.push_back(t);
}

View File

@ -1284,132 +1284,6 @@ void OSDService::queue_for_peering(PG *pg)
#undef dout_prefix
#define dout_prefix *_dout
int OSD::convert_collection(ObjectStore *store, coll_t cid)
{
coll_t tmp0("convertfs_temp");
coll_t tmp1("convertfs_temp1");
vector<ghobject_t> objects;
map<string, bufferptr> aset;
int r = store->collection_getattrs(cid, aset);
if (r < 0)
return r;
{
ObjectStore::Transaction t;
t.create_collection(tmp0);
for (map<string, bufferptr>::iterator i = aset.begin();
i != aset.end();
++i) {
bufferlist val;
val.push_back(i->second);
t.collection_setattr(tmp0, i->first, val);
}
store->apply_transaction(t);
}
ghobject_t next;
while (!next.is_max()) {
objects.clear();
ghobject_t start = next;
r = store->collection_list_partial(cid, start,
200, 300, 0,
&objects, &next);
if (r < 0)
return r;
ObjectStore::Transaction t;
for (vector<ghobject_t>::iterator i = objects.begin();
i != objects.end();
++i) {
t.collection_add(tmp0, cid, *i);
}
store->apply_transaction(t);
}
{
ObjectStore::Transaction t;
t.collection_rename(cid, tmp1);
t.collection_rename(tmp0, cid);
store->apply_transaction(t);
}
recursive_remove_collection(store, tmp1);
store->sync_and_flush();
store->sync();
return 0;
}
int OSD::do_convertfs(ObjectStore *store)
{
int r = store->mount();
if (r < 0)
return r;
uint32_t version;
r = store->version_stamp_is_valid(&version);
if (r < 0)
return r;
if (r == 1)
return store->umount();
derr << "ObjectStore is old at version " << version << ". Updating..." << dendl;
derr << "Removing tmp pgs" << dendl;
vector<coll_t> collections;
r = store->list_collections(collections);
if (r < 0)
return r;
for (vector<coll_t>::iterator i = collections.begin();
i != collections.end();
++i) {
spg_t pgid;
if (i->is_temp(pgid))
recursive_remove_collection(store, *i);
else if (i->to_str() == "convertfs_temp" ||
i->to_str() == "convertfs_temp1")
recursive_remove_collection(store, *i);
}
store->flush();
derr << "Getting collections" << dendl;
derr << collections.size() << " to process." << dendl;
collections.clear();
r = store->list_collections(collections);
if (r < 0)
return r;
int processed = 0;
for (vector<coll_t>::iterator i = collections.begin();
i != collections.end();
++i, ++processed) {
derr << processed << "/" << collections.size() << " processed" << dendl;
uint32_t collection_version;
r = store->collection_version_current(*i, &collection_version);
if (r < 0) {
return r;
} else if (r == 1) {
derr << "Collection " << *i << " is up to date" << dendl;
} else {
derr << "Updating collection " << *i << " current version is "
<< collection_version << dendl;
r = convert_collection(store, *i);
if (r < 0)
return r;
derr << "collection " << *i << " updated" << dendl;
}
}
derr << "All collections up to date, updating version stamp..." << dendl;
r = store->update_version_stamp();
if (r < 0)
return r;
store->sync_and_flush();
store->sync();
derr << "Version stamp updated, done with upgrade!" << dendl;
return store->umount();
}
int OSD::mkfs(CephContext *cct, ObjectStore *store, const string &dev,
uuid_d fsid, int whoami)
{
@ -2570,7 +2444,7 @@ void OSD::recursive_remove_collection(ObjectStore *store, coll_t tmp)
int r = mapper.remove_oid(p->hobj, &_t);
if (r != 0 && r != -ENOENT)
assert(0);
t.collection_remove(tmp, *p);
t.remove(tmp, *p);
if (removed > 300) {
int r = store->apply_transaction(t);
assert(r == 0);
@ -2859,9 +2733,11 @@ void OSD::load_pgs()
spg_t pgid;
snapid_t snap;
uint64_t seq;
char val;
if (it->is_temp(pgid) ||
it->is_removal(&seq, &pgid)) {
it->is_removal(&seq, &pgid) ||
store->collection_getattr(*it, "remove", &val, 1) > 0) {
dout(10) << "load_pgs " << *it << " clearing temp" << dendl;
recursive_remove_collection(store, *it);
continue;

View File

@ -2323,8 +2323,6 @@ protected:
// static bits
static int find_osd_dev(char *result, int whoami);
static int do_convertfs(ObjectStore *store);
static int convert_collection(ObjectStore *store, coll_t cid);
static int mkfs(CephContext *cct, ObjectStore *store,
const string& dev,
uuid_d fsid, int whoami);

View File

@ -70,11 +70,8 @@ bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
case DSOP_OBJ_REMOVE:
ok = do_remove(gen);
break;
case DSOP_COLL_ADD:
ok = do_coll_add(gen);
break;
case DSOP_COLL_RENAME:
//do_coll_rename(gen);
case DSOP_COLL_MOVE:
ok = do_coll_move(gen);
break;
case DSOP_SET_ATTRS:
ok = do_set_attrs(gen);
@ -384,31 +381,7 @@ bool DeterministicOpSequence::_prepare_colls(rngen_t& gen,
}
bool DeterministicOpSequence::do_coll_rename(rngen_t& gen)
{
int coll_pos = _gen_coll_id(gen);
dout(0) << "do_coll_rename coll pos #" << coll_pos << dendl;
coll_entry_t *coll_entry = get_coll_at(coll_pos);
if (!coll_entry) {
dout(0) << "do_coll_rename no collection at pos #" << coll_pos << dendl;
return false;
}
coll_t orig_coll = coll_entry->m_coll;
char buf[100];
memset(buf, 0, 100);
snprintf(buf, 100, "0.%d_head", m_next_coll_nr++);
coll_t new_coll(buf);
coll_entry->m_coll = new_coll;
dout(0) << "do_coll_rename " << orig_coll.to_str()
<< " => " << new_coll.to_str() << dendl;
_do_coll_rename(orig_coll, new_coll);
return true;
}
bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
bool DeterministicOpSequence::do_coll_move(rngen_t& gen)
{
coll_entry_t *orig_coll = NULL, *new_coll = NULL;
if (!_prepare_colls(gen, orig_coll, new_coll))
@ -421,22 +394,25 @@ bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
int obj_key = -1;
hobject_t *obj = orig_coll->get_obj_at(obj_pos, &obj_key);
if (!obj) {
dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
dout(0) << "do_coll_move coll " << orig_coll->m_coll.to_str()
<< " has no object as pos #" << obj_pos << " (key " << obj_key << ")"
<< dendl;
return false;
}
if (new_coll->check_for_obj(obj_key)) {
dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
dout(0) << "do_coll_move coll " << orig_coll->m_coll.to_str()
<< " already has object as pos #" << obj_pos << " (key " << obj_key << ")"
<< dendl;
return false;
}
dout(0) << "do_coll_add " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
dout(0) << "do_coll_move " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
<< " => " << new_coll->m_coll.to_str() << "/" << obj->oid.name << dendl;
new_coll->touch_obj(obj_key);
_do_coll_add(orig_coll->m_coll, new_coll->m_coll, *obj);
orig_coll->remove_obj(obj_key);
_do_coll_move(orig_coll->m_coll, new_coll->m_coll, *obj);
return true;
}
@ -550,20 +526,13 @@ void DeterministicOpSequence::_do_write_and_clone_range(coll_t coll,
m_store->apply_transaction(t);
}
void DeterministicOpSequence::_do_coll_add(coll_t orig_coll, coll_t new_coll,
hobject_t& obj)
void DeterministicOpSequence::_do_coll_move(coll_t orig_coll, coll_t new_coll,
hobject_t& obj)
{
ObjectStore::Transaction t;
note_txn(&t);
t.remove(new_coll, obj);
t.collection_add(new_coll, orig_coll, obj);
t.collection_move(new_coll, orig_coll, obj);
m_store->apply_transaction(t);
}
void DeterministicOpSequence::_do_coll_rename(coll_t orig_coll, coll_t new_coll)
{
ObjectStore::Transaction t;
note_txn(&t);
t.collection_rename(orig_coll, new_coll);
m_store->apply_transaction(t);
}

View File

@ -39,8 +39,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
DSOP_CLONE = 2,
DSOP_CLONE_RANGE = 3,
DSOP_OBJ_REMOVE = 4,
DSOP_COLL_RENAME = 5,
DSOP_COLL_ADD = 6,
DSOP_COLL_MOVE = 6,
DSOP_SET_ATTRS = 7,
DSOP_COLL_CREATE = 8,
@ -64,8 +63,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
bool do_write(rngen_t& gen);
bool do_clone(rngen_t& gen);
bool do_clone_range(rngen_t& gen);
bool do_coll_rename(rngen_t& gen);
bool do_coll_add(rngen_t& gen);
bool do_coll_move(rngen_t& gen);
bool do_set_attrs(rngen_t& gen);
bool do_coll_create(rngen_t& gen);
@ -82,8 +80,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
virtual void _do_write_and_clone_range(coll_t coll, hobject_t& orig_obj,
hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
uint64_t dstoff, bufferlist& bl);
virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll);
virtual void _do_coll_move(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
virtual void _do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs);
int _gen_coll_id(rngen_t& gen);

View File

@ -1060,7 +1060,7 @@ TEST_P(StoreTest, HashCollisionTest) {
i != created.end();
++i) {
ObjectStore::Transaction t;
t.collection_remove(cid, *i);
t.remove(cid, *i);
r = store->apply_transaction(t);
ASSERT_EQ(r, 0);
}
@ -1134,7 +1134,7 @@ TEST_P(StoreTest, ScrubTest) {
i != created.end();
++i) {
ObjectStore::Transaction t;
t.collection_remove(cid, *i);
t.remove(cid, *i);
r = store->apply_transaction(t);
ASSERT_EQ(r, 0);
}

View File

@ -42,10 +42,6 @@ ceph_psim_SOURCES = tools/psim.cc
ceph_psim_LDADD = $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_psim
ceph_dupstore_SOURCES = tools/dupstore.cc
ceph_dupstore_LDADD = $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_dupstore
ceph_radosacl_SOURCES = tools/radosacl.cc
ceph_radosacl_LDADD = $(LIBRADOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_radosacl

View File

@ -495,7 +495,7 @@ out:
}
//Based on part of OSD::load_pgs()
int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
int finish_remove_pgs(ObjectStore *store)
{
vector<coll_t> ls;
int r = store->list_collections(ls);
@ -522,11 +522,12 @@ int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
}
uint64_t seq;
if (it->is_removal(&seq, &pgid)) {
if (seq >= *next_removal_seq)
*next_removal_seq = seq + 1;
cout << "finish_remove_pgs removing " << *it << ", seq is "
<< seq << " pgid is " << pgid << std::endl;
coll_t coll(pgid);
char val;
if (it->is_removal(&seq, &pgid) ||
store->collection_getattr(coll, "remove", &val, 1) == 1) {
cout << "finish_remove_pgs removing " << *it
<< " pgid is " << pgid << std::endl;
remove_coll(store, *it);
continue;
}
@ -536,17 +537,15 @@ int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
return 0;
}
int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid,
uint64_t *next_removal_seq)
int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid)
{
ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
if (store->collection_exists(coll_t(r_pgid))) {
coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++, r_pgid);
cout << "collection rename " << coll_t(r_pgid)
<< " to " << to_remove
<< std::endl;
rmt->collection_rename(coll_t(r_pgid), to_remove);
cout << " marking collection for removal" << std::endl;
bufferlist one;
one.append('1');
rmt->collection_setattr(coll_t(r_pgid), "remove", one);
} else {
delete rmt;
return ENOENT;
@ -1199,9 +1198,6 @@ int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl)
cout << std::endl;
#endif
coll_t newcoll(ms.info.pgid);
t->collection_rename(coll, newcoll);
int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver, ms.past_intervals);
if (ret) return ret;
@ -1335,8 +1331,7 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
pg_info_t info;
PGLog::IndexedLog log;
uint64_t next_removal_seq = 0; //My local seq
finish_remove_pgs(store, &next_removal_seq);
finish_remove_pgs(store);
int ret = sh.read_super();
if (ret)
@ -1385,11 +1380,11 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
return 1;
}
//Switch to collection which will be removed automatically if
//this program is interupted.
coll_t rmcoll = coll_t::make_removal_coll(next_removal_seq, pgid);
// mark this coll for removal until we are done
bufferlist one;
one.append('1');
ObjectStore::Transaction *t = new ObjectStore::Transaction;
t->create_collection(rmcoll);
t->collection_setattr(coll, "remove", one);
store->apply_transaction(*t);
delete t;
@ -1409,11 +1404,11 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
}
switch(type) {
case TYPE_OBJECT_BEGIN:
ret = get_object(store, rmcoll, ebl);
ret = get_object(store, coll, ebl);
if (ret) return ret;
break;
case TYPE_PG_METADATA:
ret = get_pg_metadata(store, rmcoll, ebl);
ret = get_pg_metadata(store, coll, ebl);
if (ret) return ret;
found_metadata = true;
break;
@ -1430,6 +1425,13 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
return EFAULT;
}
// done, clear removal flag
cout << "done, clearing remove flag" << std::endl;
t = new ObjectStore::Transaction;
t->collection_rmattr(coll, "remove");
store->apply_transaction(*t);
delete t;
return 0;
}
@ -2138,15 +2140,14 @@ int main(int argc, char **argv)
biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
if (op == "remove") {
uint64_t next_removal_seq = 0; //My local seq
finish_remove_pgs(fs, &next_removal_seq);
int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq);
finish_remove_pgs(fs);
int r = initiate_new_remove_pg(fs, pgid);
if (r) {
cerr << "PG '" << pgid << "' not found" << std::endl;
ret = 1;
goto out;
}
finish_remove_pgs(fs, &next_removal_seq);
finish_remove_pgs(fs);
cout << "Remove successful" << std::endl;
goto out;
}

View File

@ -1,110 +0,0 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#include "common/ceph_argparse.h"
#include "global/global_init.h"
#include "include/unordered_map.h"
#include "os/FileStore.h"
int dupstore(ObjectStore* src, ObjectStore* dst)
{
if (src->mount() < 0) return 1;
if (dst->mkfs() < 0) return 1;
if (dst->mount() < 0) return 1;
// objects
ceph::unordered_map<ghobject_t, coll_t> did_object;
// collections
vector<coll_t> collections;
int ret = src->list_collections(collections);
if (ret < 0) {
cerr << "Error " << ret << " while listing collections" << std::endl;
return 1;
}
int num = collections.size();
cout << num << " collections" << std::endl;
int i = 1;
for (vector<coll_t>::iterator p = collections.begin();
p != collections.end();
++p) {
cout << "collection " << i++ << "/" << num << " " << hex << *p << dec << std::endl;
{
ObjectStore::Transaction t;
t.create_collection(*p);
map<string,bufferptr> attrs;
src->collection_getattrs(*p, attrs);
t.collection_setattrs(*p, attrs);
dst->apply_transaction(t);
}
vector<ghobject_t> o;
src->collection_list(*p, o);
int numo = o.size();
int j = 1;
for (vector<ghobject_t>::iterator q = o.begin(); q != o.end(); ++q) {
ObjectStore::Transaction t;
if (did_object.count(*q))
t.collection_add(*p, did_object[*q], *q);
else {
bufferlist bl;
src->read(*p, *q, 0, 0, bl);
cout << "object " << j++ << "/" << numo << " " << *q << " = " << bl.length() << " bytes" << std::endl;
t.write(*p, *q, 0, bl.length(), bl);
map<string,bufferptr> attrs;
src->getattrs(*p, *q, attrs);
t.setattrs(*p, *q, attrs);
did_object[*q] = *p;
}
dst->apply_transaction(t);
}
}
src->umount();
dst->umount();
return 0;
}
void usage()
{
cerr << "usage: ceph_dupstore filestore SRC filestore DST" << std::endl;
exit(0);
}
int main(int argc, const char **argv)
{
vector<const char*> args;
argv_to_vec(argc, argv, args);
env_to_vec(args);
global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
common_init_finish(g_ceph_context);
// args
if (args.size() != 4)
usage();
ObjectStore *src = 0, *dst = 0;
if (strcmp(args[0], "filestore") == 0)
src = new FileStore(args[1], NULL);
else usage();
if (strcmp(args[2], "filestore") == 0)
dst = new FileStore(args[3], NULL);
else usage();
return dupstore(src, dst);
}

View File

@ -447,22 +447,6 @@ TRACEPOINT_EVENT(objectstore, omap_clear_exit,
)
)
TRACEPOINT_EVENT(objectstore, coll_rename_enter,
TP_ARGS(
const char *, osr_name),
TP_FIELDS(
ctf_string(osr_name, osr_name)
)
)
TRACEPOINT_EVENT(objectstore, coll_rename_exit,
TP_ARGS(
int, retval),
TP_FIELDS(
ctf_integer(int, retval, retval)
)
)
TRACEPOINT_EVENT(objectstore, startsync_enter,
TP_ARGS(
const char *, osr_name),