RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2024-12-19 01:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5331 from liewegas/wip-collection-list

Browse Source 
os: clean up collection_list

Reviewed-by: Samuel Just <sjust@redhat.com>
This commit is contained in:
Sage Weil 2015-08-07 10:15:10 -04:00
commit 7811c776c5
31 changed files with 139 additions and 1050 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/os/CollectionIndex.h
View File

@ -166,18 +166,12 @@ protected:
/// List contents of collection by hash
virtual int collection_list_partial(
const ghobject_t &start, ///< [in] object at which to start
int min_count, ///< [in] get at least min_count objects
const ghobject_t &end, ///< [in] list only objects < end
int max_count, ///< [in] return at most max_count objects
snapid_t seq, ///< [in] list only objects with snap >= seq
vector<ghobject_t> *ls, ///< [out] Listed objects
ghobject_t *next ///< [out] Next object to list
) = 0;
/// List contents of collection.
virtual int collection_list(
vector<ghobject_t> *ls ///< [out] Listed Objects
) = 0;
/// Call prior to removing directory
virtual int prep_delete() { return 0; }

114
src/os/FileStore.cc
View File

@ -4484,7 +4484,7 @@ int FileStore::_collection_remove_recursive(const coll_t &cid,
vector<ghobject_t> objects;
ghobject_t max;
while (!max.is_max()) {
r = collection_list_partial(cid, max, 200, 300, 0, &objects, &max);
r = collection_list(cid, max, ghobject_t::get_max(), 300, &objects, &max);
if (r < 0)
return r;
for (vector<ghobject_t>::iterator i = objects.begin();
@ -4631,7 +4631,7 @@ bool FileStore::collection_empty(coll_t c)
vector<ghobject_t> ls;
collection_list_handle_t handle;
r = index->collection_list_partial(ghobject_t(), 1, 1, 0, &ls, NULL);
r = index->collection_list_partial(ghobject_t(), ghobject_t::get_max(), 1, &ls, NULL);
if (r < 0) {
assert(!m_filestore_fail_eio || r != -EIO);
return false;
@ -4640,62 +4640,15 @@ bool FileStore::collection_empty(coll_t c)
tracepoint(objectstore, collection_empty_exit, ret);
return ret;
}
int FileStore::collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
snapid_t seq, vector<ghobject_t> *ls)
int FileStore::collection_list(coll_t c, ghobject_t start, ghobject_t end, int max,
vector<ghobject_t> *ls, ghobject_t *next)
{
tracepoint(objectstore, collection_list_range_enter, c.c_str());
bool done = false;
ghobject_t next = start;
if (!c.is_temp() && !c.is_meta() && next.hobj.pool < -1) {
coll_t temp = c.get_temp();
int r = collection_list_range(temp, start, end, seq, ls);
if (r < 0)
return r;
// ... always continue on to non-temp ...
}
while (!done) {
vector<ghobject_t> next_objects;
int r = collection_list_partial(c, next,
get_ideal_list_min(), get_ideal_list_max(),
seq, &next_objects, &next);
if (r < 0)
return r;
ls->insert(ls->end(), next_objects.begin(), next_objects.end());
// special case for empty collection
if (ls->empty()) {
break;
}
while (!ls->empty() && ls->back() >= end) {
ls->pop_back();
done = true;
}
if (next >= end) {
done = true;
}
}
tracepoint(objectstore, collection_list_range_exit, 0);
return 0;
}
int FileStore::collection_list_partial(coll_t c, ghobject_t start,
int min, int max, snapid_t seq,
vector<ghobject_t> *ls, ghobject_t *next)
{
tracepoint(objectstore, collection_list_partial_enter, c.c_str());
dout(10) << "collection_list_partial: " << c << " start " << start << dendl;
assert(next);
if (start.is_max())
return 0;
ghobject_t temp_next;
if (!next)
next = &temp_next;
// figure out the pool id. we need this in order to generate a
// meaningful 'next' value.
int64_t pool = -1;
@ -4720,7 +4673,6 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start,
dout(20) << __func__ << " pool is " << pool << " shard is " << shard
<< " pgid " << pgid << dendl;
}
ghobject_t sep;
sep.hobj.pool = -1;
sep.set_shard(shard);
@ -4728,7 +4680,7 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start,
if (start < sep) {
dout(10) << __func__ << " first checking temp pool" << dendl;
coll_t temp = c.get_temp();
int r = collection_list_partial(temp, start, min, max, seq, ls, next);
int r = collection_list(temp, start, end, max, ls, next);
if (r < 0)
return r;
if (*next != ghobject_t::get_max())
@ -4749,52 +4701,24 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start,
assert(NULL != index.index);
RWLock::RLocker l((index.index)->access_lock);
r = index->collection_list_partial(start,
min, max, seq,
ls, next);
r = index->collection_list_partial(start, end, max, ls, next);
if (r < 0) {
assert(!m_filestore_fail_eio || r != -EIO);
return r;
}
if (ls)
dout(20) << "objects: " << *ls << dendl;
dout(20) << "objects: " << ls << dendl;
// HashIndex doesn't know the pool when constructing a 'next' value
if (!next->is_max()) {
if (next && !next->is_max()) {
next->hobj.pool = pool;
next->set_shard(shard);
dout(20) << " next " << *next << dendl;
}
dout(20) << " next " << *next << dendl;
tracepoint(objectstore, collection_list_partial_exit, 0);
return 0;
}
int FileStore::collection_list(coll_t c, vector<ghobject_t>& ls)
{
tracepoint(objectstore, collection_list_enter, c.c_str());
if (!c.is_temp() && !c.is_meta()) {
coll_t temp = c.get_temp();
int r = collection_list(temp, ls);
if (r < 0)
return r;
}
Index index;
int r = get_index(c, &index);
if (r < 0)
return r;
assert(NULL != index.index);
RWLock::RLocker l((index.index)->access_lock);
r = index->collection_list(&ls);
assert(!m_filestore_fail_eio || r != -EIO);
tracepoint(objectstore, collection_list_exit, r);
return r;
}
int FileStore::omap_get(coll_t c, const ghobject_t &hoid,
bufferlist *header,
map<string, bufferlist> *out)
@ -5375,10 +5299,10 @@ int FileStore::_split_collection(coll_t cid,
vector<ghobject_t> objects;
ghobject_t next;
while (1) {
collection_list_partial(
collection_list(
cid,
next,
get_ideal_list_min(), get_ideal_list_max(), 0,
next, ghobject_t::get_max(),
get_ideal_list_max(),
&objects,
&next);
if (objects.empty())
@ -5394,10 +5318,10 @@ int FileStore::_split_collection(coll_t cid,
}
next = ghobject_t();
while (1) {
collection_list_partial(
collection_list(
dest,
next,
get_ideal_list_min(), get_ideal_list_max(), 0,
next, ghobject_t::get_max(),
get_ideal_list_max(),
&objects,
&next);
if (objects.empty())

8
src/os/FileStore.h
View File

@ -617,18 +617,14 @@ public:
const SequencerPosition &spos);
// collections
int collection_list(coll_t c, ghobject_t start, ghobject_t end, int max,
vector<ghobject_t> *ls, ghobject_t *next);
int list_collections(vector<coll_t>& ls);
int list_collections(vector<coll_t>& ls, bool include_temp);
int collection_version_current(coll_t c, uint32_t *version);
int collection_stat(coll_t c, struct stat *st);
bool collection_exists(coll_t c);
bool collection_empty(coll_t c);
int collection_list(coll_t c, vector<ghobject_t>& oid);
int collection_list_partial(coll_t c, ghobject_t start,
int min, int max, snapid_t snap,
vector<ghobject_t> *ls, ghobject_t *next);
int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
snapid_t seq, vector<ghobject_t> *ls);
// omap (see ObjectStore.h for documentation)
int omap_get(coll_t c, const ghobject_t &oid, bufferlist *header,

426
src/os/FlatIndex.cc
View File

@ -1,426 +0,0 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#if defined(__FreeBSD__)
#include <sys/cdefs.h>
#include <sys/param.h>
#endif
#include "FlatIndex.h"
#include "common/ceph_crypto.h"
#include "osd/osd_types.h"
#include <errno.h>
#include "chain_xattr.h"
using ceph::crypto::SHA1;
/*
* long file names will have the following format:
*
* prefix_hash_index_cookie
*
* The prefix will just be the first X bytes of the original file name.
* The cookie is a constant string that shows whether this file name
* is hashed
*/
#define FILENAME_LFN_DIGEST_SIZE CEPH_CRYPTO_SHA1_DIGESTSIZE
#define FILENAME_MAX_LEN 4096 // the long file name size
#define FILENAME_SHORT_LEN 255 // the short file name size
#define FILENAME_COOKIE "long" // ceph long file name
#define FILENAME_HASH_LEN FILENAME_LFN_DIGEST_SIZE
#define FILENAME_EXTRA 4 // underscores and digit
#define LFN_ATTR "user.cephos.lfn"
#define FILENAME_PREFIX_LEN (FILENAME_SHORT_LEN - FILENAME_HASH_LEN - (sizeof(FILENAME_COOKIE) - 1) - FILENAME_EXTRA)
int FlatIndex::cleanup() {
return 0;
}
static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
{
int i;
str[0] = '\0';
for (i = 0; i < len; i++) {
sprintf(&str[i*2], "%02x", (int)buf[i]);
}
}
static int hash_filename(const char *filename, char *hash, int buf_len)
{
if (buf_len < FILENAME_HASH_LEN + 1)
return -EINVAL;
char buf[FILENAME_LFN_DIGEST_SIZE];
char hex[FILENAME_LFN_DIGEST_SIZE * 2];
SHA1 h;
h.Update((const byte *)filename, strlen(filename));
h.Final((byte *)buf);
buf_to_hex((byte *)buf, (FILENAME_HASH_LEN + 1) / 2, hex);
strncpy(hash, hex, FILENAME_HASH_LEN);
hash[FILENAME_HASH_LEN] = '\0';
return 0;
}
static void build_filename(char *filename, int len, const char *old_filename, int i)
{
char hash[FILENAME_HASH_LEN + 1];
assert(len >= FILENAME_SHORT_LEN + 4);
strncpy(filename, old_filename, FILENAME_PREFIX_LEN);
filename[FILENAME_PREFIX_LEN] = '\0';
if (strlen(filename) < FILENAME_PREFIX_LEN)
return;
if (old_filename[FILENAME_PREFIX_LEN] == '\0')
return;
hash_filename(old_filename, hash, sizeof(hash));
int ofs = FILENAME_PREFIX_LEN;
while (1) {
int suffix_len = sprintf(filename + ofs, "_%s_%d_" FILENAME_COOKIE, hash, i);
if (ofs + suffix_len <= FILENAME_SHORT_LEN || !ofs)
break;
ofs--;
}
}
/* is this a candidate? */
static int lfn_is_hashed_filename(const char *filename)
{
int len = strlen(filename);
if (len < FILENAME_SHORT_LEN)
return 0;
return (strcmp(filename + len - (sizeof(FILENAME_COOKIE) - 1), FILENAME_COOKIE) == 0);
}
static void lfn_translate(const char *path, const char *name, char *new_name, int len)
{
if (!lfn_is_hashed_filename(name)) {
strncpy(new_name, name, len);
return;
}
char buf[PATH_MAX];
snprintf(buf, sizeof(buf), "%s/%s", path, name);
int r = chain_getxattr(buf, LFN_ATTR, new_name, len - 1);
if (r < 0)
strncpy(new_name, name, len);
else
new_name[r] = '\0';
return;
}
static int append_oname(const ghobject_t &oid, char *s, int len)
{
//assert(sizeof(oid) == 28);
char *end = s + len;
char *t = s + strlen(s);
const char *i = oid.hobj.oid.name.c_str();
while (*i && t < end) {
if (*i == '\\') {
*t++ = '\\';
*t++ = '\\';
} else if (*i == '.' && i == oid.hobj.oid.name.c_str()) { // only escape leading .
*t++ = '\\';
*t++ = '.';
} else if (*i == '/') {
*t++ = '\\';
*t++ = 's';
} else
*t++ = *i;
i++;
}
int size = t - s;
if (oid.hobj.snap == CEPH_NOSNAP)
size += snprintf(t, end - t, "_head");
else if (oid.hobj.snap == CEPH_SNAPDIR)
size += snprintf(t, end - t, "_snapdir");
else
size += snprintf(t, end - t, "_%llx", (long long unsigned)oid.hobj.snap);
return size;
}
static bool parse_object(char *s, ghobject_t& oid)
{
sobject_t o;
char *bar = s + strlen(s) - 1;
while (*bar != '_' &&
bar > s)
bar--;
if (*bar == '_') {
char buf[bar-s + 1];
char *t = buf;
char *i = s;
while (i < bar) {
if (*i == '\\') {
i++;
switch (*i) {
case '\\': *t++ = '\\'; break;
case '.': *t++ = '.'; break;
case 's': *t++ = '/'; break;
default: assert(0);
}
} else {
*t++ = *i;
}
i++;
}
*t = 0;
o.oid.name = string(buf, t-buf);
if (strcmp(bar+1, "head") == 0)
o.snap = CEPH_NOSNAP;
else if (strcmp(bar+1, "snapdir") == 0)
o.snap = CEPH_SNAPDIR;
else
o.snap = strtoull(bar+1, &s, 16);
oid = ghobject_t(hobject_t(o));
return true;
}
return false;
}
static int lfn_get(const char *coll_path, const ghobject_t& oid, char *pathname, int len, char *lfn, int lfn_len, int *exist, int *is_lfn)
{
int i = 0;
strncpy(pathname, coll_path, len);
size_t path_len = strlen(coll_path);
pathname[path_len] = '/';
path_len++;
pathname[path_len] = '\0';
char *filename = pathname + path_len;
*lfn = '\0';
int actual_len = append_oname(oid, lfn, lfn_len);
if (actual_len < (int)FILENAME_PREFIX_LEN) {
/* not a long file name, just build it as it is */
strncpy(filename, lfn, len - path_len);
*is_lfn = 0;
struct stat buf;
int r = ::stat(pathname, &buf);
if (r < 0) {
if (errno == ENOENT) {
*exist = 0;
} else {
return -errno;
}
} else {
*exist = 1;
}
return 0;
}
*is_lfn = 1;
*exist = 0;
while (1) {
char buf[PATH_MAX];
int r;
build_filename(filename, len - path_len, lfn, i);
r = chain_getxattr(pathname, LFN_ATTR, buf, sizeof(buf));
if (r < 0)
r = -errno;
if (r > 0) {
buf[MIN((int)sizeof(buf)-1, r)] = '\0';
if (strcmp(buf, lfn) == 0) { // a match?
*exist = 1;
return i;
}
}
switch (r) {
case -ENOENT:
return i;
case -ERANGE:
assert(0); // shouldn't happen
default:
break;
}
if (r < 0)
break;
i++;
}
return 0; // unreachable anyway
}
int FlatIndex::init() {
return 0;
}
int FlatIndex::created(const ghobject_t &hoid, const char *path) {
char long_name[PATH_MAX];
long_name[0] = '\0';
int actual_len = append_oname(hoid, long_name, sizeof(long_name));
if (actual_len < (int)FILENAME_PREFIX_LEN) {
return 0;
}
assert(long_name[actual_len] == '\0');
assert(long_name[actual_len - 1] != '\0');
int r = chain_setxattr(path, LFN_ATTR, long_name, actual_len);
if (r < 0)
return r;
return 0;
}
int FlatIndex::unlink(const ghobject_t &o) {
char long_fn[PATH_MAX];
char short_fn[PATH_MAX];
char short_fn2[PATH_MAX];
int r, i, exist, err;
int path_len;
int is_lfn;
r = lfn_get(base_path.c_str(), o, short_fn, sizeof(short_fn),
long_fn, sizeof(long_fn), &exist, &is_lfn);
if (r < 0)
return r;
if (!is_lfn) {
r = ::unlink(short_fn);
if (r < 0) {
return -errno;
}
return 0;
}
if (!exist)
return -ENOENT;
const char *next = strncpy(short_fn2, base_path.c_str(), sizeof(short_fn2));
path_len = next - short_fn2;
short_fn2[path_len] = '/';
path_len++;
short_fn2[path_len] = '\0';
for (i = r + 1; ; i++) {
struct stat buf;
int ret;
build_filename(&short_fn2[path_len], sizeof(short_fn2) - path_len, long_fn, i);
ret = ::stat(short_fn2, &buf);
if (ret < 0) {
if (i == r + 1) {
err = ::unlink(short_fn);
if (err < 0)
return err;
return 0;
}
break;
}
}
build_filename(&short_fn2[path_len], sizeof(short_fn2) - path_len, long_fn, i - 1);
if (rename(short_fn2, short_fn) < 0) {
assert(0);
}
return 0;
}
int FlatIndex::lookup(const ghobject_t &hoid, IndexedPath *path, int *exist) {
char long_fn[PATH_MAX];
char short_fn[PATH_MAX];
int r;
int is_lfn;
r = lfn_get(base_path.c_str(), hoid,
short_fn, sizeof(short_fn), long_fn,
sizeof(long_fn), exist, &is_lfn);
if (r < 0)
return r;
*path = IndexedPath(new Path(string(short_fn), this));
return 0;
}
static int get_hobject_from_oinfo(const char *dir, const char *file,
ghobject_t *o) {
char path[PATH_MAX];
bufferptr bp(PATH_MAX);
snprintf(path, sizeof(path), "%s/%s", dir, file);
// Hack, user.ceph._ is the attribute used to store the object info
int r = chain_getxattr(path, "user.ceph._", bp.c_str(), bp.length());
if (r < 0)
return r;
bufferlist bl;
bl.push_back(bp);
object_info_t oi(bl);
*o = ghobject_t(oi.soid);
return 0;
}
int FlatIndex::collection_list_partial(const ghobject_t &start,
int min_count,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next) {
assert(0); // Should not be called
return 0;
}
int FlatIndex::collection_list(vector<ghobject_t> *ls) {
char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1];
char dir_name[PATH_MAX], new_name[PATH_MAX];
strncpy(dir_name, base_path.c_str(), sizeof(dir_name));
dir_name[sizeof(dir_name)-1]='\0';
DIR *dir = ::opendir(dir_name);
if (!dir)
return -errno;
// first, build (ino, object) list
vector< pair<ino_t,ghobject_t> > inolist;
struct dirent *de;
while (::readdir_r(dir, (struct dirent *)buf, &de) == 0) {
if (!de)
break;
// parse
if (de->d_name[0] == '.')
continue;
//cout << " got object " << de->d_name << std::endl;
ghobject_t o;
lfn_translate(dir_name, de->d_name, new_name, sizeof(new_name));
if (parse_object(new_name, o)) {
get_hobject_from_oinfo(dir_name, de->d_name, &o);
inolist.push_back(pair<ino_t,ghobject_t>(de->d_ino, o));
ls->push_back(o);
}
}
// sort
sort(inolist.begin(), inolist.end());
// build final list
ls->resize(inolist.size());
int i = 0;
for (vector< pair<ino_t,ghobject_t> >::iterator p = inolist.begin(); p != inolist.end(); ++p)
(*ls)[i++].swap(p->second);
::closedir(dir);
return 0;
}

85
src/os/FlatIndex.h
View File

@ -1,85 +0,0 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#ifndef CEPH_FLATINDEX_H
#define CEPH_FLATINDEX_H
#include <string>
#include <map>
#include <set>
#include <vector>
#include "include/memory.h"
#include "CollectionIndex.h"
/**
* FlatIndex implements the collection layout prior to CollectionIndex
*
* This class should only be used for converting old filestores.
*/
class FlatIndex : public CollectionIndex {
string base_path;
coll_t collection;
public:
FlatIndex(coll_t collection, string base_path) :
CollectionIndex(collection),
base_path(base_path),
collection(collection) {}
/// @see CollectionIndex
uint32_t collection_version() { return FLAT_INDEX_TAG; }
coll_t coll() const { return collection; }
/// @see CollectionIndex
int cleanup();
/// @see CollectionIndex
int init();
/// @see CollectionIndex
int created(
const ghobject_t &oid,
const char *path
);
/// @see CollectionIndex
int unlink(
const ghobject_t &oid
);
/// @see CollectionIndex
int lookup(
const ghobject_t &oid,
IndexedPath *path,
int *exist
);
/// @see CollectionIndex
int collection_list(
vector<ghobject_t> *ls
);
/// @see CollectionIndex
int collection_list_partial(
const ghobject_t &start,
int min_count,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next
);
};
#endif

9
src/os/GenericObjectMap.cc
View File

@ -1049,7 +1049,7 @@ void GenericObjectMap::set_header(const coll_t &cid, const ghobject_t &oid,
t->set(GHOBJECT_TO_SEQ_PREFIX, to_set);
}
int GenericObjectMap::list_objects(const coll_t &cid, ghobject_t start, int max,
int GenericObjectMap::list_objects(const coll_t &cid, ghobject_t start, ghobject_t end, int max,
vector<ghobject_t> *out, ghobject_t *next)
{
// FIXME
@ -1101,7 +1101,14 @@ int GenericObjectMap::list_objects(const coll_t &cid, ghobject_t start, int max,
break;
}
if (header.oid >= end) {
if (next)
*next = ghobject_t::get_max();
break;
}
assert(start <= header.oid);
assert(header.oid < end);
size++;

2
src/os/GenericObjectMap.h
View File

@ -117,7 +117,7 @@ class GenericObjectMap {
bool check(std::ostream &out);
/// Util, list all objects, there must be no other concurrent access
int list_objects(const coll_t &cid, ghobject_t start, int max,
int list_objects(const coll_t &cid, ghobject_t start, ghobject_t end, int max,
vector<ghobject_t> *objs, ///< [out] objects
ghobject_t *next);

32
src/os/HashIndex.cc
View File

@ -320,15 +320,9 @@ int HashIndex::_lookup(const ghobject_t &oid,
return get_mangled_name(*path, oid, mangled_name, exists_out);
}
int HashIndex::_collection_list(vector<ghobject_t> *ls) {
vector<string> path;
return list_by_hash(path, 0, 0, 0, 0, ls);
}
int HashIndex::_collection_list_partial(const ghobject_t &start,
int min_count,
const ghobject_t &end,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next) {
vector<string> path;
@ -336,8 +330,8 @@ int HashIndex::_collection_list_partial(const ghobject_t &start,
if (!next)
next = &_next;
*next = start;
dout(20) << "_collection_list_partial " << start << " " << min_count << "-" << max_count << " ls.size " << ls->size() << dendl;
return list_by_hash(path, min_count, max_count, seq, next, ls);
dout(20) << __func__ << " start:" << start << " end:" << end << "-" << max_count << " ls.size " << ls->size() << dendl;
return list_by_hash(path, end, max_count, next, ls);
}
int HashIndex::prep_delete() {
@ -769,7 +763,6 @@ uint32_t HashIndex::hash_prefix_to_hash(string prefix) {
int HashIndex::get_path_contents_by_hash(const vector<string> &path,
const string *lower_bound,
const ghobject_t *next_object,
const snapid_t *seq,
set<string> *hash_prefixes,
set<pair<string, ghobject_t> > *objects) {
set<string> subdirs;
@ -792,8 +785,6 @@ int HashIndex::get_path_contents_by_hash(const vector<string> &path,
continue;
if (next_object && i->second < *next_object)
continue;
if (seq && i->second.hobj.snap < *seq)
continue;
hash_prefixes->insert(hash_prefix);
objects->insert(pair<string, ghobject_t>(hash_prefix, i->second));
}
@ -816,9 +807,8 @@ int HashIndex::get_path_contents_by_hash(const vector<string> &path,
}
int HashIndex::list_by_hash(const vector<string> &path,
int min_count,
ghobject_t end,
int max_count,
snapid_t seq,
ghobject_t *next,
vector<ghobject_t> *out) {
assert(out);
@ -829,7 +819,6 @@ int HashIndex::list_by_hash(const vector<string> &path,
int r = get_path_contents_by_hash(path,
NULL,
next,
&seq,
&hash_prefixes,
&objects);
if (r < 0)
@ -841,19 +830,13 @@ int HashIndex::list_by_hash(const vector<string> &path,
set<pair<string, ghobject_t> >::iterator j = objects.lower_bound(
make_pair(*i, ghobject_t()));
if (j == objects.end() || j->first != *i) {
if (min_count > 0 && out->size() > (unsigned)min_count) {
if (next)
*next = ghobject_t(hobject_t("", "", CEPH_NOSNAP, hash_prefix_to_hash(*i), -1, ""));
return 0;
}
*(next_path.rbegin()) = *(i->rbegin());
ghobject_t next_recurse;
if (next)
next_recurse = *next;
r = list_by_hash(next_path,
min_count,
end,
max_count,
seq,
&next_recurse,
out);
@ -871,6 +854,11 @@ int HashIndex::list_by_hash(const vector<string> &path,
*next = j->second;
return 0;
}
if (j->second >= end) {
if (next)
*next = ghobject_t::get_max();
return 0;
}
if (!next || j->second >= *next) {
out->push_back(j->second);
}

10
src/os/HashIndex.h
View File

@ -178,9 +178,6 @@ protected:
string *mangled_name,
int *exists
);
int _collection_list(
vector<ghobject_t> *ls
);
/**
* Pre-hash the collection to create folders according to the expected number
@ -193,9 +190,8 @@ protected:
int _collection_list_partial(
const ghobject_t &start,
int min_count,
const ghobject_t &end,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next
);
@ -358,7 +354,6 @@ private:
const vector<string> &path, /// [in] Path to list
const string *lower_bound, /// [in] list > *lower_bound
const ghobject_t *next_object, /// [in] list > *next_object
const snapid_t *seq, /// [in] list >= *seq
set<string> *hash_prefixes, /// [out] prefixes in dir
set<pair<string, ghobject_t> > *objects /// [out] objects
);
@ -366,9 +361,8 @@ private:
/// List objects in collection in ghobject_t order
int list_by_hash(
const vector<string> &path, /// [in] Path to list
int min_count, /// [in] List at least min_count
ghobject_t end, /// [in] List only objects < end
int max_count, /// [in] List at most max_count
snapid_t seq, /// [in] list only objects where snap >= seq
ghobject_t *next, /// [in,out] List objects >= *next
vector<ghobject_t> *out /// [out] Listed objects
); ///< @return Error Code, 0 on success

6
src/os/IndexManager.cc
View File

@ -28,7 +28,6 @@
#include "include/buffer.h"
#include "IndexManager.h"
#include "FlatIndex.h"
#include "HashIndex.h"
#include "CollectionIndex.h"
@ -95,10 +94,7 @@ int IndexManager::build_index(coll_t c, const char *path, CollectionIndex **inde
return r;
switch (version) {
case CollectionIndex::FLAT_INDEX_TAG: {
*index = new FlatIndex(c, path);
return 0;
}
case CollectionIndex::FLAT_INDEX_TAG:
case CollectionIndex::HASH_INDEX_TAG: // fall through
case CollectionIndex::HASH_INDEX_TAG_2: // fall through
case CollectionIndex::HOBJECT_WITH_POOL: {

1
src/os/IndexManager.h
View File

@ -24,7 +24,6 @@
#include "CollectionIndex.h"
#include "HashIndex.h"
#include "FlatIndex.h"
/// Public type for Index

72
src/os/KeyValueStore.cc
View File

@ -2329,7 +2329,7 @@ int KeyValueStore::_destroy_collection(coll_t c, BufferTransaction &t)
}
}
r = backend->list_objects(c, ghobject_t(), modified_object+1, &oids,
r = backend->list_objects(c, ghobject_t(), ghobject_t::get_max(), modified_object+1, &oids,
0);
// No other object
if (oids.size() != modified_object && oids.size() != 0) {
@ -2444,7 +2444,7 @@ int KeyValueStore::_collection_remove_recursive(const coll_t &cid,
vector<ghobject_t> objects;
ghobject_t max;
while (!max.is_max()) {
r = collection_list_partial(cid, max, 200, 300, 0, &objects, &max);
r = collection_list(cid, max, ghobject_t::get_max(), 300, &objects, &max);
if (r < 0)
goto out;
@ -2490,66 +2490,24 @@ bool KeyValueStore::collection_empty(coll_t c)
dout(10) << __func__ << " " << dendl;
vector<ghobject_t> oids;
backend->list_objects(c, ghobject_t(), 1, &oids, 0);
backend->list_objects(c, ghobject_t(), ghobject_t::get_max(), 1, &oids, 0);
return oids.empty();
}
int KeyValueStore::collection_list_range(coll_t c, ghobject_t start,
ghobject_t end, snapid_t seq,
vector<ghobject_t> *ls)
int KeyValueStore::collection_list(coll_t c, ghobject_t start,
ghobject_t end, int max,
vector<ghobject_t> *ls, ghobject_t *next)
{
bool done = false;
ghobject_t next = start;
while (!done) {
vector<ghobject_t> next_objects;
int r = collection_list_partial(c, next, get_ideal_list_min(),
get_ideal_list_max(), seq,
&next_objects, &next);
if (r < 0)
return r;
ls->insert(ls->end(), next_objects.begin(), next_objects.end());
// special case for empty collection
if (ls->empty()) {
break;
}
while (!ls->empty() && ls->back() >= end) {
ls->pop_back();
done = true;
}
if (next >= end) {
done = true;
}
}
return 0;
}
int KeyValueStore::collection_list_partial(coll_t c, ghobject_t start,
int min, int max, snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next)
{
dout(10) << __func__ << " " << c << " start:" << start << " is_max:"
<< start.is_max() << dendl;
if (min < 0 || max < 0)
if ( max < 0)
return -EINVAL;
if (start.is_max())
return 0;
return backend->list_objects(c, start, max, ls, next);
}
int r = backend->list_objects(c, start, end, max, ls, next);
int KeyValueStore::collection_list(coll_t c, vector<ghobject_t>& ls)
{
return collection_list_partial(c, ghobject_t(), 0, 0, 0, &ls, 0);
return r;
}
int KeyValueStore::collection_version_current(coll_t c, uint32_t *version)
@ -2831,8 +2789,8 @@ int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
ghobject_t next, current;
int move_size = 0;
while (1) {
collection_list_partial(cid, current, get_ideal_list_min(),
get_ideal_list_max(), 0, &objects, &next);
collection_list(cid, current, ghobject_t::get_max(),
get_ideal_list_max(), &objects, &next);
dout(20) << __func__ << cid << "objects size: " << objects.size()
<< dendl;
@ -2862,8 +2820,8 @@ int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
vector<ghobject_t> objects;
ghobject_t next;
while (1) {
collection_list_partial(cid, next, get_ideal_list_min(),
get_ideal_list_max(), 0, &objects, &next);
collection_list(cid, next, ghobject_t::get_max(),
get_ideal_list_max(), &objects, &next);
if (objects.empty())
break;
@ -2878,8 +2836,8 @@ int KeyValueStore::_split_collection(coll_t cid, uint32_t bits, uint32_t rem,
next = ghobject_t();
while (1) {
collection_list_partial(dest, next, get_ideal_list_min(),
get_ideal_list_max(), 0, &objects, &next);
collection_list(dest, next, ghobject_t::get_max(),
get_ideal_list_max(), &objects, &next);
if (objects.empty())
break;

9
src/os/KeyValueStore.h
View File

@ -618,12 +618,9 @@ class KeyValueStore : public ObjectStore,
int list_collections(vector<coll_t>& ls);
bool collection_exists(coll_t c);
bool collection_empty(coll_t c);
int collection_list(coll_t c, vector<ghobject_t>& oid);
int collection_list_partial(coll_t c, ghobject_t start,
int min, int max, snapid_t snap,
vector<ghobject_t> *ls, ghobject_t *next);
int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
snapid_t seq, vector<ghobject_t> *ls);
int collection_list(coll_t c, ghobject_t start, ghobject_t end,
int max,
vector<ghobject_t> *ls, ghobject_t *next);
int collection_version_current(coll_t c, uint32_t *version);
// omap (see ObjectStore.h for documentation)

10
src/os/LFNIndex.cc
View File

@ -143,11 +143,6 @@ int LFNIndex::lookup(const ghobject_t &oid,
);
}
int LFNIndex::collection_list(vector<ghobject_t> *ls)
{
return _collection_list(ls);
}
int LFNIndex::pre_hash_collection(uint32_t pg_num, uint64_t expected_num_objs)
{
return _pre_hash_collection(pg_num, expected_num_objs);
@ -155,13 +150,12 @@ int LFNIndex::pre_hash_collection(uint32_t pg_num, uint64_t expected_num_objs)
int LFNIndex::collection_list_partial(const ghobject_t &start,
int min_count,
const ghobject_t &end,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next)
{
return _collection_list_partial(start, min_count, max_count, seq, ls, next);
return _collection_list_partial(start, end, max_count, ls, next);
}
/* Derived class utility methods */

26
src/os/LFNIndex.h
View File

@ -178,11 +178,6 @@ public:
int *exist
);
/// @see CollectionIndex
int collection_list(
vector<ghobject_t> *ls
);
/// @see CollectionIndex;
int pre_hash_collection(
uint32_t pg_num,
@ -192,9 +187,8 @@ public:
/// @see CollectionIndex
int collection_list_partial(
const ghobject_t &start,
int min_count,
const ghobject_t &end,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next
);
@ -243,21 +237,6 @@ protected:
int *exists ///< [out] True if the object exists.
) = 0;
/**
* List contents of the collection, must be implemented by derived class.
*
* @param [out] seq Snapid to list.
* @param [in] max_count Max number to list (0 for no limit).
* @param [out] ls Container for listed objects.
* @param [in,out] last List handle. 0 for beginning. Passing the same
* cookie location will cause the next max_count to be listed.
* @return Error code. 0 on success.
*/
/// List contents of collection.
virtual int _collection_list(
vector<ghobject_t> *ls ///< [out] Listed objects.
) = 0;
/// Pre-hash the collection with the given pg number and
/// expected number of objects in the collection.
virtual int _pre_hash_collection(
@ -268,9 +247,8 @@ protected:
/// @see CollectionIndex
virtual int _collection_list_partial(
const ghobject_t &start,
int min_count,
const ghobject_t &end,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next
) = 0;

2
src/os/Makefile.am
View File

@ -11,7 +11,6 @@ libos_la_SOURCES = \
os/GenericObjectMap.cc \
os/FileJournal.cc \
os/FileStore.cc \
os/FlatIndex.cc \
os/GenericFileStoreBackend.cc \
os/HashIndex.cc \
os/IndexManager.cc \
@ -55,7 +54,6 @@ noinst_HEADERS += \
os/GenericObjectMap.h \
os/FileJournal.h \
os/FileStore.h \
os/FlatIndex.h \
os/FDCache.h \
os/GenericFileStoreBackend.h \
os/HashIndex.h \

54
src/os/MemStore.cc
View File

@ -421,27 +421,10 @@ bool MemStore::collection_empty(coll_t cid)
return c->object_map.empty();
}
int MemStore::collection_list(coll_t cid, vector<ghobject_t>& o)
int MemStore::collection_list(coll_t cid, ghobject_t start, ghobject_t end,
int max,
vector<ghobject_t> *ls, ghobject_t *next)
{
dout(10) << __func__ << " " << cid << dendl;
CollectionRef c = get_collection(cid);
if (!c)
return -ENOENT;
RWLock::RLocker l(c->lock);
for (map<ghobject_t,ObjectRef>::iterator p = c->object_map.begin();
p != c->object_map.end();
++p)
o.push_back(p->first);
return 0;
}
int MemStore::collection_list_partial(coll_t cid, ghobject_t start,
int min, int max, snapid_t snap,
vector<ghobject_t> *ls, ghobject_t *next)
{
dout(10) << __func__ << " " << cid << " " << start << " " << min << "-"
<< max << " " << snap << dendl;
CollectionRef c = get_collection(cid);
if (!c)
return -ENOENT;
@ -449,33 +432,16 @@ int MemStore::collection_list_partial(coll_t cid, ghobject_t start,
map<ghobject_t,ObjectRef>::iterator p = c->object_map.lower_bound(start);
while (p != c->object_map.end() &&
ls->size() < (unsigned)max) {
ls->size() < (unsigned)max &&
p->first < end) {
ls->push_back(p->first);
++p;
}
if (p == c->object_map.end())
*next = ghobject_t::get_max();
else
*next = p->first;
return 0;
}
int MemStore::collection_list_range(coll_t cid,
ghobject_t start, ghobject_t end,
snapid_t seq, vector<ghobject_t> *ls)
{
dout(10) << __func__ << " " << cid << " " << start << " " << end
<< " " << seq << dendl;
CollectionRef c = get_collection(cid);
if (!c)
return -ENOENT;
RWLock::RLocker l(c->lock);
map<ghobject_t,ObjectRef>::iterator p = c->object_map.lower_bound(start);
while (p != c->object_map.end() &&
p->first < end) {
ls->push_back(p->first);
++p;
if (next != NULL) {
if (p == c->object_map.end())
*next = ghobject_t::get_max();
else
*next = p->first;
}
return 0;
}

9
src/os/MemStore.h
View File

@ -306,12 +306,9 @@ public:
int list_collections(vector<coll_t>& ls);
bool collection_exists(coll_t c);
bool collection_empty(coll_t c);
int collection_list(coll_t cid, vector<ghobject_t>& o);
int collection_list_partial(coll_t cid, ghobject_t start,
int min, int max, snapid_t snap,
vector<ghobject_t> *ls, ghobject_t *next);
int collection_list_range(coll_t cid, ghobject_t start, ghobject_t end,
snapid_t seq, vector<ghobject_t> *ls);
int collection_list(coll_t cid, ghobject_t start, ghobject_t end,
int max,
vector<ghobject_t> *ls, ghobject_t *next);
int omap_get(
coll_t cid, ///< [in] Collection containing oid

42
src/os/ObjectStore.h
View File

@ -1809,14 +1809,7 @@ public:
std::string *value);
/**
* get ideal min value for collection_list_partial()
*
* default to some arbitrary values; the implementation will override.
*/
virtual int get_ideal_list_min() { return 32; }
/**
* get ideal max value for collection_list_partial()
* get ideal max value for collection_list()
*
* default to some arbitrary values; the implementation will override.
*/
@ -2029,42 +2022,19 @@ public:
virtual bool collection_empty(coll_t c) = 0;
/**
* collection_list - get all objects of a collection in sorted order
*
* @param c collection name
* @param o [out] list of objects
* @returns 0 on success, negative error code on failure
*/
virtual int collection_list(coll_t c, vector<ghobject_t>& o) = 0;
/**
* list partial contents of collection relative to a hash offset/position
*
* @param c collection
* @param start list objects that sort >= this value
* @param min return at least this many results, unless we reach the end
* @param max return no more than this many results
* @param snap return no objects with snap < snapid
* @param ls [out] result
* @param next [out] next item sorts >= this value
* @return zero on success, or negative error
*/
virtual int collection_list_partial(coll_t c, ghobject_t start,
int min, int max, snapid_t snap,
vector<ghobject_t> *ls, ghobject_t *next) = 0;
/**
* list contents of a collection that fall in the range [start, end)
* list contents of a collection that fall in the range [start, end) and no more than a specified many result
*
* @param c collection
* @param start list object that sort >= this value
* @param end list objects that sort < this value
* @param max return no more than this many results
* @param seq return no objects with snap < seq
* @param ls [out] result
* @param next [out] next item sorts >= this value
* @return zero on success, or negative error
*/
virtual int collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
snapid_t seq, vector<ghobject_t> *ls) = 0;
virtual int collection_list(coll_t c, ghobject_t start, ghobject_t end, int max,
vector<ghobject_t> *ls, ghobject_t *next) = 0;
/// OMAP
/// Get omap contents

14
src/osd/OSD.cc
View File

@ -2506,10 +2506,9 @@ void OSD::clear_temp_objects()
ghobject_t next;
while (1) {
vector<ghobject_t> objects;
store->collection_list_partial(*p, next,
store->get_ideal_list_min(),
store->get_ideal_list_max(),
0, &objects, &next);
store->collection_list(*p, next, ghobject_t::get_max(),
store->get_ideal_list_max(),
&objects, &next);
if (objects.empty())
break;
vector<ghobject_t>::iterator q;
@ -2547,7 +2546,7 @@ void OSD::recursive_remove_collection(ObjectStore *store, spg_t pgid, coll_t tmp
SnapMapper mapper(&driver, 0, 0, 0, pgid.shard);
vector<ghobject_t> objects;
store->collection_list(tmp, objects);
store->collection_list(tmp, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0);
// delete them.
unsigned removed = 0;
@ -4227,12 +4226,11 @@ bool remove_dir(
ObjectStore::Transaction *t = new ObjectStore::Transaction;
ghobject_t next;
handle.reset_tp_timeout();
store->collection_list_partial(
store->collection_list(
coll,
next,
store->get_ideal_list_min(),
ghobject_t::get_max(),
store->get_ideal_list_max(),
0,
&olist,
&next);
for (vector<ghobject_t>::iterator i = olist.begin();

1
src/osd/PG.cc
View File

@ -3893,7 +3893,6 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle)
start,
cct->_conf->osd_scrub_chunk_min,
cct->_conf->osd_scrub_chunk_max,
0,
&objects,
&candidate_end);
assert(ret >= 0);

13
src/osd/PGBackend.cc
View File

@ -100,7 +100,6 @@ int PGBackend::objects_list_partial(
const hobject_t &begin,
int min,
int max,
snapid_t seq,
vector<hobject_t> *ls,
hobject_t *next)
{
@ -113,12 +112,11 @@ int PGBackend::objects_list_partial(
int r = 0;
while (!_next.is_max() && ls->size() < (unsigned)min) {
vector<ghobject_t> objects;
int r = store->collection_list_partial(
int r = store->collection_list(
coll,
_next,
min - ls->size(),
ghobject_t::get_max(),
max - ls->size(),
seq,
&objects,
&_next);
if (r != 0)
@ -148,12 +146,13 @@ int PGBackend::objects_list_range(
{
assert(ls);
vector<ghobject_t> objects;
int r = store->collection_list_range(
int r = store->collection_list(
coll,
ghobject_t(start, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
ghobject_t(end, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
seq,
&objects);
INT_MAX,
&objects,
NULL);
ls->reserve(objects.size());
for (vector<ghobject_t>::iterator i = objects.begin();
i != objects.end();

1
src/osd/PGBackend.h
View File

@ -511,7 +511,6 @@
const hobject_t &begin,
int min,
int max,
snapid_t seq,
vector<hobject_t> *ls,
hobject_t *next);

5
src/osd/ReplicatedPG.cc
View File

@ -834,7 +834,6 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
current,
list_size,
list_size,
snapid,
&sentries,
&next);
if (r != 0) {
@ -992,7 +991,6 @@ void ReplicatedPG::do_pg_op(OpRequestRef op)
current,
list_size,
list_size,
snapid,
&sentries,
&next);
if (r != 0) {
@ -10087,7 +10085,7 @@ void ReplicatedPG::scan_range(
vector<hobject_t> ls;
ls.reserve(max);
int r = pgbackend->objects_list_partial(bi->begin, min, max, 0, &ls, &bi->end);
int r = pgbackend->objects_list_partial(bi->begin, min, max, &ls, &bi->end);
assert(r >= 0);
dout(10) << " got " << ls.size() << " items, next " << bi->end << dendl;
dout(20) << ls << dendl;
@ -10622,7 +10620,6 @@ bool ReplicatedPG::agent_work(int start_max, int agent_flush_quota)
vector<hobject_t> ls;
hobject_t next;
int r = pgbackend->objects_list_partial(agent_state->position, ls_min, ls_max,
0 /* no filtering by snapid */,
&ls, &next);
assert(r >= 0);
dout(20) << __func__ << " got " << ls.size() << " objects" << dendl;

5
src/test/Makefile-server.am
View File

@ -221,11 +221,6 @@ unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_TESTPROGRAMS += unittest_chain_xattr
unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc
unittest_flatindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_flatindex_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_TESTPROGRAMS += unittest_flatindex
unittest_lfnindex_SOURCES = test/os/TestLFNIndex.cc
unittest_lfnindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_lfnindex_CXXFLAGS = $(UNITTEST_CXXFLAGS)

4
src/test/objectstore/FileStoreDiff.cc
View File

@ -132,13 +132,13 @@ bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t
int err;
std::vector<ghobject_t> b_objects, a_objects;
err = b_store->collection_list(coll, b_objects);
err = b_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(), INT_MAX, &b_objects, NULL);
if (err < 0) {
dout(0) << "diff_objects list on verify coll " << coll.to_str()
<< " returns " << err << dendl;
return true;
}
err = a_store->collection_list(coll, a_objects);
err = a_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(), INT_MAX, &a_objects, NULL);
if (err < 0) {
dout(0) << "diff_objects list on store coll " << coll.to_str()
<< " returns " << err << dendl;

54
src/test/objectstore/store_test.cc
View File

@ -282,8 +282,8 @@ TEST_P(StoreTest, SimpleListTest) {
vector<ghobject_t> objects;
ghobject_t next, current;
while (!next.is_max()) {
int r = store->collection_list_partial(cid, current, 25, 50,
0, &objects, &next);
int r = store->collection_list(cid, current, ghobject_t::get_max(), 50,
&objects, &next);
ASSERT_EQ(r, 0);
cout << " got " << objects.size() << " next " << next << std::endl;
for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end();
@ -366,8 +366,8 @@ TEST_P(StoreTest, MultipoolListTest) {
vector<ghobject_t> objects;
ghobject_t next, current;
while (!next.is_max()) {
int r = store->collection_list_partial(cid, current, 25, 50,
0, &objects, &next);
int r = store->collection_list(cid, current, ghobject_t::get_max(), 50,
&objects, &next);
ASSERT_EQ(r, 0);
cout << " got " << objects.size() << " next " << next << std::endl;
for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end();
@ -585,7 +585,7 @@ TEST_P(StoreTest, ManyObjectTest) {
set<ghobject_t> listed;
vector<ghobject_t> objects;
r = store->collection_list(cid, objects);
r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0);
ASSERT_EQ(r, 0);
cerr << "objects.size() is " << objects.size() << std::endl;
@ -599,12 +599,11 @@ TEST_P(StoreTest, ManyObjectTest) {
ghobject_t start, next;
objects.clear();
r = store->collection_list_partial(
r = store->collection_list(
cid,
ghobject_t::get_max(),
50,
ghobject_t::get_max(),
60,
0,
&objects,
&next
);
@ -614,12 +613,11 @@ TEST_P(StoreTest, ManyObjectTest) {
objects.clear();
listed.clear();
while (1) {
r = store->collection_list_partial(cid, start,
50,
60,
0,
&objects,
&next);
r = store->collection_list(cid, start,
ghobject_t::get_max(),
60,
&objects,
&next);
ASSERT_TRUE(sorted(objects));
ASSERT_EQ(r, 0);
listed.insert(objects.begin(), objects.end());
@ -1101,8 +1099,8 @@ public:
ghobject_t next, current;
while (1) {
cerr << "scanning..." << std::endl;
int r = store->collection_list_partial(cid, current, 50, 100,
0, &objects, &next);
int r = store->collection_list(cid, current, ghobject_t::get_max(), 100,
&objects, &next);
ASSERT_EQ(r, 0);
ASSERT_TRUE(sorted(objects));
objects_set.insert(objects.begin(), objects.end());
@ -1117,7 +1115,7 @@ public:
ASSERT_GT(available_objects.count(*i), (unsigned)0);
}
int r = store->collection_list(cid, objects);
int r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0);
ASSERT_EQ(r, 0);
objects_set2.insert(objects.begin(), objects.end());
ASSERT_EQ(objects_set2.size(), available_objects.size());
@ -1313,7 +1311,7 @@ TEST_P(StoreTest, HashCollisionTest) {
}
}
vector<ghobject_t> objects;
r = store->collection_list(cid, objects);
r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0);
ASSERT_EQ(r, 0);
set<ghobject_t> listed(objects.begin(), objects.end());
cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
@ -1322,8 +1320,8 @@ TEST_P(StoreTest, HashCollisionTest) {
listed.clear();
ghobject_t current, next;
while (1) {
r = store->collection_list_partial(cid, current, 50, 60,
0, &objects, &next);
r = store->collection_list(cid, current, ghobject_t::get_max(), 60,
&objects, &next);
ASSERT_EQ(r, 0);
ASSERT_TRUE(sorted(objects));
for (vector<ghobject_t>::iterator i = objects.begin();
@ -1408,7 +1406,8 @@ TEST_P(StoreTest, ScrubTest) {
}
vector<ghobject_t> objects;
r = store->collection_list(cid, objects);
r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX,
&objects, 0);
ASSERT_EQ(r, 0);
set<ghobject_t> listed(objects.begin(), objects.end());
cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
@ -1417,8 +1416,8 @@ TEST_P(StoreTest, ScrubTest) {
listed.clear();
ghobject_t current, next;
while (1) {
r = store->collection_list_partial(cid, current, 50, 60,
0, &objects, &next);
r = store->collection_list(cid, current, ghobject_t::get_max(), 60,
&objects, &next);
ASSERT_EQ(r, 0);
ASSERT_TRUE(sorted(objects));
for (vector<ghobject_t>::iterator i = objects.begin();
@ -1706,7 +1705,8 @@ void colsplittest(
ObjectStore::Transaction t;
vector<ghobject_t> objects;
r = store->collection_list(cid, objects);
r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX,
&objects, 0);
ASSERT_EQ(r, 0);
ASSERT_EQ(objects.size(), num_objects);
for (vector<ghobject_t>::iterator i = objects.begin();
@ -1717,7 +1717,8 @@ void colsplittest(
}
objects.clear();
r = store->collection_list(tid, objects);
r = store->collection_list(tid, ghobject_t(), ghobject_t::get_max(), INT_MAX,
&objects, 0);
ASSERT_EQ(r, 0);
ASSERT_EQ(objects.size(), num_objects);
for (vector<ghobject_t>::iterator i = objects.begin();
@ -1925,7 +1926,8 @@ TEST_P(StoreTest, BigRGWObjectName) {
{
vector<ghobject_t> objects;
r = store->collection_list(cid, objects);
r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX,
&objects, 0);
ASSERT_EQ(r, 0);
ASSERT_EQ(objects.size(), 1u);
ASSERT_EQ(objects[0], oid2);

2
src/test/objectstore/workload_generator.cc
View File

@ -353,7 +353,7 @@ void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t,
m_nr_runs.set(0);
entry->m_osr.flush();
vector<ghobject_t> ls;
m_store->collection_list(entry->m_coll, ls);
m_store->collection_list(entry->m_coll, ghobject_t(), ghobject_t::get_max(), INT_MAX, &ls, NULL);
dout(2) << __func__ << " coll " << entry->m_coll
<< " (" << ls.size() << " objects)" << dendl;

139
src/test/os/TestFlatIndex.cc
View File

@ -1,139 +0,0 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
*
* Author: Loic Dachary <loic@dachary.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Library Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Library Public License for more details.
*
*/
#include <stdio.h>
#include <signal.h>
#include "os/FlatIndex.h"
#include "os/CollectionIndex.h"
#include "os/chain_xattr.h"
#include "common/ceph_argparse.h"
#include "global/global_init.h"
#include <gtest/gtest.h>
TEST(FlatIndex, FlatIndex) {
coll_t collection;
const std::string base_path("PATH");
FlatIndex index(collection, base_path);
EXPECT_EQ(collection, index.coll());
EXPECT_EQ((unsigned)0, index.collection_version());
//
// checking placeholders
//
EXPECT_EQ(0, index.init());
EXPECT_EQ(0, index.cleanup());
}
TEST(FlatIndex, collection) {
coll_t collection;
const std::string base_path("PATH");
FlatIndex index(collection, base_path);
const std::string key("KEY");
uint64_t hash = 111;
uint64_t pool = 222;
const std::string object_name(10, 'A');
ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, ""));
vector<ghobject_t> ls;
ASSERT_THROW(index.collection_list_partial(hoid, 0, 0, 0, &ls, &hoid), FailedAssertion);
}
TEST(FlatIndex, created_unlink) {
coll_t collection;
const std::string base_path("PATH");
EXPECT_EQ(0, ::system("rm -fr PATH"));
EXPECT_EQ(0, ::mkdir("PATH", 0700));
ceph::shared_ptr<CollectionIndex> index(new FlatIndex(collection, base_path));
const std::string key("KEY");
uint64_t hash = 111;
uint64_t pool = 222;
//
// short object name
//
{
CollectionIndex::IndexedPath indexed_path;
const std::string object_name(10, 'A');
ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, ""));
int exists;
EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
EXPECT_EQ(0, exists);
EXPECT_EQ(0, ::close(::creat(indexed_path->path(), 0600)));
EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
EXPECT_EQ(1, exists);
EXPECT_EQ(0, index->unlink(hoid));
EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
EXPECT_EQ(0, exists);
}
//
// long object name
//
{
CollectionIndex::IndexedPath indexed_path;
const std::string object_name(1024, 'A');
ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, ""));
int exists;
EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
EXPECT_EQ(0, exists);
EXPECT_EQ(0, ::close(::creat(indexed_path->path(), 0600)));
EXPECT_EQ(0, index->created(hoid, indexed_path->path()));
EXPECT_EQ(0, index->unlink(hoid));
EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists));
EXPECT_EQ(0, exists);
}
EXPECT_EQ(0, ::system("rm -fr PATH"));
}
TEST(FlatIndex, collection_list) {
coll_t collection;
const std::string base_path("PATH");
EXPECT_EQ(0, ::system("rm -fr PATH"));
EXPECT_EQ(0, ::mkdir("PATH", 0700));
const std::string object_name("ABC");
const std::string filename("PATH/" + object_name + "_head");
EXPECT_EQ(0, ::close(::creat(filename.c_str(), 0600)));
ceph::shared_ptr<CollectionIndex> index(new FlatIndex(collection, base_path));
vector<ghobject_t> ls;
index->collection_list(&ls);
EXPECT_EQ((unsigned)1, ls.size());
EXPECT_EQ(object_name, ls[0].hobj.oid.name);
EXPECT_EQ(0, ::system("rm -fr PATH"));
}
int main(int argc, char **argv) {
int fd = ::creat("detect", 0600);
int ret = chain_fsetxattr(fd, "user.test", "A", 1);
::close(fd);
::unlink("detect");
if (ret < 0) {
cerr << "SKIP FlatIndex because unable to test for xattr" << std::endl;
} else {
vector<const char*> args;
argv_to_vec(argc, (const char **)argv, args);
global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
common_init_finish(g_ceph_context);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
}
// Local Variables:
// compile-command: "cd ../.. ; make unittest_flatindex ; ./unittest_flatindex # --gtest_filter=FlatIndexTest.FlatIndex --log-to-stderr=true --debug-filestore=20"
// End:

7
src/test/os/TestLFNIndex.cc
View File

@ -75,15 +75,10 @@ protected:
int *exists
) { return 0; }
virtual int _collection_list(
vector<ghobject_t> *ls
) { return 0; }
virtual int _collection_list_partial(
const ghobject_t &start,
int min_count,
const ghobject_t &end,
int max_count,
snapid_t seq,
vector<ghobject_t> *ls,
ghobject_t *next
) { return 0; }

19
src/tools/ceph_objectstore_tool.cc
View File

@ -79,14 +79,13 @@ int _action_on_all_objects_in_pg(ObjectStore *store, coll_t coll, action_on_obje
ghobject_t next;
while (!next.is_max()) {
vector<ghobject_t> list;
int r = store->collection_list_partial(
coll,
next,
LIST_AT_A_TIME,
LIST_AT_A_TIME,
0,
&list,
&next);
int r = store->collection_list(
coll,
next,
ghobject_t::get_max(),
LIST_AT_A_TIME,
&list,
&next);
if (r < 0) {
cerr << "Error listing collection: " << coll << ", "
<< cpp_strerror(r) << std::endl;
@ -412,7 +411,7 @@ void remove_coll(ObjectStore *store, const coll_t &coll)
cout << "remove_coll " << coll << std::endl;
while (!next.is_max()) {
vector<ghobject_t> objects;
r = store->collection_list_partial(coll, next, 200, 300, 0,
r = store->collection_list(coll, next, ghobject_t::get_max(), 300,
&objects, &next);
if (r < 0)
goto out;
@ -700,7 +699,7 @@ int ObjectStoreTool::export_files(ObjectStore *store, coll_t coll)
while (!next.is_max()) {
vector<ghobject_t> objects;
int r = store->collection_list_partial(coll, next, 200, 300, 0,
int r = store->collection_list(coll, next, ghobject_t::get_max(), 300,
&objects, &next);
if (r < 0)
return r;