RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-20 01:17:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7513 from athanatos/wip-ceph-test-rados-2

Browse Source 
ceph_test_rados: use less CPU
This commit is contained in:
Sage Weil 2016-02-08 09:23:56 -05:00
commit 3f66231edf
3 changed files with 257 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

185
src/test/osd/Object.cc
View File

@ -82,68 +82,44 @@ void VarLenGenerator::get_ranges_map(
}
}
ObjectDesc::iterator &ObjectDesc::iterator::advance(bool init) {
assert(pos < limit);
assert(!end());
if (!init) {
pos++;
}
if (end()) {
return *this;
}
while (pos == limit) {
cur_cont = stack.begin()->first;
limit = stack.begin()->second;
void ObjectDesc::iterator::adjust_stack() {
while (!stack.empty() && pos >= stack.front().second.next) {
assert(pos == stack.front().second.next);
size = stack.front().second.size;
current = stack.front().first;
stack.pop_front();
}
if (cur_cont == obj.layers.end()) {
return *this;
if (stack.empty()) {
cur_valid_till = std::numeric_limits<uint64_t>::max();
} else {
cur_valid_till = stack.front().second.next;
}
interval_set<uint64_t> ranges;
cur_cont->first->get_ranges(cur_cont->second, ranges);
while (!ranges.contains(pos)) {
stack.push_front(std::pair<std::list<std::pair<ceph::shared_ptr<ContentsGenerator>,
ContDesc> >::iterator,
uint64_t>(cur_cont, limit));
uint64_t length = cur_cont->first->get_length(cur_cont->second);
uint64_t next;
if (pos >= length) {
next = limit;
cur_cont = obj.layers.end();
} else if (ranges.empty() || pos >= ranges.range_end()) {
next = length;
++cur_cont;
} else {
next = ranges.start_after(pos);
++cur_cont;
}
if (next < limit) {
limit = next;
}
if (cur_cont == obj.layers.end()) {
break;
while (current != layers.end() && !current->covers(pos)) {
uint64_t next = current->next(pos);
if (next < cur_valid_till) {
stack.push_front(
make_pair(
current,
StackState{next, size}
)
);
cur_valid_till = next;
}
ranges.clear();
cur_cont->first->get_ranges(cur_cont->second, ranges);
++current;
}
if (cur_cont == obj.layers.end()) {
return *this;
if (current == layers.end()) {
size = 0;
} else {
current->iter.seek(pos);
size = std::min(size, current->get_size());
cur_valid_till = std::min(
current->valid_till(pos),
cur_valid_till);
}
if (!cont_iters.count(cur_cont->second)) {
cont_iters.insert(std::pair<ContDesc,ContentsGenerator::iterator>(
cur_cont->second,
cur_cont->first->get_iterator(cur_cont->second)));
}
std::map<ContDesc,ContentsGenerator::iterator>::iterator j = cont_iters.find(
cur_cont->second);
assert(j != cont_iters.end());
j->second.seek(pos);
return *this;
}
const ContDesc &ObjectDesc::most_recent() {
@ -156,68 +132,75 @@ void ObjectDesc::update(ContentsGenerator *gen, const ContDesc &next) {
}
bool ObjectDesc::check(bufferlist &to_check) {
iterator i = begin();
uint64_t pos = 0;
for (bufferlist::iterator p = to_check.begin();
!p.end();
++p, ++i, ++pos) {
if (i.end()) {
std::cout << "reached end of iterator first" << std::endl;
return false;
}
if (*i != *p) {
std::cout << "incorrect buffer at pos " << pos << std::endl;
return false;
}
iterator objiter = begin();
uint64_t error_at = 0;
if (!objiter.check_bl_advance(to_check, &error_at)) {
std::cout << "incorrect buffer at pos " << error_at << std::endl;
return false;
}
uint64_t size = layers.empty() ? 0 :
most_recent_gen()->get_length(most_recent());
if (pos != size) {
std::cout << "only read " << pos << " out of size " << size << std::endl;
uint64_t size = layers.begin()->first->get_length(layers.begin()->second);
if (to_check.length() < size) {
std::cout << "only read " << to_check.length()
<< " out of size " << size << std::endl;
return false;
}
return true;
}
bool ObjectDesc::check_sparse(const std::map<uint64_t, uint64_t>& extents,
bufferlist &to_check) {
auto i = begin();
auto p = to_check.begin();
bufferlist &to_check)
{
uint64_t off = 0;
uint64_t pos = 0;
for (auto extent : extents) {
const uint64_t start = extent.first;
const uint64_t end = start + extent.second;
for (; pos < end; ++i, ++pos) {
if (i.end()) {
std::cout << "reached end of iterator first" << std::endl;
auto objiter = begin();
for (auto &&extiter : extents) {
// verify hole
{
bufferlist bl;
bl.append_zero(extiter.first - pos);
uint64_t error_at = 0;
if (!objiter.check_bl_advance(bl, &error_at)) {
std::cout << "sparse read omitted non-zero data at "
<< error_at << std::endl;
return false;
}
if (pos < start) {
// check the hole
if (*i != '\0') {
std::cout << "incorrect buffer at pos " << pos << std::endl;
return false;
}
} else {
// then the extent
if (*i != *p) {
std::cout << "incorrect buffer at pos " << pos << std::endl;
return false;
}
++p;
}
}
}
uint64_t size = layers.empty() ? 0 :
most_recent_gen()->get_length(most_recent());
while (pos < size) {
if (*i != '\0') {
std::cout << "sparse read omitted non-zero data at " << pos << std::endl;
assert(off <= to_check.length());
pos = extiter.first;
objiter.seek(pos);
{
bufferlist bl;
bl.substr_of(
to_check,
off,
std::min(to_check.length() - off, extiter.second));
uint64_t error_at = 0;
if (!objiter.check_bl_advance(bl, &error_at)) {
std::cout << "incorrect buffer at pos " << error_at << std::endl;
return false;
}
off += extiter.second;
pos += extiter.second;
}
if (pos < extiter.first + extiter.second) {
std::cout << "reached end of iterator first" << std::endl;
return false;
}
++i;
++pos;
}
assert(pos == size);
// final hole
bufferlist bl;
uint64_t size = layers.begin()->first->get_length(layers.begin()->second);
bl.append_zero(size - pos);
uint64_t error_at;
if (!objiter.check_bl_advance(bl, &error_at)) {
std::cout << "sparse read omitted non-zero data at "
<< error_at << std::endl;
return false;
}
return true;
}

208
src/test/osd/Object.h
View File

@ -5,6 +5,7 @@
#include <list>
#include <map>
#include <set>
#include <random>
#ifndef OBJECT_H
#define OBJECT_H
@ -61,6 +62,28 @@ public:
virtual bool end() = 0;
virtual ContDesc get_cont() const = 0;
virtual uint64_t get_pos() const = 0;
virtual bufferlist gen_bl_advance(uint64_t s) {
bufferptr ret = buffer::create(s);
for (uint64_t i = 0; i < s; ++i, ++(*this)) {
ret[i] = **this;
}
bufferlist _ret;
_ret.push_back(ret);
return _ret;
}
virtual bool check_bl_advance(bufferlist &bl, uint64_t *off = nullptr) {
uint64_t _off = 0;
for (bufferlist::iterator i = bl.begin();
!i.end();
++i, ++_off, ++(*this)) {
if (*i != **this) {
if (off)
*off = _off;
return false;
}
}
return true;
}
virtual ~iterator_impl() {};
};
@ -90,6 +113,12 @@ public:
other.impl = impl;
impl = otherimpl;
}
bufferlist gen_bl_advance(uint64_t s) {
return impl->gen_bl_advance(s);
}
bool check_bl_advance(bufferlist &bl, uint64_t *off = nullptr) {
return impl->check_bl_advance(bl, off);
}
iterator(ContentsGenerator *parent, iterator_impl *impl) :
parent(parent), impl(impl) {}
};
@ -124,19 +153,7 @@ public:
class RandGenerator : public ContentsGenerator {
public:
class RandWrap {
public:
unsigned int state;
explicit RandWrap(unsigned int seed)
{
state = seed;
}
int operator()()
{
return rand_r(&state);
}
};
typedef std::minstd_rand0 RandWrap;
class iterator_impl : public ContentsGenerator::iterator_impl {
public:
@ -296,41 +313,91 @@ public:
class iterator {
public:
uint64_t pos;
ObjectDesc &obj;
std::list<std::pair<std::list<std::pair<ceph::shared_ptr<ContentsGenerator>,
ContDesc> >::iterator,
uint64_t> > stack;
std::map<ContDesc,ContentsGenerator::iterator> cont_iters;
uint64_t limit;
std::list<std::pair<ceph::shared_ptr<ContentsGenerator>,
ContDesc> >::iterator cur_cont;
uint64_t size;
uint64_t cur_valid_till;
class ContState {
interval_set<uint64_t> ranges;
const uint64_t size;
public:
ContDesc cont;
ceph::shared_ptr<ContentsGenerator> gen;
ContentsGenerator::iterator iter;
ContState(
ContDesc _cont,
ceph::shared_ptr<ContentsGenerator> _gen,
ContentsGenerator::iterator _iter)
: size(_gen->get_length(_cont)), cont(_cont), gen(_gen), iter(_iter) {
gen->get_ranges(cont, ranges);
}
const interval_set<uint64_t> &get_ranges() {
return ranges;
}
uint64_t get_size() {
return gen->get_length(cont);
}
bool covers(uint64_t pos) {
return ranges.contains(pos) || (!ranges.starts_after(pos) && pos >= size);
}
uint64_t next(uint64_t pos) {
assert(!covers(pos));
return ranges.starts_after(pos) ? ranges.start_after(pos) : size;
}
uint64_t valid_till(uint64_t pos) {
assert(covers(pos));
return ranges.contains(pos) ?
ranges.end_after(pos) :
std::numeric_limits<uint64_t>::max();
}
};
std::list<ContState> layers;
struct StackState {
const uint64_t next;
const uint64_t size;
};
std::list<std::pair<std::list<ContState>::iterator, StackState> > stack;
std::list<ContState>::iterator current;
explicit iterator(ObjectDesc &obj) :
pos(0), obj(obj) {
limit = obj.layers.begin()->first->get_length(obj.layers.begin()->second);
cur_cont = obj.layers.begin();
advance(true);
pos(0),
size(obj.layers.begin()->first->get_length(obj.layers.begin()->second)),
cur_valid_till(0) {
for (auto &&i : obj.layers) {
layers.push_back({i.second, i.first, i.first->get_iterator(i.second)});
}
current = layers.begin();
adjust_stack();
}
iterator &advance(bool init);
void adjust_stack();
iterator &operator++() {
return advance(false);
assert(cur_valid_till >= pos);
++pos;
if (pos >= cur_valid_till) {
adjust_stack();
}
return *this;
}
char operator*() {
if (cur_cont == obj.layers.end()) {
if (current == layers.end()) {
return '\0';
} else {
std::map<ContDesc,ContentsGenerator::iterator>::iterator j = cont_iters.find(
cur_cont->second);
assert(j != cont_iters.end());
return *(j->second);
return pos >= size ? '\0' : *(current->iter);
}
}
bool end() {
return pos >= obj.layers.begin()->first->get_length(
obj.layers.begin()->second);
return pos >= size;
}
void seek(uint64_t _pos) {
@ -338,8 +405,77 @@ public:
assert(0);
}
while (pos < _pos) {
++(*this);
assert(cur_valid_till >= pos);
uint64_t next = std::min(_pos - pos, cur_valid_till - pos);
pos += next;
if (pos >= cur_valid_till) {
assert(pos == cur_valid_till);
adjust_stack();
}
}
assert(pos == _pos);
}
bufferlist gen_bl_advance(uint64_t s) {
bufferlist ret;
while (s > 0) {
assert(cur_valid_till >= pos);
uint64_t next = std::min(s, cur_valid_till - pos);
if (current != layers.end() && pos < size) {
ret.append(current->iter.gen_bl_advance(next));
} else {
ret.append_zero(next);
}
pos += next;
assert(next <= s);
s -= next;
if (pos >= cur_valid_till) {
assert(cur_valid_till == pos);
adjust_stack();
}
}
return ret;
}
bool check_bl_advance(bufferlist &bl, uint64_t *error_at = nullptr) {
uint64_t off = 0;
while (off < bl.length()) {
assert(cur_valid_till >= pos);
uint64_t next = std::min(bl.length() - off, cur_valid_till - pos);
bufferlist to_check;
to_check.substr_of(bl, off, next);
if (current != layers.end() && pos < size) {
if (!current->iter.check_bl_advance(to_check, error_at)) {
if (error_at)
*error_at += off;
return false;
}
} else {
uint64_t at = pos;
for (auto i = to_check.begin(); !i.end(); ++i, ++at) {
if (*i) {
if (error_at)
*error_at = at;
return false;
}
}
}
pos += next;
off += next;
assert(off <= bl.length());
if (pos >= cur_valid_till) {
assert(cur_valid_till == pos);
adjust_stack();
}
}
assert(off == bl.length());
return true;
}
};

5
src/test/osd/RadosModel.h
View File

@ -791,11 +791,8 @@ public:
for (map<uint64_t, uint64_t>::iterator i = ranges.begin();
i != ranges.end();
++i, ++tid) {
bufferlist to_write;
gen_pos.seek(i->first);
for (uint64_t k = 0; k != i->second; ++k, ++gen_pos) {
to_write.append(*gen_pos);
}
bufferlist to_write = gen_pos.gen_bl_advance(i->second);
assert(to_write.length() == i->second);
assert(to_write.length() > 0);
std::cout << num << ": writing " << context->prefix+oid