From 0b0f23cbed104a7797dec8309dcb4389eec33225 Mon Sep 17 00:00:00 2001 From: sage Date: Fri, 16 Dec 2005 22:29:18 +0000 Subject: [PATCH] *** empty log message *** git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@530 29311d96-e01e-0410-9327-a35deaab8ce9 --- ceph/config.cc | 6 +++++- ceph/config.h | 5 +++++ ceph/ebofs/AlignedBufferPool.h | 10 +++++++--- ceph/ebofs/BlockDevice.cc | 10 ++++++++++ ceph/ebofs/Ebofs.cc | 21 ++++++++++++++++++--- ceph/ebofs/Ebofs.h | 8 +++----- ceph/ebofs/mkfs.ebofs.cc | 15 ++++----------- ceph/include/buffer.h | 2 +- ceph/osd/OSD.cc | 15 +-------------- ceph/osd/OSD.h | 3 --- 10 files changed, 54 insertions(+), 41 deletions(-) diff --git a/ceph/config.cc b/ceph/config.cc index 35f8ecd0b9c..6ae304e458c 100644 --- a/ceph/config.cc +++ b/ceph/config.cc @@ -116,10 +116,14 @@ md_config_t g_conf = { osd_fakestore_syncthreads: 4, osd_ebofs: 0, + // --- ebofs --- ebofs_commit_interval: 2, // seconds. 0 = no timeout (for debugging/tracing) ebofs_bc_size: (50 *256), // measured in 4k blocks, or *256 for MB - ebofs_bc_max_dirty: (40 *256), // before write() will wait for data to flush + ebofs_bc_max_dirty: (10 *256), // before write() will wait for data to flush + // --- block device --- + bdev_max_el_ms: 1000, // restart elevator at least once every 1000 ms + // --- fakeclient (mds regression testing) (ancient history) --- num_fakeclient: 100, diff --git a/ceph/config.h b/ceph/config.h index af8afbf70bc..8d9300c34ac 100644 --- a/ceph/config.h +++ b/ceph/config.h @@ -91,10 +91,15 @@ struct md_config_t { int osd_fakestore_syncthreads; // such crap int osd_ebofs; + // ebofs int ebofs_commit_interval; off_t ebofs_bc_size; off_t ebofs_bc_max_dirty; + // block device + int bdev_max_el_ms; + + // fake client int num_fakeclient; unsigned fakeclient_requests; diff --git a/ceph/ebofs/AlignedBufferPool.h b/ceph/ebofs/AlignedBufferPool.h index 4af885247f4..8832e90a6e2 100644 --- a/ceph/ebofs/AlignedBufferPool.h +++ b/ceph/ebofs/AlignedBufferPool.h @@ -25,13 +25,16 @@ class AlignedBufferPool { bool dommap; + off_t talloc; + public: - AlignedBufferPool(int a) : alignment(a), dommap(true) {} + AlignedBufferPool(int a) : alignment(a), dommap(true), talloc(0) {} ~AlignedBufferPool() { } void free(char *p, unsigned len) { - dout(1) << "bufferpool.free " << (void*)p << " len " << len << endl; + dout(20) << "bufferpool(" << (void*)this << ").free " << (void*)p << " len " << len << " ... total " << talloc << endl; + talloc -= len; if (dommap) munmap(p, len); else @@ -52,9 +55,10 @@ class AlignedBufferPool { ::posix_memalign((void**)&p, alignment, bytes); assert(p); + talloc += bytes; ::memset(p, 0, bytes); // only to shut up valgrind - dout(1) << "bufferpool.alloc " << (void*)p << endl; + dout(20) << "bufferpool(" << (void*)this << ").alloc " << (void*)p << " len " << bytes << " ... total " << talloc << endl; return new buffer(p, bytes, BUFFER_MODE_NOCOPY|BUFFER_MODE_NOFREE|BUFFER_MODE_CUSTOMFREE, bytes, diff --git a/ceph/ebofs/BlockDevice.cc b/ceph/ebofs/BlockDevice.cc index b956be09d2e..f5d658f038f 100644 --- a/ceph/ebofs/BlockDevice.cc +++ b/ceph/ebofs/BlockDevice.cc @@ -49,6 +49,10 @@ int BlockDevice::io_thread_entry() // queue? if (!io_queue.empty()) { + utime_t stop = g_clock.now(); + utime_t max(0, 1000*g_conf.bdev_max_el_ms); // (s,us), convert ms -> us! + stop += max; + if (dir_forward) { // forward sweep dout(20) << "io_thread forward sweep" << endl; @@ -81,6 +85,9 @@ int BlockDevice::io_thread_entry() lock.Unlock(); do_io(biols); lock.Lock(); + + utime_t now = g_clock.now(); + if (now > stop) break; } } else { // reverse sweep @@ -115,6 +122,9 @@ int BlockDevice::io_thread_entry() lock.Unlock(); do_io(biols); lock.Lock(); + + utime_t now = g_clock.now(); + if (now > stop) break; } } dir_forward = !dir_forward; diff --git a/ceph/ebofs/Ebofs.cc b/ceph/ebofs/Ebofs.cc index 85c1a0d3f37..a2fe680a3f1 100644 --- a/ceph/ebofs/Ebofs.cc +++ b/ceph/ebofs/Ebofs.cc @@ -19,6 +19,12 @@ int Ebofs::mount() ebofs_lock.Lock(); assert(!mounted); + int r = dev.open(); + if (r < 0) { + ebofs_lock.Unlock(); + return r; + } + // read super bufferptr bp1 = bufferpool.alloc(EBOFS_BLOCK_SIZE); bufferptr bp2 = bufferpool.alloc(EBOFS_BLOCK_SIZE); @@ -79,6 +85,12 @@ int Ebofs::mkfs() ebofs_lock.Lock(); assert(!mounted); + int r = dev.open(); + if (r < 0) { + ebofs_lock.Unlock(); + return r; + } + block_t num_blocks = dev.get_num_blocks(); free_blocks = 0; @@ -145,8 +157,9 @@ int Ebofs::mkfs() dout(3) << "mkfs: cleaning up" << endl; close_tables(); - dout(1) << "mkfs: done" << endl; + dev.close(); + dout(1) << "mkfs: done" << endl; ebofs_lock.Unlock(); return 0; } @@ -190,6 +203,7 @@ int Ebofs::umount() // free memory dout(2) << "umount cleaning up" << endl; close_tables(); + dev.close(); dout(1) << "umount done" << endl; ebofs_lock.Unlock(); @@ -275,7 +289,8 @@ int Ebofs::commit_thread_entry() // wait for kick, or timeout if (g_conf.ebofs_commit_interval) { - commit_cond.WaitInterval(ebofs_lock, utime_t(EBOFS_COMMIT_INTERVAL,0)); + dout(10) << "commit_thread sleeping (up to) " << g_conf.ebofs_commit_interval << " seconds" << endl; + commit_cond.WaitInterval(ebofs_lock, utime_t(g_conf.ebofs_commit_interval,0)); } else { // DEBUG.. wait until kicked dout(10) << "commit_thread no commit_interval, waiting until kicked" << endl; @@ -865,7 +880,7 @@ void Ebofs::trim_bc() } } - dout(10) << "trim_bc finish: size " << bc.get_size() << ", trimmable " << bc.get_trimmable() << ", max " << max << endl; + dout(1) << "trim_bc finish: size " << bc.get_size() << ", trimmable " << bc.get_trimmable() << ", max " << max << endl; /* dout(10) << "trim_buffer_cache finish: " diff --git a/ceph/ebofs/Ebofs.h b/ceph/ebofs/Ebofs.h index 1cdc1398b33..4029407289a 100644 --- a/ceph/ebofs/Ebofs.h +++ b/ceph/ebofs/Ebofs.h @@ -26,15 +26,13 @@ inline ostream& operator<<(ostream& out, idpair_t oc) { } -const int EBOFS_COMMIT_INTERVAL = 2; // 0 == never - class Ebofs : public ObjectStore { protected: Mutex ebofs_lock; // a beautiful global lock // ** super ** - BlockDevice &dev; + BlockDevice dev; bool mounted, unmounting; bool readonly; version_t super_epoch; @@ -173,8 +171,8 @@ class Ebofs : public ObjectStore { bool _write_will_block(); public: - Ebofs(BlockDevice& d) : - dev(d), + Ebofs(char *devfn) : + dev(devfn), mounted(false), unmounting(false), readonly(false), super_epoch(0), commit_thread_started(false), mid_commit(false), commit_thread(this), diff --git a/ceph/ebofs/mkfs.ebofs.cc b/ceph/ebofs/mkfs.ebofs.cc index ec130947a17..90e385caa2d 100644 --- a/ceph/ebofs/mkfs.ebofs.cc +++ b/ceph/ebofs/mkfs.ebofs.cc @@ -16,20 +16,14 @@ int main(int argc, char **argv) } char *filename = args[0]; - // device - BlockDevice dev(filename); - if (dev.open() < 0) { - cerr << "couldn't open " << filename << endl; - return -1; - } - // mkfs - Ebofs mfs(dev); - mfs.mkfs(); + Ebofs mfs(filename); + int r = mfs.mkfs(); + if (r < 0) exit(r); if (1) { // test-o-rama! - Ebofs fs(dev); + Ebofs fs(filename); fs.mount(); if (1) { // big writes @@ -169,7 +163,6 @@ int main(int argc, char **argv) fs.umount(); } - dev.close(); } diff --git a/ceph/include/buffer.h b/ceph/include/buffer.h index 26015465e8a..c1ffd605d34 100644 --- a/ceph/include/buffer.h +++ b/ceph/include/buffer.h @@ -100,7 +100,7 @@ class buffer { bdbout(1) << "buffer.malloc " << (void*)_dataptr << endl; } ~buffer() { - bdbout(1) << "buffer.des " << *this << endl; + bdbout(1) << "buffer.des " << *this << " " << (void*)free_func << endl; if (free_func) { bdbout(1) << "buffer.custom_free_func " << free_func_arg << " " << (void*)_dataptr << endl; free_func( free_func_arg, _dataptr, _alloc_len ); diff --git a/ceph/osd/OSD.cc b/ceph/osd/OSD.cc index 56d28e5624e..c10ef9b8ec7 100644 --- a/ceph/osd/OSD.cc +++ b/ceph/osd/OSD.cc @@ -12,7 +12,6 @@ #ifdef USE_EBOFS # include "ebofs/Ebofs.h" -# include "ebofs/BlockDevice.h" #endif @@ -84,14 +83,12 @@ OSD::OSD(int id, Messenger *m) store = new OBFSStore(whoami, NULL, "/dev/sdb3"); #else # ifdef USE_EBOFS - storedev = 0; if (g_conf.osd_ebofs) { char hostname[100]; hostname[0] = 0; gethostname(hostname,100); sprintf(ebofs_path, "%s/%s", ebofs_base_path, hostname); - storedev = new BlockDevice(ebofs_path); - store = new Ebofs(*storedev); + store = new Ebofs(ebofs_path); } else # endif store = new FakeStore(osd_base_path, whoami); @@ -142,9 +139,6 @@ OSD::~OSD() if (messenger) { delete messenger; messenger = 0; } if (logger) { delete logger; logger = 0; } if (store) { delete store; store = 0; } -#ifdef USE_EBOFS - if (storedev) { delete storedev; storedev = 0; } -#endif } @@ -152,10 +146,6 @@ int OSD::init() { osd_lock.Lock(); -#ifdef USE_EBOFS - if (storedev) - storedev->open(); -#endif if (g_conf.osd_mkfs) store->mkfs(); int r = store->mount(); @@ -182,9 +172,6 @@ int OSD::shutdown() messenger->shutdown(); int r = store->umount(); -#ifdef USE_EBOFS - if (storedev) storedev->close(); -#endif return r; } diff --git a/ceph/osd/OSD.h b/ceph/osd/OSD.h index 4696ea22d53..b210c73262b 100644 --- a/ceph/osd/OSD.h +++ b/ceph/osd/OSD.h @@ -50,9 +50,6 @@ class OSD : public Dispatcher { int whoami; class ObjectStore *store; -#ifdef USE_EBOFS - class BlockDevice *storedev; // for ebofs -#endif class HostMonitor *monitor; class Logger *logger;