*** empty log message ***

git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@283 29311d96-e01e-0410-9327-a35deaab8ce9

ceph/Makefile

@@ -74,9 +74,12 @@ mttest: test/mttest.cc msg/MTMessenger.cc ${COMMON_OBJS}
 mpifuse: mpifuse.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/MPIMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS}
 	${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@
 
-mpisyn: mpisyn.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/MPIMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS}
+mpisyn: mpisyn.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o msg/MPIMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS}
 	${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@
 
+fakesyn: fakesyn.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o msg/FakeMessenger.o msg/CheesySerializer.o ${COMMON_OBJS}
+	${CC} ${CFLAGS} ${LIBS} $^ -o $@
+
 fakefuse: fakefuse.cc mds/allmds.o client/Client.o osd/OSD.o client/fuse.o msg/FakeMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS}
 	${CC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@
 

ceph/TODO

@@ -7,24 +7,33 @@ big fast todo's:
 - mds benchmarks
  - synthetic client stable,
  - pseudo-mega-filesystem
-   - osd copy-on-write?
+   - osd copy-on-write..
+
+- osd failure structures
+- heartbeatmonitor vs pingmonitor
+
+
 
 ask tyce+bill:
 - obfs on alc?
 
 
+md tests:
+- log length versus cache size, workload
+
+
 notes and todos.
 - SyntheticClient
+
 - virtual mega-filesystem (metadata only)
 
 
-HARD LINKS
+finish HARD LINKS
 - reclaim danglers from inode file on discover...
 - fix rename
 
 
 
-
 MDS TODO
 - fix hard links
   - they mostly work, but they're fragile

ceph/client/SyntheticClient.cc

@@ -20,7 +20,19 @@
 void *synthetic_client_thread_entry(void *ptr)
 {
   SyntheticClient *sc = (SyntheticClient*)ptr;
-  sc->run();
+  switch (sc->mode) {
+  case SYNCLIENT_MODE_RANDOMWALK:
+	sc->random_walk(sc->iarg1);
+	break;
+  case SYNCLIENT_MODE_MAKEDIRS:
+	sc->make_dirs(sc->sarg1.c_str(), sc->iarg1, sc->iarg2, sc->iarg3);
+	break;
+  case SYNCLIENT_MODE_FULLWALK:
+	sc->full_walk(sc->sarg1);
+	break;
+  default:
+	assert(0);
+  }
   return 0;
 }
 
@@ -84,11 +96,68 @@ void SyntheticClient::up()
 }
 
 
-int SyntheticClient::run()
+
+
+int SyntheticClient::full_walk(string& basedir) 
+{
+  // read dir
+  map<string, inode_t*> contents;
+  int r = client->getdir(basedir.c_str(), contents);
+  if (r < 0) {
+	dout(1) << "readdir on " << basedir << " returns " << r << endl;
+	return r;
+  }
+
+  for (map<string, inode_t*>::iterator it = contents.begin();
+	   it != contents.end();
+	   it++) {
+	string file = basedir + "/" + it->first;
+
+	struct stat st;
+	int r = client->lstat(file.c_str(), &st);
+	if (r < 0) {
+	  dout(1) << "stat error on " << file << " r=" << r << endl;
+	  continue;
+	}
+
+	if (st.st_mode & INODE_MODE_DIR) full_walk(file);
+  }
+
+  return 0;
+}
+
+int SyntheticClient::make_dirs(const char *basedir, int dirs, int files, int depth)
+{
+  // make sure base dir exists
+  int r = client->mkdir(basedir, 0755);
+  if (r != 0) {
+	dout(1) << "can't make base dir? " << basedir << endl;
+	return -1;
+  }
+
+  if (depth == 0) return 0;
+
+  // children
+  char d[500];
+  dout(5-depth) << "make_dirs " << basedir << " dirs " << dirs << " files " << files << " depth " << depth << endl;
+  for (int i=0; i<files; i++) {
+	sprintf(d,"%s/file.%d", basedir, i);
+	client->mknod(d, 0644);
+  }
+
+  for (int i=0; i<dirs; i++) {
+	sprintf(d, "%s/dir.%d", basedir, i);
+	make_dirs(d, dirs, files, depth-1);
+  }
+  
+  return 0;
+}
+
+int SyntheticClient::random_walk(int num_req)
 {
   int left = num_req;
 
-  dout(1) << "run() will do " << left << " ops" << endl;
+  dout(1) << "random_walk() will do " << left << " ops" << endl;
 
   init_op_dist();  // set up metadata op distribution
  

ceph/client/SyntheticClient.h

@@ -6,9 +6,12 @@
 #include "Client.h"
 #include "include/Distribution.h"
 
+#define SYNCLIENT_MODE_RANDOMWALK  1
+#define SYNCLIENT_MODE_FULLWALK    2
+#define SYNCLIENT_MODE_MAKEDIRS    3
+
 class SyntheticClient {
   Client *client;
-  int num_req;
 
   pthread_t thread_id;
 
@@ -66,17 +69,21 @@ class SyntheticClient {
   }
 
  public:
-  SyntheticClient(Client *client,
-				  int num_req) {
+  SyntheticClient(Client *client) {
 	this->client = client;
-	this->num_req = num_req;
 	thread_id = 0;
   }
 
   int start_thread();
   int join_thread();
 
-  int run();
+  string sarg1;
+  int iarg1, iarg2, iarg3;
+  int mode;
+  
+  int full_walk(string& fromdir);
+  int random_walk(int n);
+  int make_dirs(const char *basedir, int dirs, int files, int depth);
 };
 
 #endif

ceph/config.cc

@@ -25,7 +25,7 @@ md_config_t g_conf = {
   fake_clock: false,
   fakemessenger_serialize: true,
 
-  debug: 15,
+  debug: 5,
   
   // --- client ---
   client_cache_size: 400,

ceph/fakesyn.cc

@@ -0,0 +1,171 @@
+
+
+#include <sys/stat.h>
+#include <iostream>
+#include <string>
+using namespace std;
+
+#include "include/config.h"
+
+#include "mds/MDCluster.h"
+#include "mds/MDS.h"
+#include "osd/OSD.h"
+#include "client/Client.h"
+#include "client/SyntheticClient.h"
+
+#include "msg/FakeMessenger.h"
+#include "msg/CheesySerializer.h"
+
+#include "common/Timer.h"
+
+#define NUMMDS g_conf.num_mds
+#define NUMOSD g_conf.num_osd
+#define NUMCLIENT g_conf.num_client
+
+class C_Test : public Context {
+public:
+  void finish(int r) {
+	cout << "C_Test->finish(" << r << ")" << endl;
+  }
+};
+
+
+int main(int argc, char **argv) {
+
+  //cerr << "mpisyn starting " << myrank << "/" << world << endl;
+
+  int start = 0;
+
+  // build new argc+argv for fuse
+  typedef char* pchar;
+  int nargc = 0;
+  char **nargv = new pchar[argc];
+  nargv[nargc++] = argv[0];
+
+  int synthetic = 100;
+  int mkfs = 0;
+  for (int i=1; i<argc; i++) {
+	if (strcmp(argv[i], "--fastmkfs") == 0) {
+	  mkfs = MDS_MKFS_FAST;
+	}
+	else if (strcmp(argv[i], "--fullmkfs") == 0) {
+	  mkfs = MDS_MKFS_FULL;
+	}
+
+	else if (strcmp(argv[i],"--numosd") == 0) {
+	  g_conf.num_osd = atoi(argv[++i]);
+	}
+	else if (strcmp(argv[i],"--numclient") == 0) {
+	  g_conf.num_client = atoi(argv[++i]);
+	}
+	else if (strcmp(argv[i],"--nummds") == 0) {
+	  g_conf.num_mds = atoi(argv[++i]);
+	}
+	else if (strcmp(argv[i],"--synthetic") == 0) {
+	  synthetic = atoi(argv[++i]);
+	}
+	else {
+	  // unknown arg, pass it on.
+	  nargv[nargc++] = argv[i];
+	}
+  }
+
+  fakemessenger_startthread();
+
+  MDCluster *mdc = new MDCluster(NUMMDS, NUMOSD);
+
+
+  char hostname[100];
+  gethostname(hostname,100);
+  int pid = getpid();
+
+  // create mds
+  MDS *mds[NUMMDS];
+  for (int i=0; i<NUMMDS; i++) {
+	//cerr << "mds" << i << " on rank " << myrank << " " << hostname << "." << pid << endl;
+	mds[i] = new MDS(mdc, i, new FakeMessenger(MSG_ADDR_MDS(i)));
+	start++;
+  }
+  
+  // create osd
+  OSD *osd[NUMOSD];
+  for (int i=0; i<NUMOSD; i++) {
+	//cerr << "osd" << i << " on rank " << myrank << " " << hostname << "." << pid << endl;
+	osd[i] = new OSD(i, new FakeMessenger(MSG_ADDR_OSD(i)));
+	start++;
+  }
+  
+  // create client
+  Client *client[NUMCLIENT];
+  SyntheticClient *syn[NUMCLIENT];
+  for (int i=0; i<NUMCLIENT; i++) {
+	//cerr << "client" << i << " on rank " << myrank << " " << hostname << "." << pid << endl;
+	CheesySerializer *serializer = new CheesySerializer( new FakeMessenger(MSG_ADDR_CLIENT(i)) );
+	client[i] = new Client(mdc, i, serializer);
+	start++;
+  }
+
+
+  // start message loop
+  fakemessenger_startthread();
+  
+  // init
+  for (int i=0; i<NUMMDS; i++) {
+	mds[i]->init();
+  }
+  
+  for (int i=0; i<NUMOSD; i++) {
+	osd[i]->init();
+  }
+  
+  // create client
+  for (int i=0; i<NUMCLIENT; i++) {
+	client[i]->init();
+	
+	// use my argc, argv (make sure you pass a mount point!)
+	cout << "mounting" << endl;
+	client[i]->mount(mkfs);
+	
+	cout << "starting synthatic client  " << endl;
+	syn[i] = new SyntheticClient(client[i]);
+
+	syn[i]->mode = SYNCLIENT_MODE_MAKEDIRS;
+	char s[20];
+	sprintf(s,"syn.%d", i);
+	syn[i]->sarg1 = s;
+	syn[i]->iarg1 = 5;
+	syn[i]->iarg2 = 5;
+	syn[i]->iarg3 = 2;
+	  
+	syn[i]->start_thread();
+  }
+  for (int i=0; i<NUMCLIENT; i++) {
+	
+	cout << "waiting for synthetic client to finish" << endl;
+	syn[i]->join_thread();
+	delete syn[i];
+	
+	client[i]->unmount();
+	cout << "unmounted" << endl;
+	client[i]->shutdown();
+  }
+  
+		
+  // wait for it to finish
+  fakemessenger_wait();
+  
+  // cleanup
+  for (int i=0; i<NUMMDS; i++) {
+	delete mds[i];
+  }
+  for (int i=0; i<NUMOSD; i++) {
+	delete osd[i];
+  }
+  for (int i=0; i<NUMCLIENT; i++) {
+	delete client[i];
+  }
+  delete mdc;
+  
+  return 0;
+}
+

ceph/msg/FakeMessenger.cc

@@ -74,8 +74,13 @@ void fakemessenger_stopthread() {
   shutdown = true;
   lock.Unlock();
   cond.Signal();
+  
+  fakemessenger_wait();
+}
 
-  cout << "fakemessenger_stopthread waiting" << endl;
+void fakemessenger_wait()
+{
+  cout << "fakemessenger_wait waiting" << endl;
   void *ptr;
   pthread_join(thread_id, &ptr);
 }
@@ -121,7 +126,7 @@ int fakemessenger_do_loop_2()
 	  Message *m = it->second->get_message();
 	  if (m) {
 		//dout(18) << "got " << m << endl;
-		dout(5) << "---- do_loop dispatching '" << m->get_type_name() << 
+		dout(5) << "---- '" << m->get_type_name() << 
 		  "' from " << MSG_ADDR_NICE(m->get_source()) << ':' << m->get_source_port() <<
 		  " to " << MSG_ADDR_NICE(m->get_dest()) << ':' << m->get_dest_port() << " ---- " << m 
 			 << endl;
@@ -198,6 +203,8 @@ int FakeMessenger::shutdown()
 
   //cout << "shutdown on messenger " << this << " has " << num_incoming() << " queued" << endl;
   directory.erase(whoami);
+  if (directory.empty()) 
+	::shutdown = true;
 }
 
 void FakeMessenger::trigger_timer(Timer *t) 

ceph/msg/FakeMessenger.h

@@ -58,7 +58,6 @@ int fakemessenger_do_loop();
 int fakemessenger_do_loop_2();
 void fakemessenger_startthread();
 void fakemessenger_stopthread();
-
-
+void fakemessenger_wait();
 
 #endif

ceph/osd/FakeStore.cc

@@ -25,10 +25,15 @@
 
 
 
-FakeStore::FakeStore(char *base, int whoami) 
+FakeStore::FakeStore(char *base, int whoami, char *shadow) 
 {
   this->basedir = base;
   this->whoami = whoami;
+
+  if (shadow) {
+	is_shadow = true;
+	shadowdir = shadow;
+  }
 }
 
 
@@ -38,6 +43,9 @@ int FakeStore::init()
   get_dir(mydir);
 
   dout(5) << "init with basedir " << mydir << endl;
+  if (is_shadow) {
+	dout(5) << " SHADOW dir is " << shadowdir << endl;
+  }
 
   // make sure global base dir exists
   struct stat st;
@@ -67,10 +75,13 @@ void FakeStore::get_dir(string& dir) {
   sprintf(s, "%d", whoami);
   dir = basedir + "/" + s;
 }
-void FakeStore::get_oname(object_t oid, string& fn) {
+void FakeStore::get_oname(object_t oid, string& fn, bool shadow) {
   static char s[100];
   sprintf(s, "%d/%02lld/%lld", whoami, HASH_FUNC(oid), oid);
-  fn = basedir + "/" + s;
+  if (shadow)
+	fn = shadowdir + "/" + s;
+  else 
+	fn = basedir + "/" + s;
   //  dout(1) << "oname is " << fn << endl;
 }
 
@@ -104,6 +115,10 @@ int FakeStore::mkfs()
 
   dout(1) << "mkfs in " << mydir << endl;
 
+  if (is_shadow) {
+	dout(1) << "WARNING mkfs reverting to shadow fs, which pbly isn't what MDS expects!" << endl;
+  }
+
   // make sure my dir exists
   r = ::stat(mydir.c_str(), &st);
   if (r != 0) {
@@ -142,32 +157,91 @@ int FakeStore::mkfs()
 bool FakeStore::exists(object_t oid)
 {
   struct stat st;
-  if (stat(oid, &st) == 0) 
+  if (stat(oid, &st) == 0)
 	return true;
-  else
+  else 
 	return false;
 }
 
+  
 int FakeStore::stat(object_t oid,
 					struct stat *st)
 {
   dout(20) << "stat " << oid << endl;
   string fn;
   get_oname(oid,fn);
-  return ::stat(fn.c_str(), st);
+  int r = ::stat(fn.c_str(), st);
+  
+  if (is_shadow && 
+	  r != 0 &&                          // primary didn't exist
+	  ::lstat(fn.c_str(), st) != 0)  {   // and wasn't an intentionally bad symlink
+	get_oname(oid,fn,true);
+	return ::stat(fn.c_str(), st);
+  } else
+	return r;
 }
+ 
+ 
+void FakeStore::shadow_copy_maybe(object_t oid) {
+  struct stat st;
+  string fn;
+  get_oname(oid, fn);
+  if (::lstat(fn.c_str(), &st) == 0) 
+	return;  // live copy exists, we're fine, do nothing.
+
+  // is there a shadow object?
+  string sfn;
+  get_oname(oid, sfn, true);
+  if (::stat(sfn.c_str(), &st) == 0) {
+	// shadow exists.  copy!
+	dout(10) << "copying shadow for " << oid << " " << st.st_size << " bytes" << endl;
+	char *buf = new char[1024*1024];
+	int left = st.st_size;
+
+	int sfd = ::open(sfn.c_str(), O_RDONLY);
+	int fd = ::open(fn.c_str(), O_WRONLY);
+	assert(sfd && fd);
+	while (left) {
+	  int howmuch = left;
+	  if (howmuch > 1024*1024) howmuch = 1024*1024;
+	  int got = ::read(sfd, buf, howmuch);
+	  int wrote = ::write(fd, buf, got);
+	  assert(wrote == got);
+	  left -= got;
+	}
+	::close(fd);
+	::close(sfd);
+  }
+}
+
 
 int FakeStore::remove(object_t oid) 
 {
   dout(20) << "remove " << oid << endl;
   string fn;
   get_oname(oid,fn);
-  return ::unlink(fn.c_str());
+  int r = ::unlink(fn.c_str());
+
+  if (r == 0 && is_shadow) {
+	string sfn;
+	struct stat st;
+	get_oname(oid, sfn, true);
+	int s = ::stat(sfn.c_str(), &st);
+	if (s == 0) {
+	  // shadow exists.  make a bad symlink to mask it.
+	  ::symlink(sfn.c_str(), "doesnotexist");
+	  r = 0;
+	}
+  }
+  return r;
 }
 
 int FakeStore::truncate(object_t oid, off_t size)
 {
   dout(20) << "truncate " << oid << " size " << size << endl;
+
+  if (is_shadow) shadow_copy_maybe(oid);
+  
   string fn;
   get_oname(oid,fn);
   ::truncate(fn.c_str(), size);
@@ -182,7 +256,17 @@ int FakeStore::read(object_t oid,
   get_oname(oid,fn);
   
   int fd = open(fn.c_str(), O_RDONLY);
-  if (fd < 0) return fd;
+  if (fd < 0) {
+	if (is_shadow) {
+	  struct stat st;
+	  if (::lstat(fn.c_str(), &st) == 0) return fd;  // neg symlink
+	  get_oname(oid,fn);
+	  fd = open(fn.c_str(), O_RDONLY);
+	  if (fd < 0) 
+		return fd;    // no shadow either.
+	} else 
+	  return fd;
+  }
   flock(fd, LOCK_EX);    // lock for safety
   
   off_t actual = lseek(fd, offset, SEEK_SET);
@@ -200,6 +284,8 @@ int FakeStore::write(object_t oid,
 					 char *buffer) {
   dout(20) << "write " << oid << " len " << len << " off " << offset << endl;
 
+  if (is_shadow) shadow_copy_maybe(oid);
+
   string fn;
   get_oname(oid,fn);
   

ceph/osd/FakeStore.h

@@ -8,11 +8,33 @@ class FakeStore : public ObjectStore {
   int whoami;
 
   void get_dir(string& dir);
-  void get_oname(object_t oid, string& fn);
+  void get_oname(object_t oid, string& fn, bool shadow=false);
   void wipe_dir(string mydir);
 
+  /* shadow: copy-on-write behavior against a "starting" clean object store... 
+	 
+  if (is_shadow == true), 
+    shadowdir has same layout as basedir
+	
+    if the (normal, live) object file:
+	 - doesn't exist, then use the shadow file if it exists
+	 - does exist, use the live file (in its entirety, COW is on object granularity)
+	 - is a symlink to a nonexistant file, the object doesn't exist (even if it does in the shadow dir)
+
+	write, truncate initiate a copy from shadow -> live.
+	unlink may create a bad symlink if the shadow file exists
+	
+	etc.
+
+	wipe wipes the live dir, effectively revertiing to the shadow fs, so be careful as 
+	   this isn't what a MDS mkfs expects!
+  */
+  string shadowdir;
+  bool is_shadow;
+  void shadow_copy_maybe(object_t oid);   // do copy-on-write.. called by write(), truncate()
+
  public:
-  FakeStore(char *base, int whoami);
+  FakeStore(char *base, int whoami, char *shadow = 0);
 
   int init();
   int finalize();