mirror of
https://github.com/ceph/ceph
synced 2024-12-18 17:37:38 +00:00
traces!
git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@413 29311d96-e01e-0410-9327-a35deaab8ce9
This commit is contained in:
parent
82e27fcf57
commit
ff265b9ba9
@ -51,6 +51,9 @@ COMMON_OBJS= \
|
||||
common/Timer.o\
|
||||
config.o
|
||||
|
||||
SYN_OBJS = \
|
||||
client/Trace.o
|
||||
|
||||
TEST_TARGETS = fakemds mpitest
|
||||
TARGETS = import singleclient mpifuse fakefuse mpisyn fakesyn tcpsyn
|
||||
|
||||
@ -65,6 +68,8 @@ obfs: depend obfstest
|
||||
gprof-helper.so: test/gprof-helper.c
|
||||
gcc -shared -fPIC test/gprof-helper.c -o gprof-helper.so -lpthread -ldl
|
||||
|
||||
|
||||
# old test crap
|
||||
import: mds/allmds.o osd/OSD.o msg/FakeMessenger.o import.cc ${COMMON_OBJS}
|
||||
${CC} ${CFLAGS} ${LIBS} $^ -o $@
|
||||
|
||||
@ -86,26 +91,32 @@ mpitest: test/mpitest.o msg/MPIMessenger.cc mds/allmds.o osd/OSD.o fakeclient/Fa
|
||||
mttest: test/mttest.cc msg/MTMessenger.cc ${COMMON_OBJS}
|
||||
${MPICC} ${CFLAGS} ${LIBS} $^ -o $@
|
||||
|
||||
mpifuse: mpifuse.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/MPIMessenger.cc ${COMMON_OBJS}
|
||||
${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@
|
||||
|
||||
# fuse
|
||||
fakefuse: fakefuse.cc mds/allmds.o client/Client.o client/Buffercache.o osd/OSD.o client/fuse.o msg/FakeMessenger.cc ${COMMON_OBJS}
|
||||
${CC} -pg ${CFLAGS} ${LIBS} -lfuse $^ -o $@
|
||||
|
||||
tcpfuse: tcpfuse.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/TCPMessenger.cc ${COMMON_OBJS}
|
||||
${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@
|
||||
|
||||
mpisyn: mpisyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o msg/MPIMessenger.cc ${COMMON_OBJS}
|
||||
mpifuse: mpifuse.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/MPIMessenger.cc ${COMMON_OBJS}
|
||||
${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@
|
||||
|
||||
# synthetic workload
|
||||
fakesyn: fakesyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o msg/FakeMessenger.o ${COMMON_OBJS} ${SYN_OBJS}
|
||||
${CC} -pg ${CFLAGS} ${LIBS} $^ -o $@
|
||||
|
||||
mpisyn: mpisyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o msg/MPIMessenger.cc ${COMMON_OBJS} ${SYN_OBJS}
|
||||
${MPICC} ${MPICFLAGS} ${MPILIBS} $^ -o $@
|
||||
|
||||
tcpsyn: tcpsyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o msg/TCPMessenger.cc ${COMMON_OBJS}
|
||||
tcpsyn: tcpsyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o msg/TCPMessenger.cc ${COMMON_OBJS} ${SYN_OBJS}
|
||||
${MPICC} ${MPICFLAGS} ${MPILIBS} $^ -o $@
|
||||
|
||||
# obfs + synthetic
|
||||
obfstest: tcpsyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.cc osd/OBFSStore.o msg/TCPMessenger.cc ${COMMON_OBJS}
|
||||
${MPICC} -DUSE_OBFS ${MPICFLAGS} ${MPILIBS} $^ -o $@ ../uofs/uofs.a
|
||||
|
||||
fakesyn: fakesyn.cc mds/allmds.o client/Client.o client/Buffercache.o client/SyntheticClient.o osd/OSD.o msg/FakeMessenger.o ${COMMON_OBJS}
|
||||
${CC} -pg ${CFLAGS} ${LIBS} $^ -o $@
|
||||
|
||||
fakefuse: fakefuse.cc mds/allmds.o client/Client.o client/Buffercache.o osd/OSD.o client/fuse.o msg/FakeMessenger.cc ${COMMON_OBJS}
|
||||
${CC} -pg ${CFLAGS} ${LIBS} -lfuse $^ -o $@
|
||||
|
||||
testmpi: test/testmpi.cc msg/MPIMessenger.cc config.o common/Timer.o common/clock.o msg/Messenger.o msg/Dispatcher.o msg/error.o
|
||||
${MPICC} ${CFLAGS} ${LIBS} $^ -o $@
|
||||
|
@ -1,3 +1,11 @@
|
||||
known bugs to fix after fast
|
||||
- RDWR on synthetic client results in
|
||||
fakesyn: mds/MDS.cc:2334: void MDS::handle_client_close(MClientRequest*, CInode*): Assertion `cur->softlock.can_write(true)' failed.
|
||||
- caps needs to be rewritten
|
||||
- MDS currently not reclaiming fh's (otherwise a client assertion eventually fails)
|
||||
- hard links!
|
||||
|
||||
|
||||
|
||||
!!!
|
||||
- test mds scaling w/ makedirs, vs mds_log_on_request
|
||||
|
@ -21,9 +21,9 @@
|
||||
|
||||
#include "include/config.h"
|
||||
#undef dout
|
||||
#define dout(l) if (l<=g_conf.debug || l<=g_conf.debug_client) cout << "client" << "." << pthread_self() << " "
|
||||
|
||||
#define dout(l) if (l<=g_conf.debug || l<=g_conf.debug_client) cout << "client" << whoami << "." << pthread_self() << " "
|
||||
|
||||
#define tout if (g_conf.client_trace) cout << "trace: "
|
||||
|
||||
|
||||
|
||||
@ -555,6 +555,15 @@ int Client::mount(int mkfs)
|
||||
delete reply;
|
||||
|
||||
client_lock.Unlock();
|
||||
|
||||
dout(3) << "op: // client trace data structs" << endl;
|
||||
dout(3) << "op: struct stat st;" << endl;
|
||||
dout(3) << "op: struct utimbuf utim;" << endl;
|
||||
dout(3) << "op: int readlinkbuf_len = 1000;" << endl;
|
||||
dout(3) << "op: char readlinkbuf[readlinkbuf_len];" << endl;
|
||||
dout(3) << "op: map<string, inode_t*> dir_contents;" << endl;
|
||||
dout(3) << "op: map<fileh_t, fileh_t> open_files;" << endl;
|
||||
dout(3) << "op: fileh_t fh;" << endl;
|
||||
}
|
||||
|
||||
int Client::unmount()
|
||||
@ -584,11 +593,15 @@ int Client::unmount()
|
||||
int Client::link(const char *existing, const char *newname)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->link(\"" << existing << "\", \"" << newname << "\");" << endl;
|
||||
tout << "link" << endl;
|
||||
tout << existing << endl;
|
||||
tout << newname << endl;
|
||||
|
||||
|
||||
// main path arg is new link name
|
||||
// sarg is target (existing file)
|
||||
|
||||
dout(3) << "link " << existing << " " << newname << endl;
|
||||
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_LINK, whoami);
|
||||
req->set_path(newname);
|
||||
@ -614,8 +627,11 @@ int Client::link(const char *existing, const char *newname)
|
||||
int Client::unlink(const char *path)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->unlink\(\"" << path << "\");" << endl;
|
||||
tout << "unlink" << endl;
|
||||
tout << path << endl;
|
||||
|
||||
|
||||
dout(3) << "unlink " << path << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_UNLINK, whoami);
|
||||
req->set_path(path);
|
||||
|
||||
@ -628,9 +644,10 @@ int Client::unlink(const char *path)
|
||||
MClientReply *reply = make_request(req);
|
||||
int res = reply->get_result();
|
||||
if (res == 0) {
|
||||
//this crashes, haven't looked at why yet
|
||||
//Dentry *dn = lookup(req->get_filepath());
|
||||
//if (dn) unlink(dn);
|
||||
// remove from local cache
|
||||
filepath fp(path);
|
||||
Dentry *dn = lookup(fp);
|
||||
if (dn) unlink(dn);
|
||||
}
|
||||
this->insert_trace(reply->get_trace());
|
||||
delete reply;
|
||||
@ -644,8 +661,12 @@ int Client::unlink(const char *path)
|
||||
int Client::rename(const char *from, const char *to)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->rename(\"" << from << "\", \"" << to << "\");" << endl;
|
||||
tout << "rename" << endl;
|
||||
tout << from << endl;
|
||||
tout << to << endl;
|
||||
|
||||
|
||||
dout(3) << "rename " << from << " " << to << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_RENAME, whoami);
|
||||
req->set_path(from);
|
||||
req->set_sarg(to);
|
||||
@ -672,8 +693,12 @@ int Client::rename(const char *from, const char *to)
|
||||
int Client::mkdir(const char *path, mode_t mode)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->mkdir(\"" << path << "\", " << mode << ");" << endl;
|
||||
tout << "mkdir" << endl;
|
||||
tout << path << endl;
|
||||
tout << mode << endl;
|
||||
|
||||
|
||||
dout(3) << "mkdir " << path << " mode " << mode << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_MKDIR, whoami);
|
||||
req->set_path(path);
|
||||
req->set_iarg( (int)mode );
|
||||
@ -698,8 +723,11 @@ int Client::mkdir(const char *path, mode_t mode)
|
||||
int Client::rmdir(const char *path)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->rmdir(\"" << path << "\");" << endl;
|
||||
tout << "rmdir" << endl;
|
||||
tout << path << endl;
|
||||
|
||||
|
||||
dout(3) << "rmdir " << path << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_RMDIR, whoami);
|
||||
req->set_path(path);
|
||||
|
||||
@ -712,8 +740,14 @@ int Client::rmdir(const char *path)
|
||||
MClientReply *reply = make_request(req);
|
||||
int res = reply->get_result();
|
||||
if (res == 0) {
|
||||
// crashes, not sure why yet
|
||||
//unlink(lookup(req->get_filepath()));
|
||||
// remove from local cache
|
||||
filepath fp(path);
|
||||
Dentry *dn = lookup(fp);
|
||||
if (dn) {
|
||||
if (dn->inode->dir && dn->inode->dir->is_empty())
|
||||
close_dir(dn->inode->dir); // FIXME: maybe i shoudl proactively hose the whole subtree from cache?
|
||||
unlink(dn);
|
||||
}
|
||||
}
|
||||
this->insert_trace(reply->get_trace());
|
||||
delete reply;
|
||||
@ -729,8 +763,12 @@ int Client::rmdir(const char *path)
|
||||
int Client::symlink(const char *target, const char *link)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->symlink(\"" << target << "\", \"" << link << "\");" << endl;
|
||||
tout << "symlink" << endl;
|
||||
tout << target << endl;
|
||||
tout << link << endl;
|
||||
|
||||
|
||||
dout(3) << "symlink target " << target << " link " << link << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_SYMLINK, whoami);
|
||||
req->set_path(link);
|
||||
req->set_sarg(target);
|
||||
@ -754,7 +792,12 @@ int Client::symlink(const char *target, const char *link)
|
||||
|
||||
int Client::readlink(const char *path, char *buf, size_t size)
|
||||
{
|
||||
dout(3) << "readlink " << path << endl;
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->readlink(\"" << path << "\", readlinkbuf, readlinkbuf_len);" << endl;
|
||||
tout << "readlink" << endl;
|
||||
tout << path << endl;
|
||||
client_lock.Unlock();
|
||||
|
||||
// stat first (FIXME, PERF access cache directly) ****
|
||||
struct stat stbuf;
|
||||
int r = this->lstat(path, &stbuf);
|
||||
@ -783,8 +826,11 @@ int Client::readlink(const char *path, char *buf, size_t size)
|
||||
int Client::lstat(const char *path, struct stat *stbuf)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->lstat(\"" << path << "\", &st);" << endl;
|
||||
tout << "lstat" << endl;
|
||||
tout << path << endl;
|
||||
|
||||
|
||||
dout(3) << "lstat " << path << endl;
|
||||
// FIXME, PERF request allocation convenient but not necessary for cache hit
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_STAT, whoami);
|
||||
req->set_path(path);
|
||||
@ -849,8 +895,12 @@ int Client::lstat(const char *path, struct stat *stbuf)
|
||||
int Client::chmod(const char *path, mode_t mode)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->chmod(\"" << path << "\", " << mode << ");" << endl;
|
||||
tout << "chmod" << endl;
|
||||
tout << path << endl;
|
||||
tout << mode << endl;
|
||||
|
||||
|
||||
dout(3) << "chmod " << path << " mode " << mode << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_CHMOD, whoami);
|
||||
req->set_path(path);
|
||||
req->set_iarg( (int)mode );
|
||||
@ -873,8 +923,13 @@ int Client::chmod(const char *path, mode_t mode)
|
||||
int Client::chown(const char *path, uid_t uid, gid_t gid)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->chown(\"" << path << "\", " << uid << ", " << gid << ");" << endl;
|
||||
tout << "chown" << endl;
|
||||
tout << path << endl;
|
||||
tout << uid << endl;
|
||||
tout << gid << endl;
|
||||
|
||||
|
||||
dout(3) << "chown " << path << " " << uid << "." << gid << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_CHOWN, whoami);
|
||||
req->set_path(path);
|
||||
req->set_iarg( (int)uid );
|
||||
@ -900,8 +955,14 @@ int Client::chown(const char *path, uid_t uid, gid_t gid)
|
||||
int Client::utime(const char *path, struct utimbuf *buf)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: utim.actime = " << buf->actime << "; utim.modtime = " << buf->modtime << ";" << endl;
|
||||
dout(3) << "op: client->utime(\"" << path << "\", &utim);" << endl;
|
||||
tout << "utime" << endl;
|
||||
tout << path << endl;
|
||||
tout << buf->actime << endl;
|
||||
tout << buf->modtime << endl;
|
||||
|
||||
|
||||
dout(3) << "utime " << path << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_UTIME, whoami);
|
||||
req->set_path(path);
|
||||
req->set_targ( buf->modtime );
|
||||
@ -929,8 +990,12 @@ int Client::utime(const char *path, struct utimbuf *buf)
|
||||
int Client::mknod(const char *path, mode_t mode)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->mknod(\"" << path << "\", " << mode << ");" << endl;
|
||||
tout << "mknod" << endl;
|
||||
tout << path << endl;
|
||||
tout << mode << endl;
|
||||
|
||||
|
||||
dout(3) << "mknod " << path << " mode " << mode << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_MKNOD, whoami);
|
||||
req->set_path(path);
|
||||
req->set_iarg( mode );
|
||||
@ -967,8 +1032,11 @@ int Client::mknod(const char *path, mode_t mode)
|
||||
int Client::getdir(const char *path, map<string,inode_t*>& contents)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->getdir(\"" << path << "\", dir_contents);" << endl;
|
||||
tout << "getdir" << endl;
|
||||
tout << path << endl;
|
||||
|
||||
|
||||
dout(3) << "getdir " << path << endl;
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_READDIR, whoami);
|
||||
req->set_path(path);
|
||||
|
||||
@ -1026,9 +1094,12 @@ int Client::getdir(const char *path, map<string,inode_t*>& contents)
|
||||
int Client::open(const char *path, int mode)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: fh = client->open(\"" << path << "\", " << mode << ");" << endl;
|
||||
tout << "open" << endl;
|
||||
tout << path << endl;
|
||||
tout << mode << endl;
|
||||
|
||||
|
||||
dout(3) << "open " << path << " mode " << mode << endl;
|
||||
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_OPEN, whoami);
|
||||
req->set_path(path);
|
||||
req->set_iarg(mode);
|
||||
@ -1039,7 +1110,8 @@ int Client::open(const char *path, int mode)
|
||||
|
||||
MClientReply *reply = make_request(req);
|
||||
assert(reply);
|
||||
dout(3) << "open result = " << reply->get_result() << endl;
|
||||
dout(3) << "op: open_files[" << reply->get_result() << "] = fh; // fh = " << reply->get_result() << endl;
|
||||
tout << reply->get_result() << endl;
|
||||
|
||||
vector<c_inode_info*> trace = reply->get_trace();
|
||||
this->insert_trace(trace);
|
||||
@ -1076,8 +1148,12 @@ int Client::open(const char *path, int mode)
|
||||
int Client::close(fileh_t fh)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->close(open_files[ " << fh << " ]);" << endl;
|
||||
dout(3) << "op: open_files.erase( " << fh << " );" << endl;
|
||||
tout << "close" << endl;
|
||||
tout << fh << endl;
|
||||
|
||||
|
||||
dout(3) << "close " << fh << endl;
|
||||
assert(fh_map.count(fh));
|
||||
Fh *f = fh_map[fh];
|
||||
Inode *in = f->inode;
|
||||
@ -1378,6 +1454,9 @@ int Client::write(fileh_t fh, const char *buf, size_t size, off_t offset)
|
||||
|
||||
cond.Wait(client_lock);
|
||||
}
|
||||
#if 0
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
// assume success for now. FIXME.
|
||||
@ -1401,9 +1480,43 @@ int Client::write(fileh_t fh, const char *buf, size_t size, off_t offset)
|
||||
}
|
||||
|
||||
|
||||
int Client::truncate(const char *file, off_t size)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->truncate(\"" << file << "\", " << size << ");" << endl;
|
||||
tout << "truncate" << endl;
|
||||
tout << file << endl;
|
||||
tout << size << endl;
|
||||
|
||||
|
||||
MClientRequest *req = new MClientRequest(MDS_OP_TRUNCATE, whoami);
|
||||
req->set_path(file);
|
||||
req->set_sizearg( size );
|
||||
|
||||
// FIXME where does FUSE maintain user information
|
||||
req->set_caller_uid(getuid());
|
||||
req->set_caller_gid(getgid());
|
||||
|
||||
MClientReply *reply = make_request(req);
|
||||
int res = reply->get_result();
|
||||
this->insert_trace(reply->get_trace());
|
||||
delete reply;
|
||||
|
||||
dout(10) << " truncate result is " << res << endl;
|
||||
|
||||
client_lock.Unlock();
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
int Client::fsync(fileh_t fh, bool syncdataonly)
|
||||
{
|
||||
client_lock.Lock();
|
||||
dout(3) << "op: client->fsync(open_files[ " << fh << " ], " << syncdataonly << ");" << endl;
|
||||
tout << "fsync" << endl;
|
||||
tout << fh << endl;
|
||||
tout << syncdataonly << endl;
|
||||
|
||||
int r = 0;
|
||||
|
||||
assert(fh_map.count(fh));
|
||||
|
@ -171,6 +171,7 @@ class Client : public Dispatcher {
|
||||
in->put();
|
||||
if (in->ref == 0) {
|
||||
inode_map.erase(in->inode.ino);
|
||||
if (in == root) root = 0;
|
||||
delete in;
|
||||
}
|
||||
}
|
||||
@ -318,7 +319,8 @@ class Client : public Dispatcher {
|
||||
int close(fileh_t fh);
|
||||
int read(fileh_t fh, char *buf, size_t size, off_t offset);
|
||||
int write(fileh_t fh, const char *buf, size_t size, off_t offset);
|
||||
int truncate(fileh_t fh, off_t size);
|
||||
int truncate(const char *file, off_t size);
|
||||
//int truncate(fileh_t fh, off_t size);
|
||||
int fsync(fileh_t fh, bool syncdataonly);
|
||||
|
||||
};
|
||||
|
@ -4,6 +4,7 @@
|
||||
#include "include/filepath.h"
|
||||
#include "mds/MDS.h"
|
||||
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
@ -15,6 +16,10 @@
|
||||
#undef dout
|
||||
#define dout(l) if (l<=g_conf.debug || l<=g_conf.debug_client) cout << "synthetic" << client->get_nodeid() << " "
|
||||
|
||||
// traces
|
||||
//void trace_include(SyntheticClient *syn, Client *cl, string& prefix);
|
||||
//void trace_openssh(SyntheticClient *syn, Client *cl, string& prefix);
|
||||
|
||||
|
||||
#define DBL 2
|
||||
|
||||
@ -118,6 +123,38 @@ int SyntheticClient::run()
|
||||
read_file(sarg1, iarg1, iarg2);
|
||||
}
|
||||
break;
|
||||
|
||||
|
||||
case SYNCLIENT_MODE_TRACEINCLUDE:
|
||||
{
|
||||
int iarg1 = iargs.front(); iargs.pop_front();
|
||||
string prefix;
|
||||
if (client->whoami == 0) {
|
||||
Trace t("client/traces/trace.include");
|
||||
play_trace(t, prefix);
|
||||
} else {
|
||||
sleep(iarg1);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SYNCLIENT_MODE_TRACEOPENSSH:
|
||||
{
|
||||
string prefix = get_sarg();
|
||||
int iarg1 = iargs.front(); iargs.pop_front();
|
||||
|
||||
Trace t("client/traces/trace.openssh");
|
||||
|
||||
client->mkdir(prefix.c_str(), 0755);
|
||||
|
||||
for (int i=0; i<iarg1; i++) {
|
||||
if (time_to_stop()) break;
|
||||
play_trace(t, prefix);
|
||||
if (time_to_stop()) break;
|
||||
clean_dir(prefix);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
assert(0);
|
||||
}
|
||||
@ -186,11 +223,148 @@ void SyntheticClient::up()
|
||||
}
|
||||
|
||||
|
||||
int SyntheticClient::play_trace(Trace& t, string& prefix)
|
||||
{
|
||||
dout(4) << "play trace" << endl;
|
||||
t.start();
|
||||
|
||||
const char *p = prefix.c_str();
|
||||
|
||||
map<__int64_t, int> open_files;
|
||||
|
||||
while (!t.end()) {
|
||||
|
||||
if (time_to_stop()) break;
|
||||
|
||||
// op
|
||||
const char *op = t.get_string();
|
||||
dout(4) << "trace op " << op << endl;
|
||||
if (strcmp(op, "link") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
const char *b = t.get_string(p);
|
||||
client->link(a,b);
|
||||
} else if (strcmp(op, "unlink") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
client->unlink(a);
|
||||
} else if (strcmp(op, "rename") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
const char *b = t.get_string(p);
|
||||
client->rename(a,b);
|
||||
} else if (strcmp(op, "mkdir") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
client->mkdir(a, b);
|
||||
} else if (strcmp(op, "rmdir") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
client->rmdir(a);
|
||||
} else if (strcmp(op, "symlink") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
const char *b = t.get_string(p);
|
||||
client->symlink(a,b);
|
||||
} else if (strcmp(op, "readlink") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
char buf[100];
|
||||
client->readlink(a, buf, 100);
|
||||
} else if (strcmp(op, "lstat") == 0) {
|
||||
struct stat st;
|
||||
const char *a = t.get_string(p);
|
||||
client->lstat(a, &st);
|
||||
} else if (strcmp(op, "chmod") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
client->chmod(a, b);
|
||||
} else if (strcmp(op, "chown") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
__int64_t c = t.get_int();
|
||||
client->chown(a, b, c);
|
||||
} else if (strcmp(op, "utime") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
__int64_t c = t.get_int();
|
||||
struct utimbuf u;
|
||||
u.actime = b;
|
||||
u.modtime = c;
|
||||
client->utime(a, &u);
|
||||
} else if (strcmp(op, "mknod") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
client->mknod(a, b);
|
||||
} else if (strcmp(op, "getdir") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
map<string,inode_t*> contents;
|
||||
client->getdir(a, contents);
|
||||
} else if (strcmp(op, "open") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
__int64_t id = t.get_int();
|
||||
__int64_t fh = client->open(a, b);
|
||||
open_files[id] = fh;
|
||||
} else if (strcmp(op, "close") == 0) {
|
||||
__int64_t id = t.get_int();
|
||||
__int64_t fh = open_files[id];
|
||||
if (fh > 0) client->close(fh);
|
||||
open_files.erase(id);
|
||||
} else if (strcmp(op, "truncate") == 0) {
|
||||
const char *a = t.get_string(p);
|
||||
__int64_t b = t.get_int();
|
||||
client->truncate(a,b);
|
||||
} else if (strcmp(op, "fsync") == 0) {
|
||||
assert(0);
|
||||
} else
|
||||
assert(0);
|
||||
}
|
||||
|
||||
// close open files
|
||||
for (map<__int64_t, __int64_t>::iterator fi = open_files.begin();
|
||||
fi != open_files.end();
|
||||
fi++) {
|
||||
client->close(fi->second);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
int SyntheticClient::clean_dir(string& basedir)
|
||||
{
|
||||
// read dir
|
||||
map<string, inode_t*> contents;
|
||||
int r = client->getdir(basedir.c_str(), contents);
|
||||
if (r < 0) {
|
||||
dout(1) << "readdir on " << basedir << " returns " << r << endl;
|
||||
return r;
|
||||
}
|
||||
|
||||
for (map<string, inode_t*>::iterator it = contents.begin();
|
||||
it != contents.end();
|
||||
it++) {
|
||||
string file = basedir + "/" + it->first;
|
||||
|
||||
if (time_to_stop()) break;
|
||||
|
||||
struct stat st;
|
||||
int r = client->lstat(file.c_str(), &st);
|
||||
if (r < 0) {
|
||||
dout(1) << "stat error on " << file << " r=" << r << endl;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (st.st_mode & INODE_MODE_DIR) {
|
||||
clean_dir(file);
|
||||
client->rmdir(file.c_str());
|
||||
} else {
|
||||
client->unlink(file.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
}
|
||||
|
||||
|
||||
int SyntheticClient::full_walk(string& basedir)
|
||||
{
|
||||
if (run_until.first && g_clock.gettimepair() > run_until) return -1;
|
||||
if (time_to_stop()) return -1;
|
||||
|
||||
// read dir
|
||||
map<string, inode_t*> contents;
|
||||
@ -220,7 +394,7 @@ int SyntheticClient::full_walk(string& basedir)
|
||||
|
||||
int SyntheticClient::make_dirs(const char *basedir, int dirs, int files, int depth)
|
||||
{
|
||||
if (run_until.first && g_clock.gettimepair() > run_until) return 0;
|
||||
if (time_to_stop()) return 0;
|
||||
|
||||
// make sure base dir exists
|
||||
int r = client->mkdir(basedir, 0755);
|
||||
@ -261,7 +435,7 @@ int SyntheticClient::write_file(string& fn, int size, int wrsize) // size is i
|
||||
if (fd < 0) return fd;
|
||||
|
||||
for (int i=0; i<chunks; i++) {
|
||||
if (run_until.first && g_clock.gettimepair() > run_until) break;
|
||||
if (time_to_stop()) break;
|
||||
dout(2) << "writing block " << i << "/" << chunks << endl;
|
||||
client->write(fd, buf, wrsize, i*wrsize);
|
||||
}
|
||||
@ -281,7 +455,7 @@ int SyntheticClient::read_file(string& fn, int size, int rdsize) // size is in
|
||||
if (fd < 0) return fd;
|
||||
|
||||
for (int i=0; i<chunks; i++) {
|
||||
if (run_until.first && g_clock.gettimepair() > run_until) break;
|
||||
if (time_to_stop()) break;
|
||||
dout(2) << "reading block " << i << "/" << chunks << endl;
|
||||
client->read(fd, buf, rdsize, i*rdsize);
|
||||
}
|
||||
@ -303,7 +477,7 @@ int SyntheticClient::random_walk(int num_req)
|
||||
while (left > 0) {
|
||||
left--;
|
||||
|
||||
if (run_until.first && g_clock.gettimepair() > run_until) break;
|
||||
if (time_to_stop()) break;
|
||||
|
||||
// ascend?
|
||||
if (cwd.depth() && !roll_die(pow(.9, cwd.depth()))) {
|
||||
@ -476,3 +650,5 @@ int SyntheticClient::random_walk(int num_req)
|
||||
dout(DBL) << "done" << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
@ -6,6 +6,8 @@
|
||||
#include "Client.h"
|
||||
#include "include/Distribution.h"
|
||||
|
||||
#include "Trace.h"
|
||||
|
||||
#define SYNCLIENT_MODE_RANDOMWALK 1
|
||||
#define SYNCLIENT_MODE_FULLWALK 2
|
||||
#define SYNCLIENT_MODE_MAKEDIRS 3
|
||||
@ -14,6 +16,9 @@
|
||||
#define SYNCLIENT_MODE_UNTIL 6
|
||||
#define SYNCLIENT_MODE_REPEATWALK 7
|
||||
|
||||
#define SYNCLIENT_MODE_TRACEOPENSSH 8
|
||||
#define SYNCLIENT_MODE_TRACEINCLUDE 9
|
||||
|
||||
class SyntheticClient {
|
||||
Client *client;
|
||||
|
||||
@ -105,13 +110,27 @@ class SyntheticClient {
|
||||
timepair_t run_until;
|
||||
|
||||
string get_sarg();
|
||||
|
||||
|
||||
bool time_to_stop() {
|
||||
if (run_until.first && g_clock.gettimepair() > run_until)
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
string compose_path(string& prefix, char *rest) {
|
||||
return prefix + rest;
|
||||
}
|
||||
|
||||
int full_walk(string& fromdir);
|
||||
int random_walk(int n);
|
||||
int make_dirs(const char *basedir, int dirs, int files, int depth);
|
||||
int write_file(string& fn, int mb, int chunk);
|
||||
int read_file(string& fn, int mb, int chunk);
|
||||
|
||||
int clean_dir(string& basedir);
|
||||
|
||||
int play_trace(Trace& t, string& prefix);
|
||||
|
||||
};
|
||||
|
||||
|
105
ceph/client/Trace.cc
Normal file
105
ceph/client/Trace.cc
Normal file
@ -0,0 +1,105 @@
|
||||
|
||||
#include "Trace.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <map>
|
||||
#include <ext/rope>
|
||||
using namespace __gnu_cxx;
|
||||
|
||||
#include "common/Mutex.h"
|
||||
|
||||
#include "include/config.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
|
||||
Mutex trace_lock;
|
||||
|
||||
class TokenList {
|
||||
public:
|
||||
string filename;
|
||||
char *data;
|
||||
int len;
|
||||
list<const char *> tokens;
|
||||
|
||||
int ref;
|
||||
};
|
||||
|
||||
map<string, TokenList*> traces;
|
||||
|
||||
|
||||
//
|
||||
Trace::Trace(const char* f)
|
||||
{
|
||||
string filename = f;
|
||||
|
||||
trace_lock.Lock();
|
||||
|
||||
if (traces.count(filename))
|
||||
tl = traces[filename];
|
||||
else {
|
||||
tl = new TokenList;
|
||||
tl->ref = 0;
|
||||
tl->filename = filename;
|
||||
|
||||
// open file
|
||||
crope cr;
|
||||
int fd = open(filename.c_str(), O_RDONLY);
|
||||
assert(fd > 0);
|
||||
char buf[100];
|
||||
while (1) {
|
||||
int r = read(fd, buf, 100);
|
||||
if (r == 0) break;
|
||||
assert(r > 0);
|
||||
cr.append(buf, r);
|
||||
}
|
||||
close(fd);
|
||||
|
||||
// copy
|
||||
tl->len = cr.length()+1;
|
||||
tl->data = new char[cr.length()];
|
||||
memcpy(tl->data, cr.c_str(), cr.length());
|
||||
tl->data[tl->len-1] = '\n';
|
||||
|
||||
// index!
|
||||
int o = 0;
|
||||
while (o < tl->len) {
|
||||
char *n = tl->data + o;
|
||||
|
||||
// find newline
|
||||
while (tl->data[o] != '\n') o++;
|
||||
assert(tl->data[o] == '\n');
|
||||
tl->data[o] = 0;
|
||||
|
||||
if (tl->data + o > n) tl->tokens.push_back(n);
|
||||
o++;
|
||||
}
|
||||
|
||||
dout(1) << "trace " << filename << " loaded with " << tl->tokens.size() << " tokens" << endl;
|
||||
traces[filename] = tl;
|
||||
}
|
||||
tl->ref++;
|
||||
|
||||
trace_lock.Unlock();
|
||||
}
|
||||
|
||||
Trace::~Trace()
|
||||
{
|
||||
trace_lock.Lock();
|
||||
|
||||
tl->ref--;
|
||||
if (tl->ref == 0) {
|
||||
traces.erase(tl->filename);
|
||||
delete tl;
|
||||
}
|
||||
|
||||
trace_lock.Unlock();
|
||||
}
|
||||
|
||||
|
||||
list<const char*>& Trace::get_list()
|
||||
{
|
||||
return tl->tokens;
|
||||
}
|
59
ceph/client/Trace.h
Normal file
59
ceph/client/Trace.h
Normal file
@ -0,0 +1,59 @@
|
||||
#ifndef __CLIENT_TRACE_H
|
||||
#define __CLIENT_TRACE_H
|
||||
|
||||
#include <list>
|
||||
#include <string>
|
||||
using namespace std;
|
||||
|
||||
/*
|
||||
|
||||
this class is more like an iterator over a constant tokenlist (which
|
||||
is protected by a mutex, see Trace.cc)
|
||||
|
||||
*/
|
||||
|
||||
class Trace {
|
||||
class TokenList *tl;
|
||||
|
||||
public:
|
||||
Trace(const char* filename);
|
||||
~Trace();
|
||||
|
||||
list<const char*>& get_list();
|
||||
|
||||
list<const char*>::iterator _cur;
|
||||
list<const char*>::iterator _end;
|
||||
|
||||
void start() {
|
||||
_cur = get_list().begin();
|
||||
_end = get_list().end();
|
||||
ns = 0;
|
||||
}
|
||||
|
||||
char strings[10][200];
|
||||
int ns;
|
||||
const char *get_string(const char *prefix = 0) {
|
||||
const char *s = *_cur;
|
||||
_cur++;
|
||||
if (prefix) {
|
||||
if (strstr(s, "/prefix") == s ||
|
||||
strstr(s, "/prefix") == s+1) {
|
||||
strcpy(strings[ns], prefix);
|
||||
strcpy(strings[ns] + strlen(prefix),
|
||||
s + strlen("/prefix"));
|
||||
s = (const char*)strings[ns];
|
||||
ns++;
|
||||
if (ns == 10) ns = 0;
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
__int64_t get_int() {
|
||||
return atoll(get_string());
|
||||
}
|
||||
bool end() {
|
||||
return _cur == _end;
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
@ -31,6 +31,8 @@
|
||||
|
||||
#include "Client.h"
|
||||
|
||||
#include "include/config.h"
|
||||
|
||||
// stl
|
||||
#include <map>
|
||||
using namespace std;
|
||||
@ -141,7 +143,7 @@ static int ceph_chown(const char *path, uid_t uid, gid_t gid)
|
||||
|
||||
static int ceph_truncate(const char *path, off_t size)
|
||||
{
|
||||
return truncate(path, size);
|
||||
return client->truncate(path, size);
|
||||
}
|
||||
|
||||
static int ceph_utime(const char *path, struct utimbuf *buf)
|
||||
@ -276,8 +278,10 @@ int ceph_fuse_main(Client *c, int argc, char *argv[])
|
||||
// large reads, direct_io (no kernel cachine)
|
||||
//newargv[newargc++] = "-o";
|
||||
//newargv[newargc++] = "large_read";
|
||||
newargv[newargc++] = "-o";
|
||||
newargv[newargc++] = "direct_io";
|
||||
if (g_conf.fuse_direct_io) {
|
||||
newargv[newargc++] = "-o";
|
||||
newargv[newargc++] = "direct_io";
|
||||
}
|
||||
|
||||
// disable stupid fuse unlink hiding thing
|
||||
newargv[newargc++] = "-o";
|
||||
|
@ -62,6 +62,8 @@ md_config_t g_conf = {
|
||||
client_bcache_alloc_minsize: 1024,
|
||||
client_bcache_alloc_maxsize: 262144,
|
||||
client_bcache_ttl: 30, // seconds until dirty buffers are written to disk
|
||||
client_trace: 0,
|
||||
fuse_direct_io: 1,
|
||||
|
||||
// --- mds ---
|
||||
mds_cache_size: MDS_CACHE_SIZE,
|
||||
@ -202,6 +204,10 @@ void parse_config_options(int argc, char **argv,
|
||||
g_conf.client_cache_size = atoi(argv[++i]);
|
||||
else if (strcmp(argv[i], "--client_cache_stat_ttl") == 0)
|
||||
g_conf.client_cache_stat_ttl = atoi(argv[++i]);
|
||||
else if (strcmp(argv[i], "--client_trace") == 0)
|
||||
g_conf.client_trace = atoi(argv[++i]);
|
||||
else if (strcmp(argv[i], "--fuse_direct_io") == 0)
|
||||
g_conf.fuse_direct_io = atoi(argv[++i]);
|
||||
|
||||
else if (strcmp(argv[i], "--osd_fsync") == 0)
|
||||
g_conf.osd_fsync = atoi(argv[++i]);
|
||||
|
@ -37,6 +37,8 @@ struct md_config_t {
|
||||
int client_bcache_alloc_minsize;
|
||||
int client_bcache_alloc_maxsize;
|
||||
int client_bcache_ttl;
|
||||
int client_trace;
|
||||
int fuse_direct_io;
|
||||
|
||||
// mds
|
||||
int mds_cache_size;
|
||||
|
@ -79,6 +79,12 @@ int main(int oargc, char **oargv)
|
||||
//syn_sargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"randomwalk") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_RANDOMWALK );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"trace_include") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_TRACEINCLUDE );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"trace_openssh") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_TRACEOPENSSH );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"until") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_UNTIL );
|
||||
|
@ -475,6 +475,7 @@ void MDS::my_dispatch(Message *m)
|
||||
// shut down?
|
||||
if (shutting_down && !shut_down) {
|
||||
if (mdcache->shutdown_pass()) {
|
||||
dout(7) << "shutdown_pass=true, finished w/ shutdown" << endl;
|
||||
shutting_down = false;
|
||||
shut_down = true;
|
||||
if (whoami) shutdown_final();
|
||||
@ -713,7 +714,7 @@ void MDS::handle_client_request(MClientRequest *req)
|
||||
*/
|
||||
|
||||
case MDS_OP_TRUNCATE:
|
||||
if (!req->get_iarg()) break; // can be called w/ either fh OR path
|
||||
if (!req->get_ino()) break; // can be called w/ either fh OR path
|
||||
|
||||
case MDS_OP_CLOSE:
|
||||
case MDS_OP_FSYNC:
|
||||
@ -741,7 +742,6 @@ void MDS::handle_client_request(MClientRequest *req)
|
||||
case MDS_OP_MKNOD:
|
||||
case MDS_OP_MKDIR:
|
||||
case MDS_OP_SYMLINK:
|
||||
case MDS_OP_TRUNCATE:
|
||||
case MDS_OP_LINK:
|
||||
case MDS_OP_UNLINK: // also wrt parent dir, NOT the unlinked inode!!
|
||||
case MDS_OP_RMDIR:
|
||||
@ -832,10 +832,10 @@ void MDS::dispatch_request(Message *m, CInode *ref)
|
||||
else
|
||||
handle_client_open(req, ref);
|
||||
break;
|
||||
/*
|
||||
case MDS_OP_TRUNCATE:
|
||||
handle_client_truncate(req, ref);
|
||||
break;
|
||||
/*
|
||||
case MDS_OP_FSYNC:
|
||||
handle_client_fsync(req, ref);
|
||||
break;
|
||||
@ -2203,6 +2203,36 @@ void MDS::handle_client_symlink(MClientRequest *req, CInode *diri)
|
||||
|
||||
|
||||
|
||||
// ===================================
|
||||
// TRUNCATE, FSYNC
|
||||
|
||||
/*
|
||||
* FIXME: this truncate implemention is WRONG WRONG WRONG
|
||||
*/
|
||||
|
||||
void MDS::handle_client_truncate(MClientRequest *req, CInode *cur)
|
||||
{
|
||||
// write
|
||||
if (!mdcache->inode_hard_write_start(cur, req))
|
||||
return; // fw or (wait for) lock
|
||||
|
||||
// check permissions
|
||||
|
||||
// do update
|
||||
cur->inode.size = req->get_sizearg();
|
||||
cur->mark_dirty();
|
||||
|
||||
mdcache->inode_hard_write_finish(cur);
|
||||
|
||||
balancer->hit_inode(cur);
|
||||
|
||||
// start reply
|
||||
MClientReply *reply = new MClientReply(req, 0);
|
||||
|
||||
// commit
|
||||
commit_request(req, reply, cur,
|
||||
new EInodeUpdate(cur));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
@ -191,8 +191,9 @@ int fakemessenger_do_loop_2()
|
||||
FakeMessenger::FakeMessenger(long me) : Messenger(me)
|
||||
{
|
||||
whoami = me;
|
||||
lock.Lock();
|
||||
directory[ whoami ] = this;
|
||||
|
||||
lock.Unlock();
|
||||
|
||||
cout << "fakemessenger " << whoami << " messenger is " << this << endl;
|
||||
|
||||
@ -224,11 +225,13 @@ FakeMessenger::~FakeMessenger()
|
||||
int FakeMessenger::shutdown()
|
||||
{
|
||||
//cout << "shutdown on messenger " << this << " has " << num_incoming() << " queued" << endl;
|
||||
lock.Lock();
|
||||
directory.erase(whoami);
|
||||
if (directory.empty()) {
|
||||
::shutdown = true;
|
||||
cond.Signal(); // why not
|
||||
}
|
||||
}
|
||||
lock.Unlock();
|
||||
}
|
||||
|
||||
/*
|
||||
|
8
ceph/script/clean_trace.pl
Executable file
8
ceph/script/clean_trace.pl
Executable file
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
my $n = 0;
|
||||
while (<>) {
|
||||
next unless /trace: /;
|
||||
my $l = $'; $';
|
||||
print $l;
|
||||
}
|
@ -81,6 +81,12 @@ int main(int oargc, char **oargv) {
|
||||
} else if (strcmp(argv[i],"randomwalk") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_RANDOMWALK );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"trace_include") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_TRACEINCLUDE );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"trace_openssh") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_TRACEOPENSSH );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
} else if (strcmp(argv[i],"until") == 0) {
|
||||
syn_modes.push_back( SYNCLIENT_MODE_UNTIL );
|
||||
syn_iargs.push_back( atoi(argv[++i]) );
|
||||
|
Loading…
Reference in New Issue
Block a user