From e6a39cf313337196039a998752c6354132a2d6a6 Mon Sep 17 00:00:00 2001 From: Thomas Schoebel-Theuer Date: Thu, 4 Jul 2013 08:19:24 +0200 Subject: [PATCH] tools: add provisionary logfile import/export (HACKERS ONLY) Use this for testing or for repair if you are extremely desperate. This is not yet end-user friendly. --- userspace/mars-log-impex.c | 325 +++++++++++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 userspace/mars-log-impex.c diff --git a/userspace/mars-log-impex.c b/userspace/mars-log-impex.c new file mode 100644 index 00000000..a2efe196 --- /dev/null +++ b/userspace/mars-log-impex.c @@ -0,0 +1,325 @@ +// (c) 2013 Thomas Schoebel-Theuer / 1&1 Internet AG + +/* This is PROVISIONARY hacker's tool for import / export + * of MARS transaction logfiles. + * + * NOT FOR END USERS!!!!! + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +/* FIXME: some _provisionary_ hacks to bridge the gap between kernelspace and userspace... + */ +#define bool int +#define false 0 +#define true 1 +#define likely(x) x +#define unlikely(x) x +#define MARS_INF printf +#define MARS_WRN printf +#define MARS_ERR printf +#define mars_digest_size 16 +#define mars_digest(a,b,c) /*empty*/ +#define loff_t long long +#include "../kernel/lib_log.h" + +static +int read_record( + struct log_header *lh, + void *buf, + int maxlen, + int fh, + loff_t pos, + void **payload, + int *payload_len, + unsigned int *seq_nr) +{ + ssize_t status; + + status = pread(fh, buf, maxlen, pos); + if (status < 0) { + MARS_ERR("cannot pread() %d bytes, status = %d\n", maxlen, (int)status); + return -errno; + } + if (!status) { + MARS_INF("got EOF\n"); + return 0; + } + + return log_scan(buf, status, pos, 0, true, lh, payload, payload_len, seq_nr); +} + +static +int write_record( + int out_fd, + void *buf, + int buf_len, + char *desc) +{ + struct log_header lh = { + .l_len = buf_len, + }; + unsigned short total_len = buf_len + OVERHEAD; + char data[total_len]; + int offset = 0; + int len = strlen(desc); + int crc = 0; + int status; + + // extract filename from desc path + while (len > 0 && desc[len-1] != '/') + len--; + desc += len; + + status = sscanf( + desc, + "%u,%ld.%lu,%ld.%lu,%hx,%lld", + &lh.l_seq_nr, + &lh.l_stamp.tv_sec, + &lh.l_stamp.tv_nsec, + &lh.l_written.tv_sec, + &lh.l_written.tv_nsec, + &lh.l_code, + &lh.l_pos + ); + if (status != 7) { + MARS_ERR("only %d arguments parsable from '%s'\n", status, desc); + return -EINVAL; + } + + DATA_PUT(data, offset, START_MAGIC); + DATA_PUT(data, offset, (char)FORMAT_VERSION); + DATA_PUT(data, offset, (char)1); // valid_flag + DATA_PUT(data, offset, total_len); // start of next header + DATA_PUT(data, offset, lh.l_stamp.tv_sec); + DATA_PUT(data, offset, lh.l_stamp.tv_nsec); + DATA_PUT(data, offset, lh.l_pos); + DATA_PUT(data, offset, lh.l_len); + DATA_PUT(data, offset, (short)0); // spare + DATA_PUT(data, offset, (int)0); // spare + DATA_PUT(data, offset, lh.l_code); + DATA_PUT(data, offset, (short)0); // spare + + memcpy(data + offset, buf, buf_len); + offset += buf_len; + + DATA_PUT(data, offset, END_MAGIC); + DATA_PUT(data, offset, crc); + DATA_PUT(data, offset, (char)1); // valid_flag copy + DATA_PUT(data, offset, (char)0); // spare + DATA_PUT(data, offset, (short)0); // spare + DATA_PUT(data, offset, lh.l_seq_nr); + DATA_PUT(data, offset, lh.l_written.tv_sec); + DATA_PUT(data, offset, lh.l_written.tv_nsec); + + if (offset != total_len) { + MARS_ERR("offset %d != total_len %d\n", offset, total_len); + return -EINVAL; + } + + status = write(out_fd, data, total_len); + if (status != total_len) { + MARS_ERR("bad write, status = %d errno = %d\n", status, errno); + return -EIO; + } + + return 0; +} + +static +int make_dirs(char *out, int out_len, char *out_dirname, int old[], unsigned int seqnr) +{ + int len; + unsigned int nr; + + nr = seqnr / 1000000000; + + len = snprintf(out, out_len, "%s/%01u", out_dirname, nr); + + if (old[0] != nr) { + old[0] = nr; + (void)mkdir(out, 0700); + } + + nr = seqnr / 1000000 % 1000; + + len += snprintf(out + len, out_len - len, "/%03u", nr); + + if (old[1] != nr) { + old[1] = nr; + (void)mkdir(out, 0700); + } + + nr = seqnr / 1000 % 1000; + + len += snprintf(out + len, out_len - len, "/%03u", nr); + + if (old[2] != nr) { + old[2] = nr; + (void)mkdir(out, 0700); + } + + return len; +} + +static +int export_logfile(char *in_filename, char *out_dirname) +{ + char buf[4096 * 8]; + int old[3] = { -1, -1, -1}; + loff_t pos = 0; + unsigned int old_seqnr = 0; + int in_fd; + + in_fd = open(in_filename, O_RDONLY); + if (in_fd < 0) { + MARS_ERR("cannot open input file '%s', errno = %d\n", in_filename, errno); + return -errno; + } + + if (out_dirname) { + (void)mkdir(out_dirname, 0700); + } + + for (;;) { + struct log_header lh = {}; + void *payload = NULL; + int payload_len = 0; + unsigned int seqnr = 0; + int status; + + status = read_record(&lh, buf, sizeof(buf), in_fd, pos, &payload, &payload_len, &seqnr); + if (status <= 0) { + return status; + } + + if (old_seqnr > 0 && seqnr != old_seqnr + 1) { + printf("ERROR: seqnr = %d status = %d\n", seqnr, status); + } else { + //printf("OK: seqnr = %d status = %d\n", seqnr, status); + } + + if (out_dirname) { + char out_name[1024]; + int len; + int out_fd; + + len = make_dirs(out_name, sizeof(out_name), out_dirname, old, seqnr); + + snprintf(out_name + len, sizeof(out_name) - len, + "/%010u,%09u.%09u,%09u.%09u,%04x,%012llu", + seqnr, + (unsigned)lh.l_stamp.tv_sec, + (unsigned)lh.l_stamp.tv_nsec, + (unsigned)lh.l_written.tv_sec, + (unsigned)lh.l_written.tv_nsec, + lh.l_code, + (unsigned long long)lh.l_pos + ); + + out_fd = creat(out_name, 0600); + if (out_fd < 0) { + MARS_ERR("cannot open output file '%s', errno = %d\n", out_name, errno); + return -errno; + } + write(out_fd, payload, payload_len); + close(out_fd); + } + + pos += status; + old_seqnr = seqnr; + } +} + +static +int import_logfile(char *in_dirname, char *out_filename) +{ + char buf[4096 * 8]; + char cmd[256]; + int out_fd; + FILE *names; + + snprintf(cmd, sizeof(cmd), "find '%s' -type f -name '[0-9]*[0-9]' | sort -n", in_dirname); + + names = popen(cmd, "r"); + if (!names) { + MARS_ERR("cannot popen command '%s', errno = %d\n", cmd, errno); + return -errno; + } + + out_fd = creat(out_filename, 0600); + if (out_fd < 0) { + MARS_ERR("cannot open output file '%s', errno = %d\n", out_filename, errno); + return -errno; + } + + for (;;) { + char path[1024]; + int len; + int in_fd; + int status; + + if (!fgets(path, sizeof(path), names)) { + break; + } + // chomp the terminating \n + len = strlen(path); + if (len <= 1) + continue; + if (path[len-1] == '\n') + path[len-1] = '\0'; + + in_fd = open(path, O_RDONLY); + if (in_fd < 0) { + MARS_ERR("cannot open input file '%s', errno = %d\n", path, errno); + return -errno; + } + + status = read(in_fd, buf, sizeof(buf)); + if (status < 0) { + MARS_ERR("cannot read from input file '%s', errno = %d\n", path, errno); + return -errno; + } + close(in_fd); + if (status >= sizeof(buf)) { + MARS_ERR("input file '%s' contains a too long record\n", path); + return -EINVAL; + } + + printf("record '%s' len = %d\n", path, status); + status = write_record(out_fd, buf, status, path); + if (status < 0) + break; + } + + close(out_fd); + pclose(names); + return 0; +} + +int main(int argc, char *argv[]) +{ + if (argc < 3) { + printf("usage: mars-log-impex {im,ex}port filename [dirname]\n"); + return -1; + } + + if (!strcmp(argv[1], "export")) { + char *out_dirname = NULL; + if (argc > 3 && argv[3]) + out_dirname = argv[3]; + return export_logfile(argv[2], out_dirname); + } + if (!strcmp(argv[1], "import") && argc > 3) { + return import_logfile(argv[3], argv[2]); + } + + return 0; +}