Merge pull request #3292 from kylinstorage/rbd-merge-diff-v2

rbd: merge diff files

Reviewed-by: Josh Durgin <jdurgin@redhat.com>
Reviewed-by: Sage Weil <sage@redhat.com>
This commit is contained in:
Josh Durgin 2015-01-16 12:08:02 -08:00
commit a845139d1f
4 changed files with 870 additions and 3 deletions

View File

@ -195,6 +195,14 @@ Commands
metadata about image size changes, and the start and end snapshots. It efficiently represents
discarded or 'zero' regions of the image.
:command:`merge-diff` [*first-diff-path*] [*second-diff-path*] [*merged-diff-path*]
Merge two continuous incremental diffs of an image into one single diff. The
first diff's end snapshot must be equal with the second diff's start snapshot.
The first diff could be - for stdin, and merged diff could be - for stdout, which
enables multiple diff files to be merged using something like
'rbd merge-diff first second - | rbd merge-diff - third result'. Note this command
currently only support the source incremental diff with stripe_count == 1
:command:`import-diff` [*src-path*] [*image-name*]
Imports an incremental diff of an image and applies it to the current image. If the diff
was generated relative to a start snapshot, we verify that snapshot already exists before

475
qa/workunits/rbd/merge_diff.sh Executable file
View File

@ -0,0 +1,475 @@
#!/bin/bash
pool=rbd
gen=$pool/gen
out=$pool/out
set -x
mkdir -p merge_diff_test
pushd merge_diff_test
function expect_false()
{
if "$@"; then return 1; else return 0; fi
}
function clear_all()
{
umount mnt || true
while [ 1 ];
do
rbd snap purge $gen 2>/dev/null >/dev/null
rbd rm $gen 2>/dev/null >/dev/null
rbd snap purge $out 2>/dev/null >/dev/null
rbd rm $out 2>/dev/null >/dev/null
sleep 5
rbd info $gen 2>/dev/null >/dev/null && continue
rbd info $out 2>/dev/null >/dev/null && continue
break
done
rm -rf diffs
}
function rebuild()
{
clear_all
rbd create $gen --size 100 --order $1 --stripe_unit $2 --stripe_count $3 --image-format $4
rbd create $out --size 1 --order 19
mkdir -p mnt diffs
rbd-fuse -p $pool mnt
}
function write()
{
dd if=/dev/urandom of=mnt/gen bs=1M conv=notrunc seek=$1 count=$2
}
function snap()
{
rbd snap create $gen@$1
}
function resize()
{
rbd resize $gen --size $1 --allow-shrink
}
function export_diff()
{
if [ $2 == "head" ]; then
target="$gen"
else
target="$gen@$2"
fi
if [ $1 == "null" ]; then
rbd export-diff $target diffs/$1.$2
else
rbd export-diff $target --from-snap $1 diffs/$1.$2
fi
}
function merge_diff()
{
rbd merge-diff diffs/$1.$2 diffs/$2.$3 diffs/$1.$3
}
function check()
{
rbd import-diff diffs/$1.$2 $out || return -1
if [ "$2" == "head" ]; then
sum1=`rbd export $gen - | md5sum`
else
sum1=`rbd export $gen@$2 - | md5sum`
fi
sum2=`rbd export $out - | md5sum`
if [ "$sum1" != "$sum2" ]; then
exit -1
fi
if [ "$2" != "head" ]; then
rbd snap ls $out | awk '{print $2}' | grep "^$2\$" || return -1
fi
}
#test f/t header
rebuild 22 4194304 1 2
write 0 1
snap a
write 1 1
export_diff null a
export_diff a head
merge_diff null a head
check null head
rebuild 22 4194304 1 2
write 0 1
snap a
write 1 1
snap b
write 2 1
export_diff null a
export_diff a b
export_diff b head
merge_diff null a b
check null b
rebuild 22 4194304 1 2
write 0 1
snap a
write 1 1
snap b
write 2 1
export_diff null a
export_diff a b
export_diff b head
merge_diff a b head
check null a
check a head
rebuild 22 4194304 1 2
write 0 1
snap a
write 1 1
snap b
write 2 1
export_diff null a
export_diff a b
export_diff b head
rbd merge-diff diffs/null.a diffs/a.b - | rbd merge-diff - diffs/b.head - > diffs/null.head
check null head
#data test
rebuild 22 4194304 1 2
write 4 2
snap s101
write 0 3
write 8 2
snap s102
export_diff null s101
export_diff s101 s102
merge_diff null s101 s102
check null s102
rebuild 22 4194304 1 2
write 0 3
write 2 5
write 8 2
snap s201
write 0 2
write 6 3
snap s202
export_diff null s201
export_diff s201 s202
merge_diff null s201 s202
check null s202
rebuild 22 4194304 1 2
write 0 4
write 12 6
snap s301
write 0 6
write 10 5
write 16 4
snap s302
export_diff null s301
export_diff s301 s302
merge_diff null s301 s302
check null s302
rebuild 22 4194304 1 2
write 0 12
write 14 2
write 18 2
snap s401
write 1 2
write 5 6
write 13 3
write 18 2
snap s402
export_diff null s401
export_diff s401 s402
merge_diff null s401 s402
check null s402
rebuild 22 4194304 1 2
write 2 4
write 10 12
write 27 6
write 36 4
snap s501
write 0 24
write 28 4
write 36 4
snap s502
export_diff null s501
export_diff s501 s502
merge_diff null s501 s502
check null s502
rebuild 22 4194304 1 2
write 0 8
resize 5
snap r1
resize 20
write 12 8
snap r2
resize 8
write 4 4
snap r3
export_diff null r1
export_diff r1 r2
export_diff r2 r3
merge_diff null r1 r2
merge_diff null r2 r3
check null r3
rebuild 22 4194304 1 2
write 0 8
resize 5
snap r1
resize 20
write 12 8
snap r2
resize 8
write 4 4
snap r3
resize 10
snap r4
export_diff null r1
export_diff r1 r2
export_diff r2 r3
export_diff r3 r4
merge_diff null r1 r2
merge_diff null r2 r3
merge_diff null r3 r4
check null r4
rebuild 22 65536 8 2
write 0 32
snap r1
write 16 32
snap r2
export_diff null r1
export_diff r1 r2
expect_false merge_diff null r1 r2
rebuild 22 4194304 1 2
write 0 1
write 2 1
write 4 1
write 6 1
snap s1
write 1 1
write 3 1
write 5 1
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 1 1
write 3 1
write 5 1
snap s1
write 0 1
write 2 1
write 4 1
write 6 1
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 3
write 6 3
write 12 3
snap s1
write 1 1
write 7 1
write 13 1
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 3
write 6 3
write 12 3
snap s1
write 0 1
write 6 1
write 12 1
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 3
write 6 3
write 12 3
snap s1
write 2 1
write 8 1
write 14 1
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 1 1
write 7 1
write 13 1
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 1
write 6 1
write 12 1
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 2 1
write 8 1
write 14 1
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 3
write 6 3
write 12 3
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 2 4
write 8 4
write 14 4
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 4
write 6 4
write 12 4
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 6
write 6 6
write 12 6
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 3 6
write 9 6
write 15 6
snap s1
write 0 3
write 6 3
write 12 3
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 8
snap s1
resize 2
resize 100
snap s2
export_diff null s1
export_diff s1 s2
merge_diff null s1 s2
check null s2
rebuild 22 4194304 1 2
write 0 8
snap s1
resize 2
resize 100
snap s2
write 20 2
snap s3
export_diff null s1
export_diff s1 s2
export_diff s2 s3
merge_diff s1 s2 s3
check null s1
check s1 s3
#addme
clear_all
popd
rm -rf merge_diff_test
exit 0

View File

@ -104,6 +104,10 @@ void usage()
" export-diff <image-name> [--from-snap <snap-name>] <path>\n"
" export an incremental diff to\n"
" path, or \"-\" for stdout\n"
" merge-diff <diff1> <diff2> <path> merge <diff1> and <diff2> into\n"
" <path>, <diff1> could be \"-\"\n"
" for stdin, and <path> could be \"-\"\n"
" for stdout\n"
" import-diff <path> <image-name> import an incremental diff from\n"
" path or \"-\" for stdin\n"
" (cp | copy) <src> <dest> copy src image to dest\n"
@ -1713,6 +1717,351 @@ static int do_import_diff(librbd::Image &image, const char *path)
return r;
}
static int parse_diff_header(int fd, __u8 *tag, string *from, string *to, uint64_t *size)
{
int r;
{//header
char buf[strlen(RBD_DIFF_BANNER) + 1];
r = safe_read_exact(fd, buf, strlen(RBD_DIFF_BANNER));
if (r < 0)
return r;
buf[strlen(RBD_DIFF_BANNER)] = '\0';
if (strcmp(buf, RBD_DIFF_BANNER)) {
cerr << "invalid banner '" << buf << "', expected '" << RBD_DIFF_BANNER << "'" << std::endl;
return -EINVAL;
}
}
while (true) {
r = safe_read_exact(fd, tag, 1);
if (r < 0)
return r;
if (*tag == 'f') {
r = read_string(fd, 4096, from); // 4k limit to make sure we don't get a garbage string
if (r < 0)
return r;
dout(2) << " from snap " << *from << dendl;
} else if (*tag == 't') {
r = read_string(fd, 4096, to); // 4k limit to make sure we don't get a garbage string
if (r < 0)
return r;
dout(2) << " to snap " << *to << dendl;
} else if (*tag == 's') {
char buf[8];
r = safe_read_exact(fd, buf, 8);
if (r < 0)
return r;
bufferlist bl;
bl.append(buf, 8);
bufferlist::iterator p = bl.begin();
::decode(*size, p);
} else {
break;
}
}
return 0;
}
static int parse_diff_body(int fd, __u8 *tag, uint64_t *offset, uint64_t *length)
{
int r;
if (!(*tag)) {
r = safe_read_exact(fd, tag, 1);
if (r < 0)
return r;
}
if (*tag == 'e') {
offset = 0;
length = 0;
return 0;
}
if (*tag != 'w' && *tag != 'z')
return -ENOTSUP;
char buf[16];
r = safe_read_exact(fd, buf, 16);
if (r < 0)
return r;
bufferlist bl;
bl.append(buf, 16);
bufferlist::iterator p = bl.begin();
::decode(*offset, p);
::decode(*length, p);
if (!(*length))
return -ENOTSUP;
return 0;
}
/*
* fd: the diff file to read from
* pd: the diff file to be written into
*/
static int accept_diff_body(int fd, int pd, __u8 tag, uint64_t offset, uint64_t length)
{
if (tag == 'e')
return 0;
bufferlist bl;
::encode(tag, bl);
::encode(offset, bl);
::encode(length, bl);
int r;
r = bl.write_fd(pd);
if (r < 0)
return r;
if (tag == 'w') {
bufferptr bp = buffer::create(length);
r = safe_read_exact(fd, bp.c_str(), length);
if (r < 0)
return r;
bufferlist data;
data.append(bp);
r = data.write_fd(pd);
if (r < 0)
return r;
}
return 0;
}
/*
* Merge two diff files into one single file
* Note: It does not do the merging work if
* either of the source diff files is stripped,
* since which complicates the process and is
* rarely used
*/
static int do_merge_diff(const char *first, const char *second, const char *path)
{
MyProgressContext pc("Merging image diff");
int fd = -1, sd = -1, pd = -1, r;
string f_from, f_to;
string s_from, s_to;
uint64_t f_size, s_size, pc_size;
__u8 f_tag = 0, s_tag = 0;
uint64_t f_off = 0, f_len = 0;
uint64_t s_off = 0, s_len = 0;
bool f_end = false, s_end = false;
bool first_stdin = !strcmp(first, "-");
if (first_stdin) {
fd = 0;
} else {
fd = open(first, O_RDONLY);
if (fd < 0) {
r = -errno;
cerr << "rbd: error opening " << first << std::endl;
goto done;
}
}
sd = open(second, O_RDONLY);
if (sd < 0) {
r = -errno;
cerr << "rbd: error opening " << second << std::endl;
goto done;
}
if (strcmp(path, "-") == 0) {
pd = 1;
} else {
pd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
if (pd < 0) {
r = -errno;
cerr << "rbd: error create " << path << std::endl;
goto done;
}
}
//We just handle the case like 'banner, [ftag], [ttag], stag, [wztag]*,etag',
// and the (offset,length) in wztag must be ascending order.
r = parse_diff_header(fd, &f_tag, &f_from, &f_to, &f_size);
if (r < 0)
goto done;
r = parse_diff_header(sd, &s_tag, &s_from, &s_to, &s_size);
if (r < 0)
goto done;
if (f_to != s_from) {
r = -EINVAL;
cerr << "The first TO snapshot must be equal with the second FROM snapshot, aborting" << std::endl;
goto done;
}
{
// header
bufferlist bl;
bl.append(RBD_DIFF_BANNER, strlen(RBD_DIFF_BANNER));
__u8 tag;
if (f_from.size()) {
tag = 'f';
::encode(tag, bl);
::encode(f_from, bl);
}
if (s_to.size()) {
tag = 't';
::encode(tag, bl);
::encode(s_to, bl);
}
tag = 's';
::encode(tag, bl);
::encode(s_size, bl);
r = bl.write_fd(pd);
if (r < 0)
goto done;
}
if (f_size > s_size)
pc_size = f_size << 1;
else
pc_size = s_size << 1;
//data block
while (!f_end || !s_end) {
// progress through input
pc.update_progress(f_off + s_off, pc_size);
if (!f_end && !f_len) {
uint64_t last_off = f_off;
r = parse_diff_body(fd, &f_tag, &f_off, &f_len);
if (r < 0)
goto done;
if (f_tag == 'e') {
f_end = true;
f_tag = 'z';
f_off = f_size;
if (f_size < s_size)
f_len = s_size - f_size;
else
f_len = 0;
}
if (last_off > f_off) {
r = -ENOTSUP;
goto done;
}
}
if (!s_end && !s_len) {
uint64_t last_off = s_off;
r = parse_diff_body(sd, &s_tag, &s_off, &s_len);
if (r < 0)
goto done;
if (s_tag == 'e') {
s_end = true;
s_off = s_size;
if (s_size < f_size)
s_len = f_size - s_size;
else
s_len = 0;
}
if (last_off > s_off) {
r = -ENOTSUP;
goto done;
}
}
if (f_off < s_off && f_len) {
uint64_t delta = s_off - f_off;
if (delta > f_len)
delta = f_len;
r = accept_diff_body(fd, pd, f_tag, f_off, delta);
f_off += delta;
f_len -= delta;
if (!f_len) {
f_tag = 0;
continue;
}
}
assert(f_off >= s_off);
if (f_off < s_off + s_len && f_len) {
uint64_t delta = s_off + s_len - f_off;
if (delta > f_len)
delta = f_len;
if (f_tag == 'w') {
if (first_stdin) {
bufferptr bp = buffer::create(delta);
r = safe_read_exact(fd, bp.c_str(), delta);
if (r < 0)
goto done;
} else {
r = lseek(fd, delta, SEEK_CUR);
if(r < 0)
goto done;
}
}
f_off += delta;
f_len -= delta;
if (!f_len) {
f_tag = 0;
continue;
}
}
assert(f_off >= s_off + s_len);
if (s_len) {
r = accept_diff_body(sd, pd, s_tag, s_off, s_len);
s_off += s_len;
s_len = 0;
s_tag = 0;
} else
assert(f_end && s_end);
continue;
}
{//tail
__u8 tag = 'e';
bufferlist bl;
::encode(tag, bl);
r = bl.write_fd(pd);
}
done:
if (pd > 2)
close(pd);
if (sd > 2)
close(sd);
if (fd > 2)
close(fd);
if(r < 0) {
pc.fail();
if (pd > 2)
unlink(path);
} else
pc.finish();
return r;
}
static int do_copy(librbd::Image &src, librados::IoCtx& dest_pp,
const char *destname)
{
@ -1970,6 +2319,7 @@ enum {
OPT_LOCK_ADD,
OPT_LOCK_REMOVE,
OPT_BENCH_WRITE,
OPT_MERGE_DIFF,
};
static int get_cmd(const char *cmd, bool snapcmd, bool lockcmd)
@ -1996,6 +2346,8 @@ static int get_cmd(const char *cmd, bool snapcmd, bool lockcmd)
return OPT_EXPORT;
if (strcmp(cmd, "export-diff") == 0)
return OPT_EXPORT_DIFF;
if (strcmp(cmd, "merge-diff") == 0)
return OPT_MERGE_DIFF;
if (strcmp(cmd, "diff") == 0)
return OPT_DIFF;
if (strcmp(cmd, "import") == 0)
@ -2099,7 +2451,8 @@ int main(int argc, const char **argv)
*dest_poolname = NULL, *dest_snapname = NULL, *path = NULL,
*devpath = NULL, *lock_cookie = NULL, *lock_client = NULL,
*lock_tag = NULL, *output_format = "plain",
*fromsnapname = NULL;
*fromsnapname = NULL,
*first_diff = NULL, *second_diff = NULL;
bool lflag = false;
int pretty_format = 0;
long long stripe_unit = 0, stripe_count = 0;
@ -2288,6 +2641,9 @@ if (!set_conf_param(v, p1, p2, p3)) { \
case OPT_EXPORT_DIFF:
SET_CONF_PARAM(v, &imgname, &path, NULL);
break;
case OPT_MERGE_DIFF:
SET_CONF_PARAM(v, &first_diff, &second_diff, &path);
break;
case OPT_IMPORT:
case OPT_IMPORT_DIFF:
SET_CONF_PARAM(v, &path, &imgname, NULL);
@ -2392,7 +2748,8 @@ if (!set_conf_param(v, p1, p2, p3)) { \
opt_cmd != OPT_IMPORT &&
opt_cmd != OPT_IMPORT_DIFF &&
opt_cmd != OPT_UNMAP &&
opt_cmd != OPT_SHOWMAPPED && !imgname) {
opt_cmd != OPT_SHOWMAPPED &&
opt_cmd != OPT_MERGE_DIFF && !imgname) {
cerr << "rbd: image name was not specified" << std::endl;
return EXIT_FAILURE;
}
@ -2446,6 +2803,20 @@ if (!set_conf_param(v, p1, p2, p3)) { \
if (!dest_poolname)
dest_poolname = "rbd";
if (opt_cmd == OPT_MERGE_DIFF) {
if (!first_diff) {
cerr << "rbd: first diff was not specified" << std::endl;
return EXIT_FAILURE;
}
if (!second_diff) {
cerr << "rbd: second diff was not specified" << std::endl;
return EXIT_FAILURE;
}
if (!path) {
cerr << "rbd: path was not specified" << std::endl;
return EXIT_FAILURE;
}
}
if (opt_cmd == OPT_EXPORT && !path)
path = imgname;
@ -2474,7 +2845,8 @@ if (!set_conf_param(v, p1, p2, p3)) { \
bool talk_to_cluster = (opt_cmd != OPT_MAP &&
opt_cmd != OPT_UNMAP &&
opt_cmd != OPT_SHOWMAPPED);
opt_cmd != OPT_SHOWMAPPED &&
opt_cmd != OPT_MERGE_DIFF);
if (talk_to_cluster && rados.init_with_context(g_ceph_context) < 0) {
cerr << "rbd: couldn't initialize rados!" << std::endl;
return EXIT_FAILURE;
@ -2795,6 +3167,14 @@ if (!set_conf_param(v, p1, p2, p3)) { \
}
break;
case OPT_MERGE_DIFF:
r = do_merge_diff(first_diff, second_diff, path);
if (r < 0) {
cerr << "rbd: merge-diff error" << std::endl;
return -r;
}
break;
case OPT_IMPORT:
if (!path) {
cerr << "rbd: import requires pathname" << std::endl;

View File

@ -26,6 +26,10 @@
export-diff <image-name> [--from-snap <snap-name>] <path>
export an incremental diff to
path, or "-" for stdout
merge-diff <diff1> <diff2> <path> merge <diff1> and <diff2> into
<path>, <diff1> could be "-"
for stdin, and <path> could be "-"
for stdout
import-diff <path> <image-name> import an incremental diff from
path or "-" for stdin
(cp | copy) <src> <dest> copy src image to dest