From ee45f46b3fee3ab3307e6269aa91ad74b07718d5 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Wed, 27 Oct 2021 15:52:10 +0900 Subject: [PATCH 1/5] tool/ceph-dedup-tool: add chunk-dedup command From perspective user who want to use deduplication, it is hard to know how to use dedup feature. So, providing chunk-dedup might be helpful to use deduplication. Signed-off-by: Myoungwon Oh --- src/tools/ceph_dedup_tool.cc | 145 +++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index adfaa3839f0..856704bdec1 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -142,6 +142,7 @@ void usage() " [--op chunk-put-ref --chunk-pool POOL --object OID --target-ref OID --target-ref-pool-id POOL_ID] \n" " [--op chunk-repair --chunk-pool POOL --object OID --target-ref OID --target-ref-pool-id POOL_ID] \n" " [--op dump-chunk-refs --chunk-pool POOL --object OID] \n" +" [--op chunk-dedup --pool POOL --object OID --chunk-pool POOL --fingerprint-algorithm FP --source-off OFFSET --source-length LENGTH] \n" << std::endl; cout << "optional arguments: " << std::endl; cout << " --object " << std::endl; @@ -951,6 +952,142 @@ out: return (ret < 0) ? 1 : 0; } +int make_dedup_object(const std::map < std::string, std::string > &opts, + std::vector &nargs) +{ + Rados rados; + IoCtx io_ctx, chunk_io_ctx; + std::string object_name, chunk_pool_name, op_name, pool_name; + int ret; + std::map::const_iterator i; + + i = opts.find("op_name"); + if (i != opts.end()) { + op_name = i->second; + } else { + cerr << "must specify op" << std::endl; + exit(1); + } + i = opts.find("pool"); + if (i != opts.end()) { + pool_name = i->second; + } else { + cerr << "must specify --pool" << std::endl; + exit(1); + } + i = opts.find("object"); + if (i != opts.end()) { + object_name = i->second; + } else { + cerr << "must specify object" << std::endl; + exit(1); + } + + i = opts.find("chunk-pool"); + if (i != opts.end()) { + chunk_pool_name = i->second; + } else { + cerr << "must specify --chunk-pool" << std::endl; + exit(1); + } + i = opts.find("pgid"); + boost::optional pgid(i != opts.end(), pg_t()); + + ret = rados.init_with_context(g_ceph_context); + if (ret < 0) { + cerr << "couldn't initialize rados: " << cpp_strerror(ret) << std::endl; + goto out; + } + ret = rados.connect(); + if (ret) { + cerr << "couldn't connect to cluster: " << cpp_strerror(ret) << std::endl; + ret = -1; + goto out; + } + ret = rados.ioctx_create(pool_name.c_str(), io_ctx); + if (ret < 0) { + cerr << "error opening pool " + << chunk_pool_name << ": " + << cpp_strerror(ret) << std::endl; + goto out; + } + ret = rados.ioctx_create(chunk_pool_name.c_str(), chunk_io_ctx); + if (ret < 0) { + cerr << "error opening pool " + << chunk_pool_name << ": " + << cpp_strerror(ret) << std::endl; + goto out; + } + + if (op_name == "chunk-dedup") { + uint64_t offset, length; + string chunk_object, fp_algo; + i = opts.find("source-off"); + if (i != opts.end()) { + if (rados_sistrtoll(i, &offset)) { + return -EINVAL; + } + } else { + cerr << "must specify --source-off" << std::endl; + exit(1); + } + i = opts.find("source-length"); + if (i != opts.end()) { + if (rados_sistrtoll(i, &length)) { + return -EINVAL; + } + } else { + cerr << "must specify --source-off" << std::endl; + exit(1); + } + i = opts.find("fingerprint-algorithm"); + if (i != opts.end()) { + fp_algo = i->second.c_str(); + if (fp_algo != "sha1" + && fp_algo != "sha256" && fp_algo != "sha512") { + cerr << "unrecognized fingerprint-algorithm " << fp_algo << std::endl; + exit(1); + } + } + // 1. make a copy from manifest object to chunk object + bufferlist bl; + ret = io_ctx.read(object_name, bl, length, offset); + if (ret < 0) { + cerr << " reading object in base pool fails : " << cpp_strerror(ret) << std::endl; + goto out; + } + chunk_object = [&fp_algo, &bl]() -> string { + if (fp_algo == "sha1") { + return ceph::crypto::digest(bl).to_str(); + } else if (fp_algo == "sha256") { + return ceph::crypto::digest(bl).to_str(); + } else if (fp_algo == "sha512") { + return ceph::crypto::digest(bl).to_str(); + } else { + assert(0 == "unrecognized fingerprint type"); + return {}; + } + }(); + ret = chunk_io_ctx.write(chunk_object, bl, length, offset); + if (ret < 0) { + cerr << " writing object in chunk pool fails : " << cpp_strerror(ret) << std::endl; + goto out; + } + // 2. call set_chunk + ObjectReadOperation op; + op.set_chunk(offset, length, chunk_io_ctx, chunk_object, 0, + CEPH_OSD_OP_FLAG_WITH_REFERENCE); + ret = io_ctx.operate(object_name, &op, NULL); + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + goto out; + } + } + +out: + return (ret < 0) ? 1 : 0; +} + int main(int argc, const char **argv) { auto args = argv_to_vec(argc, argv); @@ -1009,6 +1146,12 @@ int main(int argc, const char **argv) opts["min-chunk-size"] = val; } else if (ceph_argparse_witharg(args, i, &val, "--max-chunk-size", (char*)NULL)) { opts["max-chunk-size"] = val; + } else if (ceph_argparse_witharg(args, i, &val, "--chunk-object", (char*)NULL)) { + opts["chunk-object"] = val; + } else if (ceph_argparse_witharg(args, i, &val, "--source-off", (char*)NULL)) { + opts["source-off"] = val; + } else if (ceph_argparse_witharg(args, i, &val, "--source-length", (char*)NULL)) { + opts["source-length"] = val; } else if (ceph_argparse_flag(args, i, "--debug", (char*)NULL)) { opts["debug"] = "true"; } else { @@ -1028,6 +1171,8 @@ int main(int argc, const char **argv) op_name == "chunk-repair" || op_name == "dump-chunk-refs") { return chunk_scrub_common(opts, args); + } else if (op_name == "chunk-dedup") { + return make_dedup_object(opts, args); } else { cerr << "unrecognized op " << op_name << std::endl; exit(1); From e7e875c5474827eefdc2a36c76c577e4876faa33 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Wed, 27 Oct 2021 16:00:44 +0900 Subject: [PATCH 2/5] qa: add chunk-dedup test Signed-off-by: Myoungwon Oh --- qa/workunits/rados/test_dedup_tool.sh | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh index f8db0ed0e77..7537cabfda0 100755 --- a/qa/workunits/rados/test_dedup_tool.sh +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -242,9 +242,50 @@ function test_dedup_chunk_repair() $RADOS_TOOL -p $POOL rm bar } +function test_dedup_object() +{ + + CHUNK_POOL=dedup_chunk_pool + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + + echo "There hiHI" > foo + + $RADOS_TOOL -p $POOL put foo ./foo + + sleep 2 + + rados ls -p $CHUNK_POOL + + RESULT=$($DEDUP_TOOL --pool $POOL --op chunk-dedup --object foo --chunk-pool $CHUNK_POOL --source-off 0 --source-length 10 --fingerprint-algorithm sha1 ) + + POOL_ID=$($CEPH_TOOL osd pool ls detail | grep $POOL | awk '{print$2}') + CHUNK_OID=$(echo -n "There hiHI" | sha1sum | awk '{print $1}') + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep foo) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $RADOS_TOOL -p $CHUNK_POOL get $CHUNK_OID ./chunk + VERIFY=$(cat ./chunk | sha1sum | awk '{print $1}') + if [ "$CHUNK_OID" != "$VERIFY" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Comparing failed expecting chunk mismatch" + fi + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + rm -rf ./foo ./chunk + $RADOS_TOOL -p $POOL rm foo +} + test_dedup_ratio_fixed test_dedup_chunk_scrub test_dedup_chunk_repair +test_dedup_object $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it From daa27d6526e0506baafe653bfcce617a8adef7be Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Wed, 27 Oct 2021 19:07:27 +0900 Subject: [PATCH 3/5] tool/ceph-dedup-tool: add object-dedup command Signed-off-by: Myoungwon Oh --- src/tools/ceph_dedup_tool.cc | 133 +++++++++++++++++++++++++++++++---- 1 file changed, 121 insertions(+), 12 deletions(-) diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index 856704bdec1..cf88369a42b 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -143,6 +143,7 @@ void usage() " [--op chunk-repair --chunk-pool POOL --object OID --target-ref OID --target-ref-pool-id POOL_ID] \n" " [--op dump-chunk-refs --chunk-pool POOL --object OID] \n" " [--op chunk-dedup --pool POOL --object OID --chunk-pool POOL --fingerprint-algorithm FP --source-off OFFSET --source-length LENGTH] \n" +" [--op object-dedup --pool POOL --object OID --chunk-pool POOL --fingerprint-algorithm FP --dedup-cdc-chunk-size CHUNK_SIZE] \n" << std::endl; cout << "optional arguments: " << std::endl; cout << " --object " << std::endl; @@ -952,12 +953,24 @@ out: return (ret < 0) ? 1 : 0; } +string make_pool_str(string pool, string var, string val) +{ + return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool + + string("\",\"var\": \"") + var + string("\",\"val\": \"") + + val + string("\"}"); +} + +string make_pool_str(string pool, string var, int val) +{ + return make_pool_str(pool, var, stringify(val)); +} + int make_dedup_object(const std::map < std::string, std::string > &opts, std::vector &nargs) { Rados rados; IoCtx io_ctx, chunk_io_ctx; - std::string object_name, chunk_pool_name, op_name, pool_name; + std::string object_name, chunk_pool_name, op_name, pool_name, fp_algo; int ret; std::map::const_iterator i; @@ -1018,10 +1031,19 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, << cpp_strerror(ret) << std::endl; goto out; } + i = opts.find("fingerprint-algorithm"); + if (i != opts.end()) { + fp_algo = i->second.c_str(); + if (fp_algo != "sha1" + && fp_algo != "sha256" && fp_algo != "sha512") { + cerr << "unrecognized fingerprint-algorithm " << fp_algo << std::endl; + exit(1); + } + } if (op_name == "chunk-dedup") { uint64_t offset, length; - string chunk_object, fp_algo; + string chunk_object; i = opts.find("source-off"); if (i != opts.end()) { if (rados_sistrtoll(i, &offset)) { @@ -1040,15 +1062,6 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, cerr << "must specify --source-off" << std::endl; exit(1); } - i = opts.find("fingerprint-algorithm"); - if (i != opts.end()) { - fp_algo = i->second.c_str(); - if (fp_algo != "sha1" - && fp_algo != "sha256" && fp_algo != "sha512") { - cerr << "unrecognized fingerprint-algorithm " << fp_algo << std::endl; - exit(1); - } - } // 1. make a copy from manifest object to chunk object bufferlist bl; ret = io_ctx.read(object_name, bl, length, offset); @@ -1082,6 +1095,99 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, cerr << " operate fail : " << cpp_strerror(ret) << std::endl; goto out; } + } else if (op_name == "object-dedup") { + unsigned chunk_size; + i = opts.find("dedup-cdc-chunk-size"); + if (i != opts.end()) { + if (rados_sistrtoll(i, &chunk_size)) { + cerr << "unrecognized dedup_cdc_chunk_size " << chunk_size << std::endl; + return -EINVAL; + } + } + + bufferlist inbl; + ret = rados.mon_command( + make_pool_str(pool_name, "fingerprint_algorithm", fp_algo), + inbl, NULL, NULL); + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + return ret; + } + ret = rados.mon_command( + make_pool_str(pool_name, "dedup_tier", chunk_pool_name), + inbl, NULL, NULL); + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + return ret; + } + ret = rados.mon_command( + make_pool_str(pool_name, "dedup_chunk_algorithm", "fastcdc"), + inbl, NULL, NULL); + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + return ret; + } + ret = rados.mon_command( + make_pool_str(pool_name, "dedup_cdc_chunk_size", chunk_size), + inbl, NULL, NULL); + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + return ret; + } + + // convert object to manifest object + ObjectWriteOperation op; + bufferlist temp; + temp.append("temp"); + op.write_full(temp); + + auto gen_r_num = [] () -> string { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + uint64_t r_num = dist(gen); + return to_string(r_num); + }; + string temp_oid = gen_r_num(); + // create temp chunk object for set-chunk + ret = chunk_io_ctx.operate(temp_oid, &op); + if (ret == -EEXIST) { + // one more try + temp_oid = gen_r_num(); + ret = chunk_io_ctx.operate(temp_oid, &op); + } + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + goto out; + } + + // set-chunk to make manifest object + ObjectReadOperation chunk_op; + chunk_op.set_chunk(0, 4, chunk_io_ctx, temp_oid, 0, + CEPH_OSD_OP_FLAG_WITH_REFERENCE); + ret = io_ctx.operate(object_name, &chunk_op, NULL); + if (ret < 0) { + cerr << " set_chunk fail : " << cpp_strerror(ret) << std::endl; + goto out; + } + + // tier-flush to perform deduplication + ObjectReadOperation flush_op; + flush_op.tier_flush(); + ret = io_ctx.operate(object_name, &flush_op, NULL); + if (ret < 0) { + cerr << " tier_flush fail : " << cpp_strerror(ret) << std::endl; + goto out; + } + + // tier-evict + ObjectReadOperation evict_op; + evict_op.tier_evict(); + ret = io_ctx.operate(object_name, &evict_op, NULL); + if (ret < 0) { + cerr << " tier_evict fail : " << cpp_strerror(ret) << std::endl; + goto out; + } } out: @@ -1152,6 +1258,8 @@ int main(int argc, const char **argv) opts["source-off"] = val; } else if (ceph_argparse_witharg(args, i, &val, "--source-length", (char*)NULL)) { opts["source-length"] = val; + } else if (ceph_argparse_witharg(args, i, &val, "--dedup-cdc-chunk-size", (char*)NULL)) { + opts["dedup-cdc-chunk-size"] = val; } else if (ceph_argparse_flag(args, i, "--debug", (char*)NULL)) { opts["debug"] = "true"; } else { @@ -1171,7 +1279,8 @@ int main(int argc, const char **argv) op_name == "chunk-repair" || op_name == "dump-chunk-refs") { return chunk_scrub_common(opts, args); - } else if (op_name == "chunk-dedup") { + } else if (op_name == "chunk-dedup" || + op_name == "object-dedup") { return make_dedup_object(opts, args); } else { cerr << "unrecognized op " << op_name << std::endl; From 16e7d5578c45dafaf772c143fe7c380f7087f4cf Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Wed, 27 Oct 2021 19:08:20 +0900 Subject: [PATCH 4/5] qa: add object-dedup test Signed-off-by: Myoungwon Oh --- qa/workunits/rados/test_dedup_tool.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh index 7537cabfda0..85351f8a806 100755 --- a/qa/workunits/rados/test_dedup_tool.sh +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -262,6 +262,29 @@ function test_dedup_object() CHUNK_OID=$(echo -n "There hiHI" | sha1sum | awk '{print $1}') RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep foo) + + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $RADOS_TOOL -p $CHUNK_POOL get $CHUNK_OID ./chunk + VERIFY=$(cat ./chunk | sha1sum | awk '{print $1}') + if [ "$CHUNK_OID" != "$VERIFY" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Comparing failed expecting chunk mismatch" + fi + + echo -n "There hihiHI" > bar + + $RADOS_TOOL -p $POOL put bar ./bar + RESULT=$($DEDUP_TOOL --pool $POOL --op object-dedup --object bar --chunk-pool $CHUNK_POOL --fingerprint-algorithm sha1 --dedup-cdc-chunk-size 4096) + + CHUNK_OID=$(echo -n "There hihiHI" | sha1sum | awk '{print $1}') + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) if [ -z "$RESULT" ] ; then $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it @@ -278,8 +301,9 @@ function test_dedup_object() $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it - rm -rf ./foo ./chunk + rm -rf ./foo ./bar ./chunk $RADOS_TOOL -p $POOL rm foo + $RADOS_TOOL -p $POOL rm bar } test_dedup_ratio_fixed From 60b7690262d2b7c7caa96fa2f1f43542f2421882 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Tue, 14 Dec 2021 00:50:57 +0900 Subject: [PATCH 5/5] tools/ceph_dedup_tool: add explanations for added two commands Signed-off-by: Myoungwon Oh --- src/tools/ceph_dedup_tool.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index cf88369a42b..4b8c926f75e 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -155,6 +155,9 @@ void usage() cout << " --report-period " << std::endl; cout << " --max-seconds " << std::endl; cout << " --max-read-size " << std::endl; + cout << "explanations: " << std::endl; + cout << " chunk-dedup performs deduplication using a chunk generated by given source" << std::endl; + cout << " offset and length. object-dedup deduplicates the entire object, not a chunk" << std::endl; exit(1); } @@ -1135,6 +1138,12 @@ int make_dedup_object(const std::map < std::string, std::string > &opts, return ret; } + /* + * TODO: add a better way to make an object a manifest object. + * We're using set_chunk with an incorrect object here simply to make + * the object a manifest object, the tier_flush() will remove + * it and replace it with the real contents. + */ // convert object to manifest object ObjectWriteOperation op; bufferlist temp; @@ -1281,6 +1290,16 @@ int main(int argc, const char **argv) return chunk_scrub_common(opts, args); } else if (op_name == "chunk-dedup" || op_name == "object-dedup") { + /* + * chunk-dedup: + * using a chunk generated by given source, + * create a new object in the chunk pool or increase the reference + * if the object exists + * + * object-dedup: + * perform deduplication on the entire object, not a chunk. + * + */ return make_dedup_object(opts, args); } else { cerr << "unrecognized op " << op_name << std::endl;