From 5b06232b13caa958a33d743b0b69908cc20404de Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Mon, 30 Oct 2023 09:50:29 +0000 Subject: [PATCH] qa/workunits: adjust the existing dedup test based on ceph-dedup-daemon Signed-off-by: Myoungwon Oh --- qa/workunits/rados/test_dedup_tool.sh | 44 ++++++++++++++++++++--- src/tools/ceph_dedup/ceph_dedup_daemon.cc | 10 ++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh index 381094c57c2..8b04dc6f142 100755 --- a/qa/workunits/rados/test_dedup_tool.sh +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -34,11 +34,13 @@ if [ -n "$CEPH_BIN" ] ; then RADOS_TOOL="$CEPH_BIN/rados" CEPH_TOOL="$CEPH_BIN/ceph" DEDUP_TOOL="$CEPH_BIN/ceph-dedup-tool" + DEDUP_DAEMON="$CEPH_BIN/ceph-dedup-daemon" else # executables should be installed by the QA env RADOS_TOOL=$(which rados) CEPH_TOOL=$(which ceph) DEDUP_TOOL=$(which ceph-dedup-tool) + DEDUP_DAEMON=$(which ceph-dedup-daemon) fi POOL=dedup_pool @@ -374,7 +376,15 @@ function test_sample_dedup() sleep 2 # Execute dedup crawler - RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 3 --sampling-ratio 50) + $DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 3 --sampling-ratio 50 --run-once + sleep 2 + PID=$(pidof ceph-dedup-daemon) + COUNT=1 + while [ -n "$PID" ] && [ $COUNT -le 30 ]; do + sleep 15 + PID=$(pidof ceph-dedup-daemon) + ((COUNT++)) + done CHUNK_OID_1=$(echo $CONTENT_1 | sha1sum | awk '{print $1}') CHUNK_OID_3=$(echo $CONTENT_3 | sha1sum | awk '{print $1}') @@ -395,6 +405,8 @@ function test_sample_dedup() die "Chunk object has no reference of first meta object" fi + sleep 2 + # 7 Duplicated objects but less than chunk dedup threshold CONTENT_2="There hiHI2" echo $CONTENT_2 > foo2 @@ -404,7 +416,15 @@ function test_sample_dedup() done CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}') - RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2) + RESULT=$($DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2 --max-thread 1 --run-once) + sleep 2 + PID=$(pidof ceph-dedup-daemon) + COUNT=1 + while [ -n "$PID" ] && [ $COUNT -le 30 ]; do + sleep 15 + PID=$(pidof ceph-dedup-daemon) + ((COUNT++)) + done # Objects duplicates less than chunk dedup threshold should be deduplicated because of they satisfies object-dedup-threshold # The only object, which is crawled at the very first, should not be deduplicated because it was not duplicated at initial time @@ -486,7 +506,15 @@ function test_sample_dedup_snap() sleep 2 # Execute dedup crawler - RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 1 --sampling-ratio 100 --snap) + RESULT=$($DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 1 --snap --run-once) + sleep 2 + PID=$(pidof ceph-dedup-daemon) + COUNT=1 + while [ -n "$PID" ] && [ $COUNT -le 20 ]; do + sleep 5 + PID=$(pidof ceph-dedup-daemon) + ((COUNT++)) + done CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}') SNAP_CONTENT_OID=$(echo $SNAP_CONTENT | sha1sum | awk '{print $1}') @@ -558,7 +586,15 @@ function test_dedup_memory_limit() sleep 2 # Execute dedup crawler - RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 2 --sampling-ratio 100 --fpstore-threshold 100) + RESULT=$($DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2 --run-once) + sleep 2 + PID=$(pidof ceph-dedup-daemon) + COUNT=1 + while [ -n "$PID" ] && [ $COUNT -le 30 ]; do + sleep 15 + PID=$(pidof ceph-dedup-daemon) + ((COUNT++)) + done CHUNK_OID_1=$(echo $CONTENT_1 | sha1sum | awk '{print $1}') CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}') diff --git a/src/tools/ceph_dedup/ceph_dedup_daemon.cc b/src/tools/ceph_dedup/ceph_dedup_daemon.cc index aa1c7b734dd..463ae789910 100644 --- a/src/tools/ceph_dedup/ceph_dedup_daemon.cc +++ b/src/tools/ceph_dedup/ceph_dedup_daemon.cc @@ -29,6 +29,7 @@ po::options_description make_usage() { ("sampling-ratio", po::value(), ": set the sampling ratio (percentile)") ("wakeup-period", po::value(), ": set the wakeup period of crawler thread (sec)") ("fpstore-threshold", po::value()->default_value(100_M), ": set max size of in-memory fingerprint store (bytes)") + ("run-once", ": do a single iteration for debug") ; desc.add(op_desc); return desc; @@ -581,6 +582,7 @@ int make_crawling_daemon(const po::variables_map &opts) string chunk_pool_name = get_opts_chunk_pool(opts); unsigned max_thread = get_opts_max_thread(opts); uint32_t report_period = get_opts_report_period(opts); + bool run_once = false; // for debug int sampling_ratio = -1; if (opts.count("sampling-ratio")) { @@ -645,6 +647,10 @@ int make_crawling_daemon(const po::variables_map &opts) return -EINVAL; } + if (opts.count("run-once")) { + run_once = true; + } + dout(0) << "ceph-dedup-daemon starts ( " << "SampleRatio : " << sampling_ratio << ", Chunk Dedup Threshold : " << chunk_dedup_threshold @@ -719,6 +725,10 @@ int make_crawling_daemon(const po::variables_map &opts) } l.lock(); + if (run_once) { + all_stop = true; + break; + } } l.unlock();