RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-20 17:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

qa/workunits: adjust the existing dedup test based on ceph-dedup-daemon

Browse Source 
Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
This commit is contained in:
myoungwon oh 2023-10-30 09:50:29 +00:00
parent 2e05df00d4
commit 5b06232b13
2 changed files with 50 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
qa/workunits/rados/test_dedup_tool.sh
View File

@ -34,11 +34,13 @@ if [ -n "$CEPH_BIN" ] ; then
RADOS_TOOL="$CEPH_BIN/rados"
CEPH_TOOL="$CEPH_BIN/ceph"
DEDUP_TOOL="$CEPH_BIN/ceph-dedup-tool"
DEDUP_DAEMON="$CEPH_BIN/ceph-dedup-daemon"
else
# executables should be installed by the QA env
RADOS_TOOL=$(which rados)
CEPH_TOOL=$(which ceph)
DEDUP_TOOL=$(which ceph-dedup-tool)
DEDUP_DAEMON=$(which ceph-dedup-daemon)
fi
POOL=dedup_pool
@ -374,7 +376,15 @@ function test_sample_dedup()
sleep 2
# Execute dedup crawler
RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 3 --sampling-ratio 50)
$DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 3 --sampling-ratio 50 --run-once
sleep 2
PID=$(pidof ceph-dedup-daemon)
COUNT=1
while [ -n "$PID" ] && [ $COUNT -le 30 ]; do
sleep 15
PID=$(pidof ceph-dedup-daemon)
((COUNT++))
done
CHUNK_OID_1=$(echo $CONTENT_1 | sha1sum | awk '{print $1}')
CHUNK_OID_3=$(echo $CONTENT_3 | sha1sum | awk '{print $1}')
@ -395,6 +405,8 @@ function test_sample_dedup()
die "Chunk object has no reference of first meta object"
fi
sleep 2
# 7 Duplicated objects but less than chunk dedup threshold
CONTENT_2="There hiHI2"
echo $CONTENT_2 > foo2
@ -404,7 +416,15 @@ function test_sample_dedup()
done
CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}')
RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2)
RESULT=$($DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2 --max-thread 1 --run-once)
sleep 2
PID=$(pidof ceph-dedup-daemon)
COUNT=1
while [ -n "$PID" ] && [ $COUNT -le 30 ]; do
sleep 15
PID=$(pidof ceph-dedup-daemon)
((COUNT++))
done
# Objects duplicates less than chunk dedup threshold should be deduplicated because of they satisfies object-dedup-threshold
# The only object, which is crawled at the very first, should not be deduplicated because it was not duplicated at initial time
@ -486,7 +506,15 @@ function test_sample_dedup_snap()
sleep 2
# Execute dedup crawler
RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 1 --sampling-ratio 100 --snap)
RESULT=$($DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 1 --snap --run-once)
sleep 2
PID=$(pidof ceph-dedup-daemon)
COUNT=1
while [ -n "$PID" ] && [ $COUNT -le 20 ]; do
sleep 5
PID=$(pidof ceph-dedup-daemon)
((COUNT++))
done
CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}')
SNAP_CONTENT_OID=$(echo $SNAP_CONTENT | sha1sum | awk '{print $1}')
@ -558,7 +586,15 @@ function test_dedup_memory_limit()
sleep 2
# Execute dedup crawler
RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 2 --sampling-ratio 100 --fpstore-threshold 100)
RESULT=$($DEDUP_DAEMON --pool $POOL --chunk-pool $CHUNK_POOL --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2 --run-once)
sleep 2
PID=$(pidof ceph-dedup-daemon)
COUNT=1
while [ -n "$PID" ] && [ $COUNT -le 30 ]; do
sleep 15
PID=$(pidof ceph-dedup-daemon)
((COUNT++))
done
CHUNK_OID_1=$(echo $CONTENT_1 | sha1sum | awk '{print $1}')
CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}')

10
src/tools/ceph_dedup/ceph_dedup_daemon.cc
View File

@ -29,6 +29,7 @@ po::options_description make_usage() {
("sampling-ratio", po::value<int>(), ": set the sampling ratio (percentile)")
("wakeup-period", po::value<int>(), ": set the wakeup period of crawler thread (sec)")
("fpstore-threshold", po::value<size_t>()->default_value(100_M), ": set max size of in-memory fingerprint store (bytes)")
("run-once", ": do a single iteration for debug")
;
desc.add(op_desc);
return desc;
@ -581,6 +582,7 @@ int make_crawling_daemon(const po::variables_map &opts)
string chunk_pool_name = get_opts_chunk_pool(opts);
unsigned max_thread = get_opts_max_thread(opts);
uint32_t report_period = get_opts_report_period(opts);
bool run_once = false; // for debug
int sampling_ratio = -1;
if (opts.count("sampling-ratio")) {
@ -645,6 +647,10 @@ int make_crawling_daemon(const po::variables_map &opts)
return -EINVAL;
}
if (opts.count("run-once")) {
run_once = true;
}
dout(0) << "ceph-dedup-daemon starts ( "
<< "SampleRatio : " << sampling_ratio
<< ", Chunk Dedup Threshold : " << chunk_dedup_threshold
@ -719,6 +725,10 @@ int make_crawling_daemon(const po::variables_map &opts)
}
l.lock();
if (run_once) {
all_stop = true;
break;
}
}
l.unlock();