mirror of
https://github.com/ceph/ceph
synced 2025-01-28 05:53:37 +00:00
b12780667f
The QoS parameters (res, wgt, lim) of mClock profiles are not allowed to be modified by users using commands like "config set" or via admin socket. handle_conf_change() does not allow changes to any built-in mClock profile at the mClock scheduler. But the config subsystem showed the change as expected for the built-in mClock profile QoS parameters. This misled the user into thinking that the change was made at the mClock server when it was not the case. The above issue is the result of the config "levels" used by the config subsystem. The inital built-in QoS params are set at the CONF_DEFAULT level. This allows the user to modify the built-in QoS params using "config set" command which sets values at CONF_MON level which has higher priority than CONF_DEFAULT level. The new value is persisted on the mon store and therefore the config subsystem shows the change when "config show" command is issued. To prevent the above, this commit adds changes to restore the defaults set for the built-in profiles by removing the new config changes from the MON store. This results in the original defaults to come back into effect and maintain a consistent view of the built-in profile across all levels. To accomplish this, the mClock scheduler is provided with additional information like the OSD id, shard id and a pointer to the MonClient using which the Mon store command to remove the option is executed. A standalone test is added to verify that built-in params cannot be modified and the original profile params are retained. Fixes: https://tracker.ceph.com/issues/57533 Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
315 lines
12 KiB
Bash
Executable File
315 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Copyright (C) 2022 Red Hat <contact@redhat.com>
|
|
#
|
|
# Author: Sridhar Seshasayee <sseshasa@redhat.com>
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU Library Public License as published by
|
|
# the Free Software Foundation; either version 2, or (at your option)
|
|
# any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library Public License for more details.
|
|
#
|
|
|
|
source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
|
|
|
|
function run() {
|
|
local dir=$1
|
|
shift
|
|
|
|
export CEPH_MON="127.0.0.1:7124" # git grep '\<7124\>' : there must be only one
|
|
export CEPH_ARGS
|
|
CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
|
|
CEPH_ARGS+="--mon-host=$CEPH_MON "
|
|
CEPH_ARGS+="--debug-mclock 20 "
|
|
|
|
local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
|
|
for func in $funcs ; do
|
|
setup $dir || return 1
|
|
$func $dir || return 1
|
|
teardown $dir || return 1
|
|
done
|
|
}
|
|
|
|
function TEST_profile_builtin_to_custom() {
|
|
local dir=$1
|
|
local OSDS=3
|
|
|
|
setup $dir || return 1
|
|
run_mon $dir a --osd_pool_default_size=$OSDS || return 1
|
|
run_mgr $dir x || return 1
|
|
for osd in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
run_osd $dir $osd --osd_op_queue=mclock_scheduler || return 1
|
|
done
|
|
|
|
# Verify that the default mclock profile is set on the OSDs
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
local mclock_profile=$(ceph config get osd.$id osd_mclock_profile)
|
|
test "$mclock_profile" = "high_client_ops" || return 1
|
|
done
|
|
|
|
# Change the mclock profile to 'custom'
|
|
ceph config set osd osd_mclock_profile custom || return 1
|
|
|
|
# Verify that the mclock profile is set to 'custom' on the OSDs
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
local mclock_profile=$(ceph config get osd.$id osd_mclock_profile)
|
|
test "$mclock_profile" = "custom" || return 1
|
|
done
|
|
|
|
# Change a mclock config param and confirm the change
|
|
local client_res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.$id) config get osd_mclock_scheduler_client_res | \
|
|
jq .osd_mclock_scheduler_client_res | bc)
|
|
echo "client_res = $client_res"
|
|
local client_res_new=$(expr $client_res + 10)
|
|
echo "client_res_new = $client_res_new"
|
|
ceph config set osd osd_mclock_scheduler_client_res \
|
|
$client_res_new || return 1
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
# Check value in config monitor db
|
|
local res=$(ceph config get osd.$id \
|
|
osd_mclock_scheduler_client_res) || return 1
|
|
test $res -eq $client_res_new || return 1
|
|
# Check value in the in-memory 'values' map
|
|
res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.$id) config get osd_mclock_scheduler_client_res | \
|
|
jq .osd_mclock_scheduler_client_res | bc)
|
|
test $res -eq $client_res_new || return 1
|
|
done
|
|
|
|
teardown $dir || return 1
|
|
}
|
|
|
|
function TEST_profile_custom_to_builtin() {
|
|
local dir=$1
|
|
local OSDS=3
|
|
|
|
setup $dir || return 1
|
|
run_mon $dir a --osd_pool_default_size=$OSDS || return 1
|
|
run_mgr $dir x || return 1
|
|
for osd in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
run_osd $dir $osd --osd_op_queue=mclock_scheduler || return 1
|
|
done
|
|
|
|
# Verify that the default mclock profile is set on the OSDs
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
local mclock_profile=$(ceph config get osd.$id osd_mclock_profile)
|
|
test "$mclock_profile" = "high_client_ops" || return 1
|
|
done
|
|
|
|
# Change the mclock profile to 'custom'
|
|
ceph config set osd osd_mclock_profile custom || return 1
|
|
|
|
# Verify that the mclock profile is set to 'custom' on the OSDs
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
local mclock_profile=$(ceph config get osd.$id osd_mclock_profile)
|
|
test "$mclock_profile" = "custom" || return 1
|
|
done
|
|
|
|
# Save the original client reservations allocated to the OSDs
|
|
local client_res=()
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
client_res+=( $(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.$id) config get osd_mclock_scheduler_client_res | \
|
|
jq .osd_mclock_scheduler_client_res | bc) )
|
|
echo "Original client_res for osd.$id = ${client_res[$id]}"
|
|
done
|
|
|
|
# Change a mclock config param and confirm the change
|
|
local client_res_new=$(expr ${client_res[0]} + 10)
|
|
echo "client_res_new = $client_res_new"
|
|
ceph config set osd osd_mclock_scheduler_client_res \
|
|
$client_res_new || return 1
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
# Check value in config monitor db
|
|
local res=$(ceph config get osd.$id \
|
|
osd_mclock_scheduler_client_res) || return 1
|
|
test $res -eq $client_res_new || return 1
|
|
# Check value in the in-memory 'values' map
|
|
res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.$id) config get osd_mclock_scheduler_client_res | \
|
|
jq .osd_mclock_scheduler_client_res | bc)
|
|
test $res -eq $client_res_new || return 1
|
|
done
|
|
|
|
# Switch the mclock profile back to the original built-in profile.
|
|
# The config subsystem prevents the overwrite of the changed QoS config
|
|
# option above i.e. osd_mclock_scheduler_client_res. This fact is verified
|
|
# before proceeding to remove the entry from the config monitor db. After
|
|
# the config entry is removed, the original value for the config option is
|
|
# restored and is verified.
|
|
ceph config set osd osd_mclock_profile high_client_ops || return 1
|
|
# Verify that the mclock profile is set to 'high_client_ops' on the OSDs
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
local mclock_profile=$(ceph config get osd.$id osd_mclock_profile)
|
|
test "$mclock_profile" = "high_client_ops" || return 1
|
|
done
|
|
|
|
# Verify that the new value is still in effect
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
# Check value in config monitor db
|
|
local res=$(ceph config get osd.$id \
|
|
osd_mclock_scheduler_client_res) || return 1
|
|
test $res -eq $client_res_new || return 1
|
|
# Check value in the in-memory 'values' map
|
|
res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.$id) config get osd_mclock_scheduler_client_res | \
|
|
jq .osd_mclock_scheduler_client_res | bc)
|
|
test $res -eq $client_res_new || return 1
|
|
done
|
|
|
|
# Remove the changed QoS config option from monitor db
|
|
ceph config rm osd osd_mclock_scheduler_client_res || return 1
|
|
|
|
# Verify that the original values are now restored
|
|
for id in $(seq 0 $(expr $OSDS - 1))
|
|
do
|
|
# Check value in the in-memory 'values' map
|
|
res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.$id) config get osd_mclock_scheduler_client_res | \
|
|
jq .osd_mclock_scheduler_client_res | bc)
|
|
test $res -eq ${client_res[$id]} || return 1
|
|
done
|
|
|
|
teardown $dir || return 1
|
|
}
|
|
|
|
function TEST_recovery_limit_adjustment_mclock() {
|
|
local dir=$1
|
|
|
|
setup $dir || return 1
|
|
run_mon $dir a || return 1
|
|
run_mgr $dir x || return 1
|
|
|
|
run_osd $dir 0 --osd_op_queue=mclock_scheduler || return 1
|
|
local recoveries=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
|
|
config get osd_recovery_max_active)
|
|
# Get default value
|
|
echo "$recoveries" | grep --quiet 'osd_recovery_max_active' || return 1
|
|
|
|
# Change the recovery limit without setting
|
|
# osd_mclock_override_recovery_settings option. Verify that the recovery
|
|
# limit is retained at its default value.
|
|
ceph config set osd.0 osd_recovery_max_active 10 || return 1
|
|
sleep 2 # Allow time for change to take effect
|
|
local max_recoveries=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
|
|
config get osd_recovery_max_active)
|
|
test "$max_recoveries" = "$recoveries" || return 1
|
|
|
|
# Change recovery limit after setting osd_mclock_override_recovery_settings.
|
|
# Verify that the recovery limit is modified.
|
|
ceph config set osd.0 osd_mclock_override_recovery_settings true || return 1
|
|
ceph config set osd.0 osd_recovery_max_active 10 || return 1
|
|
sleep 2 # Allow time for change to take effect
|
|
max_recoveries=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
|
|
config get osd_recovery_max_active)
|
|
test "$max_recoveries" = '{"osd_recovery_max_active":"10"}' || return 1
|
|
|
|
teardown $dir || return 1
|
|
}
|
|
|
|
function TEST_backfill_limit_adjustment_mclock() {
|
|
local dir=$1
|
|
|
|
setup $dir || return 1
|
|
run_mon $dir a || return 1
|
|
run_mgr $dir x || return 1
|
|
|
|
run_osd $dir 0 --osd_op_queue=mclock_scheduler || return 1
|
|
local backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
|
|
config get osd_max_backfills)
|
|
# Get default value
|
|
echo "$backfills" | grep --quiet 'osd_max_backfills' || return 1
|
|
|
|
# Change the backfill limit without setting
|
|
# osd_mclock_override_recovery_settings option. Verify that the backfill
|
|
# limit is retained at its default value.
|
|
ceph config set osd.0 osd_max_backfills 20 || return 1
|
|
sleep 2 # Allow time for change to take effect
|
|
local max_backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
|
|
config get osd_max_backfills)
|
|
test "$max_backfills" = "$backfills" || return 1
|
|
|
|
# Change backfills limit after setting osd_mclock_override_recovery_settings.
|
|
# Verify that the backfills limit is modified.
|
|
ceph config set osd.0 osd_mclock_override_recovery_settings true || return 1
|
|
ceph config set osd.0 osd_max_backfills 20 || return 1
|
|
sleep 2 # Allow time for change to take effect
|
|
max_backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
|
|
config get osd_max_backfills)
|
|
test "$max_backfills" = '{"osd_max_backfills":"20"}' || return 1
|
|
|
|
teardown $dir || return 1
|
|
}
|
|
|
|
function TEST_profile_disallow_builtin_params_modify() {
|
|
local dir=$1
|
|
|
|
setup $dir || return 1
|
|
run_mon $dir a || return 1
|
|
run_mgr $dir x || return 1
|
|
|
|
run_osd $dir 0 --osd_op_queue=mclock_scheduler || return 1
|
|
|
|
# Verify that the default mclock profile is set on the OSD
|
|
local mclock_profile=$(ceph config get osd.0 osd_mclock_profile)
|
|
test "$mclock_profile" = "high_client_ops" || return 1
|
|
|
|
declare -a options=("osd_mclock_scheduler_background_recovery_res"
|
|
"osd_mclock_scheduler_client_res")
|
|
|
|
for opt in "${options[@]}"
|
|
do
|
|
# Try and change a mclock config param and confirm that no change occurred
|
|
local opt_val_orig=$(CEPH_ARGS='' ceph --format=json daemon \
|
|
$(get_asok_path osd.0) config get $opt | jq .$opt | bc)
|
|
local opt_val_new=$(expr $opt_val_orig + 10)
|
|
ceph config set osd.0 $opt $opt_val_new || return 1
|
|
sleep 2 # Allow time for changes to take effect
|
|
|
|
# Check configuration value on Mon store (or the default) for the osd
|
|
local res=$(ceph config get osd.0 $opt) || return 1
|
|
echo "Mon db (or default): osd.0 $opt = $res"
|
|
test $res -ne $opt_val_new || return 1
|
|
|
|
# Check running configuration value using "config show" cmd
|
|
res=$(ceph config show osd.0 | grep $opt |\
|
|
awk '{ print $2 }' | bc ) || return 1
|
|
echo "Running config: osd.0 $opt = $res"
|
|
test $res -ne $opt_val_new || return 1
|
|
test $res -eq $opt_val_orig || return 1
|
|
|
|
# Check value in the in-memory 'values' map is unmodified
|
|
res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \
|
|
osd.0) config get $opt | jq .$opt | bc)
|
|
echo "Values map: osd.0 $opt = $res"
|
|
test $res -ne $opt_val_new || return 1
|
|
test $res -eq $opt_val_orig || return 1
|
|
done
|
|
|
|
teardown $dir || return 1
|
|
}
|
|
|
|
main mclock-config "$@"
|
|
|
|
# Local Variables:
|
|
# compile-command: "cd build ; make -j4 && \
|
|
# ../qa/run-standalone.sh mclock-config.sh"
|
|
# End:
|