diff --git a/qa/standalone/osd/osd-bluefs-volume-ops.sh b/qa/standalone/osd/osd-bluefs-volume-ops.sh index c0235ef6772..aedfbc9b5cb 100755 --- a/qa/standalone/osd/osd-bluefs-volume-ops.sh +++ b/qa/standalone/osd/osd-bluefs-volume-ops.sh @@ -400,6 +400,96 @@ function TEST_bluestore2() { wait_for_clean || return 1 } +function TEST_bluestore_expand() { + local dir=$1 + + local flimit=$(ulimit -n) + if [ $flimit -lt 1536 ]; then + echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." + fi + export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--bluestore_block_size=4294967296 " + CEPH_ARGS+="--bluestore_block_db_create=true " + CEPH_ARGS+="--bluestore_block_db_size=1073741824 " + CEPH_ARGS+="--bluestore_block_wal_create=false " + CEPH_ARGS+="--bluestore_fsck_on_mount=true " + CEPH_ARGS+="--osd_pool_default_size=1 " + CEPH_ARGS+="--osd_pool_default_min_size=1 " + CEPH_ARGS+="--bluestore_debug_enforce_settings=ssd " + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + osd_pid0=$(cat $dir/osd.0.pid) + + sleep 5 + create_pool foo 16 + + # write some objects + timeout 60 rados bench -p foo 30 write -b 4096 --no-cleanup #|| return 1 + sleep 5 + + total_space_before=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_total_bytes" ) + free_space_before=`ceph tell osd.0 bluestore bluefs device info | grep "BDEV_SLOW" -A 2 | grep free | cut -d':' -f 2 | cut -d"," -f 1 | cut -d' ' -f 2` + + # kill + while kill $osd_pid0; do sleep 1 ; done + ceph osd down 0 + + # destage allocation to file before expand (in case fast-shutdown skipped that step) + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 allocmap || return 1 + + # expand slow devices + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 fsck || return 1 + + requested_space=4294967296 # 4GB + truncate $dir/0/block -s $requested_space + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 bluefs-bdev-expand || return 1 + + # slow, DB, WAL -> slow, DB + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 fsck || return 1 + + # compare allocation-file with RocksDB state + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 qfsck || return 1 + + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 bluefs-bdev-sizes + + activate_osd $dir 0 || return 1 + osd_pid0=$(cat $dir/osd.0.pid) + + wait_for_clean || return 1 + + total_space_after=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_total_bytes" ) + free_space_after=`ceph tell osd.0 bluestore bluefs device info | grep "BDEV_SLOW" -A 2 | grep free | cut -d':' -f 2 | cut -d"," -f 1 | cut -d' ' -f 2` + + if [$total_space_after != $requested_space]; then + echo "total_space_after = $total_space_after" + echo "requested_space = $requested_space" + return 1; + fi + + total_space_added=$((total_space_after - total_space_before)) + free_space_added=$((free_space_after - free_space_before)) + + let new_used_space=($total_space_added - $free_space_added) + echo $new_used_space + # allow upto 128KB to be consumed + if [ $new_used_space -gt 131072 ]; then + echo "total_space_added = $total_space_added" + echo "free_space_added = $free_space_added" + return 1; + fi + + # kill + while kill $osd_pid0; do sleep 1 ; done + ceph osd down 0 + + ceph-bluestore-tool --log-file $dir/bluestore_tool.log --path $dir/0 qfsck || return 1 +} + main osd-bluefs-volume-ops "$@" # Local Variables: diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 89cc0713a44..7e5776cc4b0 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -6600,7 +6600,8 @@ void BlueStore::_close_db() dout(10) << __func__ << ":read_only=" << db_was_opened_read_only << " fm=" << fm << " destage_alloc_file=" << need_to_destage_allocation_file << dendl; _close_db_leave_bluefs(); - if (fm && fm->is_null_manager() && !db_was_opened_read_only && need_to_destage_allocation_file) { + if (need_to_destage_allocation_file) { + ceph_assert(fm && fm->is_null_manager()); int ret = store_allocator(alloc); if (ret != 0) { derr << __func__ << "::NCB::store_allocator() failed (continue with bitmapFreelistManager)" << dendl; @@ -7449,6 +7450,11 @@ int BlueStore::expand_devices(ostream& out) << std::endl; } } + + // we grow the allocation range, must reflect it in the allocation file + alloc->init_add_free(size0, size - size0); + need_to_destage_allocation_file = true; + _close_db_and_around(); // mount in read/write to sync expansion changes diff --git a/src/os/bluestore/bluestore_tool.cc b/src/os/bluestore/bluestore_tool.cc index 6cf5d3dd1a4..2f65042a050 100644 --- a/src/os/bluestore/bluestore_tool.cc +++ b/src/os/bluestore/bluestore_tool.cc @@ -565,7 +565,7 @@ int main(int argc, char **argv) #endif } else if (action == "allocmap") { -#ifndef CEPH_BLUESTORE_TOOL_ENABLE_ALLOCMAP +#ifdef CEPH_BLUESTORE_TOOL_DISABLE_ALLOCMAP cerr << action << " bluestore.allocmap is not supported!!! " << std::endl; exit(EXIT_FAILURE); #else