#!/bin/bash # # Common routines for all tests # # Temporary array for building the final command, injecting arguments as needed declare -a cmd_array # assert that argument is not empty and is an existing path (file or directory) _assert_path() { local path path="$1" if [ -z "$path" ]; then echo "ASSERTION FAIL: $path is not valid" exit 1 fi if [ -f "$path" -o -d "$path" -o -b "$path" ]; then return 0 fi echo "ASSERTION FAIL: $path is not valid" exit 1 } # $1: this string gets matched to files, absolute or relative path, or a # systemwide command available via $PATH _is_file_or_command() { local msg msg="$1" if [ -z "$msg" ]; then return 1 fi if [ -f "$msg" -o -d "$msg" -o -b "$msg" ]; then return 0 fi msg=$(type -p -- "$msg") if [ -f "$msg" ]; then return 0 fi return 1 } _fail() { echo "$*" | tee -a "$RESULTS" exit 1 } # log a message to the results file _log() { echo "$*" >> "$RESULTS" } # copy stdout to log and pass to stdout, eg. another stdout consumer, commands # should redirect stderr to stdout if this is consmed by further commands _log_stdout() { tee -a "$RESULTS" } # print a message to stdout and to log that something was skipped _log_skipped() { echo " [SKIPPED] $*" | tee -a "$RESULTS" exit 0 } _not_run() { echo " [NOTRUN] $*" exit 0 } # debugging helper _dump_args() { local i i=1 echo "DUMP args for ${FUNCNAME[1]}:" while [ $# -gt 0 ]; do echo "ARG[$i]: $1" i=$(($i+1)) shift done } # read arguments, look if we're calling btrfs and if there's a known # subcommand, return argument index to insert, taking root helper into # consideration, returns 2 for unknown subcommand _get_spec_ins() { if [ "$1" = 'root_helper' ]; then if [[ $2 =~ /btrfs$ ]]; then echo -n 4 return fi if [[ $2 =~ /mkfs.btrfs$ ]]; then echo -n 3 return fi if [[ $2 =~ /btrfs-convert$ ]]; then echo -n 3 return fi else if [[ $1 =~ /btrfs$ ]]; then echo -n 3 return fi if [[ $1 =~ /mkfs.btrfs$ ]]; then echo -n 2 return fi if [[ $1 =~ /btrfs-convert$ ]]; then echo -n 2 return fi fi echo -n 2 } # return command-specific arguments if enabled _cmd_spec() { if [ "$TEST_ENABLE_OVERRIDE" = 'true' ]; then # if defined via common.local, use it, otherwise pass make # arguments if [ "$(type -t _skip_spec)" = 'function' ]; then if _skip_spec "$@"; then return fi fi case "$1" in check) echo -n "$TEST_ARGS_CHECK" ;; */mkfs.btrfs) echo -n "$TEST_ARGS_MKFS" ;; */btrfs-convert) echo -n "$TEST_ARGS_CONVERT" ;; esac fi } # Check if $1 is a known command that may need parameter injection or some # special handling regarding $INSTRUMENT _is_target_command() { [[ $1 =~ /btrfs$ ]] && return 0 [[ $1 =~ /mkfs.btrfs$ ]] && return 0 [[ $1 =~ /btrfs-convert$ ]] && return 0 [[ $1 =~ /btrfs-corrupt-block$ ]] && return 0 [[ $1 =~ /btrfs-image$ ]] && return 0 [[ $1 =~ /btrfs-select-super$ ]] && return 0 [[ $1 =~ /btrfs-find-root$ ]] && return 0 [[ $1 =~ /btrfstune$ ]] && return 0 return 1 } # Expanding extra commands/options for current command string # This function is responsible for inserting the following things: # - @INSTRUMENT before 'btrfs' commands # To ensure that $INSTRUMENT is not executed for things like sudo/mount # which normally have setuid/setgid which will not be instrumented # # - Add extra arguments for 'btrfs check/mkfs.btrfs/btrfs-convert' # This allows us to test certain functionality like lowmem mode for 'btrfs # check' expand_command() { local command_index local ins local spec local i ins=$(_get_spec_ins "$@") spec=$(($ins-1)) spec=$(_cmd_spec "${@:$spec}") cmd_array=() if [ "$1" = 'root_helper' ] && _is_target_command "$2" ; then command_index=2 elif _is_target_command "$1" ; then command_index=1 else # Since we iterate from offset 1, this never get hit, thus we # won't insert $INSTRUMENT command_index=0 fi for ((i = 1; i <= $#; i++ )); do if [ $i -eq $command_index -a ! -z "$INSTRUMENT" ]; then cmd_array+=($INSTRUMENT) fi if [ $i -eq $ins -a ! -z "$spec" ]; then cmd_array+=("$spec") fi if [ $i -eq $command_index -a "$TEST_FLAVOR" = 'static' ]; then cmd_array+=("${!i}".static) else cmd_array+=("${!i}") fi done } # Argument passing magic: # the command passed to run_* helpers is inspected, if there's 'btrfs command' # found and there are defined additional arguments, they're inserted just after # the command name, ie. any arguments in the test could override them. # # The root helper is recognized. Unrecognized subcommands or external tools # are not affected. run_check() { expand_command "$@" echo "====== RUN CHECK ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD: ${cmd_array[@]}" > /dev/tty; fi "${cmd_array[@]}" >> "$RESULTS" 2>&1 || _fail "failed: ${cmd_array[@]}" } # same as run_check but the stderr+stdout output is duplicated on stdout and # can be processed further # # NOTE: If this function is called on btrfs related commands, caller should # filter the output, as INSTRUMENT can easily pollute the output. run_check_stdout() { expand_command "$@" echo "====== RUN CHECK ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(stdout): ${cmd_array[@]}" \ > /dev/tty; fi "${cmd_array[@]}" 2>&1 | tee -a "$RESULTS" if [ ${PIPESTATUS[0]} -ne 0 ]; then _fail "failed: $@" fi } # same as run_check but does not fail the test if it's handled gracefully by # the tool, unexpected failure like segfault or abort will exit forcibly # output is logged run_mayfail() { expand_command "$@" echo "====== RUN MAYFAIL ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mayfail): ${cmd_array[@]}" \ > /dev/tty; fi "${cmd_array[@]}" >> "$RESULTS" 2>&1 ret=$? if [ $ret != 0 ]; then echo "failed (ignored, ret=$ret): $@" >> "$RESULTS" if [ $ret == 139 ]; then _fail "mayfail: returned code 139 (SEGFAULT), not ignored" elif [ $ret == 134 ]; then _fail "mayfail: returned code 134 (SIGABRT), not ignored" fi return $ret fi } # Same as run_mayfail but does not fail the test if it's handled gracefully by # the caller, unexpected failure like segfault or abort will exit forcibly, # output is logged # # NOTE: we don't use pipefail to avoid disturbing the rest of the caller # script, so here we use a temporary output file. Pipes are not supported, # store the output to a variable for further processing. run_mayfail_stdout() { tmp_output=$(mktemp --tmpdir btrfs-progs-test--mayfail-stdtout.XXXXXX) expand_command "$@" echo "====== RUN MAYFAIL ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mayfail): ${cmd_array[@]}" \ > /dev/tty; fi "${cmd_array[@]}" 2>&1 > "$tmp_output" ret=$? cat "$tmp_output" >> "$RESULTS" cat "$tmp_output" rm -- "$tmp_output" if [ "$ret" != 0 ]; then echo "failed (ignored, ret=$ret): $@" >> "$RESULTS" if [ "$ret" == 139 ]; then _fail "mayfail: returned code 139 (SEGFAULT), not ignored" elif [ "$ret" == 134 ]; then _fail "mayfail: returned code 134 (SIGABRT), not ignored" fi return "$ret" fi # return the command code and let the caller decide what to do based # on the stdout return "$ret" } # first argument is error message to print if it fails, otherwise # same as run_check but expects the command to fail, output is logged run_mustfail() { local msg msg="$1" shift if _is_file_or_command "$msg"; then echo "ASSERTION FAIL: 1st argument of run_mustfail must be a message" exit 1 fi expand_command "$@" echo "====== RUN MUSTFAIL ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mustfail): ${cmd_array[@]}" \ > /dev/tty; fi "${cmd_array[@]}" >> "$RESULTS" 2>&1 if [ $? != 0 ]; then echo "failed (expected): $@" >> "$RESULTS" return 0 else echo "succeeded (unexpected!): $@" >> "$RESULTS" _fail "unexpected success: $msg" return 1 fi } # The first parameter is error message to print if it fails, just like # run_must_fail(). # NOTE: we don't use pipefail to avoid disturbing other script, so here we # use a temporary output file. # So it doesn't support pipeline in the @cmd run_mustfail_stdout() { local msg local ret local tmp_output tmp_output=$(mktemp --tmpdir btrfs-progs-test--mustfail-stdtout.XXXXXX) msg="$1" shift if _is_file_or_command "$msg"; then echo "ASSERTION FAIL: 1st argument of run_mustfail_stdout must be a message" exit 1 fi expand_command "$@" echo "====== RUN MUSTFAIL ${cmd_array[@]}" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mustfail): ${cmd_array[@]}" \ > /dev/tty; fi "${cmd_array[@]}" 2>&1 > "$tmp_output" ret=$? cat "$tmp_output" >> "$RESULTS" cat "$tmp_output" rm "$tmp_output" if [ "$ret" != 0 ]; then echo "failed (expected): $@" >> "$RESULTS" return 0 else echo "succeeded (unexpected!): $@" >> "$RESULTS" _fail "unexpected success: $msg" return 1 fi } check_prereq() { # Internal tools for testing, not shipped with the package case "$1" in btrfs-corrupt-block|btrfs-find-root|btrfs-select-super|fssum) if ! [ -f "$INTERNAL_BIN/$1" ]; then _fail "Failed prerequisites: $INTERNAL_BIN/$1"; fi ;; *) bin="$TOP/$1" if [ "$TEST_FLAVOR" = 'static' ]; then bin="${bin}.static" fi if ! [ -f "$bin" ]; then _fail "Failed prerequisites: $bin"; fi ;; esac } check_global_prereq() { type -p "$1" &> /dev/null if [ $? -ne 0 ]; then _fail "Failed system wide prerequisities: $1"; fi } # Check if dmsetup and the target passed as argument is available, and skip the # test if they aren't # $1: the target name, expectind module dm-$1 check_dm_target_support() { local target="$1" setup_root_helper type -p dmsetup &> /dev/null if [ $? -ne 0 ]; then _not_run "This test requires dmsetup tool"; fi $SUDO_HELPER modprobe "dm-$target" >/dev/null 2>&1 $SUDO_HELPER dmsetup targets 2>&1 | grep -q "^$target" if [ $? -ne 0 ]; then _not_run "This test requires dm-$target support" fi } check_image() { local image local tmp_output tmp_output=$(mktemp --tmpdir btrfs-progs-test-check-image.XXXXXX) image=$1 echo "testing image $(basename $image)" >> "$RESULTS" run_mustfail_stdout "btrfs check should have detected corruption" \ "$TOP/btrfs" check "$image" &> "$tmp_output" # Also make sure no subpage related warnings check_test_results "$tmp_output" "$testname" run_check "$TOP/btrfs" check --repair --force "$image" run_check_stdout "$TOP/btrfs" check "$image" &> "$tmp_output" # Also make sure no subpage related warnings for the repaired image check_test_results "$tmp_output" "$testname" rm -f -- "$tmp_output" } # Extract a usable image from packed formats # - raw btrfs filesystem images, suffix .raw # - dtto compressed by XZ, suffix .raw.xz # - meta-dump images with suffix .img # - dtto compressed by XZ, suffix .img.xz # - compressed send stream, .stream.xz extract_image() { local image local cleanme image="$1" case "$image" in *.img) rm -f "$image.restored" ;; *.img.xz) xz --decompress --keep --force "$image" || \ _fail "failed to decompress image $image" >&2 image=${image%%.xz} rm -f "$image.restored" cleanme=$image ;; *.raw) cp --sparse=auto --force "$image" "$image.restored" ;; *.raw.xz) xz --decompress --keep --force "$image" || \ _fail "failed to decompress image $image" >&2 image=${image%%.xz} mv "$image" "$image.restored" ;; *.stream.xz) xz --decompress --keep --force "$image" || \ _fail "failed to decompress file $image" >&2 image=${image%%.xz} mv "$image" "$image.restored" ;; esac if ! [ -f "$image.restored" ]; then echo "restoring image $(basename $image)" >> "$RESULTS" "$TOP/btrfs-image" -r "$image" "$image.restored" \ &>> "$RESULTS" \ || _fail "failed to restore image $image" >&2 fi [ -f "$cleanme" ] && rm -f "$cleanme" echo "$image.restored" } # Process all image dumps in a given directory check_all_images() { local dir local extracted dir="$1" if [ -z "$dir" ]; then dir=. fi _assert_path "$dir" for image in $(find "$dir" \( -iname '*.img' -o \ -iname '*.img.xz' -o \ -iname '*.raw' -o \ -iname '*.raw.xz' \) | sort) do extracted=$(extract_image "$image") check_image "$extracted" rm -f "$extracted" done } # some tests need to mount the recovered image and do verifications call # 'setup_root_helper' and then check for have_root_helper == 1 if the test # needs to fail otherwise; using sudo by default for now SUDO_HELPER= NEED_SUDO_VALIDATE=unknown export SUDO_HELPER export NEED_SUDO_VALIDATE root_helper() { if [ $UID -eq 0 ]; then "$@" else if [ "$NEED_SUDO_VALIDATE" = 'yes' ]; then sudo -v -n &>/dev/null || \ _not_run "Need to validate sudo credentials" sudo -n "$@" elif [ "$NEED_SUDO_VALIDATE" = 'no' ]; then sudo -n /bin/true &> /dev/null || \ _not_run "Need to validate sudo user settings" sudo -n "$@" else # should not happen _not_run "Need to validate root privileges" fi fi } setup_root_helper() { if [ $UID -eq 0 -o -n "$SUDO_HELPER" ]; then return fi # Test for old sudo or special settings, which make sudo -v fail even # if user setting is NOPASSWD sudo -n /bin/true &>/dev/null && NEED_SUDO_VALIDATE=no # Newer sudo or default sudo setting sudo -v -n &>/dev/null && NEED_SUDO_VALIDATE=yes if [ "$NEED_SUDO_VALIDATE" = 'unknown' ]; then _not_run "Need to validate root privileges" fi SUDO_HELPER=root_helper } prepare_test_dev() { # num[K/M/G/T...] local size="$1" [[ "$size" ]] || size='2G' # Still truncate it to new size if [ -n "$TEST_DEV" ]; then truncate -s 0 "$TEST_DEV" truncate -s "$size" "$TEST_DEV" return; fi echo "\$TEST_DEV not given, using $TEST_TOP/test.img as fallback" >> \ "$RESULTS" TEST_DEV="$TEST_TOP/test.img" truncate -s 0 "$TEST_DEV" truncate -s "$size" "$TEST_DEV" || _not_run "create file for loop device failed" } # Create filesystem on $TEST_DEV with given options, # do not use for multi-device filesystem # $1-$n: optional, default is -f run_check_mkfs_test_dev() { setup_root_helper # check accidental files/devices passed for opt in "$@"; do if [ -f "$opt" -o -b "$opt" ]; then _fail "ERROR: unexpected option for run_check_mkfs_test_dev: device" fi done run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$@" "$TEST_DEV" } run_check_mount_test_dev() { setup_root_helper local loop_opt if [[ -b "$TEST_DEV" ]]; then loop_opt="" elif [[ -f "$TEST_DEV" ]]; then loop_opt="-o loop" else _fail "Invalid \$TEST_DEV: $TEST_DEV" fi [[ -d "$TEST_MNT" ]] || { _fail "Invalid \$TEST_MNT: $TEST_MNT" } run_check $SUDO_HELPER mount -t btrfs $loop_opt "$@" "$TEST_DEV" "$TEST_MNT" } # $1-$n: optional paths to unmount, otherwise fallback to TEST_DEV run_check_umount_test_dev() { setup_root_helper if [ "$#" = 0 ]; then set -- "$TEST_DEV" fi run_check $SUDO_HELPER umount "$@" } check_kernel_support() { if ! grep -iq 'btrfs' /proc/filesystems; then run_check $SUDO_HELPER modprobe btrfs if ! grep -iq 'btrfs' /proc/filesystems; then echo \ "WARNING: btrfs filesystem not found in /proc/filesystems, some tests might fail" return 1 fi fi return 0 } # compare running kernel version to the given parameter, return success # if running is newer than requested (let caller decide if to fail or skip) # $1: minimum version of running kernel in major.minor format (eg. 4.19) check_min_kernel_version() { local unamemajor local unameminor local argmajor local argminor # 4.19.1-1-default uname=$(uname -r) # 4.19.1 uname=${uname%%-*} IFS=. read unamemajor unameminor tmp <<< "$uname" IFS=. read argmajor argminor tmp <<< "$1" # "compare versions: ${unamemajor}.${unameminor} ? ${argmajor}.${argminor}" [ "$unamemajor" -lt "$argmajor" ] || return 1 [ "$unameminor" -lt "$argminor" ] || return 1 return 0 } # Get subvolume id for specified path _get_subvolid() { # run_check_stdout may have INSTRUMENT polluting the output, we need to # filter the output run_check_stdout "$TOP/btrfs" inspect-internal rootid "$1" | \ grep -e "^[[:digit:]]\+$" } # compare running kernel version to the given parameter, return success # if running is newer than requested (let caller decide if to fail or skip) # $1: minimum version of running kernel in major.minor format (eg. 4.19) check_min_kernel_version() { local unamemajor local unameminor local argmajor local argminor # 4.19.1-1-default uname=$(uname -r) # 4.19.1 uname=${uname%%-*} IFS=. read unamemajor unameminor tmp <<< "$uname" IFS=. read argmajor argminor tmp <<< "$1" # "compare versions: ${unamemajor}.${unameminor} ? ${argmajor}.${argminor}" [ "$unamemajor" -ge "$argmajor" ] || return 1 [ "$unameminor" -ge "$argminor" ] || return 1 return 0 } # how many files to create. DATASET_SIZE=50 generate_dataset() { dataset_type="$1" dirpath=$TEST_MNT/$dataset_type run_check $SUDO_HELPER mkdir -p "$dirpath" case "$dataset_type" in small) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER dd if=/dev/urandom of="$dirpath/$dataset_type.$num" bs=10K \ count=1 status=noxfer >/dev/null 2>&1 done ;; hardlink) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$num" run_check $SUDO_HELPER ln "$dirpath/$dataset_type.$num" "$dirpath/hlink.$num" done ;; fast_symlink) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$num" run_check cd "$dirpath" && \ $SUDO_HELPER ln -s "$dataset_type.$num" "$dirpath/slink.$num" && \ cd / done ;; brokenlink) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER ln -s "$dirpath/$dataset_type.$num" "$dirpath/blink.$num" done ;; perm) for modes in 777 775 755 750 700 666 664 644 640 600 444 440 400 000 \ 1777 1775 1755 1750 1700 1666 1664 1644 1640 1600 1444 1440 1400 1000 \ 2777 2775 2755 2750 2700 2666 2664 2644 2640 2600 2444 2440 2400 2000 \ 4777 4775 4755 4750 4700 4666 4664 4644 4640 4600 4444 4440 4400 4000; do run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$modes" run_check $SUDO_HELPER chmod "$modes" "$dirpath/$dataset_type.$modes" done ;; sparse) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER dd if=/dev/urandom of="$dirpath/$dataset_type.$num" bs=10K \ count=1 status=noxfer >/dev/null 2>&1 run_check $SUDO_HELPER truncate -s 500K "$dirpath/$dataset_type.$num" run_check $SUDO_HELPER dd if=/dev/urandom of="$dirpath/$dataset_type.$num" bs=10K \ oflag=append conv=notrunc count=1 status=noxfer >/dev/null 2>&1 run_check $SUDO_HELPER truncate -s 800K "$dirpath/$dataset_type.$num" done ;; acls) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$num" run_check $SUDO_HELPER setfacl -m "u:root:x" "$dirpath/$dataset_type.$num" run_check $SUDO_HELPER setfattr -n user.foo -v "bar$num" "$dirpath/$dataset_type.$num" done ;; fifo) for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER mkfifo "$dirpath/$dataset_type.$num" done ;; slow_symlink) long_filename=`date +%s | sha256sum | cut -f1 -d ' '` run_check $SUDO_HELPER touch "$dirpath/$long_filename" for num in $(seq 1 "$DATASET_SIZE"); do run_check $SUDO_HELPER ln -s "$dirpath/$long_filename" "$dirpath/slow_slink.$num" done ;; large) run_check $SUDO_HELPER dd if=/dev/urandom bs=32M count=1 \ of="$dirpath/$dataset_type" status=noxfer >/dev/null 2>&1 ;; esac } # prepare environment for loop devices, set up the following variables # - nloopdevs -- number of desired devices # - loopdevs -- array containing paths to all devices (after prepare is called) # - loopdev_prefix -- file backed images starting with this string, 'img' by default # # $1: number of loop devices to be set up setup_loopdevs() { if [ -z "$1" ]; then _fail "setup_loopdevs needs a number" fi nloopdevs="$1" loopdev_prefix=img declare -a loopdevs } # create all loop devices from a given loopdev environment prepare_loopdevs() { for i in `seq $nloopdevs`; do touch "$loopdev_prefix$i" chmod a+rw "$loopdev_prefix$i" truncate -s0 "$loopdev_prefix$i" truncate -s2g "$loopdev_prefix$i" loopdevs[$i]=`run_check_stdout $SUDO_HELPER losetup --find --show "$loopdev_prefix$i"` done } # detach loop devices and reset their size to 0, delete the files afterwards cleanup_loopdevs() { for dev in ${loopdevs[@]}; do run_check $SUDO_HELPER losetup -d "$dev" done for i in `seq $nloopdevs`; do truncate -s0 "$loopdev_prefix$i" rm -- "$loopdev_prefix$i" done run_check $SUDO_HELPER losetup --all } init_env() { TEST_MNT="${TEST_MNT:-$TEST_TOP/mnt}" export TEST_MNT mkdir -p "$TEST_MNT" || { echo "Failed mkdir -p $TEST_MNT"; exit 1; } source "$TEST_TOP/common.local" if [ "$TEST_ENABLE_OVERRIDE" = 'true' -a -n "$RESULTS" ]; then echo "INCLUDE common.local" >> "$RESULTS" echo " check: $TEST_ARGS_CHECK" >> "$RESULTS" echo " mkfs: $TEST_ARGS_MKFS" >> "$RESULTS" echo " convert: $TEST_ARGS_CONVERT" >> "$RESULTS" fi } # Catch critical warning messages in test results check_test_results() { local results="$1" local testname="$2" # Check subpage related warning if grep -q "not nodesize aligned" "$results"; then _fail "found subpage related warning for case $testname" fi } init_env