btrfs-progs/tests/common

#!/bin/bash
#
# Common routines for all tests
#

# assert that argument is not empty and is an existing path (file or directory)
_assert_path()
{
	local path

	path="$1"
	if [ -z "$path" ]; then
		echo "ASSERTION FAIL: $path is not valid"
		exit 1
	fi

	if [ -f "$path" -o -d "$path" -o -b "$path" ]; then
		return 0
	fi
	echo "ASSERTION FAIL: $path is not valid"
	exit 1
}

# $1: this string gets matched to files, absolute or relative path, or a
# systemwide command available via $PATH
_is_file_or_command()
{
	local msg

	msg="$1"
	if [ -z "$msg" ]; then
		return 1
	fi

	if [ -f "$msg" -o -d "$msg" -o -b "$msg" ]; then
		return 0
	fi
	msg=$(type -p -- "$msg")
	if [ -f "$msg" ]; then
		return 0
	fi
	return 1
}

_fail()
{
	echo "$*" | tee -a "$RESULTS"
	exit 1
}

# log a message to the results file
_log()
{
	echo "$*" >> "$RESULTS"
}

# copy stdout to log and pass to stdout, eg. another stdout consumer, commands
# should redirect stderr to stdout if this is consmed by further commands
_log_stdout()
{
	tee -a "$RESULTS"
}

_not_run()
{
	echo "    [NOTRUN] $*"
	exit 0
}

# debugging helper
_dump_args()
{
	local i

	i=1
	echo "DUMP args for ${FUNCNAME[1]}:"
	while [ $# -gt 0 ]; do
		echo "ARG[$i]: $1"
		i=$(($i+1))
		shift
	done
}

# read arguments, look if we're calling btrfs and if there's a known
# subcommand, return argument index to insert, taking root helper into
# consideration, returns 2 for unknown subcommand
_get_spec_ins()
{
	if [ "$1" = 'root_helper' ]; then
		if [[ $2 =~ /btrfs$ ]]; then
			echo -n 4
			return
		fi
	else
		if [[ $1 =~ /btrfs$ ]]; then
			echo -n 3
			return
		fi
	fi
	echo -n 2
}

# return command-specific arguments if enabled
_cmd_spec()
{
	if [ "$TEST_ENABLE_OVERRIDE" = 'true' ]; then
		# if defined via common.local, use it, otherwise pass make
		# arguments
		if [ "$(type -t _skip_spec)" = 'function' ]; then
			if _skip_spec "$@"; then
				return
			fi
		fi
		case "$1" in
	  		check) echo -n "$TEST_ARGS_CHECK" ;;
		esac
	fi
}

# Argument passing magic:
# the command passed to run_* helpers is inspected, if there's 'btrfs command'
# found and there are defined additional arguments, they're inserted just after
# the command name, ie. any arguments in the test could override them.
#
# The root helper is recognized.  Unrecognized subcommands or external tools
# are not affected.

run_check()
{
	local spec
	local ins
	local cmd

	ins=$(_get_spec_ins "$@")
	spec=$(($ins-1))
	cmd=$(eval echo "\${$spec}")
	spec=$(_cmd_spec "${@:$spec}")
	set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}"
	echo "====== RUN CHECK $@" >> "$RESULTS" 2>&1
	if [[ $TEST_LOG =~ tty ]]; then echo "CMD: $@" > /dev/tty; fi
	if [ "$1" = 'root_helper' ]; then
		"$@" >> "$RESULTS" 2>&1 || _fail "failed: $@"
	else
		$INSTRUMENT "$@" >> "$RESULTS" 2>&1 || _fail "failed: $@"
	fi
}

# same as run_check but the stderr+stdout output is duplicated on stdout and
# can be processed further
run_check_stdout()
{
	local spec
	local ins
	local cmd

	ins=$(_get_spec_ins "$@")
	spec=$(($ins-1))
	cmd=$(eval echo "\${$spec}")
	spec=$(_cmd_spec "${@:$spec}")
	set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}"
	echo "====== RUN CHECK $@" >> "$RESULTS" 2>&1
	if [[ $TEST_LOG =~ tty ]]; then echo "CMD(stdout): $@" > /dev/tty; fi
	if [ "$1" = 'root_helper' ]; then
		"$@" 2>&1 | tee -a "$RESULTS"
	else
		$INSTRUMENT "$@" 2>&1 | tee -a "$RESULTS"
	fi
	if [ ${PIPESTATUS[0]} -ne 0 ]; then
		_fail "failed: $@"
	fi
}

# same as run_check but does not fail the test if it's handled gracefully by
# the tool, unexpected failure like segfault or abort will exit forcibly
# output is logged
run_mayfail()
{
	local spec
	local ins
	local cmd
	local ret

	ins=$(_get_spec_ins "$@")
	spec=$(($ins-1))
	cmd=$(eval echo "\${$spec}")
	spec=$(_cmd_spec "${@:$spec}")
	set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}"
	echo "====== RUN MAYFAIL $@" >> "$RESULTS" 2>&1
	if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mayfail): $@" > /dev/tty; fi
	if [ "$1" = 'root_helper' ]; then
		"$@" >> "$RESULTS" 2>&1
	else
		$INSTRUMENT "$@" >> "$RESULTS" 2>&1
	fi
	ret=$?
	if [ $ret != 0 ]; then
		echo "failed (ignored, ret=$ret): $@" >> "$RESULTS"
		if [ $ret == 139 ]; then
			_fail "mayfail: returned code 139 (SEGFAULT), not ignored"
		elif [ $ret == 134 ]; then
			_fail "mayfail: returned code 134 (SIGABRT), not ignored"
		fi
		return $ret
	fi
}

# first argument is error message to print if it fails, otherwise
# same as run_check but expects the command to fail, output is logged
run_mustfail()
{
	local spec
	local ins
	local cmd
	local msg

	msg="$1"
	shift

	if _is_file_or_command "$msg"; then
		echo "ASSERTION FAIL: 1st argument of run_mustfail must be a message"
		exit 1
	fi

	ins=$(_get_spec_ins "$@")
	spec=$(($ins-1))
	cmd=$(eval echo "\${$spec}")
	spec=$(_cmd_spec "${@:$spec}")
	set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}"
	echo "====== RUN MUSTFAIL $@" >> "$RESULTS" 2>&1
	if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mustfail): $@" > /dev/tty; fi
	if [ "$1" = 'root_helper' ]; then
		"$@" >> "$RESULTS" 2>&1
	else
		$INSTRUMENT "$@" >> "$RESULTS" 2>&1
	fi
	if [ $? != 0 ]; then
		echo "failed (expected): $@" >> "$RESULTS"
		return 0
	else
		echo "succeeded (unexpected!): $@" >> "$RESULTS"
		_fail "unexpected success: $msg"
		return 1
	fi
}

# The first parameter is error message to print if it fails, just like
# run_must_fail().
# NOTE: we don't use pipefail to avoid disturbing other script, so here we
# use a temporary output file.
# So it doesn't support pipeline in the @cmd
run_mustfail_stdout()
{
	local spec
	local ins
	local cmd
	local msg
	local ret
	local tmp_output

	tmp_output=$(mktemp --tmpdir btrfs-progs-test--mustfail-stdtout.XXXXXX)

	msg="$1"
	shift

	if _is_file_or_command "$msg"; then
		echo "ASSERTION FAIL: 1st argument of run_mustfail_stdout must be a message"
		exit 1
	fi

	ins=$(_get_spec_ins "$@")
	spec=$(($ins-1))
	cmd=$(eval echo "\${$spec}")
	spec=$(_cmd_spec "${@:$spec}")
	set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}"
	echo "====== RUN MUSTFAIL $@" >> "$RESULTS" 2>&1
	if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mustfail): $@" > /dev/tty; fi
	if [ "$1" = 'root_helper' ]; then
		"$@" 2>&1 > "$tmp_output"
	else
		$INSTRUMENT "$@" 2>&1 > "$tmp_output"
	fi
	ret=$?

	cat "$tmp_output" >> "$RESULTS"
	cat "$tmp_output"
	rm "$tmp_output"

	if [ "$ret" != 0 ]; then
		echo "failed (expected): $@" >> "$RESULTS"
		return 0
	else
		echo "succeeded (unexpected!): $@" >> "$RESULTS"
		_fail "unexpected success: $msg"
		return 1
	fi
}

check_prereq()
{
	if [ "$1" = "btrfs-corrupt-block" -o "$1" = "fssum" ]; then
		if ! [ -f "$INTERNAL_BIN/$1" ]; then
			_fail "Failed prerequisites: $INTERNAL_BIN/$1";
		fi
	elif ! [ -f "$TOP/$1" ]; then
		_fail "Failed prerequisites: $TOP/$1";
	fi
}

check_global_prereq()
{
	which "$1" &> /dev/null
	if [ $? -ne 0 ]; then
		_fail "Failed system wide prerequisities: $1";
	fi
}

check_image()
{
	local image

	image=$1
	echo "testing image $(basename $image)" >> "$RESULTS"
	"$TOP/btrfs" check "$image" >> "$RESULTS" 2>&1
	[ $? -eq 0 ] && _fail "btrfs check should have detected corruption"

	run_check "$TOP/btrfs" check --repair "$image"
	run_check "$TOP/btrfs" check "$image"
}

# Extract a usable image from packed formats
# - raw btrfs filesystem images, suffix .raw
# - dtto compressed by XZ, suffix .raw.xz
# - meta-dump images with suffix .img
# - dtto compressed by XZ, suffix .img.xz
# - compressed send stream, .stream.xz
extract_image()
{
	local image
	local cleanme

	image="$1"
	case "$image" in
	*.img)
		rm -f "$image.restored"
		;;
	*.img.xz)
		xz --decompress --keep "$image" || \
			_fail "failed to decompress image $image" >&2
		image=${image%%.xz}
		rm -f "$image.restored"
		cleanme=$image
		;;
	*.raw)
		cp --sparse=auto "$image" "$image.restored"
		;;
	*.raw.xz)
		xz --decompress --keep "$image" || \
			_fail "failed to decompress image $image" >&2
		image=${image%%.xz}
		mv "$image" "$image.restored"
		;;
	*.stream.xz)
		xz --decompress --keep "$image" || \
			_fail "failed to decompress file $image" >&2
		image=${image%%.xz}
		mv "$image" "$image.restored"
		;;
	esac

	if ! [ -f "$image.restored" ]; then
		echo "restoring image $(basename $image)" >> "$RESULTS"
		"$TOP/btrfs-image" -r "$image" "$image.restored" \
			&>> "$RESULTS" \
			|| _fail "failed to restore image $image" >&2
	fi

	[ -f "$cleanme" ] && rm -f "$cleanme"

	echo "$image.restored"
}

# Process all image dumps in a given directory
check_all_images()
{
	local dir
	local extracted

	dir="$1"
	if [ -z "$dir" ]; then
		dir=.
	fi
	_assert_path "$dir"
	for image in $(find "$dir" \( -iname '*.img' -o	\
				-iname '*.img.xz' -o 	\
				-iname '*.raw' -o 	\
				-iname '*.raw.xz' \) | sort)
	do
		extracted=$(extract_image "$image")
		check_image "$extracted"
		rm -f "$extracted"
	done
}

# some tests need to mount the recovered image and do verifications call
# 'setup_root_helper' and then check for have_root_helper == 1 if the test
# needs to fail otherwise; using sudo by default for now
SUDO_HELPER=
NEED_SUDO_VALIDATE=unknown
export SUDO_HELPER
export NEED_SUDO_VALIDATE
root_helper()
{
	if [ $UID -eq 0 ]; then
		"$@"
	else
		if [ "$NEED_SUDO_VALIDATE" = 'yes' ]; then
			sudo -v -n &>/dev/null || \
				_not_run "Need to validate sudo credentials"
			sudo -n "$@"
		elif [ "$NEED_SUDO_VALIDATE" = 'no' ]; then
			sudo -n /bin/true &> /dev/null || \
				_not_run "Need to validate sudo user settings"
			sudo -n "$@"
		else
			# should not happen
			_not_run "Need to validate root privileges"
		fi
	fi
}

setup_root_helper()
{
	if [ $UID -eq 0 -o -n "$SUDO_HELPER" ]; then
		return
	fi

	# Test for old sudo or special settings, which make sudo -v fail even
	# if user setting is NOPASSWD
	sudo -n /bin/true &>/dev/null && NEED_SUDO_VALIDATE=no

	# Newer sudo or default sudo setting
	sudo -v -n &>/dev/null && NEED_SUDO_VALIDATE=yes

	if [ "$NEED_SUDO_VALIDATE" = 'unknown' ]; then
		_not_run "Need to validate root privileges"
	fi
	SUDO_HELPER=root_helper
}

prepare_test_dev()
{
	# num[K/M/G/T...]
	local size="$1"

	[[ "$size" ]] || size='2G'
	# Still truncate it to new size
	if [ -n "$TEST_DEV" ]; then
		truncate -s 0 "$TEST_DEV"
		truncate -s "$size" "$TEST_DEV"
		return;
	fi

	echo "\$TEST_DEV not given, using $TEST_TOP/test.img as fallback" >> \
		"$RESULTS"
	TEST_DEV="$TEST_TOP/test.img"

	truncate -s 0 "$TEST_DEV"
	truncate -s "$size" "$TEST_DEV" || _not_run "create file for loop device failed"
}

# Create filesystem on $TEST_DEV with given options,
# do not use for multi-device filesystem
# $1-$n: optional, default is -f
run_check_mkfs_test_dev()
{
	setup_root_helper

	# check accidental files/devices passed
	for opt in "$@"; do
		if [ -f "$opt" -o -b "$opt" ]; then
			_fail "ERROR: unexpected option for run_check_mkfs_test_dev: device"
		fi
	done

	run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$@" "$TEST_DEV"
}

run_check_mount_test_dev()
{
	setup_root_helper

	local loop_opt
	if [[ -b "$TEST_DEV" ]]; then
		loop_opt=""
	elif [[ -f "$TEST_DEV" ]]; then
		loop_opt="-o loop"
	else
		_fail "Invalid \$TEST_DEV: $TEST_DEV"
	fi

	[[ -d "$TEST_MNT" ]] || {
		_fail "Invalid \$TEST_MNT: $TEST_MNT"
	}

	run_check $SUDO_HELPER mount -t btrfs $loop_opt "$@" "$TEST_DEV" "$TEST_MNT"
}

# $1-$n: optional paths to unmount, otherwise fallback to TEST_DEV
run_check_umount_test_dev()
{
	setup_root_helper
	if [ "$#" = 0 ]; then
		set -- "$TEST_DEV"
	fi
	run_check $SUDO_HELPER umount "$@"
}

check_kernel_support()
{
	if ! grep -iq 'btrfs' /proc/filesystems; then
		run_check $SUDO_HELPER modprobe btrfs
			if ! grep -iq 'btrfs' /proc/filesystems; then
				echo \
"WARNING: btrfs filesystem not found in /proc/filesystems, some tests might fail"
				return 1
			fi
	fi
	return 0
}

# compare running kernel version to the given parameter, return success
# if running is newer than requested (let caller decide if to fail or skip)
# $1: minimum version of running kernel in major.minor format (eg. 4.19)
check_min_kernel_version()
{
	local unamemajor
	local unameminor
	local argmajor
	local argminor

	# 4.19.1-1-default
	uname=$(uname -r)
	# 4.19.1
	uname=${uname%%-*}
	IFS=. read unamemajor unameminor tmp <<< "$uname"
	IFS=. read argmajor argminor tmp <<< "$1"
	# "compare versions: ${unamemajor}.${unameminor} ? ${argmajor}.${argminor}"
	[ "$unamemajor" -lt "$argmajor" ] || return 1
	[ "$unameminor" -lt "$argminor" ] || return 1
	return 0
}

# compare running kernel version to the given parameter, return success
# if running is newer than requested (let caller decide if to fail or skip)
# $1: minimum version of running kernel in major.minor format (eg. 4.19)
check_min_kernel_version()
{
	local unamemajor
	local unameminor
	local argmajor
	local argminor

	# 4.19.1-1-default
	uname=$(uname -r)
	# 4.19.1
	uname=${uname%%-*}
	IFS=. read unamemajor unameminor tmp <<< "$uname"
	IFS=. read argmajor argminor tmp <<< "$1"
	# "compare versions: ${unamemajor}.${unameminor} ? ${argmajor}.${argminor}"
	[ "$unamemajor" -ge "$argmajor" ] || return 1
	[ "$unameminor" -ge "$argminor" ] || return 1
	return 0
}

# how many files to create.
DATASET_SIZE=50

generate_dataset() {

	dataset_type="$1"
	dirpath=$TEST_MNT/$dataset_type
	run_check $SUDO_HELPER mkdir -p "$dirpath"

	case "$dataset_type" in
		small)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER dd if=/dev/urandom of="$dirpath/$dataset_type.$num" bs=10K \
				count=1 status=noxfer >/dev/null 2>&1
			done
			;;

		hardlink)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$num"
				run_check $SUDO_HELPER ln "$dirpath/$dataset_type.$num" "$dirpath/hlink.$num"
			done
			;;

		fast_symlink)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$num"
				run_check cd "$dirpath" && \
					$SUDO_HELPER ln -s "$dataset_type.$num" "$dirpath/slink.$num" && \
					cd /
			done
			;;

		brokenlink)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER ln -s "$dirpath/$dataset_type.$num" "$dirpath/blink.$num"
			done
			;;

		perm)
			for modes in 777 775 755 750 700 666 664 644 640 600 444 440 400 000		\
				1777 1775 1755 1750 1700 1666 1664 1644 1640 1600 1444 1440 1400 1000	\
				2777 2775 2755 2750 2700 2666 2664 2644 2640 2600 2444 2440 2400 2000	\
				4777 4775 4755 4750 4700 4666 4664 4644 4640 4600 4444 4440 4400 4000; do
				run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$modes"
				run_check $SUDO_HELPER chmod "$modes" "$dirpath/$dataset_type.$modes"
			done
			;;

		sparse)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER dd if=/dev/urandom of="$dirpath/$dataset_type.$num" bs=10K \
				count=1 status=noxfer >/dev/null 2>&1
				run_check $SUDO_HELPER truncate -s 500K "$dirpath/$dataset_type.$num"
				run_check $SUDO_HELPER dd if=/dev/urandom of="$dirpath/$dataset_type.$num" bs=10K \
				oflag=append conv=notrunc count=1 status=noxfer >/dev/null 2>&1
				run_check $SUDO_HELPER truncate -s 800K "$dirpath/$dataset_type.$num"
			done
			;;

		acls)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER touch "$dirpath/$dataset_type.$num"
				run_check $SUDO_HELPER setfacl -m "u:root:x" "$dirpath/$dataset_type.$num"
				run_check $SUDO_HELPER setfattr -n user.foo -v "bar$num" "$dirpath/$dataset_type.$num"
			done
			;;

		fifo)
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER mkfifo "$dirpath/$dataset_type.$num"
			done
			;;

		slow_symlink)
			long_filename=`date +%s | sha256sum | cut -f1 -d ' '`
			run_check $SUDO_HELPER touch "$dirpath/$long_filename"
			for num in $(seq 1 "$DATASET_SIZE"); do
				run_check $SUDO_HELPER ln -s "$dirpath/$long_filename" "$dirpath/slow_slink.$num"
			done
			;;
		large)
			run_check $SUDO_HELPER dd if=/dev/urandom bs=32M count=1 \
				of="$dirpath/$dataset_type" status=noxfer >/dev/null 2>&1
			;;
	esac
}

# prepare environment for loop devices, set up the following variables
# - nloopdevs -- number of desired devices
# - loopdevs  -- array containing paths to all devices (after prepare is called)
# - loopdev_prefix -- file backed images starting with this string, 'img' by default
#
# $1: number of loop devices to be set up
setup_loopdevs()
{
	if [ -z "$1" ]; then
		_fail "setup_loopdevs needs a number"
	fi
	nloopdevs="$1"
	loopdev_prefix=img
	declare -a loopdevs

}

# create all loop devices from a given loopdev environment
prepare_loopdevs()
{
	for i in `seq $nloopdevs`; do
		touch "$loopdev_prefix$i"
		chmod a+rw "$loopdev_prefix$i"
		truncate -s0 "$loopdev_prefix$i"
		truncate -s2g "$loopdev_prefix$i"
		loopdevs[$i]=`run_check_stdout $SUDO_HELPER losetup --find --show "$loopdev_prefix$i"`
	done
}

# detach loop devices and reset their size to 0, delete the files afterwards
cleanup_loopdevs()
{
	for dev in ${loopdevs[@]}; do
		run_check $SUDO_HELPER losetup -d "$dev"
	done
	for i in `seq $nloopdevs`; do
		truncate -s0 "$loopdev_prefix$i"
		rm -- "$loopdev_prefix$i"
	done
	run_check $SUDO_HELPER losetup --all
}

init_env()
{
	TEST_MNT="${TEST_MNT:-$TEST_TOP/mnt}"
	export TEST_MNT
	mkdir -p "$TEST_MNT" || { echo "Failed mkdir -p $TEST_MNT"; exit 1; }

	source "$TEST_TOP/common.local"

	if [ "$TEST_ENABLE_OVERRIDE" = 'true' -a -n "$RESULTS" ]; then
		echo "INCLUDE common.local" >> "$RESULTS"
		echo "  check: $TEST_ARGS_CHECK" >> "$RESULTS"
	fi
}
init_env