Real disk partitioning now enabled in test suite for Linux

When using real devices, specify DISKS="sdb sdc sdd" opposed to
/dev/sdb in zfs-tests.sh - otherwise errors with directory names and
disk names registering as "/dev//dev/sdb" for some tests.  The same
goes for mpath: DISK="mpatha mpathad mpathb"

Expected Usage:

$ DISKS="sdb sdc sdd" zfs-tests.sh

SLICE_PREFIX is now set as "p" for a loop device (ie loop0p2) or
"" for a real device (ie sdb2), or either for multipath devices
(ie mpatha1 or mpath1p1) instead of only "p" by default.  Note that
kpartx partitioning is not currently supported in this patch
(ie "partx") and may need to be disabled on Debian distributions.
Functions added for determining test directory (/dev or /dev/mapper)
as well as slice prefix are determined and exported mostly in the cfg
file of each test group directory.

Currently zpools cannot be created on whole mpath devices that have
been partitioned. In order to fix this tests have either been revised
to use a partition instead, or if there is a size constraint and the
pool needs to be created on the whole disk, partitions are then deleted
if the device is a multipath device.  This functionality is added to
default_cleanup() or to individual cleanup scripts if a non-default
cleanup method is used.

The max partitions is currently set at 8 to account for all of the
tests thus far.

Patch changes are generally encompassed in "if is_linux" construct.

Signed-off-by: Sydney Vanda <[email protected]>
Reviewed-by: John Salinas <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: David Quigley <[email protected]>
Closes openzfs#4447
Closes openzfs#4964
Closes openzfs#5074

config/user-commands.m4

@@ -99,9 +99,11 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_LINUX], [
 	AC_PATH_TOOL(COMPRESS, gzip, "")
 	AC_PATH_TOOL(FORMAT, parted, "")
 	AC_PATH_TOOL(LOCKFS, lsof, "")
+	AC_PATH_TOOL(LSBLK, lsblk, "")
 	AC_PATH_TOOL(MODUNLOAD, rmmod, "")
 	AC_PATH_TOOL(NEWFS, mke2fs, "")
 	AC_PATH_TOOL(PFEXEC, sudo, "")
+	AC_PATH_TOOL(READLINK, readlink, "")
 	AC_PATH_TOOL(SHARE, exportfs, "")
 	AC_PATH_TOOL(SWAP, swapon, "")
 	AC_PATH_TOOL(SWAPADD, swapon, "")

tests/zfs-tests/include/commands.cfg.in

@@ -52,6 +52,7 @@ export LOCKFS="@LOCKFS@"
 export LOFIADM="@LOFIADM@"
 export LOGNAME="@LOGNAME@"
 export LS="@LS@"
+export LSBLK="@LSBLK@"
 export MD5SUM="@MD5SUM@"
 export MKDIR="@MKDIR@"
 export MKNOD="@MKNOD@"
@@ -75,6 +76,7 @@ export PRTVTOC="@PRTVTOC@"
 export PS="@PS@"
 export PSRINFO="@PSRINFO@"
 export PYTHON="@PYTHON@"
+export READLINK="@READLINK@"
 export REBOOT="@REBOOT@"
 export RM="@RM@"
 export RMDIR="@RMDIR@"

tests/zfs-tests/include/default.cfg.in

@@ -171,6 +171,8 @@ for i in $ZFS_ALL_VERSIONS; do
 	eval 'export ZFS_VERSION_$i="v${i}-fs"'
 done
 
+export MAX_PARTITIONS=8
+
 if is_linux; then
 	unpack_opts="--sparse -xf"
 	pack_opts="--sparse -cf"
@@ -180,8 +182,8 @@ if is_linux; then
 
 	ZVOL_DEVDIR="/dev/zvol"
 	ZVOL_RDEVDIR="/dev/zvol"
-	DEV_DSKDIR="/dev"
 	DEV_RDSKDIR="/dev"
+	DEV_MPATHDIR="/dev/mapper"
 
 	NEWFS_DEFAULT_FS="ext2"
 else
@@ -199,4 +201,4 @@ else
 	NEWFS_DEFAULT_FS="ufs"
 fi
 export unpack_opts pack_opts verbose unpack_preserve pack_preserve \
-       ZVOL_DEVDIR ZVOL_RDEVDIR NEWFS_DEFAULT_FS DEV_DSKDIR DEV_RDSKDIR
+       ZVOL_DEVDIR ZVOL_RDEVDIR NEWFS_DEFAULT_FS DEV_RDSKDIR DEV_MPATHDIR

tests/zfs-tests/include/libtest.shlib

@@ -456,6 +456,11 @@ function default_cleanup_noexit
 
 	[[ -d $TESTDIR ]] && \
 		log_must $RM -rf $TESTDIR
+
+	disk1=${DISKS%% *}
+	if is_mpath_device $disk1; then
+		delete_partitions
+	fi
 }
 
 
@@ -733,6 +738,69 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 	return 0
 }
 
+#
+# Delete all partitions on all disks - this is specifically for the use of multipath
+# devices which currently can only be used in the test suite as raw/un-partitioned
+# devices (ie a zpool cannot be created on a whole mpath device that has partitions)
+#
+function delete_partitions
+{
+	typeset -i j=1
+
+	if [[ -z $DISK_ARRAY_NUM ]]; then
+		DISK_ARRAY_NUM=$($ECHO ${DISKS} | $NAWK '{print NF}')
+	fi
+	if [[ -z $DISKSARRAY ]]; then
+		DISKSARRAY=$DISKS
+	fi
+
+	if is_linux; then
+		if (( $DISK_ARRAY_NUM == 1 )); then
+			while ((j < MAX_PARTITIONS)); do
+				$FORMAT $DEV_DSKDIR/$DISK -s rm $j > /dev/null 2>&1
+				if (( $? == 1 )); then
+					$LSBLK | $EGREP ${DISK}${SLICE_PREFIX}${j} > /dev/null
+						if (( $? == 1 )); then
+							log_note "Partitions for $DISK should be deleted"
+						else
+							log_fail "Partition for ${DISK}${SLICE_PREFIX}${j} not deleted"
+						fi
+						return 0
+				else
+					$LSBLK | $EGREP ${DISK}${SLICE_PREFIX}${j} > /dev/null
+						if (( $? == 0 )); then
+							log_fail "Partition for ${DISK}${SLICE_PREFIX}${j} not deleted"
+						fi
+				fi
+				((j = j+1))
+			done
+		else
+			for disk in `$ECHO $DISKSARRAY`; do
+				while ((j < MAX_PARTITIONS)); do
+					$FORMAT $DEV_DSKDIR/$disk -s rm $j > /dev/null 2>&1
+					if (( $? == 1 )); then
+						$LSBLK | $EGREP ${disk}${SLICE_PREFIX}${j} > /dev/null
+						if (( $? == 1 )); then
+							log_note "Partitions for $disk should be deleted"
+						else
+							log_fail "Partition for ${disk}${SLICE_PREFIX}${j} not deleted"
+						fi
+						j=7
+					else
+						$LSBLK | $EGREP ${disk}${SLICE_PREFIX}${j} > /dev/null
+						if (( $? == 0 )); then
+							log_fail "Partition for ${disk}${SLICE_PREFIX}${j} not deleted"
+						fi
+					fi
+					((j = j+1))
+				done
+				j=1
+			done
+		fi
+	fi
+	return 0
+}
+
 #
 # Get the end cyl of the given slice
 #
@@ -2482,6 +2550,113 @@ function is_physical_device #device
 	fi
 }
 
+#
+# Check if the given device is a real device (ie SCSI device)
+#
+function is_real_device #disk
+{
+	typeset disk=$1
+	[[ -z $disk ]] && log_fail "No argument for disk given."
+
+       if is_linux; then
+		$LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP disk > /dev/null 2>&1
+		return $?
+       fi
+}
+
+#
+# Check if the given device is a loop device
+#
+function is_loop_device #disk
+{
+	typeset disk=$1
+	[[ -z $disk ]] && log_fail "No argument for disk given."
+
+       if is_linux; then
+		$LSBLK $DEV_RDSKDIR/$disk -o TYPE | $EGREP loop > /dev/null 2>&1
+		return $?
+       fi
+}
+
+#
+# Check if the given device is a multipath device and if there is a sybolic
+# link to a device mapper and to a disk
+# Currently no support for dm devices alone without multipath
+#
+function is_mpath_device #disk
+{
+	typeset disk=$1
+	[[ -z $disk ]] && log_fail "No argument for disk given."
+
+	if is_linux; then
+		$LSBLK $DEV_MPATHDIR/$disk -o TYPE | $EGREP mpath > /dev/null 2>&1
+		if (($? == 0)); then
+			$READLINK $DEV_MPATHDIR/$disk > /dev/null 2>&1
+			return $?
+		else
+			return $?
+		fi
+	fi
+}
+
+# Set the slice prefix for disk partitioning depending
+# on whether the device is a real, multipath, or loop device.
+# Currently all disks have to be of the same type, so only
+# checks first disk to determine slice prefix.
+#
+function set_slice_prefix
+{
+	typeset disk
+	typeset -i i=0
+
+	if is_linux; then
+		while (( i < $DISK_ARRAY_NUM )); do
+			disk="$($ECHO $DISKS | $NAWK '{print $(i + 1)}')"
+			if ( is_mpath_device $disk ) && [[ -z $($ECHO $disk | awk 'substr($1,18,1)\
+			     ~ /^[[:digit:]]+$/') ]] || ( is_real_device $disk ); then
+				export SLICE_PREFIX=""
+				return 0
+			elif ( is_mpath_device $disk || is_loop_device $disk ); then
+				export SLICE_PREFIX="p"
+				return 0
+			else
+				log_fail "$disk not supported for partitioning."
+			fi
+			(( i = i + 1))
+		done
+	fi
+}
+
+#
+# Set the directory path of the listed devices in $DISK_ARRAY_NUM
+# Currently all disks have to be of the same type, so only
+# checks first disk to determine device directory
+# default = /dev (linux)
+# real disk = /dev (linux)
+# multipath device = /dev/mapper (linux)
+#
+function set_device_dir
+{
+	typeset disk
+	typeset -i i=0
+
+	if is_linux; then
+		while (( i < $DISK_ARRAY_NUM )); do
+			disk="$($ECHO $DISKS | $NAWK '{print $(i + 1)}')"
+			if is_mpath_device $disk; then
+				export DEV_DSKDIR=$DEV_MPATHDIR
+				return 0
+			else
+				export DEV_DSKDIR=$DEV_RDSKDIR
+				return 0
+			fi
+			(( i = i + 1))
+		done
+	else
+		export DEV_DSKDIR=$DEV_RDSKDIR
+	fi
+}
+
 #
 # Get the directory path of given device
 #

tests/zfs-tests/tests/functional/cache/cache.cfg

@@ -30,12 +30,12 @@
 
 . $STF_SUITE/include/libtest.shlib
 
+export DISK_ARRAY_NUM=0
+
 function set_disks
 {
 	set -A disk_array $(find_disks $DISKS)
 
-	typeset -i DISK_ARRAY_NUM=0
-
 	if (( ${#disk_array[*]} <= 1 )); then
 		export DISK=${DISKS%% *}
 	else
@@ -58,6 +58,7 @@ function set_disks
 }
 
 set_disks
+set_device_dir
 
 export SIZE=64M
 

tests/zfs-tests/tests/functional/cache/setup.ksh

@@ -34,7 +34,7 @@
 
 verify_runnable "global"
 
-if ! $(is_physical_device $LDEV) ; then
+if ! is_physical_device $LDEV; then
 	log_unsupported "Only physical disk could be cache device"
 fi
 

tests/zfs-tests/tests/functional/clean_mirror/cleanup.ksh

@@ -38,6 +38,12 @@ $DF -F zfs -h | $GREP "$TESTFS " >/dev/null
 [[ $? == 0 ]] && log_must $ZFS umount -f $TESTDIR
 destroy_pool $TESTPOOL
 
+if is_mpath_device $MIRROR_PRIMARY; then
+	$FORMAT $DEV_DSKDIR/$MIRROR_PRIMARY -s rm 1
+fi
+if is_mpath_device $MIRROR_SECONDARY; then
+	$FORMAT $DEV_DSKDIR/$MIRROR_SECONDARY -s rm 1
+fi
 # recreate and destroy a zpool over the disks to restore the partitions to
 # normal
 if [[ -n $SINGLE_DISK ]]; then

tests/zfs-tests/tests/functional/clean_mirror/default.cfg

@@ -44,19 +44,54 @@ if [[ -z $MIRROR_SECONDARY ]]; then
 	SIDE_SECONDARY_PART=1
 
 	if is_linux; then
-		SIDE_PRIMARY=${SINGLE_DISK}p1
-		SIDE_SECONDARY=${SINGLE_DISK}p2
+		if is_mpath_device $SINGLE_DISK; then
+			export DEV_DSKDIR=$DEV_MPATHDIR
+		else
+			export DEV_DSKDIR=$DEV_RDSKDIR
+		fi
+		if ( is_mpath_device $SINGLE_DISK ) && [[ -z $($ECHO $SINGLE_DISK | awk 'substr($1,18,1)\
+		    ~ /^[[:digit:]]+$/') ]] || ( is_real_device $SINGLE_DISK ); then
+			SIDE_PRIMARY=${SINGLE_DISK}1
+			SIDE_SECONDARY=${SINGLE_DISK}2
+		elif ( is_mpath_device $SINGLE_DISK || is_loop_device $SINGLE_DISK ); then
+			SIDE_PRIMARY=${SINGLE_DISK}p1
+			SIDE_SECONDARY=${SINGLE_DISK}p2
+		else
+			log_fail "$SINGLE_DISK not supported for partitioning."
+		fi
 	else
+		export DEV_DSKDIR="/dev"
 		SIDE_PRIMARY=${SINGLE_DISK}s${SIDE_PRIMARY_PART}
 		SIDE_SECONDARY=${SINGLE_DISK}s${SIDE_SECONDARY_PART}
 	fi
 else
 	SIDE_PRIMARY_PART=0
 	SIDE_SECONDARY_PART=0
 	if is_linux; then
-		SIDE_PRIMARY=${MIRROR_PRIMARY}p1
-		SIDE_SECONDARY=${MIRROR_SECONDARY}p1
+		if is_mpath_device $MIRROR_PRIMARY; then
+			export DEV_DSKDIR=$DEV_MPATHDIR
+		else
+			export DEV_DSKDIR=$DEV_RDSKDIR
+		fi
+		if ( is_mpath_device $MIRROR_PRIMARY ) && [[ -z $($ECHO $MIRROR_PRIMARY | awk 'substr($1,18,1)\
+		    ~ /^[[:digit:]]+$/') ]] || ( is_real_device $MIRROR_PRIMARY ); then
+			SIDE_PRIMARY=${MIRROR_PRIMARY}1
+		elif ( is_mpath_device $MIRROR_PRIMARY || is_loop_device $MIRROR_PRIMARY ); then
+			SIDE_PRIMARY=${MIRROR_PRIMARY}p1
+		else
+			log_fail "$MIRROR_PRIMARY not supported for partitioning."
+		fi
+		if ( is_mpath_device $MIRROR_SECONDARY ) && [[ -z $($ECHO $MIRROR_SECONDARY | awk 'substr($1,18,1)\
+		    ~ /^[[:digit:]]+$/') ]] || ( is_real_device $MIRROR_SECONDARY ); then
+			SIDE_SECONDARY=${MIRROR_SECONDARY}1
+		elif ( is_mpath_device $MIRROR_SECONDARY || is_loop_device $MIRROR_SECONDARY ); then
+			SIDE_SECONDARY=${MIRROR_SECONDARY}p1
+		else
+			log_fail "$MIRROR_SECONDARY not supported for partitioning."
+		fi
+
 	else
+		export DEV_DSKDIR="/dev"
 		SIDE_PRIMARY=${MIRROR_PRIMARY}s${SIDE_PRIMARY_PART}
 		SIDE_SECONDARY=${MIRROR_SECONDARY}s${SIDE_SECONDARY_PART}
 	fi

tests/zfs-tests/tests/functional/clean_mirror/setup.ksh

@@ -34,7 +34,7 @@
 
 verify_runnable "global"
 
-if ! $(is_physical_device $DISKS) ; then
+if ! is_physical_device $DISKS; then
 	log_unsupported "This directory cannot be run on raw files."
 fi
 

tests/zfs-tests/tests/functional/cli_root/zpool_add/cleanup.ksh

@@ -32,6 +32,11 @@
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib
 
+DISK=${DISKS%% *}
+if is_mpath_device $DISK; then
+        delete_partitions
+fi
+
 cleanup_devices $DISKS
 
 log_pass

tests/zfs-tests/tests/functional/cli_root/zpool_add/setup.ksh

@@ -38,6 +38,11 @@ if ! $(is_physical_device $DISKS) ; then
 	log_unsupported "This directory cannot be run on raw files."
 fi
 
+disk1=${DISKS%% *}
+if is_mpath_device $disk1; then
+        delete_partitions
+fi
+
 if [[ -n $DISK ]]; then
 	#
         # Use 'zpool create' to clean up the infomation in