Retry zfs destroy when busy in rsend tests

rsend tests in the test suite frequently create and destroy datasets. It is possible for zfs destroy to return an error code indicating the dataset is busy. Simply use a log_must_busy in these cases to retry destroying those datasets. Other fixes to rsend test cases to avoid unmounting and remounting filesystems and some cleanup. Reviewed-by: Brian Behlendorf <[email protected]> Signed-off-by: Giuseppe Di Natale <[email protected]> Closes openzfs#6418
gldisater · Aug 3, 2017 · e3bdcb8 · e3bdcb8
1 parent ecb2b7d
commit e3bdcb8
Show file tree

Hide file tree

Showing 16 changed files with 46 additions and 47 deletions.
diff --git a/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh b/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
@@ -56,7 +56,7 @@ verify_runnable "both"
 function cleanup {
 
 	for obj in $OBJ_LIST; do
-		datasetexists $obj && log_must zfs destroy -f $obj
+		datasetexists $obj && log_must_busy zfs destroy -f $obj
 	done
 }
 
@@ -112,7 +112,7 @@ for obj in $OBJ_LIST ; do
 		"($resv_size_set != $resv_size_get)"
 	fi
 
-	log_must zfs destroy -f $obj
+	log_must_busy zfs destroy -f $obj
 
 	new_space_avail=`get_prop available $TESTPOOL`
 	new_space_used=`get_prop used $TESTPOOL`

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
@@ -119,13 +119,13 @@ function cleanup_pool
 	log_must rm -rf $BACKDIR/*
 
 	if is_global_zone ; then
-		log_must zfs destroy -Rf $pool
+		log_must_busy zfs destroy -Rf $pool
 	else
 		typeset list=$(zfs list -H -r -t all -o name $pool)
 		for ds in $list ; do
 			if [[ $ds != $pool ]] ; then
 				if datasetexists $ds ; then
-					log_must zfs destroy -Rf $ds
+					log_must_busy zfs destroy -Rf $ds
 				fi
 			fi
 		done
@@ -296,24 +296,16 @@ function destroy_tree
 	typeset -i ret=0
 	typeset snap
 	for snap in "$@" ; do
-		zfs destroy $snap
-		ret=$?
+		log_must_busy zfs destroy $snap
 
 		typeset ds=${snap%%@*}
 		typeset type=$(get_prop "type" $ds)
 		if [[ $type == "filesystem" ]]; then
 			typeset mntpnt=$(get_prop mountpoint $ds)
-			((ret |= $?))
-
-			if ((ret != 0)); then
-				rm -r $mntpnt/$snap
-				((ret |= $?))
+			if [[ -n $mntpnt ]]; then
+				rm -rf $mntpnt/$snap
 			fi
 		fi
-
-		if ((ret != 0)); then
-			return $ret
-		fi
 	done
 
 	return 0
@@ -516,8 +508,8 @@ function test_fs_setup
 	typeset sendpool=${sendfs%%/*}
 	typeset recvpool=${recvfs%%/*}
 
-	datasetexists $sendfs && log_must zfs destroy -r $sendpool
-	datasetexists $recvfs && log_must zfs destroy -r $recvpool
+	datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
+	datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
 
 	if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
 		mk_files 1000 256 0 $sendfs &
@@ -547,7 +539,7 @@ function test_fs_setup
 	fi
 
 	if datasetexists $streamfs; then
-		log_must zfs destroy -r $streamfs
+		log_must_busy zfs destroy -r $streamfs
 	fi
 	log_must zfs create -o compress=lz4 $sendpool/stream
 }
@@ -664,8 +656,8 @@ function resume_cleanup
 	typeset sendfs=$1
 	typeset streamfs=$2
 
-	datasetexists $sendfs && log_must zfs destroy -r $sendfs
-	datasetexists $streamfs && log_must zfs destroy -r $streamfs
+	datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
+	datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
 	cleanup_pool $POOL2
 	rm -f /$POOL/initial.zsend /$POOL/incremental.zsend
 }
diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
@@ -59,7 +59,7 @@ if is_global_zone ; then
 	#
 	log_must eval "zfs send -I $POOL2@psnap $POOL2/pclone@final > " \
 		"$BACKDIR/pool-clone-I"
-	log_must zfs destroy -rf $POOL2/pclone
+	log_must_busy zfs destroy -rf $POOL2/pclone
 	log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-clone-I"
 	log_must cmp_ds_subs $POOL $POOL2
 	log_must cmp_ds_cont $POOL $POOL2
@@ -73,7 +73,7 @@ dstds=$(get_dst_ds $POOL $POOL2)
 ds=$dstds/$FS/fs1
 log_must eval "zfs send -I $ds/fs2@fsnap $ds/fclone@final > " \
 	"$BACKDIR/fs-clone-I"
-log_must zfs destroy -rf $ds/fclone
+log_must_busy zfs destroy -rf $ds/fclone
 log_must eval "zfs receive -F $ds/fclone < $BACKDIR/fs-clone-I"
 
 log_must cmp_ds_subs $POOL $dstds
@@ -86,7 +86,7 @@ if is_global_zone ; then
 	ds=$POOL2/$FS
 	log_must eval "zfs send -I $ds/vol@vsnap $ds/vclone@final > " \
 		"$BACKDIR/vol-clone-I"
-	log_must zfs destroy -rf $ds/vclone
+	log_must_busy zfs destroy -rf $ds/vclone
 	log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/vol-clone-I"
 	log_must cmp_ds_subs $POOL $POOL2
 	log_must cmp_ds_cont $POOL $POOL2

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
@@ -48,10 +48,10 @@ verify_runnable "global"
 function cleanup
 {
 	if datasetexists bpool ; then
-		log_must zpool destroy -f bpool
+		log_must_busy zpool destroy -f bpool
 	fi
 	if datasetexists spool ; then
-		log_must zpool destroy -f spool
+		log_must_busy zpool destroy -f spool
 	fi
 }
 
@@ -80,7 +80,7 @@ log_must ismounted spool
 # Test out of space on top filesystem
 #
 log_must mv /bpool/fs/file /bpool
-log_must zfs destroy -rf bpool/fs
+log_must_busy zfs destroy -rf bpool/fs
 
 log_must zfs snapshot bpool@snap
 log_must eval "zfs send -R bpool@snap > $BACKDIR/bpool-R"

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
@@ -45,9 +45,6 @@ verify_runnable "both"
 
 function cleanup
 {
-	export __ZFS_POOL_RESTRICT="$POOL $POOL2"
-	log_must zfs unmount -a
-	unset __ZFS_POOL_RESTRICT
 	log_must cleanup_pool $POOL
 	log_must cleanup_pool $POOL2
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
@@ -66,11 +66,11 @@ cleanup
 log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R"
 log_must eval "zfs receive -dF $POOL2 < $BACKDIR/pool-final-R"
 
-log_must zfs destroy -Rf $POOL/$FS
-log_must zfs destroy -Rf $POOL/pclone
+log_must_busy zfs destroy -Rf $POOL/$FS
+log_must_busy zfs destroy -Rf $POOL/pclone
 
 if is_global_zone ; then
-	log_must zfs destroy -Rf $POOL/vol
+	log_must_busy zfs destroy -Rf $POOL/vol
 fi
 log_must zfs snapshot -r $POOL@destroy
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
@@ -33,6 +33,11 @@
 
 verify_runnable "both"
 
+# See issue: https://github.com/zfsonlinux/zfs/issues/6446
+if is_linux; then
+	log_unsupported "Test often hangs. Skipping."
+fi
+
 log_assert "Verify resumability of full ZFS send/receive with the -D " \
     "(dedup) flag"
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
@@ -35,6 +35,11 @@
 
 verify_runnable "both"
 
+# See issue: https://github.com/zfsonlinux/zfs/issues/6446
+if is_linux; then
+	log_unsupported "Test often hangs. Skipping."
+fi
+
 log_assert "Verify resumability of a full and incremental ZFS send/receive " \
     "with the -e (embedded) flag"
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
@@ -54,10 +54,10 @@ log_onexit resume_cleanup $sendfs $streamfs
 
 test_fs_setup $sendfs $recvfs
 log_must zfs bookmark $sendfs@a $sendfs#bm_a
-log_must zfs destroy $sendfs@a
+log_must_busy zfs destroy $sendfs@a
 log_must zfs receive -v $recvfs </$POOL/initial.zsend
 resume_test "zfs send -i \#bm_a $sendfs@b" $streamfs $recvfs
-log_must zfs destroy -r -f $sendfs
+log_must_busy zfs destroy -r -f $sendfs
 log_must zfs receive -v $sendfs </$POOL/initial.zsend
 log_must zfs receive -v $sendfs </$POOL/incremental.zsend
 file_check $sendfs $recvfs

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
@@ -50,7 +50,7 @@ streamfs=$POOL/stream
 log_onexit resume_cleanup $sendfs $streamfs
 
 test_fs_setup $sendfs $recvfs
-log_must zfs unmount $sendfs
+log_must zfs unmount -f $sendfs
 resume_test "zfs send $sendfs" $streamfs $recvfs
 file_check $sendfs $recvfs
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
@@ -52,8 +52,8 @@ for compress in off gzip; do
 		poolexists $POOL3 && destroy_pool $POOL3
 		log_must zpool create $pool_opt $POOL3 $DISK3
 
-		datasetexists $send_ds && log_must zfs destroy -r $send_ds
-		datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+		datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+		datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds
 
 		log_must zfs create -o compress=$compress $send_ds
 		typeset dir=$(get_prop mountpoint $send_ds)
@@ -63,7 +63,7 @@ for compress in off gzip; do
 		log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c"
 		log_must eval "zfs recv $recv_ds <$BACKDIR/full-c"
 
-		log_must zfs destroy -r $recv_ds
+		log_must_busy zfs destroy -r $recv_ds
 
 		log_must eval "zfs send $send_ds@full >$BACKDIR/full"
 		log_must eval "zfs recv $recv_ds <$BACKDIR/full"

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
@@ -50,8 +50,8 @@ datasetexists $POOL3 && log_must zpool destroy $POOL3
 log_must zpool create -d $POOL3 $DISK3
 
 for compress in $compress_types; do
-	datasetexists $send_ds && log_must zfs destroy -r $send_ds
-	datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+	datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+	datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds
 
 	log_must zfs create -o compress=$compress $send_ds
 	typeset dir=$(get_prop mountpoint $send_ds)

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
@@ -56,8 +56,8 @@ log_onexit cleanup_pool $POOL2
 write_compressible $BACKDIR ${megs}m
 
 for compress in $compress_types; do
-	datasetexists $send_ds && log_must zfs destroy -r $send_ds
-	datasetexists $send_vol && log_must zfs destroy -r $send_vol
+	datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+	datasetexists $send_vol && log_must_busy zfs destroy -r $send_vol
 	log_must zfs create -o compress=$compress $send_ds
 	log_must zfs create -V 1g -o compress=$compress $send_vol
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
@@ -59,7 +59,7 @@ for prop in $(get_rand_compress_any 6); do
 		verify_stream_size $BACKDIR/compressed $sendfs
 
 		log_must rm $BACKDIR/uncompressed $BACKDIR/compressed
-		log_must zfs destroy -r $sendfs
+		log_must_busy zfs destroy -r $sendfs
 	done
 done
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
@@ -29,7 +29,7 @@
 
 function cleanup
 {
-	log_must zfs destroy -r $vol
+	log_must_busy zfs destroy -r $vol
 	cleanup_pool $POOL2
 }
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
@@ -55,8 +55,8 @@ verify_runnable "both"
 
 function cleanup
 {
-	datasetexists $TESTPOOL/128k && log_must zfs destroy $TESTPOOL/128k
-	datasetexists $TESTPOOL/1m && log_must zfs destroy $TESTPOOL/1m
+	datasetexists $TESTPOOL/128k && log_must_busy zfs destroy $TESTPOOL/128k
+	datasetexists $TESTPOOL/1m && log_must_busy zfs destroy $TESTPOOL/1m
 	cleanup_pool $POOL2
 	destroy_pool $POOL3
 }
@@ -153,7 +153,7 @@ function check
 
 	check_recsize $recv_ds $expected_file_bs $expected_recsize
 	$do_size_test && verify_stream_size $stream $recv_ds
-	log_must zfs destroy -r $recv_ds
+	log_must_busy zfs destroy -r $recv_ds
 }
 
 log_assert "Verify compressed send works with datasets of varying recsize."