OpenZFS - Performance regression suite for zfstest

Author: John Wren Kennedy <[email protected]>
Reviewed by: Prakash Surya <[email protected]>
Reviewed by: Dan Kimmel <[email protected]>
Reviewed by: Matt Ahrens <[email protected]>
Reviewed by: Paul Dagnelie <[email protected]>
Reviewed by: Don Brady <[email protected]>
Reviewed by: Richard Elling <[email protected]>
Reviewed by: Brian Behlendorf <[email protected]>
Reviewed-by: David Quigley <[email protected]>
Approved by: Richard Lowe <[email protected]>
Ported-by: Don Brady <[email protected]>

OpenZFS-issue: https://www.illumos.org/issues/6950
OpenZFS-commit: openzfs/openzfs@dcbf3bd6
Delphix-commit: delphix/delphix-os@978ed49
Closes openzfs#4929

ZFS Test Suite Performance Regression Tests

This was pulled into OpenZFS via the compressed arc featureand was
separated out in zfsonlinux as a separate pull request from PR-4768.
It originally came in as QA-4903 in Delphix-OS from John Kennedy.

Expected Usage:

$ DISKS="sdb sdc sdd" zfs-tests.sh -r perf-regression.run

Porting Notes:
1. Added assertions in the setup script to make sure required tools
   (fio, mpstat, ...) are present.
2. For the config.json generation in perf.shlib used arcstats and
    other binaries instead of dtrace to query the values.
3. For the perf data collection:
   - use "zpool iostat -lpvyL" instead of the io.d dtrace script
    (currently not collecting zfs_read/write latency stats)
   - mpstat and iostat take different arguments
   - prefetch_io.sh is a placeholder that uses arcstats instead of
     dtrace
4. Build machines require fio, mdadm and sysstat pakage (YMMV).

Future Work:
   - Need a way to measure zfs_read and zfs_write latencies per pool.
   - Need tools to takes two sets of output and display/graph the
     differences
   - Bring over additional regression tests from Delphix

config/user-commands.m4

@@ -26,10 +26,12 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_COMMON], [
 	AC_PATH_TOOL(DU, du, "")
 	AC_PATH_TOOL(ECHO, echo, "")
 	AC_PATH_TOOL(EGREP, egrep, "")
+	AC_PATH_TOOL(FALSE, false, "")
 	AC_PATH_TOOL(FDISK, fdisk, "")
 	AC_PATH_TOOL(FGREP, fgrep, "")
 	AC_PATH_TOOL(FILE, file, "")
 	AC_PATH_TOOL(FIND, find, "")
+	AC_PATH_TOOL(FIO, fio, "")
 	AC_PATH_TOOL(FSCK, fsck, "")
 	AC_PATH_TOOL(GNUDD, dd, "")
 	AC_PATH_TOOL(GETCONF, getconf, "")
@@ -74,6 +76,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_COMMON], [
 	AC_PATH_TOOL(SYNC, sync, "")
 	AC_PATH_TOOL(TAIL, tail, "")
 	AC_PATH_TOOL(TAR, tar, "")
+	AC_PATH_TOOL(TIMEOUT, timeout, "")
 	AC_PATH_TOOL(TOUCH, touch, "")
 	AC_PATH_TOOL(TR, tr, "")
 	AC_PATH_TOOL(TRUNCATE, truncate, "")
@@ -96,14 +99,21 @@ dnl # These commands may take different command line arguments.
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_LINUX], [
 	AC_PATH_TOOL(BLOCKDEV, blockdev, "")
+	AC_PATH_TOOL(CHACL, chacl, "")
 	AC_PATH_TOOL(COMPRESS, gzip, "")
 	AC_PATH_TOOL(FORMAT, parted, "")
+	AC_PATH_TOOL(FREE, free, "")
+	AC_PATH_TOOL(GETFACL, getfacl, "")
+	AC_PATH_TOOL(IOSTAT, iostat, "")
 	AC_PATH_TOOL(LOCKFS, lsof, "")
 	AC_PATH_TOOL(LSBLK, lsblk, "")
 	AC_PATH_TOOL(MODUNLOAD, rmmod, "")
+	AC_PATH_TOOL(MPSTAT, mpstat, "")
 	AC_PATH_TOOL(NEWFS, mke2fs, "")
+	AC_PATH_TOOL(NPROC, nproc, "")
 	AC_PATH_TOOL(PFEXEC, sudo, "")
 	AC_PATH_TOOL(READLINK, readlink, "")
+	AC_PATH_TOOL(SETFACL, setfacl, "")
 	AC_PATH_TOOL(SHARE, exportfs, "")
 	AC_PATH_TOOL(SWAP, swapon, "")
 	AC_PATH_TOOL(SWAPADD, swapon, "")
@@ -112,10 +122,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_LINUX], [
 	AC_PATH_TOOL(UFSRESTORE, restore, "")
 	AC_PATH_TOOL(UNCOMPRESS, gunzip, "")
 	AC_PATH_TOOL(UNSHARE, exportfs, "")
-	AC_PATH_TOOL(GETFACL, getfacl, "")
-	AC_PATH_TOOL(SETFACL, setfacl, "")
-	AC_PATH_TOOL(CHACL, chacl, "")
-	AC_PATH_TOOL(NPROC, nproc, "")
+	AC_PATH_TOOL(VMSTAT, vmstat, "")
 
 	PAGESIZE=$($GETCONF PAGESIZE)
 	AC_SUBST(PAGESIZE)

configure.ac

@@ -280,6 +280,10 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/zvol/zvol_ENOSPC/Makefile
 	tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile
 	tests/zfs-tests/tests/functional/zvol/zvol_swap/Makefile
+	tests/zfs-tests/tests/perf/Makefile
+	tests/zfs-tests/tests/perf/fio/Makefile
+	tests/zfs-tests/tests/perf/regression/Makefile
+	tests/zfs-tests/tests/perf/scripts/Makefile
 	tests/zfs-tests/tests/stress/Makefile
 	rpm/Makefile
 	rpm/redhat/Makefile

rpm/generic/zfs.spec.in

@@ -174,6 +174,9 @@ Requires:       lsscsi
 Requires:       mdadm
 Requires:       bc
 Requires:       ksh
+Requires:	fio
+Requires:	acl
+Requires:	sysstat
 
 %description test
 This package contains test infrastructure and support scripts for

scripts/zfs2zol-patch.sed

@@ -13,3 +13,9 @@ s:usr/src/lib/libzfs/common:lib/libzfs:g
 s:usr/src/lib/libzfs_core/common:lib/libzfs_core:g
 s:lib/libzpool/common/sys:include/sys:g
 s:lib/libzpool/common:lib/libzpool:g
+
+s:usr/src/test/zfs-tests/include:tests/zfs-tests/include:g
+s:usr/src/test/zfs-tests/runfiles:tests/runfiles:g
+s:usr/src/test/zfs-tests/tests/functional:tests/zfs-tests/tests/functional:g
+s:usr/src/test/zfs-tests/tests/perf:tests/zfs-tests/tests/perf:g
+s:usr/src/test/test-runner/cmd/run.py:tests/test-runner/cmd/test-runner.py:g

tests/runfiles/perf-regression.run

@@ -0,0 +1,30 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+[DEFAULT]
+pre = setup
+quiet = False
+pre_user = root
+user = root
+timeout = 0
+post_user = root
+post = cleanup
+outputdir = /var/tmp/test_results
+
+[tests/perf/regression]
+tests = ['sequential_writes', 'sequential_reads', 'sequential_reads_cached',
+    'sequential_reads_cached_clone', 'random_reads', 'random_writes',
+    'random_readwrite']
+post =

tests/test-runner/cmd/test-runner.py

@@ -26,6 +26,7 @@
 from subprocess import PIPE
 from subprocess import Popen
 from sys import argv
+from sys import maxint
 from sys import exit
 from threading import Timer
 from time import time
@@ -121,13 +122,16 @@ class Cmd(object):
     def __init__(self, pathname, outputdir=None, timeout=None, user=None):
         self.pathname = pathname
         self.outputdir = outputdir or 'BASEDIR'
-        self.timeout = timeout or 60
+        self.timeout = timeout
         self.user = user or ''
         self.killed = False
         self.result = Result()
 
+        if self.timeout == None:
+	    self.timeout = 60
+
     def __str__(self):
-        return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % (
+        return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % (
                 self.pathname, self.outputdir, self.timeout, self.user)
 
     def kill_cmd(self, proc):
@@ -213,6 +217,9 @@ def run(self, options):
 
         self.result.starttime = time()
         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
+        # Allow a special timeout value of 0 to mean infinity
+        if int(self.timeout) == 0:
+            self.timeout = maxint
         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 
         try:
@@ -301,7 +308,7 @@ def __str__(self):
             pre_user = ' (as %s)' % (self.pre_user)
         if len(self.post_user):
             post_user = ' (as %s)' % (self.post_user)
-        return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nPre: %s%s\nPost: " \
+        return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
                "%s%s\nUser: %s\n" % (self.pathname, self.outputdir,
                 self.timeout, self.pre, pre_user, self.post, post_user,
                 self.user)
@@ -376,7 +383,7 @@ def __str__(self):
             pre_user = ' (as %s)' % (self.pre_user)
         if len(self.post_user):
             post_user = ' (as %s)' % (self.post_user)
-        return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
+        return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \
                "Pre: %s%s\nPost: %s%s\nUser: %s\n" % (self.pathname,
                 self.outputdir, self.tests, self.timeout, self.pre, pre_user,
                 self.post, post_user, self.user)

tests/zfs-tests/include/commands.cfg.in

@@ -27,11 +27,14 @@ export DU="@DU@"
 export DUMPADM="@DUMPADM@"
 export ECHO="@ECHO@"
 export EGREP="@EGREP@"
+export FALSE="@FALSE@"
 export FDISK="@FDISK@"
 export FGREP="@FGREP@"
 export FILE="@FILE@"
 export FIND="@FIND@"
+export FIO="@FIO@"
 export FORMAT="@FORMAT@"
+export FREE="@FREE@"
 export FSCK="@FSCK@"
 export GETENT="@GETENT@"
 export GETFACL="@GETFACL@"
@@ -44,6 +47,7 @@ export GROUPMOD="@GROUPMOD@"
 export HEAD="@HEAD@"
 export HOSTNAME="@HOSTNAME@"
 export ID="@ID@"
+export IOSTAT="@IOSTAT@"
 export ISAINFO="@ISAINFO@"
 export KILL="@KILL@"
 export KSH="@KSH@"
@@ -61,6 +65,7 @@ export MNTTAB="@MNTTAB@"
 export MODINFO="@MODINFO@"
 export MODUNLOAD="@MODUNLOAD@"
 export MOUNT="@MOUNT@"
+export MPSTAT="@MPSTAT@"
 export MV="@MV@"
 export NAWK="@AWK@"
 export NEWFS="@NEWFS@"
@@ -98,6 +103,7 @@ export SWAPADD="@SWAPADD@"
 export SYNC="@SYNC@"
 export TAIL="@TAIL@"
 export TAR="@TAR@"
+export TIMEOUT="@TIMEOUT@"
 export TOUCH="@TOUCH@"
 export TR="@TR@"
 export TRUNCATE="@TRUNCATE@"
@@ -115,6 +121,7 @@ export UNSHARE="@UNSHARE@"
 export USERADD="@USERADD@"
 export USERDEL="@USERDEL@"
 export USERMOD="@USERMOD@"
+export VMSTAT="@VMSTAT@"
 export WAIT="@WAIT@"
 export WC="@WC@"
 export ZONEADM="@ZONEADM@"

tests/zfs-tests/include/default.cfg.in

@@ -104,12 +104,15 @@ export COMPRESSION_PROP=on
 export CHECKSUM_PROP=on
 
 # some common variables used by test scripts :
+export FIO_SCRIPTS=$STF_SUITE/tests/perf/fio
+export PERF_SCRIPTS=$STF_SUITE/tests/perf/scripts
 
 # some test pool names
 export TESTPOOL=testpool.$$
 export TESTPOOL1=testpool1.$$
 export TESTPOOL2=testpool2.$$
 export TESTPOOL3=testpool3.$$
+export PERFPOOL=perfpool
 
 # some test file system names
 export TESTFS=testfs.$$

tests/zfs-tests/include/libtest.shlib

@@ -2809,6 +2809,30 @@ function vdevs_in_pool
 	return 0;
 }
 
+function get_max
+{
+	typeset -l i max=$1
+	shift
+
+	for i in "$@"; do
+		max=$(echo $((max > i ? max : i)))
+	done
+
+	echo $max
+}
+
+function get_min
+{
+	typeset -l i min=$1
+	shift
+
+	for i in "$@"; do
+		min=$(echo $((min < i ? min : i)))
+	done
+
+	echo $min
+}
+
 #
 # Wait for newly created block devices to have their minors created.
 #

tests/zfs-tests/tests/Makefile.am

@@ -1 +1 @@
-SUBDIRS = functional stress
+SUBDIRS = functional perf stress

tests/zfs-tests/tests/functional/clean_mirror/clean_mirror_common.kshlib

@@ -25,7 +25,7 @@
 #
 
 #
-# Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright (c) 2013, 2015 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/tests/functional/clean_mirror/default.cfg
@@ -36,6 +36,32 @@
 # the contents of the mirror.
 # This code is sourced into each of these test cases.
 
+#
+# Synchronize all the data in pool
+#
+# $1 pool name
+#
+function sync_pool #pool
+{
+	typeset pool=$1
+
+	log_must $SYNC
+	log_must $SLEEP 2
+	# Flush all the pool data.
+	typeset -i ret
+	$ZPOOL scrub $pool >/dev/null 2>&1
+	ret=$?
+	(( $ret != 0 )) && \
+		log_fail "$ZPOOL scrub $pool failed."
+
+	while ! is_pool_scrubbed $pool; do
+		if is_pool_resilvered $pool ; then
+			log_fail "$pool should not be resilver completed."
+		fi
+		log_must $SLEEP 2
+	done
+}
+
 function overwrite_verify_mirror
 {
 	typeset AFFECTED_DEVICE=$1
@@ -60,6 +86,12 @@ function overwrite_verify_mirror
 
 	atfile=0
 
+	#
+	# Flush out the cache so that we ensure we're reading from disk.
+	#
+	log_must $ZPOOL export $TESTPOOL
+	log_must $ZPOOL import $TESTPOOL
+
 	typeset -i failedcount=0
 	while (( atfile < FILE_COUNT )); do
 		files[$atfile]=$TESTDIR/file.$atfile
@@ -75,4 +107,6 @@ function overwrite_verify_mirror
 		log_fail "of the $FILE_COUNT files $failedcount did not " \
 		    "have the same checksum before and after."
 	fi
+
+	sync_pool $TESTPOOL
 }

tests/zfs-tests/tests/perf/Makefile.am

@@ -0,0 +1,7 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf
+dist_pkgdata_SCRIPTS = perf.shlib
+
+SUBDIRS = \
+	fio \
+	regression \
+	scripts

tests/zfs-tests/tests/perf/fio/Makefile.am

@@ -0,0 +1,8 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf/fio
+dist_pkgdata_SCRIPTS = \
+	mkfiles.fio \
+	random_reads.fio \
+	random_readwrite.fio \
+	random_writes.fio \
+	sequential_reads.fio \
+	sequential_writes.fio

tests/zfs-tests/tests/perf/fio/mkfiles.fio

@@ -0,0 +1,30 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+[global]
+filename_format=file$jobnum
+group_reporting=1
+fallocate=0
+ioengine=psync
+bs=1024k
+rw=write
+thread=1
+directory=/${TESTFS}
+numjobs=${NUMJOBS}
+filesize=${FILE_SIZE}
+buffer_compress_percentage=33
+buffer_compress_chunk=4096
+
+[job]