forked from openzfs/zfs
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Make zpool status counters match error events count
The number of IO and checksum events should match the number of errors seen in zpool status. Previously there was a mismatch between the two counts because zpool status would only count unrecovered errors, while zpool events would get an event for *all* errors (recovered or not). This lead to situations where disks could be faulted for "too many errors", while at the same time showing zero errors in zpool status. This fixes the zpool status error counters to increment at the same times we post the error events. Reviewed-by: Tom Caputi <[email protected]> Reviewed-by: Brian Behlendorf <[email protected]> Reviewed-by: Olaf Faaland <[email protected]> Signed-off-by: Tony Hutter <[email protected]> Closes openzfs#4851 Closes openzfs#7817
- Loading branch information
1 parent
04a3b07
commit 2bbec1c
Showing
6 changed files
with
176 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
152 changes: 152 additions & 0 deletions
152
tests/zfs-tests/tests/functional/cli_root/zpool_events/zpool_events_errors.ksh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| #!/bin/ksh -p | ||
| # CDDL HEADER START | ||
| # | ||
| # The contents of this file are subject to the terms of the | ||
| # Common Development and Distribution License (the "License"). | ||
| # You may not use this file except in compliance with the License. | ||
| # | ||
| # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | ||
| # or http://www.opensolaris.org/os/licensing. | ||
| # See the License for the specific language governing permissions | ||
| # and limitations under the License. | ||
| # | ||
| # When distributing Covered Code, include this CDDL HEADER in each | ||
| # file and include the License file at usr/src/OPENSOLARIS.LICENSE. | ||
| # If applicable, add the following below this CDDL HEADER, with the | ||
| # fields enclosed by brackets "[]" replaced with your own identifying | ||
| # information: Portions Copyright [yyyy] [name of copyright owner] | ||
| # | ||
| # CDDL HEADER END | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2018 by Lawrence Livermore National Security, LLC. | ||
| # | ||
|
|
||
| # DESCRIPTION: | ||
| # Verify the number of IO and checksum events match the error counters | ||
| # in zpool status. | ||
| # | ||
| # STRATEGY: | ||
| # 1. Create a raidz or mirror pool | ||
| # 2. Inject read/write IO errors or checksum errors | ||
| # 3. Verify the number of errors in zpool status match the corresponding | ||
| # number of error events. | ||
| # 4. Repeat for all combinations of raidz/mirror and io/checksum errors. | ||
| # | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
|
|
||
| verify_runnable "both" | ||
|
|
||
| MOUNTDIR=$TEST_BASE_DIR/mount | ||
| VDEV1=$TEST_BASE_DIR/file1 | ||
| VDEV2=$TEST_BASE_DIR/file2 | ||
| VDEV3=$TEST_BASE_DIR/file3 | ||
| POOL=error_pool | ||
| FILESIZE=$((20 * 1024 * 1024)) | ||
| OLD_CHECKSUMS=$(get_tunable zfs_checksum_events_per_second) | ||
| OLD_LEN_MAX=$(get_tunable zfs_zevent_len_max) | ||
|
|
||
| function cleanup | ||
| { | ||
| log_must set_tunable64 zfs_checksum_events_per_second $OLD_CHECKSUMS | ||
| log_must set_tunable64 zfs_zevent_len_max $OLD_LEN_MAX | ||
|
|
||
| log_must zinject -c all | ||
| log_must zpool events -c | ||
| if poolexists $POOL ; then | ||
| log_must destroy_pool $POOL | ||
| fi | ||
| log_must rm -f $VDEV1 $VDEV2 $VDEV3 | ||
| } | ||
|
|
||
| log_assert "Check that the number of zpool errors match the number of events" | ||
|
|
||
| log_onexit cleanup | ||
|
|
||
| # Set our thresholds high so we never ratelimit or drop events. | ||
| set_tunable64 zfs_checksum_events_per_second 20000 | ||
| set_tunable64 zfs_zevent_len_max 20000 | ||
|
|
||
| log_must truncate -s $MINVDEVSIZE $VDEV1 $VDEV2 $VDEV3 | ||
| log_must mkdir -p $MOUNTDIR | ||
|
|
||
| # Run error test on a specific type of pool | ||
| # | ||
| # $1: pool - raidz, mirror | ||
| # $2: test type - corrupt (checksum error), io | ||
| # $3: read, write | ||
| function do_test | ||
| { | ||
| POOLTYPE=$1 | ||
| ERR=$2 | ||
| RW=$3 | ||
|
|
||
| log_note "Testing $ERR $RW on $POOLTYPE" | ||
| log_must zpool create -f -m $MOUNTDIR -o failmode=continue $POOL $POOLTYPE $VDEV1 $VDEV2 $VDEV3 | ||
| log_must zpool events -c | ||
| log_must zfs set compression=off $POOL | ||
|
|
||
| if [ "$RW" == "read" ] ; then | ||
| log_must mkfile $FILESIZE $MOUNTDIR/file | ||
| fi | ||
|
|
||
| log_must zinject -d $VDEV1 -e $ERR -T $RW -f 100 $POOL | ||
|
|
||
| if [ "$RW" == "write" ] ; then | ||
| log_must mkfile $FILESIZE $MOUNTDIR/file | ||
| log_must zpool sync $POOL | ||
| else | ||
| log_must zpool scrub $POOL | ||
| wait_scrubbed $POOL | ||
| fi | ||
|
|
||
| log_must zinject -c all | ||
|
|
||
| # Wait for the pool to settle down and finish resilvering (if | ||
| # necessary). We want the errors to stop incrementing before we | ||
| # check the error and event counts. | ||
| while is_pool_resilvering $POOL ; do | ||
| sleep 1 | ||
| done | ||
|
|
||
| out="$(zpool status -p | grep $VDEV1)" | ||
|
|
||
| if [ "$ERR" == "corrupt" ] ; then | ||
| events=$(zpool events | grep checksum | wc -l) | ||
| val=$(echo "$out" | awk '{print $5}') | ||
| str="checksum" | ||
| elif [ "$ERR" == "io" ] ; then | ||
| allevents=$(zpool events | grep io) | ||
| events=$(echo "$allevents" | wc -l) | ||
| if [ "$RW" == "read" ] ; then | ||
| str="read IO" | ||
| val=$(echo "$out" | awk '{print $3}') | ||
| else | ||
| str="write IO" | ||
| val=$(echo "$out" | awk '{print $4}') | ||
| fi | ||
| fi | ||
|
|
||
| if [ "$val" == "0" ] || [ "$events" == "" ] ; then | ||
| log_fail "Didn't see any errors or events ($val/$events)" | ||
| fi | ||
|
|
||
| if [ "$val" != "$events" ] ; then | ||
| log_fail "$val $POOLTYPE $str errors != $events events" | ||
| else | ||
| log_note "$val $POOLTYPE $str errors == $events events" | ||
| fi | ||
|
|
||
| log_must zpool destroy $POOL | ||
| } | ||
|
|
||
| # Test all types of errors on mirror and raidz pools | ||
| for pooltype in mirror raidz ; do | ||
| do_test $pooltype corrupt read | ||
| do_test $pooltype io read | ||
| do_test $pooltype io write | ||
| done | ||
|
|
||
| log_pass "The number of errors matched the number of events" |