/usr/lib/zfs-linux/zfs/zed.d/checksum-spare.sh is in zfs-zed 0.6.5.6-0ubuntu8.
This file is owned by root:root, with mode 0o755.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 | #!/bin/sh
#
# Replace a device with a hot spare in response to IO or CHECKSUM errors.
# The following actions will be performed automatically when the number
# of errors exceed the limit set by ZED_SPARE_ON_IO_ERRORS or
# ZED_SPARE_ON_CHECKSUM_ERRORS.
#
# 1) FAULT the device on IO errors, no futher IO will be attempted.
# DEGRADE the device on checksum errors, the device is still
# functional and can be used to service IO requests.
# 2) Set the SES fault beacon for the device.
# 3) Replace the device with a hot spare if any are available.
#
# Once the hot sparing operation is complete either the failed device or
# the hot spare must be manually retired using the 'zpool detach' command.
# The 'autoreplace' functionality which would normally take care of this
# under Illumos has not yet been implemented.
#
# Full support for autoreplace is planned, but it requires that the full
# ZFS Diagnosis Engine be ported. In the meanwhile this script provides
# the majority of the expected hot spare functionality.
#
# Exit codes:
# 0: hot spare replacement successful
# 1: hot spare device not available
# 2: hot sparing disabled or threshold not reached
# 3: device already faulted or degraded
# 9: internal error
[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
. "${ZED_ZEDLET_DIR}/zed-functions.sh"
# Disabled by default. Enable in the zed.rc file.
: "${ZED_SPARE_ON_CHECKSUM_ERRORS:=0}"
: "${ZED_SPARE_ON_IO_ERRORS:=0}"
# query_vdev_status (pool, vdev)
#
# Given a [pool] and [vdev], return the matching vdev path & status on stdout.
#
# Warning: This function does not handle the case of [pool] or [vdev]
# containing whitespace. Beware of ShellCheck SC2046. Caveat emptor.
#
# Arguments
# pool: pool name
# vdev: virtual device name
#
# StdOut
# arg1: vdev pathname
# arg2: vdev status
#
query_vdev_status()
{
local pool="$1"
local vdev="$2"
local t
vdev="$(basename -- "${vdev}")"
([ -n "${pool}" ] && [ -n "${vdev}" ]) || return
t="$(printf '\t')"
"${ZPOOL}" status "${pool}" 2>/dev/null | sed -n -e \
"s,^[ $t]*\(.*${vdev}\(-part[0-9]\+\)\?\)[ $t]*\([A-Z]\+\).*,\1 \3,p" \
| tail -1
}
# notify (old_vdev, new_vdev, num_errors)
#
# Send a notification regarding the hot spare replacement.
#
# Arguments
# old_vdev: path of old vdev that has failed
# new_vdev: path of new vdev used as the hot spare replacement
# num_errors: number of errors that triggered this replacement
#
notify()
{
local old_vdev="$1"
local new_vdev="$2"
local num_errors="$3"
local note_subject
local note_pathname
local s
local rv
umask 077
note_subject="ZFS hot spare replacement for ${ZEVENT_POOL} on $(hostname)"
note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
{
[ "${num_errors}" -ne 1 ] 2>/dev/null && s="s"
echo "ZFS has replaced a failing device with a hot spare after" \
"${num_errors} ${ZEVENT_SUBCLASS} error${s}:"
echo
echo " eid: ${ZEVENT_EID}"
echo " class: ${ZEVENT_SUBCLASS}"
echo " host: $(hostname)"
echo " time: ${ZEVENT_TIME_STRING}"
echo " old: ${old_vdev}"
echo " new: ${new_vdev}"
"${ZPOOL}" status "${ZEVENT_POOL}"
} > "${note_pathname}"
zed_notify "${note_subject}" "${note_pathname}"; rv=$?
rm -f "${note_pathname}"
return "${rv}"
}
# main
#
# Arguments
# none
#
# Return
# see above
#
main()
{
local num_errors
local action
local lockfile
local vdev_path
local vdev_status
local spare
local spare_path
local spare_status
local zpool_err
local zpool_rv
local rv
# Avoid hot-sparing a hot-spare.
#
# Note: ZEVENT_VDEV_PATH is not defined for ZEVENT_VDEV_TYPE=spare.
#
[ "${ZEVENT_VDEV_TYPE}" = "spare" ] && exit 2
[ -n "${ZEVENT_POOL}" ] || exit 9
[ -n "${ZEVENT_VDEV_GUID}" ] || exit 9
[ -n "${ZEVENT_VDEV_PATH}" ] || exit 9
zed_check_cmd "${ZPOOL}" "${ZINJECT}" || exit 9
# Fault the device after a given number of I/O errors.
#
if [ "${ZEVENT_SUBCLASS}" = "io" ]; then
if [ "${ZED_SPARE_ON_IO_ERRORS}" -gt 0 ]; then
num_errors=$((ZEVENT_VDEV_READ_ERRORS + ZEVENT_VDEV_WRITE_ERRORS))
[ "${num_errors}" -ge "${ZED_SPARE_ON_IO_ERRORS}" ] \
&& action="fault"
fi 2>/dev/null
# Degrade the device after a given number of checksum errors.
#
elif [ "${ZEVENT_SUBCLASS}" = "checksum" ]; then
if [ "${ZED_SPARE_ON_CHECKSUM_ERRORS}" -gt 0 ]; then
num_errors="${ZEVENT_VDEV_CKSUM_ERRORS}"
[ "${num_errors}" -ge "${ZED_SPARE_ON_CHECKSUM_ERRORS}" ] \
&& action="degrade"
fi 2>/dev/null
else
zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
exit 9
fi
# Error threshold not reached.
#
if [ -z "${action}" ]; then
exit 2
fi
lockfile="zed.spare.lock"
zed_lock "${lockfile}"
# shellcheck disable=SC2046
set -- $(query_vdev_status "${ZEVENT_POOL}" "${ZEVENT_VDEV_PATH}")
vdev_path="$1"
vdev_status="$2"
# Device is already FAULTED or DEGRADED.
#
if [ "${vdev_status}" = "FAULTED" ] \
|| [ "${vdev_status}" = "DEGRADED" ]; then
rv=3
else
rv=1
# 1) FAULT or DEGRADE the device.
#
"${ZINJECT}" -d "${ZEVENT_VDEV_GUID}" -A "${action}" "${ZEVENT_POOL}"
# 2) Set the SES fault beacon.
#
# TODO: Set the 'fault' or 'ident' beacon for the device. This can
# be done through the sg_ses utility. The only hard part is to map
# the sd device to its corresponding enclosure and slot. We may
# be able to leverage the existing vdev_id scripts for this.
#
# $ sg_ses --dev-slot-num=0 --set=ident /dev/sg3
# $ sg_ses --dev-slot-num=0 --clear=ident /dev/sg3
# 3) Replace the device with a hot spare.
#
# Round-robin through the spares trying those that are available.
#
for spare in ${ZEVENT_VDEV_SPARE_PATHS}; do
# shellcheck disable=SC2046
set -- $(query_vdev_status "${ZEVENT_POOL}" "${spare}")
spare_path="$1"
spare_status="$2"
[ "${spare_status}" = "AVAIL" ] || continue
zpool_err="$("${ZPOOL}" replace "${ZEVENT_POOL}" \
"${ZEVENT_VDEV_GUID}" "${spare_path}" 2>&1)"; zpool_rv=$?
if [ "${zpool_rv}" -ne 0 ]; then
[ -n "${zpool_err}" ] && zed_log_err "zpool ${zpool_err}"
else
notify "${vdev_path}" "${spare_path}" "${num_errors}"
rv=0
break
fi
done
fi
zed_unlock "${lockfile}"
exit "${rv}"
}
main "$@"
|