1 #
2 # CDDL HEADER START
3 #
4 # The contents of this file are subject to the terms of the
5 # Common Development and Distribution License (the "License").
6 # You may not use this file except in compliance with the License.
7 #
8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 # or http://www.opensolaris.org/os/licensing.
10 # See the License for the specific language governing permissions
11 # and limitations under the License.
12 #
13 # When distributing Covered Code, include this CDDL HEADER in each
14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 # If applicable, add the following below this CDDL HEADER, with the
16 # fields enclosed by brackets "[]" replaced with your own identifying
17 # information: Portions Copyright [yyyy] [name of copyright owner]
18 #
19 # CDDL HEADER END
20 #
21
22 #
23 # Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 # Use is subject to license terms.
25 #
26
27 #
28 # Copyright (c) 2013, 2016 by Delphix. All rights reserved.
29 #
30
31 . $STF_SUITE/include/libtest.shlib
32 . $STF_SUITE/tests/functional/redundancy/redundancy.cfg
33
34 function cleanup
35 {
36 if poolexists $TESTPOOL; then
37 destroy_pool $TESTPOOL
38 fi
39 typeset dir
40 for dir in $TESTDIR $BASEDIR; do
41 if [[ -d $dir ]]; then
42 log_must rm -rf $dir
43 fi
44 done
45 }
46
47 #
48 # Get random number between min and max number.
49 #
50 # $1 Minimal value
51 # $2 Maximal value
52 #
53 function random
54 {
55 typeset -i min=$1
56 typeset -i max=$2
57 typeset -i value
58
59 while true; do
60 ((value = RANDOM % (max + 1)))
61 if ((value >= min)); then
62 break
63 fi
64 done
65
66 echo $value
67 }
68
69 #
70 # Record the directories construction and checksum all the files which reside
71 # within the specified pool
72 #
73 # $1 The specified pool
74 # $2 The file which save the record.
75 #
76 function record_data
77 {
78 typeset pool=$1
79 typeset recordfile=$2
80
81 [[ -z $pool ]] && log_fail "No specified pool."
82 [[ -f $recordfile ]] && log_must rm -f $recordfile
83
84 typeset mntpnt
85 mntpnt=$(get_prop mountpoint $pool)
86 log_must eval "du -a $mntpnt > $recordfile 2>&1"
87 #
88 # When the data was damaged, checksum is failing and return 1
89 # So, will not use log_must
90 #
91 find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1
92 }
93
94 #
95 # Create test pool and fill with files and directories.
96 #
97 # $1 pool name
98 # $2 pool type
99 # $3 virtual devices number
100 #
101 function setup_test_env
102 {
103 typeset pool=$1
104 typeset keyword=$2
105 typeset -i vdev_cnt=$3
106 typeset vdevs
107
108 typeset -i i=0
109 while (( i < vdev_cnt )); do
110 vdevs="$vdevs $BASEDIR/vdev$i"
111 ((i += 1))
112 done
113
114 if [[ ! -d $BASEDIR ]]; then
115 log_must mkdir $BASEDIR
116 fi
117
118 if poolexists $pool ; then
119 destroy_pool $pool
120 fi
121
122 log_must mkfile $MINVDEVSIZE $vdevs
123
124 log_must zpool create -m $TESTDIR $pool $keyword $vdevs
125
126 log_note "Filling up the filesystem ..."
127 typeset -i ret=0
128 typeset -i i=0
129 typeset file=$TESTDIR/file
130 while true ; do
131 file_write -o create -f $file.$i \
132 -b $BLOCKSZ -c $NUM_WRITES
133 ret=$?
134 (( $ret != 0 )) && break
135 (( i = i + 1 ))
136 done
137 (($ret != 28 )) && log_note "file_write return value($ret) is unexpected."
138
139 record_data $TESTPOOL $PRE_RECORD_FILE
140 }
141
142 #
143 # Check pool status is healthy
144 #
145 # $1 pool
146 #
147 function is_healthy
148 {
149 typeset pool=$1
150
151 typeset healthy_output="pool '$pool' is healthy"
152 typeset real_output=$(zpool status -x $pool)
153
154 if [[ "$real_output" == "$healthy_output" ]]; then
155 return 0
156 else
157 typeset -i ret
158 zpool status -x $pool | grep "state:" | \
159 grep "FAULTED" >/dev/null 2>&1
160 ret=$?
161 (( $ret == 0 )) && return 1
162 typeset l_scan
163 typeset errnum
164 l_scan=$(zpool status -x $pool | grep "scan:")
165 l_scan=${l_scan##*"with"}
166 errnum=$(echo $l_scan | awk '{print $1}')
167
168 return $errnum
169 fi
170 }
171
172 #
173 # Check pool data is valid
174 #
175 # $1 pool
176 #
177 function is_data_valid
178 {
179 typeset pool=$1
180
181 record_data $pool $PST_RECORD_FILE
182 if ! diff $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then
183 return 1
184 fi
185
186 return 0
187 }
188
189 #
190 # Get the specified count devices name
191 #
192 # $1 pool name
193 # $2 devices count
194 #
195 function get_vdevs #pool cnt
196 {
197 typeset pool=$1
198 typeset -i cnt=$2
199
200 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \
201 egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \
202 egrep -v "/old$|^$pool$")
203 typeset -i i=0
204 typeset vdevs
205 while ((i < cnt)); do
206 typeset dev=$(echo $all_devs | awk '{print $1}')
207 eval all_devs=\${all_devs##*$dev}
208
209 vdevs="$dev $vdevs"
210 ((i += 1))
211 done
212
213 echo "$vdevs"
214 }
215
216 #
217 # Synchronize all the data in pool
218 #
219 # $1 pool name
220 #
221 function sync_pool #pool
222 {
223 typeset pool=$1
224
225 log_must sync
226 log_must sleep 2
227 # Flush all the pool data.
228 typeset -i ret
229 zpool scrub $pool >/dev/null 2>&1
230 ret=$?
231 (( $ret != 0 )) && \
232 log_fail "zpool scrub $pool failed."
233
234 while ! is_pool_scrubbed $pool; do
235 if is_pool_resilvered $pool ; then
236 log_fail "$pool should not be resilver completed."
237 fi
238 log_must sleep 2
239 done
240 }
241
242 #
243 # Create and replace the same name virtual device files
244 #
245 # $1 pool name
246 # $2-n virtual device files
247 #
248 function replace_missing_devs
249 {
250 typeset pool=$1
251 shift
252
253 typeset vdev
254 for vdev in $@; do
255 log_must mkfile $MINVDEVSIZE $vdev
256 log_must zpool replace -f $pool $vdev $vdev
257 while true; do
258 if ! is_pool_resilvered $pool ; then
259 log_must sleep 2
260 else
261 break
262 fi
263 done
264 done
265 }
266
267 #
268 # Damage the pool's virtual device files.
269 #
270 # $1 pool name
271 # $2 Failing devices count
272 # $3 damage vdevs method, if not null, we keep
273 # the label for the vdevs
274 #
275 function damage_devs
276 {
277 typeset pool=$1
278 typeset -i cnt=$2
279 typeset label="$3"
280 typeset vdevs
281 typeset -i bs_count=$((64 * 1024))
282
283 vdevs=$(get_vdevs $pool $cnt)
284 typeset dev
285 if [[ -n $label ]]; then
286 for dev in $vdevs; do
287 dd if=/dev/zero of=$dev seek=512 bs=1024 \
288 count=$bs_count conv=notrunc >/dev/null 2>&1
289 done
290 else
291 for dev in $vdevs; do
292 dd if=/dev/zero of=$dev bs=1024 count=$bs_count \
293 conv=notrunc >/dev/null 2>&1
294 done
295 fi
296
297 sync_pool $pool
298 }
299
300 #
301 # Clear errors in the pool caused by data corruptions
302 #
303 # $1 pool name
304 #
305 function clear_errors
306 {
307 typeset pool=$1
308
309 log_must zpool clear $pool
310
311 if ! is_healthy $pool ; then
312 log_note "$pool should be healthy."
313 return 1
314 fi
315 if ! is_data_valid $pool ; then
316 log_note "Data should be valid in $pool."
317 return 1
318 fi
319
320 return 0
321 }
322
323 #
324 # Remove the specified pool's virtual device files
325 #
326 # $1 Pool name
327 # $2 Missing devices count
328 #
329 function remove_devs
330 {
331 typeset pool=$1
332 typeset -i cnt=$2
333 typeset vdevs
334
335 vdevs=$(get_vdevs $pool $cnt)
336 log_must rm -f $vdevs
337
338 sync_pool $pool
339 }
340
341 #
342 # Recover the bad or missing device files in the pool
343 #
344 # $1 Pool name
345 # $2 Missing devices count
346 #
347 function recover_bad_missing_devs
348 {
349 typeset pool=$1
350 typeset -i cnt=$2
351 typeset vdevs
352
353 vdevs=$(get_vdevs $pool $cnt)
354 replace_missing_devs $pool $vdevs
355
356 if ! is_healthy $pool ; then
357 log_note "$pool should be healthy."
358 return 1
359 fi
360 if ! is_data_valid $pool ; then
361 log_note "Data should be valid in $pool."
362 return 1
363 fi
364
365 return 0
366 }