1 #
2 # CDDL HEADER START
3 #
4 # The contents of this file are subject to the terms of the
5 # Common Development and Distribution License (the "License").
6 # You may not use this file except in compliance with the License.
7 #
8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 # or http://www.opensolaris.org/os/licensing.
10 # See the License for the specific language governing permissions
11 # and limitations under the License.
12 #
13 # When distributing Covered Code, include this CDDL HEADER in each
14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 # If applicable, add the following below this CDDL HEADER, with the
16 # fields enclosed by brackets "[]" replaced with your own identifying
17 # information: Portions Copyright [yyyy] [name of copyright owner]
18 #
19 # CDDL HEADER END
20 #
21
22 #
23 # Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 # Use is subject to license terms.
25 #
26
27 #
28 # Copyright (c) 2013, 2016 by Delphix. All rights reserved.
29 #
30
31 . $STF_SUITE/include/libtest.shlib
32 . $STF_SUITE/tests/functional/rsend/rsend.cfg
33
34 #
35 # Set up test model which includes various datasets
36 #
37 # @final
38 # @snapB
39 # @init
40 # |
41 # ______ pclone
42 # | /
43 # |@psnap
44 # || @final
45 # ||@final @final @snapC
46 # ||@snapC @snapC @snapB
47 # ||@snapA @snapB @snapA
48 # ||@init @init @init
49 # ||| | |
50 # $pool -------- $FS ------- fs1 ------- fs2
51 # \ \\_____ \ |
52 # vol vol \____ \ @fsnap
53 # | | \ \ \
54 # @init @vsnap | ------------ fclone
55 # @snapA @init \ | |
56 # @final @snapB \ | @init
57 # @snapC vclone @snapA
58 # @final | @final
59 # @init
60 # @snapC
61 # @final
62 #
63 # $1 pool name
64 #
65 function setup_test_model
66 {
67 typeset pool=$1
68
69 log_must zfs create -p $pool/$FS/fs1/fs2
70
71 log_must zfs snapshot $pool@psnap
72 log_must zfs clone $pool@psnap $pool/pclone
73
74 if is_global_zone ; then
75 log_must zfs create -V 16M $pool/vol
76 log_must zfs create -V 16M $pool/$FS/vol
77
78 log_must zfs snapshot $pool/$FS/vol@vsnap
79 log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
80 fi
81
82 log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
83 log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
84 log_must zfs snapshot -r $pool@init
85
86 log_must snapshot_tree $pool@snapA
87 log_must snapshot_tree $pool@snapC
88 log_must snapshot_tree $pool/pclone@snapB
89 log_must snapshot_tree $pool/$FS@snapB
90 log_must snapshot_tree $pool/$FS@snapC
91 log_must snapshot_tree $pool/$FS/fs1@snapA
92 log_must snapshot_tree $pool/$FS/fs1@snapB
93 log_must snapshot_tree $pool/$FS/fs1@snapC
94 log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
95
96 if is_global_zone ; then
97 log_must zfs snapshot $pool/vol@snapA
98 log_must zfs snapshot $pool/$FS/vol@snapB
99 log_must zfs snapshot $pool/$FS/vol@snapC
100 log_must zfs snapshot $pool/$FS/vclone@snapC
101 fi
102
103 log_must zfs snapshot -r $pool@final
104
105 return 0
106 }
107
108 #
109 # Cleanup the BACKDIR and given pool content and all the sub datasets
110 #
111 # $1 pool name
112 #
113 function cleanup_pool
114 {
115 typeset pool=$1
116 log_must rm -rf $BACKDIR/*
117
118 if is_global_zone ; then
119 log_must zfs destroy -Rf $pool
120 else
121 typeset list=$(zfs list -H -r -t filesystem,snapshot,volume -o name $pool)
122 for ds in $list ; do
123 if [[ $ds != $pool ]] ; then
124 if datasetexists $ds ; then
125 log_must zfs destroy -Rf $ds
126 fi
127 fi
128 done
129 fi
130
131 typeset mntpnt=$(get_prop mountpoint $pool)
132 if ! ismounted $pool ; then
133 # Make sure mountpoint directory is empty
134 if [[ -d $mntpnt ]]; then
135 log_must rm -rf $mntpnt/*
136 fi
137
138 log_must zfs mount $pool
139 fi
140 if [[ -d $mntpnt ]]; then
141 log_must rm -rf $mntpnt/*
142 fi
143
144 return 0
145 }
146
147 #
148 # Detect if the given two filesystems have same sub-datasets
149 #
150 # $1 source filesystem
151 # $2 destination filesystem
152 #
153 function cmp_ds_subs
154 {
155 typeset src_fs=$1
156 typeset dst_fs=$2
157
158 zfs list -r -H -t filesystem,snapshot,volume -o name $src_fs > $BACKDIR/src1
159 zfs list -r -H -t filesystem,snapshot,volume -o name $dst_fs > $BACKDIR/dst1
160
161 eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
162 eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
163
164 diff $BACKDIR/src $BACKDIR/dst
165 typeset -i ret=$?
166
167 rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
168
169 return $ret
170 }
171
172 #
173 # Compare all the directores and files in two filesystems
174 #
175 # $1 source filesystem
176 # $2 destination filesystem
177 #
178 function cmp_ds_cont
179 {
180 typeset src_fs=$1
181 typeset dst_fs=$2
182
183 typeset srcdir dstdir
184 srcdir=$(get_prop mountpoint $src_fs)
185 dstdir=$(get_prop mountpoint $dst_fs)
186
187 diff -r $srcdir $dstdir > /dev/null 2>&1
188 echo $?
189 }
190
191 #
192 # Compare the given two dataset properties
193 #
194 # $1 dataset 1
195 # $2 dataset 2
196 #
197 function cmp_ds_prop
198 {
199 typeset dtst1=$1
200 typeset dtst2=$2
201
202 for item in "type" "origin" "volblocksize" "aclinherit" "aclmode" \
203 "atime" "canmount" "checksum" "compression" "copies" "devices" \
204 "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
205 "sharenfs" "snapdir" "version" "volsize" "xattr" "zoned" \
206 "mountpoint";
207 do
208 zfs get -H -o property,value,source $item $dtst1 >> \
209 $BACKDIR/dtst1
210 zfs get -H -o property,value,source $item $dtst2 >> \
211 $BACKDIR/dtst2
212 done
213
214 eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
215 eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
216
217 diff $BACKDIR/dtst1 $BACKDIR/dtst2
218 typeset -i ret=$?
219
220 rm -f $BACKDIR/dtst1 $BACKDIR/dtst2
221
222 return $ret
223
224 }
225
226 #
227 # Random create directories and files
228 #
229 # $1 directory
230 #
231 function random_tree
232 {
233 typeset dir=$1
234
235 if [[ -d $dir ]]; then
236 rm -rf $dir
237 fi
238 mkdir -p $dir
239 typeset -i ret=$?
240
241 typeset -i nl nd nf
242 ((nl = RANDOM % 6 + 1))
243 ((nd = RANDOM % 3 ))
244 ((nf = RANDOM % 5 ))
245 mktree -b $dir -l $nl -d $nd -f $nf
246 ((ret |= $?))
247
248 return $ret
249 }
250
251 #
252 # Put data in filesystem and take snapshot
253 #
254 # $1 snapshot name
255 #
256 function snapshot_tree
257 {
258 typeset snap=$1
259 typeset ds=${snap%%@*}
260 typeset type=$(get_prop "type" $ds)
261
262 typeset -i ret=0
263 if [[ $type == "filesystem" ]]; then
264 typeset mntpnt=$(get_prop mountpoint $ds)
265 ((ret |= $?))
266
267 if ((ret == 0)) ; then
268 eval random_tree $mntpnt/${snap##$ds}
269 ((ret |= $?))
270 fi
271 fi
272
273 if ((ret == 0)) ; then
274 zfs snapshot $snap
275 ((ret |= $?))
276 fi
277
278 return $ret
279 }
280
281 #
282 # Destroy the given snapshot and stuff
283 #
284 # $1 snapshot
285 #
286 function destroy_tree
287 {
288 typeset -i ret=0
289 typeset snap
290 for snap in "$@" ; do
291 zfs destroy $snap
292 ret=$?
293
294 typeset ds=${snap%%@*}
295 typeset type=$(get_prop "type" $ds)
296 if [[ $type == "filesystem" ]]; then
297 typeset mntpnt=$(get_prop mountpoint $ds)
298 ((ret |= $?))
299
300 if ((ret != 0)); then
301 rm -r $mntpnt/$snap
302 ((ret |= $?))
303 fi
304 fi
305
306 if ((ret != 0)); then
307 return $ret
308 fi
309 done
310
311 return 0
312 }
313
314 #
315 # Get all the sub-datasets of give dataset with specific suffix
316 #
317 # $1 Given dataset
318 # $2 Suffix
319 #
320 function getds_with_suffix
321 {
322 typeset ds=$1
323 typeset suffix=$2
324
325 typeset list=$(zfs list -r -H -t filesystem,snapshot,volume -o name $ds \
326 | grep "$suffix$")
327
328 echo $list
329 }
330
331 #
332 # Output inherited properties whitch is edited for file system
333 #
334 function fs_inherit_prop
335 {
336 typeset fs_prop
337 if is_global_zone ; then
338 fs_prop=$(zfs inherit 2>&1 | \
339 awk '$2=="YES" && $3=="YES" {print $1}')
340 if ! is_te_enabled ; then
341 fs_prop=$(echo $fs_prop | grep -v "mlslabel")
342 fi
343 else
344 fs_prop=$(zfs inherit 2>&1 | \
345 awk '$2=="YES" && $3=="YES" {print $1}'|
346 egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
347 fi
348
349 echo $fs_prop
350 }
351
352 #
353 # Output inherited properties for volume
354 #
355 function vol_inherit_prop
356 {
357 echo "checksum readonly"
358 }
359
360 #
361 # Get the destination dataset to compare
362 #
363 function get_dst_ds
364 {
365 typeset srcfs=$1
366 typeset dstfs=$2
367
368 #
369 # If the srcfs is not pool
370 #
371 if ! zpool list $srcfs > /dev/null 2>&1 ; then
372 eval dstfs="$dstfs/${srcfs#*/}"
373 fi
374
375 echo $dstfs
376 }
377
378 #
379 # Make test files
380 #
381 # $1 Number of files to create
382 # $2 Maximum file size
383 # $3 File ID offset
384 # $4 File system to create the files on
385 #
386 function mk_files
387 {
388 nfiles=$1
389 maxsize=$2
390 file_id_offset=$3
391 fs=$4
392
393 for ((i=0; i<$nfiles; i=i+1)); do
394 dd if=/dev/urandom \
395 of=/$fs/file-$maxsize-$((i+$file_id_offset)) \
396 bs=$(($RANDOM * $RANDOM % $maxsize)) \
397 count=1 >/dev/null 2>&1 || log_fail \
398 "Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))"
399 done
400 echo Created $nfiles files of random sizes up to $maxsize bytes
401 }
402
403 #
404 # Remove test files
405 #
406 # $1 Number of files to remove
407 # $2 Maximum file size
408 # $3 File ID offset
409 # $4 File system to remove the files from
410 #
411 function rm_files
412 {
413 nfiles=$1
414 maxsize=$2
415 file_id_offset=$3
416 fs=$4
417
418 for ((i=0; i<$nfiles; i=i+1)); do
419 rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
420 done
421 echo Removed $nfiles files of random sizes up to $maxsize bytes
422 }
423
424 #
425 # Mess up file contents
426 #
427 # $1 The file path
428 #
429 function mess_file
430 {
431 file=$1
432
433 filesize=$(stat -c '%s' $file)
434 offset=$(($RANDOM * $RANDOM % $filesize))
435 if (($RANDOM % 7 <= 1)); then
436 #
437 # We corrupt 2 bytes to minimize the chance that we
438 # write the same value that's already there.
439 #
440 log_must eval "dd if=/dev/random of=$file conv=notrunc " \
441 "bs=1 count=2 oseek=$offset >/dev/null 2>&1"
442 else
443 log_must truncate -s $offset $file
444 fi
445 }
446
447 #
448 # Diff the send/receive filesystems
449 #
450 # $1 The sent filesystem
451 # $2 The received filesystem
452 #
453 function file_check
454 {
455 sendfs=$1
456 recvfs=$2
457
458 if [[ -d /$recvfs/.zfs/snapshot/a && -d \
459 /$sendfs/.zfs/snapshot/a ]]; then
460 diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
461 [[ $? -eq 0 ]] || log_fail "Differences found in snap a"
462 fi
463 if [[ -d /$recvfs/.zfs/snapshot/b && -d \
464 /$sendfs/.zfs/snapshot/b ]]; then
465 diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
466 [[ $? -eq 0 ]] || log_fail "Differences found in snap b"
467 fi
468 }
469
470 #
471 # Resume test helper
472 #
473 # $1 The ZFS send command
474 # $2 The filesystem where the streams are sent
475 # $3 The receive filesystem
476 #
477 function resume_test
478 {
479 sendcmd=$1
480 streamfs=$2
481 recvfs=$3
482
483 stream_num=1
484 log_must eval "$sendcmd >/$streamfs/$stream_num"
485
486 for ((i=0; i<2; i=i+1)); do
487 mess_file /$streamfs/$stream_num
488 log_mustnot zfs recv -sv $recvfs </$streamfs/$stream_num
489 stream_num=$((stream_num+1))
490
491 token=$(zfs get -Hp -o value receive_resume_token $recvfs)
492 log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
493 [[ -f /$streamfs/$stream_num ]] || \
494 log_fail "NO FILE /$streamfs/$stream_num"
495 done
496 log_must zfs recv -sv $recvfs </$streamfs/$stream_num
497 }
498
499 #
500 # Setup filesystems for the resumable send/receive tests
501 #
502 # $1 The pool to set up with the "send" filesystems
503 # $2 The pool for receive
504 #
505 function test_fs_setup
506 {
507 sendpool=$1
508 recvpool=$2
509
510 sendfs=$sendpool/sendfs
511 recvfs=$recvpool/recvfs
512 streamfs=$sendpool/stream
513
514 if datasetexists $recvfs; then
515 log_must zfs destroy -r $recvfs
516 fi
517 if datasetexists $sendfs; then
518 log_must zfs destroy -r $sendfs
519 fi
520 if $(zfs create -o compress=lz4 $sendfs); then
521 mk_files 1000 256 0 $sendfs &
522 mk_files 1000 131072 0 $sendfs &
523 mk_files 100 1048576 0 $sendfs &
524 mk_files 10 10485760 0 $sendfs &
525 mk_files 1 104857600 0 $sendfs &
526 log_must wait
527 log_must zfs snapshot $sendfs@a
528
529 rm_files 200 256 0 $sendfs &
530 rm_files 200 131072 0 $sendfs &
531 rm_files 20 1048576 0 $sendfs &
532 rm_files 2 10485760 0 $sendfs &
533 log_must wait
534
535 mk_files 400 256 0 $sendfs &
536 mk_files 400 131072 0 $sendfs &
537 mk_files 40 1048576 0 $sendfs &
538 mk_files 4 10485760 0 $sendfs &
539 log_must wait
540
541 log_must zfs snapshot $sendfs@b
542 log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
543 log_must eval "zfs send -v -i @a $sendfs@b " \
544 ">/$sendpool/incremental.zsend"
545 fi
546
547 if datasetexists $streamfs; then
548 log_must zfs destroy -r $streamfs
549 fi
550 log_must zfs create -o compress=lz4 $sendpool/stream
551 }