1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2017 by Delphix. All rights reserved.
27 */
28
29 /*
30 * Vnode operations for the High Sierra filesystem
31 */
32
33 #include <sys/types.h>
34 #include <sys/t_lock.h>
35 #include <sys/param.h>
36 #include <sys/time.h>
37 #include <sys/systm.h>
38 #include <sys/sysmacros.h>
39 #include <sys/resource.h>
40 #include <sys/signal.h>
41 #include <sys/cred.h>
42 #include <sys/user.h>
43 #include <sys/buf.h>
44 #include <sys/vfs.h>
45 #include <sys/vfs_opreg.h>
46 #include <sys/stat.h>
47 #include <sys/vnode.h>
48 #include <sys/mode.h>
49 #include <sys/proc.h>
50 #include <sys/disp.h>
51 #include <sys/file.h>
52 #include <sys/fcntl.h>
53 #include <sys/flock.h>
54 #include <sys/kmem.h>
55 #include <sys/uio.h>
56 #include <sys/conf.h>
57 #include <sys/errno.h>
58 #include <sys/mman.h>
59 #include <sys/pathname.h>
60 #include <sys/debug.h>
61 #include <sys/vmsystm.h>
62 #include <sys/cmn_err.h>
63 #include <sys/fbuf.h>
64 #include <sys/dirent.h>
65 #include <sys/errno.h>
66 #include <sys/dkio.h>
67 #include <sys/cmn_err.h>
68 #include <sys/atomic.h>
69
70 #include <vm/hat.h>
71 #include <vm/page.h>
72 #include <vm/pvn.h>
73 #include <vm/as.h>
74 #include <vm/seg.h>
75 #include <vm/seg_map.h>
76 #include <vm/seg_kmem.h>
77 #include <vm/seg_vn.h>
78 #include <vm/rm.h>
79 #include <vm/page.h>
80 #include <sys/swap.h>
81 #include <sys/avl.h>
82 #include <sys/sunldi.h>
83 #include <sys/ddi.h>
84 #include <sys/sunddi.h>
85 #include <sys/sdt.h>
86
87 /*
88 * For struct modlinkage
89 */
90 #include <sys/modctl.h>
91
92 #include <sys/fs/hsfs_spec.h>
93 #include <sys/fs/hsfs_node.h>
94 #include <sys/fs/hsfs_impl.h>
95 #include <sys/fs/hsfs_susp.h>
96 #include <sys/fs/hsfs_rrip.h>
97
98 #include <fs/fs_subr.h>
99
100 /* # of contiguous requests to detect sequential access pattern */
101 static int seq_contig_requests = 2;
102
103 /*
104 * This is the max number os taskq threads that will be created
105 * if required. Since we are using a Dynamic TaskQ by default only
106 * one thread is created initially.
107 *
108 * NOTE: In the usual hsfs use case this per fs instance number
109 * of taskq threads should not place any undue load on a system.
110 * Even on an unusual system with say 100 CDROM drives, 800 threads
111 * will not be created unless all the drives are loaded and all
112 * of them are saturated with I/O at the same time! If there is at
113 * all a complaint of system load due to such an unusual case it
114 * should be easy enough to change to one per-machine Dynamic TaskQ
115 * for all hsfs mounts with a nthreads of say 32.
116 */
117 static int hsfs_taskq_nthreads = 8; /* # of taskq threads per fs */
118
119 /* Min count of adjacent bufs that will avoid buf coalescing */
120 static int hsched_coalesce_min = 2;
121
122 /*
123 * Kmem caches for heavily used small allocations. Using these kmem
124 * caches provides a factor of 3 reduction in system time and greatly
125 * aids overall throughput esp. on SPARC.
126 */
127 struct kmem_cache *hio_cache;
128 struct kmem_cache *hio_info_cache;
129
130 /*
131 * This tunable allows us to ignore inode numbers from rrip-1.12.
132 * In this case, we fall back to our default inode algorithm.
133 */
134 extern int use_rrip_inodes;
135
136 /*
137 * Free behind logic from UFS to tame our thirst for
138 * the page cache.
139 * See usr/src/uts/common/fs/ufs/ufs_vnops.c for more
140 * explanation.
141 */
142 static int freebehind = 1;
143 static int smallfile = 0;
144 static int cache_read_ahead = 0;
145 static u_offset_t smallfile64 = 32 * 1024;
146 #define SMALLFILE1_D 1000
147 #define SMALLFILE2_D 10
148 static u_offset_t smallfile1 = 32 * 1024;
149 static u_offset_t smallfile2 = 32 * 1024;
150 static clock_t smallfile_update = 0; /* when to recompute */
151 static uint_t smallfile1_d = SMALLFILE1_D;
152 static uint_t smallfile2_d = SMALLFILE2_D;
153
154 static int hsched_deadline_compare(const void *x1, const void *x2);
155 static int hsched_offset_compare(const void *x1, const void *x2);
156 static void hsched_enqueue_io(struct hsfs *fsp, struct hio *hsio, int ra);
157 int hsched_invoke_strategy(struct hsfs *fsp);
158
159 /* ARGSUSED */
160 static int
161 hsfs_fsync(vnode_t *cp, int syncflag, cred_t *cred, caller_context_t *ct)
162 {
163 return (0);
164 }
165
166
167 /*ARGSUSED*/
168 static int
169 hsfs_read(struct vnode *vp, struct uio *uiop, int ioflag, struct cred *cred,
170 struct caller_context *ct)
171 {
172 caddr_t base;
173 offset_t diff;
174 int error;
175 struct hsnode *hp;
176 uint_t filesize;
177 int dofree;
178
179 hp = VTOH(vp);
180 /*
181 * if vp is of type VDIR, make sure dirent
182 * is filled up with all info (because of ptbl)
183 */
184 if (vp->v_type == VDIR) {
185 if (hp->hs_dirent.ext_size == 0)
186 hs_filldirent(vp, &hp->hs_dirent);
187 }
188 filesize = hp->hs_dirent.ext_size;
189
190 /* Sanity checks. */
191 if (uiop->uio_resid == 0 || /* No data wanted. */
192 uiop->uio_loffset > HS_MAXFILEOFF || /* Offset too big. */
193 uiop->uio_loffset >= filesize) /* Past EOF. */
194 return (0);
195
196 do {
197 /*
198 * We want to ask for only the "right" amount of data.
199 * In this case that means:-
200 *
201 * We can't get data from beyond our EOF. If asked,
202 * we will give a short read.
203 *
204 * segmap_getmapflt returns buffers of MAXBSIZE bytes.
205 * These buffers are always MAXBSIZE aligned.
206 * If our starting offset is not MAXBSIZE aligned,
207 * we can only ask for less than MAXBSIZE bytes.
208 *
209 * If our requested offset and length are such that
210 * they belong in different MAXBSIZE aligned slots
211 * then we'll be making more than one call on
212 * segmap_getmapflt.
213 *
214 * This diagram shows the variables we use and their
215 * relationships.
216 *
217 * |<-----MAXBSIZE----->|
218 * +--------------------------...+
219 * |.....mapon->|<--n-->|....*...|EOF
220 * +--------------------------...+
221 * uio_loffset->|
222 * uio_resid....|<---------->|
223 * diff.........|<-------------->|
224 *
225 * So, in this case our offset is not aligned
226 * and our request takes us outside of the
227 * MAXBSIZE window. We will break this up into
228 * two segmap_getmapflt calls.
229 */
230 size_t nbytes;
231 offset_t mapon;
232 size_t n;
233 uint_t flags;
234
235 mapon = uiop->uio_loffset & MAXBOFFSET;
236 diff = filesize - uiop->uio_loffset;
237 nbytes = (size_t)MIN(MAXBSIZE - mapon, uiop->uio_resid);
238 n = MIN(diff, nbytes);
239 if (n <= 0) {
240 /* EOF or request satisfied. */
241 return (0);
242 }
243
244 /*
245 * Freebehind computation taken from:
246 * usr/src/uts/common/fs/ufs/ufs_vnops.c
247 */
248 if (drv_hztousec(ddi_get_lbolt()) >= smallfile_update) {
249 uint64_t percpufreeb;
250 if (smallfile1_d == 0) smallfile1_d = SMALLFILE1_D;
251 if (smallfile2_d == 0) smallfile2_d = SMALLFILE2_D;
252 percpufreeb = ptob((uint64_t)freemem) / ncpus_online;
253 smallfile1 = percpufreeb / smallfile1_d;
254 smallfile2 = percpufreeb / smallfile2_d;
255 smallfile1 = MAX(smallfile1, smallfile);
256 smallfile1 = MAX(smallfile1, smallfile64);
257 smallfile2 = MAX(smallfile1, smallfile2);
258 smallfile_update = drv_hztousec(ddi_get_lbolt())
259 + 1000000;
260 }
261
262 dofree = freebehind &&
263 hp->hs_prev_offset == uiop->uio_loffset &&
264 hp->hs_ra_bytes > 0;
265
266 base = segmap_getmapflt(segkmap, vp,
267 (u_offset_t)uiop->uio_loffset, n, 1, S_READ);
268
269 error = uiomove(base + mapon, n, UIO_READ, uiop);
270
271 if (error == 0) {
272 /*
273 * if read a whole block, or read to eof,
274 * won't need this buffer again soon.
275 */
276 if (n + mapon == MAXBSIZE ||
277 uiop->uio_loffset == filesize)
278 flags = SM_DONTNEED;
279 else
280 flags = 0;
281
282 if (dofree) {
283 flags = SM_FREE | SM_ASYNC;
284 if ((cache_read_ahead == 0) &&
285 uiop->uio_loffset > smallfile2)
286 flags |= SM_DONTNEED;
287 }
288
289 error = segmap_release(segkmap, base, flags);
290 } else
291 (void) segmap_release(segkmap, base, 0);
292 } while (error == 0 && uiop->uio_resid > 0);
293
294 return (error);
295 }
296
297 /*ARGSUSED2*/
298 static int
299 hsfs_getattr(struct vnode *vp, struct vattr *vap, int flags, struct cred *cred,
300 caller_context_t *ct)
301 {
302 struct hsnode *hp;
303 struct vfs *vfsp;
304 struct hsfs *fsp;
305
306 hp = VTOH(vp);
307 fsp = VFS_TO_HSFS(vp->v_vfsp);
308 vfsp = vp->v_vfsp;
309
310 if ((hp->hs_dirent.ext_size == 0) && (vp->v_type == VDIR)) {
311 hs_filldirent(vp, &hp->hs_dirent);
312 }
313 vap->va_type = IFTOVT(hp->hs_dirent.mode);
314 vap->va_mode = hp->hs_dirent.mode;
315 vap->va_uid = hp->hs_dirent.uid;
316 vap->va_gid = hp->hs_dirent.gid;
317
318 vap->va_fsid = vfsp->vfs_dev;
319 vap->va_nodeid = (ino64_t)hp->hs_nodeid;
320 vap->va_nlink = hp->hs_dirent.nlink;
321 vap->va_size = (offset_t)hp->hs_dirent.ext_size;
322
323 vap->va_atime.tv_sec = hp->hs_dirent.adate.tv_sec;
324 vap->va_atime.tv_nsec = hp->hs_dirent.adate.tv_usec*1000;
325 vap->va_mtime.tv_sec = hp->hs_dirent.mdate.tv_sec;
326 vap->va_mtime.tv_nsec = hp->hs_dirent.mdate.tv_usec*1000;
327 vap->va_ctime.tv_sec = hp->hs_dirent.cdate.tv_sec;
328 vap->va_ctime.tv_nsec = hp->hs_dirent.cdate.tv_usec*1000;
329 if (vp->v_type == VCHR || vp->v_type == VBLK)
330 vap->va_rdev = hp->hs_dirent.r_dev;
331 else
332 vap->va_rdev = 0;
333 vap->va_blksize = vfsp->vfs_bsize;
334 /* no. of blocks = no. of data blocks + no. of xar blocks */
335 vap->va_nblocks = (fsblkcnt64_t)howmany(vap->va_size + (u_longlong_t)
336 (hp->hs_dirent.xar_len << fsp->hsfs_vol.lbn_shift), DEV_BSIZE);
337 vap->va_seq = hp->hs_seq;
338 return (0);
339 }
340
341 /*ARGSUSED*/
342 static int
343 hsfs_readlink(struct vnode *vp, struct uio *uiop, struct cred *cred,
344 caller_context_t *ct)
345 {
346 struct hsnode *hp;
347
348 if (vp->v_type != VLNK)
349 return (EINVAL);
350
351 hp = VTOH(vp);
352
353 if (hp->hs_dirent.sym_link == (char *)NULL)
354 return (ENOENT);
355
356 return (uiomove(hp->hs_dirent.sym_link,
357 (size_t)MIN(hp->hs_dirent.ext_size,
358 uiop->uio_resid), UIO_READ, uiop));
359 }
360
361 /*ARGSUSED*/
362 static void
363 hsfs_inactive(struct vnode *vp, struct cred *cred, caller_context_t *ct)
364 {
365 struct hsnode *hp;
366 struct hsfs *fsp;
367
368 int nopage;
369
370 hp = VTOH(vp);
371 fsp = VFS_TO_HSFS(vp->v_vfsp);
372 /*
373 * Note: acquiring and holding v_lock for quite a while
374 * here serializes on the vnode; this is unfortunate, but
375 * likely not to overly impact performance, as the underlying
376 * device (CDROM drive) is quite slow.
377 */
378 rw_enter(&fsp->hsfs_hash_lock, RW_WRITER);
379 mutex_enter(&hp->hs_contents_lock);
380 mutex_enter(&vp->v_lock);
381
382 if (vp->v_count < 1) {
383 panic("hsfs_inactive: v_count < 1");
384 /*NOTREACHED*/
385 }
386
387 VN_RELE_LOCKED(vp);
388 if (vp->v_count > 0 || (hp->hs_flags & HREF) == 0) {
389 mutex_exit(&vp->v_lock);
390 mutex_exit(&hp->hs_contents_lock);
391 rw_exit(&fsp->hsfs_hash_lock);
392 return;
393 }
394 if (vp->v_count == 0) {
395 /*
396 * Free the hsnode.
397 * If there are no pages associated with the
398 * hsnode, give it back to the kmem_cache,
399 * else put at the end of this file system's
400 * internal free list.
401 */
402 nopage = !vn_has_cached_data(vp);
403 hp->hs_flags = 0;
404 /*
405 * exit these locks now, since hs_freenode may
406 * kmem_free the hsnode and embedded vnode
407 */
408 mutex_exit(&vp->v_lock);
409 mutex_exit(&hp->hs_contents_lock);
410 hs_freenode(vp, fsp, nopage);
411 } else {
412 mutex_exit(&vp->v_lock);
413 mutex_exit(&hp->hs_contents_lock);
414 }
415 rw_exit(&fsp->hsfs_hash_lock);
416 }
417
418
419 /*ARGSUSED*/
420 static int
421 hsfs_lookup(struct vnode *dvp, char *nm, struct vnode **vpp,
422 struct pathname *pnp, int flags, struct vnode *rdir, struct cred *cred,
423 caller_context_t *ct, int *direntflags, pathname_t *realpnp)
424 {
425 int error;
426 int namelen = (int)strlen(nm);
427
428 if (*nm == '\0') {
429 VN_HOLD(dvp);
430 *vpp = dvp;
431 return (0);
432 }
433
434 /*
435 * If we're looking for ourself, life is simple.
436 */
437 if (namelen == 1 && *nm == '.') {
438 if (error = hs_access(dvp, (mode_t)VEXEC, cred))
439 return (error);
440 VN_HOLD(dvp);
441 *vpp = dvp;
442 return (0);
443 }
444
445 return (hs_dirlook(dvp, nm, namelen, vpp, cred));
446 }
447
448
449 /*ARGSUSED*/
450 static int
451 hsfs_readdir(struct vnode *vp, struct uio *uiop, struct cred *cred, int *eofp,
452 caller_context_t *ct, int flags)
453 {
454 struct hsnode *dhp;
455 struct hsfs *fsp;
456 struct hs_direntry hd;
457 struct dirent64 *nd;
458 int error;
459 uint_t offset; /* real offset in directory */
460 uint_t dirsiz; /* real size of directory */
461 uchar_t *blkp;
462 int hdlen; /* length of hs directory entry */
463 long ndlen; /* length of dirent entry */
464 int bytes_wanted;
465 size_t bufsize; /* size of dirent buffer */
466 char *outbuf; /* ptr to dirent buffer */
467 char *dname;
468 int dnamelen;
469 size_t dname_size;
470 struct fbuf *fbp;
471 uint_t last_offset; /* last index into current dir block */
472 ino64_t dirino; /* temporary storage before storing in dirent */
473 off_t diroff;
474
475 dhp = VTOH(vp);
476 fsp = VFS_TO_HSFS(vp->v_vfsp);
477 if (dhp->hs_dirent.ext_size == 0)
478 hs_filldirent(vp, &dhp->hs_dirent);
479 dirsiz = dhp->hs_dirent.ext_size;
480 if (uiop->uio_loffset >= dirsiz) { /* at or beyond EOF */
481 if (eofp)
482 *eofp = 1;
483 return (0);
484 }
485 ASSERT(uiop->uio_loffset <= HS_MAXFILEOFF);
486 offset = uiop->uio_loffset;
487
488 dname_size = fsp->hsfs_namemax + 1; /* 1 for the ending NUL */
489 dname = kmem_alloc(dname_size, KM_SLEEP);
490 bufsize = uiop->uio_resid + sizeof (struct dirent64);
491
492 outbuf = kmem_alloc(bufsize, KM_SLEEP);
493 nd = (struct dirent64 *)outbuf;
494
495 while (offset < dirsiz) {
496 bytes_wanted = MIN(MAXBSIZE, dirsiz - (offset & MAXBMASK));
497
498 error = fbread(vp, (offset_t)(offset & MAXBMASK),
499 (unsigned int)bytes_wanted, S_READ, &fbp);
500 if (error)
501 goto done;
502
503 blkp = (uchar_t *)fbp->fb_addr;
504 last_offset = (offset & MAXBMASK) + fbp->fb_count;
505
506 #define rel_offset(offset) ((offset) & MAXBOFFSET) /* index into blkp */
507
508 while (offset < last_offset) {
509 /*
510 * Very similar validation code is found in
511 * process_dirblock(), hsfs_node.c.
512 * For an explanation, see there.
513 * It may make sense for the future to
514 * "consolidate" the code in hs_parsedir(),
515 * process_dirblock() and hsfs_readdir() into
516 * a single utility function.
517 */
518 hdlen = (int)((uchar_t)
519 HDE_DIR_LEN(&blkp[rel_offset(offset)]));
520 if (hdlen < HDE_ROOT_DIR_REC_SIZE ||
521 offset + hdlen > last_offset) {
522 /*
523 * advance to next sector boundary
524 */
525 offset = roundup(offset + 1, HS_SECTOR_SIZE);
526 if (hdlen)
527 hs_log_bogus_disk_warning(fsp,
528 HSFS_ERR_TRAILING_JUNK, 0);
529
530 continue;
531 }
532
533 bzero(&hd, sizeof (hd));
534
535 /*
536 * Just ignore invalid directory entries.
537 * XXX - maybe hs_parsedir() will detect EXISTENCE bit
538 */
539 if (!hs_parsedir(fsp, &blkp[rel_offset(offset)],
540 &hd, dname, &dnamelen, last_offset - offset)) {
541 /*
542 * Determine if there is enough room
543 */
544 ndlen = (long)DIRENT64_RECLEN((dnamelen));
545
546 if ((ndlen + ((char *)nd - outbuf)) >
547 uiop->uio_resid) {
548 fbrelse(fbp, S_READ);
549 goto done; /* output buffer full */
550 }
551
552 diroff = offset + hdlen;
553 /*
554 * If the media carries rrip-v1.12 or newer,
555 * and we trust the inodes from the rrip data
556 * (use_rrip_inodes != 0), use that data. If the
557 * media has been created by a recent mkisofs
558 * version, we may trust all numbers in the
559 * starting extent number; otherwise, we cannot
560 * do this for zero sized files and symlinks,
561 * because if we did we'd end up mapping all of
562 * them to the same node. We use HS_DUMMY_INO
563 * in this case and make sure that we will not
564 * map all files to the same meta data.
565 */
566 if (hd.inode != 0 && use_rrip_inodes) {
567 dirino = hd.inode;
568 } else if ((hd.ext_size == 0 ||
569 hd.sym_link != (char *)NULL) &&
570 (fsp->hsfs_flags & HSFSMNT_INODE) == 0) {
571 dirino = HS_DUMMY_INO;
572 } else {
573 dirino = hd.ext_lbn;
574 }
575
576 /* strncpy(9f) will zero uninitialized bytes */
577
578 ASSERT(strlen(dname) + 1 <=
579 DIRENT64_NAMELEN(ndlen));
580 (void) strncpy(nd->d_name, dname,
581 DIRENT64_NAMELEN(ndlen));
582 nd->d_reclen = (ushort_t)ndlen;
583 nd->d_off = (offset_t)diroff;
584 nd->d_ino = dirino;
585 nd = (struct dirent64 *)((char *)nd + ndlen);
586
587 /*
588 * free up space allocated for symlink
589 */
590 if (hd.sym_link != (char *)NULL) {
591 kmem_free(hd.sym_link,
592 (size_t)(hd.ext_size+1));
593 hd.sym_link = (char *)NULL;
594 }
595 }
596 offset += hdlen;
597 }
598 fbrelse(fbp, S_READ);
599 }
600
601 /*
602 * Got here for one of the following reasons:
603 * 1) outbuf is full (error == 0)
604 * 2) end of directory reached (error == 0)
605 * 3) error reading directory sector (error != 0)
606 * 4) directory entry crosses sector boundary (error == 0)
607 *
608 * If any directory entries have been copied, don't report
609 * case 4. Instead, return the valid directory entries.
610 *
611 * If no entries have been copied, report the error.
612 * If case 4, this will be indistiguishable from EOF.
613 */
614 done:
615 ndlen = ((char *)nd - outbuf);
616 if (ndlen != 0) {
617 error = uiomove(outbuf, (size_t)ndlen, UIO_READ, uiop);
618 uiop->uio_loffset = offset;
619 }
620 kmem_free(dname, dname_size);
621 kmem_free(outbuf, bufsize);
622 if (eofp && error == 0)
623 *eofp = (uiop->uio_loffset >= dirsiz);
624 return (error);
625 }
626
627 /*ARGSUSED2*/
628 static int
629 hsfs_fid(struct vnode *vp, struct fid *fidp, caller_context_t *ct)
630 {
631 struct hsnode *hp;
632 struct hsfid *fid;
633
634 if (fidp->fid_len < (sizeof (*fid) - sizeof (fid->hf_len))) {
635 fidp->fid_len = sizeof (*fid) - sizeof (fid->hf_len);
636 return (ENOSPC);
637 }
638
639 fid = (struct hsfid *)fidp;
640 fid->hf_len = sizeof (*fid) - sizeof (fid->hf_len);
641 hp = VTOH(vp);
642 mutex_enter(&hp->hs_contents_lock);
643 fid->hf_dir_lbn = hp->hs_dir_lbn;
644 fid->hf_dir_off = (ushort_t)hp->hs_dir_off;
645 fid->hf_ino = hp->hs_nodeid;
646 mutex_exit(&hp->hs_contents_lock);
647 return (0);
648 }
649
650 /*ARGSUSED*/
651 static int
652 hsfs_open(struct vnode **vpp, int flag, struct cred *cred, caller_context_t *ct)
653 {
654 return (0);
655 }
656
657 /*ARGSUSED*/
658 static int
659 hsfs_close(struct vnode *vp, int flag, int count, offset_t offset,
660 struct cred *cred, caller_context_t *ct)
661 {
662 (void) cleanlocks(vp, ttoproc(curthread)->p_pid, 0);
663 cleanshares(vp, ttoproc(curthread)->p_pid);
664 return (0);
665 }
666
667 /*ARGSUSED2*/
668 static int
669 hsfs_access(struct vnode *vp, int mode, int flags, cred_t *cred,
670 caller_context_t *ct)
671 {
672 return (hs_access(vp, (mode_t)mode, cred));
673 }
674
675 /*
676 * the seek time of a CD-ROM is very slow, and data transfer
677 * rate is even worse (max. 150K per sec). The design
678 * decision is to reduce access to cd-rom as much as possible,
679 * and to transfer a sizable block (read-ahead) of data at a time.
680 * UFS style of read ahead one block at a time is not appropriate,
681 * and is not supported
682 */
683
684 /*
685 * KLUSTSIZE should be a multiple of PAGESIZE and <= MAXPHYS.
686 */
687 #define KLUSTSIZE (56 * 1024)
688 /* we don't support read ahead */
689 int hsfs_lostpage; /* no. of times we lost original page */
690
691 /*
692 * Used to prevent biodone() from releasing buf resources that
693 * we didn't allocate in quite the usual way.
694 */
695 /*ARGSUSED*/
696 int
697 hsfs_iodone(struct buf *bp)
698 {
699 sema_v(&bp->b_io);
700 return (0);
701 }
702
703 /*
704 * The taskq thread that invokes the scheduling function to ensure
705 * that all readaheads are complete and cleans up the associated
706 * memory and releases the page lock.
707 */
708 void
709 hsfs_ra_task(void *arg)
710 {
711 struct hio_info *info = arg;
712 uint_t count;
713 struct buf *wbuf;
714
715 ASSERT(info->pp != NULL);
716
717 for (count = 0; count < info->bufsused; count++) {
718 wbuf = &(info->bufs[count]);
719
720 DTRACE_PROBE1(hsfs_io_wait_ra, struct buf *, wbuf);
721 while (sema_tryp(&(info->sema[count])) == 0) {
722 if (hsched_invoke_strategy(info->fsp)) {
723 sema_p(&(info->sema[count]));
724 break;
725 }
726 }
727 sema_destroy(&(info->sema[count]));
728 DTRACE_PROBE1(hsfs_io_done_ra, struct buf *, wbuf);
729 biofini(&(info->bufs[count]));
730 }
731 for (count = 0; count < info->bufsused; count++) {
732 if (info->vas[count] != NULL) {
733 ppmapout(info->vas[count]);
734 }
735 }
736 kmem_free(info->vas, info->bufcnt * sizeof (caddr_t));
737 kmem_free(info->bufs, info->bufcnt * sizeof (struct buf));
738 kmem_free(info->sema, info->bufcnt * sizeof (ksema_t));
739
740 pvn_read_done(info->pp, 0);
741 kmem_cache_free(hio_info_cache, info);
742 }
743
744 /*
745 * Submit asynchronous readahead requests to the I/O scheduler
746 * depending on the number of pages to read ahead. These requests
747 * are asynchronous to the calling thread but I/O requests issued
748 * subsequently by other threads with higher LBNs must wait for
749 * these readaheads to complete since we have a single ordered
750 * I/O pipeline. Thus these readaheads are semi-asynchronous.
751 * A TaskQ handles waiting for the readaheads to complete.
752 *
753 * This function is mostly a copy of hsfs_getapage but somewhat
754 * simpler. A readahead request is aborted if page allocation
755 * fails.
756 */
757 /*ARGSUSED*/
758 static int
759 hsfs_getpage_ra(struct vnode *vp, u_offset_t off, struct seg *seg,
760 caddr_t addr, struct hsnode *hp, struct hsfs *fsp, int xarsiz,
761 offset_t bof, int chunk_lbn_count, int chunk_data_bytes)
762 {
763 struct buf *bufs;
764 caddr_t *vas;
765 caddr_t va;
766 struct page *pp, *searchp, *lastp;
767 struct vnode *devvp;
768 ulong_t byte_offset;
769 size_t io_len_tmp;
770 uint_t io_off, io_len;
771 uint_t xlen;
772 uint_t filsiz;
773 uint_t secsize;
774 uint_t bufcnt;
775 uint_t bufsused;
776 uint_t count;
777 uint_t io_end;
778 uint_t which_chunk_lbn;
779 uint_t offset_lbn;
780 uint_t offset_extra;
781 offset_t offset_bytes;
782 uint_t remaining_bytes;
783 uint_t extension;
784 int remainder; /* must be signed */
785 diskaddr_t driver_block;
786 u_offset_t io_off_tmp;
787 ksema_t *fio_done;
788 struct hio_info *info;
789 size_t len;
790
791 ASSERT(fsp->hqueue != NULL);
792
793 if (addr >= seg->s_base + seg->s_size) {
794 return (-1);
795 }
796
797 devvp = fsp->hsfs_devvp;
798 secsize = fsp->hsfs_vol.lbn_size; /* bytes per logical block */
799
800 /* file data size */
801 filsiz = hp->hs_dirent.ext_size;
802
803 if (off >= filsiz)
804 return (0);
805
806 extension = 0;
807 pp = NULL;
808
809 extension += hp->hs_ra_bytes;
810
811 /*
812 * Some CD writers (e.g. Kodak Photo CD writers)
813 * create CDs in TAO mode and reserve tracks that
814 * are not completely written. Some sectors remain
815 * unreadable for this reason and give I/O errors.
816 * Also, there's no point in reading sectors
817 * we'll never look at. So, if we're asked to go
818 * beyond the end of a file, truncate to the length
819 * of that file.
820 *
821 * Additionally, this behaviour is required by section
822 * 6.4.5 of ISO 9660:1988(E).
823 */
824 len = MIN(extension ? extension : PAGESIZE, filsiz - off);
825
826 /* A little paranoia */
827 if (len <= 0)
828 return (-1);
829
830 /*
831 * After all that, make sure we're asking for things in units
832 * that bdev_strategy() will understand (see bug 4202551).
833 */
834 len = roundup(len, DEV_BSIZE);
835
836 pp = pvn_read_kluster(vp, off, seg, addr, &io_off_tmp,
837 &io_len_tmp, off, len, 1);
838
839 if (pp == NULL) {
840 hp->hs_num_contig = 0;
841 hp->hs_ra_bytes = 0;
842 hp->hs_prev_offset = 0;
843 return (-1);
844 }
845
846 io_off = (uint_t)io_off_tmp;
847 io_len = (uint_t)io_len_tmp;
848
849 /* check for truncation */
850 /*
851 * xxx Clean up and return EIO instead?
852 * xxx Ought to go to u_offset_t for everything, but we
853 * xxx call lots of things that want uint_t arguments.
854 */
855 ASSERT(io_off == io_off_tmp);
856
857 /*
858 * get enough buffers for worst-case scenario
859 * (i.e., no coalescing possible).
860 */
861 bufcnt = (len + secsize - 1) / secsize;
862 bufs = kmem_alloc(bufcnt * sizeof (struct buf), KM_SLEEP);
863 vas = kmem_alloc(bufcnt * sizeof (caddr_t), KM_SLEEP);
864
865 /*
866 * Allocate a array of semaphores since we are doing I/O
867 * scheduling.
868 */
869 fio_done = kmem_alloc(bufcnt * sizeof (ksema_t), KM_SLEEP);
870
871 /*
872 * If our filesize is not an integer multiple of PAGESIZE,
873 * we zero that part of the last page that's between EOF and
874 * the PAGESIZE boundary.
875 */
876 xlen = io_len & PAGEOFFSET;
877 if (xlen != 0)
878 pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
879
880 DTRACE_PROBE2(hsfs_readahead, struct vnode *, vp, uint_t, io_len);
881
882 va = NULL;
883 lastp = NULL;
884 searchp = pp;
885 io_end = io_off + io_len;
886 for (count = 0, byte_offset = io_off;
887 byte_offset < io_end;
888 count++) {
889 ASSERT(count < bufcnt);
890
891 bioinit(&bufs[count]);
892 bufs[count].b_edev = devvp->v_rdev;
893 bufs[count].b_dev = cmpdev(devvp->v_rdev);
894 bufs[count].b_flags = B_NOCACHE|B_BUSY|B_READ;
895 bufs[count].b_iodone = hsfs_iodone;
896 bufs[count].b_vp = vp;
897 bufs[count].b_file = vp;
898
899 /* Compute disk address for interleaving. */
900
901 /* considered without skips */
902 which_chunk_lbn = byte_offset / chunk_data_bytes;
903
904 /* factor in skips */
905 offset_lbn = which_chunk_lbn * chunk_lbn_count;
906
907 /* convert to physical byte offset for lbn */
908 offset_bytes = LBN_TO_BYTE(offset_lbn, vp->v_vfsp);
909
910 /* don't forget offset into lbn */
911 offset_extra = byte_offset % chunk_data_bytes;
912
913 /* get virtual block number for driver */
914 driver_block = lbtodb(bof + xarsiz
915 + offset_bytes + offset_extra);
916
917 if (lastp != searchp) {
918 /* this branch taken first time through loop */
919 va = vas[count] = ppmapin(searchp, PROT_WRITE,
920 (caddr_t)-1);
921 /* ppmapin() guarantees not to return NULL */
922 } else {
923 vas[count] = NULL;
924 }
925
926 bufs[count].b_un.b_addr = va + byte_offset % PAGESIZE;
927 bufs[count].b_offset =
928 (offset_t)(byte_offset - io_off + off);
929
930 /*
931 * We specifically use the b_lblkno member here
932 * as even in the 32 bit world driver_block can
933 * get very large in line with the ISO9660 spec.
934 */
935
936 bufs[count].b_lblkno = driver_block;
937
938 remaining_bytes = ((which_chunk_lbn + 1) * chunk_data_bytes)
939 - byte_offset;
940
941 /*
942 * remaining_bytes can't be zero, as we derived
943 * which_chunk_lbn directly from byte_offset.
944 */
945 if ((remaining_bytes + byte_offset) < (off + len)) {
946 /* coalesce-read the rest of the chunk */
947 bufs[count].b_bcount = remaining_bytes;
948 } else {
949 /* get the final bits */
950 bufs[count].b_bcount = off + len - byte_offset;
951 }
952
953 remainder = PAGESIZE - (byte_offset % PAGESIZE);
954 if (bufs[count].b_bcount > remainder) {
955 bufs[count].b_bcount = remainder;
956 }
957
958 bufs[count].b_bufsize = bufs[count].b_bcount;
959 if (((offset_t)byte_offset + bufs[count].b_bcount) >
960 HS_MAXFILEOFF) {
961 break;
962 }
963 byte_offset += bufs[count].b_bcount;
964
965 /*
966 * We are scheduling I/O so we need to enqueue
967 * requests rather than calling bdev_strategy
968 * here. A later invocation of the scheduling
969 * function will take care of doing the actual
970 * I/O as it selects requests from the queue as
971 * per the scheduling logic.
972 */
973 struct hio *hsio = kmem_cache_alloc(hio_cache,
974 KM_SLEEP);
975
976 sema_init(&fio_done[count], 0, NULL,
977 SEMA_DEFAULT, NULL);
978 hsio->bp = &bufs[count];
979 hsio->sema = &fio_done[count];
980 hsio->io_lblkno = bufs[count].b_lblkno;
981 hsio->nblocks = howmany(hsio->bp->b_bcount,
982 DEV_BSIZE);
983
984 /* used for deadline */
985 hsio->io_timestamp = drv_hztousec(ddi_get_lbolt());
986
987 /* for I/O coalescing */
988 hsio->contig_chain = NULL;
989 hsched_enqueue_io(fsp, hsio, 1);
990
991 lwp_stat_update(LWP_STAT_INBLK, 1);
992 lastp = searchp;
993 if ((remainder - bufs[count].b_bcount) < 1) {
994 searchp = searchp->p_next;
995 }
996 }
997
998 bufsused = count;
999 info = kmem_cache_alloc(hio_info_cache, KM_SLEEP);
1000 info->bufs = bufs;
1001 info->vas = vas;
1002 info->sema = fio_done;
1003 info->bufsused = bufsused;
1004 info->bufcnt = bufcnt;
1005 info->fsp = fsp;
1006 info->pp = pp;
1007
1008 (void) taskq_dispatch(fsp->hqueue->ra_task,
1009 hsfs_ra_task, info, KM_SLEEP);
1010 /*
1011 * The I/O locked pages are unlocked in our taskq thread.
1012 */
1013 return (0);
1014 }
1015
1016 /*
1017 * Each file may have a different interleaving on disk. This makes
1018 * things somewhat interesting. The gist is that there are some
1019 * number of contiguous data sectors, followed by some other number
1020 * of contiguous skip sectors. The sum of those two sets of sectors
1021 * defines the interleave size. Unfortunately, it means that we generally
1022 * can't simply read N sectors starting at a given offset to satisfy
1023 * any given request.
1024 *
1025 * What we do is get the relevant memory pages via pvn_read_kluster(),
1026 * then stride through the interleaves, setting up a buf for each
1027 * sector that needs to be brought in. Instead of kmem_alloc'ing
1028 * space for the sectors, though, we just point at the appropriate
1029 * spot in the relevant page for each of them. This saves us a bunch
1030 * of copying.
1031 *
1032 * NOTICE: The code below in hsfs_getapage is mostly same as the code
1033 * in hsfs_getpage_ra above (with some omissions). If you are
1034 * making any change to this function, please also look at
1035 * hsfs_getpage_ra.
1036 */
1037 /*ARGSUSED*/
1038 static int
1039 hsfs_getapage(struct vnode *vp, u_offset_t off, size_t len, uint_t *protp,
1040 struct page *pl[], size_t plsz, struct seg *seg, caddr_t addr,
1041 enum seg_rw rw, struct cred *cred)
1042 {
1043 struct hsnode *hp;
1044 struct hsfs *fsp;
1045 int err;
1046 struct buf *bufs;
1047 caddr_t *vas;
1048 caddr_t va;
1049 struct page *pp, *searchp, *lastp;
1050 page_t *pagefound;
1051 offset_t bof;
1052 struct vnode *devvp;
1053 ulong_t byte_offset;
1054 size_t io_len_tmp;
1055 uint_t io_off, io_len;
1056 uint_t xlen;
1057 uint_t filsiz;
1058 uint_t secsize;
1059 uint_t bufcnt;
1060 uint_t bufsused;
1061 uint_t count;
1062 uint_t io_end;
1063 uint_t which_chunk_lbn;
1064 uint_t offset_lbn;
1065 uint_t offset_extra;
1066 offset_t offset_bytes;
1067 uint_t remaining_bytes;
1068 uint_t extension;
1069 int remainder; /* must be signed */
1070 int chunk_lbn_count;
1071 int chunk_data_bytes;
1072 int xarsiz;
1073 diskaddr_t driver_block;
1074 u_offset_t io_off_tmp;
1075 ksema_t *fio_done;
1076 int calcdone;
1077
1078 /*
1079 * We don't support asynchronous operation at the moment, so
1080 * just pretend we did it. If the pages are ever actually
1081 * needed, they'll get brought in then.
1082 */
1083 if (pl == NULL)
1084 return (0);
1085
1086 hp = VTOH(vp);
1087 fsp = VFS_TO_HSFS(vp->v_vfsp);
1088 devvp = fsp->hsfs_devvp;
1089 secsize = fsp->hsfs_vol.lbn_size; /* bytes per logical block */
1090
1091 /* file data size */
1092 filsiz = hp->hs_dirent.ext_size;
1093
1094 /* disk addr for start of file */
1095 bof = LBN_TO_BYTE((offset_t)hp->hs_dirent.ext_lbn, vp->v_vfsp);
1096
1097 /* xarsiz byte must be skipped for data */
1098 xarsiz = hp->hs_dirent.xar_len << fsp->hsfs_vol.lbn_shift;
1099
1100 /* how many logical blocks in an interleave (data+skip) */
1101 chunk_lbn_count = hp->hs_dirent.intlf_sz + hp->hs_dirent.intlf_sk;
1102
1103 if (chunk_lbn_count == 0) {
1104 chunk_lbn_count = 1;
1105 }
1106
1107 /*
1108 * Convert interleaving size into bytes. The zero case
1109 * (no interleaving) optimization is handled as a side-
1110 * effect of the read-ahead logic.
1111 */
1112 if (hp->hs_dirent.intlf_sz == 0) {
1113 chunk_data_bytes = LBN_TO_BYTE(1, vp->v_vfsp);
1114 /*
1115 * Optimization: If our pagesize is a multiple of LBN
1116 * bytes, we can avoid breaking up a page into individual
1117 * lbn-sized requests.
1118 */
1119 if (PAGESIZE % chunk_data_bytes == 0) {
1120 chunk_lbn_count = BYTE_TO_LBN(PAGESIZE, vp->v_vfsp);
1121 chunk_data_bytes = PAGESIZE;
1122 }
1123 } else {
1124 chunk_data_bytes =
1125 LBN_TO_BYTE(hp->hs_dirent.intlf_sz, vp->v_vfsp);
1126 }
1127
1128 reread:
1129 err = 0;
1130 pagefound = 0;
1131 calcdone = 0;
1132
1133 /*
1134 * Do some read-ahead. This mostly saves us a bit of
1135 * system cpu time more than anything else when doing
1136 * sequential reads. At some point, could do the
1137 * read-ahead asynchronously which might gain us something
1138 * on wall time, but it seems unlikely....
1139 *
1140 * We do the easy case here, which is to read through
1141 * the end of the chunk, minus whatever's at the end that
1142 * won't exactly fill a page.
1143 */
1144 if (hp->hs_ra_bytes > 0 && chunk_data_bytes != PAGESIZE) {
1145 which_chunk_lbn = (off + len) / chunk_data_bytes;
1146 extension = ((which_chunk_lbn + 1) * chunk_data_bytes) - off;
1147 extension -= (extension % PAGESIZE);
1148 } else {
1149 extension = roundup(len, PAGESIZE);
1150 }
1151
1152 atomic_inc_64(&fsp->total_pages_requested);
1153
1154 pp = NULL;
1155 again:
1156 /* search for page in buffer */
1157 if ((pagefound = page_exists(vp, off)) == 0) {
1158 /*
1159 * Need to really do disk IO to get the page.
1160 */
1161 if (!calcdone) {
1162 extension += hp->hs_ra_bytes;
1163
1164 /*
1165 * Some cd writers don't write sectors that aren't
1166 * used. Also, there's no point in reading sectors
1167 * we'll never look at. So, if we're asked to go
1168 * beyond the end of a file, truncate to the length
1169 * of that file.
1170 *
1171 * Additionally, this behaviour is required by section
1172 * 6.4.5 of ISO 9660:1988(E).
1173 */
1174 len = MIN(extension ? extension : PAGESIZE,
1175 filsiz - off);
1176
1177 /* A little paranoia. */
1178 ASSERT(len > 0);
1179
1180 /*
1181 * After all that, make sure we're asking for things
1182 * in units that bdev_strategy() will understand
1183 * (see bug 4202551).
1184 */
1185 len = roundup(len, DEV_BSIZE);
1186 calcdone = 1;
1187 }
1188
1189 pp = pvn_read_kluster(vp, off, seg, addr, &io_off_tmp,
1190 &io_len_tmp, off, len, 0);
1191
1192 if (pp == NULL) {
1193 /*
1194 * Pressure on memory, roll back readahead
1195 */
1196 hp->hs_num_contig = 0;
1197 hp->hs_ra_bytes = 0;
1198 hp->hs_prev_offset = 0;
1199 goto again;
1200 }
1201
1202 io_off = (uint_t)io_off_tmp;
1203 io_len = (uint_t)io_len_tmp;
1204
1205 /* check for truncation */
1206 /*
1207 * xxx Clean up and return EIO instead?
1208 * xxx Ought to go to u_offset_t for everything, but we
1209 * xxx call lots of things that want uint_t arguments.
1210 */
1211 ASSERT(io_off == io_off_tmp);
1212
1213 /*
1214 * get enough buffers for worst-case scenario
1215 * (i.e., no coalescing possible).
1216 */
1217 bufcnt = (len + secsize - 1) / secsize;
1218 bufs = kmem_zalloc(bufcnt * sizeof (struct buf), KM_SLEEP);
1219 vas = kmem_alloc(bufcnt * sizeof (caddr_t), KM_SLEEP);
1220
1221 /*
1222 * Allocate a array of semaphores if we are doing I/O
1223 * scheduling.
1224 */
1225 if (fsp->hqueue != NULL)
1226 fio_done = kmem_alloc(bufcnt * sizeof (ksema_t),
1227 KM_SLEEP);
1228 for (count = 0; count < bufcnt; count++) {
1229 bioinit(&bufs[count]);
1230 bufs[count].b_edev = devvp->v_rdev;
1231 bufs[count].b_dev = cmpdev(devvp->v_rdev);
1232 bufs[count].b_flags = B_NOCACHE|B_BUSY|B_READ;
1233 bufs[count].b_iodone = hsfs_iodone;
1234 bufs[count].b_vp = vp;
1235 bufs[count].b_file = vp;
1236 }
1237
1238 /*
1239 * If our filesize is not an integer multiple of PAGESIZE,
1240 * we zero that part of the last page that's between EOF and
1241 * the PAGESIZE boundary.
1242 */
1243 xlen = io_len & PAGEOFFSET;
1244 if (xlen != 0)
1245 pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
1246
1247 va = NULL;
1248 lastp = NULL;
1249 searchp = pp;
1250 io_end = io_off + io_len;
1251 for (count = 0, byte_offset = io_off;
1252 byte_offset < io_end; count++) {
1253 ASSERT(count < bufcnt);
1254
1255 /* Compute disk address for interleaving. */
1256
1257 /* considered without skips */
1258 which_chunk_lbn = byte_offset / chunk_data_bytes;
1259
1260 /* factor in skips */
1261 offset_lbn = which_chunk_lbn * chunk_lbn_count;
1262
1263 /* convert to physical byte offset for lbn */
1264 offset_bytes = LBN_TO_BYTE(offset_lbn, vp->v_vfsp);
1265
1266 /* don't forget offset into lbn */
1267 offset_extra = byte_offset % chunk_data_bytes;
1268
1269 /* get virtual block number for driver */
1270 driver_block =
1271 lbtodb(bof + xarsiz + offset_bytes + offset_extra);
1272
1273 if (lastp != searchp) {
1274 /* this branch taken first time through loop */
1275 va = vas[count] =
1276 ppmapin(searchp, PROT_WRITE, (caddr_t)-1);
1277 /* ppmapin() guarantees not to return NULL */
1278 } else {
1279 vas[count] = NULL;
1280 }
1281
1282 bufs[count].b_un.b_addr = va + byte_offset % PAGESIZE;
1283 bufs[count].b_offset =
1284 (offset_t)(byte_offset - io_off + off);
1285
1286 /*
1287 * We specifically use the b_lblkno member here
1288 * as even in the 32 bit world driver_block can
1289 * get very large in line with the ISO9660 spec.
1290 */
1291
1292 bufs[count].b_lblkno = driver_block;
1293
1294 remaining_bytes =
1295 ((which_chunk_lbn + 1) * chunk_data_bytes)
1296 - byte_offset;
1297
1298 /*
1299 * remaining_bytes can't be zero, as we derived
1300 * which_chunk_lbn directly from byte_offset.
1301 */
1302 if ((remaining_bytes + byte_offset) < (off + len)) {
1303 /* coalesce-read the rest of the chunk */
1304 bufs[count].b_bcount = remaining_bytes;
1305 } else {
1306 /* get the final bits */
1307 bufs[count].b_bcount = off + len - byte_offset;
1308 }
1309
1310 /*
1311 * It would be nice to do multiple pages'
1312 * worth at once here when the opportunity
1313 * arises, as that has been shown to improve
1314 * our wall time. However, to do that
1315 * requires that we use the pageio subsystem,
1316 * which doesn't mix well with what we're
1317 * already using here. We can't use pageio
1318 * all the time, because that subsystem
1319 * assumes that a page is stored in N
1320 * contiguous blocks on the device.
1321 * Interleaving violates that assumption.
1322 *
1323 * Update: This is now not so big a problem
1324 * because of the I/O scheduler sitting below
1325 * that can re-order and coalesce I/O requests.
1326 */
1327
1328 remainder = PAGESIZE - (byte_offset % PAGESIZE);
1329 if (bufs[count].b_bcount > remainder) {
1330 bufs[count].b_bcount = remainder;
1331 }
1332
1333 bufs[count].b_bufsize = bufs[count].b_bcount;
1334 if (((offset_t)byte_offset + bufs[count].b_bcount) >
1335 HS_MAXFILEOFF) {
1336 break;
1337 }
1338 byte_offset += bufs[count].b_bcount;
1339
1340 if (fsp->hqueue == NULL) {
1341 (void) bdev_strategy(&bufs[count]);
1342
1343 } else {
1344 /*
1345 * We are scheduling I/O so we need to enqueue
1346 * requests rather than calling bdev_strategy
1347 * here. A later invocation of the scheduling
1348 * function will take care of doing the actual
1349 * I/O as it selects requests from the queue as
1350 * per the scheduling logic.
1351 */
1352 struct hio *hsio = kmem_cache_alloc(hio_cache,
1353 KM_SLEEP);
1354
1355 sema_init(&fio_done[count], 0, NULL,
1356 SEMA_DEFAULT, NULL);
1357 hsio->bp = &bufs[count];
1358 hsio->sema = &fio_done[count];
1359 hsio->io_lblkno = bufs[count].b_lblkno;
1360 hsio->nblocks = howmany(hsio->bp->b_bcount,
1361 DEV_BSIZE);
1362
1363 /* used for deadline */
1364 hsio->io_timestamp =
1365 drv_hztousec(ddi_get_lbolt());
1366
1367 /* for I/O coalescing */
1368 hsio->contig_chain = NULL;
1369 hsched_enqueue_io(fsp, hsio, 0);
1370 }
1371
1372 lwp_stat_update(LWP_STAT_INBLK, 1);
1373 lastp = searchp;
1374 if ((remainder - bufs[count].b_bcount) < 1) {
1375 searchp = searchp->p_next;
1376 }
1377 }
1378
1379 bufsused = count;
1380 /* Now wait for everything to come in */
1381 if (fsp->hqueue == NULL) {
1382 for (count = 0; count < bufsused; count++) {
1383 if (err == 0) {
1384 err = biowait(&bufs[count]);
1385 } else
1386 (void) biowait(&bufs[count]);
1387 }
1388 } else {
1389 for (count = 0; count < bufsused; count++) {
1390 struct buf *wbuf;
1391
1392 /*
1393 * Invoke scheduling function till our buf
1394 * is processed. In doing this it might
1395 * process bufs enqueued by other threads
1396 * which is good.
1397 */
1398 wbuf = &bufs[count];
1399 DTRACE_PROBE1(hsfs_io_wait, struct buf *, wbuf);
1400 while (sema_tryp(&fio_done[count]) == 0) {
1401 /*
1402 * hsched_invoke_strategy will return 1
1403 * if the I/O queue is empty. This means
1404 * that there is another thread who has
1405 * issued our buf and is waiting. So we
1406 * just block instead of spinning.
1407 */
1408 if (hsched_invoke_strategy(fsp)) {
1409 sema_p(&fio_done[count]);
1410 break;
1411 }
1412 }
1413 sema_destroy(&fio_done[count]);
1414 DTRACE_PROBE1(hsfs_io_done, struct buf *, wbuf);
1415
1416 if (err == 0) {
1417 err = geterror(wbuf);
1418 }
1419 }
1420 kmem_free(fio_done, bufcnt * sizeof (ksema_t));
1421 }
1422
1423 /* Don't leak resources */
1424 for (count = 0; count < bufcnt; count++) {
1425 biofini(&bufs[count]);
1426 if (count < bufsused && vas[count] != NULL) {
1427 ppmapout(vas[count]);
1428 }
1429 }
1430
1431 kmem_free(vas, bufcnt * sizeof (caddr_t));
1432 kmem_free(bufs, bufcnt * sizeof (struct buf));
1433 }
1434
1435 if (err) {
1436 pvn_read_done(pp, B_ERROR);
1437 return (err);
1438 }
1439
1440 /*
1441 * Lock the requested page, and the one after it if possible.
1442 * Don't bother if our caller hasn't given us a place to stash
1443 * the page pointers, since otherwise we'd lock pages that would
1444 * never get unlocked.
1445 */
1446 if (pagefound) {
1447 int index;
1448 ulong_t soff;
1449
1450 /*
1451 * Make sure it's in memory before we say it's here.
1452 */
1453 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) {
1454 hsfs_lostpage++;
1455 goto reread;
1456 }
1457
1458 pl[0] = pp;
1459 index = 1;
1460 atomic_inc_64(&fsp->cache_read_pages);
1461
1462 /*
1463 * Try to lock the next page, if it exists, without
1464 * blocking.
1465 */
1466 plsz -= PAGESIZE;
1467 /* LINTED (plsz is unsigned) */
1468 for (soff = off + PAGESIZE; plsz > 0;
1469 soff += PAGESIZE, plsz -= PAGESIZE) {
1470 pp = page_lookup_nowait(vp, (u_offset_t)soff,
1471 SE_SHARED);
1472 if (pp == NULL)
1473 break;
1474 pl[index++] = pp;
1475 }
1476 pl[index] = NULL;
1477
1478 /*
1479 * Schedule a semi-asynchronous readahead if we are
1480 * accessing the last cached page for the current
1481 * file.
1482 *
1483 * Doing this here means that readaheads will be
1484 * issued only if cache-hits occur. This is an advantage
1485 * since cache-hits would mean that readahead is giving
1486 * the desired benefit. If cache-hits do not occur there
1487 * is no point in reading ahead of time - the system
1488 * is loaded anyway.
1489 */
1490 if (fsp->hqueue != NULL &&
1491 hp->hs_prev_offset - off == PAGESIZE &&
1492 hp->hs_prev_offset < filsiz &&
1493 hp->hs_ra_bytes > 0 &&
1494 !page_exists(vp, hp->hs_prev_offset)) {
1495 (void) hsfs_getpage_ra(vp, hp->hs_prev_offset, seg,
1496 addr + PAGESIZE, hp, fsp, xarsiz, bof,
1497 chunk_lbn_count, chunk_data_bytes);
1498 }
1499
1500 return (0);
1501 }
1502
1503 if (pp != NULL) {
1504 pvn_plist_init(pp, pl, plsz, off, io_len, rw);
1505 }
1506
1507 return (err);
1508 }
1509
1510 /*ARGSUSED*/
1511 static int
1512 hsfs_getpage(struct vnode *vp, offset_t off, size_t len, uint_t *protp,
1513 struct page *pl[], size_t plsz, struct seg *seg, caddr_t addr,
1514 enum seg_rw rw, struct cred *cred, caller_context_t *ct)
1515 {
1516 uint_t filsiz;
1517 struct hsfs *fsp;
1518 struct hsnode *hp;
1519
1520 fsp = VFS_TO_HSFS(vp->v_vfsp);
1521 hp = VTOH(vp);
1522
1523 /* does not support write */
1524 if (rw == S_WRITE) {
1525 return (EROFS);
1526 }
1527
1528 if (vp->v_flag & VNOMAP) {
1529 return (ENOSYS);
1530 }
1531
1532 ASSERT(off <= HS_MAXFILEOFF);
1533
1534 /*
1535 * Determine file data size for EOF check.
1536 */
1537 filsiz = hp->hs_dirent.ext_size;
1538 if ((off + len) > (offset_t)(filsiz + PAGEOFFSET) && seg != segkmap)
1539 return (EFAULT); /* beyond EOF */
1540
1541 /*
1542 * Async Read-ahead computation.
1543 * This attempts to detect sequential access pattern and
1544 * enables reading extra pages ahead of time.
1545 */
1546 if (fsp->hqueue != NULL) {
1547 /*
1548 * This check for sequential access also takes into
1549 * account segmap weirdness when reading in chunks
1550 * less than the segmap size of 8K.
1551 */
1552 if (hp->hs_prev_offset == off || (off <
1553 hp->hs_prev_offset && off + MAX(len, PAGESIZE)
1554 >= hp->hs_prev_offset)) {
1555 if (hp->hs_num_contig <
1556 (seq_contig_requests - 1)) {
1557 hp->hs_num_contig++;
1558
1559 } else {
1560 /*
1561 * We increase readahead quantum till
1562 * a predefined max. max_readahead_bytes
1563 * is a multiple of PAGESIZE.
1564 */
1565 if (hp->hs_ra_bytes <
1566 fsp->hqueue->max_ra_bytes) {
1567 hp->hs_ra_bytes += PAGESIZE;
1568 }
1569 }
1570 } else {
1571 /*
1572 * Not contiguous so reduce read ahead counters.
1573 */
1574 if (hp->hs_ra_bytes > 0)
1575 hp->hs_ra_bytes -= PAGESIZE;
1576
1577 if (hp->hs_ra_bytes <= 0) {
1578 hp->hs_ra_bytes = 0;
1579 if (hp->hs_num_contig > 0)
1580 hp->hs_num_contig--;
1581 }
1582 }
1583 /*
1584 * Length must be rounded up to page boundary.
1585 * since we read in units of pages.
1586 */
1587 hp->hs_prev_offset = off + roundup(len, PAGESIZE);
1588 DTRACE_PROBE1(hsfs_compute_ra, struct hsnode *, hp);
1589 }
1590 if (protp != NULL)
1591 *protp = PROT_ALL;
1592
1593 return (pvn_getpages(hsfs_getapage, vp, off, len, protp, pl, plsz,
1594 seg, addr, rw, cred));
1595 }
1596
1597
1598
1599 /*
1600 * This function should never be called. We need to have it to pass
1601 * it as an argument to other functions.
1602 */
1603 /*ARGSUSED*/
1604 int
1605 hsfs_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp, size_t *lenp,
1606 int flags, cred_t *cr)
1607 {
1608 /* should never happen - just destroy it */
1609 cmn_err(CE_NOTE, "hsfs_putapage: dirty HSFS page");
1610 pvn_write_done(pp, B_ERROR | B_WRITE | B_INVAL | B_FORCE | flags);
1611 return (0);
1612 }
1613
1614
1615 /*
1616 * The only flags we support are B_INVAL, B_FREE and B_DONTNEED.
1617 * B_INVAL is set by:
1618 *
1619 * 1) the MC_SYNC command of memcntl(2) to support the MS_INVALIDATE flag.
1620 * 2) the MC_ADVISE command of memcntl(2) with the MADV_DONTNEED advice
1621 * which translates to an MC_SYNC with the MS_INVALIDATE flag.
1622 *
1623 * The B_FREE (as well as the B_DONTNEED) flag is set when the
1624 * MADV_SEQUENTIAL advice has been used. VOP_PUTPAGE is invoked
1625 * from SEGVN to release pages behind a pagefault.
1626 */
1627 /*ARGSUSED*/
1628 static int
1629 hsfs_putpage(struct vnode *vp, offset_t off, size_t len, int flags,
1630 struct cred *cr, caller_context_t *ct)
1631 {
1632 int error = 0;
1633
1634 if (vp->v_count == 0) {
1635 panic("hsfs_putpage: bad v_count");
1636 /*NOTREACHED*/
1637 }
1638
1639 if (vp->v_flag & VNOMAP)
1640 return (ENOSYS);
1641
1642 ASSERT(off <= HS_MAXFILEOFF);
1643
1644 if (!vn_has_cached_data(vp)) /* no pages mapped */
1645 return (0);
1646
1647 if (len == 0) { /* from 'off' to EOF */
1648 error = pvn_vplist_dirty(vp, off, hsfs_putapage, flags, cr);
1649 } else {
1650 offset_t end_off = off + len;
1651 offset_t file_size = VTOH(vp)->hs_dirent.ext_size;
1652 offset_t io_off;
1653
1654 file_size = (file_size + PAGESIZE - 1) & PAGEMASK;
1655 if (end_off > file_size)
1656 end_off = file_size;
1657
1658 for (io_off = off; io_off < end_off; io_off += PAGESIZE) {
1659 page_t *pp;
1660
1661 /*
1662 * We insist on getting the page only if we are
1663 * about to invalidate, free or write it and
1664 * the B_ASYNC flag is not set.
1665 */
1666 if ((flags & B_INVAL) || ((flags & B_ASYNC) == 0)) {
1667 pp = page_lookup(vp, io_off,
1668 (flags & (B_INVAL | B_FREE)) ?
1669 SE_EXCL : SE_SHARED);
1670 } else {
1671 pp = page_lookup_nowait(vp, io_off,
1672 (flags & B_FREE) ? SE_EXCL : SE_SHARED);
1673 }
1674
1675 if (pp == NULL)
1676 continue;
1677
1678 /*
1679 * Normally pvn_getdirty() should return 0, which
1680 * impies that it has done the job for us.
1681 * The shouldn't-happen scenario is when it returns 1.
1682 * This means that the page has been modified and
1683 * needs to be put back.
1684 * Since we can't write on a CD, we fake a failed
1685 * I/O and force pvn_write_done() to destroy the page.
1686 */
1687 if (pvn_getdirty(pp, flags) == 1) {
1688 cmn_err(CE_NOTE,
1689 "hsfs_putpage: dirty HSFS page");
1690 pvn_write_done(pp, flags |
1691 B_ERROR | B_WRITE | B_INVAL | B_FORCE);
1692 }
1693 }
1694 }
1695 return (error);
1696 }
1697
1698
1699 /*ARGSUSED*/
1700 static int
1701 hsfs_map(struct vnode *vp, offset_t off, struct as *as, caddr_t *addrp,
1702 size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, struct cred *cred,
1703 caller_context_t *ct)
1704 {
1705 struct segvn_crargs vn_a;
1706 int error;
1707
1708 /* VFS_RECORD(vp->v_vfsp, VS_MAP, VS_CALL); */
1709
1710 if (vp->v_flag & VNOMAP)
1711 return (ENOSYS);
1712
1713 if ((prot & PROT_WRITE) && (flags & MAP_SHARED))
1714 return (ENOSYS);
1715
1716 if (off > HS_MAXFILEOFF || off < 0 ||
1717 (off + len) < 0 || (off + len) > HS_MAXFILEOFF)
1718 return (ENXIO);
1719
1720 if (vp->v_type != VREG) {
1721 return (ENODEV);
1722 }
1723
1724 /*
1725 * If file is being locked, disallow mapping.
1726 */
1727 if (vn_has_mandatory_locks(vp, VTOH(vp)->hs_dirent.mode))
1728 return (EAGAIN);
1729
1730 as_rangelock(as);
1731 error = choose_addr(as, addrp, len, off, ADDR_VACALIGN, flags);
1732 if (error != 0) {
1733 as_rangeunlock(as);
1734 return (error);
1735 }
1736
1737 vn_a.vp = vp;
1738 vn_a.offset = off;
1739 vn_a.type = flags & MAP_TYPE;
1740 vn_a.prot = prot;
1741 vn_a.maxprot = maxprot;
1742 vn_a.flags = flags & ~MAP_TYPE;
1743 vn_a.cred = cred;
1744 vn_a.amp = NULL;
1745 vn_a.szc = 0;
1746 vn_a.lgrp_mem_policy_flags = 0;
1747
1748 error = as_map(as, *addrp, len, segvn_create, &vn_a);
1749 as_rangeunlock(as);
1750 return (error);
1751 }
1752
1753 /* ARGSUSED */
1754 static int
1755 hsfs_addmap(struct vnode *vp, offset_t off, struct as *as, caddr_t addr,
1756 size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, struct cred *cr,
1757 caller_context_t *ct)
1758 {
1759 struct hsnode *hp;
1760
1761 if (vp->v_flag & VNOMAP)
1762 return (ENOSYS);
1763
1764 hp = VTOH(vp);
1765 mutex_enter(&hp->hs_contents_lock);
1766 hp->hs_mapcnt += btopr(len);
1767 mutex_exit(&hp->hs_contents_lock);
1768 return (0);
1769 }
1770
1771 /*ARGSUSED*/
1772 static int
1773 hsfs_delmap(struct vnode *vp, offset_t off, struct as *as, caddr_t addr,
1774 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cr,
1775 caller_context_t *ct)
1776 {
1777 struct hsnode *hp;
1778
1779 if (vp->v_flag & VNOMAP)
1780 return (ENOSYS);
1781
1782 hp = VTOH(vp);
1783 mutex_enter(&hp->hs_contents_lock);
1784 hp->hs_mapcnt -= btopr(len); /* Count released mappings */
1785 ASSERT(hp->hs_mapcnt >= 0);
1786 mutex_exit(&hp->hs_contents_lock);
1787 return (0);
1788 }
1789
1790 /* ARGSUSED */
1791 static int
1792 hsfs_seek(struct vnode *vp, offset_t ooff, offset_t *noffp,
1793 caller_context_t *ct)
1794 {
1795 return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
1796 }
1797
1798 /* ARGSUSED */
1799 static int
1800 hsfs_frlock(struct vnode *vp, int cmd, struct flock64 *bfp, int flag,
1801 offset_t offset, struct flk_callback *flk_cbp, cred_t *cr,
1802 caller_context_t *ct)
1803 {
1804 struct hsnode *hp = VTOH(vp);
1805
1806 /*
1807 * If the file is being mapped, disallow fs_frlock.
1808 * We are not holding the hs_contents_lock while checking
1809 * hs_mapcnt because the current locking strategy drops all
1810 * locks before calling fs_frlock.
1811 * So, hs_mapcnt could change before we enter fs_frlock making
1812 * it meaningless to have held hs_contents_lock in the first place.
1813 */
1814 if (hp->hs_mapcnt > 0 && MANDLOCK(vp, hp->hs_dirent.mode))
1815 return (EAGAIN);
1816
1817 return (fs_frlock(vp, cmd, bfp, flag, offset, flk_cbp, cr, ct));
1818 }
1819
1820 static int
1821 hsched_deadline_compare(const void *x1, const void *x2)
1822 {
1823 const struct hio *h1 = x1;
1824 const struct hio *h2 = x2;
1825
1826 if (h1->io_timestamp < h2->io_timestamp)
1827 return (-1);
1828 if (h1->io_timestamp > h2->io_timestamp)
1829 return (1);
1830
1831 if (h1->io_lblkno < h2->io_lblkno)
1832 return (-1);
1833 if (h1->io_lblkno > h2->io_lblkno)
1834 return (1);
1835
1836 if (h1 < h2)
1837 return (-1);
1838 if (h1 > h2)
1839 return (1);
1840
1841 return (0);
1842 }
1843
1844 static int
1845 hsched_offset_compare(const void *x1, const void *x2)
1846 {
1847 const struct hio *h1 = x1;
1848 const struct hio *h2 = x2;
1849
1850 if (h1->io_lblkno < h2->io_lblkno)
1851 return (-1);
1852 if (h1->io_lblkno > h2->io_lblkno)
1853 return (1);
1854
1855 if (h1 < h2)
1856 return (-1);
1857 if (h1 > h2)
1858 return (1);
1859
1860 return (0);
1861 }
1862
1863 void
1864 hsched_init_caches(void)
1865 {
1866 hio_cache = kmem_cache_create("hsfs_hio_cache",
1867 sizeof (struct hio), 0, NULL,
1868 NULL, NULL, NULL, NULL, 0);
1869
1870 hio_info_cache = kmem_cache_create("hsfs_hio_info_cache",
1871 sizeof (struct hio_info), 0, NULL,
1872 NULL, NULL, NULL, NULL, 0);
1873 }
1874
1875 void
1876 hsched_fini_caches(void)
1877 {
1878 kmem_cache_destroy(hio_cache);
1879 kmem_cache_destroy(hio_info_cache);
1880 }
1881
1882 /*
1883 * Initialize I/O scheduling structures. This is called via hsfs_mount
1884 */
1885 void
1886 hsched_init(struct hsfs *fsp, int fsid, struct modlinkage *modlinkage)
1887 {
1888 struct hsfs_queue *hqueue = fsp->hqueue;
1889 struct vnode *vp = fsp->hsfs_devvp;
1890
1891 /* TaskQ name of the form: hsched_task_ + stringof(int) */
1892 char namebuf[23];
1893 int error, err;
1894 struct dk_cinfo info;
1895 ldi_handle_t lh;
1896 ldi_ident_t li;
1897
1898 /*
1899 * Default maxtransfer = 16k chunk
1900 */
1901 hqueue->dev_maxtransfer = 16384;
1902
1903 /*
1904 * Try to fetch the maximum device transfer size. This is used to
1905 * ensure that a coalesced block does not exceed the maxtransfer.
1906 */
1907 err = ldi_ident_from_mod(modlinkage, &li);
1908 if (err) {
1909 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
1910 cmn_err(CE_NOTE, "hsched_init: ldi_ident_from_mod err=%d\n",
1911 err);
1912 goto set_ra;
1913 }
1914
1915 err = ldi_open_by_dev(&(vp->v_rdev), OTYP_CHR, FREAD, CRED(), &lh, li);
1916 ldi_ident_release(li);
1917 if (err) {
1918 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
1919 cmn_err(CE_NOTE, "hsched_init: ldi_open err=%d\n", err);
1920 goto set_ra;
1921 }
1922
1923 error = ldi_ioctl(lh, DKIOCINFO, (intptr_t)&info, FKIOCTL,
1924 CRED(), &err);
1925 err = ldi_close(lh, FREAD, CRED());
1926 if (err) {
1927 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
1928 cmn_err(CE_NOTE, "hsched_init: ldi_close err=%d\n", err);
1929 }
1930
1931 if (error == 0) {
1932 hqueue->dev_maxtransfer = ldbtob(info.dki_maxtransfer);
1933 }
1934
1935 set_ra:
1936 /*
1937 * Max size of data to read ahead for sequential access pattern.
1938 * Conservative to avoid letting the underlying CD drive to spin
1939 * down, in case the application is reading slowly.
1940 * We read ahead upto a max of 4 pages.
1941 */
1942 hqueue->max_ra_bytes = PAGESIZE * 8;
1943
1944 mutex_init(&(hqueue->hsfs_queue_lock), NULL, MUTEX_DEFAULT, NULL);
1945 mutex_init(&(hqueue->strategy_lock), NULL, MUTEX_DEFAULT, NULL);
1946 avl_create(&(hqueue->read_tree), hsched_offset_compare,
1947 sizeof (struct hio), offsetof(struct hio, io_offset_node));
1948 avl_create(&(hqueue->deadline_tree), hsched_deadline_compare,
1949 sizeof (struct hio), offsetof(struct hio, io_deadline_node));
1950
1951 (void) snprintf(namebuf, sizeof (namebuf), "hsched_task_%d", fsid);
1952 hqueue->ra_task = taskq_create(namebuf, hsfs_taskq_nthreads,
1953 minclsyspri + 2, 1, 104857600 / PAGESIZE, TASKQ_DYNAMIC);
1954
1955 hqueue->next = NULL;
1956 hqueue->nbuf = kmem_zalloc(sizeof (struct buf), KM_SLEEP);
1957 }
1958
1959 void
1960 hsched_fini(struct hsfs_queue *hqueue)
1961 {
1962 if (hqueue != NULL) {
1963 /*
1964 * Remove the sentinel if there was one.
1965 */
1966 if (hqueue->next != NULL) {
1967 avl_remove(&hqueue->read_tree, hqueue->next);
1968 kmem_cache_free(hio_cache, hqueue->next);
1969 }
1970 avl_destroy(&(hqueue->read_tree));
1971 avl_destroy(&(hqueue->deadline_tree));
1972 mutex_destroy(&(hqueue->hsfs_queue_lock));
1973 mutex_destroy(&(hqueue->strategy_lock));
1974
1975 /*
1976 * If there are any existing readahead threads running
1977 * taskq_destroy will wait for them to finish.
1978 */
1979 taskq_destroy(hqueue->ra_task);
1980 kmem_free(hqueue->nbuf, sizeof (struct buf));
1981 }
1982 }
1983
1984 /*
1985 * Determine if two I/O requests are adjacent to each other so
1986 * that they can coalesced.
1987 */
1988 #define IS_ADJACENT(io, nio) \
1989 (((io)->io_lblkno + (io)->nblocks == (nio)->io_lblkno) && \
1990 (io)->bp->b_edev == (nio)->bp->b_edev)
1991
1992 /*
1993 * This performs the actual I/O scheduling logic. We use the Circular
1994 * Look algorithm here. Sort the I/O requests in ascending order of
1995 * logical block number and process them starting with the lowest
1996 * numbered block and progressing towards higher block numbers in the
1997 * queue. Once there are no more higher numbered blocks, start again
1998 * with the lowest one. This is good for CD/DVD as you keep moving
1999 * the head in one direction along the outward spiral track and avoid
2000 * too many seeks as much as possible. The re-ordering also allows
2001 * us to coalesce adjacent requests into one larger request.
2002 * This is thus essentially a 1-way Elevator with front merging.
2003 *
2004 * In addition each read request here has a deadline and will be
2005 * processed out of turn if the deadline (500ms) expires.
2006 *
2007 * This function is necessarily serialized via hqueue->strategy_lock.
2008 * This function sits just below hsfs_getapage and processes all read
2009 * requests orginating from that function.
2010 */
2011 int
2012 hsched_invoke_strategy(struct hsfs *fsp)
2013 {
2014 struct hsfs_queue *hqueue;
2015 struct buf *nbuf;
2016 struct hio *fio, *nio, *tio, *prev, *last;
2017 size_t bsize, soffset, offset, data;
2018 int bioret, bufcount;
2019 struct vnode *fvp;
2020 ksema_t *io_done;
2021 caddr_t iodata;
2022
2023 hqueue = fsp->hqueue;
2024 mutex_enter(&hqueue->strategy_lock);
2025 mutex_enter(&hqueue->hsfs_queue_lock);
2026
2027 /*
2028 * Check for Deadline expiration first
2029 */
2030 fio = avl_first(&hqueue->deadline_tree);
2031
2032 /*
2033 * Paranoid check for empty I/O queue. Both deadline
2034 * and read trees contain same data sorted in different
2035 * ways. So empty deadline tree = empty read tree.
2036 */
2037 if (fio == NULL) {
2038 /*
2039 * Remove the sentinel if there was one.
2040 */
2041 if (hqueue->next != NULL) {
2042 avl_remove(&hqueue->read_tree, hqueue->next);
2043 kmem_cache_free(hio_cache, hqueue->next);
2044 hqueue->next = NULL;
2045 }
2046 mutex_exit(&hqueue->hsfs_queue_lock);
2047 mutex_exit(&hqueue->strategy_lock);
2048 return (1);
2049 }
2050
2051 if (drv_hztousec(ddi_get_lbolt()) - fio->io_timestamp
2052 < HSFS_READ_DEADLINE) {
2053 /*
2054 * Apply standard scheduling logic. This uses the
2055 * C-LOOK approach. Process I/O requests in ascending
2056 * order of logical block address till no subsequent
2057 * higher numbered block request remains. Then start
2058 * again from the lowest numbered block in the queue.
2059 *
2060 * We do this cheaply here by means of a sentinel.
2061 * The last processed I/O structure from the previous
2062 * invocation of this func, is left dangling in the
2063 * read_tree so that we can easily scan to the next
2064 * higher numbered request and remove the sentinel.
2065 */
2066 fio = NULL;
2067 if (hqueue->next != NULL) {
2068 fio = AVL_NEXT(&hqueue->read_tree, hqueue->next);
2069 avl_remove(&hqueue->read_tree, hqueue->next);
2070 kmem_cache_free(hio_cache, hqueue->next);
2071 hqueue->next = NULL;
2072 }
2073 if (fio == NULL) {
2074 fio = avl_first(&hqueue->read_tree);
2075 }
2076 } else if (hqueue->next != NULL) {
2077 DTRACE_PROBE1(hsfs_deadline_expiry, struct hio *, fio);
2078
2079 avl_remove(&hqueue->read_tree, hqueue->next);
2080 kmem_cache_free(hio_cache, hqueue->next);
2081 hqueue->next = NULL;
2082 }
2083
2084 /*
2085 * In addition we try to coalesce contiguous
2086 * requests into one bigger request.
2087 */
2088 bufcount = 1;
2089 bsize = ldbtob(fio->nblocks);
2090 fvp = fio->bp->b_file;
2091 nio = AVL_NEXT(&hqueue->read_tree, fio);
2092 tio = fio;
2093 while (nio != NULL && IS_ADJACENT(tio, nio) &&
2094 bsize < hqueue->dev_maxtransfer) {
2095 avl_remove(&hqueue->deadline_tree, tio);
2096 avl_remove(&hqueue->read_tree, tio);
2097 tio->contig_chain = nio;
2098 bsize += ldbtob(nio->nblocks);
2099 prev = tio;
2100 tio = nio;
2101
2102 /*
2103 * This check is required to detect the case where
2104 * we are merging adjacent buffers belonging to
2105 * different files. fvp is used to set the b_file
2106 * parameter in the coalesced buf. b_file is used
2107 * by DTrace so we do not want DTrace to accrue
2108 * requests to two different files to any one file.
2109 */
2110 if (fvp && tio->bp->b_file != fvp) {
2111 fvp = NULL;
2112 }
2113
2114 nio = AVL_NEXT(&hqueue->read_tree, nio);
2115 bufcount++;
2116 }
2117
2118 /*
2119 * tio is not removed from the read_tree as it serves as a sentinel
2120 * to cheaply allow us to scan to the next higher numbered I/O
2121 * request.
2122 */
2123 hqueue->next = tio;
2124 avl_remove(&hqueue->deadline_tree, tio);
2125 mutex_exit(&hqueue->hsfs_queue_lock);
2126 DTRACE_PROBE3(hsfs_io_dequeued, struct hio *, fio, int, bufcount,
2127 size_t, bsize);
2128
2129 /*
2130 * The benefit of coalescing occurs if the the savings in I/O outweighs
2131 * the cost of doing the additional work below.
2132 * It was observed that coalescing 2 buffers results in diminishing
2133 * returns, so we do coalescing if we have >2 adjacent bufs.
2134 */
2135 if (bufcount > hsched_coalesce_min) {
2136 /*
2137 * We have coalesced blocks. First allocate mem and buf for
2138 * the entire coalesced chunk.
2139 * Since we are guaranteed single-threaded here we pre-allocate
2140 * one buf at mount time and that is re-used every time. This
2141 * is a synthesized buf structure that uses kmem_alloced chunk.
2142 * Not quite a normal buf attached to pages.
2143 */
2144 fsp->coalesced_bytes += bsize;
2145 nbuf = hqueue->nbuf;
2146 bioinit(nbuf);
2147 nbuf->b_edev = fio->bp->b_edev;
2148 nbuf->b_dev = fio->bp->b_dev;
2149 nbuf->b_flags = fio->bp->b_flags;
2150 nbuf->b_iodone = fio->bp->b_iodone;
2151 iodata = kmem_alloc(bsize, KM_SLEEP);
2152 nbuf->b_un.b_addr = iodata;
2153 nbuf->b_lblkno = fio->bp->b_lblkno;
2154 nbuf->b_vp = fvp;
2155 nbuf->b_file = fvp;
2156 nbuf->b_bcount = bsize;
2157 nbuf->b_bufsize = bsize;
2158
2159 DTRACE_PROBE3(hsfs_coalesced_io_start, struct hio *, fio, int,
2160 bufcount, size_t, bsize);
2161
2162 /*
2163 * Perform I/O for the coalesced block.
2164 */
2165 (void) bdev_strategy(nbuf);
2166
2167 /*
2168 * Duplicate the last IO node to leave the sentinel alone.
2169 * The sentinel is freed in the next invocation of this
2170 * function.
2171 */
2172 prev->contig_chain = kmem_cache_alloc(hio_cache, KM_SLEEP);
2173 prev->contig_chain->bp = tio->bp;
2174 prev->contig_chain->sema = tio->sema;
2175 tio = prev->contig_chain;
2176 tio->contig_chain = NULL;
2177 soffset = ldbtob(fio->bp->b_lblkno);
2178 nio = fio;
2179
2180 bioret = biowait(nbuf);
2181 data = bsize - nbuf->b_resid;
2182 biofini(nbuf);
2183 mutex_exit(&hqueue->strategy_lock);
2184
2185 /*
2186 * We use the b_resid parameter to detect how much
2187 * data was succesfully transferred. We will signal
2188 * a success to all the fully retrieved actual bufs
2189 * before coalescing, rest is signaled as error,
2190 * if any.
2191 */
2192 tio = nio;
2193 DTRACE_PROBE3(hsfs_coalesced_io_done, struct hio *, nio,
2194 int, bioret, size_t, data);
2195
2196 /*
2197 * Copy data and signal success to all the bufs
2198 * which can be fully satisfied from b_resid.
2199 */
2200 while (nio != NULL && data >= nio->bp->b_bcount) {
2201 offset = ldbtob(nio->bp->b_lblkno) - soffset;
2202 bcopy(iodata + offset, nio->bp->b_un.b_addr,
2203 nio->bp->b_bcount);
2204 data -= nio->bp->b_bcount;
2205 bioerror(nio->bp, 0);
2206 biodone(nio->bp);
2207 sema_v(nio->sema);
2208 tio = nio;
2209 nio = nio->contig_chain;
2210 kmem_cache_free(hio_cache, tio);
2211 }
2212
2213 /*
2214 * Signal error to all the leftover bufs (if any)
2215 * after b_resid data is exhausted.
2216 */
2217 while (nio != NULL) {
2218 nio->bp->b_resid = nio->bp->b_bcount - data;
2219 bzero(nio->bp->b_un.b_addr + data, nio->bp->b_resid);
2220 bioerror(nio->bp, bioret);
2221 biodone(nio->bp);
2222 sema_v(nio->sema);
2223 tio = nio;
2224 nio = nio->contig_chain;
2225 kmem_cache_free(hio_cache, tio);
2226 data = 0;
2227 }
2228 kmem_free(iodata, bsize);
2229 } else {
2230
2231 nbuf = tio->bp;
2232 io_done = tio->sema;
2233 nio = fio;
2234 last = tio;
2235
2236 while (nio != NULL) {
2237 (void) bdev_strategy(nio->bp);
2238 nio = nio->contig_chain;
2239 }
2240 nio = fio;
2241 mutex_exit(&hqueue->strategy_lock);
2242
2243 while (nio != NULL) {
2244 if (nio == last) {
2245 (void) biowait(nbuf);
2246 sema_v(io_done);
2247 break;
2248 /* sentinel last not freed. See above. */
2249 } else {
2250 (void) biowait(nio->bp);
2251 sema_v(nio->sema);
2252 }
2253 tio = nio;
2254 nio = nio->contig_chain;
2255 kmem_cache_free(hio_cache, tio);
2256 }
2257 }
2258 return (0);
2259 }
2260
2261 /*
2262 * Insert an I/O request in the I/O scheduler's pipeline
2263 * Using AVL tree makes it easy to reorder the I/O request
2264 * based on logical block number.
2265 */
2266 static void
2267 hsched_enqueue_io(struct hsfs *fsp, struct hio *hsio, int ra)
2268 {
2269 struct hsfs_queue *hqueue = fsp->hqueue;
2270
2271 mutex_enter(&hqueue->hsfs_queue_lock);
2272
2273 fsp->physical_read_bytes += hsio->bp->b_bcount;
2274 if (ra)
2275 fsp->readahead_bytes += hsio->bp->b_bcount;
2276
2277 avl_add(&hqueue->deadline_tree, hsio);
2278 avl_add(&hqueue->read_tree, hsio);
2279
2280 DTRACE_PROBE3(hsfs_io_enqueued, struct hio *, hsio,
2281 struct hsfs_queue *, hqueue, int, ra);
2282
2283 mutex_exit(&hqueue->hsfs_queue_lock);
2284 }
2285
2286 /* ARGSUSED */
2287 static int
2288 hsfs_pathconf(struct vnode *vp, int cmd, ulong_t *valp, struct cred *cr,
2289 caller_context_t *ct)
2290 {
2291 struct hsfs *fsp;
2292
2293 int error = 0;
2294
2295 switch (cmd) {
2296
2297 case _PC_NAME_MAX:
2298 fsp = VFS_TO_HSFS(vp->v_vfsp);
2299 *valp = fsp->hsfs_namemax;
2300 break;
2301
2302 case _PC_FILESIZEBITS:
2303 *valp = 33; /* Without multi extent support: 4 GB - 2k */
2304 break;
2305
2306 case _PC_TIMESTAMP_RESOLUTION:
2307 /*
2308 * HSFS keeps, at best, 1/100 second timestamp resolution.
2309 */
2310 *valp = 10000000L;
2311 break;
2312
2313 default:
2314 error = fs_pathconf(vp, cmd, valp, cr, ct);
2315 break;
2316 }
2317
2318 return (error);
2319 }
2320
2321
2322
2323 const fs_operation_def_t hsfs_vnodeops_template[] = {
2324 VOPNAME_OPEN, { .vop_open = hsfs_open },
2325 VOPNAME_CLOSE, { .vop_close = hsfs_close },
2326 VOPNAME_READ, { .vop_read = hsfs_read },
2327 VOPNAME_GETATTR, { .vop_getattr = hsfs_getattr },
2328 VOPNAME_ACCESS, { .vop_access = hsfs_access },
2329 VOPNAME_LOOKUP, { .vop_lookup = hsfs_lookup },
2330 VOPNAME_READDIR, { .vop_readdir = hsfs_readdir },
2331 VOPNAME_READLINK, { .vop_readlink = hsfs_readlink },
2332 VOPNAME_FSYNC, { .vop_fsync = hsfs_fsync },
2333 VOPNAME_INACTIVE, { .vop_inactive = hsfs_inactive },
2334 VOPNAME_FID, { .vop_fid = hsfs_fid },
2335 VOPNAME_SEEK, { .vop_seek = hsfs_seek },
2336 VOPNAME_FRLOCK, { .vop_frlock = hsfs_frlock },
2337 VOPNAME_GETPAGE, { .vop_getpage = hsfs_getpage },
2338 VOPNAME_PUTPAGE, { .vop_putpage = hsfs_putpage },
2339 VOPNAME_MAP, { .vop_map = hsfs_map },
2340 VOPNAME_ADDMAP, { .vop_addmap = hsfs_addmap },
2341 VOPNAME_DELMAP, { .vop_delmap = hsfs_delmap },
2342 VOPNAME_PATHCONF, { .vop_pathconf = hsfs_pathconf },
2343 NULL, NULL
2344 };
2345
2346 struct vnodeops *hsfs_vnodeops;