Print this page
8634 epoll fails to wake on certain edge-triggered conditions
8635 epoll should not emit POLLNVAL
8636 recursive epoll should emit EPOLLRDNORM
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Igor Kozhukhov <igor@dilos.org>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/devpoll.c
+++ new/usr/src/uts/common/io/devpoll.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
|
↓ open down ↓ |
17 lines elided |
↑ open up ↑ |
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /*
27 27 * Copyright (c) 2012 by Delphix. All rights reserved.
28 - * Copyright 2016 Joyent, Inc.
28 + * Copyright 2017 Joyent, Inc.
29 29 */
30 30
31 31 #include <sys/types.h>
32 32 #include <sys/devops.h>
33 33 #include <sys/conf.h>
34 34 #include <sys/modctl.h>
35 35 #include <sys/sunddi.h>
36 36 #include <sys/stat.h>
37 37 #include <sys/poll_impl.h>
38 38 #include <sys/errno.h>
39 39 #include <sys/kmem.h>
40 40 #include <sys/mkdev.h>
41 41 #include <sys/debug.h>
42 42 #include <sys/file.h>
43 43 #include <sys/sysmacros.h>
44 44 #include <sys/systm.h>
45 45 #include <sys/bitmap.h>
46 46 #include <sys/devpoll.h>
47 47 #include <sys/rctl.h>
48 48 #include <sys/resource.h>
49 49 #include <sys/schedctl.h>
50 50 #include <sys/epoll.h>
51 51
52 52 #define RESERVED 1
53 53
54 54 /* local data struct */
55 55 static dp_entry_t **devpolltbl; /* dev poll entries */
56 56 static size_t dptblsize;
57 57
58 58 static kmutex_t devpoll_lock; /* lock protecting dev tbl */
59 59 int devpoll_init; /* is /dev/poll initialized already */
60 60
61 61 /* device local functions */
62 62
63 63 static int dpopen(dev_t *devp, int flag, int otyp, cred_t *credp);
64 64 static int dpwrite(dev_t dev, struct uio *uiop, cred_t *credp);
65 65 static int dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
66 66 int *rvalp);
67 67 static int dppoll(dev_t dev, short events, int anyyet, short *reventsp,
68 68 struct pollhead **phpp);
69 69 static int dpclose(dev_t dev, int flag, int otyp, cred_t *credp);
70 70 static dev_info_t *dpdevi;
71 71
72 72
73 73 static struct cb_ops dp_cb_ops = {
74 74 dpopen, /* open */
75 75 dpclose, /* close */
76 76 nodev, /* strategy */
77 77 nodev, /* print */
78 78 nodev, /* dump */
79 79 nodev, /* read */
80 80 dpwrite, /* write */
81 81 dpioctl, /* ioctl */
82 82 nodev, /* devmap */
83 83 nodev, /* mmap */
84 84 nodev, /* segmap */
85 85 dppoll, /* poll */
86 86 ddi_prop_op, /* prop_op */
87 87 (struct streamtab *)0, /* streamtab */
88 88 D_MP, /* flags */
89 89 CB_REV, /* cb_ops revision */
90 90 nodev, /* aread */
91 91 nodev /* awrite */
92 92 };
93 93
94 94 static int dpattach(dev_info_t *, ddi_attach_cmd_t);
95 95 static int dpdetach(dev_info_t *, ddi_detach_cmd_t);
96 96 static int dpinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
97 97
98 98 static struct dev_ops dp_ops = {
99 99 DEVO_REV, /* devo_rev */
100 100 0, /* refcnt */
101 101 dpinfo, /* info */
102 102 nulldev, /* identify */
103 103 nulldev, /* probe */
104 104 dpattach, /* attach */
105 105 dpdetach, /* detach */
106 106 nodev, /* reset */
107 107 &dp_cb_ops, /* driver operations */
108 108 (struct bus_ops *)NULL, /* bus operations */
109 109 nulldev, /* power */
110 110 ddi_quiesce_not_needed, /* quiesce */
111 111 };
112 112
113 113
114 114 static struct modldrv modldrv = {
115 115 &mod_driverops, /* type of module - a driver */
116 116 "/dev/poll driver",
117 117 &dp_ops,
118 118 };
119 119
120 120 static struct modlinkage modlinkage = {
121 121 MODREV_1,
122 122 (void *)&modldrv,
123 123 NULL
124 124 };
125 125
126 126 static void pcachelink_assoc(pollcache_t *, pollcache_t *);
127 127 static void pcachelink_mark_stale(pollcache_t *);
128 128 static void pcachelink_purge_stale(pollcache_t *);
129 129 static void pcachelink_purge_all(pollcache_t *);
130 130
131 131
132 132 /*
133 133 * Locking Design
134 134 *
135 135 * The /dev/poll driver shares most of its code with poll sys call whose
136 136 * code is in common/syscall/poll.c. In poll(2) design, the pollcache
137 137 * structure is per lwp. An implicit assumption is made there that some
138 138 * portion of pollcache will never be touched by other lwps. E.g., in
139 139 * poll(2) design, no lwp will ever need to grow bitmap of other lwp.
140 140 * This assumption is not true for /dev/poll; hence the need for extra
141 141 * locking.
142 142 *
143 143 * To allow more parallelism, each /dev/poll file descriptor (indexed by
144 144 * minor number) has its own lock. Since read (dpioctl) is a much more
145 145 * frequent operation than write, we want to allow multiple reads on same
146 146 * /dev/poll fd. However, we prevent writes from being starved by giving
147 147 * priority to write operation. Theoretically writes can starve reads as
148 148 * well. But in practical sense this is not important because (1) writes
149 149 * happens less often than reads, and (2) write operation defines the
150 150 * content of poll fd a cache set. If writes happens so often that they
151 151 * can starve reads, that means the cached set is very unstable. It may
152 152 * not make sense to read an unstable cache set anyway. Therefore, the
153 153 * writers starving readers case is not handled in this design.
154 154 */
155 155
156 156 int
157 157 _init()
158 158 {
159 159 int error;
160 160
161 161 dptblsize = DEVPOLLSIZE;
162 162 devpolltbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
163 163 mutex_init(&devpoll_lock, NULL, MUTEX_DEFAULT, NULL);
164 164 devpoll_init = 1;
165 165 if ((error = mod_install(&modlinkage)) != 0) {
166 166 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
167 167 devpoll_init = 0;
168 168 }
169 169 return (error);
170 170 }
171 171
172 172 int
173 173 _fini()
174 174 {
175 175 int error;
176 176
177 177 if ((error = mod_remove(&modlinkage)) != 0) {
178 178 return (error);
179 179 }
180 180 mutex_destroy(&devpoll_lock);
181 181 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
182 182 return (0);
183 183 }
184 184
185 185 int
186 186 _info(struct modinfo *modinfop)
187 187 {
188 188 return (mod_info(&modlinkage, modinfop));
189 189 }
190 190
191 191 /*ARGSUSED*/
192 192 static int
193 193 dpattach(dev_info_t *devi, ddi_attach_cmd_t cmd)
194 194 {
195 195 if (ddi_create_minor_node(devi, "poll", S_IFCHR, 0, DDI_PSEUDO, NULL)
196 196 == DDI_FAILURE) {
197 197 ddi_remove_minor_node(devi, NULL);
198 198 return (DDI_FAILURE);
199 199 }
200 200 dpdevi = devi;
201 201 return (DDI_SUCCESS);
202 202 }
203 203
204 204 static int
205 205 dpdetach(dev_info_t *devi, ddi_detach_cmd_t cmd)
206 206 {
207 207 if (cmd != DDI_DETACH)
208 208 return (DDI_FAILURE);
209 209
210 210 ddi_remove_minor_node(devi, NULL);
211 211 return (DDI_SUCCESS);
212 212 }
213 213
214 214 /* ARGSUSED */
215 215 static int
216 216 dpinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
217 217 {
218 218 int error;
219 219
220 220 switch (infocmd) {
221 221 case DDI_INFO_DEVT2DEVINFO:
222 222 *result = (void *)dpdevi;
223 223 error = DDI_SUCCESS;
224 224 break;
225 225 case DDI_INFO_DEVT2INSTANCE:
226 226 *result = (void *)0;
227 227 error = DDI_SUCCESS;
228 228 break;
229 229 default:
230 230 error = DDI_FAILURE;
231 231 }
232 232 return (error);
233 233 }
234 234
235 235 /*
236 236 * dp_pcache_poll has similar logic to pcache_poll() in poll.c. The major
237 237 * differences are: (1) /dev/poll requires scanning the bitmap starting at
238 238 * where it was stopped last time, instead of always starting from 0,
239 239 * (2) since user may not have cleaned up the cached fds when they are
240 240 * closed, some polldats in cache may refer to closed or reused fds. We
241 241 * need to check for those cases.
242 242 *
243 243 * NOTE: Upon closing an fd, automatic poll cache cleanup is done for
244 244 * poll(2) caches but NOT for /dev/poll caches. So expect some
245 245 * stale entries!
246 246 */
247 247 static int
248 248 dp_pcache_poll(dp_entry_t *dpep, void *dpbuf,
249 249 pollcache_t *pcp, nfds_t nfds, int *fdcntp)
250 250 {
251 251 int start, ostart, end;
252 252 int fdcnt, fd;
253 253 boolean_t done;
254 254 file_t *fp;
255 255 short revent;
256 256 boolean_t no_wrap;
257 257 pollhead_t *php;
258 258 polldat_t *pdp;
259 259 pollfd_t *pfdp;
260 260 epoll_event_t *epoll;
261 261 int error = 0;
262 262 short mask = POLLRDHUP | POLLWRBAND;
263 263 boolean_t is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
264 264
265 265 ASSERT(MUTEX_HELD(&pcp->pc_lock));
266 266 if (pcp->pc_bitmap == NULL) {
267 267 /*
268 268 * No Need to search because no poll fd
269 269 * has been cached.
270 270 */
271 271 return (error);
272 272 }
273 273
274 274 if (is_epoll) {
275 275 pfdp = NULL;
276 276 epoll = (epoll_event_t *)dpbuf;
277 277 } else {
278 278 pfdp = (pollfd_t *)dpbuf;
279 279 epoll = NULL;
280 280 }
281 281 retry:
282 282 start = ostart = pcp->pc_mapstart;
283 283 end = pcp->pc_mapend;
284 284 php = NULL;
285 285
286 286 if (start == 0) {
287 287 /*
288 288 * started from every begining, no need to wrap around.
289 289 */
290 290 no_wrap = B_TRUE;
291 291 } else {
292 292 no_wrap = B_FALSE;
293 293 }
294 294 done = B_FALSE;
295 295 fdcnt = 0;
296 296 while ((fdcnt < nfds) && !done) {
297 297 php = NULL;
298 298 revent = 0;
299 299 /*
300 300 * Examine the bit map in a circular fashion
301 301 * to avoid starvation. Always resume from
302 302 * last stop. Scan till end of the map. Then
303 303 * wrap around.
304 304 */
305 305 fd = bt_getlowbit(pcp->pc_bitmap, start, end);
306 306 ASSERT(fd <= end);
307 307 if (fd >= 0) {
308 308 if (fd == end) {
309 309 if (no_wrap) {
310 310 done = B_TRUE;
311 311 } else {
312 312 start = 0;
313 313 end = ostart - 1;
314 314 no_wrap = B_TRUE;
315 315 }
316 316 } else {
317 317 start = fd + 1;
318 318 }
319 319 pdp = pcache_lookup_fd(pcp, fd);
320 320 repoll:
321 321 ASSERT(pdp != NULL);
322 322 ASSERT(pdp->pd_fd == fd);
323 323 if (pdp->pd_fp == NULL) {
324 324 /*
325 325 * The fd is POLLREMOVed. This fd is
326 326 * logically no longer cached. So move
327 327 * on to the next one.
328 328 */
329 329 continue;
330 330 }
331 331 if ((fp = getf(fd)) == NULL) {
332 332 /*
333 333 * The fd has been closed, but user has not
334 334 * done a POLLREMOVE on this fd yet. Instead
335 335 * of cleaning it here implicitly, we return
336 336 * POLLNVAL. This is consistent with poll(2)
337 337 * polling a closed fd. Hope this will remind
338 338 * user to do a POLLREMOVE.
339 339 */
340 340 if (!is_epoll && pfdp != NULL) {
341 341 pfdp[fdcnt].fd = fd;
342 342 pfdp[fdcnt].revents = POLLNVAL;
343 343 fdcnt++;
344 344 continue;
345 345 }
346 346
347 347 /*
348 348 * In the epoll compatibility case, we actually
349 349 * perform the implicit removal to remain
350 350 * closer to the epoll semantics.
351 351 */
352 352 if (is_epoll) {
353 353 pdp->pd_fp = NULL;
354 354 pdp->pd_events = 0;
355 355
356 356 if (pdp->pd_php != NULL) {
357 357 pollhead_delete(pdp->pd_php,
358 358 pdp);
|
↓ open down ↓ |
320 lines elided |
↑ open up ↑ |
359 359 pdp->pd_php = NULL;
360 360 }
361 361
362 362 BT_CLEAR(pcp->pc_bitmap, fd);
363 363 continue;
364 364 }
365 365 }
366 366
367 367 if (fp != pdp->pd_fp) {
368 368 /*
369 - * user is polling on a cached fd which was
370 - * closed and then reused. Unfortunately
371 - * there is no good way to inform user.
372 - * If the file struct is also reused, we
373 - * may not be able to detect the fd reuse
374 - * at all. As long as this does not
375 - * cause system failure and/or memory leak,
376 - * we will play along. Man page states if
377 - * user does not clean up closed fds, polling
369 + * The user is polling on a cached fd which was
370 + * closed and then reused. Unfortunately there
371 + * is no good way to communicate this fact to
372 + * the consumer.
373 + *
374 + * If the file struct is also reused, we may
375 + * not be able to detect the fd reuse at all.
376 + * As long as this does not cause system
377 + * failure and/or memory leaks, we will play
378 + * along. The man page states that if the user
379 + * does not clean up closed fds, polling
378 380 * results will be indeterministic.
379 381 *
380 - * XXX - perhaps log the detection of fd
381 - * reuse?
382 + * XXX: perhaps log the detection of fd reuse?
382 383 */
383 384 pdp->pd_fp = fp;
385 +
386 + /*
387 + * When this situation has been detected, it's
388 + * likely that any existing pollhead is
389 + * ill-suited to perform proper wake-ups.
390 + *
391 + * Clean up the old entry under the expectation
392 + * that a valid one will be provided as part of
393 + * the later VOP_POLL.
394 + */
395 + if (pdp->pd_php != NULL) {
396 + pollhead_delete(pdp->pd_php, pdp);
397 + pdp->pd_php = NULL;
398 + }
384 399 }
385 400 /*
386 401 * XXX - pollrelock() logic needs to know which
387 402 * which pollcache lock to grab. It'd be a
388 403 * cleaner solution if we could pass pcp as
389 404 * an arguement in VOP_POLL interface instead
390 405 * of implicitly passing it using thread_t
391 406 * struct. On the other hand, changing VOP_POLL
392 407 * interface will require all driver/file system
393 408 * poll routine to change. May want to revisit
394 409 * the tradeoff later.
395 410 */
396 411 curthread->t_pollcache = pcp;
397 412 error = VOP_POLL(fp->f_vnode, pdp->pd_events, 0,
398 413 &revent, &php, NULL);
414 +
415 + /*
416 + * Recheck edge-triggered descriptors which lack a
417 + * pollhead. While this check is performed when an fd
418 + * is added to the pollcache in dpwrite(), subsequent
419 + * descriptor manipulation could cause a different
420 + * resource to be present now.
421 + */
422 + if ((pdp->pd_events & POLLET) && error == 0 &&
423 + pdp->pd_php == NULL && php == NULL && revent != 0) {
424 + short levent = 0;
425 +
426 + /*
427 + * The same POLLET-only VOP_POLL is used in an
428 + * attempt to coax a pollhead from older
429 + * driver logic.
430 + */
431 + error = VOP_POLL(fp->f_vnode, POLLET,
432 + 0, &levent, &php, NULL);
433 + }
434 +
399 435 curthread->t_pollcache = NULL;
400 436 releasef(fd);
401 437 if (error != 0) {
402 438 break;
403 439 }
404 440
405 441 /*
406 442 * layered devices (e.g. console driver)
407 443 * may change the vnode and thus the pollhead
408 444 * pointer out from underneath us.
409 445 */
410 446 if (php != NULL && pdp->pd_php != NULL &&
411 447 php != pdp->pd_php) {
412 448 pollhead_delete(pdp->pd_php, pdp);
413 449 pdp->pd_php = php;
414 450 pollhead_insert(php, pdp);
415 451 /*
416 452 * The bit should still be set.
417 453 */
418 454 ASSERT(BT_TEST(pcp->pc_bitmap, fd));
419 455 goto retry;
420 456 }
421 457
422 458 if (revent != 0) {
423 459 if (pfdp != NULL) {
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
424 460 pfdp[fdcnt].fd = fd;
425 461 pfdp[fdcnt].events = pdp->pd_events;
426 462 pfdp[fdcnt].revents = revent;
427 463 } else if (epoll != NULL) {
428 464 epoll_event_t *ep = &epoll[fdcnt];
429 465
430 466 ASSERT(epoll != NULL);
431 467 ep->data.u64 = pdp->pd_epolldata;
432 468
433 469 /*
470 + * Since POLLNVAL is a legal event for
471 + * VOP_POLL handlers to emit, it must
472 + * be translated epoll-legal.
473 + */
474 + if (revent & POLLNVAL) {
475 + revent &= ~POLLNVAL;
476 + revent |= POLLERR;
477 + }
478 +
479 + /*
434 480 * If any of the event bits are set for
435 481 * which poll and epoll representations
436 482 * differ, swizzle in the native epoll
437 483 * values.
438 484 */
439 485 if (revent & mask) {
440 486 ep->events = (revent & ~mask) |
441 487 ((revent & POLLRDHUP) ?
442 488 EPOLLRDHUP : 0) |
443 489 ((revent & POLLWRBAND) ?
444 490 EPOLLWRBAND : 0);
445 491 } else {
446 492 ep->events = revent;
447 493 }
448 494
449 495 /*
450 496 * We define POLLWRNORM to be POLLOUT,
451 497 * but epoll has separate definitions
452 498 * for them; if POLLOUT is set and the
453 499 * user has asked for EPOLLWRNORM, set
454 500 * that as well.
455 501 */
456 502 if ((revent & POLLOUT) &&
457 503 (pdp->pd_events & EPOLLWRNORM)) {
458 504 ep->events |= EPOLLWRNORM;
459 505 }
460 506 } else {
461 507 pollstate_t *ps =
462 508 curthread->t_pollstate;
463 509 /*
464 510 * The devpoll handle itself is being
465 511 * polled. Notify the caller of any
466 512 * readable event(s), leaving as much
467 513 * state as possible untouched.
468 514 */
469 515 VERIFY(fdcnt == 0);
470 516 VERIFY(ps != NULL);
471 517
472 518 /*
473 519 * If a call to pollunlock() fails
474 520 * during VOP_POLL, skip over the fd
475 521 * and continue polling.
476 522 *
477 523 * Otherwise, report that there is an
478 524 * event pending.
479 525 */
480 526 if ((ps->ps_flags & POLLSTATE_ULFAIL)
|
↓ open down ↓ |
37 lines elided |
↑ open up ↑ |
481 527 != 0) {
482 528 ps->ps_flags &=
483 529 ~POLLSTATE_ULFAIL;
484 530 continue;
485 531 } else {
486 532 fdcnt++;
487 533 break;
488 534 }
489 535 }
490 536
491 - /*
492 - * If POLLET is set, clear the bit in the
493 - * bitmap -- which effectively latches the
494 - * edge on a pollwakeup() from the driver.
495 - */
496 - if (pdp->pd_events & POLLET)
497 - BT_CLEAR(pcp->pc_bitmap, fd);
498 -
499 - /*
500 - * If POLLONESHOT is set, perform the implicit
501 - * POLLREMOVE.
502 - */
537 + /* Handle special polling modes. */
503 538 if (pdp->pd_events & POLLONESHOT) {
539 + /*
540 + * If POLLONESHOT is set, perform the
541 + * implicit POLLREMOVE.
542 + */
504 543 pdp->pd_fp = NULL;
505 544 pdp->pd_events = 0;
506 545
507 546 if (pdp->pd_php != NULL) {
508 547 pollhead_delete(pdp->pd_php,
509 548 pdp);
510 549 pdp->pd_php = NULL;
511 550 }
512 551
513 552 BT_CLEAR(pcp->pc_bitmap, fd);
553 + } else if (pdp->pd_events & POLLET) {
554 + /*
555 + * Wire up the pollhead which should
556 + * have been provided. Edge-triggered
557 + * polling cannot function properly
558 + * with drivers which do not emit one.
559 + */
560 + if (php != NULL &&
561 + pdp->pd_php == NULL) {
562 + pollhead_insert(php, pdp);
563 + pdp->pd_php = php;
564 + }
565 +
566 + /*
567 + * If the driver has emitted a pollhead,
568 + * clear the bit in the bitmap which
569 + * effectively latches the edge on a
570 + * pollwakeup() from the driver.
571 + */
572 + if (pdp->pd_php != NULL) {
573 + BT_CLEAR(pcp->pc_bitmap, fd);
574 + }
514 575 }
515 576
516 577 fdcnt++;
517 578 } else if (php != NULL) {
518 579 /*
519 580 * We clear a bit or cache a poll fd if
520 581 * the driver returns a poll head ptr,
521 582 * which is expected in the case of 0
522 583 * revents. Some buggy driver may return
523 584 * NULL php pointer with 0 revents. In
524 585 * this case, we just treat the driver as
525 586 * "noncachable" and not clearing the bit
526 587 * in bitmap.
527 588 */
528 589 if ((pdp->pd_php != NULL) &&
529 590 ((pcp->pc_flag & PC_POLLWAKE) == 0)) {
530 591 BT_CLEAR(pcp->pc_bitmap, fd);
531 592 }
532 593 if (pdp->pd_php == NULL) {
533 594 pollhead_insert(php, pdp);
534 595 pdp->pd_php = php;
535 596 /*
536 597 * An event of interest may have
537 598 * arrived between the VOP_POLL() and
538 599 * the pollhead_insert(); check again.
539 600 */
540 601 goto repoll;
541 602 }
542 603 }
543 604 } else {
544 605 /*
545 606 * No bit set in the range. Check for wrap around.
546 607 */
547 608 if (!no_wrap) {
548 609 start = 0;
549 610 end = ostart - 1;
550 611 no_wrap = B_TRUE;
551 612 } else {
552 613 done = B_TRUE;
553 614 }
554 615 }
555 616 }
556 617
557 618 if (!done) {
558 619 pcp->pc_mapstart = start;
559 620 }
560 621 ASSERT(*fdcntp == 0);
561 622 *fdcntp = fdcnt;
562 623 return (error);
563 624 }
564 625
565 626 /*ARGSUSED*/
566 627 static int
567 628 dpopen(dev_t *devp, int flag, int otyp, cred_t *credp)
568 629 {
569 630 minor_t minordev;
570 631 dp_entry_t *dpep;
571 632 pollcache_t *pcp;
572 633
573 634 ASSERT(devpoll_init);
574 635 ASSERT(dptblsize <= MAXMIN);
575 636 mutex_enter(&devpoll_lock);
576 637 for (minordev = 0; minordev < dptblsize; minordev++) {
577 638 if (devpolltbl[minordev] == NULL) {
578 639 devpolltbl[minordev] = (dp_entry_t *)RESERVED;
579 640 break;
580 641 }
581 642 }
582 643 if (minordev == dptblsize) {
583 644 dp_entry_t **newtbl;
584 645 size_t oldsize;
585 646
586 647 /*
587 648 * Used up every entry in the existing devpoll table.
588 649 * Grow the table by DEVPOLLSIZE.
589 650 */
590 651 if ((oldsize = dptblsize) >= MAXMIN) {
591 652 mutex_exit(&devpoll_lock);
592 653 return (ENXIO);
593 654 }
594 655 dptblsize += DEVPOLLSIZE;
595 656 if (dptblsize > MAXMIN) {
596 657 dptblsize = MAXMIN;
597 658 }
598 659 newtbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
599 660 bcopy(devpolltbl, newtbl, sizeof (caddr_t) * oldsize);
600 661 kmem_free(devpolltbl, sizeof (caddr_t) * oldsize);
601 662 devpolltbl = newtbl;
602 663 devpolltbl[minordev] = (dp_entry_t *)RESERVED;
603 664 }
604 665 mutex_exit(&devpoll_lock);
605 666
606 667 dpep = kmem_zalloc(sizeof (dp_entry_t), KM_SLEEP);
607 668 /*
608 669 * allocate a pollcache skeleton here. Delay allocating bitmap
609 670 * structures until dpwrite() time, since we don't know the
610 671 * optimal size yet. We also delay setting the pid until either
611 672 * dpwrite() or attempt to poll on the instance, allowing parents
612 673 * to create instances of /dev/poll for their children. (In the
613 674 * epoll compatibility case, this check isn't performed to maintain
614 675 * semantic compatibility.)
615 676 */
616 677 pcp = pcache_alloc();
617 678 dpep->dpe_pcache = pcp;
618 679 pcp->pc_pid = -1;
619 680 *devp = makedevice(getmajor(*devp), minordev); /* clone the driver */
620 681 mutex_enter(&devpoll_lock);
621 682 ASSERT(minordev < dptblsize);
622 683 ASSERT(devpolltbl[minordev] == (dp_entry_t *)RESERVED);
623 684 devpolltbl[minordev] = dpep;
624 685 mutex_exit(&devpoll_lock);
625 686 return (0);
626 687 }
627 688
628 689 /*
629 690 * Write to dev/poll add/remove fd's to/from a cached poll fd set,
630 691 * or change poll events for a watched fd.
631 692 */
632 693 /*ARGSUSED*/
633 694 static int
634 695 dpwrite(dev_t dev, struct uio *uiop, cred_t *credp)
635 696 {
636 697 minor_t minor;
637 698 dp_entry_t *dpep;
638 699 pollcache_t *pcp;
639 700 pollfd_t *pollfdp, *pfdp;
640 701 dvpoll_epollfd_t *epfdp;
641 702 uintptr_t limit;
642 703 int error, size;
643 704 ssize_t uiosize;
644 705 size_t copysize;
645 706 nfds_t pollfdnum;
646 707 struct pollhead *php = NULL;
647 708 polldat_t *pdp;
648 709 int fd;
649 710 file_t *fp;
650 711 boolean_t is_epoll, fds_added = B_FALSE;
651 712
652 713 minor = getminor(dev);
653 714
654 715 mutex_enter(&devpoll_lock);
655 716 ASSERT(minor < dptblsize);
656 717 dpep = devpolltbl[minor];
657 718 ASSERT(dpep != NULL);
658 719 mutex_exit(&devpoll_lock);
659 720
660 721 mutex_enter(&dpep->dpe_lock);
661 722 pcp = dpep->dpe_pcache;
662 723 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
663 724 size = (is_epoll) ? sizeof (dvpoll_epollfd_t) : sizeof (pollfd_t);
664 725 mutex_exit(&dpep->dpe_lock);
665 726
666 727 if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
667 728 if (pcp->pc_pid != -1) {
668 729 return (EACCES);
669 730 }
670 731
671 732 pcp->pc_pid = curproc->p_pid;
672 733 }
673 734
674 735 uiosize = uiop->uio_resid;
675 736 pollfdnum = uiosize / size;
676 737
677 738 /*
678 739 * We want to make sure that pollfdnum isn't large enough to DoS us,
679 740 * but we also don't want to grab p_lock unnecessarily -- so we
680 741 * perform the full check against our resource limits if and only if
681 742 * pollfdnum is larger than the known-to-be-sane value of UINT8_MAX.
682 743 */
683 744 if (pollfdnum > UINT8_MAX) {
684 745 mutex_enter(&curproc->p_lock);
685 746 if (pollfdnum >
686 747 (uint_t)rctl_enforced_value(rctlproc_legacy[RLIMIT_NOFILE],
687 748 curproc->p_rctls, curproc)) {
688 749 (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
689 750 curproc->p_rctls, curproc, RCA_SAFE);
690 751 mutex_exit(&curproc->p_lock);
691 752 return (EINVAL);
692 753 }
693 754 mutex_exit(&curproc->p_lock);
694 755 }
695 756
696 757 /*
697 758 * Copy in the pollfd array. Walk through the array and add
698 759 * each polled fd to the cached set.
699 760 */
700 761 pollfdp = kmem_alloc(uiosize, KM_SLEEP);
701 762 limit = (uintptr_t)pollfdp + (pollfdnum * size);
702 763
703 764 /*
704 765 * Although /dev/poll uses the write(2) interface to cache fds, it's
705 766 * not supposed to function as a seekable device. To prevent offset
706 767 * from growing and eventually exceed the maximum, reset the offset
707 768 * here for every call.
708 769 */
709 770 uiop->uio_loffset = 0;
710 771
711 772 /*
712 773 * Use uiocopy instead of uiomove when populating pollfdp, keeping
713 774 * uio_resid untouched for now. Write syscalls will translate EINTR
714 775 * into a success if they detect "successfully transfered" data via an
715 776 * updated uio_resid. Falsely suppressing such errors is disastrous.
716 777 */
717 778 if ((error = uiocopy((caddr_t)pollfdp, uiosize, UIO_WRITE, uiop,
718 779 ©size)) != 0) {
719 780 kmem_free(pollfdp, uiosize);
720 781 return (error);
721 782 }
722 783
723 784 /*
724 785 * We are about to enter the core portion of dpwrite(). Make sure this
725 786 * write has exclusive access in this portion of the code, i.e., no
726 787 * other writers in this code.
727 788 *
728 789 * Waiting for all readers to drop their references to the dpe is
729 790 * unecessary since the pollcache itself is protected by pc_lock.
730 791 */
731 792 mutex_enter(&dpep->dpe_lock);
732 793 dpep->dpe_writerwait++;
733 794 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0) {
734 795 ASSERT(dpep->dpe_refcnt != 0);
735 796
736 797 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
737 798 dpep->dpe_writerwait--;
738 799 mutex_exit(&dpep->dpe_lock);
739 800 kmem_free(pollfdp, uiosize);
740 801 return (EINTR);
741 802 }
742 803 }
743 804 dpep->dpe_writerwait--;
744 805 dpep->dpe_flag |= DP_WRITER_PRESENT;
745 806 dpep->dpe_refcnt++;
746 807
747 808 if (!is_epoll && (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0) {
748 809 /*
749 810 * The epoll compat mode was enabled while we were waiting to
750 811 * establish write access. It is not safe to continue since
751 812 * state was prepared for non-epoll operation.
752 813 */
753 814 error = EBUSY;
754 815 goto bypass;
755 816 }
756 817 mutex_exit(&dpep->dpe_lock);
757 818
758 819 /*
759 820 * Since the dpwrite() may recursively walk an added /dev/poll handle,
760 821 * pollstate_enter() deadlock and loop detection must be used.
761 822 */
762 823 (void) pollstate_create();
763 824 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
764 825
765 826 if (pcp->pc_bitmap == NULL) {
766 827 pcache_create(pcp, pollfdnum);
767 828 }
768 829 for (pfdp = pollfdp; (uintptr_t)pfdp < limit;
769 830 pfdp = (pollfd_t *)((uintptr_t)pfdp + size)) {
770 831 fd = pfdp->fd;
771 832 if ((uint_t)fd >= P_FINFO(curproc)->fi_nfiles) {
772 833 /*
773 834 * epoll semantics demand that we return EBADF if our
774 835 * specified fd is invalid.
775 836 */
776 837 if (is_epoll) {
777 838 error = EBADF;
778 839 break;
779 840 }
780 841
781 842 continue;
782 843 }
783 844
784 845 pdp = pcache_lookup_fd(pcp, fd);
785 846 if (pfdp->events != POLLREMOVE) {
786 847
787 848 fp = NULL;
788 849
789 850 if (pdp == NULL) {
790 851 /*
791 852 * If we're in epoll compatibility mode, check
792 853 * that the fd is valid before allocating
793 854 * anything for it; epoll semantics demand that
794 855 * we return EBADF if our specified fd is
795 856 * invalid.
796 857 */
797 858 if (is_epoll) {
798 859 if ((fp = getf(fd)) == NULL) {
799 860 error = EBADF;
800 861 break;
801 862 }
802 863 }
803 864
804 865 pdp = pcache_alloc_fd(0);
805 866 pdp->pd_fd = fd;
806 867 pdp->pd_pcache = pcp;
807 868 pcache_insert_fd(pcp, pdp, pollfdnum);
808 869 } else {
809 870 /*
810 871 * epoll semantics demand that we error out if
811 872 * a file descriptor is added twice, which we
812 873 * check (imperfectly) by checking if we both
813 874 * have the file descriptor cached and the
814 875 * file pointer that correponds to the file
815 876 * descriptor matches our cached value. If
816 877 * there is a pointer mismatch, the file
817 878 * descriptor was closed without being removed.
818 879 * The converse is clearly not true, however,
819 880 * so to narrow the window by which a spurious
820 881 * EEXIST may be returned, we also check if
821 882 * this fp has been added to an epoll control
822 883 * descriptor in the past; if it hasn't, we
823 884 * know that this is due to fp reuse -- it's
824 885 * not a true EEXIST case. (By performing this
825 886 * additional check, we limit the window of
826 887 * spurious EEXIST to situations where a single
827 888 * file descriptor is being used across two or
828 889 * more epoll control descriptors -- and even
829 890 * then, the file descriptor must be closed and
830 891 * reused in a relatively tight time span.)
831 892 */
832 893 if (is_epoll) {
833 894 if (pdp->pd_fp != NULL &&
834 895 (fp = getf(fd)) != NULL &&
835 896 fp == pdp->pd_fp &&
836 897 (fp->f_flag2 & FEPOLLED)) {
837 898 error = EEXIST;
838 899 releasef(fd);
839 900 break;
840 901 }
841 902
842 903 /*
843 904 * We have decided that the cached
844 905 * information was stale: it either
845 906 * didn't match, or the fp had never
846 907 * actually been epoll()'d on before.
847 908 * We need to now clear our pd_events
848 909 * to assure that we don't mistakenly
849 910 * operate on cached event disposition.
850 911 */
851 912 pdp->pd_events = 0;
852 913 }
853 914 }
854 915
855 916 if (is_epoll) {
856 917 epfdp = (dvpoll_epollfd_t *)pfdp;
857 918 pdp->pd_epolldata = epfdp->dpep_data;
858 919 }
859 920
860 921 ASSERT(pdp->pd_fd == fd);
861 922 ASSERT(pdp->pd_pcache == pcp);
862 923 if (fd >= pcp->pc_mapsize) {
863 924 mutex_exit(&pcp->pc_lock);
864 925 pcache_grow_map(pcp, fd);
865 926 mutex_enter(&pcp->pc_lock);
866 927 }
867 928 if (fd > pcp->pc_mapend) {
868 929 pcp->pc_mapend = fd;
869 930 }
870 931 if (fp == NULL && (fp = getf(fd)) == NULL) {
871 932 /*
872 933 * The fd is not valid. Since we can't pass
873 934 * this error back in the write() call, set
874 935 * the bit in bitmap to force DP_POLL ioctl
875 936 * to examine it.
876 937 */
877 938 BT_SET(pcp->pc_bitmap, fd);
878 939 pdp->pd_events |= pfdp->events;
879 940 continue;
880 941 }
881 942
882 943 /*
883 944 * To (greatly) reduce EEXIST false positives, we
884 945 * denote that this fp has been epoll()'d. We do this
885 946 * regardless of epoll compatibility mode, as the flag
886 947 * is harmless if not in epoll compatibility mode.
887 948 */
888 949 fp->f_flag2 |= FEPOLLED;
889 950
890 951 /*
891 952 * Don't do VOP_POLL for an already cached fd with
892 953 * same poll events.
893 954 */
894 955 if ((pdp->pd_events == pfdp->events) &&
895 956 (pdp->pd_fp == fp)) {
896 957 /*
897 958 * the events are already cached
898 959 */
899 960 releasef(fd);
900 961 continue;
901 962 }
902 963
903 964 /*
904 965 * do VOP_POLL and cache this poll fd.
905 966 */
906 967 /*
907 968 * XXX - pollrelock() logic needs to know which
908 969 * which pollcache lock to grab. It'd be a
909 970 * cleaner solution if we could pass pcp as
|
↓ open down ↓ |
386 lines elided |
↑ open up ↑ |
910 971 * an arguement in VOP_POLL interface instead
911 972 * of implicitly passing it using thread_t
912 973 * struct. On the other hand, changing VOP_POLL
913 974 * interface will require all driver/file system
914 975 * poll routine to change. May want to revisit
915 976 * the tradeoff later.
916 977 */
917 978 curthread->t_pollcache = pcp;
918 979 error = VOP_POLL(fp->f_vnode, pfdp->events, 0,
919 980 &pfdp->revents, &php, NULL);
981 +
982 + /*
983 + * Edge-triggered polling requires a pollhead in order
984 + * to initiate wake-ups properly. Drivers which are
985 + * savvy to POLLET presence, which should include
986 + * everything in-gate, will always emit one, regardless
987 + * of revent status. Older drivers which only emit a
988 + * pollhead if 'revents == 0' are given a second chance
989 + * here via a second VOP_POLL, with only POLLET set in
990 + * the events of interest. These circumstances should
991 + * induce any cacheable drivers to emit a pollhead for
992 + * wake-ups.
993 + *
994 + * Drivers which never emit a pollhead will simply
995 + * disobey the exectation of edge-triggered behavior.
996 + * This includes recursive epoll which, even on Linux,
997 + * yields its events in a level-triggered fashion only.
998 + */
999 + if ((pdp->pd_events & POLLET) && error == 0 &&
1000 + php == NULL) {
1001 + short levent = 0;
1002 +
1003 + error = VOP_POLL(fp->f_vnode, POLLET, 0,
1004 + &levent, &php, NULL);
1005 + }
1006 +
920 1007 curthread->t_pollcache = NULL;
921 1008 /*
922 1009 * We always set the bit when this fd is cached;
923 1010 * this forces the first DP_POLL to poll this fd.
924 1011 * Real performance gain comes from subsequent
925 1012 * DP_POLL. We also attempt a pollhead_insert();
926 1013 * if it's not possible, we'll do it in dpioctl().
927 1014 */
928 1015 BT_SET(pcp->pc_bitmap, fd);
929 1016 if (error != 0) {
930 1017 releasef(fd);
931 1018 break;
932 1019 }
933 1020 pdp->pd_fp = fp;
934 1021 pdp->pd_events |= pfdp->events;
935 1022 if (php != NULL) {
936 1023 if (pdp->pd_php == NULL) {
937 1024 pollhead_insert(php, pdp);
938 1025 pdp->pd_php = php;
939 1026 } else {
940 1027 if (pdp->pd_php != php) {
941 1028 pollhead_delete(pdp->pd_php,
942 1029 pdp);
943 1030 pollhead_insert(php, pdp);
944 1031 pdp->pd_php = php;
945 1032 }
946 1033 }
947 1034 }
948 1035 fds_added = B_TRUE;
949 1036 releasef(fd);
950 1037 } else {
951 1038 if (pdp == NULL || pdp->pd_fp == NULL) {
952 1039 if (is_epoll) {
953 1040 /*
954 1041 * As with the add case (above), epoll
955 1042 * semantics demand that we error out
956 1043 * in this case.
957 1044 */
958 1045 error = ENOENT;
959 1046 break;
960 1047 }
961 1048
962 1049 continue;
963 1050 }
964 1051 ASSERT(pdp->pd_fd == fd);
965 1052 pdp->pd_fp = NULL;
966 1053 pdp->pd_events = 0;
967 1054 ASSERT(pdp->pd_thread == NULL);
968 1055 if (pdp->pd_php != NULL) {
969 1056 pollhead_delete(pdp->pd_php, pdp);
970 1057 pdp->pd_php = NULL;
971 1058 }
972 1059 BT_CLEAR(pcp->pc_bitmap, fd);
973 1060 }
974 1061 }
975 1062 /*
976 1063 * Wake any pollcache waiters so they can check the new descriptors.
977 1064 *
978 1065 * Any fds added to an recursive-capable pollcache could themselves be
979 1066 * /dev/poll handles. To ensure that proper event propagation occurs,
980 1067 * parent pollcaches are woken too, so that they can create any needed
981 1068 * pollcache links.
982 1069 */
983 1070 if (fds_added) {
984 1071 cv_broadcast(&pcp->pc_cv);
985 1072 pcache_wake_parents(pcp);
986 1073 }
987 1074 pollstate_exit(pcp);
988 1075 mutex_enter(&dpep->dpe_lock);
989 1076 bypass:
990 1077 dpep->dpe_flag &= ~DP_WRITER_PRESENT;
991 1078 dpep->dpe_refcnt--;
992 1079 cv_broadcast(&dpep->dpe_cv);
993 1080 mutex_exit(&dpep->dpe_lock);
994 1081 kmem_free(pollfdp, uiosize);
995 1082 if (error == 0) {
996 1083 /*
997 1084 * The state of uio_resid is updated only after the pollcache
998 1085 * is successfully modified.
999 1086 */
1000 1087 uioskip(uiop, copysize);
1001 1088 }
1002 1089 return (error);
1003 1090 }
1004 1091
1005 1092 #define DP_SIGMASK_RESTORE(ksetp) { \
1006 1093 if (ksetp != NULL) { \
1007 1094 mutex_enter(&p->p_lock); \
1008 1095 if (lwp->lwp_cursig == 0) { \
1009 1096 t->t_hold = lwp->lwp_sigoldmask; \
1010 1097 t->t_flag &= ~T_TOMASK; \
1011 1098 } \
1012 1099 mutex_exit(&p->p_lock); \
1013 1100 } \
1014 1101 }
1015 1102
1016 1103 /*ARGSUSED*/
1017 1104 static int
1018 1105 dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
1019 1106 {
1020 1107 minor_t minor;
1021 1108 dp_entry_t *dpep;
1022 1109 pollcache_t *pcp;
1023 1110 hrtime_t now;
1024 1111 int error = 0;
1025 1112 boolean_t is_epoll;
1026 1113 STRUCT_DECL(dvpoll, dvpoll);
1027 1114
1028 1115 if (cmd == DP_POLL || cmd == DP_PPOLL) {
1029 1116 /* do this now, before we sleep on DP_WRITER_PRESENT */
1030 1117 now = gethrtime();
1031 1118 }
1032 1119
1033 1120 minor = getminor(dev);
1034 1121 mutex_enter(&devpoll_lock);
1035 1122 ASSERT(minor < dptblsize);
1036 1123 dpep = devpolltbl[minor];
1037 1124 mutex_exit(&devpoll_lock);
1038 1125 ASSERT(dpep != NULL);
1039 1126 pcp = dpep->dpe_pcache;
1040 1127
1041 1128 mutex_enter(&dpep->dpe_lock);
1042 1129 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
1043 1130
1044 1131 if (cmd == DP_EPOLLCOMPAT) {
1045 1132 if (dpep->dpe_refcnt != 0) {
1046 1133 /*
1047 1134 * We can't turn on epoll compatibility while there
1048 1135 * are outstanding operations.
1049 1136 */
1050 1137 mutex_exit(&dpep->dpe_lock);
1051 1138 return (EBUSY);
1052 1139 }
1053 1140
1054 1141 /*
1055 1142 * epoll compatibility is a one-way street: there's no way
1056 1143 * to turn it off for a particular open.
1057 1144 */
1058 1145 dpep->dpe_flag |= DP_ISEPOLLCOMPAT;
1059 1146 mutex_exit(&dpep->dpe_lock);
1060 1147
1061 1148 return (0);
1062 1149 }
1063 1150
1064 1151 if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
1065 1152 if (pcp->pc_pid != -1) {
1066 1153 mutex_exit(&dpep->dpe_lock);
1067 1154 return (EACCES);
1068 1155 }
1069 1156
1070 1157 pcp->pc_pid = curproc->p_pid;
1071 1158 }
1072 1159
1073 1160 /* Wait until all writers have cleared the handle before continuing */
1074 1161 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0 ||
1075 1162 (dpep->dpe_writerwait != 0)) {
1076 1163 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
1077 1164 mutex_exit(&dpep->dpe_lock);
1078 1165 return (EINTR);
1079 1166 }
1080 1167 }
1081 1168 dpep->dpe_refcnt++;
1082 1169 mutex_exit(&dpep->dpe_lock);
1083 1170
1084 1171 switch (cmd) {
1085 1172 case DP_POLL:
1086 1173 case DP_PPOLL:
1087 1174 {
1088 1175 pollstate_t *ps;
1089 1176 nfds_t nfds;
1090 1177 int fdcnt = 0;
1091 1178 size_t size, fdsize, dpsize;
1092 1179 hrtime_t deadline = 0;
1093 1180 k_sigset_t *ksetp = NULL;
1094 1181 k_sigset_t kset;
1095 1182 sigset_t set;
1096 1183 kthread_t *t = curthread;
1097 1184 klwp_t *lwp = ttolwp(t);
1098 1185 struct proc *p = ttoproc(curthread);
1099 1186
1100 1187 STRUCT_INIT(dvpoll, mode);
1101 1188
1102 1189 /*
1103 1190 * The dp_setp member is only required/consumed for DP_PPOLL,
1104 1191 * which otherwise uses the same structure as DP_POLL.
1105 1192 */
1106 1193 if (cmd == DP_POLL) {
1107 1194 dpsize = (uintptr_t)STRUCT_FADDR(dvpoll, dp_setp) -
1108 1195 (uintptr_t)STRUCT_FADDR(dvpoll, dp_fds);
1109 1196 } else {
1110 1197 ASSERT(cmd == DP_PPOLL);
1111 1198 dpsize = STRUCT_SIZE(dvpoll);
1112 1199 }
1113 1200
1114 1201 if ((mode & FKIOCTL) != 0) {
1115 1202 /* Kernel-internal ioctl call */
1116 1203 bcopy((caddr_t)arg, STRUCT_BUF(dvpoll), dpsize);
1117 1204 error = 0;
1118 1205 } else {
1119 1206 error = copyin((caddr_t)arg, STRUCT_BUF(dvpoll),
1120 1207 dpsize);
1121 1208 }
1122 1209
1123 1210 if (error) {
1124 1211 DP_REFRELE(dpep);
1125 1212 return (EFAULT);
1126 1213 }
1127 1214
1128 1215 deadline = STRUCT_FGET(dvpoll, dp_timeout);
1129 1216 if (deadline > 0) {
1130 1217 /*
1131 1218 * Convert the deadline from relative milliseconds
1132 1219 * to absolute nanoseconds. They must wait for at
1133 1220 * least a tick.
1134 1221 */
1135 1222 deadline = MSEC2NSEC(deadline);
1136 1223 deadline = MAX(deadline, nsec_per_tick);
1137 1224 deadline += now;
1138 1225 }
1139 1226
1140 1227 if (cmd == DP_PPOLL) {
1141 1228 void *setp = STRUCT_FGETP(dvpoll, dp_setp);
1142 1229
1143 1230 if (setp != NULL) {
1144 1231 if ((mode & FKIOCTL) != 0) {
1145 1232 /* Use the signal set directly */
1146 1233 ksetp = (k_sigset_t *)setp;
1147 1234 } else {
1148 1235 if (copyin(setp, &set, sizeof (set))) {
1149 1236 DP_REFRELE(dpep);
1150 1237 return (EFAULT);
1151 1238 }
1152 1239 sigutok(&set, &kset);
1153 1240 ksetp = &kset;
1154 1241 }
1155 1242
1156 1243 mutex_enter(&p->p_lock);
1157 1244 schedctl_finish_sigblock(t);
1158 1245 lwp->lwp_sigoldmask = t->t_hold;
1159 1246 t->t_hold = *ksetp;
1160 1247 t->t_flag |= T_TOMASK;
1161 1248
1162 1249 /*
1163 1250 * Like ppoll() with a non-NULL sigset, we'll
1164 1251 * call cv_reltimedwait_sig() just to check for
1165 1252 * signals. This call will return immediately
1166 1253 * with either 0 (signalled) or -1 (no signal).
1167 1254 * There are some conditions whereby we can
1168 1255 * get 0 from cv_reltimedwait_sig() without
1169 1256 * a true signal (e.g., a directed stop), so
1170 1257 * we restore our signal mask in the unlikely
1171 1258 * event that lwp_cursig is 0.
1172 1259 */
1173 1260 if (!cv_reltimedwait_sig(&t->t_delay_cv,
1174 1261 &p->p_lock, 0, TR_CLOCK_TICK)) {
1175 1262 if (lwp->lwp_cursig == 0) {
1176 1263 t->t_hold = lwp->lwp_sigoldmask;
1177 1264 t->t_flag &= ~T_TOMASK;
1178 1265 }
1179 1266
1180 1267 mutex_exit(&p->p_lock);
1181 1268
1182 1269 DP_REFRELE(dpep);
1183 1270 return (EINTR);
1184 1271 }
1185 1272
1186 1273 mutex_exit(&p->p_lock);
1187 1274 }
1188 1275 }
1189 1276
1190 1277 if ((nfds = STRUCT_FGET(dvpoll, dp_nfds)) == 0) {
1191 1278 /*
1192 1279 * We are just using DP_POLL to sleep, so
1193 1280 * we don't any of the devpoll apparatus.
1194 1281 * Do not check for signals if we have a zero timeout.
1195 1282 */
1196 1283 DP_REFRELE(dpep);
1197 1284 if (deadline == 0) {
1198 1285 DP_SIGMASK_RESTORE(ksetp);
1199 1286 return (0);
1200 1287 }
1201 1288
1202 1289 mutex_enter(&curthread->t_delay_lock);
1203 1290 while ((error =
1204 1291 cv_timedwait_sig_hrtime(&curthread->t_delay_cv,
1205 1292 &curthread->t_delay_lock, deadline)) > 0)
1206 1293 continue;
1207 1294 mutex_exit(&curthread->t_delay_lock);
1208 1295
1209 1296 DP_SIGMASK_RESTORE(ksetp);
1210 1297
1211 1298 return (error == 0 ? EINTR : 0);
1212 1299 }
1213 1300
1214 1301 if (is_epoll) {
1215 1302 size = nfds * (fdsize = sizeof (epoll_event_t));
1216 1303 } else {
1217 1304 size = nfds * (fdsize = sizeof (pollfd_t));
1218 1305 }
1219 1306
1220 1307 /*
1221 1308 * XXX It would be nice not to have to alloc each time, but it
1222 1309 * requires another per thread structure hook. This can be
1223 1310 * implemented later if data suggests that it's necessary.
1224 1311 */
1225 1312 ps = pollstate_create();
1226 1313
1227 1314 if (ps->ps_dpbufsize < size) {
1228 1315 /*
1229 1316 * If nfds is larger than twice the current maximum
1230 1317 * open file count, we'll silently clamp it. This
1231 1318 * only limits our exposure to allocating an
1232 1319 * inordinate amount of kernel memory; it doesn't
1233 1320 * otherwise affect the semantics. (We have this
1234 1321 * check at twice the maximum instead of merely the
1235 1322 * maximum because some applications pass an nfds that
1236 1323 * is only slightly larger than their limit.)
1237 1324 */
1238 1325 mutex_enter(&p->p_lock);
1239 1326 if ((nfds >> 1) > p->p_fno_ctl) {
1240 1327 nfds = p->p_fno_ctl;
1241 1328 size = nfds * fdsize;
1242 1329 }
1243 1330 mutex_exit(&p->p_lock);
1244 1331
1245 1332 if (ps->ps_dpbufsize < size) {
1246 1333 kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize);
1247 1334 ps->ps_dpbuf = kmem_zalloc(size, KM_SLEEP);
1248 1335 ps->ps_dpbufsize = size;
1249 1336 }
1250 1337 }
1251 1338
1252 1339 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
1253 1340 for (;;) {
1254 1341 pcp->pc_flag &= ~PC_POLLWAKE;
1255 1342
1256 1343 /*
1257 1344 * Mark all child pcachelinks as stale.
1258 1345 * Those which are still part of the tree will be
1259 1346 * marked as valid during the poll.
1260 1347 */
1261 1348 pcachelink_mark_stale(pcp);
1262 1349
1263 1350 error = dp_pcache_poll(dpep, ps->ps_dpbuf,
1264 1351 pcp, nfds, &fdcnt);
1265 1352 if (fdcnt > 0 || error != 0)
1266 1353 break;
1267 1354
1268 1355 /* Purge still-stale child pcachelinks */
1269 1356 pcachelink_purge_stale(pcp);
1270 1357
1271 1358 /*
1272 1359 * A pollwake has happened since we polled cache.
1273 1360 */
1274 1361 if (pcp->pc_flag & PC_POLLWAKE)
1275 1362 continue;
1276 1363
1277 1364 /*
1278 1365 * Sleep until we are notified, signaled, or timed out.
1279 1366 */
1280 1367 if (deadline == 0) {
1281 1368 /* immediate timeout; do not check signals */
1282 1369 break;
1283 1370 }
1284 1371
1285 1372 error = cv_timedwait_sig_hrtime(&pcp->pc_cv,
1286 1373 &pcp->pc_lock, deadline);
1287 1374
1288 1375 /*
1289 1376 * If we were awakened by a signal or timeout then
1290 1377 * break the loop, else poll again.
1291 1378 */
1292 1379 if (error <= 0) {
1293 1380 error = (error == 0) ? EINTR : 0;
1294 1381 break;
1295 1382 } else {
1296 1383 error = 0;
1297 1384 }
1298 1385 }
1299 1386 pollstate_exit(pcp);
1300 1387
1301 1388 DP_SIGMASK_RESTORE(ksetp);
1302 1389
1303 1390 if (error == 0 && fdcnt > 0) {
1304 1391 /*
1305 1392 * It should be noted that FKIOCTL does not influence
1306 1393 * the copyout (vs bcopy) of dp_fds at this time.
1307 1394 */
1308 1395 if (copyout(ps->ps_dpbuf,
1309 1396 STRUCT_FGETP(dvpoll, dp_fds), fdcnt * fdsize)) {
1310 1397 DP_REFRELE(dpep);
1311 1398 return (EFAULT);
1312 1399 }
1313 1400 *rvalp = fdcnt;
1314 1401 }
1315 1402 break;
1316 1403 }
1317 1404
1318 1405 case DP_ISPOLLED:
1319 1406 {
1320 1407 pollfd_t pollfd;
1321 1408 polldat_t *pdp;
1322 1409
1323 1410 STRUCT_INIT(dvpoll, mode);
1324 1411 error = copyin((caddr_t)arg, &pollfd, sizeof (pollfd_t));
1325 1412 if (error) {
1326 1413 DP_REFRELE(dpep);
1327 1414 return (EFAULT);
1328 1415 }
1329 1416 mutex_enter(&pcp->pc_lock);
1330 1417 if (pcp->pc_hash == NULL) {
1331 1418 /*
1332 1419 * No Need to search because no poll fd
1333 1420 * has been cached.
1334 1421 */
1335 1422 mutex_exit(&pcp->pc_lock);
1336 1423 DP_REFRELE(dpep);
1337 1424 return (0);
1338 1425 }
1339 1426 if (pollfd.fd < 0) {
1340 1427 mutex_exit(&pcp->pc_lock);
1341 1428 break;
1342 1429 }
1343 1430 pdp = pcache_lookup_fd(pcp, pollfd.fd);
1344 1431 if ((pdp != NULL) && (pdp->pd_fd == pollfd.fd) &&
1345 1432 (pdp->pd_fp != NULL)) {
1346 1433 pollfd.revents = pdp->pd_events;
1347 1434 if (copyout(&pollfd, (caddr_t)arg, sizeof (pollfd_t))) {
1348 1435 mutex_exit(&pcp->pc_lock);
1349 1436 DP_REFRELE(dpep);
1350 1437 return (EFAULT);
1351 1438 }
1352 1439 *rvalp = 1;
1353 1440 }
1354 1441 mutex_exit(&pcp->pc_lock);
1355 1442 break;
1356 1443 }
1357 1444
1358 1445 default:
1359 1446 DP_REFRELE(dpep);
1360 1447 return (EINVAL);
1361 1448 }
1362 1449 DP_REFRELE(dpep);
1363 1450 return (error);
1364 1451 }
1365 1452
1366 1453 /*
1367 1454 * Overview of Recursive Polling
1368 1455 *
1369 1456 * It is possible for /dev/poll to poll for events on file descriptors which
1370 1457 * themselves are /dev/poll handles. Pending events in the child handle are
1371 1458 * represented as readable data via the POLLIN flag. To limit surface area,
1372 1459 * this recursion is presently allowed on only /dev/poll handles which have
1373 1460 * been placed in epoll mode via the DP_EPOLLCOMPAT ioctl. Recursion depth is
1374 1461 * limited to 5 in order to be consistent with Linux epoll.
1375 1462 *
1376 1463 * Extending dppoll() for VOP_POLL:
1377 1464 *
1378 1465 * The recursive /dev/poll implementation begins by extending dppoll() to
1379 1466 * report when resources contained in the pollcache have relevant event state.
1380 1467 * At the highest level, it means calling dp_pcache_poll() so it indicates if
1381 1468 * fd events are present without consuming them or altering the pollcache
1382 1469 * bitmap. This ensures that a subsequent DP_POLL operation on the bitmap will
1383 1470 * yield the initiating event. Additionally, the VOP_POLL should return in
1384 1471 * such a way that dp_pcache_poll() does not clear the parent bitmap entry
1385 1472 * which corresponds to the child /dev/poll fd. This means that child
1386 1473 * pollcaches will be checked during every poll which facilitates wake-up
1387 1474 * behavior detailed below.
1388 1475 *
1389 1476 * Pollcache Links and Wake Events:
1390 1477 *
1391 1478 * Recursive /dev/poll avoids complicated pollcache locking constraints during
1392 1479 * pollwakeup events by eschewing the traditional pollhead mechanism in favor
1393 1480 * of a different approach. For each pollcache at the root of a recursive
1394 1481 * /dev/poll "tree", pcachelink_t structures are established to all child
1395 1482 * /dev/poll pollcaches. During pollnotify() in a child pollcache, the
1396 1483 * linked list of pcachelink_t entries is walked, where those marked as valid
1397 1484 * incur a cv_broadcast to their parent pollcache. Most notably, these
1398 1485 * pcachelink_t cv wakeups are performed without acquiring pc_lock on the
1399 1486 * parent pollcache (which would require careful deadlock avoidance). This
1400 1487 * still allows the woken poll on the parent to discover the pertinent events
1401 1488 * due to the fact that bitmap entires for the child pollcache are always
1402 1489 * maintained by the dppoll() logic above.
1403 1490 *
1404 1491 * Depth Limiting and Loop Prevention:
1405 1492 *
1406 1493 * As each pollcache is encountered (either via DP_POLL or dppoll()), depth and
1407 1494 * loop constraints are enforced via pollstate_enter(). The pollcache_t
1408 1495 * pointer is compared against any existing entries in ps_pc_stack and is added
1409 1496 * to the end if no match (and therefore loop) is found. Once poll operations
1410 1497 * for a given pollcache_t are complete, pollstate_exit() clears the pointer
1411 1498 * from the list. The pollstate_enter() and pollstate_exit() functions are
1412 1499 * responsible for acquiring and releasing pc_lock, respectively.
1413 1500 *
1414 1501 * Deadlock Safety:
1415 1502 *
1416 1503 * Descending through a tree of recursive /dev/poll handles involves the tricky
1417 1504 * business of sequentially entering multiple pollcache locks. This tree
1418 1505 * topology cannot define a lock acquisition order in such a way that it is
1419 1506 * immune to deadlocks between threads. The pollstate_enter() and
1420 1507 * pollstate_exit() functions provide an interface for recursive /dev/poll
1421 1508 * operations to safely lock pollcaches while failing gracefully in the face of
1422 1509 * deadlocking topologies. (See pollstate_contend() for more detail about how
1423 1510 * deadlocks are detected and resolved.)
1424 1511 */
1425 1512
1426 1513 /*ARGSUSED*/
1427 1514 static int
1428 1515 dppoll(dev_t dev, short events, int anyyet, short *reventsp,
1429 1516 struct pollhead **phpp)
1430 1517 {
1431 1518 minor_t minor;
1432 1519 dp_entry_t *dpep;
1433 1520 pollcache_t *pcp;
1434 1521 int res, rc = 0;
1435 1522
1436 1523 minor = getminor(dev);
1437 1524 mutex_enter(&devpoll_lock);
1438 1525 ASSERT(minor < dptblsize);
1439 1526 dpep = devpolltbl[minor];
1440 1527 ASSERT(dpep != NULL);
1441 1528 mutex_exit(&devpoll_lock);
1442 1529
1443 1530 mutex_enter(&dpep->dpe_lock);
1444 1531 if ((dpep->dpe_flag & DP_ISEPOLLCOMPAT) == 0) {
1445 1532 /* Poll recursion is not yet supported for non-epoll handles */
1446 1533 *reventsp = POLLERR;
1447 1534 mutex_exit(&dpep->dpe_lock);
1448 1535 return (0);
1449 1536 } else {
1450 1537 dpep->dpe_refcnt++;
|
↓ open down ↓ |
521 lines elided |
↑ open up ↑ |
1451 1538 pcp = dpep->dpe_pcache;
1452 1539 mutex_exit(&dpep->dpe_lock);
1453 1540 }
1454 1541
1455 1542 res = pollstate_enter(pcp);
1456 1543 if (res == PSE_SUCCESS) {
1457 1544 nfds_t nfds = 1;
1458 1545 int fdcnt = 0;
1459 1546 pollstate_t *ps = curthread->t_pollstate;
1460 1547
1461 - rc = dp_pcache_poll(dpep, NULL, pcp, nfds, &fdcnt);
1462 - if (rc == 0) {
1463 - *reventsp = (fdcnt > 0) ? POLLIN : 0;
1548 + /*
1549 + * Recursive polling will only emit certain events. Skip a
1550 + * scan of the pollcache if those events are not of interest.
1551 + */
1552 + if (events & (POLLIN|POLLRDNORM)) {
1553 + rc = dp_pcache_poll(dpep, NULL, pcp, nfds, &fdcnt);
1554 + } else {
1555 + rc = 0;
1556 + fdcnt = 0;
1464 1557 }
1558 +
1559 + if (rc == 0 && fdcnt > 0) {
1560 + *reventsp = POLLIN|POLLRDNORM;
1561 + } else {
1562 + *reventsp = 0;
1563 + }
1465 1564 pcachelink_assoc(pcp, ps->ps_pc_stack[0]);
1466 1565 pollstate_exit(pcp);
1467 1566 } else {
1468 1567 switch (res) {
1469 1568 case PSE_FAIL_DEPTH:
1470 1569 rc = EINVAL;
1471 1570 break;
1472 1571 case PSE_FAIL_LOOP:
1473 1572 case PSE_FAIL_DEADLOCK:
1474 1573 rc = ELOOP;
1475 1574 break;
1476 1575 default:
1477 1576 /*
1478 1577 * If anything else has gone awry, such as being polled
1479 1578 * from an unexpected context, fall back to the
1480 1579 * recursion-intolerant response.
1481 1580 */
1482 1581 *reventsp = POLLERR;
1483 1582 rc = 0;
1484 1583 break;
1485 1584 }
1486 1585 }
1487 1586
1488 1587 DP_REFRELE(dpep);
1489 1588 return (rc);
1490 1589 }
1491 1590
1492 1591 /*
1493 1592 * devpoll close should do enough clean up before the pollcache is deleted,
1494 1593 * i.e., it should ensure no one still references the pollcache later.
1495 1594 * There is no "permission" check in here. Any process having the last
1496 1595 * reference of this /dev/poll fd can close.
1497 1596 */
1498 1597 /*ARGSUSED*/
1499 1598 static int
1500 1599 dpclose(dev_t dev, int flag, int otyp, cred_t *credp)
1501 1600 {
1502 1601 minor_t minor;
1503 1602 dp_entry_t *dpep;
1504 1603 pollcache_t *pcp;
1505 1604 int i;
1506 1605 polldat_t **hashtbl;
1507 1606 polldat_t *pdp;
1508 1607
1509 1608 minor = getminor(dev);
1510 1609
1511 1610 mutex_enter(&devpoll_lock);
1512 1611 dpep = devpolltbl[minor];
1513 1612 ASSERT(dpep != NULL);
1514 1613 devpolltbl[minor] = NULL;
1515 1614 mutex_exit(&devpoll_lock);
1516 1615 pcp = dpep->dpe_pcache;
1517 1616 ASSERT(pcp != NULL);
1518 1617 /*
1519 1618 * At this point, no other lwp can access this pollcache via the
1520 1619 * /dev/poll fd. This pollcache is going away, so do the clean
1521 1620 * up without the pc_lock.
1522 1621 */
1523 1622 hashtbl = pcp->pc_hash;
1524 1623 for (i = 0; i < pcp->pc_hashsize; i++) {
1525 1624 for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
1526 1625 if (pdp->pd_php != NULL) {
1527 1626 pollhead_delete(pdp->pd_php, pdp);
1528 1627 pdp->pd_php = NULL;
1529 1628 pdp->pd_fp = NULL;
1530 1629 }
1531 1630 }
1532 1631 }
1533 1632 /*
1534 1633 * pollwakeup() may still interact with this pollcache. Wait until
1535 1634 * it is done.
1536 1635 */
1537 1636 mutex_enter(&pcp->pc_no_exit);
1538 1637 ASSERT(pcp->pc_busy >= 0);
1539 1638 while (pcp->pc_busy > 0)
1540 1639 cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
1541 1640 mutex_exit(&pcp->pc_no_exit);
1542 1641
1543 1642 /* Clean up any pollcache links created via recursive /dev/poll */
1544 1643 if (pcp->pc_parents != NULL || pcp->pc_children != NULL) {
1545 1644 /*
1546 1645 * Because of the locking rules for pcachelink manipulation,
1547 1646 * acquring pc_lock is required for this step.
1548 1647 */
1549 1648 mutex_enter(&pcp->pc_lock);
1550 1649 pcachelink_purge_all(pcp);
1551 1650 mutex_exit(&pcp->pc_lock);
1552 1651 }
1553 1652
1554 1653 pcache_destroy(pcp);
1555 1654 ASSERT(dpep->dpe_refcnt == 0);
1556 1655 kmem_free(dpep, sizeof (dp_entry_t));
1557 1656 return (0);
1558 1657 }
1559 1658
1560 1659 static void
1561 1660 pcachelink_locked_rele(pcachelink_t *pl)
1562 1661 {
1563 1662 ASSERT(MUTEX_HELD(&pl->pcl_lock));
1564 1663 VERIFY(pl->pcl_refcnt >= 1);
1565 1664
1566 1665 pl->pcl_refcnt--;
1567 1666 if (pl->pcl_refcnt == 0) {
1568 1667 VERIFY(pl->pcl_state == PCL_INVALID);
1569 1668 ASSERT(pl->pcl_parent_pc == NULL);
1570 1669 ASSERT(pl->pcl_child_pc == NULL);
1571 1670 ASSERT(pl->pcl_parent_next == NULL);
1572 1671 ASSERT(pl->pcl_child_next == NULL);
1573 1672
1574 1673 pl->pcl_state = PCL_FREE;
1575 1674 mutex_destroy(&pl->pcl_lock);
1576 1675 kmem_free(pl, sizeof (pcachelink_t));
1577 1676 } else {
1578 1677 mutex_exit(&pl->pcl_lock);
1579 1678 }
1580 1679 }
1581 1680
1582 1681 /*
1583 1682 * Associate parent and child pollcaches via a pcachelink_t. If an existing
1584 1683 * link (stale or valid) between the two is found, it will be reused. If a
1585 1684 * suitable link is not found for reuse, a new one will be allocated.
1586 1685 */
1587 1686 static void
1588 1687 pcachelink_assoc(pollcache_t *child, pollcache_t *parent)
1589 1688 {
1590 1689 pcachelink_t *pl, **plpn;
1591 1690
1592 1691 ASSERT(MUTEX_HELD(&child->pc_lock));
1593 1692 ASSERT(MUTEX_HELD(&parent->pc_lock));
1594 1693
1595 1694 /* Search for an existing link we can reuse. */
1596 1695 plpn = &child->pc_parents;
1597 1696 for (pl = child->pc_parents; pl != NULL; pl = *plpn) {
1598 1697 mutex_enter(&pl->pcl_lock);
1599 1698 if (pl->pcl_state == PCL_INVALID) {
1600 1699 /* Clean any invalid links while walking the list */
1601 1700 *plpn = pl->pcl_parent_next;
1602 1701 pl->pcl_child_pc = NULL;
1603 1702 pl->pcl_parent_next = NULL;
1604 1703 pcachelink_locked_rele(pl);
1605 1704 } else if (pl->pcl_parent_pc == parent) {
1606 1705 /* Successfully found parent link */
1607 1706 ASSERT(pl->pcl_state == PCL_VALID ||
1608 1707 pl->pcl_state == PCL_STALE);
1609 1708 pl->pcl_state = PCL_VALID;
1610 1709 mutex_exit(&pl->pcl_lock);
1611 1710 return;
1612 1711 } else {
1613 1712 plpn = &pl->pcl_parent_next;
1614 1713 mutex_exit(&pl->pcl_lock);
1615 1714 }
1616 1715 }
1617 1716
1618 1717 /* No existing link to the parent was found. Create a fresh one. */
1619 1718 pl = kmem_zalloc(sizeof (pcachelink_t), KM_SLEEP);
1620 1719 mutex_init(&pl->pcl_lock, NULL, MUTEX_DEFAULT, NULL);
1621 1720
1622 1721 pl->pcl_parent_pc = parent;
1623 1722 pl->pcl_child_next = parent->pc_children;
1624 1723 parent->pc_children = pl;
1625 1724 pl->pcl_refcnt++;
1626 1725
1627 1726 pl->pcl_child_pc = child;
1628 1727 pl->pcl_parent_next = child->pc_parents;
1629 1728 child->pc_parents = pl;
1630 1729 pl->pcl_refcnt++;
1631 1730
1632 1731 pl->pcl_state = PCL_VALID;
1633 1732 }
1634 1733
1635 1734 /*
1636 1735 * Mark all child links in a pollcache as stale. Any invalid child links found
1637 1736 * during iteration are purged.
1638 1737 */
1639 1738 static void
1640 1739 pcachelink_mark_stale(pollcache_t *pcp)
1641 1740 {
1642 1741 pcachelink_t *pl, **plpn;
1643 1742
1644 1743 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1645 1744
1646 1745 plpn = &pcp->pc_children;
1647 1746 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1648 1747 mutex_enter(&pl->pcl_lock);
1649 1748 if (pl->pcl_state == PCL_INVALID) {
1650 1749 /*
1651 1750 * Remove any invalid links while we are going to the
1652 1751 * trouble of walking the list.
1653 1752 */
1654 1753 *plpn = pl->pcl_child_next;
1655 1754 pl->pcl_parent_pc = NULL;
1656 1755 pl->pcl_child_next = NULL;
1657 1756 pcachelink_locked_rele(pl);
1658 1757 } else {
1659 1758 pl->pcl_state = PCL_STALE;
1660 1759 plpn = &pl->pcl_child_next;
1661 1760 mutex_exit(&pl->pcl_lock);
1662 1761 }
1663 1762 }
1664 1763 }
1665 1764
1666 1765 /*
1667 1766 * Purge all stale (or invalid) child links from a pollcache.
1668 1767 */
1669 1768 static void
1670 1769 pcachelink_purge_stale(pollcache_t *pcp)
1671 1770 {
1672 1771 pcachelink_t *pl, **plpn;
1673 1772
1674 1773 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1675 1774
1676 1775 plpn = &pcp->pc_children;
1677 1776 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1678 1777 mutex_enter(&pl->pcl_lock);
1679 1778 switch (pl->pcl_state) {
1680 1779 case PCL_STALE:
1681 1780 pl->pcl_state = PCL_INVALID;
1682 1781 /* FALLTHROUGH */
1683 1782 case PCL_INVALID:
1684 1783 *plpn = pl->pcl_child_next;
1685 1784 pl->pcl_parent_pc = NULL;
1686 1785 pl->pcl_child_next = NULL;
1687 1786 pcachelink_locked_rele(pl);
1688 1787 break;
1689 1788 default:
1690 1789 plpn = &pl->pcl_child_next;
1691 1790 mutex_exit(&pl->pcl_lock);
1692 1791 }
1693 1792 }
1694 1793 }
1695 1794
1696 1795 /*
1697 1796 * Purge all child and parent links from a pollcache, regardless of status.
1698 1797 */
1699 1798 static void
1700 1799 pcachelink_purge_all(pollcache_t *pcp)
1701 1800 {
1702 1801 pcachelink_t *pl, **plpn;
1703 1802
1704 1803 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1705 1804
1706 1805 plpn = &pcp->pc_parents;
1707 1806 for (pl = pcp->pc_parents; pl != NULL; pl = *plpn) {
1708 1807 mutex_enter(&pl->pcl_lock);
1709 1808 pl->pcl_state = PCL_INVALID;
1710 1809 *plpn = pl->pcl_parent_next;
1711 1810 pl->pcl_child_pc = NULL;
1712 1811 pl->pcl_parent_next = NULL;
1713 1812 pcachelink_locked_rele(pl);
1714 1813 }
1715 1814
1716 1815 plpn = &pcp->pc_children;
1717 1816 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1718 1817 mutex_enter(&pl->pcl_lock);
1719 1818 pl->pcl_state = PCL_INVALID;
1720 1819 *plpn = pl->pcl_child_next;
1721 1820 pl->pcl_parent_pc = NULL;
1722 1821 pl->pcl_child_next = NULL;
1723 1822 pcachelink_locked_rele(pl);
1724 1823 }
1725 1824
1726 1825 ASSERT(pcp->pc_parents == NULL);
1727 1826 ASSERT(pcp->pc_children == NULL);
1728 1827 }
|
↓ open down ↓ |
254 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX