Print this page
OS-5518 devpoll write feigns success in the face of EINTR
OS-5520 epoll_ctl not allowed to emit EINTR
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Bryan Cantrill <bryan@joyent.com>
OS-5516 vmxnet3s declares wrong sdu
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Bryan Cantrill <bryan@joyent.com>
OS-5511 epoll should not leave dangling polldat_t entries
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Bryan Cantrill <bryan@joyent.com>
OS-5260 lxbrand epoll_pwait needs sigset translation
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/devpoll.c
+++ new/usr/src/uts/common/io/devpoll.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /*
27 27 * Copyright (c) 2012 by Delphix. All rights reserved.
28 28 * Copyright 2016 Joyent, Inc.
29 29 */
30 30
31 31 #include <sys/types.h>
32 32 #include <sys/devops.h>
33 33 #include <sys/conf.h>
34 34 #include <sys/modctl.h>
35 35 #include <sys/sunddi.h>
36 36 #include <sys/stat.h>
37 37 #include <sys/poll_impl.h>
38 38 #include <sys/errno.h>
39 39 #include <sys/kmem.h>
40 40 #include <sys/mkdev.h>
41 41 #include <sys/debug.h>
42 42 #include <sys/file.h>
43 43 #include <sys/sysmacros.h>
44 44 #include <sys/systm.h>
45 45 #include <sys/bitmap.h>
46 46 #include <sys/devpoll.h>
47 47 #include <sys/rctl.h>
48 48 #include <sys/resource.h>
49 49 #include <sys/schedctl.h>
50 50 #include <sys/epoll.h>
51 51
52 52 #define RESERVED 1
53 53
54 54 /* local data struct */
55 55 static dp_entry_t **devpolltbl; /* dev poll entries */
56 56 static size_t dptblsize;
57 57
58 58 static kmutex_t devpoll_lock; /* lock protecting dev tbl */
59 59 int devpoll_init; /* is /dev/poll initialized already */
60 60
61 61 /* device local functions */
62 62
63 63 static int dpopen(dev_t *devp, int flag, int otyp, cred_t *credp);
64 64 static int dpwrite(dev_t dev, struct uio *uiop, cred_t *credp);
65 65 static int dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
66 66 int *rvalp);
67 67 static int dppoll(dev_t dev, short events, int anyyet, short *reventsp,
68 68 struct pollhead **phpp);
69 69 static int dpclose(dev_t dev, int flag, int otyp, cred_t *credp);
70 70 static dev_info_t *dpdevi;
71 71
72 72
73 73 static struct cb_ops dp_cb_ops = {
74 74 dpopen, /* open */
75 75 dpclose, /* close */
76 76 nodev, /* strategy */
77 77 nodev, /* print */
78 78 nodev, /* dump */
79 79 nodev, /* read */
80 80 dpwrite, /* write */
81 81 dpioctl, /* ioctl */
82 82 nodev, /* devmap */
83 83 nodev, /* mmap */
84 84 nodev, /* segmap */
85 85 dppoll, /* poll */
86 86 ddi_prop_op, /* prop_op */
87 87 (struct streamtab *)0, /* streamtab */
88 88 D_MP, /* flags */
89 89 CB_REV, /* cb_ops revision */
90 90 nodev, /* aread */
91 91 nodev /* awrite */
92 92 };
93 93
94 94 static int dpattach(dev_info_t *, ddi_attach_cmd_t);
95 95 static int dpdetach(dev_info_t *, ddi_detach_cmd_t);
96 96 static int dpinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
97 97
98 98 static struct dev_ops dp_ops = {
99 99 DEVO_REV, /* devo_rev */
100 100 0, /* refcnt */
101 101 dpinfo, /* info */
102 102 nulldev, /* identify */
103 103 nulldev, /* probe */
104 104 dpattach, /* attach */
105 105 dpdetach, /* detach */
106 106 nodev, /* reset */
107 107 &dp_cb_ops, /* driver operations */
108 108 (struct bus_ops *)NULL, /* bus operations */
109 109 nulldev, /* power */
110 110 ddi_quiesce_not_needed, /* quiesce */
111 111 };
112 112
113 113
114 114 static struct modldrv modldrv = {
115 115 &mod_driverops, /* type of module - a driver */
116 116 "/dev/poll driver",
117 117 &dp_ops,
118 118 };
119 119
120 120 static struct modlinkage modlinkage = {
121 121 MODREV_1,
122 122 (void *)&modldrv,
123 123 NULL
124 124 };
125 125
126 126 static void pcachelink_assoc(pollcache_t *, pollcache_t *);
127 127 static void pcachelink_mark_stale(pollcache_t *);
128 128 static void pcachelink_purge_stale(pollcache_t *);
129 129 static void pcachelink_purge_all(pollcache_t *);
130 130
131 131
132 132 /*
133 133 * Locking Design
134 134 *
135 135 * The /dev/poll driver shares most of its code with poll sys call whose
136 136 * code is in common/syscall/poll.c. In poll(2) design, the pollcache
137 137 * structure is per lwp. An implicit assumption is made there that some
138 138 * portion of pollcache will never be touched by other lwps. E.g., in
139 139 * poll(2) design, no lwp will ever need to grow bitmap of other lwp.
140 140 * This assumption is not true for /dev/poll; hence the need for extra
141 141 * locking.
142 142 *
143 143 * To allow more parallelism, each /dev/poll file descriptor (indexed by
144 144 * minor number) has its own lock. Since read (dpioctl) is a much more
145 145 * frequent operation than write, we want to allow multiple reads on same
146 146 * /dev/poll fd. However, we prevent writes from being starved by giving
147 147 * priority to write operation. Theoretically writes can starve reads as
148 148 * well. But in practical sense this is not important because (1) writes
149 149 * happens less often than reads, and (2) write operation defines the
150 150 * content of poll fd a cache set. If writes happens so often that they
151 151 * can starve reads, that means the cached set is very unstable. It may
152 152 * not make sense to read an unstable cache set anyway. Therefore, the
153 153 * writers starving readers case is not handled in this design.
154 154 */
155 155
156 156 int
157 157 _init()
158 158 {
159 159 int error;
160 160
161 161 dptblsize = DEVPOLLSIZE;
162 162 devpolltbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
163 163 mutex_init(&devpoll_lock, NULL, MUTEX_DEFAULT, NULL);
164 164 devpoll_init = 1;
165 165 if ((error = mod_install(&modlinkage)) != 0) {
166 166 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
167 167 devpoll_init = 0;
168 168 }
169 169 return (error);
170 170 }
171 171
172 172 int
173 173 _fini()
174 174 {
175 175 int error;
176 176
177 177 if ((error = mod_remove(&modlinkage)) != 0) {
178 178 return (error);
179 179 }
180 180 mutex_destroy(&devpoll_lock);
181 181 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
182 182 return (0);
183 183 }
184 184
185 185 int
186 186 _info(struct modinfo *modinfop)
187 187 {
188 188 return (mod_info(&modlinkage, modinfop));
189 189 }
190 190
191 191 /*ARGSUSED*/
192 192 static int
193 193 dpattach(dev_info_t *devi, ddi_attach_cmd_t cmd)
194 194 {
195 195 if (ddi_create_minor_node(devi, "poll", S_IFCHR, 0, DDI_PSEUDO, NULL)
196 196 == DDI_FAILURE) {
197 197 ddi_remove_minor_node(devi, NULL);
198 198 return (DDI_FAILURE);
199 199 }
200 200 dpdevi = devi;
201 201 return (DDI_SUCCESS);
202 202 }
203 203
204 204 static int
205 205 dpdetach(dev_info_t *devi, ddi_detach_cmd_t cmd)
206 206 {
207 207 if (cmd != DDI_DETACH)
208 208 return (DDI_FAILURE);
209 209
210 210 ddi_remove_minor_node(devi, NULL);
211 211 return (DDI_SUCCESS);
212 212 }
213 213
214 214 /* ARGSUSED */
215 215 static int
216 216 dpinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
217 217 {
218 218 int error;
219 219
220 220 switch (infocmd) {
221 221 case DDI_INFO_DEVT2DEVINFO:
222 222 *result = (void *)dpdevi;
223 223 error = DDI_SUCCESS;
224 224 break;
225 225 case DDI_INFO_DEVT2INSTANCE:
226 226 *result = (void *)0;
227 227 error = DDI_SUCCESS;
228 228 break;
229 229 default:
230 230 error = DDI_FAILURE;
231 231 }
232 232 return (error);
233 233 }
234 234
235 235 /*
236 236 * dp_pcache_poll has similar logic to pcache_poll() in poll.c. The major
237 237 * differences are: (1) /dev/poll requires scanning the bitmap starting at
238 238 * where it was stopped last time, instead of always starting from 0,
239 239 * (2) since user may not have cleaned up the cached fds when they are
240 240 * closed, some polldats in cache may refer to closed or reused fds. We
241 241 * need to check for those cases.
242 242 *
243 243 * NOTE: Upon closing an fd, automatic poll cache cleanup is done for
244 244 * poll(2) caches but NOT for /dev/poll caches. So expect some
245 245 * stale entries!
246 246 */
247 247 static int
248 248 dp_pcache_poll(dp_entry_t *dpep, void *dpbuf,
249 249 pollcache_t *pcp, nfds_t nfds, int *fdcntp)
250 250 {
251 251 int start, ostart, end;
252 252 int fdcnt, fd;
253 253 boolean_t done;
254 254 file_t *fp;
255 255 short revent;
256 256 boolean_t no_wrap;
257 257 pollhead_t *php;
258 258 polldat_t *pdp;
259 259 pollfd_t *pfdp;
260 260 epoll_event_t *epoll;
261 261 int error = 0;
262 262 short mask = POLLRDHUP | POLLWRBAND;
263 263 boolean_t is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
264 264
265 265 ASSERT(MUTEX_HELD(&pcp->pc_lock));
266 266 if (pcp->pc_bitmap == NULL) {
267 267 /*
268 268 * No Need to search because no poll fd
269 269 * has been cached.
270 270 */
271 271 return (error);
272 272 }
273 273
274 274 if (is_epoll) {
275 275 pfdp = NULL;
276 276 epoll = (epoll_event_t *)dpbuf;
277 277 } else {
278 278 pfdp = (pollfd_t *)dpbuf;
279 279 epoll = NULL;
280 280 }
281 281 retry:
282 282 start = ostart = pcp->pc_mapstart;
283 283 end = pcp->pc_mapend;
284 284 php = NULL;
285 285
286 286 if (start == 0) {
287 287 /*
288 288 * started from every begining, no need to wrap around.
289 289 */
290 290 no_wrap = B_TRUE;
291 291 } else {
292 292 no_wrap = B_FALSE;
293 293 }
294 294 done = B_FALSE;
295 295 fdcnt = 0;
296 296 while ((fdcnt < nfds) && !done) {
297 297 php = NULL;
298 298 revent = 0;
299 299 /*
300 300 * Examine the bit map in a circular fashion
301 301 * to avoid starvation. Always resume from
302 302 * last stop. Scan till end of the map. Then
303 303 * wrap around.
304 304 */
305 305 fd = bt_getlowbit(pcp->pc_bitmap, start, end);
306 306 ASSERT(fd <= end);
307 307 if (fd >= 0) {
308 308 if (fd == end) {
309 309 if (no_wrap) {
310 310 done = B_TRUE;
311 311 } else {
312 312 start = 0;
313 313 end = ostart - 1;
314 314 no_wrap = B_TRUE;
315 315 }
316 316 } else {
317 317 start = fd + 1;
318 318 }
319 319 pdp = pcache_lookup_fd(pcp, fd);
320 320 repoll:
321 321 ASSERT(pdp != NULL);
322 322 ASSERT(pdp->pd_fd == fd);
323 323 if (pdp->pd_fp == NULL) {
324 324 /*
325 325 * The fd is POLLREMOVed. This fd is
326 326 * logically no longer cached. So move
327 327 * on to the next one.
328 328 */
329 329 continue;
330 330 }
331 331 if ((fp = getf(fd)) == NULL) {
332 332 /*
333 333 * The fd has been closed, but user has not
334 334 * done a POLLREMOVE on this fd yet. Instead
335 335 * of cleaning it here implicitly, we return
336 336 * POLLNVAL. This is consistent with poll(2)
337 337 * polling a closed fd. Hope this will remind
338 338 * user to do a POLLREMOVE.
339 339 */
340 340 if (!is_epoll && pfdp != NULL) {
341 341 pfdp[fdcnt].fd = fd;
342 342 pfdp[fdcnt].revents = POLLNVAL;
343 343 fdcnt++;
344 344 continue;
345 345 }
|
↓ open down ↓ |
345 lines elided |
↑ open up ↑ |
346 346
347 347 /*
348 348 * In the epoll compatibility case, we actually
349 349 * perform the implicit removal to remain
350 350 * closer to the epoll semantics.
351 351 */
352 352 if (is_epoll) {
353 353 pdp->pd_fp = NULL;
354 354 pdp->pd_events = 0;
355 355
356 - if (php != NULL) {
357 - pollhead_delete(php, pdp);
356 + if (pdp->pd_php != NULL) {
357 + pollhead_delete(pdp->pd_php,
358 + pdp);
358 359 pdp->pd_php = NULL;
359 360 }
360 361
361 362 BT_CLEAR(pcp->pc_bitmap, fd);
362 363 continue;
363 364 }
364 365 }
365 366
366 367 if (fp != pdp->pd_fp) {
367 368 /*
368 369 * user is polling on a cached fd which was
369 370 * closed and then reused. Unfortunately
370 371 * there is no good way to inform user.
371 372 * If the file struct is also reused, we
372 373 * may not be able to detect the fd reuse
373 374 * at all. As long as this does not
374 375 * cause system failure and/or memory leak,
375 376 * we will play along. Man page states if
376 377 * user does not clean up closed fds, polling
377 378 * results will be indeterministic.
378 379 *
379 380 * XXX - perhaps log the detection of fd
380 381 * reuse?
381 382 */
382 383 pdp->pd_fp = fp;
383 384 }
384 385 /*
385 386 * XXX - pollrelock() logic needs to know which
386 387 * which pollcache lock to grab. It'd be a
387 388 * cleaner solution if we could pass pcp as
388 389 * an arguement in VOP_POLL interface instead
389 390 * of implicitly passing it using thread_t
390 391 * struct. On the other hand, changing VOP_POLL
391 392 * interface will require all driver/file system
392 393 * poll routine to change. May want to revisit
393 394 * the tradeoff later.
394 395 */
395 396 curthread->t_pollcache = pcp;
396 397 error = VOP_POLL(fp->f_vnode, pdp->pd_events, 0,
397 398 &revent, &php, NULL);
398 399 curthread->t_pollcache = NULL;
399 400 releasef(fd);
400 401 if (error != 0) {
401 402 break;
402 403 }
403 404
404 405 /*
405 406 * layered devices (e.g. console driver)
406 407 * may change the vnode and thus the pollhead
407 408 * pointer out from underneath us.
408 409 */
409 410 if (php != NULL && pdp->pd_php != NULL &&
410 411 php != pdp->pd_php) {
411 412 pollhead_delete(pdp->pd_php, pdp);
412 413 pdp->pd_php = php;
413 414 pollhead_insert(php, pdp);
414 415 /*
415 416 * The bit should still be set.
416 417 */
417 418 ASSERT(BT_TEST(pcp->pc_bitmap, fd));
418 419 goto retry;
419 420 }
420 421
421 422 if (revent != 0) {
422 423 if (pfdp != NULL) {
423 424 pfdp[fdcnt].fd = fd;
424 425 pfdp[fdcnt].events = pdp->pd_events;
425 426 pfdp[fdcnt].revents = revent;
426 427 } else if (epoll != NULL) {
427 428 epoll_event_t *ep = &epoll[fdcnt];
428 429
429 430 ASSERT(epoll != NULL);
430 431 ep->data.u64 = pdp->pd_epolldata;
431 432
432 433 /*
433 434 * If any of the event bits are set for
434 435 * which poll and epoll representations
435 436 * differ, swizzle in the native epoll
436 437 * values.
437 438 */
438 439 if (revent & mask) {
439 440 ep->events = (revent & ~mask) |
440 441 ((revent & POLLRDHUP) ?
441 442 EPOLLRDHUP : 0) |
442 443 ((revent & POLLWRBAND) ?
443 444 EPOLLWRBAND : 0);
444 445 } else {
445 446 ep->events = revent;
446 447 }
447 448
448 449 /*
449 450 * We define POLLWRNORM to be POLLOUT,
450 451 * but epoll has separate definitions
451 452 * for them; if POLLOUT is set and the
452 453 * user has asked for EPOLLWRNORM, set
453 454 * that as well.
454 455 */
455 456 if ((revent & POLLOUT) &&
456 457 (pdp->pd_events & EPOLLWRNORM)) {
457 458 ep->events |= EPOLLWRNORM;
458 459 }
459 460 } else {
460 461 pollstate_t *ps =
461 462 curthread->t_pollstate;
462 463 /*
463 464 * The devpoll handle itself is being
464 465 * polled. Notify the caller of any
465 466 * readable event(s), leaving as much
466 467 * state as possible untouched.
467 468 */
468 469 VERIFY(fdcnt == 0);
469 470 VERIFY(ps != NULL);
470 471
471 472 /*
472 473 * If a call to pollunlock() fails
473 474 * during VOP_POLL, skip over the fd
474 475 * and continue polling.
475 476 *
476 477 * Otherwise, report that there is an
477 478 * event pending.
478 479 */
479 480 if ((ps->ps_flags & POLLSTATE_ULFAIL)
480 481 != 0) {
481 482 ps->ps_flags &=
482 483 ~POLLSTATE_ULFAIL;
483 484 continue;
484 485 } else {
485 486 fdcnt++;
486 487 break;
487 488 }
488 489 }
489 490
490 491 /*
491 492 * If POLLET is set, clear the bit in the
492 493 * bitmap -- which effectively latches the
493 494 * edge on a pollwakeup() from the driver.
494 495 */
495 496 if (pdp->pd_events & POLLET)
|
↓ open down ↓ |
128 lines elided |
↑ open up ↑ |
496 497 BT_CLEAR(pcp->pc_bitmap, fd);
497 498
498 499 /*
499 500 * If POLLONESHOT is set, perform the implicit
500 501 * POLLREMOVE.
501 502 */
502 503 if (pdp->pd_events & POLLONESHOT) {
503 504 pdp->pd_fp = NULL;
504 505 pdp->pd_events = 0;
505 506
506 - if (php != NULL) {
507 - pollhead_delete(php, pdp);
507 + if (pdp->pd_php != NULL) {
508 + pollhead_delete(pdp->pd_php,
509 + pdp);
508 510 pdp->pd_php = NULL;
509 511 }
510 512
511 513 BT_CLEAR(pcp->pc_bitmap, fd);
512 514 }
513 515
514 516 fdcnt++;
515 517 } else if (php != NULL) {
516 518 /*
517 519 * We clear a bit or cache a poll fd if
518 520 * the driver returns a poll head ptr,
519 521 * which is expected in the case of 0
520 522 * revents. Some buggy driver may return
521 523 * NULL php pointer with 0 revents. In
522 524 * this case, we just treat the driver as
523 525 * "noncachable" and not clearing the bit
524 526 * in bitmap.
525 527 */
526 528 if ((pdp->pd_php != NULL) &&
527 529 ((pcp->pc_flag & PC_POLLWAKE) == 0)) {
528 530 BT_CLEAR(pcp->pc_bitmap, fd);
529 531 }
530 532 if (pdp->pd_php == NULL) {
531 533 pollhead_insert(php, pdp);
532 534 pdp->pd_php = php;
533 535 /*
534 536 * An event of interest may have
535 537 * arrived between the VOP_POLL() and
536 538 * the pollhead_insert(); check again.
537 539 */
538 540 goto repoll;
539 541 }
540 542 }
541 543 } else {
542 544 /*
543 545 * No bit set in the range. Check for wrap around.
544 546 */
545 547 if (!no_wrap) {
546 548 start = 0;
547 549 end = ostart - 1;
548 550 no_wrap = B_TRUE;
549 551 } else {
550 552 done = B_TRUE;
551 553 }
552 554 }
553 555 }
554 556
555 557 if (!done) {
556 558 pcp->pc_mapstart = start;
557 559 }
558 560 ASSERT(*fdcntp == 0);
559 561 *fdcntp = fdcnt;
560 562 return (error);
561 563 }
562 564
563 565 /*ARGSUSED*/
564 566 static int
565 567 dpopen(dev_t *devp, int flag, int otyp, cred_t *credp)
566 568 {
567 569 minor_t minordev;
568 570 dp_entry_t *dpep;
569 571 pollcache_t *pcp;
570 572
571 573 ASSERT(devpoll_init);
572 574 ASSERT(dptblsize <= MAXMIN);
573 575 mutex_enter(&devpoll_lock);
574 576 for (minordev = 0; minordev < dptblsize; minordev++) {
575 577 if (devpolltbl[minordev] == NULL) {
576 578 devpolltbl[minordev] = (dp_entry_t *)RESERVED;
577 579 break;
578 580 }
579 581 }
580 582 if (minordev == dptblsize) {
581 583 dp_entry_t **newtbl;
582 584 size_t oldsize;
583 585
584 586 /*
585 587 * Used up every entry in the existing devpoll table.
586 588 * Grow the table by DEVPOLLSIZE.
587 589 */
588 590 if ((oldsize = dptblsize) >= MAXMIN) {
589 591 mutex_exit(&devpoll_lock);
590 592 return (ENXIO);
591 593 }
592 594 dptblsize += DEVPOLLSIZE;
593 595 if (dptblsize > MAXMIN) {
594 596 dptblsize = MAXMIN;
595 597 }
596 598 newtbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
597 599 bcopy(devpolltbl, newtbl, sizeof (caddr_t) * oldsize);
598 600 kmem_free(devpolltbl, sizeof (caddr_t) * oldsize);
599 601 devpolltbl = newtbl;
600 602 devpolltbl[minordev] = (dp_entry_t *)RESERVED;
601 603 }
602 604 mutex_exit(&devpoll_lock);
603 605
604 606 dpep = kmem_zalloc(sizeof (dp_entry_t), KM_SLEEP);
605 607 /*
606 608 * allocate a pollcache skeleton here. Delay allocating bitmap
607 609 * structures until dpwrite() time, since we don't know the
608 610 * optimal size yet. We also delay setting the pid until either
609 611 * dpwrite() or attempt to poll on the instance, allowing parents
610 612 * to create instances of /dev/poll for their children. (In the
611 613 * epoll compatibility case, this check isn't performed to maintain
612 614 * semantic compatibility.)
613 615 */
614 616 pcp = pcache_alloc();
615 617 dpep->dpe_pcache = pcp;
616 618 pcp->pc_pid = -1;
617 619 *devp = makedevice(getmajor(*devp), minordev); /* clone the driver */
618 620 mutex_enter(&devpoll_lock);
619 621 ASSERT(minordev < dptblsize);
620 622 ASSERT(devpolltbl[minordev] == (dp_entry_t *)RESERVED);
621 623 devpolltbl[minordev] = dpep;
622 624 mutex_exit(&devpoll_lock);
623 625 return (0);
624 626 }
625 627
626 628 /*
627 629 * Write to dev/poll add/remove fd's to/from a cached poll fd set,
628 630 * or change poll events for a watched fd.
629 631 */
630 632 /*ARGSUSED*/
631 633 static int
|
↓ open down ↓ |
114 lines elided |
↑ open up ↑ |
632 634 dpwrite(dev_t dev, struct uio *uiop, cred_t *credp)
633 635 {
634 636 minor_t minor;
635 637 dp_entry_t *dpep;
636 638 pollcache_t *pcp;
637 639 pollfd_t *pollfdp, *pfdp;
638 640 dvpoll_epollfd_t *epfdp;
639 641 uintptr_t limit;
640 642 int error, size;
641 643 ssize_t uiosize;
644 + size_t copysize;
642 645 nfds_t pollfdnum;
643 646 struct pollhead *php = NULL;
644 647 polldat_t *pdp;
645 648 int fd;
646 649 file_t *fp;
647 650 boolean_t is_epoll, fds_added = B_FALSE;
648 651
649 652 minor = getminor(dev);
650 653
651 654 mutex_enter(&devpoll_lock);
652 655 ASSERT(minor < dptblsize);
653 656 dpep = devpolltbl[minor];
654 657 ASSERT(dpep != NULL);
655 658 mutex_exit(&devpoll_lock);
656 659
657 660 mutex_enter(&dpep->dpe_lock);
658 661 pcp = dpep->dpe_pcache;
659 662 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
660 663 size = (is_epoll) ? sizeof (dvpoll_epollfd_t) : sizeof (pollfd_t);
661 664 mutex_exit(&dpep->dpe_lock);
662 665
663 666 if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
664 667 if (pcp->pc_pid != -1) {
665 668 return (EACCES);
666 669 }
667 670
668 671 pcp->pc_pid = curproc->p_pid;
669 672 }
670 673
671 674 uiosize = uiop->uio_resid;
672 675 pollfdnum = uiosize / size;
673 676
674 677 /*
675 678 * We want to make sure that pollfdnum isn't large enough to DoS us,
676 679 * but we also don't want to grab p_lock unnecessarily -- so we
677 680 * perform the full check against our resource limits if and only if
678 681 * pollfdnum is larger than the known-to-be-sane value of UINT8_MAX.
679 682 */
680 683 if (pollfdnum > UINT8_MAX) {
681 684 mutex_enter(&curproc->p_lock);
682 685 if (pollfdnum >
683 686 (uint_t)rctl_enforced_value(rctlproc_legacy[RLIMIT_NOFILE],
684 687 curproc->p_rctls, curproc)) {
685 688 (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
686 689 curproc->p_rctls, curproc, RCA_SAFE);
687 690 mutex_exit(&curproc->p_lock);
688 691 return (EINVAL);
689 692 }
690 693 mutex_exit(&curproc->p_lock);
691 694 }
692 695
693 696 /*
694 697 * Copy in the pollfd array. Walk through the array and add
695 698 * each polled fd to the cached set.
696 699 */
|
↓ open down ↓ |
45 lines elided |
↑ open up ↑ |
697 700 pollfdp = kmem_alloc(uiosize, KM_SLEEP);
698 701 limit = (uintptr_t)pollfdp + (pollfdnum * size);
699 702
700 703 /*
701 704 * Although /dev/poll uses the write(2) interface to cache fds, it's
702 705 * not supposed to function as a seekable device. To prevent offset
703 706 * from growing and eventually exceed the maximum, reset the offset
704 707 * here for every call.
705 708 */
706 709 uiop->uio_loffset = 0;
707 - if ((error = uiomove((caddr_t)pollfdp, uiosize, UIO_WRITE, uiop))
708 - != 0) {
710 +
711 + /*
712 + * Use uiocopy instead of uiomove when populating pollfdp, keeping
713 + * uio_resid untouched for now. Write syscalls will translate EINTR
714 + * into a success if they detect "successfully transfered" data via an
715 + * updated uio_resid. Falsely suppressing such errors is disastrous.
716 + */
717 + if ((error = uiocopy((caddr_t)pollfdp, uiosize, UIO_WRITE, uiop,
718 + ©size)) != 0) {
709 719 kmem_free(pollfdp, uiosize);
710 720 return (error);
711 721 }
722 +
712 723 /*
713 724 * We are about to enter the core portion of dpwrite(). Make sure this
714 725 * write has exclusive access in this portion of the code, i.e., no
715 726 * other writers in this code.
716 727 *
717 728 * Waiting for all readers to drop their references to the dpe is
718 729 * unecessary since the pollcache itself is protected by pc_lock.
719 730 */
720 731 mutex_enter(&dpep->dpe_lock);
721 732 dpep->dpe_writerwait++;
722 733 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0) {
723 734 ASSERT(dpep->dpe_refcnt != 0);
724 735
725 736 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
726 737 dpep->dpe_writerwait--;
727 738 mutex_exit(&dpep->dpe_lock);
728 739 kmem_free(pollfdp, uiosize);
729 740 return (EINTR);
730 741 }
731 742 }
732 743 dpep->dpe_writerwait--;
733 744 dpep->dpe_flag |= DP_WRITER_PRESENT;
734 745 dpep->dpe_refcnt++;
735 746
736 747 if (!is_epoll && (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0) {
737 748 /*
738 749 * The epoll compat mode was enabled while we were waiting to
739 750 * establish write access. It is not safe to continue since
740 751 * state was prepared for non-epoll operation.
741 752 */
742 753 error = EBUSY;
743 754 goto bypass;
744 755 }
745 756 mutex_exit(&dpep->dpe_lock);
746 757
747 758 /*
748 759 * Since the dpwrite() may recursively walk an added /dev/poll handle,
749 760 * pollstate_enter() deadlock and loop detection must be used.
750 761 */
751 762 (void) pollstate_create();
752 763 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
753 764
754 765 if (pcp->pc_bitmap == NULL) {
755 766 pcache_create(pcp, pollfdnum);
756 767 }
757 768 for (pfdp = pollfdp; (uintptr_t)pfdp < limit;
758 769 pfdp = (pollfd_t *)((uintptr_t)pfdp + size)) {
759 770 fd = pfdp->fd;
760 771 if ((uint_t)fd >= P_FINFO(curproc)->fi_nfiles) {
761 772 /*
762 773 * epoll semantics demand that we return EBADF if our
763 774 * specified fd is invalid.
764 775 */
765 776 if (is_epoll) {
766 777 error = EBADF;
767 778 break;
768 779 }
769 780
770 781 continue;
771 782 }
772 783
773 784 pdp = pcache_lookup_fd(pcp, fd);
774 785 if (pfdp->events != POLLREMOVE) {
775 786
776 787 fp = NULL;
777 788
778 789 if (pdp == NULL) {
779 790 /*
780 791 * If we're in epoll compatibility mode, check
781 792 * that the fd is valid before allocating
782 793 * anything for it; epoll semantics demand that
783 794 * we return EBADF if our specified fd is
784 795 * invalid.
785 796 */
786 797 if (is_epoll) {
787 798 if ((fp = getf(fd)) == NULL) {
788 799 error = EBADF;
789 800 break;
790 801 }
791 802 }
792 803
793 804 pdp = pcache_alloc_fd(0);
794 805 pdp->pd_fd = fd;
795 806 pdp->pd_pcache = pcp;
796 807 pcache_insert_fd(pcp, pdp, pollfdnum);
797 808 } else {
798 809 /*
799 810 * epoll semantics demand that we error out if
800 811 * a file descriptor is added twice, which we
801 812 * check (imperfectly) by checking if we both
802 813 * have the file descriptor cached and the
803 814 * file pointer that correponds to the file
804 815 * descriptor matches our cached value. If
805 816 * there is a pointer mismatch, the file
806 817 * descriptor was closed without being removed.
807 818 * The converse is clearly not true, however,
808 819 * so to narrow the window by which a spurious
809 820 * EEXIST may be returned, we also check if
810 821 * this fp has been added to an epoll control
811 822 * descriptor in the past; if it hasn't, we
812 823 * know that this is due to fp reuse -- it's
813 824 * not a true EEXIST case. (By performing this
814 825 * additional check, we limit the window of
815 826 * spurious EEXIST to situations where a single
816 827 * file descriptor is being used across two or
817 828 * more epoll control descriptors -- and even
818 829 * then, the file descriptor must be closed and
819 830 * reused in a relatively tight time span.)
820 831 */
821 832 if (is_epoll) {
822 833 if (pdp->pd_fp != NULL &&
823 834 (fp = getf(fd)) != NULL &&
824 835 fp == pdp->pd_fp &&
825 836 (fp->f_flag2 & FEPOLLED)) {
826 837 error = EEXIST;
827 838 releasef(fd);
828 839 break;
829 840 }
830 841
831 842 /*
832 843 * We have decided that the cached
833 844 * information was stale: it either
834 845 * didn't match, or the fp had never
835 846 * actually been epoll()'d on before.
836 847 * We need to now clear our pd_events
837 848 * to assure that we don't mistakenly
838 849 * operate on cached event disposition.
839 850 */
840 851 pdp->pd_events = 0;
841 852 }
842 853 }
843 854
844 855 if (is_epoll) {
845 856 epfdp = (dvpoll_epollfd_t *)pfdp;
846 857 pdp->pd_epolldata = epfdp->dpep_data;
847 858 }
848 859
849 860 ASSERT(pdp->pd_fd == fd);
850 861 ASSERT(pdp->pd_pcache == pcp);
851 862 if (fd >= pcp->pc_mapsize) {
852 863 mutex_exit(&pcp->pc_lock);
853 864 pcache_grow_map(pcp, fd);
854 865 mutex_enter(&pcp->pc_lock);
855 866 }
856 867 if (fd > pcp->pc_mapend) {
857 868 pcp->pc_mapend = fd;
858 869 }
859 870 if (fp == NULL && (fp = getf(fd)) == NULL) {
860 871 /*
861 872 * The fd is not valid. Since we can't pass
862 873 * this error back in the write() call, set
863 874 * the bit in bitmap to force DP_POLL ioctl
864 875 * to examine it.
865 876 */
866 877 BT_SET(pcp->pc_bitmap, fd);
867 878 pdp->pd_events |= pfdp->events;
868 879 continue;
869 880 }
870 881
871 882 /*
872 883 * To (greatly) reduce EEXIST false positives, we
873 884 * denote that this fp has been epoll()'d. We do this
874 885 * regardless of epoll compatibility mode, as the flag
875 886 * is harmless if not in epoll compatibility mode.
876 887 */
877 888 fp->f_flag2 |= FEPOLLED;
878 889
879 890 /*
880 891 * Don't do VOP_POLL for an already cached fd with
881 892 * same poll events.
882 893 */
883 894 if ((pdp->pd_events == pfdp->events) &&
884 895 (pdp->pd_fp == fp)) {
885 896 /*
886 897 * the events are already cached
887 898 */
888 899 releasef(fd);
889 900 continue;
890 901 }
891 902
892 903 /*
893 904 * do VOP_POLL and cache this poll fd.
894 905 */
895 906 /*
896 907 * XXX - pollrelock() logic needs to know which
897 908 * which pollcache lock to grab. It'd be a
898 909 * cleaner solution if we could pass pcp as
899 910 * an arguement in VOP_POLL interface instead
900 911 * of implicitly passing it using thread_t
901 912 * struct. On the other hand, changing VOP_POLL
902 913 * interface will require all driver/file system
903 914 * poll routine to change. May want to revisit
904 915 * the tradeoff later.
905 916 */
906 917 curthread->t_pollcache = pcp;
907 918 error = VOP_POLL(fp->f_vnode, pfdp->events, 0,
908 919 &pfdp->revents, &php, NULL);
909 920 curthread->t_pollcache = NULL;
910 921 /*
911 922 * We always set the bit when this fd is cached;
912 923 * this forces the first DP_POLL to poll this fd.
913 924 * Real performance gain comes from subsequent
914 925 * DP_POLL. We also attempt a pollhead_insert();
915 926 * if it's not possible, we'll do it in dpioctl().
916 927 */
917 928 BT_SET(pcp->pc_bitmap, fd);
918 929 if (error != 0) {
919 930 releasef(fd);
920 931 break;
921 932 }
922 933 pdp->pd_fp = fp;
923 934 pdp->pd_events |= pfdp->events;
924 935 if (php != NULL) {
925 936 if (pdp->pd_php == NULL) {
926 937 pollhead_insert(php, pdp);
927 938 pdp->pd_php = php;
928 939 } else {
929 940 if (pdp->pd_php != php) {
930 941 pollhead_delete(pdp->pd_php,
931 942 pdp);
932 943 pollhead_insert(php, pdp);
933 944 pdp->pd_php = php;
934 945 }
935 946 }
936 947 }
937 948 fds_added = B_TRUE;
938 949 releasef(fd);
939 950 } else {
940 951 if (pdp == NULL || pdp->pd_fp == NULL) {
941 952 if (is_epoll) {
942 953 /*
943 954 * As with the add case (above), epoll
944 955 * semantics demand that we error out
945 956 * in this case.
946 957 */
947 958 error = ENOENT;
948 959 break;
949 960 }
950 961
951 962 continue;
952 963 }
953 964 ASSERT(pdp->pd_fd == fd);
954 965 pdp->pd_fp = NULL;
955 966 pdp->pd_events = 0;
956 967 ASSERT(pdp->pd_thread == NULL);
957 968 if (pdp->pd_php != NULL) {
958 969 pollhead_delete(pdp->pd_php, pdp);
959 970 pdp->pd_php = NULL;
960 971 }
961 972 BT_CLEAR(pcp->pc_bitmap, fd);
962 973 }
963 974 }
964 975 /*
965 976 * Wake any pollcache waiters so they can check the new descriptors.
966 977 *
967 978 * Any fds added to an recursive-capable pollcache could themselves be
968 979 * /dev/poll handles. To ensure that proper event propagation occurs,
969 980 * parent pollcaches are woken too, so that they can create any needed
970 981 * pollcache links.
971 982 */
972 983 if (fds_added) {
973 984 cv_broadcast(&pcp->pc_cv);
|
↓ open down ↓ |
252 lines elided |
↑ open up ↑ |
974 985 pcache_wake_parents(pcp);
975 986 }
976 987 pollstate_exit(pcp);
977 988 mutex_enter(&dpep->dpe_lock);
978 989 bypass:
979 990 dpep->dpe_flag &= ~DP_WRITER_PRESENT;
980 991 dpep->dpe_refcnt--;
981 992 cv_broadcast(&dpep->dpe_cv);
982 993 mutex_exit(&dpep->dpe_lock);
983 994 kmem_free(pollfdp, uiosize);
995 + if (error == 0) {
996 + /*
997 + * The state of uio_resid is updated only after the pollcache
998 + * is successfully modified.
999 + */
1000 + uioskip(uiop, copysize);
1001 + }
984 1002 return (error);
985 1003 }
986 1004
987 1005 #define DP_SIGMASK_RESTORE(ksetp) { \
988 1006 if (ksetp != NULL) { \
989 1007 mutex_enter(&p->p_lock); \
990 1008 if (lwp->lwp_cursig == 0) { \
991 1009 t->t_hold = lwp->lwp_sigoldmask; \
992 1010 t->t_flag &= ~T_TOMASK; \
993 1011 } \
994 1012 mutex_exit(&p->p_lock); \
995 1013 } \
996 1014 }
997 1015
998 1016 /*ARGSUSED*/
999 1017 static int
1000 1018 dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
1001 1019 {
1002 1020 minor_t minor;
1003 1021 dp_entry_t *dpep;
1004 1022 pollcache_t *pcp;
1005 1023 hrtime_t now;
1006 1024 int error = 0;
1007 1025 boolean_t is_epoll;
1008 1026 STRUCT_DECL(dvpoll, dvpoll);
1009 1027
1010 1028 if (cmd == DP_POLL || cmd == DP_PPOLL) {
1011 1029 /* do this now, before we sleep on DP_WRITER_PRESENT */
1012 1030 now = gethrtime();
1013 1031 }
1014 1032
1015 1033 minor = getminor(dev);
1016 1034 mutex_enter(&devpoll_lock);
1017 1035 ASSERT(minor < dptblsize);
1018 1036 dpep = devpolltbl[minor];
1019 1037 mutex_exit(&devpoll_lock);
1020 1038 ASSERT(dpep != NULL);
1021 1039 pcp = dpep->dpe_pcache;
1022 1040
1023 1041 mutex_enter(&dpep->dpe_lock);
1024 1042 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
1025 1043
1026 1044 if (cmd == DP_EPOLLCOMPAT) {
1027 1045 if (dpep->dpe_refcnt != 0) {
1028 1046 /*
1029 1047 * We can't turn on epoll compatibility while there
1030 1048 * are outstanding operations.
1031 1049 */
1032 1050 mutex_exit(&dpep->dpe_lock);
1033 1051 return (EBUSY);
1034 1052 }
1035 1053
1036 1054 /*
1037 1055 * epoll compatibility is a one-way street: there's no way
1038 1056 * to turn it off for a particular open.
1039 1057 */
1040 1058 dpep->dpe_flag |= DP_ISEPOLLCOMPAT;
1041 1059 mutex_exit(&dpep->dpe_lock);
1042 1060
1043 1061 return (0);
1044 1062 }
1045 1063
1046 1064 if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
1047 1065 if (pcp->pc_pid != -1) {
1048 1066 mutex_exit(&dpep->dpe_lock);
1049 1067 return (EACCES);
1050 1068 }
1051 1069
1052 1070 pcp->pc_pid = curproc->p_pid;
1053 1071 }
1054 1072
1055 1073 /* Wait until all writers have cleared the handle before continuing */
1056 1074 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0 ||
1057 1075 (dpep->dpe_writerwait != 0)) {
1058 1076 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
1059 1077 mutex_exit(&dpep->dpe_lock);
1060 1078 return (EINTR);
1061 1079 }
1062 1080 }
1063 1081 dpep->dpe_refcnt++;
1064 1082 mutex_exit(&dpep->dpe_lock);
1065 1083
1066 1084 switch (cmd) {
1067 1085 case DP_POLL:
1068 1086 case DP_PPOLL:
1069 1087 {
1070 1088 pollstate_t *ps;
1071 1089 nfds_t nfds;
1072 1090 int fdcnt = 0;
1073 1091 size_t size, fdsize, dpsize;
1074 1092 hrtime_t deadline = 0;
1075 1093 k_sigset_t *ksetp = NULL;
1076 1094 k_sigset_t kset;
1077 1095 sigset_t set;
1078 1096 kthread_t *t = curthread;
1079 1097 klwp_t *lwp = ttolwp(t);
1080 1098 struct proc *p = ttoproc(curthread);
1081 1099
1082 1100 STRUCT_INIT(dvpoll, mode);
1083 1101
1084 1102 /*
1085 1103 * The dp_setp member is only required/consumed for DP_PPOLL,
1086 1104 * which otherwise uses the same structure as DP_POLL.
1087 1105 */
1088 1106 if (cmd == DP_POLL) {
1089 1107 dpsize = (uintptr_t)STRUCT_FADDR(dvpoll, dp_setp) -
1090 1108 (uintptr_t)STRUCT_FADDR(dvpoll, dp_fds);
1091 1109 } else {
1092 1110 ASSERT(cmd == DP_PPOLL);
1093 1111 dpsize = STRUCT_SIZE(dvpoll);
1094 1112 }
1095 1113
1096 1114 if ((mode & FKIOCTL) != 0) {
1097 1115 /* Kernel-internal ioctl call */
1098 1116 bcopy((caddr_t)arg, STRUCT_BUF(dvpoll), dpsize);
1099 1117 error = 0;
1100 1118 } else {
1101 1119 error = copyin((caddr_t)arg, STRUCT_BUF(dvpoll),
1102 1120 dpsize);
1103 1121 }
1104 1122
1105 1123 if (error) {
1106 1124 DP_REFRELE(dpep);
1107 1125 return (EFAULT);
1108 1126 }
1109 1127
1110 1128 deadline = STRUCT_FGET(dvpoll, dp_timeout);
1111 1129 if (deadline > 0) {
1112 1130 /*
1113 1131 * Convert the deadline from relative milliseconds
1114 1132 * to absolute nanoseconds. They must wait for at
1115 1133 * least a tick.
|
↓ open down ↓ |
122 lines elided |
↑ open up ↑ |
1116 1134 */
1117 1135 deadline = MSEC2NSEC(deadline);
1118 1136 deadline = MAX(deadline, nsec_per_tick);
1119 1137 deadline += now;
1120 1138 }
1121 1139
1122 1140 if (cmd == DP_PPOLL) {
1123 1141 void *setp = STRUCT_FGETP(dvpoll, dp_setp);
1124 1142
1125 1143 if (setp != NULL) {
1126 - if (copyin(setp, &set, sizeof (set))) {
1127 - DP_REFRELE(dpep);
1128 - return (EFAULT);
1144 + if ((mode & FKIOCTL) != 0) {
1145 + /* Use the signal set directly */
1146 + ksetp = (k_sigset_t *)setp;
1147 + } else {
1148 + if (copyin(setp, &set, sizeof (set))) {
1149 + DP_REFRELE(dpep);
1150 + return (EFAULT);
1151 + }
1152 + sigutok(&set, &kset);
1153 + ksetp = &kset;
1129 1154 }
1130 1155
1131 - sigutok(&set, &kset);
1132 - ksetp = &kset;
1133 -
1134 1156 mutex_enter(&p->p_lock);
1135 1157 schedctl_finish_sigblock(t);
1136 1158 lwp->lwp_sigoldmask = t->t_hold;
1137 1159 t->t_hold = *ksetp;
1138 1160 t->t_flag |= T_TOMASK;
1139 1161
1140 1162 /*
1141 1163 * Like ppoll() with a non-NULL sigset, we'll
1142 1164 * call cv_reltimedwait_sig() just to check for
1143 1165 * signals. This call will return immediately
1144 1166 * with either 0 (signalled) or -1 (no signal).
1145 1167 * There are some conditions whereby we can
1146 1168 * get 0 from cv_reltimedwait_sig() without
1147 1169 * a true signal (e.g., a directed stop), so
1148 1170 * we restore our signal mask in the unlikely
1149 1171 * event that lwp_cursig is 0.
1150 1172 */
1151 1173 if (!cv_reltimedwait_sig(&t->t_delay_cv,
1152 1174 &p->p_lock, 0, TR_CLOCK_TICK)) {
1153 1175 if (lwp->lwp_cursig == 0) {
1154 1176 t->t_hold = lwp->lwp_sigoldmask;
1155 1177 t->t_flag &= ~T_TOMASK;
1156 1178 }
1157 1179
1158 1180 mutex_exit(&p->p_lock);
1159 1181
1160 1182 DP_REFRELE(dpep);
1161 1183 return (EINTR);
1162 1184 }
1163 1185
1164 1186 mutex_exit(&p->p_lock);
1165 1187 }
1166 1188 }
1167 1189
1168 1190 if ((nfds = STRUCT_FGET(dvpoll, dp_nfds)) == 0) {
1169 1191 /*
1170 1192 * We are just using DP_POLL to sleep, so
1171 1193 * we don't any of the devpoll apparatus.
1172 1194 * Do not check for signals if we have a zero timeout.
1173 1195 */
1174 1196 DP_REFRELE(dpep);
1175 1197 if (deadline == 0) {
1176 1198 DP_SIGMASK_RESTORE(ksetp);
1177 1199 return (0);
1178 1200 }
1179 1201
1180 1202 mutex_enter(&curthread->t_delay_lock);
1181 1203 while ((error =
1182 1204 cv_timedwait_sig_hrtime(&curthread->t_delay_cv,
1183 1205 &curthread->t_delay_lock, deadline)) > 0)
1184 1206 continue;
1185 1207 mutex_exit(&curthread->t_delay_lock);
1186 1208
1187 1209 DP_SIGMASK_RESTORE(ksetp);
1188 1210
1189 1211 return (error == 0 ? EINTR : 0);
1190 1212 }
1191 1213
1192 1214 if (is_epoll) {
1193 1215 size = nfds * (fdsize = sizeof (epoll_event_t));
1194 1216 } else {
1195 1217 size = nfds * (fdsize = sizeof (pollfd_t));
1196 1218 }
1197 1219
1198 1220 /*
1199 1221 * XXX It would be nice not to have to alloc each time, but it
1200 1222 * requires another per thread structure hook. This can be
1201 1223 * implemented later if data suggests that it's necessary.
1202 1224 */
1203 1225 ps = pollstate_create();
1204 1226
1205 1227 if (ps->ps_dpbufsize < size) {
1206 1228 /*
1207 1229 * If nfds is larger than twice the current maximum
1208 1230 * open file count, we'll silently clamp it. This
1209 1231 * only limits our exposure to allocating an
1210 1232 * inordinate amount of kernel memory; it doesn't
1211 1233 * otherwise affect the semantics. (We have this
1212 1234 * check at twice the maximum instead of merely the
1213 1235 * maximum because some applications pass an nfds that
1214 1236 * is only slightly larger than their limit.)
1215 1237 */
1216 1238 mutex_enter(&p->p_lock);
1217 1239 if ((nfds >> 1) > p->p_fno_ctl) {
1218 1240 nfds = p->p_fno_ctl;
1219 1241 size = nfds * fdsize;
1220 1242 }
1221 1243 mutex_exit(&p->p_lock);
1222 1244
1223 1245 if (ps->ps_dpbufsize < size) {
1224 1246 kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize);
1225 1247 ps->ps_dpbuf = kmem_zalloc(size, KM_SLEEP);
1226 1248 ps->ps_dpbufsize = size;
1227 1249 }
1228 1250 }
1229 1251
1230 1252 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
1231 1253 for (;;) {
1232 1254 pcp->pc_flag &= ~PC_POLLWAKE;
1233 1255
1234 1256 /*
1235 1257 * Mark all child pcachelinks as stale.
1236 1258 * Those which are still part of the tree will be
1237 1259 * marked as valid during the poll.
1238 1260 */
1239 1261 pcachelink_mark_stale(pcp);
1240 1262
1241 1263 error = dp_pcache_poll(dpep, ps->ps_dpbuf,
1242 1264 pcp, nfds, &fdcnt);
1243 1265 if (fdcnt > 0 || error != 0)
1244 1266 break;
1245 1267
1246 1268 /* Purge still-stale child pcachelinks */
1247 1269 pcachelink_purge_stale(pcp);
1248 1270
1249 1271 /*
1250 1272 * A pollwake has happened since we polled cache.
1251 1273 */
1252 1274 if (pcp->pc_flag & PC_POLLWAKE)
1253 1275 continue;
1254 1276
1255 1277 /*
1256 1278 * Sleep until we are notified, signaled, or timed out.
1257 1279 */
1258 1280 if (deadline == 0) {
1259 1281 /* immediate timeout; do not check signals */
1260 1282 break;
1261 1283 }
1262 1284
1263 1285 error = cv_timedwait_sig_hrtime(&pcp->pc_cv,
1264 1286 &pcp->pc_lock, deadline);
1265 1287
1266 1288 /*
1267 1289 * If we were awakened by a signal or timeout then
1268 1290 * break the loop, else poll again.
1269 1291 */
1270 1292 if (error <= 0) {
1271 1293 error = (error == 0) ? EINTR : 0;
|
↓ open down ↓ |
128 lines elided |
↑ open up ↑ |
1272 1294 break;
1273 1295 } else {
1274 1296 error = 0;
1275 1297 }
1276 1298 }
1277 1299 pollstate_exit(pcp);
1278 1300
1279 1301 DP_SIGMASK_RESTORE(ksetp);
1280 1302
1281 1303 if (error == 0 && fdcnt > 0) {
1304 + /*
1305 + * It should be noted that FKIOCTL does not influence
1306 + * the copyout (vs bcopy) of dp_fds at this time.
1307 + */
1282 1308 if (copyout(ps->ps_dpbuf,
1283 1309 STRUCT_FGETP(dvpoll, dp_fds), fdcnt * fdsize)) {
1284 1310 DP_REFRELE(dpep);
1285 1311 return (EFAULT);
1286 1312 }
1287 1313 *rvalp = fdcnt;
1288 1314 }
1289 1315 break;
1290 1316 }
1291 1317
1292 1318 case DP_ISPOLLED:
1293 1319 {
1294 1320 pollfd_t pollfd;
1295 1321 polldat_t *pdp;
1296 1322
1297 1323 STRUCT_INIT(dvpoll, mode);
1298 1324 error = copyin((caddr_t)arg, &pollfd, sizeof (pollfd_t));
1299 1325 if (error) {
1300 1326 DP_REFRELE(dpep);
1301 1327 return (EFAULT);
1302 1328 }
1303 1329 mutex_enter(&pcp->pc_lock);
1304 1330 if (pcp->pc_hash == NULL) {
1305 1331 /*
1306 1332 * No Need to search because no poll fd
1307 1333 * has been cached.
1308 1334 */
1309 1335 mutex_exit(&pcp->pc_lock);
1310 1336 DP_REFRELE(dpep);
1311 1337 return (0);
1312 1338 }
1313 1339 if (pollfd.fd < 0) {
1314 1340 mutex_exit(&pcp->pc_lock);
1315 1341 break;
1316 1342 }
1317 1343 pdp = pcache_lookup_fd(pcp, pollfd.fd);
1318 1344 if ((pdp != NULL) && (pdp->pd_fd == pollfd.fd) &&
1319 1345 (pdp->pd_fp != NULL)) {
1320 1346 pollfd.revents = pdp->pd_events;
1321 1347 if (copyout(&pollfd, (caddr_t)arg, sizeof (pollfd_t))) {
1322 1348 mutex_exit(&pcp->pc_lock);
1323 1349 DP_REFRELE(dpep);
1324 1350 return (EFAULT);
1325 1351 }
1326 1352 *rvalp = 1;
1327 1353 }
1328 1354 mutex_exit(&pcp->pc_lock);
1329 1355 break;
1330 1356 }
1331 1357
1332 1358 default:
1333 1359 DP_REFRELE(dpep);
1334 1360 return (EINVAL);
1335 1361 }
1336 1362 DP_REFRELE(dpep);
1337 1363 return (error);
1338 1364 }
1339 1365
1340 1366 /*
1341 1367 * Overview of Recursive Polling
1342 1368 *
1343 1369 * It is possible for /dev/poll to poll for events on file descriptors which
1344 1370 * themselves are /dev/poll handles. Pending events in the child handle are
1345 1371 * represented as readable data via the POLLIN flag. To limit surface area,
1346 1372 * this recursion is presently allowed on only /dev/poll handles which have
1347 1373 * been placed in epoll mode via the DP_EPOLLCOMPAT ioctl. Recursion depth is
1348 1374 * limited to 5 in order to be consistent with Linux epoll.
1349 1375 *
1350 1376 * Extending dppoll() for VOP_POLL:
1351 1377 *
1352 1378 * The recursive /dev/poll implementation begins by extending dppoll() to
1353 1379 * report when resources contained in the pollcache have relevant event state.
1354 1380 * At the highest level, it means calling dp_pcache_poll() so it indicates if
1355 1381 * fd events are present without consuming them or altering the pollcache
1356 1382 * bitmap. This ensures that a subsequent DP_POLL operation on the bitmap will
1357 1383 * yield the initiating event. Additionally, the VOP_POLL should return in
1358 1384 * such a way that dp_pcache_poll() does not clear the parent bitmap entry
1359 1385 * which corresponds to the child /dev/poll fd. This means that child
1360 1386 * pollcaches will be checked during every poll which facilitates wake-up
1361 1387 * behavior detailed below.
1362 1388 *
1363 1389 * Pollcache Links and Wake Events:
1364 1390 *
1365 1391 * Recursive /dev/poll avoids complicated pollcache locking constraints during
1366 1392 * pollwakeup events by eschewing the traditional pollhead mechanism in favor
1367 1393 * of a different approach. For each pollcache at the root of a recursive
1368 1394 * /dev/poll "tree", pcachelink_t structures are established to all child
1369 1395 * /dev/poll pollcaches. During pollnotify() in a child pollcache, the
1370 1396 * linked list of pcachelink_t entries is walked, where those marked as valid
1371 1397 * incur a cv_broadcast to their parent pollcache. Most notably, these
1372 1398 * pcachelink_t cv wakeups are performed without acquiring pc_lock on the
1373 1399 * parent pollcache (which would require careful deadlock avoidance). This
1374 1400 * still allows the woken poll on the parent to discover the pertinent events
1375 1401 * due to the fact that bitmap entires for the child pollcache are always
1376 1402 * maintained by the dppoll() logic above.
1377 1403 *
1378 1404 * Depth Limiting and Loop Prevention:
1379 1405 *
1380 1406 * As each pollcache is encountered (either via DP_POLL or dppoll()), depth and
1381 1407 * loop constraints are enforced via pollstate_enter(). The pollcache_t
1382 1408 * pointer is compared against any existing entries in ps_pc_stack and is added
1383 1409 * to the end if no match (and therefore loop) is found. Once poll operations
1384 1410 * for a given pollcache_t are complete, pollstate_exit() clears the pointer
1385 1411 * from the list. The pollstate_enter() and pollstate_exit() functions are
1386 1412 * responsible for acquiring and releasing pc_lock, respectively.
1387 1413 *
1388 1414 * Deadlock Safety:
1389 1415 *
1390 1416 * Descending through a tree of recursive /dev/poll handles involves the tricky
1391 1417 * business of sequentially entering multiple pollcache locks. This tree
1392 1418 * topology cannot define a lock acquisition order in such a way that it is
1393 1419 * immune to deadlocks between threads. The pollstate_enter() and
1394 1420 * pollstate_exit() functions provide an interface for recursive /dev/poll
1395 1421 * operations to safely lock pollcaches while failing gracefully in the face of
1396 1422 * deadlocking topologies. (See pollstate_contend() for more detail about how
1397 1423 * deadlocks are detected and resolved.)
1398 1424 */
1399 1425
1400 1426 /*ARGSUSED*/
1401 1427 static int
1402 1428 dppoll(dev_t dev, short events, int anyyet, short *reventsp,
1403 1429 struct pollhead **phpp)
1404 1430 {
1405 1431 minor_t minor;
1406 1432 dp_entry_t *dpep;
1407 1433 pollcache_t *pcp;
1408 1434 int res, rc = 0;
1409 1435
1410 1436 minor = getminor(dev);
1411 1437 mutex_enter(&devpoll_lock);
1412 1438 ASSERT(minor < dptblsize);
1413 1439 dpep = devpolltbl[minor];
1414 1440 ASSERT(dpep != NULL);
1415 1441 mutex_exit(&devpoll_lock);
1416 1442
1417 1443 mutex_enter(&dpep->dpe_lock);
1418 1444 if ((dpep->dpe_flag & DP_ISEPOLLCOMPAT) == 0) {
1419 1445 /* Poll recursion is not yet supported for non-epoll handles */
1420 1446 *reventsp = POLLERR;
1421 1447 mutex_exit(&dpep->dpe_lock);
1422 1448 return (0);
1423 1449 } else {
1424 1450 dpep->dpe_refcnt++;
1425 1451 pcp = dpep->dpe_pcache;
1426 1452 mutex_exit(&dpep->dpe_lock);
1427 1453 }
1428 1454
1429 1455 res = pollstate_enter(pcp);
1430 1456 if (res == PSE_SUCCESS) {
1431 1457 nfds_t nfds = 1;
1432 1458 int fdcnt = 0;
1433 1459 pollstate_t *ps = curthread->t_pollstate;
1434 1460
1435 1461 rc = dp_pcache_poll(dpep, NULL, pcp, nfds, &fdcnt);
1436 1462 if (rc == 0) {
1437 1463 *reventsp = (fdcnt > 0) ? POLLIN : 0;
1438 1464 }
1439 1465 pcachelink_assoc(pcp, ps->ps_pc_stack[0]);
1440 1466 pollstate_exit(pcp);
1441 1467 } else {
1442 1468 switch (res) {
1443 1469 case PSE_FAIL_DEPTH:
1444 1470 rc = EINVAL;
1445 1471 break;
1446 1472 case PSE_FAIL_LOOP:
1447 1473 case PSE_FAIL_DEADLOCK:
1448 1474 rc = ELOOP;
1449 1475 break;
1450 1476 default:
1451 1477 /*
1452 1478 * If anything else has gone awry, such as being polled
1453 1479 * from an unexpected context, fall back to the
1454 1480 * recursion-intolerant response.
1455 1481 */
1456 1482 *reventsp = POLLERR;
1457 1483 rc = 0;
1458 1484 break;
1459 1485 }
1460 1486 }
1461 1487
1462 1488 DP_REFRELE(dpep);
1463 1489 return (rc);
1464 1490 }
1465 1491
1466 1492 /*
1467 1493 * devpoll close should do enough clean up before the pollcache is deleted,
1468 1494 * i.e., it should ensure no one still references the pollcache later.
1469 1495 * There is no "permission" check in here. Any process having the last
1470 1496 * reference of this /dev/poll fd can close.
1471 1497 */
1472 1498 /*ARGSUSED*/
1473 1499 static int
1474 1500 dpclose(dev_t dev, int flag, int otyp, cred_t *credp)
1475 1501 {
1476 1502 minor_t minor;
1477 1503 dp_entry_t *dpep;
1478 1504 pollcache_t *pcp;
1479 1505 int i;
1480 1506 polldat_t **hashtbl;
1481 1507 polldat_t *pdp;
1482 1508
1483 1509 minor = getminor(dev);
1484 1510
1485 1511 mutex_enter(&devpoll_lock);
1486 1512 dpep = devpolltbl[minor];
1487 1513 ASSERT(dpep != NULL);
1488 1514 devpolltbl[minor] = NULL;
1489 1515 mutex_exit(&devpoll_lock);
1490 1516 pcp = dpep->dpe_pcache;
1491 1517 ASSERT(pcp != NULL);
1492 1518 /*
1493 1519 * At this point, no other lwp can access this pollcache via the
1494 1520 * /dev/poll fd. This pollcache is going away, so do the clean
1495 1521 * up without the pc_lock.
1496 1522 */
1497 1523 hashtbl = pcp->pc_hash;
1498 1524 for (i = 0; i < pcp->pc_hashsize; i++) {
1499 1525 for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
1500 1526 if (pdp->pd_php != NULL) {
1501 1527 pollhead_delete(pdp->pd_php, pdp);
1502 1528 pdp->pd_php = NULL;
1503 1529 pdp->pd_fp = NULL;
1504 1530 }
1505 1531 }
1506 1532 }
1507 1533 /*
1508 1534 * pollwakeup() may still interact with this pollcache. Wait until
1509 1535 * it is done.
1510 1536 */
1511 1537 mutex_enter(&pcp->pc_no_exit);
1512 1538 ASSERT(pcp->pc_busy >= 0);
1513 1539 while (pcp->pc_busy > 0)
1514 1540 cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
1515 1541 mutex_exit(&pcp->pc_no_exit);
1516 1542
1517 1543 /* Clean up any pollcache links created via recursive /dev/poll */
1518 1544 if (pcp->pc_parents != NULL || pcp->pc_children != NULL) {
1519 1545 /*
1520 1546 * Because of the locking rules for pcachelink manipulation,
1521 1547 * acquring pc_lock is required for this step.
1522 1548 */
1523 1549 mutex_enter(&pcp->pc_lock);
1524 1550 pcachelink_purge_all(pcp);
1525 1551 mutex_exit(&pcp->pc_lock);
1526 1552 }
1527 1553
1528 1554 pcache_destroy(pcp);
1529 1555 ASSERT(dpep->dpe_refcnt == 0);
1530 1556 kmem_free(dpep, sizeof (dp_entry_t));
1531 1557 return (0);
1532 1558 }
1533 1559
1534 1560 static void
1535 1561 pcachelink_locked_rele(pcachelink_t *pl)
1536 1562 {
1537 1563 ASSERT(MUTEX_HELD(&pl->pcl_lock));
1538 1564 VERIFY(pl->pcl_refcnt >= 1);
1539 1565
1540 1566 pl->pcl_refcnt--;
1541 1567 if (pl->pcl_refcnt == 0) {
1542 1568 VERIFY(pl->pcl_state == PCL_INVALID);
1543 1569 ASSERT(pl->pcl_parent_pc == NULL);
1544 1570 ASSERT(pl->pcl_child_pc == NULL);
1545 1571 ASSERT(pl->pcl_parent_next == NULL);
1546 1572 ASSERT(pl->pcl_child_next == NULL);
1547 1573
1548 1574 pl->pcl_state = PCL_FREE;
1549 1575 mutex_destroy(&pl->pcl_lock);
1550 1576 kmem_free(pl, sizeof (pcachelink_t));
1551 1577 } else {
1552 1578 mutex_exit(&pl->pcl_lock);
1553 1579 }
1554 1580 }
1555 1581
1556 1582 /*
1557 1583 * Associate parent and child pollcaches via a pcachelink_t. If an existing
1558 1584 * link (stale or valid) between the two is found, it will be reused. If a
1559 1585 * suitable link is not found for reuse, a new one will be allocated.
1560 1586 */
1561 1587 static void
1562 1588 pcachelink_assoc(pollcache_t *child, pollcache_t *parent)
1563 1589 {
1564 1590 pcachelink_t *pl, **plpn;
1565 1591
1566 1592 ASSERT(MUTEX_HELD(&child->pc_lock));
1567 1593 ASSERT(MUTEX_HELD(&parent->pc_lock));
1568 1594
1569 1595 /* Search for an existing link we can reuse. */
1570 1596 plpn = &child->pc_parents;
1571 1597 for (pl = child->pc_parents; pl != NULL; pl = *plpn) {
1572 1598 mutex_enter(&pl->pcl_lock);
1573 1599 if (pl->pcl_state == PCL_INVALID) {
1574 1600 /* Clean any invalid links while walking the list */
1575 1601 *plpn = pl->pcl_parent_next;
1576 1602 pl->pcl_child_pc = NULL;
1577 1603 pl->pcl_parent_next = NULL;
1578 1604 pcachelink_locked_rele(pl);
1579 1605 } else if (pl->pcl_parent_pc == parent) {
1580 1606 /* Successfully found parent link */
1581 1607 ASSERT(pl->pcl_state == PCL_VALID ||
1582 1608 pl->pcl_state == PCL_STALE);
1583 1609 pl->pcl_state = PCL_VALID;
1584 1610 mutex_exit(&pl->pcl_lock);
1585 1611 return;
1586 1612 } else {
1587 1613 plpn = &pl->pcl_parent_next;
1588 1614 mutex_exit(&pl->pcl_lock);
1589 1615 }
1590 1616 }
1591 1617
1592 1618 /* No existing link to the parent was found. Create a fresh one. */
1593 1619 pl = kmem_zalloc(sizeof (pcachelink_t), KM_SLEEP);
1594 1620 mutex_init(&pl->pcl_lock, NULL, MUTEX_DEFAULT, NULL);
1595 1621
1596 1622 pl->pcl_parent_pc = parent;
1597 1623 pl->pcl_child_next = parent->pc_children;
1598 1624 parent->pc_children = pl;
1599 1625 pl->pcl_refcnt++;
1600 1626
1601 1627 pl->pcl_child_pc = child;
1602 1628 pl->pcl_parent_next = child->pc_parents;
1603 1629 child->pc_parents = pl;
1604 1630 pl->pcl_refcnt++;
1605 1631
1606 1632 pl->pcl_state = PCL_VALID;
1607 1633 }
1608 1634
1609 1635 /*
1610 1636 * Mark all child links in a pollcache as stale. Any invalid child links found
1611 1637 * during iteration are purged.
1612 1638 */
1613 1639 static void
1614 1640 pcachelink_mark_stale(pollcache_t *pcp)
1615 1641 {
1616 1642 pcachelink_t *pl, **plpn;
1617 1643
1618 1644 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1619 1645
1620 1646 plpn = &pcp->pc_children;
1621 1647 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1622 1648 mutex_enter(&pl->pcl_lock);
1623 1649 if (pl->pcl_state == PCL_INVALID) {
1624 1650 /*
1625 1651 * Remove any invalid links while we are going to the
1626 1652 * trouble of walking the list.
1627 1653 */
1628 1654 *plpn = pl->pcl_child_next;
1629 1655 pl->pcl_parent_pc = NULL;
1630 1656 pl->pcl_child_next = NULL;
1631 1657 pcachelink_locked_rele(pl);
1632 1658 } else {
1633 1659 pl->pcl_state = PCL_STALE;
1634 1660 plpn = &pl->pcl_child_next;
1635 1661 mutex_exit(&pl->pcl_lock);
1636 1662 }
1637 1663 }
1638 1664 }
1639 1665
1640 1666 /*
1641 1667 * Purge all stale (or invalid) child links from a pollcache.
1642 1668 */
1643 1669 static void
1644 1670 pcachelink_purge_stale(pollcache_t *pcp)
1645 1671 {
1646 1672 pcachelink_t *pl, **plpn;
1647 1673
1648 1674 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1649 1675
1650 1676 plpn = &pcp->pc_children;
1651 1677 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1652 1678 mutex_enter(&pl->pcl_lock);
1653 1679 switch (pl->pcl_state) {
1654 1680 case PCL_STALE:
1655 1681 pl->pcl_state = PCL_INVALID;
1656 1682 /* FALLTHROUGH */
1657 1683 case PCL_INVALID:
1658 1684 *plpn = pl->pcl_child_next;
1659 1685 pl->pcl_parent_pc = NULL;
1660 1686 pl->pcl_child_next = NULL;
1661 1687 pcachelink_locked_rele(pl);
1662 1688 break;
1663 1689 default:
1664 1690 plpn = &pl->pcl_child_next;
1665 1691 mutex_exit(&pl->pcl_lock);
1666 1692 }
1667 1693 }
1668 1694 }
1669 1695
1670 1696 /*
1671 1697 * Purge all child and parent links from a pollcache, regardless of status.
1672 1698 */
1673 1699 static void
1674 1700 pcachelink_purge_all(pollcache_t *pcp)
1675 1701 {
1676 1702 pcachelink_t *pl, **plpn;
1677 1703
1678 1704 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1679 1705
1680 1706 plpn = &pcp->pc_parents;
1681 1707 for (pl = pcp->pc_parents; pl != NULL; pl = *plpn) {
1682 1708 mutex_enter(&pl->pcl_lock);
1683 1709 pl->pcl_state = PCL_INVALID;
1684 1710 *plpn = pl->pcl_parent_next;
1685 1711 pl->pcl_child_pc = NULL;
1686 1712 pl->pcl_parent_next = NULL;
1687 1713 pcachelink_locked_rele(pl);
1688 1714 }
1689 1715
1690 1716 plpn = &pcp->pc_children;
1691 1717 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1692 1718 mutex_enter(&pl->pcl_lock);
1693 1719 pl->pcl_state = PCL_INVALID;
1694 1720 *plpn = pl->pcl_child_next;
1695 1721 pl->pcl_parent_pc = NULL;
1696 1722 pl->pcl_child_next = NULL;
1697 1723 pcachelink_locked_rele(pl);
1698 1724 }
1699 1725
1700 1726 ASSERT(pcp->pc_parents == NULL);
1701 1727 ASSERT(pcp->pc_children == NULL);
1702 1728 }
|
↓ open down ↓ |
411 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX