Print this page
OS-5566 ppoll timeout calculation can overflow
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Alex Wilson <alex.wilson@joyent.com>
Approved by: Robert Mustacchi <rm@joyent.com>
OS-4656 nested epoll does not mimic Linux behavior
Reviewed by: Bryan Cantrill <bryan@joyent.com>
OS-5162 poll/select yield improper EINTR when nfds and timeout are 0
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Joshua M. Clulow <jmc@joyent.com>
OS-4830 lxbrand convert select/poll to IKE
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/syscall/poll.c
+++ new/usr/src/uts/common/syscall/poll.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
|
↓ open down ↓ |
21 lines elided |
↑ open up ↑ |
22 22 /*
23 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
28 28 /* All Rights Reserved */
29 29
30 30 /*
31 31 * Copyright (c) 2012 by Delphix. All rights reserved.
32 - * Copyright 2015, Joyent, Inc.
32 + * Copyright 2016, Joyent, Inc.
33 33 */
34 34
35 35 /*
36 36 * Portions of this source code were derived from Berkeley 4.3 BSD
37 37 * under license from the Regents of the University of California.
38 38 */
39 39
40 40 #include <sys/param.h>
41 41 #include <sys/isa_defs.h>
42 42 #include <sys/types.h>
43 43 #include <sys/sysmacros.h>
44 44 #include <sys/user.h>
45 45 #include <sys/systm.h>
46 46 #include <sys/errno.h>
47 47 #include <sys/time.h>
48 48 #include <sys/vnode.h>
49 49 #include <sys/file.h>
50 50 #include <sys/mode.h>
51 51 #include <sys/proc.h>
52 52 #include <sys/uio.h>
53 53 #include <sys/poll_impl.h>
54 54 #include <sys/kmem.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/debug.h>
57 57 #include <sys/bitmap.h>
58 58 #include <sys/kstat.h>
59 59 #include <sys/rctl.h>
60 60 #include <sys/port_impl.h>
61 61 #include <sys/schedctl.h>
62 62 #include <sys/cpu.h>
63 63
64 64 #define NPHLOCKS 64 /* Number of locks; must be power of 2 */
65 65 #define PHLOCKADDR(php) &plocks[(((uintptr_t)(php)) >> 8) & (NPHLOCKS - 1)]
66 66 #define PHLOCK(php) PHLOCKADDR(php).pp_lock
67 67 #define PH_ENTER(php) mutex_enter(PHLOCK(php))
68 68 #define PH_EXIT(php) mutex_exit(PHLOCK(php))
69 69 #define VALID_POLL_EVENTS (POLLIN | POLLPRI | POLLOUT | POLLRDNORM \
70 70 | POLLRDBAND | POLLWRBAND | POLLHUP | POLLERR | POLLNVAL)
71 71
72 72 /*
73 73 * global counters to collect some stats
74 74 */
75 75 static struct {
76 76 kstat_named_t polllistmiss; /* failed to find a cached poll list */
77 77 kstat_named_t pollcachehit; /* list matched 100% w/ cached one */
78 78 kstat_named_t pollcachephit; /* list matched < 100% w/ cached one */
79 79 kstat_named_t pollcachemiss; /* every list entry is dif from cache */
80 80 kstat_named_t pollunlockfail; /* failed to perform pollunlock */
81 81 } pollstats = {
82 82 { "polllistmiss", KSTAT_DATA_UINT64 },
83 83 { "pollcachehit", KSTAT_DATA_UINT64 },
84 84 { "pollcachephit", KSTAT_DATA_UINT64 },
85 85 { "pollcachemiss", KSTAT_DATA_UINT64 },
86 86 { "pollunlockfail", KSTAT_DATA_UINT64 }
87 87 };
88 88
89 89 kstat_named_t *pollstats_ptr = (kstat_named_t *)&pollstats;
90 90 uint_t pollstats_ndata = sizeof (pollstats) / sizeof (kstat_named_t);
91 91
92 92 struct pplock {
93 93 kmutex_t pp_lock;
94 94 short pp_flag;
95 95 kcondvar_t pp_wait_cv;
96 96 int32_t pp_pad; /* to a nice round 16 bytes */
97 97 };
98 98
99 99 static struct pplock plocks[NPHLOCKS]; /* Hash array of pollhead locks */
100 100
101 101 /* Contention lock & list for preventing deadlocks in recursive /dev/poll. */
102 102 static kmutex_t pollstate_contenders_lock;
103 103 static pollstate_t *pollstate_contenders = NULL;
104 104
105 105 #ifdef DEBUG
106 106 static int pollchecksanity(pollstate_t *, nfds_t);
107 107 static int pollcheckxref(pollstate_t *, int);
108 108 static void pollcheckphlist(void);
109 109 static int pollcheckrevents(pollstate_t *, int, int, int);
110 110 static void checkpolldat(pollstate_t *);
111 111 #endif /* DEBUG */
112 112 static int plist_chkdupfd(file_t *, polldat_t *, pollstate_t *, pollfd_t *, int,
113 113 int *);
114 114
115 115 /*
116 116 * Data structure overview:
117 117 * The per-thread poll state consists of
118 118 * one pollstate_t
119 119 * one pollcache_t
120 120 * one bitmap with one event bit per fd
121 121 * a (two-dimensional) hashed array of polldat_t structures - one entry
122 122 * per fd
123 123 *
124 124 * This conglomerate of data structures interact with
125 125 * the pollhead which is used by VOP_POLL and pollwakeup
126 126 * (protected by the PHLOCK, cached array of plocks), and
127 127 * the fpollinfo list hanging off the fi_list which is used to notify
128 128 * poll when a cached fd is closed. This is protected by uf_lock.
129 129 *
130 130 * Invariants:
131 131 * pd_php (pollhead pointer) is set iff (if and only if) the polldat
132 132 * is on that pollhead. This is modified atomically under pc_lock.
133 133 *
134 134 * pd_fp (file_t pointer) is set iff the thread is on the fpollinfo
135 135 * list for that open file.
136 136 * This is modified atomically under pc_lock.
137 137 *
138 138 * pd_count is the sum (over all values of i) of pd_ref[i].xf_refcnt.
139 139 * Iff pd_ref[i].xf_refcnt >= 1 then
140 140 * ps_pcacheset[i].pcs_pollfd[pd_ref[i].xf_position].fd == pd_fd
141 141 * Iff pd_ref[i].xf_refcnt > 1 then
142 142 * In ps_pcacheset[i].pcs_pollfd between index
143 143 * pd_ref[i].xf_position] and the end of the list
144 144 * there are xf_refcnt entries with .fd == pd_fd
145 145 *
146 146 * Locking design:
147 147 * Whenever possible the design relies on the fact that the poll cache state
148 148 * is per thread thus for both poll and exit it is self-synchronizing.
149 149 * Thus the key interactions where other threads access the state are:
150 150 * pollwakeup (and polltime), and
151 151 * close cleaning up the cached references to an open file
152 152 *
153 153 * The two key locks in poll proper is ps_lock and pc_lock.
154 154 *
155 155 * The ps_lock is used for synchronization between poll, (lwp_)exit and close
156 156 * to ensure that modifications to pollcacheset structure are serialized.
157 157 * This lock is held through most of poll() except where poll sleeps
158 158 * since there is little need to handle closes concurrently with the execution
159 159 * of poll.
160 160 * The pc_lock protects most of the fields in pollcache structure and polldat
161 161 * structures (which are accessed by poll, pollwakeup, and polltime)
162 162 * with the exception of fields that are only modified when only one thread
163 163 * can access this per-thread state.
164 164 * Those exceptions occur in poll when first allocating the per-thread state,
165 165 * when poll grows the number of polldat (never shrinks), and when
166 166 * exit/pollcleanup has ensured that there are no references from either
167 167 * pollheads or fpollinfo to the threads poll state.
168 168 *
169 169 * Poll(2) system call is the only path which ps_lock and pc_lock are both
170 170 * held, in that order. It needs ps_lock to synchronize with close and
171 171 * lwp_exit; and pc_lock with pollwakeup.
172 172 *
173 173 * The locking interaction between pc_lock and PHLOCK take into account
174 174 * that poll acquires these locks in the order of pc_lock and then PHLOCK
175 175 * while pollwakeup does it in the reverse order. Thus pollwakeup implements
176 176 * deadlock avoidance by dropping the locks and reacquiring them in the
177 177 * reverse order. For this to work pollwakeup needs to prevent the thread
178 178 * from exiting and freeing all of the poll related state. Thus is done
179 179 * using
180 180 * the pc_no_exit lock
181 181 * the pc_busy counter
182 182 * the pc_busy_cv condition variable
183 183 *
184 184 * The locking interaction between pc_lock and uf_lock has similar
185 185 * issues. Poll holds ps_lock and/or pc_lock across calls to getf/releasef
186 186 * which acquire uf_lock. The poll cleanup in close needs to hold uf_lock
187 187 * to prevent poll or exit from doing a delfpollinfo after which the thread
188 188 * might exit. But the cleanup needs to acquire pc_lock when modifying
189 189 * the poll cache state. The solution is to use pc_busy and do the close
190 190 * cleanup in two phases:
191 191 * First close calls pollblockexit which increments pc_busy.
192 192 * This prevents the per-thread poll related state from being freed.
193 193 * Then close drops uf_lock and calls pollcacheclean.
194 194 * This routine can then acquire pc_lock and remove any references
195 195 * to the closing fd (as well as recording that it has been closed
196 196 * so that a POLLNVAL can be generated even if the fd is reused before
197 197 * poll has been woken up and checked getf() again).
198 198 *
199 199 * When removing a polled fd from poll cache, the fd is always removed
200 200 * from pollhead list first and then from fpollinfo list, i.e.,
201 201 * pollhead_delete() is called before delfpollinfo().
202 202 *
203 203 *
204 204 * Locking hierarchy:
205 205 * pc_no_exit is a leaf level lock.
206 206 * ps_lock is held when acquiring pc_lock (except when pollwakeup
207 207 * acquires pc_lock).
208 208 * pc_lock might be held when acquiring PHLOCK (pollhead_insert/
209 209 * pollhead_delete)
210 210 * pc_lock is always held (but this is not required)
211 211 * when acquiring PHLOCK (in polladd/pollhead_delete and pollwakeup called
212 212 * from pcache_clean_entry).
213 213 * pc_lock is held across addfpollinfo/delfpollinfo which acquire
214 214 * uf_lock.
215 215 * pc_lock is held across getf/releasef which acquire uf_lock.
216 216 * ps_lock might be held across getf/releasef which acquire uf_lock.
217 217 * pollwakeup tries to acquire pc_lock while holding PHLOCK
218 218 * but drops the locks and reacquire them in reverse order to avoid
219 219 * deadlock.
220 220 *
221 221 * Note also that there is deadlock avoidance support for VOP_POLL routines
222 222 * and pollwakeup involving a file system or driver lock.
223 223 * See below.
224 224 */
225 225
226 226 /*
227 227 * Deadlock avoidance support for VOP_POLL() routines. This is
228 228 * sometimes necessary to prevent deadlock between polling threads
229 229 * (which hold poll locks on entry to xx_poll(), then acquire foo)
230 230 * and pollwakeup() threads (which hold foo, then acquire poll locks).
231 231 *
232 232 * pollunlock(*cookie) releases whatever poll locks the current thread holds,
233 233 * setting a cookie for use by pollrelock();
234 234 *
235 235 * pollrelock(cookie) reacquires previously dropped poll locks;
236 236 *
237 237 * polllock(php, mutex) does the common case: pollunlock(),
238 238 * acquire the problematic mutex, pollrelock().
239 239 *
240 240 * If polllock() or pollunlock() return non-zero, it indicates that a recursive
241 241 * /dev/poll is in progress and pollcache locks cannot be dropped. Callers
242 242 * must handle this by indicating a POLLNVAL in the revents of the VOP_POLL.
243 243 */
244 244 int
245 245 pollunlock(int *lockstate)
246 246 {
247 247 pollstate_t *ps = curthread->t_pollstate;
248 248 pollcache_t *pcp;
249 249
250 250 ASSERT(lockstate != NULL);
251 251
252 252 /*
253 253 * There is no way to safely perform a pollunlock() while in the depths
254 254 * of a recursive /dev/poll operation.
255 255 */
256 256 if (ps != NULL && ps->ps_depth > 1) {
257 257 ps->ps_flags |= POLLSTATE_ULFAIL;
258 258 pollstats.pollunlockfail.value.ui64++;
259 259 return (-1);
260 260 }
261 261
262 262 /*
263 263 * t_pollcache is set by /dev/poll and event ports (port_fd.c).
264 264 * If the pollrelock/pollunlock is called as a result of poll(2),
265 265 * the t_pollcache should be NULL.
266 266 */
267 267 if (curthread->t_pollcache == NULL)
268 268 pcp = ps->ps_pcache;
269 269 else
270 270 pcp = curthread->t_pollcache;
271 271
272 272 if (!mutex_owned(&pcp->pc_lock)) {
273 273 *lockstate = 0;
274 274 } else {
275 275 *lockstate = 1;
276 276 mutex_exit(&pcp->pc_lock);
277 277 }
278 278 return (0);
279 279 }
280 280
281 281 void
282 282 pollrelock(int lockstate)
283 283 {
284 284 pollstate_t *ps = curthread->t_pollstate;
285 285 pollcache_t *pcp;
286 286
287 287 /* Skip this whole ordeal if the pollcache was not locked to begin */
288 288 if (lockstate == 0)
289 289 return;
290 290
291 291 /*
292 292 * t_pollcache is set by /dev/poll and event ports (port_fd.c).
293 293 * If the pollrelock/pollunlock is called as a result of poll(2),
294 294 * the t_pollcache should be NULL.
295 295 */
296 296 if (curthread->t_pollcache == NULL)
297 297 pcp = ps->ps_pcache;
298 298 else
299 299 pcp = curthread->t_pollcache;
300 300
301 301 mutex_enter(&pcp->pc_lock);
302 302 }
303 303
304 304 /* ARGSUSED */
305 305 int
306 306 polllock(pollhead_t *php, kmutex_t *lp)
307 307 {
308 308 if (mutex_tryenter(lp) == 0) {
309 309 int state;
|
↓ open down ↓ |
267 lines elided |
↑ open up ↑ |
310 310
311 311 if (pollunlock(&state) != 0) {
312 312 return (-1);
313 313 }
314 314 mutex_enter(lp);
315 315 pollrelock(state);
316 316 }
317 317 return (0);
318 318 }
319 319
320 -static int
321 -poll_common(pollfd_t *fds, nfds_t nfds, timespec_t *tsp, k_sigset_t *ksetp)
320 +int
321 +poll_copyin(pollstate_t *ps, pollfd_t *fds, nfds_t nfds)
322 322 {
323 + pollfd_t *pollfdp;
324 + nfds_t old_nfds;
325 +
326 + /*
327 + * NOTE: for performance, buffers are saved across poll() calls.
328 + * The theory is that if a process polls heavily, it tends to poll
329 + * on the same set of descriptors. Therefore, we only reallocate
330 + * buffers when nfds changes. There is no hysteresis control,
331 + * because there is no data to suggest that this is necessary;
332 + * the penalty of reallocating is not *that* great in any event.
333 + */
334 + old_nfds = ps->ps_nfds;
335 + if (nfds != old_nfds) {
336 + kmem_free(ps->ps_pollfd, old_nfds * sizeof (pollfd_t));
337 + pollfdp = kmem_alloc(nfds * sizeof (pollfd_t), KM_SLEEP);
338 + ps->ps_pollfd = pollfdp;
339 + ps->ps_nfds = nfds;
340 + }
341 +
342 + pollfdp = ps->ps_pollfd;
343 + if (copyin(fds, pollfdp, nfds * sizeof (pollfd_t))) {
344 + return (EFAULT);
345 + }
346 +
347 + if (fds == NULL) {
348 + /*
349 + * If the process has page 0 mapped, then the copyin() above
350 + * will succeed even if fds is NULL. However, our cached
351 + * poll lists are keyed by the address of the passed-in fds
352 + * structure, and we use the value NULL to indicate an unused
353 + * poll cache list entry. As such, we elect not to support
354 + * NULL as a valid (user) memory address and fail the poll()
355 + * call.
356 + */
357 + return (EFAULT);
358 + }
359 + return (0);
360 +}
361 +
362 +int
363 +poll_common(pollstate_t *ps, pollfd_t *fds, nfds_t nfds, timespec_t *tsp,
364 + int *fdcnt)
365 +{
323 366 kthread_t *t = curthread;
324 - klwp_t *lwp = ttolwp(t);
325 - proc_t *p = ttoproc(t);
326 - int fdcnt = 0;
327 - int i;
328 367 hrtime_t deadline; /* hrtime value when we want to return */
329 368 pollfd_t *pollfdp;
330 - pollstate_t *ps;
331 369 pollcache_t *pcp;
332 370 int error = 0;
333 - nfds_t old_nfds;
334 371 int cacheindex = 0; /* which cache set is used */
335 372
336 373 /*
337 374 * Determine the precise future time of the requested timeout, if any.
338 375 */
339 376 if (tsp == NULL) {
340 377 deadline = -1;
341 378 } else if (tsp->tv_sec == 0 && tsp->tv_nsec == 0) {
342 379 deadline = 0;
380 + } else if (tsp->tv_sec >= HRTIME_MAX/NANOSEC) {
381 + /* Use an indefinite timeout if tv_sec would cause overflow */
382 + deadline = -1;
343 383 } else {
384 + /*
385 + * The above check, when combined with the protections offered
386 + * by itimerspecfix (ensuring that neither field is negative
387 + * and that tv_nsec represents less than a whole second), will
388 + * prevent overflow during the conversion from timespec_t to
389 + * uhrtime_t.
390 + */
391 + uhrtime_t utime = tsp->tv_sec * NANOSEC;
392 + utime += tsp->tv_nsec;
393 +
344 394 /* They must wait at least a tick. */
345 - deadline = ((hrtime_t)tsp->tv_sec * NANOSEC) + tsp->tv_nsec;
346 - deadline = MAX(deadline, nsec_per_tick);
347 - deadline += gethrtime();
348 - }
395 + utime = MAX(utime, nsec_per_tick);
349 396
350 - /*
351 - * Reset our signal mask, if requested.
352 - */
353 - if (ksetp != NULL) {
354 - mutex_enter(&p->p_lock);
355 - schedctl_finish_sigblock(t);
356 - lwp->lwp_sigoldmask = t->t_hold;
357 - t->t_hold = *ksetp;
358 - t->t_flag |= T_TOMASK;
359 397 /*
360 - * Call cv_reltimedwait_sig() just to check for signals.
361 - * We will return immediately with either 0 or -1.
398 + * Since utime has an upper bound of HRTIME_MAX, adding the
399 + * gethrtime() result cannot incur an overflow as the unsigned
400 + * type has an adequate bound.
362 401 */
363 - if (!cv_reltimedwait_sig(&t->t_delay_cv, &p->p_lock, 0,
364 - TR_CLOCK_TICK)) {
365 - mutex_exit(&p->p_lock);
366 - error = EINTR;
367 - goto pollout;
402 + utime += (uhrtime_t)gethrtime();
403 + if (utime > HRTIME_MAX) {
404 + deadline = -1;
405 + } else {
406 + deadline = (hrtime_t)utime;
368 407 }
369 - mutex_exit(&p->p_lock);
370 408 }
371 409
372 410 /*
373 - * Check to see if this guy just wants to use poll() as a timeout.
411 + * Check to see if the caller just wants to use poll() as a timeout.
374 412 * If yes then bypass all the other stuff and make him sleep.
375 413 */
376 414 if (nfds == 0) {
415 + *fdcnt = 0;
377 416 /*
378 417 * Sleep until we have passed the requested future
379 418 * time or until interrupted by a signal.
380 419 * Do not check for signals if we do not want to wait.
381 420 */
382 421 if (deadline != 0) {
383 422 mutex_enter(&t->t_delay_lock);
384 423 while ((error = cv_timedwait_sig_hrtime(&t->t_delay_cv,
385 424 &t->t_delay_lock, deadline)) > 0)
386 425 continue;
387 426 mutex_exit(&t->t_delay_lock);
388 - error = (error == 0) ? EINTR : 0;
427 + return ((error == 0) ? EINTR : 0);
389 428 }
390 - goto pollout;
429 + return (0);
391 430 }
392 431
393 - if (nfds > p->p_fno_ctl) {
394 - mutex_enter(&p->p_lock);
395 - (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
396 - p->p_rctls, p, RCA_SAFE);
397 - mutex_exit(&p->p_lock);
398 - error = EINVAL;
399 - goto pollout;
400 - }
401 -
402 - /*
403 - * Need to allocate memory for pollstate before anything because
404 - * the mutex and cv are created in this space
405 - */
406 - ps = pollstate_create();
407 -
408 - if (ps->ps_pcache == NULL)
409 - ps->ps_pcache = pcache_alloc();
410 - pcp = ps->ps_pcache;
411 -
412 - /*
413 - * NOTE: for performance, buffers are saved across poll() calls.
414 - * The theory is that if a process polls heavily, it tends to poll
415 - * on the same set of descriptors. Therefore, we only reallocate
416 - * buffers when nfds changes. There is no hysteresis control,
417 - * because there is no data to suggest that this is necessary;
418 - * the penalty of reallocating is not *that* great in any event.
419 - */
420 - old_nfds = ps->ps_nfds;
421 - if (nfds != old_nfds) {
422 -
423 - kmem_free(ps->ps_pollfd, old_nfds * sizeof (pollfd_t));
424 - pollfdp = kmem_alloc(nfds * sizeof (pollfd_t), KM_SLEEP);
425 - ps->ps_pollfd = pollfdp;
426 - ps->ps_nfds = nfds;
427 - }
428 -
432 + VERIFY(ps != NULL);
429 433 pollfdp = ps->ps_pollfd;
430 - if (copyin(fds, pollfdp, nfds * sizeof (pollfd_t))) {
431 - error = EFAULT;
432 - goto pollout;
433 - }
434 + VERIFY(pollfdp != NULL);
434 435
435 - if (fds == NULL) {
436 - /*
437 - * If the process has page 0 mapped, then the copyin() above
438 - * will succeed even if fds is NULL. However, our cached
439 - * poll lists are keyed by the address of the passed-in fds
440 - * structure, and we use the value NULL to indicate an unused
441 - * poll cache list entry. As such, we elect not to support
442 - * NULL as a valid (user) memory address and fail the poll()
443 - * call.
444 - */
445 - error = EINVAL;
446 - goto pollout;
447 - }
448 -
449 436 /*
450 437 * If this thread polls for the first time, allocate ALL poll
451 438 * cache data structures and cache the poll fd list. This
452 439 * allocation is delayed till now because lwp's polling 0 fd
453 440 * (i.e. using poll as timeout()) don't need this memory.
454 441 */
455 442 mutex_enter(&ps->ps_lock);
456 443 pcp = ps->ps_pcache;
457 444 ASSERT(pcp != NULL);
458 445 if (pcp->pc_bitmap == NULL) {
459 446 pcache_create(pcp, nfds);
460 447 /*
461 448 * poll and cache this poll fd list in ps_pcacheset[0].
462 449 */
463 - error = pcacheset_cache_list(ps, fds, &fdcnt, cacheindex);
464 - if (fdcnt || error) {
450 + error = pcacheset_cache_list(ps, fds, fdcnt, cacheindex);
451 + if (error || *fdcnt) {
465 452 mutex_exit(&ps->ps_lock);
466 - goto pollout;
453 + return (error);
467 454 }
468 455 } else {
469 456 pollcacheset_t *pcset = ps->ps_pcacheset;
470 457
471 458 /*
472 459 * Not first time polling. Select a cached poll list by
473 460 * matching user pollfd list buffer address.
474 461 */
475 462 for (cacheindex = 0; cacheindex < ps->ps_nsets; cacheindex++) {
476 463 if (pcset[cacheindex].pcs_usradr == (uintptr_t)fds) {
477 464 if ((++pcset[cacheindex].pcs_count) == 0) {
478 465 /*
479 466 * counter is wrapping around.
480 467 */
|
↓ open down ↓ |
4 lines elided |
↑ open up ↑ |
481 468 pcacheset_reset_count(ps, cacheindex);
482 469 }
483 470 /*
484 471 * examine and resolve possible
485 472 * difference of the current poll
486 473 * list and previously cached one.
487 474 * If there is an error during resolve(),
488 475 * the callee will guarantee the consistency
489 476 * of cached poll list and cache content.
490 477 */
491 - error = pcacheset_resolve(ps, nfds, &fdcnt,
478 + error = pcacheset_resolve(ps, nfds, fdcnt,
492 479 cacheindex);
493 480 if (error) {
494 481 mutex_exit(&ps->ps_lock);
495 - goto pollout;
482 + return (error);
496 483 }
497 484 break;
498 485 }
499 486
500 487 /*
501 488 * Note that pcs_usradr field of an used entry won't be
502 489 * NULL because it stores the address of passed-in fds,
503 490 * and NULL fds will not be cached (Then it is either
504 491 * the special timeout case when nfds is 0 or it returns
505 492 * failure directly).
506 493 */
507 494 if (pcset[cacheindex].pcs_usradr == NULL) {
508 495 /*
509 496 * found an unused entry. Use it to cache
510 497 * this poll list.
511 498 */
512 - error = pcacheset_cache_list(ps, fds, &fdcnt,
499 + error = pcacheset_cache_list(ps, fds, fdcnt,
513 500 cacheindex);
514 - if (fdcnt || error) {
501 + if (error || *fdcnt) {
515 502 mutex_exit(&ps->ps_lock);
516 - goto pollout;
503 + return (error);
517 504 }
518 505 break;
519 506 }
520 507 }
521 508 if (cacheindex == ps->ps_nsets) {
522 509 /*
523 510 * We failed to find a matching cached poll fd list.
524 511 * replace an old list.
525 512 */
526 513 pollstats.polllistmiss.value.ui64++;
527 514 cacheindex = pcacheset_replace(ps);
528 515 ASSERT(cacheindex < ps->ps_nsets);
529 516 pcset[cacheindex].pcs_usradr = (uintptr_t)fds;
530 - error = pcacheset_resolve(ps, nfds, &fdcnt, cacheindex);
517 + error = pcacheset_resolve(ps, nfds, fdcnt, cacheindex);
531 518 if (error) {
532 519 mutex_exit(&ps->ps_lock);
533 - goto pollout;
520 + return (error);
534 521 }
535 522 }
536 523 }
537 524
538 525 /*
539 526 * Always scan the bitmap with the lock on the pollcache held.
540 527 * This is to make sure that a wakeup does not come undetected.
541 528 * If the lock is not held, a pollwakeup could have come for an
542 529 * fd we already checked but before this thread sleeps, in which
543 530 * case the wakeup is missed. Now we hold the pcache lock and
544 531 * check the bitmap again. This will prevent wakeup from happening
545 532 * while we hold pcache lock since pollwakeup() will also lock
546 533 * the pcache before updating poll bitmap.
547 534 */
548 535 mutex_enter(&pcp->pc_lock);
549 536 for (;;) {
550 537 pcp->pc_flag = 0;
551 - error = pcache_poll(pollfdp, ps, nfds, &fdcnt, cacheindex);
552 - if (fdcnt || error) {
538 + error = pcache_poll(pollfdp, ps, nfds, fdcnt, cacheindex);
539 + if (error || *fdcnt) {
553 540 mutex_exit(&pcp->pc_lock);
554 541 mutex_exit(&ps->ps_lock);
555 542 break;
556 543 }
557 544
558 545 /*
559 546 * If PC_POLLWAKE is set, a pollwakeup() was performed on
560 547 * one of the file descriptors. This can happen only if
561 548 * one of the VOP_POLL() functions dropped pcp->pc_lock.
562 549 * The only current cases of this is in procfs (prpoll())
563 550 * and STREAMS (strpoll()).
564 551 */
565 552 if (pcp->pc_flag & PC_POLLWAKE)
566 553 continue;
567 554
568 555 /*
569 556 * If you get here, the poll of fds was unsuccessful.
570 557 * Wait until some fd becomes readable, writable, or gets
571 558 * an exception, or until a signal or a timeout occurs.
572 559 * Do not check for signals if we have a zero timeout.
573 560 */
574 561 mutex_exit(&ps->ps_lock);
575 562 if (deadline == 0) {
576 563 error = -1;
577 564 } else {
578 565 error = cv_timedwait_sig_hrtime(&pcp->pc_cv,
579 566 &pcp->pc_lock, deadline);
580 567 }
581 568 mutex_exit(&pcp->pc_lock);
582 569 /*
583 570 * If we have received a signal or timed out
584 571 * then break out and return.
585 572 */
586 573 if (error <= 0) {
587 574 error = (error == 0) ? EINTR : 0;
|
↓ open down ↓ |
25 lines elided |
↑ open up ↑ |
588 575 break;
589 576 }
590 577 /*
591 578 * We have not received a signal or timed out.
592 579 * Continue around and poll fds again.
593 580 */
594 581 mutex_enter(&ps->ps_lock);
595 582 mutex_enter(&pcp->pc_lock);
596 583 }
597 584
585 + return (error);
586 +}
587 +
588 +/*
589 + * This is the system call trap that poll(),
590 + * select() and pselect() are built upon.
591 + * It is a private interface between libc and the kernel.
592 + */
593 +int
594 +pollsys(pollfd_t *fds, nfds_t nfds, timespec_t *timeoutp, sigset_t *setp)
595 +{
596 + kthread_t *t = curthread;
597 + klwp_t *lwp = ttolwp(t);
598 + proc_t *p = ttoproc(t);
599 + timespec_t ts;
600 + timespec_t *tsp;
601 + k_sigset_t kset;
602 + pollstate_t *ps = NULL;
603 + pollfd_t *pollfdp = NULL;
604 + int error = 0, fdcnt = 0;
605 +
606 + /*
607 + * Copy in timeout
608 + */
609 + if (timeoutp == NULL) {
610 + tsp = NULL;
611 + } else {
612 + if (get_udatamodel() == DATAMODEL_NATIVE) {
613 + if (copyin(timeoutp, &ts, sizeof (ts)))
614 + return (set_errno(EFAULT));
615 + } else {
616 + timespec32_t ts32;
617 +
618 + if (copyin(timeoutp, &ts32, sizeof (ts32)))
619 + return (set_errno(EFAULT));
620 + TIMESPEC32_TO_TIMESPEC(&ts, &ts32)
621 + }
622 +
623 + if (itimerspecfix(&ts))
624 + return (set_errno(EINVAL));
625 + tsp = &ts;
626 + }
627 +
628 + /*
629 + * Copy in and reset signal mask, if requested.
630 + */
631 + if (setp != NULL) {
632 + sigset_t set;
633 +
634 + if (copyin(setp, &set, sizeof (set)))
635 + return (set_errno(EFAULT));
636 + sigutok(&set, &kset);
637 +
638 + mutex_enter(&p->p_lock);
639 + schedctl_finish_sigblock(t);
640 + lwp->lwp_sigoldmask = t->t_hold;
641 + t->t_hold = kset;
642 + t->t_flag |= T_TOMASK;
643 + /*
644 + * Call cv_reltimedwait_sig() just to check for signals.
645 + * We will return immediately with either 0 or -1.
646 + */
647 + if (!cv_reltimedwait_sig(&t->t_delay_cv, &p->p_lock, 0,
648 + TR_CLOCK_TICK)) {
649 + mutex_exit(&p->p_lock);
650 + error = EINTR;
651 + goto pollout;
652 + }
653 + mutex_exit(&p->p_lock);
654 + }
655 +
656 + /*
657 + * Initialize pollstate and copy in pollfd data if present.
658 + * If nfds == 0, we will skip all of the copying and check steps and
659 + * proceed directly into poll_common to process the supplied timeout.
660 + */
661 + if (nfds != 0) {
662 + if (nfds > p->p_fno_ctl) {
663 + mutex_enter(&p->p_lock);
664 + (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
665 + p->p_rctls, p, RCA_SAFE);
666 + mutex_exit(&p->p_lock);
667 + error = EINVAL;
668 + goto pollout;
669 + }
670 +
671 + /*
672 + * Need to allocate memory for pollstate before anything
673 + * because the mutex and cv are created in this space
674 + */
675 + ps = pollstate_create();
676 + if (ps->ps_pcache == NULL)
677 + ps->ps_pcache = pcache_alloc();
678 +
679 + if ((error = poll_copyin(ps, fds, nfds)) != 0)
680 + goto pollout;
681 + pollfdp = ps->ps_pollfd;
682 + }
683 +
684 + /*
685 + * Perform the actual poll.
686 + */
687 + error = poll_common(ps, fds, nfds, tsp, &fdcnt);
688 +
598 689 pollout:
599 690 /*
600 - * If we changed the signal mask but we received
601 - * no signal then restore the signal mask.
602 - * Otherwise psig() will deal with the signal mask.
691 + * If we changed the signal mask but we received no signal then restore
692 + * the signal mask. Otherwise psig() will deal with the signal mask.
603 693 */
604 - if (ksetp != NULL) {
694 + if (setp != NULL) {
605 695 mutex_enter(&p->p_lock);
606 696 if (lwp->lwp_cursig == 0) {
607 697 t->t_hold = lwp->lwp_sigoldmask;
608 698 t->t_flag &= ~T_TOMASK;
609 699 }
610 700 mutex_exit(&p->p_lock);
611 701 }
612 702
613 703 if (error)
614 704 return (set_errno(error));
615 -
616 705 /*
617 706 * Copy out the events and return the fdcnt to the user.
618 707 */
619 - if (nfds != 0 &&
620 - copyout(pollfdp, fds, nfds * sizeof (pollfd_t)))
708 + if (nfds != 0 && copyout(pollfdp, fds, nfds * sizeof (pollfd_t)))
621 709 return (set_errno(EFAULT));
622 710
623 711 #ifdef DEBUG
624 712 /*
625 713 * Another sanity check:
626 714 */
627 715 if (fdcnt) {
628 - int reventcnt = 0;
716 + int i, reventcnt = 0;
629 717
630 718 for (i = 0; i < nfds; i++) {
631 719 if (pollfdp[i].fd < 0) {
632 720 ASSERT(pollfdp[i].revents == 0);
633 721 continue;
634 722 }
635 723 if (pollfdp[i].revents) {
636 724 reventcnt++;
637 725 }
638 726 }
639 727 ASSERT(fdcnt == reventcnt);
640 728 } else {
729 + int i;
730 +
641 731 for (i = 0; i < nfds; i++) {
642 732 ASSERT(pollfdp[i].revents == 0);
643 733 }
644 734 }
645 735 #endif /* DEBUG */
646 736
647 737 return (fdcnt);
648 738 }
649 739
650 -/*
651 - * This is the system call trap that poll(),
652 - * select() and pselect() are built upon.
653 - * It is a private interface between libc and the kernel.
654 - */
655 -int
656 -pollsys(pollfd_t *fds, nfds_t nfds, timespec_t *timeoutp, sigset_t *setp)
657 -{
658 - timespec_t ts;
659 - timespec_t *tsp;
660 - sigset_t set;
661 - k_sigset_t kset;
662 - k_sigset_t *ksetp;
663 - model_t datamodel = get_udatamodel();
664 -
665 - if (timeoutp == NULL)
666 - tsp = NULL;
667 - else {
668 - if (datamodel == DATAMODEL_NATIVE) {
669 - if (copyin(timeoutp, &ts, sizeof (ts)))
670 - return (set_errno(EFAULT));
671 - } else {
672 - timespec32_t ts32;
673 -
674 - if (copyin(timeoutp, &ts32, sizeof (ts32)))
675 - return (set_errno(EFAULT));
676 - TIMESPEC32_TO_TIMESPEC(&ts, &ts32)
677 - }
678 -
679 - if (itimerspecfix(&ts))
680 - return (set_errno(EINVAL));
681 - tsp = &ts;
682 - }
683 -
684 - if (setp == NULL)
685 - ksetp = NULL;
686 - else {
687 - if (copyin(setp, &set, sizeof (set)))
688 - return (set_errno(EFAULT));
689 - sigutok(&set, &kset);
690 - ksetp = &kset;
691 - }
692 -
693 - return (poll_common(fds, nfds, tsp, ksetp));
694 -}
695 -
696 740 /*
697 741 * Clean up any state left around by poll(2). Called when a thread exits.
698 742 */
699 743 void
700 744 pollcleanup()
701 745 {
702 746 pollstate_t *ps = curthread->t_pollstate;
703 747 pollcache_t *pcp;
704 748
705 749 if (ps == NULL)
706 750 return;
707 751 pcp = ps->ps_pcache;
708 752 /*
709 753 * free up all cached poll fds
710 754 */
711 755 if (pcp == NULL) {
712 756 /* this pollstate is used by /dev/poll */
713 757 goto pollcleanout;
714 758 }
715 759
716 760 if (pcp->pc_bitmap != NULL) {
717 761 ASSERT(MUTEX_NOT_HELD(&ps->ps_lock));
718 762 /*
719 763 * a close lwp can race with us when cleaning up a polldat
720 764 * entry. We hold the ps_lock when cleaning hash table.
721 765 * Since this pollcache is going away anyway, there is no
722 766 * need to hold the pc_lock.
723 767 */
724 768 mutex_enter(&ps->ps_lock);
725 769 pcache_clean(pcp);
726 770 mutex_exit(&ps->ps_lock);
727 771 #ifdef DEBUG
728 772 /*
729 773 * At this point, all fds cached by this lwp should be
730 774 * cleaned up. There should be no fd in fi_list still
731 775 * reference this thread.
732 776 */
733 777 checkfpollinfo(); /* sanity check */
734 778 pollcheckphlist(); /* sanity check */
735 779 #endif /* DEBUG */
736 780 }
737 781 /*
738 782 * Be sure no one is referencing thread before exiting
739 783 */
740 784 mutex_enter(&pcp->pc_no_exit);
741 785 ASSERT(pcp->pc_busy >= 0);
742 786 while (pcp->pc_busy > 0)
743 787 cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
744 788 mutex_exit(&pcp->pc_no_exit);
745 789 pollcleanout:
746 790 pollstate_destroy(ps);
747 791 curthread->t_pollstate = NULL;
748 792 }
749 793
750 794 /*
751 795 * pollwakeup() - poke threads waiting in poll() for some event
752 796 * on a particular object.
753 797 *
754 798 * The threads hanging off of the specified pollhead structure are scanned.
755 799 * If their event mask matches the specified event(s), then pollnotify() is
756 800 * called to poke the thread.
757 801 *
758 802 * Multiple events may be specified. When POLLHUP or POLLERR are specified,
759 803 * all waiting threads are poked.
760 804 *
761 805 * It is important that pollnotify() not drop the lock protecting the list
762 806 * of threads.
763 807 */
764 808 void
765 809 pollwakeup(pollhead_t *php, short events_arg)
766 810 {
767 811 polldat_t *pdp;
768 812 int events = (ushort_t)events_arg;
769 813 struct plist {
770 814 port_t *pp;
771 815 int pevents;
772 816 struct plist *next;
773 817 };
774 818 struct plist *plhead = NULL, *pltail = NULL;
775 819
776 820 retry:
777 821 PH_ENTER(php);
778 822
779 823 for (pdp = php->ph_list; pdp; pdp = pdp->pd_next) {
780 824 if ((pdp->pd_events & events) ||
781 825 (events & (POLLHUP | POLLERR))) {
782 826
783 827 pollcache_t *pcp;
784 828
785 829 if (pdp->pd_portev != NULL) {
786 830 port_kevent_t *pkevp = pdp->pd_portev;
787 831 /*
788 832 * Object (fd) is associated with an event port,
789 833 * => send event notification to the port.
790 834 */
791 835 ASSERT(pkevp->portkev_source == PORT_SOURCE_FD);
792 836 mutex_enter(&pkevp->portkev_lock);
793 837 if (pkevp->portkev_flags & PORT_KEV_VALID) {
794 838 int pevents;
795 839
796 840 pkevp->portkev_flags &= ~PORT_KEV_VALID;
797 841 pkevp->portkev_events |= events &
798 842 (pdp->pd_events | POLLHUP |
799 843 POLLERR);
800 844 /*
801 845 * portkev_lock mutex will be released
802 846 * by port_send_event().
803 847 */
804 848 port_send_event(pkevp);
805 849
806 850 /*
807 851 * If we have some thread polling the
808 852 * port's fd, add it to the list. They
809 853 * will be notified later.
810 854 * The port_pollwkup() will flag the
811 855 * port_t so that it will not disappear
812 856 * till port_pollwkdone() is called.
813 857 */
814 858 pevents =
815 859 port_pollwkup(pkevp->portkev_port);
816 860 if (pevents) {
817 861 struct plist *t;
818 862 t = kmem_zalloc(
819 863 sizeof (struct plist),
820 864 KM_SLEEP);
821 865 t->pp = pkevp->portkev_port;
822 866 t->pevents = pevents;
823 867 if (plhead == NULL) {
824 868 plhead = t;
825 869 } else {
826 870 pltail->next = t;
827 871 }
828 872 pltail = t;
829 873 }
830 874 } else {
831 875 mutex_exit(&pkevp->portkev_lock);
832 876 }
833 877 continue;
834 878 }
835 879
836 880 pcp = pdp->pd_pcache;
837 881
838 882 /*
839 883 * Try to grab the lock for this thread. If
840 884 * we don't get it then we may deadlock so
841 885 * back out and restart all over again. Note
842 886 * that the failure rate is very very low.
843 887 */
844 888 if (mutex_tryenter(&pcp->pc_lock)) {
845 889 pollnotify(pcp, pdp->pd_fd);
846 890 mutex_exit(&pcp->pc_lock);
847 891 } else {
848 892 /*
849 893 * We are here because:
850 894 * 1) This thread has been woke up
851 895 * and is trying to get out of poll().
852 896 * 2) Some other thread is also here
853 897 * but with a different pollhead lock.
854 898 *
855 899 * So, we need to drop the lock on pollhead
856 900 * because of (1) but we want to prevent
857 901 * that thread from doing lwp_exit() or
858 902 * devpoll close. We want to ensure that
859 903 * the pollcache pointer is still invalid.
860 904 *
861 905 * Solution: Grab the pcp->pc_no_exit lock,
862 906 * increment the pc_busy counter, drop every
863 907 * lock in sight. Get out of the way and wait
864 908 * for type (2) threads to finish.
865 909 */
866 910
867 911 mutex_enter(&pcp->pc_no_exit);
868 912 pcp->pc_busy++; /* prevents exit()'s */
869 913 mutex_exit(&pcp->pc_no_exit);
870 914
871 915 PH_EXIT(php);
872 916 mutex_enter(&pcp->pc_lock);
873 917 mutex_exit(&pcp->pc_lock);
874 918 mutex_enter(&pcp->pc_no_exit);
875 919 pcp->pc_busy--;
876 920 if (pcp->pc_busy == 0) {
877 921 /*
878 922 * Wakeup the thread waiting in
879 923 * thread_exit().
880 924 */
881 925 cv_signal(&pcp->pc_busy_cv);
882 926 }
883 927 mutex_exit(&pcp->pc_no_exit);
884 928 goto retry;
885 929 }
886 930 }
887 931 }
888 932
889 933
890 934 /*
891 935 * Event ports - If this php is of the port on the list,
892 936 * call port_pollwkdone() to release it. The port_pollwkdone()
893 937 * needs to be called before dropping the PH lock so that any new
894 938 * thread attempting to poll this port are blocked. There can be
895 939 * only one thread here in pollwakeup notifying this port's fd.
896 940 */
897 941 if (plhead != NULL && &plhead->pp->port_pollhd == php) {
898 942 struct plist *t;
899 943 port_pollwkdone(plhead->pp);
900 944 t = plhead;
901 945 plhead = plhead->next;
902 946 kmem_free(t, sizeof (struct plist));
903 947 }
904 948 PH_EXIT(php);
905 949
906 950 /*
907 951 * Event ports - Notify threads polling the event port's fd.
908 952 * This is normally done in port_send_event() where it calls
909 953 * pollwakeup() on the port. But, for PORT_SOURCE_FD source alone,
910 954 * we do it here in pollwakeup() to avoid a recursive call.
911 955 */
912 956 if (plhead != NULL) {
913 957 php = &plhead->pp->port_pollhd;
914 958 events = plhead->pevents;
915 959 goto retry;
916 960 }
917 961 }
918 962
919 963 /*
920 964 * This function is called to inform a thread (or threads) that an event being
921 965 * polled on has occurred. The pollstate lock on the thread should be held
922 966 * on entry.
923 967 */
924 968 void
925 969 pollnotify(pollcache_t *pcp, int fd)
926 970 {
927 971 ASSERT(fd < pcp->pc_mapsize);
928 972 ASSERT(MUTEX_HELD(&pcp->pc_lock));
929 973 BT_SET(pcp->pc_bitmap, fd);
930 974 pcp->pc_flag |= PC_POLLWAKE;
931 975 cv_broadcast(&pcp->pc_cv);
932 976 pcache_wake_parents(pcp);
933 977 }
934 978
935 979 /*
936 980 * add a polldat entry to pollhead ph_list. The polldat struct is used
937 981 * by pollwakeup to wake sleeping pollers when polled events has happened.
938 982 */
939 983 void
940 984 pollhead_insert(pollhead_t *php, polldat_t *pdp)
941 985 {
942 986 PH_ENTER(php);
943 987 ASSERT(pdp->pd_next == NULL);
944 988 #ifdef DEBUG
945 989 {
946 990 /*
947 991 * the polldat should not be already on the list
948 992 */
949 993 polldat_t *wp;
950 994 for (wp = php->ph_list; wp; wp = wp->pd_next) {
951 995 ASSERT(wp != pdp);
952 996 }
953 997 }
954 998 #endif /* DEBUG */
955 999 pdp->pd_next = php->ph_list;
956 1000 php->ph_list = pdp;
957 1001 PH_EXIT(php);
958 1002 }
959 1003
960 1004 /*
961 1005 * Delete the polldat entry from ph_list.
962 1006 */
963 1007 void
964 1008 pollhead_delete(pollhead_t *php, polldat_t *pdp)
965 1009 {
966 1010 polldat_t *wp;
967 1011 polldat_t **wpp;
968 1012
969 1013 PH_ENTER(php);
970 1014 for (wpp = &php->ph_list; (wp = *wpp) != NULL; wpp = &wp->pd_next) {
971 1015 if (wp == pdp) {
972 1016 *wpp = pdp->pd_next;
973 1017 pdp->pd_next = NULL;
974 1018 break;
975 1019 }
976 1020 }
977 1021 #ifdef DEBUG
978 1022 /* assert that pdp is no longer in the list */
979 1023 for (wp = *wpp; wp; wp = wp->pd_next) {
980 1024 ASSERT(wp != pdp);
981 1025 }
982 1026 #endif /* DEBUG */
983 1027 PH_EXIT(php);
984 1028 }
985 1029
986 1030 /*
987 1031 * walk through the poll fd lists to see if they are identical. This is an
988 1032 * expensive operation and should not be done more than once for each poll()
989 1033 * call.
990 1034 *
991 1035 * As an optimization (i.e., not having to go through the lists more than
992 1036 * once), this routine also clear the revents field of pollfd in 'current'.
993 1037 * Zeroing out the revents field of each entry in current poll list is
994 1038 * required by poll man page.
995 1039 *
996 1040 * Since the events field of cached list has illegal poll events filtered
997 1041 * out, the current list applies the same filtering before comparison.
998 1042 *
999 1043 * The routine stops when it detects a meaningful difference, or when it
1000 1044 * exhausts the lists.
1001 1045 */
1002 1046 int
1003 1047 pcacheset_cmp(pollfd_t *current, pollfd_t *cached, pollfd_t *newlist, int n)
1004 1048 {
1005 1049 int ix;
1006 1050
1007 1051 for (ix = 0; ix < n; ix++) {
1008 1052 /* Prefetch 64 bytes worth of 8-byte elements */
1009 1053 if ((ix & 0x7) == 0) {
1010 1054 prefetch_write_many((caddr_t)¤t[ix + 8]);
1011 1055 prefetch_write_many((caddr_t)&cached[ix + 8]);
1012 1056 }
1013 1057 if (current[ix].fd == cached[ix].fd) {
1014 1058 /*
1015 1059 * Filter out invalid poll events while we are in
1016 1060 * inside the loop.
1017 1061 */
1018 1062 if (current[ix].events & ~VALID_POLL_EVENTS) {
1019 1063 current[ix].events &= VALID_POLL_EVENTS;
1020 1064 if (newlist != NULL)
1021 1065 newlist[ix].events = current[ix].events;
1022 1066 }
1023 1067 if (current[ix].events == cached[ix].events) {
1024 1068 current[ix].revents = 0;
1025 1069 continue;
1026 1070 }
1027 1071 }
1028 1072 if ((current[ix].fd < 0) && (cached[ix].fd < 0)) {
1029 1073 current[ix].revents = 0;
1030 1074 continue;
1031 1075 }
1032 1076 return (ix);
1033 1077 }
1034 1078 return (ix);
1035 1079 }
1036 1080
1037 1081 /*
1038 1082 * This routine returns a pointer to a cached poll fd entry, or NULL if it
1039 1083 * does not find it in the hash table.
1040 1084 */
1041 1085 polldat_t *
1042 1086 pcache_lookup_fd(pollcache_t *pcp, int fd)
1043 1087 {
1044 1088 int hashindex;
1045 1089 polldat_t *pdp;
1046 1090
1047 1091 hashindex = POLLHASH(pcp->pc_hashsize, fd);
1048 1092 pdp = pcp->pc_hash[hashindex];
1049 1093 while (pdp != NULL) {
1050 1094 if (pdp->pd_fd == fd)
1051 1095 break;
1052 1096 pdp = pdp->pd_hashnext;
1053 1097 }
1054 1098 return (pdp);
1055 1099 }
1056 1100
1057 1101 polldat_t *
1058 1102 pcache_alloc_fd(int nsets)
1059 1103 {
1060 1104 polldat_t *pdp;
1061 1105
1062 1106 pdp = kmem_zalloc(sizeof (polldat_t), KM_SLEEP);
1063 1107 if (nsets > 0) {
1064 1108 pdp->pd_ref = kmem_zalloc(sizeof (xref_t) * nsets, KM_SLEEP);
1065 1109 pdp->pd_nsets = nsets;
1066 1110 }
1067 1111 return (pdp);
1068 1112 }
1069 1113
1070 1114 /*
1071 1115 * This routine inserts a polldat into the pollcache's hash table. It
1072 1116 * may be necessary to grow the size of the hash table.
1073 1117 */
1074 1118 void
1075 1119 pcache_insert_fd(pollcache_t *pcp, polldat_t *pdp, nfds_t nfds)
1076 1120 {
1077 1121 int hashindex;
1078 1122 int fd;
1079 1123
1080 1124 if ((pcp->pc_fdcount > pcp->pc_hashsize * POLLHASHTHRESHOLD) ||
1081 1125 (nfds > pcp->pc_hashsize * POLLHASHTHRESHOLD)) {
1082 1126 pcache_grow_hashtbl(pcp, nfds);
1083 1127 }
1084 1128 fd = pdp->pd_fd;
1085 1129 hashindex = POLLHASH(pcp->pc_hashsize, fd);
1086 1130 pdp->pd_hashnext = pcp->pc_hash[hashindex];
1087 1131 pcp->pc_hash[hashindex] = pdp;
1088 1132 pcp->pc_fdcount++;
1089 1133
1090 1134 #ifdef DEBUG
1091 1135 {
1092 1136 /*
1093 1137 * same fd should not appear on a hash list twice
1094 1138 */
1095 1139 polldat_t *pdp1;
1096 1140 for (pdp1 = pdp->pd_hashnext; pdp1; pdp1 = pdp1->pd_hashnext) {
1097 1141 ASSERT(pdp->pd_fd != pdp1->pd_fd);
1098 1142 }
1099 1143 }
1100 1144 #endif /* DEBUG */
1101 1145 }
1102 1146
1103 1147 /*
1104 1148 * Grow the hash table -- either double the table size or round it to the
1105 1149 * nearest multiples of POLLHASHCHUNKSZ, whichever is bigger. Rehash all the
1106 1150 * elements on the hash table.
1107 1151 */
1108 1152 void
1109 1153 pcache_grow_hashtbl(pollcache_t *pcp, nfds_t nfds)
1110 1154 {
1111 1155 int oldsize;
1112 1156 polldat_t **oldtbl;
1113 1157 polldat_t *pdp, *pdp1;
1114 1158 int i;
1115 1159 #ifdef DEBUG
1116 1160 int count = 0;
1117 1161 #endif
1118 1162
1119 1163 ASSERT(pcp->pc_hashsize % POLLHASHCHUNKSZ == 0);
1120 1164 oldsize = pcp->pc_hashsize;
1121 1165 oldtbl = pcp->pc_hash;
1122 1166 if (nfds > pcp->pc_hashsize * POLLHASHINC) {
1123 1167 pcp->pc_hashsize = (nfds + POLLHASHCHUNKSZ - 1) &
1124 1168 ~(POLLHASHCHUNKSZ - 1);
1125 1169 } else {
1126 1170 pcp->pc_hashsize = pcp->pc_hashsize * POLLHASHINC;
1127 1171 }
1128 1172 pcp->pc_hash = kmem_zalloc(pcp->pc_hashsize * sizeof (polldat_t *),
1129 1173 KM_SLEEP);
1130 1174 /*
1131 1175 * rehash existing elements
1132 1176 */
1133 1177 pcp->pc_fdcount = 0;
1134 1178 for (i = 0; i < oldsize; i++) {
1135 1179 pdp = oldtbl[i];
1136 1180 while (pdp != NULL) {
1137 1181 pdp1 = pdp->pd_hashnext;
1138 1182 pcache_insert_fd(pcp, pdp, nfds);
1139 1183 pdp = pdp1;
1140 1184 #ifdef DEBUG
1141 1185 count++;
1142 1186 #endif
1143 1187 }
1144 1188 }
1145 1189 kmem_free(oldtbl, oldsize * sizeof (polldat_t *));
1146 1190 ASSERT(pcp->pc_fdcount == count);
1147 1191 }
1148 1192
1149 1193 void
1150 1194 pcache_grow_map(pollcache_t *pcp, int fd)
1151 1195 {
1152 1196 int newsize;
1153 1197 ulong_t *newmap;
1154 1198
1155 1199 /*
1156 1200 * grow to nearest multiple of POLLMAPCHUNK, assuming POLLMAPCHUNK is
1157 1201 * power of 2.
1158 1202 */
1159 1203 newsize = (fd + POLLMAPCHUNK) & ~(POLLMAPCHUNK - 1);
1160 1204 newmap = kmem_zalloc((newsize / BT_NBIPUL) * sizeof (ulong_t),
1161 1205 KM_SLEEP);
1162 1206 /*
1163 1207 * don't want pollwakeup to set a bit while growing the bitmap.
1164 1208 */
1165 1209 ASSERT(mutex_owned(&pcp->pc_lock) == 0);
1166 1210 mutex_enter(&pcp->pc_lock);
1167 1211 bcopy(pcp->pc_bitmap, newmap,
1168 1212 (pcp->pc_mapsize / BT_NBIPUL) * sizeof (ulong_t));
1169 1213 kmem_free(pcp->pc_bitmap,
1170 1214 (pcp->pc_mapsize /BT_NBIPUL) * sizeof (ulong_t));
1171 1215 pcp->pc_bitmap = newmap;
1172 1216 pcp->pc_mapsize = newsize;
1173 1217 mutex_exit(&pcp->pc_lock);
1174 1218 }
1175 1219
1176 1220 /*
1177 1221 * remove all the reference from pollhead list and fpollinfo lists.
1178 1222 */
1179 1223 void
1180 1224 pcache_clean(pollcache_t *pcp)
1181 1225 {
1182 1226 int i;
1183 1227 polldat_t **hashtbl;
1184 1228 polldat_t *pdp;
1185 1229
1186 1230 ASSERT(MUTEX_HELD(&curthread->t_pollstate->ps_lock));
1187 1231 hashtbl = pcp->pc_hash;
1188 1232 for (i = 0; i < pcp->pc_hashsize; i++) {
1189 1233 for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
1190 1234 if (pdp->pd_php != NULL) {
1191 1235 pollhead_delete(pdp->pd_php, pdp);
1192 1236 pdp->pd_php = NULL;
1193 1237 }
1194 1238 if (pdp->pd_fp != NULL) {
1195 1239 delfpollinfo(pdp->pd_fd);
1196 1240 pdp->pd_fp = NULL;
1197 1241 }
1198 1242 }
1199 1243 }
1200 1244 }
1201 1245
1202 1246 void
1203 1247 pcacheset_invalidate(pollstate_t *ps, polldat_t *pdp)
1204 1248 {
1205 1249 int i;
1206 1250 int fd = pdp->pd_fd;
1207 1251
1208 1252 /*
1209 1253 * we come here because an earlier close() on this cached poll fd.
1210 1254 */
1211 1255 ASSERT(pdp->pd_fp == NULL);
1212 1256 ASSERT(MUTEX_HELD(&ps->ps_lock));
1213 1257 pdp->pd_events = 0;
1214 1258 for (i = 0; i < ps->ps_nsets; i++) {
1215 1259 xref_t *refp;
1216 1260 pollcacheset_t *pcsp;
1217 1261
1218 1262 ASSERT(pdp->pd_ref != NULL);
1219 1263 refp = &pdp->pd_ref[i];
1220 1264 if (refp->xf_refcnt) {
1221 1265 ASSERT(refp->xf_position >= 0);
1222 1266 pcsp = &ps->ps_pcacheset[i];
1223 1267 if (refp->xf_refcnt == 1) {
1224 1268 pcsp->pcs_pollfd[refp->xf_position].fd = -1;
1225 1269 refp->xf_refcnt = 0;
1226 1270 pdp->pd_count--;
1227 1271 } else if (refp->xf_refcnt > 1) {
1228 1272 int j;
1229 1273
1230 1274 /*
1231 1275 * turn off every appearance in pcs_pollfd list
1232 1276 */
1233 1277 for (j = refp->xf_position;
1234 1278 j < pcsp->pcs_nfds; j++) {
1235 1279 if (pcsp->pcs_pollfd[j].fd == fd) {
1236 1280 pcsp->pcs_pollfd[j].fd = -1;
1237 1281 refp->xf_refcnt--;
1238 1282 pdp->pd_count--;
1239 1283 }
1240 1284 }
1241 1285 }
1242 1286 ASSERT(refp->xf_refcnt == 0);
1243 1287 refp->xf_position = POLLPOSINVAL;
1244 1288 }
1245 1289 }
1246 1290 ASSERT(pdp->pd_count == 0);
1247 1291 }
1248 1292
1249 1293 /*
1250 1294 * Insert poll fd into the pollcache, and add poll registration.
1251 1295 * This routine is called after getf() and before releasef(). So the vnode
1252 1296 * can not disappear even if we block here.
1253 1297 * If there is an error, the polled fd is not cached.
1254 1298 */
1255 1299 int
1256 1300 pcache_insert(pollstate_t *ps, file_t *fp, pollfd_t *pollfdp, int *fdcntp,
1257 1301 ssize_t pos, int which)
1258 1302 {
1259 1303 pollcache_t *pcp = ps->ps_pcache;
1260 1304 polldat_t *pdp;
1261 1305 int error;
1262 1306 int fd;
1263 1307 pollhead_t *memphp = NULL;
1264 1308 xref_t *refp;
1265 1309 int newpollfd = 0;
1266 1310
1267 1311 ASSERT(MUTEX_HELD(&ps->ps_lock));
1268 1312 /*
1269 1313 * The poll caching uses the existing VOP_POLL interface. If there
1270 1314 * is no polled events, we want the polled device to set its "some
1271 1315 * one is sleeping in poll" flag. When the polled events happen
1272 1316 * later, the driver will call pollwakeup(). We achieve this by
1273 1317 * always passing 0 in the third parameter ("anyyet") when calling
1274 1318 * VOP_POLL. This parameter is not looked at by drivers when the
1275 1319 * polled events exist. If a driver chooses to ignore this parameter
1276 1320 * and call pollwakeup whenever the polled events happen, that will
1277 1321 * be OK too.
1278 1322 */
1279 1323 ASSERT(curthread->t_pollcache == NULL);
1280 1324 error = VOP_POLL(fp->f_vnode, pollfdp->events, 0, &pollfdp->revents,
1281 1325 &memphp, NULL);
1282 1326 if (error) {
1283 1327 return (error);
1284 1328 }
1285 1329 if (pollfdp->revents) {
1286 1330 (*fdcntp)++;
1287 1331 }
1288 1332 /*
1289 1333 * polling the underlying device succeeded. Now we can cache it.
1290 1334 * A close can't come in here because we have not done a releasef()
1291 1335 * yet.
1292 1336 */
1293 1337 fd = pollfdp->fd;
1294 1338 pdp = pcache_lookup_fd(pcp, fd);
1295 1339 if (pdp == NULL) {
1296 1340 ASSERT(ps->ps_nsets > 0);
1297 1341 pdp = pcache_alloc_fd(ps->ps_nsets);
1298 1342 newpollfd = 1;
1299 1343 }
1300 1344 /*
1301 1345 * If this entry was used to cache a poll fd which was closed, and
1302 1346 * this entry has not been cleaned, do it now.
1303 1347 */
1304 1348 if ((pdp->pd_count > 0) && (pdp->pd_fp == NULL)) {
1305 1349 pcacheset_invalidate(ps, pdp);
1306 1350 ASSERT(pdp->pd_next == NULL);
1307 1351 }
1308 1352 if (pdp->pd_count == 0) {
1309 1353 pdp->pd_fd = fd;
1310 1354 pdp->pd_fp = fp;
1311 1355 addfpollinfo(fd);
1312 1356 pdp->pd_thread = curthread;
1313 1357 pdp->pd_pcache = pcp;
1314 1358 /*
1315 1359 * the entry is never used or cleared by removing a cached
1316 1360 * pollfd (pcache_delete_fd). So all the fields should be clear.
1317 1361 */
1318 1362 ASSERT(pdp->pd_next == NULL);
1319 1363 }
1320 1364
1321 1365 /*
1322 1366 * A polled fd is considered cached. So there should be a fpollinfo
1323 1367 * entry on uf_fpollinfo list.
1324 1368 */
1325 1369 ASSERT(infpollinfo(fd));
1326 1370 /*
1327 1371 * If there is an inconsistency, we want to know it here.
1328 1372 */
1329 1373 ASSERT(pdp->pd_fp == fp);
1330 1374
1331 1375 /*
1332 1376 * XXX pd_events is a union of all polled events on this fd, possibly
1333 1377 * by different threads. Unless this is a new first poll(), pd_events
1334 1378 * never shrinks. If an event is no longer polled by a process, there
1335 1379 * is no way to cancel that event. In that case, poll degrade to its
1336 1380 * old form -- polling on this fd every time poll() is called. The
1337 1381 * assumption is an app always polls the same type of events.
1338 1382 */
1339 1383 pdp->pd_events |= pollfdp->events;
1340 1384
1341 1385 pdp->pd_count++;
1342 1386 /*
1343 1387 * There is not much special handling for multiple appearances of
1344 1388 * same fd other than xf_position always recording the first
1345 1389 * appearance in poll list. If this is called from pcacheset_cache_list,
1346 1390 * a VOP_POLL is called on every pollfd entry; therefore each
1347 1391 * revents and fdcnt should be set correctly. If this is called from
1348 1392 * pcacheset_resolve, we don't care about fdcnt here. Pollreadmap will
1349 1393 * pick up the right count and handle revents field of each pollfd
1350 1394 * entry.
1351 1395 */
1352 1396 ASSERT(pdp->pd_ref != NULL);
1353 1397 refp = &pdp->pd_ref[which];
1354 1398 if (refp->xf_refcnt == 0) {
1355 1399 refp->xf_position = pos;
1356 1400 } else {
1357 1401 /*
1358 1402 * xf_position records the fd's first appearance in poll list
1359 1403 */
1360 1404 if (pos < refp->xf_position) {
1361 1405 refp->xf_position = pos;
1362 1406 }
1363 1407 }
1364 1408 ASSERT(pollfdp->fd == ps->ps_pollfd[refp->xf_position].fd);
1365 1409 refp->xf_refcnt++;
1366 1410 if (fd >= pcp->pc_mapsize) {
1367 1411 pcache_grow_map(pcp, fd);
1368 1412 }
1369 1413 if (fd > pcp->pc_mapend) {
1370 1414 pcp->pc_mapend = fd;
1371 1415 }
1372 1416 if (newpollfd != 0) {
1373 1417 pcache_insert_fd(ps->ps_pcache, pdp, ps->ps_nfds);
1374 1418 }
1375 1419 if (memphp) {
1376 1420 if (pdp->pd_php == NULL) {
1377 1421 pollhead_insert(memphp, pdp);
1378 1422 pdp->pd_php = memphp;
1379 1423 } else {
1380 1424 if (memphp != pdp->pd_php) {
1381 1425 /*
1382 1426 * layered devices (e.g. console driver)
1383 1427 * may change the vnode and thus the pollhead
1384 1428 * pointer out from underneath us.
1385 1429 */
1386 1430 pollhead_delete(pdp->pd_php, pdp);
1387 1431 pollhead_insert(memphp, pdp);
1388 1432 pdp->pd_php = memphp;
1389 1433 }
1390 1434 }
1391 1435 }
1392 1436 /*
1393 1437 * Since there is a considerable window between VOP_POLL and when
1394 1438 * we actually put the polldat struct on the pollhead list, we could
1395 1439 * miss a pollwakeup. In the case of polling additional events, we
1396 1440 * don't update the events until after VOP_POLL. So we could miss
1397 1441 * pollwakeup there too. So we always set the bit here just to be
1398 1442 * safe. The real performance gain is in subsequent pcache_poll.
1399 1443 */
1400 1444 mutex_enter(&pcp->pc_lock);
1401 1445 BT_SET(pcp->pc_bitmap, fd);
1402 1446 mutex_exit(&pcp->pc_lock);
1403 1447 return (0);
1404 1448 }
1405 1449
1406 1450 /*
1407 1451 * The entry is not really deleted. The fields are cleared so that the
1408 1452 * entry is no longer useful, but it will remain in the hash table for reuse
1409 1453 * later. It will be freed when the polling lwp exits.
1410 1454 */
1411 1455 int
1412 1456 pcache_delete_fd(pollstate_t *ps, int fd, size_t pos, int which, uint_t cevent)
1413 1457 {
1414 1458 pollcache_t *pcp = ps->ps_pcache;
1415 1459 polldat_t *pdp;
1416 1460 xref_t *refp;
1417 1461
1418 1462 ASSERT(fd < pcp->pc_mapsize);
1419 1463 ASSERT(MUTEX_HELD(&ps->ps_lock));
1420 1464
1421 1465 pdp = pcache_lookup_fd(pcp, fd);
1422 1466 ASSERT(pdp != NULL);
1423 1467 ASSERT(pdp->pd_count > 0);
1424 1468 ASSERT(pdp->pd_ref != NULL);
1425 1469 refp = &pdp->pd_ref[which];
1426 1470 if (pdp->pd_count == 1) {
1427 1471 pdp->pd_events = 0;
1428 1472 refp->xf_position = POLLPOSINVAL;
1429 1473 ASSERT(refp->xf_refcnt == 1);
1430 1474 refp->xf_refcnt = 0;
1431 1475 if (pdp->pd_php) {
1432 1476 /*
1433 1477 * It is possible for a wakeup thread to get ahead
1434 1478 * of the following pollhead_delete and set the bit in
1435 1479 * bitmap. It is OK because the bit will be cleared
1436 1480 * here anyway.
1437 1481 */
1438 1482 pollhead_delete(pdp->pd_php, pdp);
1439 1483 pdp->pd_php = NULL;
1440 1484 }
1441 1485 pdp->pd_count = 0;
1442 1486 if (pdp->pd_fp != NULL) {
1443 1487 pdp->pd_fp = NULL;
1444 1488 delfpollinfo(fd);
1445 1489 }
1446 1490 mutex_enter(&pcp->pc_lock);
1447 1491 BT_CLEAR(pcp->pc_bitmap, fd);
1448 1492 mutex_exit(&pcp->pc_lock);
1449 1493 return (0);
1450 1494 }
1451 1495 if ((cevent & POLLCLOSED) == POLLCLOSED) {
1452 1496 /*
1453 1497 * fd cached here has been closed. This is the first
1454 1498 * pcache_delete_fd called after the close. Clean up the
1455 1499 * entire entry.
1456 1500 */
1457 1501 pcacheset_invalidate(ps, pdp);
1458 1502 ASSERT(pdp->pd_php == NULL);
1459 1503 mutex_enter(&pcp->pc_lock);
1460 1504 BT_CLEAR(pcp->pc_bitmap, fd);
1461 1505 mutex_exit(&pcp->pc_lock);
1462 1506 return (0);
1463 1507 }
1464 1508 #ifdef DEBUG
1465 1509 if (getf(fd) != NULL) {
1466 1510 ASSERT(infpollinfo(fd));
1467 1511 releasef(fd);
1468 1512 }
1469 1513 #endif /* DEBUG */
1470 1514 pdp->pd_count--;
1471 1515 ASSERT(refp->xf_refcnt > 0);
1472 1516 if (--refp->xf_refcnt == 0) {
1473 1517 refp->xf_position = POLLPOSINVAL;
1474 1518 } else {
1475 1519 ASSERT(pos >= refp->xf_position);
1476 1520 if (pos == refp->xf_position) {
1477 1521 /*
1478 1522 * The xref position is no longer valid.
1479 1523 * Reset it to a special value and let
1480 1524 * caller know it needs to updatexref()
1481 1525 * with a new xf_position value.
1482 1526 */
1483 1527 refp->xf_position = POLLPOSTRANS;
1484 1528 return (1);
1485 1529 }
1486 1530 }
1487 1531 return (0);
1488 1532 }
1489 1533
1490 1534 void
1491 1535 pcache_update_xref(pollcache_t *pcp, int fd, ssize_t pos, int which)
1492 1536 {
1493 1537 polldat_t *pdp;
1494 1538
1495 1539 pdp = pcache_lookup_fd(pcp, fd);
1496 1540 ASSERT(pdp != NULL);
1497 1541 ASSERT(pdp->pd_ref != NULL);
1498 1542 pdp->pd_ref[which].xf_position = pos;
1499 1543 }
1500 1544
1501 1545 #ifdef DEBUG
1502 1546 /*
1503 1547 * For each polled fd, it's either in the bitmap or cached in
1504 1548 * pcache hash table. If this routine returns 0, something is wrong.
1505 1549 */
1506 1550 static int
1507 1551 pollchecksanity(pollstate_t *ps, nfds_t nfds)
1508 1552 {
1509 1553 int i;
1510 1554 int fd;
1511 1555 pollcache_t *pcp = ps->ps_pcache;
1512 1556 polldat_t *pdp;
1513 1557 pollfd_t *pollfdp = ps->ps_pollfd;
1514 1558 file_t *fp;
1515 1559
1516 1560 ASSERT(MUTEX_HELD(&ps->ps_lock));
1517 1561 for (i = 0; i < nfds; i++) {
1518 1562 fd = pollfdp[i].fd;
1519 1563 if (fd < 0) {
1520 1564 ASSERT(pollfdp[i].revents == 0);
1521 1565 continue;
1522 1566 }
1523 1567 if (pollfdp[i].revents == POLLNVAL)
1524 1568 continue;
1525 1569 if ((fp = getf(fd)) == NULL)
1526 1570 continue;
1527 1571 pdp = pcache_lookup_fd(pcp, fd);
1528 1572 ASSERT(pdp != NULL);
1529 1573 ASSERT(infpollinfo(fd));
1530 1574 ASSERT(pdp->pd_fp == fp);
1531 1575 releasef(fd);
1532 1576 if (BT_TEST(pcp->pc_bitmap, fd))
1533 1577 continue;
1534 1578 if (pdp->pd_php == NULL)
1535 1579 return (0);
1536 1580 }
1537 1581 return (1);
1538 1582 }
1539 1583 #endif /* DEBUG */
1540 1584
1541 1585 /*
1542 1586 * resolve the difference between the current poll list and a cached one.
1543 1587 */
1544 1588 int
1545 1589 pcacheset_resolve(pollstate_t *ps, nfds_t nfds, int *fdcntp, int which)
1546 1590 {
1547 1591 int i;
1548 1592 pollcache_t *pcp = ps->ps_pcache;
1549 1593 pollfd_t *newlist = NULL;
1550 1594 pollfd_t *current = ps->ps_pollfd;
1551 1595 pollfd_t *cached;
1552 1596 pollcacheset_t *pcsp;
1553 1597 int common;
1554 1598 int count = 0;
1555 1599 int offset;
1556 1600 int remain;
1557 1601 int fd;
1558 1602 file_t *fp;
1559 1603 int fdcnt = 0;
1560 1604 int cnt = 0;
1561 1605 nfds_t old_nfds;
1562 1606 int error = 0;
1563 1607 int mismatch = 0;
1564 1608
1565 1609 ASSERT(MUTEX_HELD(&ps->ps_lock));
1566 1610 #ifdef DEBUG
1567 1611 checkpolldat(ps);
1568 1612 #endif
1569 1613 pcsp = &ps->ps_pcacheset[which];
1570 1614 old_nfds = pcsp->pcs_nfds;
1571 1615 common = (nfds > old_nfds) ? old_nfds : nfds;
1572 1616 if (nfds != old_nfds) {
1573 1617 /*
1574 1618 * the length of poll list has changed. allocate a new
1575 1619 * pollfd list.
1576 1620 */
1577 1621 newlist = kmem_alloc(nfds * sizeof (pollfd_t), KM_SLEEP);
1578 1622 bcopy(current, newlist, sizeof (pollfd_t) * nfds);
1579 1623 }
1580 1624 /*
1581 1625 * Compare the overlapping part of the current fd list with the
1582 1626 * cached one. Whenever a difference is found, resolve it.
1583 1627 * The comparison is done on the current poll list and the
1584 1628 * cached list. But we may be setting up the newlist to be the
1585 1629 * cached list for next poll.
1586 1630 */
1587 1631 cached = pcsp->pcs_pollfd;
1588 1632 remain = common;
1589 1633
1590 1634 while (count < common) {
1591 1635 int tmpfd;
1592 1636 pollfd_t *np;
1593 1637
1594 1638 np = (newlist != NULL) ? &newlist[count] : NULL;
1595 1639 offset = pcacheset_cmp(¤t[count], &cached[count], np,
1596 1640 remain);
1597 1641 /*
1598 1642 * Collect stats. If lists are completed the first time,
1599 1643 * it's a hit. Otherwise, it's a partial hit or miss.
1600 1644 */
1601 1645 if ((count == 0) && (offset == common)) {
1602 1646 pollstats.pollcachehit.value.ui64++;
1603 1647 } else {
1604 1648 mismatch++;
1605 1649 }
1606 1650 count += offset;
1607 1651 if (offset < remain) {
1608 1652 ASSERT(count < common);
1609 1653 ASSERT((current[count].fd != cached[count].fd) ||
1610 1654 (current[count].events != cached[count].events));
1611 1655 /*
1612 1656 * Filter out invalid events.
1613 1657 */
1614 1658 if (current[count].events & ~VALID_POLL_EVENTS) {
1615 1659 if (newlist != NULL) {
1616 1660 newlist[count].events =
1617 1661 current[count].events &=
1618 1662 VALID_POLL_EVENTS;
1619 1663 } else {
1620 1664 current[count].events &=
1621 1665 VALID_POLL_EVENTS;
1622 1666 }
1623 1667 }
1624 1668 /*
1625 1669 * when resolving a difference, we always remove the
1626 1670 * fd from cache before inserting one into cache.
1627 1671 */
1628 1672 if (cached[count].fd >= 0) {
1629 1673 tmpfd = cached[count].fd;
1630 1674 if (pcache_delete_fd(ps, tmpfd, count, which,
1631 1675 (uint_t)cached[count].events)) {
1632 1676 /*
1633 1677 * This should be rare but needed for
1634 1678 * correctness.
1635 1679 *
1636 1680 * The first appearance in cached list
1637 1681 * is being "turned off". The same fd
1638 1682 * appear more than once in the cached
1639 1683 * poll list. Find the next one on the
1640 1684 * list and update the cached
1641 1685 * xf_position field.
1642 1686 */
1643 1687 for (i = count + 1; i < old_nfds; i++) {
1644 1688 if (cached[i].fd == tmpfd) {
1645 1689 pcache_update_xref(pcp,
1646 1690 tmpfd, (ssize_t)i,
1647 1691 which);
1648 1692 break;
1649 1693 }
1650 1694 }
1651 1695 ASSERT(i <= old_nfds);
1652 1696 }
1653 1697 /*
1654 1698 * In case a new cache list is allocated,
1655 1699 * need to keep both cache lists in sync
1656 1700 * b/c the new one can be freed if we have
1657 1701 * an error later.
1658 1702 */
1659 1703 cached[count].fd = -1;
1660 1704 if (newlist != NULL) {
1661 1705 newlist[count].fd = -1;
1662 1706 }
1663 1707 }
1664 1708 if ((tmpfd = current[count].fd) >= 0) {
1665 1709 /*
1666 1710 * add to the cached fd tbl and bitmap.
1667 1711 */
1668 1712 if ((fp = getf(tmpfd)) == NULL) {
1669 1713 current[count].revents = POLLNVAL;
1670 1714 if (newlist != NULL) {
1671 1715 newlist[count].fd = -1;
1672 1716 }
1673 1717 cached[count].fd = -1;
1674 1718 fdcnt++;
1675 1719 } else {
1676 1720 /*
1677 1721 * Here we don't care about the
1678 1722 * fdcnt. We will examine the bitmap
1679 1723 * later and pick up the correct
1680 1724 * fdcnt there. So we never bother
1681 1725 * to check value of 'cnt'.
1682 1726 */
1683 1727 error = pcache_insert(ps, fp,
1684 1728 ¤t[count], &cnt,
1685 1729 (ssize_t)count, which);
1686 1730 /*
1687 1731 * if no error, we want to do releasef
1688 1732 * after we updated cache poll list
1689 1733 * entry so that close() won't race
1690 1734 * us.
1691 1735 */
1692 1736 if (error) {
1693 1737 /*
1694 1738 * If we encountered an error,
1695 1739 * we have invalidated an
1696 1740 * entry in cached poll list
1697 1741 * (in pcache_delete_fd() above)
1698 1742 * but failed to add one here.
1699 1743 * This is OK b/c what's in the
1700 1744 * cached list is consistent
1701 1745 * with content of cache.
1702 1746 * It will not have any ill
1703 1747 * effect on next poll().
1704 1748 */
1705 1749 releasef(tmpfd);
1706 1750 if (newlist != NULL) {
1707 1751 kmem_free(newlist,
1708 1752 nfds *
1709 1753 sizeof (pollfd_t));
1710 1754 }
1711 1755 return (error);
1712 1756 }
1713 1757 /*
1714 1758 * If we have allocated a new(temp)
1715 1759 * cache list, we need to keep both
1716 1760 * in sync b/c the new one can be freed
1717 1761 * if we have an error later.
1718 1762 */
1719 1763 if (newlist != NULL) {
1720 1764 newlist[count].fd =
1721 1765 current[count].fd;
1722 1766 newlist[count].events =
1723 1767 current[count].events;
1724 1768 }
1725 1769 cached[count].fd = current[count].fd;
1726 1770 cached[count].events =
1727 1771 current[count].events;
1728 1772 releasef(tmpfd);
1729 1773 }
1730 1774 } else {
1731 1775 current[count].revents = 0;
1732 1776 }
1733 1777 count++;
1734 1778 remain = common - count;
1735 1779 }
1736 1780 }
1737 1781 if (mismatch != 0) {
1738 1782 if (mismatch == common) {
1739 1783 pollstats.pollcachemiss.value.ui64++;
1740 1784 } else {
1741 1785 pollstats.pollcachephit.value.ui64++;
1742 1786 }
1743 1787 }
1744 1788 /*
1745 1789 * take care of the non overlapping part of a list
1746 1790 */
1747 1791 if (nfds > old_nfds) {
1748 1792 ASSERT(newlist != NULL);
1749 1793 for (i = old_nfds; i < nfds; i++) {
1750 1794 /* filter out invalid events */
1751 1795 if (current[i].events & ~VALID_POLL_EVENTS) {
1752 1796 newlist[i].events = current[i].events =
1753 1797 current[i].events & VALID_POLL_EVENTS;
1754 1798 }
1755 1799 if ((fd = current[i].fd) < 0) {
1756 1800 current[i].revents = 0;
1757 1801 continue;
1758 1802 }
1759 1803 /*
1760 1804 * add to the cached fd tbl and bitmap.
1761 1805 */
1762 1806 if ((fp = getf(fd)) == NULL) {
1763 1807 current[i].revents = POLLNVAL;
1764 1808 newlist[i].fd = -1;
1765 1809 fdcnt++;
1766 1810 continue;
1767 1811 }
1768 1812 /*
1769 1813 * Here we don't care about the
1770 1814 * fdcnt. We will examine the bitmap
1771 1815 * later and pick up the correct
1772 1816 * fdcnt there. So we never bother to
1773 1817 * check 'cnt'.
1774 1818 */
1775 1819 error = pcache_insert(ps, fp, ¤t[i], &cnt,
1776 1820 (ssize_t)i, which);
1777 1821 releasef(fd);
1778 1822 if (error) {
1779 1823 /*
1780 1824 * Here we are half way through adding newly
1781 1825 * polled fd. Undo enough to keep the cache
1782 1826 * list consistent with the cache content.
1783 1827 */
1784 1828 pcacheset_remove_list(ps, current, old_nfds,
1785 1829 i, which, 0);
1786 1830 kmem_free(newlist, nfds * sizeof (pollfd_t));
1787 1831 return (error);
1788 1832 }
1789 1833 }
1790 1834 }
1791 1835 if (old_nfds > nfds) {
1792 1836 /*
1793 1837 * remove the fd's which are no longer polled.
1794 1838 */
1795 1839 pcacheset_remove_list(ps, pcsp->pcs_pollfd, nfds, old_nfds,
1796 1840 which, 1);
1797 1841 }
1798 1842 /*
1799 1843 * set difference resolved. update nfds and cachedlist
1800 1844 * in pollstate struct.
1801 1845 */
1802 1846 if (newlist != NULL) {
1803 1847 kmem_free(pcsp->pcs_pollfd, old_nfds * sizeof (pollfd_t));
1804 1848 /*
1805 1849 * By now, the pollfd.revents field should
1806 1850 * all be zeroed.
1807 1851 */
1808 1852 pcsp->pcs_pollfd = newlist;
1809 1853 pcsp->pcs_nfds = nfds;
1810 1854 }
1811 1855 ASSERT(*fdcntp == 0);
1812 1856 *fdcntp = fdcnt;
1813 1857 /*
1814 1858 * By now for every fd in pollfdp, one of the following should be
1815 1859 * true. Otherwise we will miss a polled event.
1816 1860 *
1817 1861 * 1. the bit corresponding to the fd in bitmap is set. So VOP_POLL
1818 1862 * will be called on this fd in next poll.
1819 1863 * 2. the fd is cached in the pcache (i.e. pd_php is set). So
1820 1864 * pollnotify will happen.
1821 1865 */
1822 1866 ASSERT(pollchecksanity(ps, nfds));
1823 1867 /*
1824 1868 * make sure cross reference between cached poll lists and cached
1825 1869 * poll fds are correct.
1826 1870 */
1827 1871 ASSERT(pollcheckxref(ps, which));
1828 1872 /*
1829 1873 * ensure each polldat in pollcache reference a polled fd in
1830 1874 * pollcacheset.
1831 1875 */
1832 1876 #ifdef DEBUG
1833 1877 checkpolldat(ps);
1834 1878 #endif
1835 1879 return (0);
1836 1880 }
1837 1881
1838 1882 #ifdef DEBUG
1839 1883 static int
1840 1884 pollscanrevents(pollcache_t *pcp, pollfd_t *pollfdp, nfds_t nfds)
1841 1885 {
1842 1886 int i;
1843 1887 int reventcnt = 0;
1844 1888
1845 1889 for (i = 0; i < nfds; i++) {
1846 1890 if (pollfdp[i].fd < 0) {
1847 1891 ASSERT(pollfdp[i].revents == 0);
1848 1892 continue;
1849 1893 }
1850 1894 if (pollfdp[i].revents) {
1851 1895 reventcnt++;
1852 1896 }
1853 1897 if (pollfdp[i].revents && (pollfdp[i].revents != POLLNVAL)) {
1854 1898 ASSERT(BT_TEST(pcp->pc_bitmap, pollfdp[i].fd));
1855 1899 }
1856 1900 }
1857 1901 return (reventcnt);
1858 1902 }
1859 1903 #endif /* DEBUG */
1860 1904
1861 1905 /*
1862 1906 * read the bitmap and poll on fds corresponding to the '1' bits. The ps_lock
1863 1907 * is held upon entry.
1864 1908 */
1865 1909 int
1866 1910 pcache_poll(pollfd_t *pollfdp, pollstate_t *ps, nfds_t nfds, int *fdcntp,
1867 1911 int which)
1868 1912 {
1869 1913 int i;
1870 1914 pollcache_t *pcp;
1871 1915 int fd;
1872 1916 int begin, end, done;
1873 1917 pollhead_t *php;
1874 1918 int fdcnt;
1875 1919 int error = 0;
1876 1920 file_t *fp;
1877 1921 polldat_t *pdp;
1878 1922 xref_t *refp;
1879 1923 int entry;
1880 1924
1881 1925 pcp = ps->ps_pcache;
1882 1926 ASSERT(MUTEX_HELD(&ps->ps_lock));
1883 1927 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1884 1928 retry:
1885 1929 done = 0;
1886 1930 begin = 0;
1887 1931 fdcnt = 0;
1888 1932 end = pcp->pc_mapend;
1889 1933 while ((fdcnt < nfds) && !done) {
1890 1934 php = NULL;
1891 1935 /*
1892 1936 * only poll fds which may have events
1893 1937 */
1894 1938 fd = bt_getlowbit(pcp->pc_bitmap, begin, end);
1895 1939 ASSERT(fd <= end);
1896 1940 if (fd >= 0) {
1897 1941 ASSERT(pollcheckrevents(ps, begin, fd, which));
1898 1942 /*
1899 1943 * adjust map pointers for next round
1900 1944 */
1901 1945 if (fd == end) {
1902 1946 done = 1;
1903 1947 } else {
1904 1948 begin = fd + 1;
1905 1949 }
1906 1950 /*
1907 1951 * A bitmap caches poll state information of
1908 1952 * multiple poll lists. Call VOP_POLL only if
1909 1953 * the bit corresponds to an fd in this poll
1910 1954 * list.
1911 1955 */
1912 1956 pdp = pcache_lookup_fd(pcp, fd);
1913 1957 ASSERT(pdp != NULL);
1914 1958 ASSERT(pdp->pd_ref != NULL);
1915 1959 refp = &pdp->pd_ref[which];
1916 1960 if (refp->xf_refcnt == 0)
1917 1961 continue;
1918 1962 entry = refp->xf_position;
1919 1963 ASSERT((entry >= 0) && (entry < nfds));
1920 1964 ASSERT(pollfdp[entry].fd == fd);
1921 1965 /*
1922 1966 * we are in this routine implies that we have
1923 1967 * successfully polled this fd in the past.
1924 1968 * Check to see this fd is closed while we are
1925 1969 * blocked in poll. This ensures that we don't
1926 1970 * miss a close on the fd in the case this fd is
1927 1971 * reused.
1928 1972 */
1929 1973 if (pdp->pd_fp == NULL) {
1930 1974 ASSERT(pdp->pd_count > 0);
1931 1975 pollfdp[entry].revents = POLLNVAL;
1932 1976 fdcnt++;
1933 1977 if (refp->xf_refcnt > 1) {
1934 1978 /*
1935 1979 * this fd appeared multiple time
1936 1980 * in the poll list. Find all of them.
1937 1981 */
1938 1982 for (i = entry + 1; i < nfds; i++) {
1939 1983 if (pollfdp[i].fd == fd) {
1940 1984 pollfdp[i].revents =
1941 1985 POLLNVAL;
1942 1986 fdcnt++;
1943 1987 }
1944 1988 }
1945 1989 }
1946 1990 pcacheset_invalidate(ps, pdp);
1947 1991 continue;
1948 1992 }
1949 1993 /*
1950 1994 * We can be here polling a device that is being
1951 1995 * closed (i.e. the file pointer is set to NULL,
1952 1996 * but pollcacheclean has not happened yet).
1953 1997 */
1954 1998 if ((fp = getf(fd)) == NULL) {
1955 1999 pollfdp[entry].revents = POLLNVAL;
1956 2000 fdcnt++;
1957 2001 if (refp->xf_refcnt > 1) {
1958 2002 /*
1959 2003 * this fd appeared multiple time
1960 2004 * in the poll list. Find all of them.
1961 2005 */
1962 2006 for (i = entry + 1; i < nfds; i++) {
1963 2007 if (pollfdp[i].fd == fd) {
1964 2008 pollfdp[i].revents =
1965 2009 POLLNVAL;
1966 2010 fdcnt++;
1967 2011 }
1968 2012 }
1969 2013 }
1970 2014 continue;
1971 2015 }
1972 2016 ASSERT(pdp->pd_fp == fp);
1973 2017 ASSERT(infpollinfo(fd));
1974 2018 /*
1975 2019 * Since we no longer hold poll head lock across
1976 2020 * VOP_POLL, pollunlock logic can be simplifed.
1977 2021 */
1978 2022 ASSERT(pdp->pd_php == NULL ||
1979 2023 MUTEX_NOT_HELD(PHLOCK(pdp->pd_php)));
1980 2024 /*
1981 2025 * underlying file systems may set a "pollpending"
1982 2026 * flag when it sees the poll may block. Pollwakeup()
1983 2027 * is called by wakeup thread if pollpending is set.
1984 2028 * Pass a 0 fdcnt so that the underlying file system
1985 2029 * will set the "pollpending" flag set when there is
1986 2030 * no polled events.
1987 2031 *
1988 2032 * Use pollfdp[].events for actual polling because
1989 2033 * the pd_events is union of all cached poll events
1990 2034 * on this fd. The events parameter also affects
1991 2035 * how the polled device sets the "poll pending"
1992 2036 * flag.
1993 2037 */
1994 2038 ASSERT(curthread->t_pollcache == NULL);
1995 2039 error = VOP_POLL(fp->f_vnode, pollfdp[entry].events, 0,
1996 2040 &pollfdp[entry].revents, &php, NULL);
1997 2041 /*
1998 2042 * releasef after completely done with this cached
1999 2043 * poll entry. To prevent close() coming in to clear
2000 2044 * this entry.
2001 2045 */
2002 2046 if (error) {
2003 2047 releasef(fd);
2004 2048 break;
2005 2049 }
2006 2050 /*
2007 2051 * layered devices (e.g. console driver)
2008 2052 * may change the vnode and thus the pollhead
2009 2053 * pointer out from underneath us.
2010 2054 */
2011 2055 if (php != NULL && pdp->pd_php != NULL &&
2012 2056 php != pdp->pd_php) {
2013 2057 releasef(fd);
2014 2058 pollhead_delete(pdp->pd_php, pdp);
2015 2059 pdp->pd_php = php;
2016 2060 pollhead_insert(php, pdp);
2017 2061 /*
2018 2062 * We could have missed a wakeup on the new
2019 2063 * target device. Make sure the new target
2020 2064 * gets polled once.
2021 2065 */
2022 2066 BT_SET(pcp->pc_bitmap, fd);
2023 2067 goto retry;
2024 2068 }
2025 2069
2026 2070 if (pollfdp[entry].revents) {
2027 2071 ASSERT(refp->xf_refcnt >= 1);
2028 2072 fdcnt++;
2029 2073 if (refp->xf_refcnt > 1) {
2030 2074 /*
2031 2075 * this fd appeared multiple time
2032 2076 * in the poll list. This is rare but
2033 2077 * we have to look at all of them for
2034 2078 * correctness.
2035 2079 */
2036 2080 error = plist_chkdupfd(fp, pdp, ps,
2037 2081 pollfdp, entry, &fdcnt);
2038 2082 if (error > 0) {
2039 2083 releasef(fd);
2040 2084 break;
2041 2085 }
2042 2086 if (error < 0) {
2043 2087 goto retry;
2044 2088 }
2045 2089 }
2046 2090 releasef(fd);
2047 2091 } else {
2048 2092 /*
2049 2093 * VOP_POLL didn't return any revents. We can
2050 2094 * clear the bit in bitmap only if we have the
2051 2095 * pollhead ptr cached and no other cached
2052 2096 * entry is polling different events on this fd.
2053 2097 * VOP_POLL may have dropped the ps_lock. Make
2054 2098 * sure pollwakeup has not happened before clear
2055 2099 * the bit.
2056 2100 */
2057 2101 if ((pdp->pd_php != NULL) &&
2058 2102 (pollfdp[entry].events == pdp->pd_events) &&
2059 2103 ((pcp->pc_flag & PC_POLLWAKE) == 0)) {
2060 2104 BT_CLEAR(pcp->pc_bitmap, fd);
2061 2105 }
2062 2106 /*
2063 2107 * if the fd can be cached now but not before,
2064 2108 * do it now.
2065 2109 */
2066 2110 if ((pdp->pd_php == NULL) && (php != NULL)) {
2067 2111 pdp->pd_php = php;
2068 2112 pollhead_insert(php, pdp);
2069 2113 /*
2070 2114 * We are inserting a polldat struct for
2071 2115 * the first time. We may have missed a
2072 2116 * wakeup on this device. Re-poll once.
2073 2117 * This should be a rare event.
2074 2118 */
2075 2119 releasef(fd);
2076 2120 goto retry;
2077 2121 }
2078 2122 if (refp->xf_refcnt > 1) {
2079 2123 /*
2080 2124 * this fd appeared multiple time
2081 2125 * in the poll list. This is rare but
2082 2126 * we have to look at all of them for
2083 2127 * correctness.
2084 2128 */
2085 2129 error = plist_chkdupfd(fp, pdp, ps,
2086 2130 pollfdp, entry, &fdcnt);
2087 2131 if (error > 0) {
2088 2132 releasef(fd);
2089 2133 break;
2090 2134 }
2091 2135 if (error < 0) {
2092 2136 goto retry;
2093 2137 }
2094 2138 }
2095 2139 releasef(fd);
2096 2140 }
2097 2141 } else {
2098 2142 done = 1;
2099 2143 ASSERT(pollcheckrevents(ps, begin, end + 1, which));
2100 2144 }
2101 2145 }
2102 2146 if (!error) {
2103 2147 ASSERT(*fdcntp + fdcnt == pollscanrevents(pcp, pollfdp, nfds));
2104 2148 *fdcntp += fdcnt;
2105 2149 }
2106 2150 return (error);
2107 2151 }
2108 2152
2109 2153 /*
2110 2154 * Going through the poll list without much locking. Poll all fds and
2111 2155 * cache all valid fds in the pollcache.
2112 2156 */
2113 2157 int
2114 2158 pcacheset_cache_list(pollstate_t *ps, pollfd_t *fds, int *fdcntp, int which)
2115 2159 {
2116 2160 pollfd_t *pollfdp = ps->ps_pollfd;
2117 2161 pollcacheset_t *pcacheset = ps->ps_pcacheset;
2118 2162 pollfd_t *newfdlist;
2119 2163 int i;
2120 2164 int fd;
2121 2165 file_t *fp;
2122 2166 int error = 0;
2123 2167
2124 2168 ASSERT(MUTEX_HELD(&ps->ps_lock));
2125 2169 ASSERT(which < ps->ps_nsets);
2126 2170 ASSERT(pcacheset != NULL);
2127 2171 ASSERT(pcacheset[which].pcs_pollfd == NULL);
2128 2172 newfdlist = kmem_alloc(ps->ps_nfds * sizeof (pollfd_t), KM_SLEEP);
2129 2173 /*
2130 2174 * cache the new poll list in pollcachset.
2131 2175 */
2132 2176 bcopy(pollfdp, newfdlist, sizeof (pollfd_t) * ps->ps_nfds);
2133 2177
2134 2178 pcacheset[which].pcs_pollfd = newfdlist;
2135 2179 pcacheset[which].pcs_nfds = ps->ps_nfds;
2136 2180 pcacheset[which].pcs_usradr = (uintptr_t)fds;
2137 2181
2138 2182 /*
2139 2183 * We have saved a copy of current poll fd list in one pollcacheset.
2140 2184 * The 'revents' field of the new list is not yet set to 0. Loop
2141 2185 * through the new list just to do that is expensive. We do that
2142 2186 * while polling the list.
2143 2187 */
2144 2188 for (i = 0; i < ps->ps_nfds; i++) {
2145 2189 fd = pollfdp[i].fd;
2146 2190 /*
2147 2191 * We also filter out the illegal poll events in the event
2148 2192 * field for the cached poll list/set.
2149 2193 */
2150 2194 if (pollfdp[i].events & ~VALID_POLL_EVENTS) {
2151 2195 newfdlist[i].events = pollfdp[i].events =
2152 2196 pollfdp[i].events & VALID_POLL_EVENTS;
2153 2197 }
2154 2198 if (fd < 0) {
2155 2199 pollfdp[i].revents = 0;
2156 2200 continue;
2157 2201 }
2158 2202 if ((fp = getf(fd)) == NULL) {
2159 2203 pollfdp[i].revents = POLLNVAL;
2160 2204 /*
2161 2205 * invalidate this cache entry in the cached poll list
2162 2206 */
2163 2207 newfdlist[i].fd = -1;
2164 2208 (*fdcntp)++;
2165 2209 continue;
2166 2210 }
2167 2211 /*
2168 2212 * cache this fd.
2169 2213 */
2170 2214 error = pcache_insert(ps, fp, &pollfdp[i], fdcntp, (ssize_t)i,
2171 2215 which);
2172 2216 releasef(fd);
2173 2217 if (error) {
2174 2218 /*
2175 2219 * Here we are half way through caching a new
2176 2220 * poll list. Undo every thing.
2177 2221 */
2178 2222 pcacheset_remove_list(ps, pollfdp, 0, i, which, 0);
2179 2223 kmem_free(newfdlist, ps->ps_nfds * sizeof (pollfd_t));
2180 2224 pcacheset[which].pcs_pollfd = NULL;
2181 2225 pcacheset[which].pcs_usradr = NULL;
2182 2226 break;
2183 2227 }
2184 2228 }
2185 2229 return (error);
2186 2230 }
2187 2231
2188 2232 /*
2189 2233 * called by pollcacheclean() to set the fp NULL. It also sets polled events
2190 2234 * in pcacheset entries to a special events 'POLLCLOSED'. Do a pollwakeup to
2191 2235 * wake any sleeping poller, then remove the polldat from the driver.
2192 2236 * The routine is called with ps_pcachelock held.
2193 2237 */
2194 2238 void
2195 2239 pcache_clean_entry(pollstate_t *ps, int fd)
2196 2240 {
2197 2241 pollcache_t *pcp;
2198 2242 polldat_t *pdp;
2199 2243 int i;
2200 2244
2201 2245 ASSERT(ps != NULL);
2202 2246 ASSERT(MUTEX_HELD(&ps->ps_lock));
2203 2247 pcp = ps->ps_pcache;
2204 2248 ASSERT(pcp);
2205 2249 pdp = pcache_lookup_fd(pcp, fd);
2206 2250 ASSERT(pdp != NULL);
2207 2251 /*
2208 2252 * the corresponding fpollinfo in fi_list has been removed by
2209 2253 * a close on this fd. Reset the cached fp ptr here.
2210 2254 */
2211 2255 pdp->pd_fp = NULL;
2212 2256 /*
2213 2257 * XXX - This routine also touches data in pcacheset struct.
2214 2258 *
2215 2259 * set the event in cached poll lists to POLLCLOSED. This invalidate
2216 2260 * the cached poll fd entry in that poll list, which will force a
2217 2261 * removal of this cached entry in next poll(). The cleanup is done
2218 2262 * at the removal time.
2219 2263 */
2220 2264 ASSERT(pdp->pd_ref != NULL);
2221 2265 for (i = 0; i < ps->ps_nsets; i++) {
2222 2266 xref_t *refp;
2223 2267 pollcacheset_t *pcsp;
2224 2268
2225 2269 refp = &pdp->pd_ref[i];
2226 2270 if (refp->xf_refcnt) {
2227 2271 ASSERT(refp->xf_position >= 0);
2228 2272 pcsp = &ps->ps_pcacheset[i];
2229 2273 if (refp->xf_refcnt == 1) {
2230 2274 pcsp->pcs_pollfd[refp->xf_position].events =
2231 2275 (short)POLLCLOSED;
2232 2276 }
2233 2277 if (refp->xf_refcnt > 1) {
2234 2278 int j;
2235 2279 /*
2236 2280 * mark every matching entry in pcs_pollfd
2237 2281 */
2238 2282 for (j = refp->xf_position;
2239 2283 j < pcsp->pcs_nfds; j++) {
2240 2284 if (pcsp->pcs_pollfd[j].fd == fd) {
2241 2285 pcsp->pcs_pollfd[j].events =
2242 2286 (short)POLLCLOSED;
2243 2287 }
2244 2288 }
2245 2289 }
2246 2290 }
2247 2291 }
2248 2292 if (pdp->pd_php) {
2249 2293 pollwakeup(pdp->pd_php, POLLHUP);
2250 2294 pollhead_delete(pdp->pd_php, pdp);
2251 2295 pdp->pd_php = NULL;
2252 2296 }
2253 2297 }
2254 2298
2255 2299 void
2256 2300 pcache_wake_parents(pollcache_t *pcp)
2257 2301 {
2258 2302 pcachelink_t *pl, *pln;
2259 2303
2260 2304 ASSERT(MUTEX_HELD(&pcp->pc_lock));
2261 2305
2262 2306 for (pl = pcp->pc_parents; pl != NULL; pl = pln) {
2263 2307 mutex_enter(&pl->pcl_lock);
2264 2308 if (pl->pcl_state == PCL_VALID) {
2265 2309 ASSERT(pl->pcl_parent_pc != NULL);
2266 2310 cv_broadcast(&pl->pcl_parent_pc->pc_cv);
2267 2311 }
2268 2312 pln = pl->pcl_parent_next;
2269 2313 mutex_exit(&pl->pcl_lock);
2270 2314 }
2271 2315 }
2272 2316
2273 2317 /*
2274 2318 * Initialize thread pollstate structure.
2275 2319 * It will persist for the life of the thread, until it calls pollcleanup().
2276 2320 */
2277 2321 pollstate_t *
2278 2322 pollstate_create()
2279 2323 {
2280 2324 pollstate_t *ps = curthread->t_pollstate;
2281 2325
2282 2326 if (ps == NULL) {
2283 2327 /*
2284 2328 * This is the first time this thread has ever polled, so we
2285 2329 * have to create its pollstate structure.
2286 2330 */
2287 2331 ps = kmem_zalloc(sizeof (pollstate_t), KM_SLEEP);
2288 2332 ps->ps_nsets = POLLFDSETS;
2289 2333 ps->ps_pcacheset = pcacheset_create(ps->ps_nsets);
2290 2334 curthread->t_pollstate = ps;
2291 2335 } else {
2292 2336 ASSERT(ps->ps_depth == 0);
2293 2337 ASSERT(ps->ps_flags == 0);
2294 2338 ASSERT(ps->ps_pc_stack[0] == 0);
2295 2339 }
2296 2340 return (ps);
2297 2341 }
2298 2342
2299 2343 void
2300 2344 pollstate_destroy(pollstate_t *ps)
2301 2345 {
2302 2346 if (ps->ps_pollfd != NULL) {
2303 2347 kmem_free(ps->ps_pollfd, ps->ps_nfds * sizeof (pollfd_t));
2304 2348 ps->ps_pollfd = NULL;
2305 2349 }
2306 2350 if (ps->ps_pcache != NULL) {
2307 2351 pcache_destroy(ps->ps_pcache);
2308 2352 ps->ps_pcache = NULL;
2309 2353 }
2310 2354 pcacheset_destroy(ps->ps_pcacheset, ps->ps_nsets);
2311 2355 ps->ps_pcacheset = NULL;
2312 2356 if (ps->ps_dpbuf != NULL) {
2313 2357 kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize);
2314 2358 ps->ps_dpbuf = NULL;
2315 2359 }
2316 2360 mutex_destroy(&ps->ps_lock);
2317 2361 kmem_free(ps, sizeof (pollstate_t));
2318 2362 }
2319 2363
2320 2364 static int
2321 2365 pollstate_contend(pollstate_t *ps, pollcache_t *pcp)
2322 2366 {
2323 2367 pollstate_t *rem, *next;
2324 2368 pollcache_t *desired_pc;
2325 2369 int result = 0, depth_total;
2326 2370
2327 2371 mutex_enter(&pollstate_contenders_lock);
2328 2372 /*
2329 2373 * There is a small chance that the pollcache of interest became
2330 2374 * available while we were waiting on the contenders lock.
2331 2375 */
2332 2376 if (mutex_tryenter(&pcp->pc_lock) != 0) {
2333 2377 goto out;
2334 2378 }
2335 2379
2336 2380 /*
2337 2381 * Walk the list of contended pollstates, searching for evidence of a
2338 2382 * deadlock condition.
2339 2383 */
2340 2384 depth_total = ps->ps_depth;
2341 2385 desired_pc = pcp;
2342 2386 for (rem = pollstate_contenders; rem != NULL; rem = next) {
2343 2387 int i, j;
2344 2388 next = rem->ps_contend_nextp;
2345 2389
2346 2390 /* Is this pollstate holding the pollcache of interest? */
2347 2391 for (i = 0; i < rem->ps_depth; i++) {
2348 2392 if (rem->ps_pc_stack[i] != desired_pc) {
2349 2393 continue;
2350 2394 }
2351 2395
2352 2396 /*
2353 2397 * The remote pollstate holds the pollcache lock we
2354 2398 * desire. If it is waiting on a pollcache we hold,
2355 2399 * then we can report the obvious deadlock.
2356 2400 */
2357 2401 ASSERT(rem->ps_contend_pc != NULL);
2358 2402 for (j = 0; j < ps->ps_depth; j++) {
2359 2403 if (rem->ps_contend_pc == ps->ps_pc_stack[j]) {
2360 2404 rem->ps_flags |= POLLSTATE_STALEMATE;
2361 2405 result = -1;
2362 2406 goto out;
2363 2407 }
2364 2408 }
2365 2409
2366 2410 /*
2367 2411 * The remote pollstate is not blocking on a pollcache
2368 2412 * which would deadlock against us. That pollcache
2369 2413 * may, however, be held by a pollstate which would
2370 2414 * result in a deadlock.
2371 2415 *
2372 2416 * To detect such a condition, we continue walking
2373 2417 * through the list using the pollcache blocking the
2374 2418 * remote thread as our new search target.
2375 2419 *
2376 2420 * Return to the front of pollstate_contenders since it
2377 2421 * is not ordered to guarantee complete dependency
2378 2422 * traversal. The below depth tracking places an upper
2379 2423 * bound on iterations.
2380 2424 */
2381 2425 desired_pc = rem->ps_contend_pc;
2382 2426 next = pollstate_contenders;
2383 2427
2384 2428 /*
2385 2429 * The recursion depth of the remote pollstate is used
2386 2430 * to calculate a final depth for the local /dev/poll
2387 2431 * recursion, since those locks will be acquired
2388 2432 * eventually. If that value exceeds the defined
2389 2433 * limit, we can report the failure now instead of
2390 2434 * recursing to that failure depth.
2391 2435 */
2392 2436 depth_total += (rem->ps_depth - i);
2393 2437 if (depth_total >= POLLMAXDEPTH) {
2394 2438 result = -1;
2395 2439 goto out;
2396 2440 }
2397 2441 }
2398 2442 }
2399 2443
2400 2444 /*
2401 2445 * No deadlock partner was found. The only course of action is to
2402 2446 * record ourself as a contended pollstate and wait for the pollcache
2403 2447 * mutex to become available.
2404 2448 */
2405 2449 ps->ps_contend_pc = pcp;
2406 2450 ps->ps_contend_nextp = pollstate_contenders;
2407 2451 ps->ps_contend_pnextp = &pollstate_contenders;
2408 2452 if (pollstate_contenders != NULL) {
2409 2453 pollstate_contenders->ps_contend_pnextp =
2410 2454 &ps->ps_contend_nextp;
2411 2455 }
2412 2456 pollstate_contenders = ps;
2413 2457
2414 2458 mutex_exit(&pollstate_contenders_lock);
2415 2459 mutex_enter(&pcp->pc_lock);
2416 2460 mutex_enter(&pollstate_contenders_lock);
2417 2461
2418 2462 /*
2419 2463 * Our acquisition of the pollcache mutex may be due to another thread
2420 2464 * giving up in the face of deadlock with us. If that is the case,
2421 2465 * we too should report the failure.
2422 2466 */
2423 2467 if ((ps->ps_flags & POLLSTATE_STALEMATE) != 0) {
2424 2468 result = -1;
2425 2469 ps->ps_flags &= ~POLLSTATE_STALEMATE;
2426 2470 mutex_exit(&pcp->pc_lock);
2427 2471 }
2428 2472
2429 2473 /* Remove ourself from the contenders list. */
2430 2474 if (ps->ps_contend_nextp != NULL) {
2431 2475 ps->ps_contend_nextp->ps_contend_pnextp =
2432 2476 ps->ps_contend_pnextp;
2433 2477 }
2434 2478 *ps->ps_contend_pnextp = ps->ps_contend_nextp;
2435 2479 ps->ps_contend_pc = NULL;
2436 2480 ps->ps_contend_nextp = NULL;
2437 2481 ps->ps_contend_pnextp = NULL;
2438 2482
2439 2483 out:
2440 2484 mutex_exit(&pollstate_contenders_lock);
2441 2485 return (result);
2442 2486 }
2443 2487
2444 2488 int
2445 2489 pollstate_enter(pollcache_t *pcp)
2446 2490 {
2447 2491 pollstate_t *ps = curthread->t_pollstate;
2448 2492 int i;
2449 2493
2450 2494 if (ps == NULL) {
2451 2495 /*
2452 2496 * The thread pollstate may not be initialized if VOP_POLL is
2453 2497 * called on a recursion-enabled /dev/poll handle from outside
2454 2498 * the poll() or /dev/poll codepaths.
2455 2499 */
2456 2500 return (PSE_FAIL_POLLSTATE);
2457 2501 }
2458 2502 if (ps->ps_depth >= POLLMAXDEPTH) {
2459 2503 return (PSE_FAIL_DEPTH);
2460 2504 }
2461 2505 /*
2462 2506 * Check the desired pollcache against pollcaches we already have
2463 2507 * locked. Such a loop is the most simple deadlock scenario.
2464 2508 */
2465 2509 for (i = 0; i < ps->ps_depth; i++) {
2466 2510 if (ps->ps_pc_stack[i] == pcp) {
2467 2511 return (PSE_FAIL_LOOP);
2468 2512 }
2469 2513 }
2470 2514 ASSERT(ps->ps_pc_stack[i] == NULL);
2471 2515
2472 2516 if (ps->ps_depth == 0) {
2473 2517 /* Locking initial the pollcache requires no caution */
2474 2518 mutex_enter(&pcp->pc_lock);
2475 2519 } else if (mutex_tryenter(&pcp->pc_lock) == 0) {
2476 2520 if (pollstate_contend(ps, pcp) != 0) {
2477 2521 /* This pollcache cannot safely be locked. */
2478 2522 return (PSE_FAIL_DEADLOCK);
2479 2523 }
2480 2524 }
2481 2525
2482 2526 ps->ps_pc_stack[ps->ps_depth++] = pcp;
2483 2527 return (PSE_SUCCESS);
2484 2528 }
2485 2529
2486 2530 void
2487 2531 pollstate_exit(pollcache_t *pcp)
2488 2532 {
2489 2533 pollstate_t *ps = curthread->t_pollstate;
2490 2534
2491 2535 VERIFY(ps != NULL);
2492 2536 VERIFY(ps->ps_pc_stack[ps->ps_depth - 1] == pcp);
2493 2537
2494 2538 mutex_exit(&pcp->pc_lock);
2495 2539 ps->ps_pc_stack[--ps->ps_depth] = NULL;
2496 2540 VERIFY(ps->ps_depth >= 0);
2497 2541 }
2498 2542
2499 2543
2500 2544 /*
2501 2545 * We are holding the appropriate uf_lock entering this routine.
2502 2546 * Bump up the ps_busy count to prevent the thread from exiting.
2503 2547 */
2504 2548 void
2505 2549 pollblockexit(fpollinfo_t *fpip)
2506 2550 {
2507 2551 for (; fpip; fpip = fpip->fp_next) {
2508 2552 pollcache_t *pcp = fpip->fp_thread->t_pollstate->ps_pcache;
2509 2553
2510 2554 mutex_enter(&pcp->pc_no_exit);
2511 2555 pcp->pc_busy++; /* prevents exit()'s */
2512 2556 mutex_exit(&pcp->pc_no_exit);
2513 2557 }
2514 2558 }
2515 2559
2516 2560 /*
2517 2561 * Complete phase 2 of cached poll fd cleanup. Call pcache_clean_entry to mark
2518 2562 * the pcacheset events field POLLCLOSED to force the next poll() to remove
2519 2563 * this cache entry. We can't clean the polldat entry clean up here because
2520 2564 * lwp block in poll() needs the info to return. Wakeup anyone blocked in
2521 2565 * poll and let exiting lwp go. No lock is help upon entry. So it's OK for
2522 2566 * pcache_clean_entry to call pollwakeup().
2523 2567 */
2524 2568 void
2525 2569 pollcacheclean(fpollinfo_t *fip, int fd)
2526 2570 {
2527 2571 struct fpollinfo *fpip, *fpip2;
2528 2572
2529 2573 fpip = fip;
2530 2574 while (fpip) {
2531 2575 pollstate_t *ps = fpip->fp_thread->t_pollstate;
2532 2576 pollcache_t *pcp = ps->ps_pcache;
2533 2577
2534 2578 mutex_enter(&ps->ps_lock);
2535 2579 pcache_clean_entry(ps, fd);
2536 2580 mutex_exit(&ps->ps_lock);
2537 2581 mutex_enter(&pcp->pc_no_exit);
2538 2582 pcp->pc_busy--;
2539 2583 if (pcp->pc_busy == 0) {
2540 2584 /*
2541 2585 * Wakeup the thread waiting in
2542 2586 * thread_exit().
2543 2587 */
2544 2588 cv_signal(&pcp->pc_busy_cv);
2545 2589 }
2546 2590 mutex_exit(&pcp->pc_no_exit);
2547 2591
2548 2592 fpip2 = fpip;
2549 2593 fpip = fpip->fp_next;
2550 2594 kmem_free(fpip2, sizeof (fpollinfo_t));
2551 2595 }
2552 2596 }
2553 2597
2554 2598 /*
2555 2599 * one of the cache line's counter is wrapping around. Reset all cache line
2556 2600 * counters to zero except one. This is simplistic, but probably works
2557 2601 * effectively.
2558 2602 */
2559 2603 void
2560 2604 pcacheset_reset_count(pollstate_t *ps, int index)
2561 2605 {
2562 2606 int i;
2563 2607
2564 2608 ASSERT(MUTEX_HELD(&ps->ps_lock));
2565 2609 for (i = 0; i < ps->ps_nsets; i++) {
2566 2610 if (ps->ps_pcacheset[i].pcs_pollfd != NULL) {
2567 2611 ps->ps_pcacheset[i].pcs_count = 0;
2568 2612 }
2569 2613 }
2570 2614 ps->ps_pcacheset[index].pcs_count = 1;
2571 2615 }
2572 2616
2573 2617 /*
2574 2618 * this routine implements poll cache list replacement policy.
2575 2619 * It is currently choose the "least used".
2576 2620 */
2577 2621 int
2578 2622 pcacheset_replace(pollstate_t *ps)
2579 2623 {
2580 2624 int i;
2581 2625 int index = 0;
2582 2626
2583 2627 ASSERT(MUTEX_HELD(&ps->ps_lock));
2584 2628 for (i = 1; i < ps->ps_nsets; i++) {
2585 2629 if (ps->ps_pcacheset[index].pcs_count >
2586 2630 ps->ps_pcacheset[i].pcs_count) {
2587 2631 index = i;
2588 2632 }
2589 2633 }
2590 2634 ps->ps_pcacheset[index].pcs_count = 0;
2591 2635 return (index);
2592 2636 }
2593 2637
2594 2638 /*
2595 2639 * this routine is called by strclose to remove remaining polldat struct on
2596 2640 * the pollhead list of the device being closed. There are two reasons as why
2597 2641 * the polldat structures still remain on the pollhead list:
2598 2642 *
2599 2643 * (1) The layered device(e.g.the console driver).
2600 2644 * In this case, the existence of a polldat implies that the thread putting
2601 2645 * the polldat on this list has not exited yet. Before the thread exits, it
2602 2646 * will have to hold this pollhead lock to remove the polldat. So holding the
2603 2647 * pollhead lock here effectively prevents the thread which put the polldat
2604 2648 * on this list from exiting.
2605 2649 *
2606 2650 * (2) /dev/poll.
2607 2651 * When a polled fd is cached in /dev/poll, its polldat will remain on the
2608 2652 * pollhead list if the process has not done a POLLREMOVE before closing the
2609 2653 * polled fd. We just unlink it here.
2610 2654 */
2611 2655 void
2612 2656 pollhead_clean(pollhead_t *php)
2613 2657 {
2614 2658 polldat_t *pdp;
2615 2659
2616 2660 /*
2617 2661 * In case(1), while we must prevent the thread in question from
2618 2662 * exiting, we must also obey the proper locking order, i.e.
2619 2663 * (ps_lock -> phlock).
2620 2664 */
2621 2665 PH_ENTER(php);
2622 2666 while (php->ph_list != NULL) {
2623 2667 pollstate_t *ps;
2624 2668 pollcache_t *pcp;
2625 2669
2626 2670 pdp = php->ph_list;
2627 2671 ASSERT(pdp->pd_php == php);
2628 2672 if (pdp->pd_thread == NULL) {
2629 2673 /*
2630 2674 * This is case(2). Since the ph_lock is sufficient
2631 2675 * to synchronize this lwp with any other /dev/poll
2632 2676 * lwp, just unlink the polldat.
2633 2677 */
2634 2678 php->ph_list = pdp->pd_next;
2635 2679 pdp->pd_php = NULL;
2636 2680 pdp->pd_next = NULL;
2637 2681 continue;
2638 2682 }
2639 2683 ps = pdp->pd_thread->t_pollstate;
2640 2684 ASSERT(ps != NULL);
2641 2685 pcp = pdp->pd_pcache;
2642 2686 ASSERT(pcp != NULL);
2643 2687 mutex_enter(&pcp->pc_no_exit);
2644 2688 pcp->pc_busy++; /* prevents exit()'s */
2645 2689 mutex_exit(&pcp->pc_no_exit);
2646 2690 /*
2647 2691 * Now get the locks in proper order to avoid deadlock.
2648 2692 */
2649 2693 PH_EXIT(php);
2650 2694 mutex_enter(&ps->ps_lock);
2651 2695 /*
2652 2696 * while we dropped the pollhead lock, the element could be
2653 2697 * taken off the list already.
2654 2698 */
2655 2699 PH_ENTER(php);
2656 2700 if (pdp->pd_php == php) {
2657 2701 ASSERT(pdp == php->ph_list);
2658 2702 php->ph_list = pdp->pd_next;
2659 2703 pdp->pd_php = NULL;
2660 2704 pdp->pd_next = NULL;
2661 2705 }
2662 2706 PH_EXIT(php);
2663 2707 mutex_exit(&ps->ps_lock);
2664 2708 mutex_enter(&pcp->pc_no_exit);
2665 2709 pcp->pc_busy--;
2666 2710 if (pcp->pc_busy == 0) {
2667 2711 /*
2668 2712 * Wakeup the thread waiting in
2669 2713 * thread_exit().
2670 2714 */
2671 2715 cv_signal(&pcp->pc_busy_cv);
2672 2716 }
2673 2717 mutex_exit(&pcp->pc_no_exit);
2674 2718 PH_ENTER(php);
2675 2719 }
2676 2720 PH_EXIT(php);
2677 2721 }
2678 2722
2679 2723 /*
2680 2724 * The remove_list is called to cleanup a partially cached 'current' list or
2681 2725 * to remove a partial list which is no longer cached. The flag value of 1
2682 2726 * indicates the second case.
2683 2727 */
2684 2728 void
2685 2729 pcacheset_remove_list(pollstate_t *ps, pollfd_t *pollfdp, int start, int end,
2686 2730 int cacheindex, int flag)
2687 2731 {
2688 2732 int i;
2689 2733
2690 2734 ASSERT(MUTEX_HELD(&ps->ps_lock));
2691 2735 for (i = start; i < end; i++) {
2692 2736 if ((pollfdp[i].fd >= 0) &&
2693 2737 (flag || !(pollfdp[i].revents & POLLNVAL))) {
2694 2738 if (pcache_delete_fd(ps, pollfdp[i].fd, i, cacheindex,
2695 2739 (uint_t)pollfdp[i].events)) {
2696 2740 int j;
2697 2741 int fd = pollfdp[i].fd;
2698 2742
2699 2743 for (j = i + 1; j < end; j++) {
2700 2744 if (pollfdp[j].fd == fd) {
2701 2745 pcache_update_xref(
2702 2746 ps->ps_pcache, fd,
2703 2747 (ssize_t)j, cacheindex);
2704 2748 break;
2705 2749 }
2706 2750 }
2707 2751 ASSERT(j <= end);
2708 2752 }
2709 2753 }
2710 2754 }
2711 2755 }
2712 2756
2713 2757 #ifdef DEBUG
2714 2758
2715 2759 #include<sys/strsubr.h>
2716 2760 /*
2717 2761 * make sure curthread is not on anyone's pollhead list any more.
2718 2762 */
2719 2763 static void
2720 2764 pollcheckphlist()
2721 2765 {
2722 2766 int i;
2723 2767 file_t *fp;
2724 2768 uf_entry_t *ufp;
2725 2769 uf_info_t *fip = P_FINFO(curproc);
2726 2770 struct stdata *stp;
2727 2771 polldat_t *pdp;
2728 2772
2729 2773 mutex_enter(&fip->fi_lock);
2730 2774 for (i = 0; i < fip->fi_nfiles; i++) {
2731 2775 UF_ENTER(ufp, fip, i);
2732 2776 if ((fp = ufp->uf_file) != NULL) {
2733 2777 if ((stp = fp->f_vnode->v_stream) != NULL) {
2734 2778 PH_ENTER(&stp->sd_pollist);
2735 2779 pdp = stp->sd_pollist.ph_list;
2736 2780 while (pdp) {
2737 2781 ASSERT(pdp->pd_thread != curthread);
2738 2782 pdp = pdp->pd_next;
2739 2783 }
2740 2784 PH_EXIT(&stp->sd_pollist);
2741 2785 }
2742 2786 }
2743 2787 UF_EXIT(ufp);
2744 2788 }
2745 2789 mutex_exit(&fip->fi_lock);
2746 2790 }
2747 2791
2748 2792 /*
2749 2793 * for resolved set poll list, the xref info in the pcache should be
2750 2794 * consistent with this poll list.
2751 2795 */
2752 2796 static int
2753 2797 pollcheckxref(pollstate_t *ps, int cacheindex)
2754 2798 {
2755 2799 pollfd_t *pollfdp = ps->ps_pcacheset[cacheindex].pcs_pollfd;
2756 2800 pollcache_t *pcp = ps->ps_pcache;
2757 2801 polldat_t *pdp;
2758 2802 int i;
2759 2803 xref_t *refp;
2760 2804
2761 2805 for (i = 0; i < ps->ps_pcacheset[cacheindex].pcs_nfds; i++) {
2762 2806 if (pollfdp[i].fd < 0) {
2763 2807 continue;
2764 2808 }
2765 2809 pdp = pcache_lookup_fd(pcp, pollfdp[i].fd);
2766 2810 ASSERT(pdp != NULL);
2767 2811 ASSERT(pdp->pd_ref != NULL);
2768 2812 refp = &pdp->pd_ref[cacheindex];
2769 2813 if (refp->xf_position >= 0) {
2770 2814 ASSERT(refp->xf_refcnt >= 1);
2771 2815 ASSERT(pollfdp[refp->xf_position].fd == pdp->pd_fd);
2772 2816 if (refp->xf_refcnt > 1) {
2773 2817 int j;
2774 2818 int count = 0;
2775 2819
2776 2820 for (j = refp->xf_position;
2777 2821 j < ps->ps_pcacheset[cacheindex].pcs_nfds;
2778 2822 j++) {
2779 2823 if (pollfdp[j].fd == pdp->pd_fd) {
2780 2824 count++;
2781 2825 }
2782 2826 }
2783 2827 ASSERT(count == refp->xf_refcnt);
2784 2828 }
2785 2829 }
2786 2830 }
2787 2831 return (1);
2788 2832 }
2789 2833
2790 2834 /*
2791 2835 * For every cached pollfd, its polldat struct should be consistent with
2792 2836 * what is in the pcacheset lists.
2793 2837 */
2794 2838 static void
2795 2839 checkpolldat(pollstate_t *ps)
2796 2840 {
2797 2841 pollcache_t *pcp = ps->ps_pcache;
2798 2842 polldat_t **hashtbl;
2799 2843 int i;
2800 2844
2801 2845 hashtbl = pcp->pc_hash;
2802 2846 for (i = 0; i < pcp->pc_hashsize; i++) {
2803 2847 polldat_t *pdp;
2804 2848
2805 2849 for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
2806 2850 ASSERT(pdp->pd_ref != NULL);
2807 2851 if (pdp->pd_count > 0) {
2808 2852 xref_t *refp;
2809 2853 int j;
2810 2854 pollcacheset_t *pcsp;
2811 2855 pollfd_t *pollfd;
2812 2856
2813 2857 for (j = 0; j < ps->ps_nsets; j++) {
2814 2858 refp = &pdp->pd_ref[j];
2815 2859 if (refp->xf_refcnt > 0) {
2816 2860 pcsp = &ps->ps_pcacheset[j];
2817 2861 ASSERT(refp->xf_position < pcsp->pcs_nfds);
2818 2862 pollfd = pcsp->pcs_pollfd;
2819 2863 ASSERT(pdp->pd_fd == pollfd[refp->xf_position].fd);
2820 2864 }
2821 2865 }
2822 2866 }
2823 2867 }
2824 2868 }
2825 2869 }
2826 2870
2827 2871 /*
2828 2872 * every wfd element on ph_list must have a corresponding fpollinfo on the
2829 2873 * uf_fpollinfo list. This is a variation of infpollinfo() w/o holding locks.
2830 2874 */
2831 2875 void
2832 2876 checkwfdlist(vnode_t *vp, fpollinfo_t *fpip)
2833 2877 {
2834 2878 stdata_t *stp;
2835 2879 polldat_t *pdp;
2836 2880 fpollinfo_t *fpip2;
2837 2881
2838 2882 if ((stp = vp->v_stream) == NULL) {
2839 2883 return;
2840 2884 }
2841 2885 PH_ENTER(&stp->sd_pollist);
2842 2886 for (pdp = stp->sd_pollist.ph_list; pdp; pdp = pdp->pd_next) {
2843 2887 if (pdp->pd_thread != NULL &&
2844 2888 pdp->pd_thread->t_procp == curthread->t_procp) {
2845 2889 for (fpip2 = fpip; fpip2; fpip2 = fpip2->fp_next) {
2846 2890 if (pdp->pd_thread == fpip2->fp_thread) {
2847 2891 break;
2848 2892 }
2849 2893 }
2850 2894 ASSERT(fpip2 != NULL);
2851 2895 }
2852 2896 }
2853 2897 PH_EXIT(&stp->sd_pollist);
2854 2898 }
2855 2899
2856 2900 /*
2857 2901 * For each cached fd whose bit is not set in bitmap, its revents field in
2858 2902 * current poll list should be 0.
2859 2903 */
2860 2904 static int
2861 2905 pollcheckrevents(pollstate_t *ps, int begin, int end, int cacheindex)
2862 2906 {
2863 2907 pollcache_t *pcp = ps->ps_pcache;
2864 2908 pollfd_t *pollfdp = ps->ps_pollfd;
2865 2909 int i;
2866 2910
2867 2911 for (i = begin; i < end; i++) {
2868 2912 polldat_t *pdp;
2869 2913
2870 2914 ASSERT(!BT_TEST(pcp->pc_bitmap, i));
2871 2915 pdp = pcache_lookup_fd(pcp, i);
2872 2916 if (pdp && pdp->pd_fp != NULL) {
2873 2917 xref_t *refp;
2874 2918 int entry;
2875 2919
2876 2920 ASSERT(pdp->pd_ref != NULL);
2877 2921 refp = &pdp->pd_ref[cacheindex];
2878 2922 if (refp->xf_refcnt == 0) {
2879 2923 continue;
2880 2924 }
2881 2925 entry = refp->xf_position;
2882 2926 ASSERT(entry >= 0);
2883 2927 ASSERT(pollfdp[entry].revents == 0);
2884 2928 if (refp->xf_refcnt > 1) {
2885 2929 int j;
2886 2930
2887 2931 for (j = entry + 1; j < ps->ps_nfds; j++) {
2888 2932 if (pollfdp[j].fd == i) {
2889 2933 ASSERT(pollfdp[j].revents == 0);
2890 2934 }
2891 2935 }
2892 2936 }
2893 2937 }
2894 2938 }
2895 2939 return (1);
2896 2940 }
2897 2941
2898 2942 #endif /* DEBUG */
2899 2943
2900 2944 pollcache_t *
2901 2945 pcache_alloc()
2902 2946 {
2903 2947 return (kmem_zalloc(sizeof (pollcache_t), KM_SLEEP));
2904 2948 }
2905 2949
2906 2950 void
2907 2951 pcache_create(pollcache_t *pcp, nfds_t nfds)
2908 2952 {
2909 2953 size_t mapsize;
2910 2954
2911 2955 /*
2912 2956 * allocate enough bits for the poll fd list
2913 2957 */
2914 2958 if ((mapsize = POLLMAPCHUNK) <= nfds) {
2915 2959 mapsize = (nfds + POLLMAPCHUNK - 1) & ~(POLLMAPCHUNK - 1);
2916 2960 }
2917 2961 pcp->pc_bitmap = kmem_zalloc((mapsize / BT_NBIPUL) * sizeof (ulong_t),
2918 2962 KM_SLEEP);
2919 2963 pcp->pc_mapsize = mapsize;
2920 2964 /*
2921 2965 * The hash size is at least POLLHASHCHUNKSZ. If user polls a large
2922 2966 * number of fd to start with, allocate a bigger hash table (to the
2923 2967 * nearest multiple of POLLHASHCHUNKSZ) because dynamically growing a
2924 2968 * hash table is expensive.
2925 2969 */
2926 2970 if (nfds < POLLHASHCHUNKSZ) {
2927 2971 pcp->pc_hashsize = POLLHASHCHUNKSZ;
2928 2972 } else {
2929 2973 pcp->pc_hashsize = (nfds + POLLHASHCHUNKSZ - 1) &
2930 2974 ~(POLLHASHCHUNKSZ - 1);
2931 2975 }
2932 2976 pcp->pc_hash = kmem_zalloc(pcp->pc_hashsize * sizeof (polldat_t *),
2933 2977 KM_SLEEP);
2934 2978 }
2935 2979
2936 2980 void
2937 2981 pcache_destroy(pollcache_t *pcp)
2938 2982 {
2939 2983 polldat_t **hashtbl;
2940 2984 int i;
2941 2985
2942 2986 hashtbl = pcp->pc_hash;
2943 2987 for (i = 0; i < pcp->pc_hashsize; i++) {
2944 2988 if (hashtbl[i] != NULL) {
2945 2989 polldat_t *pdp, *pdp2;
2946 2990
2947 2991 pdp = hashtbl[i];
2948 2992 while (pdp != NULL) {
2949 2993 pdp2 = pdp->pd_hashnext;
2950 2994 if (pdp->pd_ref != NULL) {
2951 2995 kmem_free(pdp->pd_ref, sizeof (xref_t) *
2952 2996 pdp->pd_nsets);
2953 2997 }
2954 2998 kmem_free(pdp, sizeof (polldat_t));
2955 2999 pdp = pdp2;
2956 3000 pcp->pc_fdcount--;
2957 3001 }
2958 3002 }
2959 3003 }
2960 3004 ASSERT(pcp->pc_fdcount == 0);
2961 3005 kmem_free(pcp->pc_hash, sizeof (polldat_t *) * pcp->pc_hashsize);
2962 3006 kmem_free(pcp->pc_bitmap,
2963 3007 sizeof (ulong_t) * (pcp->pc_mapsize/BT_NBIPUL));
2964 3008 mutex_destroy(&pcp->pc_no_exit);
2965 3009 mutex_destroy(&pcp->pc_lock);
2966 3010 cv_destroy(&pcp->pc_cv);
2967 3011 cv_destroy(&pcp->pc_busy_cv);
2968 3012 kmem_free(pcp, sizeof (pollcache_t));
2969 3013 }
2970 3014
2971 3015 pollcacheset_t *
2972 3016 pcacheset_create(int nsets)
2973 3017 {
2974 3018 return (kmem_zalloc(sizeof (pollcacheset_t) * nsets, KM_SLEEP));
2975 3019 }
2976 3020
2977 3021 void
2978 3022 pcacheset_destroy(pollcacheset_t *pcsp, int nsets)
2979 3023 {
2980 3024 int i;
2981 3025
2982 3026 for (i = 0; i < nsets; i++) {
2983 3027 if (pcsp[i].pcs_pollfd != NULL) {
2984 3028 kmem_free(pcsp[i].pcs_pollfd, pcsp[i].pcs_nfds *
2985 3029 sizeof (pollfd_t));
2986 3030 }
2987 3031 }
2988 3032 kmem_free(pcsp, sizeof (pollcacheset_t) * nsets);
2989 3033 }
2990 3034
2991 3035 /*
2992 3036 * Check each duplicated poll fd in the poll list. It may be necessary to
2993 3037 * VOP_POLL the same fd again using different poll events. getf() has been
2994 3038 * done by caller. This routine returns 0 if it can sucessfully process the
2995 3039 * entire poll fd list. It returns -1 if underlying vnode has changed during
2996 3040 * a VOP_POLL, in which case the caller has to repoll. It returns a positive
2997 3041 * value if VOP_POLL failed.
2998 3042 */
2999 3043 static int
3000 3044 plist_chkdupfd(file_t *fp, polldat_t *pdp, pollstate_t *psp, pollfd_t *pollfdp,
3001 3045 int entry, int *fdcntp)
3002 3046 {
3003 3047 int i;
3004 3048 int fd;
3005 3049 nfds_t nfds = psp->ps_nfds;
3006 3050
3007 3051 fd = pollfdp[entry].fd;
3008 3052 for (i = entry + 1; i < nfds; i++) {
3009 3053 if (pollfdp[i].fd == fd) {
3010 3054 if (pollfdp[i].events == pollfdp[entry].events) {
3011 3055 if ((pollfdp[i].revents =
3012 3056 pollfdp[entry].revents) != 0) {
3013 3057 (*fdcntp)++;
3014 3058 }
3015 3059 } else {
3016 3060
3017 3061 int error;
3018 3062 pollhead_t *php;
3019 3063 pollcache_t *pcp = psp->ps_pcache;
3020 3064
3021 3065 /*
3022 3066 * the events are different. VOP_POLL on this
3023 3067 * fd so that we don't miss any revents.
3024 3068 */
3025 3069 php = NULL;
3026 3070 ASSERT(curthread->t_pollcache == NULL);
3027 3071 error = VOP_POLL(fp->f_vnode,
3028 3072 pollfdp[i].events, 0,
3029 3073 &pollfdp[i].revents, &php, NULL);
3030 3074 if (error) {
3031 3075 return (error);
3032 3076 }
3033 3077 /*
3034 3078 * layered devices(e.g. console driver)
3035 3079 * may change the vnode and thus the pollhead
3036 3080 * pointer out from underneath us.
3037 3081 */
3038 3082 if (php != NULL && pdp->pd_php != NULL &&
3039 3083 php != pdp->pd_php) {
3040 3084 pollhead_delete(pdp->pd_php, pdp);
3041 3085 pdp->pd_php = php;
3042 3086 pollhead_insert(php, pdp);
3043 3087 /*
3044 3088 * We could have missed a wakeup on the
3045 3089 * new target device. Make sure the new
3046 3090 * target gets polled once.
3047 3091 */
3048 3092 BT_SET(pcp->pc_bitmap, fd);
3049 3093 return (-1);
3050 3094 }
3051 3095 if (pollfdp[i].revents) {
3052 3096 (*fdcntp)++;
3053 3097 }
3054 3098 }
3055 3099 }
3056 3100 }
3057 3101 return (0);
3058 3102 }
|
↓ open down ↓ |
2353 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX