Print this page
9042 multiples of tty streams modules cause weirdness
Reviewed by: Randy Fishel <randyf@sibernet.com>
Reviewed by: Carlos Neira <cneirabustos@gmail.com>
Approved by: Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
re #13613 rb4516 Tunables needs volatile keyword
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/strsubr.c
+++ new/usr/src/uts/common/os/strsubr.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 22 /* All Rights Reserved */
23 23
24 -
25 24 /*
26 25 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
27 26 * Use is subject to license terms.
28 27 * Copyright (c) 2016 by Delphix. All rights reserved.
28 + * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
29 29 */
30 +/*
31 + * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
32 + */
30 33
31 34 #include <sys/types.h>
32 35 #include <sys/sysmacros.h>
33 36 #include <sys/param.h>
34 37 #include <sys/errno.h>
35 38 #include <sys/signal.h>
36 39 #include <sys/proc.h>
37 40 #include <sys/conf.h>
38 41 #include <sys/cred.h>
39 42 #include <sys/user.h>
40 43 #include <sys/vnode.h>
41 44 #include <sys/file.h>
42 45 #include <sys/session.h>
43 46 #include <sys/stream.h>
44 47 #include <sys/strsubr.h>
45 48 #include <sys/stropts.h>
46 49 #include <sys/poll.h>
47 50 #include <sys/systm.h>
48 51 #include <sys/cpuvar.h>
49 52 #include <sys/uio.h>
50 53 #include <sys/cmn_err.h>
51 54 #include <sys/priocntl.h>
52 55 #include <sys/procset.h>
53 56 #include <sys/vmem.h>
54 57 #include <sys/bitmap.h>
55 58 #include <sys/kmem.h>
56 59 #include <sys/siginfo.h>
57 60 #include <sys/vtrace.h>
58 61 #include <sys/callb.h>
59 62 #include <sys/debug.h>
60 63 #include <sys/modctl.h>
61 64 #include <sys/vmsystm.h>
62 65 #include <vm/page.h>
63 66 #include <sys/atomic.h>
64 67 #include <sys/suntpi.h>
65 68 #include <sys/strlog.h>
66 69 #include <sys/promif.h>
67 70 #include <sys/project.h>
68 71 #include <sys/vm.h>
69 72 #include <sys/taskq.h>
70 73 #include <sys/sunddi.h>
71 74 #include <sys/sunldi_impl.h>
72 75 #include <sys/strsun.h>
73 76 #include <sys/isa_defs.h>
74 77 #include <sys/multidata.h>
75 78 #include <sys/pattr.h>
76 79 #include <sys/strft.h>
77 80 #include <sys/fs/snode.h>
78 81 #include <sys/zone.h>
79 82 #include <sys/open.h>
80 83 #include <sys/sunldi.h>
81 84 #include <sys/sad.h>
82 85 #include <sys/netstack.h>
83 86
84 87 #define O_SAMESTR(q) (((q)->q_next) && \
85 88 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
86 89
87 90 /*
88 91 * WARNING:
89 92 * The variables and routines in this file are private, belonging
90 93 * to the STREAMS subsystem. These should not be used by modules
91 94 * or drivers. Compatibility will not be guaranteed.
92 95 */
93 96
94 97 /*
95 98 * Id value used to distinguish between different multiplexor links.
96 99 */
97 100 static int32_t lnk_id = 0;
98 101
99 102 #define STREAMS_LOPRI MINCLSYSPRI
100 103 static pri_t streams_lopri = STREAMS_LOPRI;
101 104
102 105 #define STRSTAT(x) (str_statistics.x.value.ui64++)
103 106 typedef struct str_stat {
104 107 kstat_named_t sqenables;
105 108 kstat_named_t stenables;
106 109 kstat_named_t syncqservice;
107 110 kstat_named_t freebs;
108 111 kstat_named_t qwr_outer;
109 112 kstat_named_t rservice;
110 113 kstat_named_t strwaits;
111 114 kstat_named_t taskqfails;
112 115 kstat_named_t bufcalls;
113 116 kstat_named_t qhelps;
114 117 kstat_named_t qremoved;
115 118 kstat_named_t sqremoved;
116 119 kstat_named_t bcwaits;
117 120 kstat_named_t sqtoomany;
118 121 } str_stat_t;
119 122
120 123 static str_stat_t str_statistics = {
121 124 { "sqenables", KSTAT_DATA_UINT64 },
122 125 { "stenables", KSTAT_DATA_UINT64 },
123 126 { "syncqservice", KSTAT_DATA_UINT64 },
124 127 { "freebs", KSTAT_DATA_UINT64 },
125 128 { "qwr_outer", KSTAT_DATA_UINT64 },
126 129 { "rservice", KSTAT_DATA_UINT64 },
127 130 { "strwaits", KSTAT_DATA_UINT64 },
128 131 { "taskqfails", KSTAT_DATA_UINT64 },
129 132 { "bufcalls", KSTAT_DATA_UINT64 },
130 133 { "qhelps", KSTAT_DATA_UINT64 },
131 134 { "qremoved", KSTAT_DATA_UINT64 },
132 135 { "sqremoved", KSTAT_DATA_UINT64 },
133 136 { "bcwaits", KSTAT_DATA_UINT64 },
134 137 { "sqtoomany", KSTAT_DATA_UINT64 },
135 138 };
136 139
137 140 static kstat_t *str_kstat;
138 141
139 142 /*
140 143 * qrunflag was used previously to control background scheduling of queues. It
141 144 * is not used anymore, but kept here in case some module still wants to access
142 145 * it via qready() and setqsched macros.
143 146 */
144 147 char qrunflag; /* Unused */
145 148
146 149 /*
147 150 * Most of the streams scheduling is done via task queues. Task queues may fail
148 151 * for non-sleep dispatches, so there are two backup threads servicing failed
149 152 * requests for queues and syncqs. Both of these threads also service failed
150 153 * dispatches freebs requests. Queues are put in the list specified by `qhead'
151 154 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
152 155 * requests are put into `freebs_list' which has no tail pointer. All three
153 156 * lists are protected by a single `service_queue' lock and use
154 157 * `services_to_run' condition variable for signaling background threads. Use of
155 158 * a single lock should not be a problem because it is only used under heavy
156 159 * loads when task queues start to fail and at that time it may be a good idea
157 160 * to throttle scheduling requests.
158 161 *
159 162 * NOTE: queues and syncqs should be scheduled by two separate threads because
160 163 * queue servicing may be blocked waiting for a syncq which may be also
161 164 * scheduled for background execution. This may create a deadlock when only one
162 165 * thread is used for both.
163 166 */
164 167
165 168 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */
166 169
167 170 static kmutex_t service_queue; /* protects all of servicing vars */
168 171 static kcondvar_t services_to_run; /* wake up background service thread */
169 172 static kcondvar_t syncqs_to_run; /* wake up background service thread */
170 173
171 174 /*
172 175 * List of queues scheduled for background processing due to lack of resources
173 176 * in the task queues. Protected by service_queue lock;
174 177 */
175 178 static struct queue *qhead;
176 179 static struct queue *qtail;
177 180
178 181 /*
179 182 * Same list for syncqs
180 183 */
181 184 static syncq_t *sqhead;
182 185 static syncq_t *sqtail;
183 186
184 187 static mblk_t *freebs_list; /* list of buffers to free */
185 188
186 189 /*
187 190 * Backup threads for servicing queues and syncqs
188 191 */
189 192 kthread_t *streams_qbkgrnd_thread;
190 193 kthread_t *streams_sqbkgrnd_thread;
191 194
192 195 /*
193 196 * Bufcalls related variables.
194 197 */
195 198 struct bclist strbcalls; /* list of waiting bufcalls */
196 199 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */
197 200 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */
198 201 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */
199 202 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */
200 203 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */
201 204
202 205 kmutex_t strresources; /* protects global resources */
203 206 kmutex_t muxifier; /* single-threads multiplexor creation */
204 207
205 208 static void *str_stack_init(netstackid_t stackid, netstack_t *ns);
206 209 static void str_stack_shutdown(netstackid_t stackid, void *arg);
207 210 static void str_stack_fini(netstackid_t stackid, void *arg);
208 211
209 212 /*
210 213 * run_queues is no longer used, but is kept in case some 3rd party
211 214 * module/driver decides to use it.
212 215 */
|
↓ open down ↓ |
173 lines elided |
↑ open up ↑ |
213 216 int run_queues = 0;
214 217
215 218 /*
216 219 * sq_max_size is the depth of the syncq (in number of messages) before
217 220 * qfill_syncq() starts QFULL'ing destination queues. As its primary
218 221 * consumer - IP is no longer D_MTPERMOD, but there may be other
219 222 * modules/drivers depend on this syncq flow control, we prefer to
220 223 * choose a large number as the default value. For potential
221 224 * performance gain, this value is tunable in /etc/system.
222 225 */
223 -int sq_max_size = 10000;
226 +volatile int sq_max_size = 10000;
224 227
225 228 /*
226 229 * The number of ciputctrl structures per syncq and stream we create when
227 230 * needed.
228 231 */
229 232 int n_ciputctrl;
230 233 int max_n_ciputctrl = 16;
231 234 /*
232 235 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
233 236 */
234 237 int min_n_ciputctrl = 2;
235 238
236 239 /*
237 240 * Per-driver/module syncqs
238 241 * ========================
239 242 *
240 243 * For drivers/modules that use PERMOD or outer syncqs we keep a list of
241 244 * perdm structures, new entries being added (and new syncqs allocated) when
242 245 * setq() encounters a module/driver with a streamtab that it hasn't seen
243 246 * before.
244 247 * The reason for this mechanism is that some modules and drivers share a
245 248 * common streamtab and it is necessary for those modules and drivers to also
246 249 * share a common PERMOD syncq.
247 250 *
248 251 * perdm_list --> dm_str == streamtab_1
249 252 * dm_sq == syncq_1
250 253 * dm_ref
251 254 * dm_next --> dm_str == streamtab_2
252 255 * dm_sq == syncq_2
253 256 * dm_ref
254 257 * dm_next --> ... NULL
255 258 *
256 259 * The dm_ref field is incremented for each new driver/module that takes
257 260 * a reference to the perdm structure and hence shares the syncq.
258 261 * References are held in the fmodsw_impl_t structure for each STREAMS module
259 262 * or the dev_impl array (indexed by device major number) for each driver.
260 263 *
261 264 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
262 265 * ^ ^ ^ ^
263 266 * | ______________/ | |
264 267 * | / | |
265 268 * dev_impl: ...|x|y|... module A module B
266 269 *
267 270 * When a module/driver is unloaded the reference count is decremented and,
268 271 * when it falls to zero, the perdm structure is removed from the list and
269 272 * the syncq is freed (see rele_dm()).
270 273 */
271 274 perdm_t *perdm_list = NULL;
272 275 static krwlock_t perdm_rwlock;
273 276 cdevsw_impl_t *devimpl;
274 277
275 278 extern struct qinit strdata;
276 279 extern struct qinit stwdata;
277 280
278 281 static void runservice(queue_t *);
279 282 static void streams_bufcall_service(void);
280 283 static void streams_qbkgrnd_service(void);
281 284 static void streams_sqbkgrnd_service(void);
282 285 static syncq_t *new_syncq(void);
283 286 static void free_syncq(syncq_t *);
284 287 static void outer_insert(syncq_t *, syncq_t *);
285 288 static void outer_remove(syncq_t *, syncq_t *);
286 289 static void write_now(syncq_t *);
287 290 static void clr_qfull(queue_t *);
288 291 static void runbufcalls(void);
289 292 static void sqenable(syncq_t *);
290 293 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
291 294 static void wait_q_syncq(queue_t *);
292 295 static void backenable_insertedq(queue_t *);
293 296
294 297 static void queue_service(queue_t *);
295 298 static void stream_service(stdata_t *);
296 299 static void syncq_service(syncq_t *);
297 300 static void qwriter_outer_service(syncq_t *);
298 301 static void mblk_free(mblk_t *);
299 302 #ifdef DEBUG
300 303 static int qprocsareon(queue_t *);
301 304 #endif
302 305
303 306 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
304 307 static void reset_nfsrv_ptr(queue_t *, queue_t *);
305 308 void set_qfull(queue_t *);
306 309
307 310 static void sq_run_events(syncq_t *);
308 311 static int propagate_syncq(queue_t *);
309 312
310 313 static void blocksq(syncq_t *, ushort_t, int);
311 314 static void unblocksq(syncq_t *, ushort_t, int);
312 315 static int dropsq(syncq_t *, uint16_t);
313 316 static void emptysq(syncq_t *);
314 317 static sqlist_t *sqlist_alloc(struct stdata *, int);
315 318 static void sqlist_free(sqlist_t *);
316 319 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t);
317 320 static void sqlist_insert(sqlist_t *, syncq_t *);
318 321 static void sqlist_insertall(sqlist_t *, queue_t *);
319 322
320 323 static void strsetuio(stdata_t *);
321 324
322 325 struct kmem_cache *stream_head_cache;
323 326 struct kmem_cache *queue_cache;
324 327 struct kmem_cache *syncq_cache;
325 328 struct kmem_cache *qband_cache;
326 329 struct kmem_cache *linkinfo_cache;
327 330 struct kmem_cache *ciputctrl_cache = NULL;
328 331
329 332 static linkinfo_t *linkinfo_list;
330 333
331 334 /* Global esballoc throttling queue */
332 335 static esb_queue_t system_esbq;
333 336
334 337 /* Array of esballoc throttling queues, of length esbq_nelem */
335 338 static esb_queue_t *volatile system_esbq_array;
336 339 static int esbq_nelem;
337 340 static kmutex_t esbq_lock;
338 341 static int esbq_log2_cpus_per_q = 0;
339 342
340 343 /* Scale the system_esbq length by setting number of CPUs per queue. */
341 344 uint_t esbq_cpus_per_q = 1;
342 345
343 346 /*
344 347 * esballoc tunable parameters.
345 348 */
346 349 int esbq_max_qlen = 0x16; /* throttled queue length */
347 350 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */
348 351
349 352 /*
350 353 * Routines to handle esballoc queueing.
351 354 */
352 355 static void esballoc_process_queue(esb_queue_t *);
353 356 static void esballoc_enqueue_mblk(mblk_t *);
354 357 static void esballoc_timer(void *);
355 358 static void esballoc_set_timer(esb_queue_t *, clock_t);
356 359 static void esballoc_mblk_free(mblk_t *);
357 360
358 361 /*
359 362 * Qinit structure and Module_info structures
360 363 * for passthru read and write queues
361 364 */
362 365
363 366 static void pass_wput(queue_t *, mblk_t *);
364 367 static queue_t *link_addpassthru(stdata_t *);
365 368 static void link_rempassthru(queue_t *);
366 369
367 370 struct module_info passthru_info = {
368 371 0,
369 372 "passthru",
370 373 0,
371 374 INFPSZ,
372 375 STRHIGH,
373 376 STRLOW
374 377 };
375 378
376 379 struct qinit passthru_rinit = {
377 380 (int (*)())putnext,
378 381 NULL,
379 382 NULL,
380 383 NULL,
381 384 NULL,
382 385 &passthru_info,
383 386 NULL
384 387 };
385 388
386 389 struct qinit passthru_winit = {
387 390 (int (*)()) pass_wput,
388 391 NULL,
389 392 NULL,
390 393 NULL,
391 394 NULL,
392 395 &passthru_info,
393 396 NULL
394 397 };
395 398
396 399 /*
397 400 * Verify correctness of list head/tail pointers.
398 401 */
399 402 #define LISTCHECK(head, tail, link) { \
400 403 EQUIV(head, tail); \
401 404 IMPLY(tail != NULL, tail->link == NULL); \
402 405 }
403 406
404 407 /*
405 408 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
406 409 * using a `link' field.
407 410 */
408 411 #define ENQUEUE(el, head, tail, link) { \
409 412 ASSERT(el->link == NULL); \
410 413 LISTCHECK(head, tail, link); \
411 414 if (head == NULL) \
412 415 head = el; \
413 416 else \
414 417 tail->link = el; \
415 418 tail = el; \
416 419 }
417 420
418 421 /*
419 422 * Dequeue the first element of the list denoted by `head' and `tail' pointers
420 423 * using a `link' field and put result into `el'.
421 424 */
422 425 #define DQ(el, head, tail, link) { \
423 426 LISTCHECK(head, tail, link); \
424 427 el = head; \
425 428 if (head != NULL) { \
426 429 head = head->link; \
427 430 if (head == NULL) \
428 431 tail = NULL; \
429 432 el->link = NULL; \
430 433 } \
431 434 }
432 435
433 436 /*
434 437 * Remove `el' from the list using `chase' and `curr' pointers and return result
435 438 * in `succeed'.
436 439 */
437 440 #define RMQ(el, head, tail, link, chase, curr, succeed) { \
438 441 LISTCHECK(head, tail, link); \
439 442 chase = NULL; \
440 443 succeed = 0; \
441 444 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
442 445 chase = curr; \
443 446 if (curr != NULL) { \
444 447 succeed = 1; \
445 448 ASSERT(curr == el); \
446 449 if (chase != NULL) \
447 450 chase->link = curr->link; \
448 451 else \
449 452 head = curr->link; \
450 453 curr->link = NULL; \
451 454 if (curr == tail) \
452 455 tail = chase; \
453 456 } \
454 457 LISTCHECK(head, tail, link); \
455 458 }
456 459
457 460 /* Handling of delayed messages on the inner syncq. */
458 461
459 462 /*
460 463 * DEBUG versions should use function versions (to simplify tracing) and
461 464 * non-DEBUG kernels should use macro versions.
462 465 */
463 466
464 467 /*
465 468 * Put a queue on the syncq list of queues.
466 469 * Assumes SQLOCK held.
467 470 */
468 471 #define SQPUT_Q(sq, qp) \
469 472 { \
470 473 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
471 474 if (!(qp->q_sqflags & Q_SQQUEUED)) { \
472 475 /* The queue should not be linked anywhere */ \
473 476 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
474 477 /* Head and tail may only be NULL simultaneously */ \
475 478 EQUIV(sq->sq_head, sq->sq_tail); \
476 479 /* Queue may be only enqueued on its syncq */ \
477 480 ASSERT(sq == qp->q_syncq); \
478 481 /* Check the correctness of SQ_MESSAGES flag */ \
479 482 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
480 483 /* Sanity check first/last elements of the list */ \
481 484 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
482 485 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
483 486 /* \
484 487 * Sanity check of priority field: empty queue should \
485 488 * have zero priority \
486 489 * and nqueues equal to zero. \
487 490 */ \
488 491 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
489 492 /* Sanity check of sq_nqueues field */ \
490 493 EQUIV(sq->sq_head, sq->sq_nqueues); \
491 494 if (sq->sq_head == NULL) { \
492 495 sq->sq_head = sq->sq_tail = qp; \
493 496 sq->sq_flags |= SQ_MESSAGES; \
494 497 } else if (qp->q_spri == 0) { \
495 498 qp->q_sqprev = sq->sq_tail; \
496 499 sq->sq_tail->q_sqnext = qp; \
497 500 sq->sq_tail = qp; \
498 501 } else { \
499 502 /* \
500 503 * Put this queue in priority order: higher \
501 504 * priority gets closer to the head. \
502 505 */ \
503 506 queue_t **qpp = &sq->sq_tail; \
504 507 queue_t *qnext = NULL; \
505 508 \
506 509 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
507 510 qnext = *qpp; \
508 511 qpp = &(*qpp)->q_sqprev; \
509 512 } \
510 513 qp->q_sqnext = qnext; \
511 514 qp->q_sqprev = *qpp; \
512 515 if (*qpp != NULL) { \
513 516 (*qpp)->q_sqnext = qp; \
514 517 } else { \
515 518 sq->sq_head = qp; \
516 519 sq->sq_pri = sq->sq_head->q_spri; \
517 520 } \
518 521 *qpp = qp; \
519 522 } \
520 523 qp->q_sqflags |= Q_SQQUEUED; \
521 524 qp->q_sqtstamp = ddi_get_lbolt(); \
522 525 sq->sq_nqueues++; \
523 526 } \
524 527 }
525 528
526 529 /*
527 530 * Remove a queue from the syncq list
528 531 * Assumes SQLOCK held.
529 532 */
530 533 #define SQRM_Q(sq, qp) \
531 534 { \
532 535 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
533 536 ASSERT(qp->q_sqflags & Q_SQQUEUED); \
534 537 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
535 538 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
536 539 /* Check that the queue is actually in the list */ \
537 540 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
538 541 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
539 542 ASSERT(sq->sq_nqueues != 0); \
540 543 if (qp->q_sqprev == NULL) { \
541 544 /* First queue on list, make head q_sqnext */ \
542 545 sq->sq_head = qp->q_sqnext; \
543 546 } else { \
544 547 /* Make prev->next == next */ \
545 548 qp->q_sqprev->q_sqnext = qp->q_sqnext; \
546 549 } \
547 550 if (qp->q_sqnext == NULL) { \
548 551 /* Last queue on list, make tail sqprev */ \
549 552 sq->sq_tail = qp->q_sqprev; \
550 553 } else { \
551 554 /* Make next->prev == prev */ \
552 555 qp->q_sqnext->q_sqprev = qp->q_sqprev; \
553 556 } \
554 557 /* clear out references on this queue */ \
555 558 qp->q_sqprev = qp->q_sqnext = NULL; \
556 559 qp->q_sqflags &= ~Q_SQQUEUED; \
557 560 /* If there is nothing queued, clear SQ_MESSAGES */ \
558 561 if (sq->sq_head != NULL) { \
559 562 sq->sq_pri = sq->sq_head->q_spri; \
560 563 } else { \
561 564 sq->sq_flags &= ~SQ_MESSAGES; \
562 565 sq->sq_pri = 0; \
563 566 } \
564 567 sq->sq_nqueues--; \
565 568 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
566 569 (sq->sq_flags & SQ_QUEUED) == 0); \
567 570 }
568 571
569 572 /* Hide the definition from the header file. */
570 573 #ifdef SQPUT_MP
571 574 #undef SQPUT_MP
572 575 #endif
573 576
574 577 /*
575 578 * Put a message on the queue syncq.
576 579 * Assumes QLOCK held.
577 580 */
578 581 #define SQPUT_MP(qp, mp) \
579 582 { \
580 583 ASSERT(MUTEX_HELD(QLOCK(qp))); \
581 584 ASSERT(qp->q_sqhead == NULL || \
582 585 (qp->q_sqtail != NULL && \
583 586 qp->q_sqtail->b_next == NULL)); \
584 587 qp->q_syncqmsgs++; \
585 588 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \
586 589 if (qp->q_sqhead == NULL) { \
587 590 qp->q_sqhead = qp->q_sqtail = mp; \
588 591 } else { \
589 592 qp->q_sqtail->b_next = mp; \
590 593 qp->q_sqtail = mp; \
591 594 } \
592 595 ASSERT(qp->q_syncqmsgs > 0); \
593 596 set_qfull(qp); \
594 597 }
595 598
596 599 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \
597 600 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
598 601 if ((sq)->sq_ciputctrl != NULL) { \
599 602 int i; \
600 603 int nlocks = (sq)->sq_nciputctrl; \
601 604 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
602 605 ASSERT((sq)->sq_type & SQ_CIPUT); \
603 606 for (i = 0; i <= nlocks; i++) { \
604 607 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
605 608 cip[i].ciputctrl_count |= SQ_FASTPUT; \
606 609 } \
607 610 } \
608 611 }
609 612
610 613
611 614 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \
612 615 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
613 616 if ((sq)->sq_ciputctrl != NULL) { \
614 617 int i; \
615 618 int nlocks = (sq)->sq_nciputctrl; \
616 619 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
617 620 ASSERT((sq)->sq_type & SQ_CIPUT); \
618 621 for (i = 0; i <= nlocks; i++) { \
619 622 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
620 623 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \
621 624 } \
622 625 } \
623 626 }
624 627
625 628 /*
626 629 * Run service procedures for all queues in the stream head.
627 630 */
628 631 #define STR_SERVICE(stp, q) { \
629 632 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \
630 633 while (stp->sd_qhead != NULL) { \
631 634 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \
632 635 ASSERT(stp->sd_nqueues > 0); \
633 636 stp->sd_nqueues--; \
634 637 ASSERT(!(q->q_flag & QINSERVICE)); \
635 638 mutex_exit(&stp->sd_qlock); \
636 639 queue_service(q); \
637 640 mutex_enter(&stp->sd_qlock); \
638 641 } \
639 642 ASSERT(stp->sd_nqueues == 0); \
640 643 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \
641 644 }
642 645
643 646 /*
644 647 * Constructor/destructor routines for the stream head cache
645 648 */
646 649 /* ARGSUSED */
647 650 static int
648 651 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
649 652 {
650 653 stdata_t *stp = buf;
651 654
652 655 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
653 656 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
654 657 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
655 658 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
656 659 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
657 660 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
658 661 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
659 662 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
660 663 stp->sd_wrq = NULL;
661 664
662 665 return (0);
663 666 }
664 667
665 668 /* ARGSUSED */
666 669 static void
667 670 stream_head_destructor(void *buf, void *cdrarg)
668 671 {
669 672 stdata_t *stp = buf;
670 673
671 674 mutex_destroy(&stp->sd_lock);
672 675 mutex_destroy(&stp->sd_reflock);
673 676 mutex_destroy(&stp->sd_qlock);
674 677 cv_destroy(&stp->sd_monitor);
675 678 cv_destroy(&stp->sd_iocmonitor);
676 679 cv_destroy(&stp->sd_refmonitor);
677 680 cv_destroy(&stp->sd_qcv);
678 681 cv_destroy(&stp->sd_zcopy_wait);
679 682 }
680 683
681 684 /*
682 685 * Constructor/destructor routines for the queue cache
683 686 */
684 687 /* ARGSUSED */
685 688 static int
686 689 queue_constructor(void *buf, void *cdrarg, int kmflags)
687 690 {
688 691 queinfo_t *qip = buf;
689 692 queue_t *qp = &qip->qu_rqueue;
690 693 queue_t *wqp = &qip->qu_wqueue;
691 694 syncq_t *sq = &qip->qu_syncq;
692 695
693 696 qp->q_first = NULL;
694 697 qp->q_link = NULL;
695 698 qp->q_count = 0;
696 699 qp->q_mblkcnt = 0;
697 700 qp->q_sqhead = NULL;
698 701 qp->q_sqtail = NULL;
699 702 qp->q_sqnext = NULL;
700 703 qp->q_sqprev = NULL;
701 704 qp->q_sqflags = 0;
702 705 qp->q_rwcnt = 0;
703 706 qp->q_spri = 0;
704 707
705 708 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
706 709 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
707 710
708 711 wqp->q_first = NULL;
709 712 wqp->q_link = NULL;
710 713 wqp->q_count = 0;
711 714 wqp->q_mblkcnt = 0;
712 715 wqp->q_sqhead = NULL;
713 716 wqp->q_sqtail = NULL;
714 717 wqp->q_sqnext = NULL;
715 718 wqp->q_sqprev = NULL;
716 719 wqp->q_sqflags = 0;
717 720 wqp->q_rwcnt = 0;
718 721 wqp->q_spri = 0;
719 722
720 723 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
721 724 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
722 725
723 726 sq->sq_head = NULL;
724 727 sq->sq_tail = NULL;
725 728 sq->sq_evhead = NULL;
726 729 sq->sq_evtail = NULL;
727 730 sq->sq_callbpend = NULL;
728 731 sq->sq_outer = NULL;
729 732 sq->sq_onext = NULL;
730 733 sq->sq_oprev = NULL;
731 734 sq->sq_next = NULL;
732 735 sq->sq_svcflags = 0;
733 736 sq->sq_servcount = 0;
734 737 sq->sq_needexcl = 0;
735 738 sq->sq_nqueues = 0;
736 739 sq->sq_pri = 0;
737 740
738 741 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
739 742 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
740 743 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
741 744
742 745 return (0);
743 746 }
744 747
745 748 /* ARGSUSED */
746 749 static void
747 750 queue_destructor(void *buf, void *cdrarg)
748 751 {
749 752 queinfo_t *qip = buf;
750 753 queue_t *qp = &qip->qu_rqueue;
751 754 queue_t *wqp = &qip->qu_wqueue;
752 755 syncq_t *sq = &qip->qu_syncq;
753 756
754 757 ASSERT(qp->q_sqhead == NULL);
755 758 ASSERT(wqp->q_sqhead == NULL);
756 759 ASSERT(qp->q_sqnext == NULL);
757 760 ASSERT(wqp->q_sqnext == NULL);
758 761 ASSERT(qp->q_rwcnt == 0);
759 762 ASSERT(wqp->q_rwcnt == 0);
760 763
761 764 mutex_destroy(&qp->q_lock);
762 765 cv_destroy(&qp->q_wait);
763 766
764 767 mutex_destroy(&wqp->q_lock);
765 768 cv_destroy(&wqp->q_wait);
766 769
767 770 mutex_destroy(&sq->sq_lock);
768 771 cv_destroy(&sq->sq_wait);
769 772 cv_destroy(&sq->sq_exitwait);
770 773 }
771 774
772 775 /*
773 776 * Constructor/destructor routines for the syncq cache
774 777 */
775 778 /* ARGSUSED */
776 779 static int
777 780 syncq_constructor(void *buf, void *cdrarg, int kmflags)
778 781 {
779 782 syncq_t *sq = buf;
780 783
781 784 bzero(buf, sizeof (syncq_t));
782 785
783 786 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
784 787 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
785 788 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
786 789
787 790 return (0);
788 791 }
789 792
790 793 /* ARGSUSED */
791 794 static void
792 795 syncq_destructor(void *buf, void *cdrarg)
793 796 {
794 797 syncq_t *sq = buf;
795 798
796 799 ASSERT(sq->sq_head == NULL);
797 800 ASSERT(sq->sq_tail == NULL);
798 801 ASSERT(sq->sq_evhead == NULL);
799 802 ASSERT(sq->sq_evtail == NULL);
800 803 ASSERT(sq->sq_callbpend == NULL);
801 804 ASSERT(sq->sq_callbflags == 0);
802 805 ASSERT(sq->sq_outer == NULL);
803 806 ASSERT(sq->sq_onext == NULL);
804 807 ASSERT(sq->sq_oprev == NULL);
805 808 ASSERT(sq->sq_next == NULL);
806 809 ASSERT(sq->sq_needexcl == 0);
807 810 ASSERT(sq->sq_svcflags == 0);
808 811 ASSERT(sq->sq_servcount == 0);
809 812 ASSERT(sq->sq_nqueues == 0);
810 813 ASSERT(sq->sq_pri == 0);
811 814 ASSERT(sq->sq_count == 0);
812 815 ASSERT(sq->sq_rmqcount == 0);
813 816 ASSERT(sq->sq_cancelid == 0);
814 817 ASSERT(sq->sq_ciputctrl == NULL);
815 818 ASSERT(sq->sq_nciputctrl == 0);
816 819 ASSERT(sq->sq_type == 0);
817 820 ASSERT(sq->sq_flags == 0);
818 821
819 822 mutex_destroy(&sq->sq_lock);
820 823 cv_destroy(&sq->sq_wait);
821 824 cv_destroy(&sq->sq_exitwait);
822 825 }
823 826
824 827 /* ARGSUSED */
825 828 static int
826 829 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
827 830 {
828 831 ciputctrl_t *cip = buf;
829 832 int i;
830 833
831 834 for (i = 0; i < n_ciputctrl; i++) {
832 835 cip[i].ciputctrl_count = SQ_FASTPUT;
833 836 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
834 837 }
835 838
836 839 return (0);
837 840 }
838 841
839 842 /* ARGSUSED */
840 843 static void
841 844 ciputctrl_destructor(void *buf, void *cdrarg)
842 845 {
843 846 ciputctrl_t *cip = buf;
844 847 int i;
845 848
846 849 for (i = 0; i < n_ciputctrl; i++) {
847 850 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
848 851 mutex_destroy(&cip[i].ciputctrl_lock);
849 852 }
850 853 }
851 854
852 855 /*
853 856 * Init routine run from main at boot time.
854 857 */
855 858 void
856 859 strinit(void)
857 860 {
858 861 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
859 862
860 863 stream_head_cache = kmem_cache_create("stream_head_cache",
861 864 sizeof (stdata_t), 0,
862 865 stream_head_constructor, stream_head_destructor, NULL,
863 866 NULL, NULL, 0);
864 867
865 868 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
866 869 queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
867 870
868 871 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
869 872 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
870 873
871 874 qband_cache = kmem_cache_create("qband_cache",
872 875 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
873 876
874 877 linkinfo_cache = kmem_cache_create("linkinfo_cache",
875 878 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
876 879
877 880 n_ciputctrl = ncpus;
878 881 n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
879 882 ASSERT(n_ciputctrl >= 1);
880 883 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
881 884 if (n_ciputctrl >= min_n_ciputctrl) {
882 885 ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
883 886 sizeof (ciputctrl_t) * n_ciputctrl,
884 887 sizeof (ciputctrl_t), ciputctrl_constructor,
885 888 ciputctrl_destructor, NULL, NULL, NULL, 0);
886 889 }
887 890
888 891 streams_taskq = system_taskq;
889 892
890 893 if (streams_taskq == NULL)
891 894 panic("strinit: no memory for streams taskq!");
892 895
893 896 bc_bkgrnd_thread = thread_create(NULL, 0,
894 897 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
895 898
896 899 streams_qbkgrnd_thread = thread_create(NULL, 0,
897 900 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
898 901
899 902 streams_sqbkgrnd_thread = thread_create(NULL, 0,
900 903 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
901 904
902 905 /*
903 906 * Create STREAMS kstats.
904 907 */
905 908 str_kstat = kstat_create("streams", 0, "strstat",
906 909 "net", KSTAT_TYPE_NAMED,
907 910 sizeof (str_statistics) / sizeof (kstat_named_t),
908 911 KSTAT_FLAG_VIRTUAL);
909 912
910 913 if (str_kstat != NULL) {
911 914 str_kstat->ks_data = &str_statistics;
912 915 kstat_install(str_kstat);
913 916 }
914 917
915 918 /*
916 919 * TPI support routine initialisation.
917 920 */
918 921 tpi_init();
919 922
920 923 /*
921 924 * Handle to have autopush and persistent link information per
922 925 * zone.
923 926 * Note: uses shutdown hook instead of destroy hook so that the
924 927 * persistent links can be torn down before the destroy hooks
925 928 * in the TCP/IP stack are called.
926 929 */
927 930 netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
928 931 str_stack_fini);
929 932 }
930 933
931 934 void
932 935 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
933 936 {
934 937 struct stdata *stp;
935 938
936 939 ASSERT(vp->v_stream);
937 940 stp = vp->v_stream;
938 941 /* Have to hold sd_lock to prevent siglist from changing */
939 942 mutex_enter(&stp->sd_lock);
940 943 if (stp->sd_sigflags & event)
941 944 strsendsig(stp->sd_siglist, event, band, error);
942 945 mutex_exit(&stp->sd_lock);
|
↓ open down ↓ |
709 lines elided |
↑ open up ↑ |
943 946 }
944 947
945 948 /*
946 949 * Send the "sevent" set of signals to a process.
947 950 * This might send more than one signal if the process is registered
948 951 * for multiple events. The caller should pass in an sevent that only
949 952 * includes the events for which the process has registered.
950 953 */
951 954 static void
952 955 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
953 - uchar_t band, int error)
956 + uchar_t band, int error)
954 957 {
955 958 ASSERT(MUTEX_HELD(&proc->p_lock));
956 959
957 960 info->si_band = 0;
958 961 info->si_errno = 0;
959 962
960 963 if (sevent & S_ERROR) {
961 964 sevent &= ~S_ERROR;
962 965 info->si_code = POLL_ERR;
963 966 info->si_errno = error;
964 967 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
965 968 "strsendsig:proc %p info %p", proc, info);
966 969 sigaddq(proc, NULL, info, KM_NOSLEEP);
967 970 info->si_errno = 0;
968 971 }
969 972 if (sevent & S_HANGUP) {
970 973 sevent &= ~S_HANGUP;
971 974 info->si_code = POLL_HUP;
972 975 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
973 976 "strsendsig:proc %p info %p", proc, info);
974 977 sigaddq(proc, NULL, info, KM_NOSLEEP);
975 978 }
976 979 if (sevent & S_HIPRI) {
977 980 sevent &= ~S_HIPRI;
978 981 info->si_code = POLL_PRI;
979 982 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
980 983 "strsendsig:proc %p info %p", proc, info);
981 984 sigaddq(proc, NULL, info, KM_NOSLEEP);
982 985 }
983 986 if (sevent & S_RDBAND) {
984 987 sevent &= ~S_RDBAND;
985 988 if (events & S_BANDURG)
986 989 sigtoproc(proc, NULL, SIGURG);
987 990 else
988 991 sigtoproc(proc, NULL, SIGPOLL);
989 992 }
990 993 if (sevent & S_WRBAND) {
991 994 sevent &= ~S_WRBAND;
992 995 sigtoproc(proc, NULL, SIGPOLL);
993 996 }
994 997 if (sevent & S_INPUT) {
995 998 sevent &= ~S_INPUT;
996 999 info->si_code = POLL_IN;
997 1000 info->si_band = band;
998 1001 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
999 1002 "strsendsig:proc %p info %p", proc, info);
1000 1003 sigaddq(proc, NULL, info, KM_NOSLEEP);
1001 1004 info->si_band = 0;
1002 1005 }
1003 1006 if (sevent & S_OUTPUT) {
1004 1007 sevent &= ~S_OUTPUT;
1005 1008 info->si_code = POLL_OUT;
1006 1009 info->si_band = band;
1007 1010 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1008 1011 "strsendsig:proc %p info %p", proc, info);
1009 1012 sigaddq(proc, NULL, info, KM_NOSLEEP);
1010 1013 info->si_band = 0;
1011 1014 }
1012 1015 if (sevent & S_MSG) {
1013 1016 sevent &= ~S_MSG;
1014 1017 info->si_code = POLL_MSG;
1015 1018 info->si_band = band;
1016 1019 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1017 1020 "strsendsig:proc %p info %p", proc, info);
1018 1021 sigaddq(proc, NULL, info, KM_NOSLEEP);
1019 1022 info->si_band = 0;
1020 1023 }
1021 1024 if (sevent & S_RDNORM) {
1022 1025 sevent &= ~S_RDNORM;
1023 1026 sigtoproc(proc, NULL, SIGPOLL);
1024 1027 }
1025 1028 if (sevent != 0) {
1026 1029 panic("strsendsig: unknown event(s) %x", sevent);
1027 1030 }
1028 1031 }
1029 1032
1030 1033 /*
1031 1034 * Send SIGPOLL/SIGURG signal to all processes and process groups
1032 1035 * registered on the given signal list that want a signal for at
1033 1036 * least one of the specified events.
1034 1037 *
1035 1038 * Must be called with exclusive access to siglist (caller holding sd_lock).
1036 1039 *
1037 1040 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1038 1041 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1039 1042 * while it is in the siglist.
1040 1043 *
1041 1044 * For performance reasons (MP scalability) the code drops pidlock
1042 1045 * when sending signals to a single process.
1043 1046 * When sending to a process group the code holds
1044 1047 * pidlock to prevent the membership in the process group from changing
1045 1048 * while walking the p_pglink list.
1046 1049 */
1047 1050 void
1048 1051 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1049 1052 {
1050 1053 strsig_t *ssp;
1051 1054 k_siginfo_t info;
1052 1055 struct pid *pidp;
1053 1056 proc_t *proc;
1054 1057
1055 1058 info.si_signo = SIGPOLL;
1056 1059 info.si_errno = 0;
1057 1060 for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1058 1061 int sevent;
1059 1062
1060 1063 sevent = ssp->ss_events & event;
1061 1064 if (sevent == 0)
1062 1065 continue;
1063 1066
1064 1067 if ((pidp = ssp->ss_pidp) == NULL) {
1065 1068 /* pid was released but still on event list */
1066 1069 continue;
1067 1070 }
1068 1071
1069 1072
1070 1073 if (ssp->ss_pid > 0) {
1071 1074 /*
1072 1075 * XXX This unfortunately still generates
1073 1076 * a signal when a fd is closed but
1074 1077 * the proc is active.
1075 1078 */
1076 1079 ASSERT(ssp->ss_pid == pidp->pid_id);
1077 1080
1078 1081 mutex_enter(&pidlock);
1079 1082 proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1080 1083 if (proc == NULL) {
1081 1084 mutex_exit(&pidlock);
1082 1085 continue;
1083 1086 }
1084 1087 mutex_enter(&proc->p_lock);
1085 1088 mutex_exit(&pidlock);
1086 1089 dosendsig(proc, ssp->ss_events, sevent, &info,
1087 1090 band, error);
1088 1091 mutex_exit(&proc->p_lock);
1089 1092 } else {
1090 1093 /*
1091 1094 * Send to process group. Hold pidlock across
1092 1095 * calls to dosendsig().
1093 1096 */
1094 1097 pid_t pgrp = -ssp->ss_pid;
1095 1098
1096 1099 mutex_enter(&pidlock);
1097 1100 proc = pgfind_zone(pgrp, ALL_ZONES);
1098 1101 while (proc != NULL) {
1099 1102 mutex_enter(&proc->p_lock);
1100 1103 dosendsig(proc, ssp->ss_events, sevent,
1101 1104 &info, band, error);
1102 1105 mutex_exit(&proc->p_lock);
1103 1106 proc = proc->p_pglink;
1104 1107 }
1105 1108 mutex_exit(&pidlock);
1106 1109 }
1107 1110 }
1108 1111 }
1109 1112
1110 1113 /*
1111 1114 * Attach a stream device or module.
1112 1115 * qp is a read queue; the new queue goes in so its next
1113 1116 * read ptr is the argument, and the write queue corresponding
1114 1117 * to the argument points to this queue. Return 0 on success,
1115 1118 * or a non-zero errno on failure.
1116 1119 */
1117 1120 int
1118 1121 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1119 1122 boolean_t is_insert)
1120 1123 {
1121 1124 major_t major;
1122 1125 cdevsw_impl_t *dp;
1123 1126 struct streamtab *str;
1124 1127 queue_t *rq;
1125 1128 queue_t *wrq;
1126 1129 uint32_t qflag;
1127 1130 uint32_t sqtype;
1128 1131 perdm_t *dmp;
1129 1132 int error;
1130 1133 int sflag;
1131 1134
1132 1135 rq = allocq();
1133 1136 wrq = _WR(rq);
1134 1137 STREAM(rq) = STREAM(wrq) = STREAM(qp);
1135 1138
1136 1139 if (fp != NULL) {
1137 1140 str = fp->f_str;
1138 1141 qflag = fp->f_qflag;
1139 1142 sqtype = fp->f_sqtype;
1140 1143 dmp = fp->f_dmp;
1141 1144 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1142 1145 sflag = MODOPEN;
1143 1146
1144 1147 /*
1145 1148 * stash away a pointer to the module structure so we can
1146 1149 * unref it in qdetach.
1147 1150 */
1148 1151 rq->q_fp = fp;
1149 1152 } else {
1150 1153 ASSERT(!is_insert);
1151 1154
1152 1155 major = getmajor(*devp);
1153 1156 dp = &devimpl[major];
1154 1157
1155 1158 str = dp->d_str;
1156 1159 ASSERT(str == STREAMSTAB(major));
1157 1160
1158 1161 qflag = dp->d_qflag;
1159 1162 ASSERT(qflag & QISDRV);
1160 1163 sqtype = dp->d_sqtype;
1161 1164
1162 1165 /* create perdm_t if needed */
1163 1166 if (NEED_DM(dp->d_dmp, qflag))
1164 1167 dp->d_dmp = hold_dm(str, qflag, sqtype);
1165 1168
1166 1169 dmp = dp->d_dmp;
1167 1170 sflag = 0;
1168 1171 }
1169 1172
1170 1173 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1171 1174 "qattach:qflag == %X(%X)", qflag, *devp);
1172 1175
1173 1176 /* setq might sleep in allocator - avoid holding locks. */
1174 1177 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1175 1178
1176 1179 /*
1177 1180 * Before calling the module's open routine, set up the q_next
1178 1181 * pointer for inserting a module in the middle of a stream.
1179 1182 *
1180 1183 * Note that we can always set _QINSERTING and set up q_next
1181 1184 * pointer for both inserting and pushing a module. Then there
1182 1185 * is no need for the is_insert parameter. In insertq(), called
1183 1186 * by qprocson(), assume that q_next of the new module always points
1184 1187 * to the correct queue and use it for insertion. Everything should
1185 1188 * work out fine. But in the first release of _I_INSERT, we
1186 1189 * distinguish between inserting and pushing to make sure that
1187 1190 * pushing a module follows the same code path as before.
1188 1191 */
1189 1192 if (is_insert) {
1190 1193 rq->q_flag |= _QINSERTING;
1191 1194 rq->q_next = qp;
1192 1195 }
1193 1196
1194 1197 /*
1195 1198 * If there is an outer perimeter get exclusive access during
1196 1199 * the open procedure. Bump up the reference count on the queue.
1197 1200 */
1198 1201 entersq(rq->q_syncq, SQ_OPENCLOSE);
1199 1202 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1200 1203 if (error != 0)
1201 1204 goto failed;
1202 1205 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1203 1206 ASSERT(qprocsareon(rq));
1204 1207 return (0);
1205 1208
1206 1209 failed:
1207 1210 rq->q_flag &= ~_QINSERTING;
1208 1211 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1209 1212 qprocsoff(rq);
1210 1213 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1211 1214 rq->q_next = wrq->q_next = NULL;
1212 1215 qdetach(rq, 0, 0, crp, B_FALSE);
1213 1216 return (error);
1214 1217 }
1215 1218
1216 1219 /*
1217 1220 * Handle second open of stream. For modules, set the
1218 1221 * last argument to MODOPEN and do not pass any open flags.
1219 1222 * Ignore dummydev since this is not the first open.
1220 1223 */
1221 1224 int
1222 1225 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1223 1226 {
1224 1227 int error;
1225 1228 dev_t dummydev;
1226 1229 queue_t *wqp = _WR(qp);
1227 1230
1228 1231 ASSERT(qp->q_flag & QREADR);
1229 1232 entersq(qp->q_syncq, SQ_OPENCLOSE);
1230 1233
1231 1234 dummydev = *devp;
1232 1235 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1233 1236 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1234 1237 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1235 1238 mutex_enter(&STREAM(qp)->sd_lock);
1236 1239 qp->q_stream->sd_flag |= STREOPENFAIL;
1237 1240 mutex_exit(&STREAM(qp)->sd_lock);
1238 1241 return (error);
1239 1242 }
1240 1243 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1241 1244
1242 1245 /*
1243 1246 * successful open should have done qprocson()
1244 1247 */
1245 1248 ASSERT(qprocsareon(_RD(qp)));
1246 1249 return (0);
1247 1250 }
1248 1251
1249 1252 /*
1250 1253 * Detach a stream module or device.
1251 1254 * If clmode == 1 then the module or driver was opened and its
1252 1255 * close routine must be called. If clmode == 0, the module
1253 1256 * or driver was never opened or the open failed, and so its close
1254 1257 * should not be called.
1255 1258 */
1256 1259 void
1257 1260 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1258 1261 {
1259 1262 queue_t *wqp = _WR(qp);
1260 1263 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1261 1264
1262 1265 if (STREAM_NEEDSERVICE(STREAM(qp)))
1263 1266 stream_runservice(STREAM(qp));
1264 1267
1265 1268 if (clmode) {
1266 1269 /*
1267 1270 * Make sure that all the messages on the write side syncq are
1268 1271 * processed and nothing is left. Since we are closing, no new
1269 1272 * messages may appear there.
1270 1273 */
1271 1274 wait_q_syncq(wqp);
1272 1275
1273 1276 entersq(qp->q_syncq, SQ_OPENCLOSE);
1274 1277 if (is_remove) {
1275 1278 mutex_enter(QLOCK(qp));
1276 1279 qp->q_flag |= _QREMOVING;
1277 1280 mutex_exit(QLOCK(qp));
1278 1281 }
1279 1282 (*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1280 1283 /*
1281 1284 * Check that qprocsoff() was actually called.
1282 1285 */
1283 1286 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1284 1287
1285 1288 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1286 1289 } else {
1287 1290 disable_svc(qp);
1288 1291 }
1289 1292
1290 1293 /*
1291 1294 * Allow any threads blocked in entersq to proceed and discover
1292 1295 * the QWCLOSE is set.
1293 1296 * Note: This assumes that all users of entersq check QWCLOSE.
1294 1297 * Currently runservice is the only entersq that can happen
1295 1298 * after removeq has finished.
1296 1299 * Removeq will have discarded all messages destined to the closing
1297 1300 * pair of queues from the syncq.
1298 1301 * NOTE: Calling a function inside an assert is unconventional.
1299 1302 * However, it does not cause any problem since flush_syncq() does
1300 1303 * not change any state except when it returns non-zero i.e.
1301 1304 * when the assert will trigger.
1302 1305 */
1303 1306 ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1304 1307 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1305 1308 ASSERT((qp->q_flag & QPERMOD) ||
1306 1309 ((qp->q_syncq->sq_head == NULL) &&
1307 1310 (wqp->q_syncq->sq_head == NULL)));
1308 1311
1309 1312 /* release any fmodsw_impl_t structure held on behalf of the queue */
1310 1313 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1311 1314 if (qp->q_fp != NULL)
1312 1315 fmodsw_rele(qp->q_fp);
1313 1316
1314 1317 /* freeq removes us from the outer perimeter if any */
1315 1318 freeq(qp);
1316 1319 }
1317 1320
1318 1321 /* Prevent service procedures from being called */
1319 1322 void
1320 1323 disable_svc(queue_t *qp)
1321 1324 {
1322 1325 queue_t *wqp = _WR(qp);
1323 1326
1324 1327 ASSERT(qp->q_flag & QREADR);
1325 1328 mutex_enter(QLOCK(qp));
1326 1329 qp->q_flag |= QWCLOSE;
1327 1330 mutex_exit(QLOCK(qp));
1328 1331 mutex_enter(QLOCK(wqp));
1329 1332 wqp->q_flag |= QWCLOSE;
1330 1333 mutex_exit(QLOCK(wqp));
1331 1334 }
1332 1335
1333 1336 /* Allow service procedures to be called again */
1334 1337 void
1335 1338 enable_svc(queue_t *qp)
1336 1339 {
1337 1340 queue_t *wqp = _WR(qp);
1338 1341
1339 1342 ASSERT(qp->q_flag & QREADR);
1340 1343 mutex_enter(QLOCK(qp));
1341 1344 qp->q_flag &= ~QWCLOSE;
1342 1345 mutex_exit(QLOCK(qp));
1343 1346 mutex_enter(QLOCK(wqp));
1344 1347 wqp->q_flag &= ~QWCLOSE;
1345 1348 mutex_exit(QLOCK(wqp));
1346 1349 }
1347 1350
1348 1351 /*
1349 1352 * Remove queue from qhead/qtail if it is enabled.
1350 1353 * Only reset QENAB if the queue was removed from the runlist.
1351 1354 * A queue goes through 3 stages:
1352 1355 * It is on the service list and QENAB is set.
1353 1356 * It is removed from the service list but QENAB is still set.
1354 1357 * QENAB gets changed to QINSERVICE.
1355 1358 * QINSERVICE is reset (when the service procedure is done)
1356 1359 * Thus we can not reset QENAB unless we actually removed it from the service
1357 1360 * queue.
1358 1361 */
1359 1362 void
1360 1363 remove_runlist(queue_t *qp)
1361 1364 {
1362 1365 if (qp->q_flag & QENAB && qhead != NULL) {
1363 1366 queue_t *q_chase;
1364 1367 queue_t *q_curr;
1365 1368 int removed;
1366 1369
1367 1370 mutex_enter(&service_queue);
1368 1371 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1369 1372 mutex_exit(&service_queue);
1370 1373 if (removed) {
1371 1374 STRSTAT(qremoved);
1372 1375 qp->q_flag &= ~QENAB;
1373 1376 }
1374 1377 }
1375 1378 }
1376 1379
1377 1380
1378 1381 /*
1379 1382 * Wait for any pending service processing to complete.
1380 1383 * The removal of queues from the runlist is not atomic with the
1381 1384 * clearing of the QENABLED flag and setting the INSERVICE flag.
1382 1385 * consequently it is possible for remove_runlist in strclose
1383 1386 * to not find the queue on the runlist but for it to be QENABLED
1384 1387 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1385 1388 * as well as INSERVICE.
1386 1389 */
1387 1390 void
1388 1391 wait_svc(queue_t *qp)
1389 1392 {
1390 1393 queue_t *wqp = _WR(qp);
1391 1394
1392 1395 ASSERT(qp->q_flag & QREADR);
1393 1396
1394 1397 /*
1395 1398 * Try to remove queues from qhead/qtail list.
1396 1399 */
1397 1400 if (qhead != NULL) {
1398 1401 remove_runlist(qp);
1399 1402 remove_runlist(wqp);
1400 1403 }
1401 1404 /*
1402 1405 * Wait till the syncqs associated with the queue disappear from the
1403 1406 * background processing list.
1404 1407 * This only needs to be done for non-PERMOD perimeters since
1405 1408 * for PERMOD perimeters the syncq may be shared and will only be freed
1406 1409 * when the last module/driver is unloaded.
1407 1410 * If for PERMOD perimeters queue was on the syncq list, removeq()
1408 1411 * should call propagate_syncq() or drain_syncq() for it. Both of these
1409 1412 * functions remove the queue from its syncq list, so sqthread will not
1410 1413 * try to access the queue.
1411 1414 */
1412 1415 if (!(qp->q_flag & QPERMOD)) {
1413 1416 syncq_t *rsq = qp->q_syncq;
1414 1417 syncq_t *wsq = wqp->q_syncq;
1415 1418
1416 1419 /*
1417 1420 * Disable rsq and wsq and wait for any background processing of
1418 1421 * syncq to complete.
1419 1422 */
1420 1423 wait_sq_svc(rsq);
1421 1424 if (wsq != rsq)
1422 1425 wait_sq_svc(wsq);
1423 1426 }
1424 1427
1425 1428 mutex_enter(QLOCK(qp));
1426 1429 while (qp->q_flag & (QINSERVICE|QENAB))
1427 1430 cv_wait(&qp->q_wait, QLOCK(qp));
1428 1431 mutex_exit(QLOCK(qp));
1429 1432 mutex_enter(QLOCK(wqp));
1430 1433 while (wqp->q_flag & (QINSERVICE|QENAB))
1431 1434 cv_wait(&wqp->q_wait, QLOCK(wqp));
1432 1435 mutex_exit(QLOCK(wqp));
1433 1436 }
1434 1437
1435 1438 /*
1436 1439 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1437 1440 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1438 1441 * also be set, and is passed through to allocb_cred_wait().
1439 1442 *
1440 1443 * Returns errno on failure, zero on success.
1441 1444 */
1442 1445 int
1443 1446 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1444 1447 {
1445 1448 mblk_t *tmp;
1446 1449 ssize_t count;
1447 1450 int error = 0;
1448 1451
1449 1452 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1450 1453 (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1451 1454
1452 1455 if (bp->b_datap->db_type == M_IOCTL) {
1453 1456 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1454 1457 } else {
1455 1458 ASSERT(bp->b_datap->db_type == M_COPYIN);
1456 1459 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1457 1460 }
1458 1461 /*
1459 1462 * strdoioctl validates ioc_count, so if this assert fails it
1460 1463 * cannot be due to user error.
1461 1464 */
1462 1465 ASSERT(count >= 0);
1463 1466
1464 1467 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1465 1468 curproc->p_pid)) == NULL) {
1466 1469 return (error);
1467 1470 }
1468 1471 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1469 1472 if (error != 0) {
1470 1473 freeb(tmp);
1471 1474 return (error);
1472 1475 }
1473 1476 DB_CPID(tmp) = curproc->p_pid;
1474 1477 tmp->b_wptr += count;
1475 1478 bp->b_cont = tmp;
1476 1479
1477 1480 return (0);
1478 1481 }
1479 1482
1480 1483 /*
1481 1484 * Copy ioctl data to user-land. Return non-zero errno on failure,
1482 1485 * 0 for success.
1483 1486 */
1484 1487 int
1485 1488 getiocd(mblk_t *bp, char *arg, int copymode)
1486 1489 {
1487 1490 ssize_t count;
1488 1491 size_t n;
1489 1492 int error;
1490 1493
1491 1494 if (bp->b_datap->db_type == M_IOCACK)
1492 1495 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1493 1496 else {
1494 1497 ASSERT(bp->b_datap->db_type == M_COPYOUT);
1495 1498 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1496 1499 }
1497 1500 ASSERT(count >= 0);
1498 1501
1499 1502 for (bp = bp->b_cont; bp && count;
1500 1503 count -= n, bp = bp->b_cont, arg += n) {
1501 1504 n = MIN(count, bp->b_wptr - bp->b_rptr);
1502 1505 error = strcopyout(bp->b_rptr, arg, n, copymode);
1503 1506 if (error)
1504 1507 return (error);
1505 1508 }
1506 1509 ASSERT(count == 0);
1507 1510 return (0);
1508 1511 }
1509 1512
1510 1513 /*
1511 1514 * Allocate a linkinfo entry given the write queue of the
1512 1515 * bottom module of the top stream and the write queue of the
1513 1516 * stream head of the bottom stream.
1514 1517 */
1515 1518 linkinfo_t *
1516 1519 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1517 1520 {
1518 1521 linkinfo_t *linkp;
1519 1522
1520 1523 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1521 1524
1522 1525 linkp->li_lblk.l_qtop = qup;
1523 1526 linkp->li_lblk.l_qbot = qdown;
1524 1527 linkp->li_fpdown = fpdown;
1525 1528
1526 1529 mutex_enter(&strresources);
1527 1530 linkp->li_next = linkinfo_list;
1528 1531 linkp->li_prev = NULL;
1529 1532 if (linkp->li_next)
1530 1533 linkp->li_next->li_prev = linkp;
1531 1534 linkinfo_list = linkp;
1532 1535 linkp->li_lblk.l_index = ++lnk_id;
1533 1536 ASSERT(lnk_id != 0); /* this should never wrap in practice */
1534 1537 mutex_exit(&strresources);
1535 1538
1536 1539 return (linkp);
1537 1540 }
1538 1541
1539 1542 /*
1540 1543 * Free a linkinfo entry.
1541 1544 */
1542 1545 void
1543 1546 lbfree(linkinfo_t *linkp)
1544 1547 {
1545 1548 mutex_enter(&strresources);
1546 1549 if (linkp->li_next)
1547 1550 linkp->li_next->li_prev = linkp->li_prev;
1548 1551 if (linkp->li_prev)
1549 1552 linkp->li_prev->li_next = linkp->li_next;
1550 1553 else
1551 1554 linkinfo_list = linkp->li_next;
1552 1555 mutex_exit(&strresources);
1553 1556
1554 1557 kmem_cache_free(linkinfo_cache, linkp);
1555 1558 }
1556 1559
1557 1560 /*
1558 1561 * Check for a potential linking cycle.
1559 1562 * Return 1 if a link will result in a cycle,
1560 1563 * and 0 otherwise.
1561 1564 */
1562 1565 int
1563 1566 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1564 1567 {
1565 1568 struct mux_node *np;
1566 1569 struct mux_edge *ep;
1567 1570 int i;
1568 1571 major_t lomaj;
1569 1572 major_t upmaj;
1570 1573 /*
1571 1574 * if the lower stream is a pipe/FIFO, return, since link
1572 1575 * cycles can not happen on pipes/FIFOs
1573 1576 */
1574 1577 if (lostp->sd_vnode->v_type == VFIFO)
1575 1578 return (0);
1576 1579
1577 1580 for (i = 0; i < ss->ss_devcnt; i++) {
1578 1581 np = &ss->ss_mux_nodes[i];
1579 1582 MUX_CLEAR(np);
1580 1583 }
1581 1584 lomaj = getmajor(lostp->sd_vnode->v_rdev);
1582 1585 upmaj = getmajor(upstp->sd_vnode->v_rdev);
1583 1586 np = &ss->ss_mux_nodes[lomaj];
1584 1587 for (;;) {
1585 1588 if (!MUX_DIDVISIT(np)) {
1586 1589 if (np->mn_imaj == upmaj)
1587 1590 return (1);
1588 1591 if (np->mn_outp == NULL) {
1589 1592 MUX_VISIT(np);
1590 1593 if (np->mn_originp == NULL)
1591 1594 return (0);
1592 1595 np = np->mn_originp;
1593 1596 continue;
1594 1597 }
1595 1598 MUX_VISIT(np);
1596 1599 np->mn_startp = np->mn_outp;
1597 1600 } else {
1598 1601 if (np->mn_startp == NULL) {
1599 1602 if (np->mn_originp == NULL)
1600 1603 return (0);
1601 1604 else {
1602 1605 np = np->mn_originp;
1603 1606 continue;
1604 1607 }
1605 1608 }
1606 1609 /*
1607 1610 * If ep->me_nodep is a FIFO (me_nodep == NULL),
1608 1611 * ignore the edge and move on. ep->me_nodep gets
1609 1612 * set to NULL in mux_addedge() if it is a FIFO.
1610 1613 *
1611 1614 */
1612 1615 ep = np->mn_startp;
1613 1616 np->mn_startp = ep->me_nextp;
1614 1617 if (ep->me_nodep == NULL)
1615 1618 continue;
1616 1619 ep->me_nodep->mn_originp = np;
1617 1620 np = ep->me_nodep;
1618 1621 }
1619 1622 }
1620 1623 }
1621 1624
1622 1625 /*
1623 1626 * Find linkinfo entry corresponding to the parameters.
1624 1627 */
1625 1628 linkinfo_t *
1626 1629 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1627 1630 {
1628 1631 linkinfo_t *linkp;
1629 1632 struct mux_edge *mep;
1630 1633 struct mux_node *mnp;
1631 1634 queue_t *qup;
1632 1635
1633 1636 mutex_enter(&strresources);
1634 1637 if ((type & LINKTYPEMASK) == LINKNORMAL) {
1635 1638 qup = getendq(stp->sd_wrq);
1636 1639 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1637 1640 if ((qup == linkp->li_lblk.l_qtop) &&
1638 1641 (!index || (index == linkp->li_lblk.l_index))) {
1639 1642 mutex_exit(&strresources);
1640 1643 return (linkp);
1641 1644 }
1642 1645 }
1643 1646 } else {
1644 1647 ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1645 1648 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1646 1649 mep = mnp->mn_outp;
1647 1650 while (mep) {
1648 1651 if ((index == 0) || (index == mep->me_muxid))
1649 1652 break;
1650 1653 mep = mep->me_nextp;
1651 1654 }
1652 1655 if (!mep) {
1653 1656 mutex_exit(&strresources);
1654 1657 return (NULL);
1655 1658 }
1656 1659 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1657 1660 if ((!linkp->li_lblk.l_qtop) &&
1658 1661 (mep->me_muxid == linkp->li_lblk.l_index)) {
1659 1662 mutex_exit(&strresources);
1660 1663 return (linkp);
1661 1664 }
1662 1665 }
1663 1666 }
1664 1667 mutex_exit(&strresources);
1665 1668 return (NULL);
1666 1669 }
1667 1670
1668 1671 /*
1669 1672 * Given a queue ptr, follow the chain of q_next pointers until you reach the
1670 1673 * last queue on the chain and return it.
1671 1674 */
1672 1675 queue_t *
1673 1676 getendq(queue_t *q)
1674 1677 {
1675 1678 ASSERT(q != NULL);
1676 1679 while (_SAMESTR(q))
1677 1680 q = q->q_next;
1678 1681 return (q);
1679 1682 }
1680 1683
1681 1684 /*
1682 1685 * Wait for the syncq count to drop to zero.
1683 1686 * sq could be either outer or inner.
1684 1687 */
1685 1688
1686 1689 static void
1687 1690 wait_syncq(syncq_t *sq)
1688 1691 {
1689 1692 uint16_t count;
1690 1693
1691 1694 mutex_enter(SQLOCK(sq));
1692 1695 count = sq->sq_count;
1693 1696 SQ_PUTLOCKS_ENTER(sq);
1694 1697 SUM_SQ_PUTCOUNTS(sq, count);
1695 1698 while (count != 0) {
1696 1699 sq->sq_flags |= SQ_WANTWAKEUP;
1697 1700 SQ_PUTLOCKS_EXIT(sq);
1698 1701 cv_wait(&sq->sq_wait, SQLOCK(sq));
1699 1702 count = sq->sq_count;
1700 1703 SQ_PUTLOCKS_ENTER(sq);
1701 1704 SUM_SQ_PUTCOUNTS(sq, count);
1702 1705 }
1703 1706 SQ_PUTLOCKS_EXIT(sq);
1704 1707 mutex_exit(SQLOCK(sq));
1705 1708 }
1706 1709
1707 1710 /*
1708 1711 * Wait while there are any messages for the queue in its syncq.
1709 1712 */
1710 1713 static void
1711 1714 wait_q_syncq(queue_t *q)
1712 1715 {
1713 1716 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1714 1717 syncq_t *sq = q->q_syncq;
1715 1718
1716 1719 mutex_enter(SQLOCK(sq));
1717 1720 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1718 1721 sq->sq_flags |= SQ_WANTWAKEUP;
1719 1722 cv_wait(&sq->sq_wait, SQLOCK(sq));
1720 1723 }
1721 1724 mutex_exit(SQLOCK(sq));
1722 1725 }
1723 1726 }
1724 1727
1725 1728
1726 1729 int
1727 1730 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1728 1731 int lhlink)
1729 1732 {
1730 1733 struct stdata *stp;
1731 1734 struct strioctl strioc;
1732 1735 struct linkinfo *linkp;
1733 1736 struct stdata *stpdown;
1734 1737 struct streamtab *str;
1735 1738 queue_t *passq;
1736 1739 syncq_t *passyncq;
1737 1740 queue_t *rq;
1738 1741 cdevsw_impl_t *dp;
1739 1742 uint32_t qflag;
1740 1743 uint32_t sqtype;
1741 1744 perdm_t *dmp;
1742 1745 int error = 0;
1743 1746 netstack_t *ns;
1744 1747 str_stack_t *ss;
1745 1748
1746 1749 stp = vp->v_stream;
1747 1750 TRACE_1(TR_FAC_STREAMS_FR,
1748 1751 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1749 1752 /*
1750 1753 * Test for invalid upper stream
1751 1754 */
1752 1755 if (stp->sd_flag & STRHUP) {
1753 1756 return (ENXIO);
1754 1757 }
1755 1758 if (vp->v_type == VFIFO) {
1756 1759 return (EINVAL);
1757 1760 }
1758 1761 if (stp->sd_strtab == NULL) {
1759 1762 return (EINVAL);
1760 1763 }
1761 1764 if (!stp->sd_strtab->st_muxwinit) {
1762 1765 return (EINVAL);
1763 1766 }
1764 1767 if (fpdown == NULL) {
1765 1768 return (EBADF);
1766 1769 }
1767 1770 ns = netstack_find_by_cred(crp);
1768 1771 ASSERT(ns != NULL);
1769 1772 ss = ns->netstack_str;
1770 1773 ASSERT(ss != NULL);
1771 1774
1772 1775 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1773 1776 netstack_rele(ss->ss_netstack);
1774 1777 return (EINVAL);
1775 1778 }
1776 1779 mutex_enter(&muxifier);
1777 1780 if (stp->sd_flag & STPLEX) {
1778 1781 mutex_exit(&muxifier);
1779 1782 netstack_rele(ss->ss_netstack);
1780 1783 return (ENXIO);
1781 1784 }
1782 1785
1783 1786 /*
1784 1787 * Test for invalid lower stream.
1785 1788 * The check for the v_type != VFIFO and having a major
1786 1789 * number not >= devcnt is done to avoid problems with
1787 1790 * adding mux_node entry past the end of mux_nodes[].
1788 1791 * For FIFO's we don't add an entry so this isn't a
1789 1792 * problem.
1790 1793 */
1791 1794 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1792 1795 (stpdown == stp) || (stpdown->sd_flag &
1793 1796 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1794 1797 ((stpdown->sd_vnode->v_type != VFIFO) &&
1795 1798 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1796 1799 linkcycle(stp, stpdown, ss)) {
1797 1800 mutex_exit(&muxifier);
1798 1801 netstack_rele(ss->ss_netstack);
1799 1802 return (EINVAL);
1800 1803 }
1801 1804 TRACE_1(TR_FAC_STREAMS_FR,
1802 1805 TR_STPDOWN, "stpdown:%p", stpdown);
1803 1806 rq = getendq(stp->sd_wrq);
1804 1807 if (cmd == I_PLINK)
1805 1808 rq = NULL;
1806 1809
1807 1810 linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1808 1811
1809 1812 strioc.ic_cmd = cmd;
1810 1813 strioc.ic_timout = INFTIM;
1811 1814 strioc.ic_len = sizeof (struct linkblk);
1812 1815 strioc.ic_dp = (char *)&linkp->li_lblk;
1813 1816
1814 1817 /*
1815 1818 * STRPLUMB protects plumbing changes and should be set before
1816 1819 * link_addpassthru()/link_rempassthru() are called, so it is set here
1817 1820 * and cleared in the end of mlink when passthru queue is removed.
1818 1821 * Setting of STRPLUMB prevents reopens of the stream while passthru
1819 1822 * queue is in-place (it is not a proper module and doesn't have open
1820 1823 * entry point).
1821 1824 *
1822 1825 * STPLEX prevents any threads from entering the stream from above. It
1823 1826 * can't be set before the call to link_addpassthru() because putnext
1824 1827 * from below may cause stream head I/O routines to be called and these
1825 1828 * routines assert that STPLEX is not set. After link_addpassthru()
1826 1829 * nothing may come from below since the pass queue syncq is blocked.
1827 1830 * Note also that STPLEX should be cleared before the call to
1828 1831 * link_rempassthru() since when messages start flowing to the stream
1829 1832 * head (e.g. because of message propagation from the pass queue) stream
1830 1833 * head I/O routines may be called with STPLEX flag set.
1831 1834 *
1832 1835 * When STPLEX is set, nothing may come into the stream from above and
1833 1836 * it is safe to do a setq which will change stream head. So, the
1834 1837 * correct sequence of actions is:
1835 1838 *
1836 1839 * 1) Set STRPLUMB
1837 1840 * 2) Call link_addpassthru()
1838 1841 * 3) Set STPLEX
1839 1842 * 4) Call setq and update the stream state
1840 1843 * 5) Clear STPLEX
1841 1844 * 6) Call link_rempassthru()
1842 1845 * 7) Clear STRPLUMB
1843 1846 *
1844 1847 * The same sequence applies to munlink() code.
1845 1848 */
1846 1849 mutex_enter(&stpdown->sd_lock);
1847 1850 stpdown->sd_flag |= STRPLUMB;
1848 1851 mutex_exit(&stpdown->sd_lock);
1849 1852 /*
1850 1853 * Add passthru queue below lower mux. This will block
1851 1854 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1852 1855 */
1853 1856 passq = link_addpassthru(stpdown);
1854 1857
1855 1858 mutex_enter(&stpdown->sd_lock);
1856 1859 stpdown->sd_flag |= STPLEX;
1857 1860 mutex_exit(&stpdown->sd_lock);
1858 1861
1859 1862 rq = _RD(stpdown->sd_wrq);
1860 1863 /*
1861 1864 * There may be messages in the streamhead's syncq due to messages
1862 1865 * that arrived before link_addpassthru() was done. To avoid
1863 1866 * background processing of the syncq happening simultaneous with
1864 1867 * setq processing, we disable the streamhead syncq and wait until
1865 1868 * existing background thread finishes working on it.
1866 1869 */
1867 1870 wait_sq_svc(rq->q_syncq);
1868 1871 passyncq = passq->q_syncq;
1869 1872 if (!(passyncq->sq_flags & SQ_BLOCKED))
1870 1873 blocksq(passyncq, SQ_BLOCKED, 0);
1871 1874
1872 1875 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1873 1876 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1874 1877 rq->q_ptr = _WR(rq)->q_ptr = NULL;
1875 1878
1876 1879 /* setq might sleep in allocator - avoid holding locks. */
1877 1880 /* Note: we are holding muxifier here. */
1878 1881
1879 1882 str = stp->sd_strtab;
1880 1883 dp = &devimpl[getmajor(vp->v_rdev)];
1881 1884 ASSERT(dp->d_str == str);
1882 1885
1883 1886 qflag = dp->d_qflag;
1884 1887 sqtype = dp->d_sqtype;
1885 1888
1886 1889 /* create perdm_t if needed */
1887 1890 if (NEED_DM(dp->d_dmp, qflag))
1888 1891 dp->d_dmp = hold_dm(str, qflag, sqtype);
1889 1892
1890 1893 dmp = dp->d_dmp;
1891 1894
1892 1895 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1893 1896 B_TRUE);
1894 1897
1895 1898 /*
1896 1899 * XXX Remove any "odd" messages from the queue.
1897 1900 * Keep only M_DATA, M_PROTO, M_PCPROTO.
1898 1901 */
1899 1902 error = strdoioctl(stp, &strioc, FNATIVE,
1900 1903 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1901 1904 if (error != 0) {
1902 1905 lbfree(linkp);
1903 1906
1904 1907 if (!(passyncq->sq_flags & SQ_BLOCKED))
1905 1908 blocksq(passyncq, SQ_BLOCKED, 0);
1906 1909 /*
1907 1910 * Restore the stream head queue and then remove
1908 1911 * the passq. Turn off STPLEX before we turn on
1909 1912 * the stream by removing the passq.
1910 1913 */
1911 1914 rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1912 1915 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1913 1916 B_TRUE);
1914 1917
1915 1918 mutex_enter(&stpdown->sd_lock);
1916 1919 stpdown->sd_flag &= ~STPLEX;
1917 1920 mutex_exit(&stpdown->sd_lock);
1918 1921
1919 1922 link_rempassthru(passq);
1920 1923
1921 1924 mutex_enter(&stpdown->sd_lock);
1922 1925 stpdown->sd_flag &= ~STRPLUMB;
1923 1926 /* Wakeup anyone waiting for STRPLUMB to clear. */
1924 1927 cv_broadcast(&stpdown->sd_monitor);
1925 1928 mutex_exit(&stpdown->sd_lock);
1926 1929
1927 1930 mutex_exit(&muxifier);
1928 1931 netstack_rele(ss->ss_netstack);
1929 1932 return (error);
1930 1933 }
1931 1934 mutex_enter(&fpdown->f_tlock);
1932 1935 fpdown->f_count++;
1933 1936 mutex_exit(&fpdown->f_tlock);
1934 1937
1935 1938 /*
1936 1939 * if we've made it here the linkage is all set up so we should also
1937 1940 * set up the layered driver linkages
1938 1941 */
1939 1942
1940 1943 ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1941 1944 if (cmd == I_LINK) {
1942 1945 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1943 1946 } else {
1944 1947 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1945 1948 }
1946 1949
1947 1950 link_rempassthru(passq);
1948 1951
1949 1952 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1950 1953
1951 1954 /*
1952 1955 * Mark the upper stream as having dependent links
1953 1956 * so that strclose can clean it up.
1954 1957 */
1955 1958 if (cmd == I_LINK) {
1956 1959 mutex_enter(&stp->sd_lock);
1957 1960 stp->sd_flag |= STRHASLINKS;
1958 1961 mutex_exit(&stp->sd_lock);
1959 1962 }
1960 1963 /*
1961 1964 * Wake up any other processes that may have been
1962 1965 * waiting on the lower stream. These will all
1963 1966 * error out.
1964 1967 */
1965 1968 mutex_enter(&stpdown->sd_lock);
1966 1969 /* The passthru module is removed so we may release STRPLUMB */
1967 1970 stpdown->sd_flag &= ~STRPLUMB;
1968 1971 cv_broadcast(&rq->q_wait);
1969 1972 cv_broadcast(&_WR(rq)->q_wait);
1970 1973 cv_broadcast(&stpdown->sd_monitor);
1971 1974 mutex_exit(&stpdown->sd_lock);
1972 1975 mutex_exit(&muxifier);
1973 1976 *rvalp = linkp->li_lblk.l_index;
1974 1977 netstack_rele(ss->ss_netstack);
1975 1978 return (0);
1976 1979 }
1977 1980
1978 1981 int
1979 1982 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1980 1983 {
1981 1984 int ret;
1982 1985 struct file *fpdown;
1983 1986
1984 1987 fpdown = getf(arg);
1985 1988 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1986 1989 if (fpdown != NULL)
1987 1990 releasef(arg);
1988 1991 return (ret);
1989 1992 }
1990 1993
1991 1994 /*
1992 1995 * Unlink a multiplexor link. Stp is the controlling stream for the
1993 1996 * link, and linkp points to the link's entry in the linkinfo list.
1994 1997 * The muxifier lock must be held on entry and is dropped on exit.
1995 1998 *
1996 1999 * NOTE : Currently it is assumed that mux would process all the messages
1997 2000 * sitting on it's queue before ACKing the UNLINK. It is the responsibility
1998 2001 * of the mux to handle all the messages that arrive before UNLINK.
1999 2002 * If the mux has to send down messages on its lower stream before
2000 2003 * ACKing I_UNLINK, then it *should* know to handle messages even
2001 2004 * after the UNLINK is acked (actually it should be able to handle till we
2002 2005 * re-block the read side of the pass queue here). If the mux does not
2003 2006 * open up the lower stream, any messages that arrive during UNLINK
2004 2007 * will be put in the stream head. In the case of lower stream opening
2005 2008 * up, some messages might land in the stream head depending on when
2006 2009 * the message arrived and when the read side of the pass queue was
2007 2010 * re-blocked.
2008 2011 */
2009 2012 int
2010 2013 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2011 2014 str_stack_t *ss)
2012 2015 {
2013 2016 struct strioctl strioc;
2014 2017 struct stdata *stpdown;
2015 2018 queue_t *rq, *wrq;
2016 2019 queue_t *passq;
2017 2020 syncq_t *passyncq;
2018 2021 int error = 0;
2019 2022 file_t *fpdown;
2020 2023
2021 2024 ASSERT(MUTEX_HELD(&muxifier));
2022 2025
2023 2026 stpdown = linkp->li_fpdown->f_vnode->v_stream;
2024 2027
2025 2028 /*
2026 2029 * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2027 2030 */
2028 2031 mutex_enter(&stpdown->sd_lock);
2029 2032 stpdown->sd_flag |= STRPLUMB;
2030 2033 mutex_exit(&stpdown->sd_lock);
2031 2034
2032 2035 /*
2033 2036 * Add passthru queue below lower mux. This will block
2034 2037 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2035 2038 */
2036 2039 passq = link_addpassthru(stpdown);
2037 2040
2038 2041 if ((flag & LINKTYPEMASK) == LINKNORMAL)
2039 2042 strioc.ic_cmd = I_UNLINK;
2040 2043 else
2041 2044 strioc.ic_cmd = I_PUNLINK;
2042 2045 strioc.ic_timout = INFTIM;
2043 2046 strioc.ic_len = sizeof (struct linkblk);
2044 2047 strioc.ic_dp = (char *)&linkp->li_lblk;
2045 2048
2046 2049 error = strdoioctl(stp, &strioc, FNATIVE,
2047 2050 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2048 2051
2049 2052 /*
2050 2053 * If there was an error and this is not called via strclose,
2051 2054 * return to the user. Otherwise, pretend there was no error
2052 2055 * and close the link.
2053 2056 */
2054 2057 if (error) {
2055 2058 if (flag & LINKCLOSE) {
2056 2059 cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2057 2060 "unlink ioctl, closing anyway (%d)\n", error);
2058 2061 } else {
2059 2062 link_rempassthru(passq);
2060 2063 mutex_enter(&stpdown->sd_lock);
2061 2064 stpdown->sd_flag &= ~STRPLUMB;
2062 2065 cv_broadcast(&stpdown->sd_monitor);
2063 2066 mutex_exit(&stpdown->sd_lock);
2064 2067 mutex_exit(&muxifier);
2065 2068 return (error);
2066 2069 }
2067 2070 }
2068 2071
2069 2072 mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2070 2073 fpdown = linkp->li_fpdown;
2071 2074 lbfree(linkp);
2072 2075
2073 2076 /*
2074 2077 * We go ahead and drop muxifier here--it's a nasty global lock that
2075 2078 * can slow others down. It's okay to since attempts to mlink() this
2076 2079 * stream will be stopped because STPLEX is still set in the stdata
2077 2080 * structure, and munlink() is stopped because mux_rmvedge() and
2078 2081 * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2079 2082 * respectively. Note that we defer the closef() of fpdown until
2080 2083 * after we drop muxifier since strclose() can call munlinkall().
2081 2084 */
2082 2085 mutex_exit(&muxifier);
2083 2086
2084 2087 wrq = stpdown->sd_wrq;
2085 2088 rq = _RD(wrq);
2086 2089
2087 2090 /*
2088 2091 * Get rid of outstanding service procedure runs, before we make
2089 2092 * it a stream head, since a stream head doesn't have any service
2090 2093 * procedure.
2091 2094 */
2092 2095 disable_svc(rq);
2093 2096 wait_svc(rq);
2094 2097
2095 2098 /*
2096 2099 * Since we don't disable the syncq for QPERMOD, we wait for whatever
2097 2100 * is queued up to be finished. mux should take care that nothing is
2098 2101 * send down to this queue. We should do it now as we're going to block
2099 2102 * passyncq if it was unblocked.
2100 2103 */
2101 2104 if (wrq->q_flag & QPERMOD) {
2102 2105 syncq_t *sq = wrq->q_syncq;
2103 2106
2104 2107 mutex_enter(SQLOCK(sq));
2105 2108 while (wrq->q_sqflags & Q_SQQUEUED) {
2106 2109 sq->sq_flags |= SQ_WANTWAKEUP;
2107 2110 cv_wait(&sq->sq_wait, SQLOCK(sq));
2108 2111 }
2109 2112 mutex_exit(SQLOCK(sq));
2110 2113 }
2111 2114 passyncq = passq->q_syncq;
2112 2115 if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2113 2116
2114 2117 syncq_t *sq, *outer;
2115 2118
2116 2119 /*
2117 2120 * Messages could be flowing from underneath. We will
2118 2121 * block the read side of the passq. This would be
2119 2122 * sufficient for QPAIR and QPERQ muxes to ensure
2120 2123 * that no data is flowing up into this queue
2121 2124 * and hence no thread active in this instance of
2122 2125 * lower mux. But for QPERMOD and QMTOUTPERIM there
2123 2126 * could be messages on the inner and outer/inner
2124 2127 * syncqs respectively. We will wait for them to drain.
2125 2128 * Because passq is blocked messages end up in the syncq
2126 2129 * And qfill_syncq could possibly end up setting QFULL
2127 2130 * which will access the rq->q_flag. Hence, we have to
2128 2131 * acquire the QLOCK in setq.
2129 2132 *
2130 2133 * XXX Messages can also flow from top into this
2131 2134 * queue though the unlink is over (Ex. some instance
2132 2135 * in putnext() called from top that has still not
2133 2136 * accessed this queue. And also putq(lowerq) ?).
2134 2137 * Solution : How about blocking the l_qtop queue ?
2135 2138 * Do we really care about such pure D_MP muxes ?
2136 2139 */
2137 2140
2138 2141 blocksq(passyncq, SQ_BLOCKED, 0);
2139 2142
2140 2143 sq = rq->q_syncq;
2141 2144 if ((outer = sq->sq_outer) != NULL) {
2142 2145
2143 2146 /*
2144 2147 * We have to just wait for the outer sq_count
2145 2148 * drop to zero. As this does not prevent new
2146 2149 * messages to enter the outer perimeter, this
2147 2150 * is subject to starvation.
2148 2151 *
2149 2152 * NOTE :Because of blocksq above, messages could
2150 2153 * be in the inner syncq only because of some
2151 2154 * thread holding the outer perimeter exclusively.
2152 2155 * Hence it would be sufficient to wait for the
2153 2156 * exclusive holder of the outer perimeter to drain
2154 2157 * the inner and outer syncqs. But we will not depend
2155 2158 * on this feature and hence check the inner syncqs
2156 2159 * separately.
2157 2160 */
2158 2161 wait_syncq(outer);
2159 2162 }
2160 2163
2161 2164
2162 2165 /*
2163 2166 * There could be messages destined for
2164 2167 * this queue. Let the exclusive holder
2165 2168 * drain it.
2166 2169 */
2167 2170
2168 2171 wait_syncq(sq);
2169 2172 ASSERT((rq->q_flag & QPERMOD) ||
2170 2173 ((rq->q_syncq->sq_head == NULL) &&
2171 2174 (_WR(rq)->q_syncq->sq_head == NULL)));
2172 2175 }
2173 2176
2174 2177 /*
2175 2178 * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2176 2179 * case as we don't disable its syncq or remove it off the syncq
2177 2180 * service list.
2178 2181 */
2179 2182 if (rq->q_flag & QPERMOD) {
2180 2183 syncq_t *sq = rq->q_syncq;
2181 2184
2182 2185 mutex_enter(SQLOCK(sq));
2183 2186 while (rq->q_sqflags & Q_SQQUEUED) {
2184 2187 sq->sq_flags |= SQ_WANTWAKEUP;
2185 2188 cv_wait(&sq->sq_wait, SQLOCK(sq));
2186 2189 }
2187 2190 mutex_exit(SQLOCK(sq));
2188 2191 }
2189 2192
2190 2193 /*
2191 2194 * flush_syncq changes states only when there are some messages to
2192 2195 * free, i.e. when it returns non-zero value to return.
2193 2196 */
2194 2197 ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2195 2198 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2196 2199
2197 2200 /*
2198 2201 * Nobody else should know about this queue now.
2199 2202 * If the mux did not process the messages before
2200 2203 * acking the I_UNLINK, free them now.
2201 2204 */
2202 2205
2203 2206 flushq(rq, FLUSHALL);
2204 2207 flushq(_WR(rq), FLUSHALL);
2205 2208
2206 2209 /*
2207 2210 * Convert the mux lower queue into a stream head queue.
2208 2211 * Turn off STPLEX before we turn on the stream by removing the passq.
2209 2212 */
2210 2213 rq->q_ptr = wrq->q_ptr = stpdown;
2211 2214 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2212 2215
2213 2216 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2214 2217 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2215 2218
2216 2219 enable_svc(rq);
2217 2220
2218 2221 /*
2219 2222 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2220 2223 * needs to be set to prevent reopen() of the stream - such reopen may
2221 2224 * try to call non-existent pass queue open routine and panic.
2222 2225 */
2223 2226 mutex_enter(&stpdown->sd_lock);
2224 2227 stpdown->sd_flag &= ~STPLEX;
2225 2228 mutex_exit(&stpdown->sd_lock);
2226 2229
2227 2230 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2228 2231 ((flag & LINKTYPEMASK) == LINKPERSIST));
2229 2232
2230 2233 /* clean up the layered driver linkages */
2231 2234 if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2232 2235 ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2233 2236 } else {
2234 2237 ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2235 2238 }
2236 2239
2237 2240 link_rempassthru(passq);
2238 2241
2239 2242 /*
2240 2243 * Now all plumbing changes are finished and STRPLUMB is no
2241 2244 * longer needed.
2242 2245 */
2243 2246 mutex_enter(&stpdown->sd_lock);
2244 2247 stpdown->sd_flag &= ~STRPLUMB;
2245 2248 cv_broadcast(&stpdown->sd_monitor);
2246 2249 mutex_exit(&stpdown->sd_lock);
2247 2250
2248 2251 (void) closef(fpdown);
2249 2252 return (0);
2250 2253 }
2251 2254
2252 2255 /*
2253 2256 * Unlink all multiplexor links for which stp is the controlling stream.
2254 2257 * Return 0, or a non-zero errno on failure.
2255 2258 */
2256 2259 int
2257 2260 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2258 2261 {
2259 2262 linkinfo_t *linkp;
2260 2263 int error = 0;
2261 2264
2262 2265 mutex_enter(&muxifier);
2263 2266 while (linkp = findlinks(stp, 0, flag, ss)) {
2264 2267 /*
2265 2268 * munlink() releases the muxifier lock.
2266 2269 */
2267 2270 if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2268 2271 return (error);
2269 2272 mutex_enter(&muxifier);
2270 2273 }
2271 2274 mutex_exit(&muxifier);
2272 2275 return (0);
2273 2276 }
2274 2277
2275 2278 /*
2276 2279 * A multiplexor link has been made. Add an
2277 2280 * edge to the directed graph.
2278 2281 */
2279 2282 void
2280 2283 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2281 2284 {
2282 2285 struct mux_node *np;
2283 2286 struct mux_edge *ep;
2284 2287 major_t upmaj;
2285 2288 major_t lomaj;
2286 2289
2287 2290 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2288 2291 lomaj = getmajor(lostp->sd_vnode->v_rdev);
2289 2292 np = &ss->ss_mux_nodes[upmaj];
2290 2293 if (np->mn_outp) {
2291 2294 ep = np->mn_outp;
2292 2295 while (ep->me_nextp)
2293 2296 ep = ep->me_nextp;
2294 2297 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2295 2298 ep = ep->me_nextp;
2296 2299 } else {
2297 2300 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2298 2301 ep = np->mn_outp;
2299 2302 }
2300 2303 ep->me_nextp = NULL;
2301 2304 ep->me_muxid = muxid;
2302 2305 /*
2303 2306 * Save the dev_t for the purposes of str_stack_shutdown.
2304 2307 * str_stack_shutdown assumes that the device allows reopen, since
2305 2308 * this dev_t is the one after any cloning by xx_open().
2306 2309 * Would prefer finding the dev_t from before any cloning,
2307 2310 * but specfs doesn't retain that.
2308 2311 */
2309 2312 ep->me_dev = upstp->sd_vnode->v_rdev;
2310 2313 if (lostp->sd_vnode->v_type == VFIFO)
2311 2314 ep->me_nodep = NULL;
2312 2315 else
2313 2316 ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2314 2317 }
2315 2318
2316 2319 /*
2317 2320 * A multiplexor link has been removed. Remove the
2318 2321 * edge in the directed graph.
2319 2322 */
2320 2323 void
2321 2324 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2322 2325 {
2323 2326 struct mux_node *np;
2324 2327 struct mux_edge *ep;
2325 2328 struct mux_edge *pep = NULL;
2326 2329 major_t upmaj;
2327 2330
2328 2331 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2329 2332 np = &ss->ss_mux_nodes[upmaj];
2330 2333 ASSERT(np->mn_outp != NULL);
2331 2334 ep = np->mn_outp;
2332 2335 while (ep) {
2333 2336 if (ep->me_muxid == muxid) {
2334 2337 if (pep)
2335 2338 pep->me_nextp = ep->me_nextp;
2336 2339 else
2337 2340 np->mn_outp = ep->me_nextp;
2338 2341 kmem_free(ep, sizeof (struct mux_edge));
2339 2342 return;
2340 2343 }
2341 2344 pep = ep;
2342 2345 ep = ep->me_nextp;
|
↓ open down ↓ |
1379 lines elided |
↑ open up ↑ |
2343 2346 }
2344 2347 ASSERT(0); /* should not reach here */
2345 2348 }
2346 2349
2347 2350 /*
2348 2351 * Translate the device flags (from conf.h) to the corresponding
2349 2352 * qflag and sq_flag (type) values.
2350 2353 */
2351 2354 int
2352 2355 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2353 - uint32_t *sqtypep)
2356 + uint32_t *sqtypep)
2354 2357 {
2355 2358 uint32_t qflag = 0;
2356 2359 uint32_t sqtype = 0;
2357 2360
2358 2361 if (devflag & _D_OLD)
2359 2362 goto bad;
2360 2363
2361 2364 /* Inner perimeter presence and scope */
2362 2365 switch (devflag & D_MTINNER_MASK) {
2363 2366 case D_MP:
2364 2367 qflag |= QMTSAFE;
2365 2368 sqtype |= SQ_CI;
2366 2369 break;
2367 2370 case D_MTPERQ|D_MP:
2368 2371 qflag |= QPERQ;
2369 2372 break;
2370 2373 case D_MTQPAIR|D_MP:
2371 2374 qflag |= QPAIR;
2372 2375 break;
2373 2376 case D_MTPERMOD|D_MP:
2374 2377 qflag |= QPERMOD;
2375 2378 break;
2376 2379 default:
2377 2380 goto bad;
2378 2381 }
2379 2382
2380 2383 /* Outer perimeter */
2381 2384 if (devflag & D_MTOUTPERIM) {
2382 2385 switch (devflag & D_MTINNER_MASK) {
2383 2386 case D_MP:
2384 2387 case D_MTPERQ|D_MP:
2385 2388 case D_MTQPAIR|D_MP:
2386 2389 break;
2387 2390 default:
2388 2391 goto bad;
2389 2392 }
2390 2393 qflag |= QMTOUTPERIM;
2391 2394 }
2392 2395
2393 2396 /* Inner perimeter modifiers */
2394 2397 if (devflag & D_MTINNER_MOD) {
2395 2398 switch (devflag & D_MTINNER_MASK) {
2396 2399 case D_MP:
2397 2400 goto bad;
2398 2401 default:
2399 2402 break;
2400 2403 }
2401 2404 if (devflag & D_MTPUTSHARED)
2402 2405 sqtype |= SQ_CIPUT;
2403 2406 if (devflag & _D_MTOCSHARED) {
2404 2407 /*
2405 2408 * The code in putnext assumes that it has the
2406 2409 * highest concurrency by not checking sq_count.
2407 2410 * Thus _D_MTOCSHARED can only be supported when
2408 2411 * D_MTPUTSHARED is set.
2409 2412 */
2410 2413 if (!(devflag & D_MTPUTSHARED))
2411 2414 goto bad;
2412 2415 sqtype |= SQ_CIOC;
2413 2416 }
2414 2417 if (devflag & _D_MTCBSHARED) {
2415 2418 /*
2416 2419 * The code in putnext assumes that it has the
2417 2420 * highest concurrency by not checking sq_count.
2418 2421 * Thus _D_MTCBSHARED can only be supported when
2419 2422 * D_MTPUTSHARED is set.
2420 2423 */
2421 2424 if (!(devflag & D_MTPUTSHARED))
2422 2425 goto bad;
2423 2426 sqtype |= SQ_CICB;
2424 2427 }
2425 2428 if (devflag & _D_MTSVCSHARED) {
2426 2429 /*
2427 2430 * The code in putnext assumes that it has the
2428 2431 * highest concurrency by not checking sq_count.
2429 2432 * Thus _D_MTSVCSHARED can only be supported when
2430 2433 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2431 2434 * supported only for QPERMOD.
2432 2435 */
2433 2436 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2434 2437 goto bad;
2435 2438 sqtype |= SQ_CISVC;
2436 2439 }
2437 2440 }
2438 2441
2439 2442 /* Default outer perimeter concurrency */
2440 2443 sqtype |= SQ_CO;
2441 2444
2442 2445 /* Outer perimeter modifiers */
2443 2446 if (devflag & D_MTOCEXCL) {
2444 2447 if (!(devflag & D_MTOUTPERIM)) {
2445 2448 /* No outer perimeter */
2446 2449 goto bad;
2447 2450 }
2448 2451 sqtype &= ~SQ_COOC;
2449 2452 }
2450 2453
2451 2454 /* Synchronous Streams extended qinit structure */
2452 2455 if (devflag & D_SYNCSTR)
2453 2456 qflag |= QSYNCSTR;
2454 2457
2455 2458 /*
2456 2459 * Private flag used by a transport module to indicate
|
↓ open down ↓ |
93 lines elided |
↑ open up ↑ |
2457 2460 * to sockfs that it supports direct-access mode without
2458 2461 * having to go through STREAMS.
2459 2462 */
2460 2463 if (devflag & _D_DIRECT) {
2461 2464 /* Reject unless the module is fully-MT (no perimeter) */
2462 2465 if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2463 2466 goto bad;
2464 2467 qflag |= _QDIRECT;
2465 2468 }
2466 2469
2470 + /*
2471 + * Private flag used to indicate that a streams module should only
2472 + * be pushed once. The TTY streams modules have this flag since if
2473 + * libc believes itself to be an xpg4 process then it will
2474 + * automatically and unconditionally push them when a PTS device is
2475 + * opened. If an application is not aware of this then without this
2476 + * flag we would end up with duplicate modules.
2477 + */
2478 + if (devflag & _D_SINGLE_INSTANCE)
2479 + qflag |= _QSINGLE_INSTANCE;
2480 +
2467 2481 *qflagp = qflag;
2468 2482 *sqtypep = sqtype;
2469 2483 return (0);
2470 2484
2471 2485 bad:
2472 2486 cmn_err(CE_WARN,
2473 2487 "stropen: bad MT flags (0x%x) in driver '%s'",
2474 2488 (int)(qflag & D_MTSAFETY_MASK),
2475 2489 stp->st_rdinit->qi_minfo->mi_idname);
2476 2490
2477 2491 return (EINVAL);
2478 2492 }
2479 2493
2480 2494 /*
2481 2495 * Set the interface values for a pair of queues (qinit structure,
2482 2496 * packet sizes, water marks).
2483 2497 * setq assumes that the caller does not have a claim (entersq or claimq)
2484 2498 * on the queue.
2485 2499 */
2486 2500 void
2487 2501 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2488 2502 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2489 2503 {
2490 2504 queue_t *wq;
2491 2505 syncq_t *sq, *outer;
2492 2506
2493 2507 ASSERT(rq->q_flag & QREADR);
2494 2508 ASSERT((qflag & QMT_TYPEMASK) != 0);
2495 2509 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2496 2510
2497 2511 wq = _WR(rq);
2498 2512 rq->q_qinfo = rinit;
2499 2513 rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2500 2514 rq->q_lowat = rinit->qi_minfo->mi_lowat;
2501 2515 rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2502 2516 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2503 2517 wq->q_qinfo = winit;
2504 2518 wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2505 2519 wq->q_lowat = winit->qi_minfo->mi_lowat;
2506 2520 wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2507 2521 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2508 2522
2509 2523 /* Remove old syncqs */
2510 2524 sq = rq->q_syncq;
2511 2525 outer = sq->sq_outer;
2512 2526 if (outer != NULL) {
2513 2527 ASSERT(wq->q_syncq->sq_outer == outer);
2514 2528 outer_remove(outer, rq->q_syncq);
2515 2529 if (wq->q_syncq != rq->q_syncq)
2516 2530 outer_remove(outer, wq->q_syncq);
2517 2531 }
2518 2532 ASSERT(sq->sq_outer == NULL);
2519 2533 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2520 2534
2521 2535 if (sq != SQ(rq)) {
2522 2536 if (!(rq->q_flag & QPERMOD))
2523 2537 free_syncq(sq);
2524 2538 if (wq->q_syncq == rq->q_syncq)
2525 2539 wq->q_syncq = NULL;
2526 2540 rq->q_syncq = NULL;
2527 2541 }
2528 2542 if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2529 2543 wq->q_syncq != SQ(rq)) {
2530 2544 free_syncq(wq->q_syncq);
2531 2545 wq->q_syncq = NULL;
2532 2546 }
2533 2547 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2534 2548 rq->q_syncq->sq_tail == NULL));
2535 2549 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2536 2550 wq->q_syncq->sq_tail == NULL));
2537 2551
2538 2552 if (!(rq->q_flag & QPERMOD) &&
2539 2553 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2540 2554 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2541 2555 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2542 2556 rq->q_syncq->sq_nciputctrl, 0);
2543 2557 ASSERT(ciputctrl_cache != NULL);
2544 2558 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2545 2559 rq->q_syncq->sq_ciputctrl = NULL;
2546 2560 rq->q_syncq->sq_nciputctrl = 0;
2547 2561 }
2548 2562
2549 2563 if (!(wq->q_flag & QPERMOD) &&
2550 2564 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2551 2565 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2552 2566 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2553 2567 wq->q_syncq->sq_nciputctrl, 0);
2554 2568 ASSERT(ciputctrl_cache != NULL);
2555 2569 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2556 2570 wq->q_syncq->sq_ciputctrl = NULL;
2557 2571 wq->q_syncq->sq_nciputctrl = 0;
2558 2572 }
2559 2573
2560 2574 sq = SQ(rq);
2561 2575 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2562 2576 ASSERT(sq->sq_outer == NULL);
2563 2577 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2564 2578
2565 2579 /*
2566 2580 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2567 2581 * bits in sq_flag based on the sqtype.
2568 2582 */
2569 2583 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2570 2584
2571 2585 rq->q_syncq = wq->q_syncq = sq;
2572 2586 sq->sq_type = sqtype;
2573 2587 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2574 2588
2575 2589 /*
2576 2590 * We are making sq_svcflags zero,
2577 2591 * resetting SQ_DISABLED in case it was set by
2578 2592 * wait_svc() in the munlink path.
2579 2593 *
2580 2594 */
2581 2595 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2582 2596 sq->sq_svcflags = 0;
2583 2597
2584 2598 /*
2585 2599 * We need to acquire the lock here for the mlink and munlink case,
2586 2600 * where canputnext, backenable, etc can access the q_flag.
2587 2601 */
2588 2602 if (lock_needed) {
2589 2603 mutex_enter(QLOCK(rq));
2590 2604 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2591 2605 mutex_exit(QLOCK(rq));
2592 2606 mutex_enter(QLOCK(wq));
2593 2607 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2594 2608 mutex_exit(QLOCK(wq));
2595 2609 } else {
2596 2610 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2597 2611 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2598 2612 }
2599 2613
2600 2614 if (qflag & QPERQ) {
2601 2615 /* Allocate a separate syncq for the write side */
2602 2616 sq = new_syncq();
2603 2617 sq->sq_type = rq->q_syncq->sq_type;
2604 2618 sq->sq_flags = rq->q_syncq->sq_flags;
2605 2619 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2606 2620 sq->sq_oprev == NULL);
2607 2621 wq->q_syncq = sq;
2608 2622 }
2609 2623 if (qflag & QPERMOD) {
2610 2624 sq = dmp->dm_sq;
2611 2625
2612 2626 /*
2613 2627 * Assert that we do have an inner perimeter syncq and that it
2614 2628 * does not have an outer perimeter associated with it.
2615 2629 */
2616 2630 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2617 2631 sq->sq_oprev == NULL);
2618 2632 rq->q_syncq = wq->q_syncq = sq;
2619 2633 }
2620 2634 if (qflag & QMTOUTPERIM) {
2621 2635 outer = dmp->dm_sq;
2622 2636
2623 2637 ASSERT(outer->sq_outer == NULL);
2624 2638 outer_insert(outer, rq->q_syncq);
2625 2639 if (wq->q_syncq != rq->q_syncq)
2626 2640 outer_insert(outer, wq->q_syncq);
2627 2641 }
2628 2642 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2629 2643 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2630 2644 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2631 2645 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2632 2646 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2633 2647
2634 2648 /*
2635 2649 * Initialize struio() types.
2636 2650 */
2637 2651 rq->q_struiot =
2638 2652 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2639 2653 wq->q_struiot =
2640 2654 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2641 2655 }
2642 2656
2643 2657 perdm_t *
2644 2658 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2645 2659 {
2646 2660 syncq_t *sq;
2647 2661 perdm_t **pp;
2648 2662 perdm_t *p;
2649 2663 perdm_t *dmp;
2650 2664
2651 2665 ASSERT(str != NULL);
2652 2666 ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2653 2667
2654 2668 rw_enter(&perdm_rwlock, RW_READER);
2655 2669 for (p = perdm_list; p != NULL; p = p->dm_next) {
2656 2670 if (p->dm_str == str) { /* found one */
2657 2671 atomic_inc_32(&(p->dm_ref));
2658 2672 rw_exit(&perdm_rwlock);
2659 2673 return (p);
2660 2674 }
2661 2675 }
2662 2676 rw_exit(&perdm_rwlock);
2663 2677
2664 2678 sq = new_syncq();
2665 2679 if (qflag & QPERMOD) {
2666 2680 sq->sq_type = sqtype | SQ_PERMOD;
2667 2681 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2668 2682 } else {
2669 2683 ASSERT(qflag & QMTOUTPERIM);
2670 2684 sq->sq_onext = sq->sq_oprev = sq;
2671 2685 }
2672 2686
2673 2687 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2674 2688 dmp->dm_sq = sq;
2675 2689 dmp->dm_str = str;
2676 2690 dmp->dm_ref = 1;
2677 2691 dmp->dm_next = NULL;
2678 2692
2679 2693 rw_enter(&perdm_rwlock, RW_WRITER);
2680 2694 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2681 2695 if (p->dm_str == str) { /* already present */
2682 2696 p->dm_ref++;
2683 2697 rw_exit(&perdm_rwlock);
2684 2698 free_syncq(sq);
2685 2699 kmem_free(dmp, sizeof (perdm_t));
2686 2700 return (p);
2687 2701 }
2688 2702 }
2689 2703
2690 2704 *pp = dmp;
2691 2705 rw_exit(&perdm_rwlock);
2692 2706 return (dmp);
2693 2707 }
2694 2708
2695 2709 void
2696 2710 rele_dm(perdm_t *dmp)
2697 2711 {
2698 2712 perdm_t **pp;
2699 2713 perdm_t *p;
2700 2714
2701 2715 rw_enter(&perdm_rwlock, RW_WRITER);
2702 2716 ASSERT(dmp->dm_ref > 0);
2703 2717
2704 2718 if (--dmp->dm_ref > 0) {
2705 2719 rw_exit(&perdm_rwlock);
2706 2720 return;
2707 2721 }
2708 2722
2709 2723 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2710 2724 if (p == dmp)
2711 2725 break;
2712 2726 ASSERT(p == dmp);
2713 2727 *pp = p->dm_next;
2714 2728 rw_exit(&perdm_rwlock);
2715 2729
2716 2730 /*
2717 2731 * Wait for any background processing that relies on the
2718 2732 * syncq to complete before it is freed.
2719 2733 */
2720 2734 wait_sq_svc(p->dm_sq);
2721 2735 free_syncq(p->dm_sq);
2722 2736 kmem_free(p, sizeof (perdm_t));
2723 2737 }
2724 2738
2725 2739 /*
2726 2740 * Make a protocol message given control and data buffers.
2727 2741 * n.b., this can block; be careful of what locks you hold when calling it.
2728 2742 *
2729 2743 * If sd_maxblk is less than *iosize this routine can fail part way through
2730 2744 * (due to an allocation failure). In this case on return *iosize will contain
2731 2745 * the amount that was consumed. Otherwise *iosize will not be modified
2732 2746 * i.e. it will contain the amount that was consumed.
2733 2747 */
2734 2748 int
2735 2749 strmakemsg(
2736 2750 struct strbuf *mctl,
2737 2751 ssize_t *iosize,
2738 2752 struct uio *uiop,
2739 2753 stdata_t *stp,
2740 2754 int32_t flag,
2741 2755 mblk_t **mpp)
2742 2756 {
2743 2757 mblk_t *mpctl = NULL;
2744 2758 mblk_t *mpdata = NULL;
2745 2759 int error;
2746 2760
2747 2761 ASSERT(uiop != NULL);
2748 2762
2749 2763 *mpp = NULL;
2750 2764 /* Create control part, if any */
2751 2765 if ((mctl != NULL) && (mctl->len >= 0)) {
2752 2766 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2753 2767 if (error)
2754 2768 return (error);
2755 2769 }
2756 2770 /* Create data part, if any */
2757 2771 if (*iosize >= 0) {
2758 2772 error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2759 2773 if (error) {
2760 2774 freemsg(mpctl);
2761 2775 return (error);
2762 2776 }
2763 2777 }
2764 2778 if (mpctl != NULL) {
2765 2779 if (mpdata != NULL)
2766 2780 linkb(mpctl, mpdata);
2767 2781 *mpp = mpctl;
2768 2782 } else {
2769 2783 *mpp = mpdata;
2770 2784 }
2771 2785 return (0);
2772 2786 }
2773 2787
2774 2788 /*
2775 2789 * Make the control part of a protocol message given a control buffer.
2776 2790 * n.b., this can block; be careful of what locks you hold when calling it.
2777 2791 */
2778 2792 int
2779 2793 strmakectl(
2780 2794 struct strbuf *mctl,
2781 2795 int32_t flag,
2782 2796 int32_t fflag,
2783 2797 mblk_t **mpp)
2784 2798 {
2785 2799 mblk_t *bp = NULL;
2786 2800 unsigned char msgtype;
2787 2801 int error = 0;
2788 2802 cred_t *cr = CRED();
2789 2803
2790 2804 /* We do not support interrupt threads using the stream head to send */
2791 2805 ASSERT(cr != NULL);
2792 2806
2793 2807 *mpp = NULL;
2794 2808 /*
2795 2809 * Create control part of message, if any.
2796 2810 */
2797 2811 if ((mctl != NULL) && (mctl->len >= 0)) {
2798 2812 caddr_t base;
2799 2813 int ctlcount;
2800 2814 int allocsz;
2801 2815
2802 2816 if (flag & RS_HIPRI)
2803 2817 msgtype = M_PCPROTO;
2804 2818 else
2805 2819 msgtype = M_PROTO;
2806 2820
2807 2821 ctlcount = mctl->len;
2808 2822 base = mctl->buf;
2809 2823
2810 2824 /*
2811 2825 * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2812 2826 * blocks by increasing the size to something more usable.
2813 2827 */
2814 2828 allocsz = MAX(ctlcount, 64);
2815 2829
2816 2830 /*
2817 2831 * Range checking has already been done; simply try
2818 2832 * to allocate a message block for the ctl part.
2819 2833 */
2820 2834 while ((bp = allocb_cred(allocsz, cr,
2821 2835 curproc->p_pid)) == NULL) {
2822 2836 if (fflag & (FNDELAY|FNONBLOCK))
2823 2837 return (EAGAIN);
2824 2838 if (error = strwaitbuf(allocsz, BPRI_MED))
2825 2839 return (error);
2826 2840 }
2827 2841
2828 2842 bp->b_datap->db_type = msgtype;
2829 2843 if (copyin(base, bp->b_wptr, ctlcount)) {
2830 2844 freeb(bp);
2831 2845 return (EFAULT);
2832 2846 }
2833 2847 bp->b_wptr += ctlcount;
2834 2848 }
2835 2849 *mpp = bp;
2836 2850 return (0);
2837 2851 }
2838 2852
2839 2853 /*
2840 2854 * Make a protocol message given data buffers.
2841 2855 * n.b., this can block; be careful of what locks you hold when calling it.
2842 2856 *
2843 2857 * If sd_maxblk is less than *iosize this routine can fail part way through
2844 2858 * (due to an allocation failure). In this case on return *iosize will contain
2845 2859 * the amount that was consumed. Otherwise *iosize will not be modified
2846 2860 * i.e. it will contain the amount that was consumed.
2847 2861 */
2848 2862 int
2849 2863 strmakedata(
2850 2864 ssize_t *iosize,
2851 2865 struct uio *uiop,
2852 2866 stdata_t *stp,
2853 2867 int32_t flag,
2854 2868 mblk_t **mpp)
2855 2869 {
2856 2870 mblk_t *mp = NULL;
2857 2871 mblk_t *bp;
2858 2872 int wroff = (int)stp->sd_wroff;
2859 2873 int tail_len = (int)stp->sd_tail;
2860 2874 int extra = wroff + tail_len;
2861 2875 int error = 0;
2862 2876 ssize_t maxblk;
2863 2877 ssize_t count = *iosize;
2864 2878 cred_t *cr;
2865 2879
2866 2880 *mpp = NULL;
2867 2881 if (count < 0)
2868 2882 return (0);
2869 2883
2870 2884 /* We do not support interrupt threads using the stream head to send */
2871 2885 cr = CRED();
2872 2886 ASSERT(cr != NULL);
2873 2887
2874 2888 maxblk = stp->sd_maxblk;
2875 2889 if (maxblk == INFPSZ)
2876 2890 maxblk = count;
2877 2891
2878 2892 /*
2879 2893 * Create data part of message, if any.
2880 2894 */
2881 2895 do {
2882 2896 ssize_t size;
2883 2897 dblk_t *dp;
2884 2898
2885 2899 ASSERT(uiop);
2886 2900
2887 2901 size = MIN(count, maxblk);
2888 2902
2889 2903 while ((bp = allocb_cred(size + extra, cr,
2890 2904 curproc->p_pid)) == NULL) {
2891 2905 error = EAGAIN;
2892 2906 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2893 2907 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2894 2908 if (count == *iosize) {
2895 2909 freemsg(mp);
2896 2910 return (error);
2897 2911 } else {
2898 2912 *iosize -= count;
2899 2913 *mpp = mp;
2900 2914 return (0);
2901 2915 }
2902 2916 }
2903 2917 }
2904 2918 dp = bp->b_datap;
2905 2919 dp->db_cpid = curproc->p_pid;
2906 2920 ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2907 2921 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2908 2922
2909 2923 if (flag & STRUIO_POSTPONE) {
2910 2924 /*
2911 2925 * Setup the stream uio portion of the
2912 2926 * dblk for subsequent use by struioget().
2913 2927 */
2914 2928 dp->db_struioflag = STRUIO_SPEC;
2915 2929 dp->db_cksumstart = 0;
2916 2930 dp->db_cksumstuff = 0;
2917 2931 dp->db_cksumend = size;
2918 2932 *(long long *)dp->db_struioun.data = 0ll;
2919 2933 bp->b_wptr += size;
2920 2934 } else {
2921 2935 if (stp->sd_copyflag & STRCOPYCACHED)
2922 2936 uiop->uio_extflg |= UIO_COPY_CACHED;
2923 2937
2924 2938 if (size != 0) {
2925 2939 error = uiomove(bp->b_wptr, size, UIO_WRITE,
2926 2940 uiop);
2927 2941 if (error != 0) {
2928 2942 freeb(bp);
2929 2943 freemsg(mp);
2930 2944 return (error);
2931 2945 }
2932 2946 }
2933 2947 bp->b_wptr += size;
2934 2948
2935 2949 if (stp->sd_wputdatafunc != NULL) {
2936 2950 mblk_t *newbp;
2937 2951
2938 2952 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2939 2953 bp, NULL, NULL, NULL, NULL);
2940 2954 if (newbp == NULL) {
2941 2955 freeb(bp);
2942 2956 freemsg(mp);
2943 2957 return (ECOMM);
2944 2958 }
2945 2959 bp = newbp;
2946 2960 }
2947 2961 }
2948 2962
2949 2963 count -= size;
2950 2964
2951 2965 if (mp == NULL)
2952 2966 mp = bp;
2953 2967 else
2954 2968 linkb(mp, bp);
2955 2969 } while (count > 0);
2956 2970
2957 2971 *mpp = mp;
2958 2972 return (0);
2959 2973 }
2960 2974
2961 2975 /*
2962 2976 * Wait for a buffer to become available. Return non-zero errno
2963 2977 * if not able to wait, 0 if buffer is probably there.
2964 2978 */
2965 2979 int
2966 2980 strwaitbuf(size_t size, int pri)
2967 2981 {
2968 2982 bufcall_id_t id;
2969 2983
2970 2984 mutex_enter(&bcall_monitor);
2971 2985 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2972 2986 &ttoproc(curthread)->p_flag_cv)) == 0) {
2973 2987 mutex_exit(&bcall_monitor);
2974 2988 return (ENOSR);
2975 2989 }
2976 2990 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2977 2991 unbufcall(id);
2978 2992 mutex_exit(&bcall_monitor);
2979 2993 return (EINTR);
2980 2994 }
2981 2995 unbufcall(id);
2982 2996 mutex_exit(&bcall_monitor);
2983 2997 return (0);
2984 2998 }
2985 2999
2986 3000 /*
2987 3001 * This function waits for a read or write event to happen on a stream.
2988 3002 * fmode can specify FNDELAY and/or FNONBLOCK.
2989 3003 * The timeout is in ms with -1 meaning infinite.
2990 3004 * The flag values work as follows:
2991 3005 * READWAIT Check for read side errors, send M_READ
2992 3006 * GETWAIT Check for read side errors, no M_READ
2993 3007 * WRITEWAIT Check for write side errors.
2994 3008 * NOINTR Do not return error if nonblocking or timeout.
2995 3009 * STR_NOERROR Ignore all errors except STPLEX.
2996 3010 * STR_NOSIG Ignore/hold signals during the duration of the call.
2997 3011 * STR_PEEK Pass through the strgeterr().
2998 3012 */
2999 3013 int
3000 3014 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3001 3015 int *done)
3002 3016 {
3003 3017 int slpflg, errs;
3004 3018 int error;
3005 3019 kcondvar_t *sleepon;
3006 3020 mblk_t *mp;
3007 3021 ssize_t *rd_count;
3008 3022 clock_t rval;
3009 3023
3010 3024 ASSERT(MUTEX_HELD(&stp->sd_lock));
3011 3025 if ((flag & READWAIT) || (flag & GETWAIT)) {
3012 3026 slpflg = RSLEEP;
3013 3027 sleepon = &_RD(stp->sd_wrq)->q_wait;
3014 3028 errs = STRDERR|STPLEX;
3015 3029 } else {
3016 3030 slpflg = WSLEEP;
3017 3031 sleepon = &stp->sd_wrq->q_wait;
3018 3032 errs = STWRERR|STRHUP|STPLEX;
3019 3033 }
3020 3034 if (flag & STR_NOERROR)
3021 3035 errs = STPLEX;
3022 3036
3023 3037 if (stp->sd_wakeq & slpflg) {
3024 3038 /*
3025 3039 * A strwakeq() is pending, no need to sleep.
3026 3040 */
3027 3041 stp->sd_wakeq &= ~slpflg;
3028 3042 *done = 0;
3029 3043 return (0);
3030 3044 }
3031 3045
3032 3046 if (stp->sd_flag & errs) {
3033 3047 /*
3034 3048 * Check for errors before going to sleep since the
3035 3049 * caller might not have checked this while holding
3036 3050 * sd_lock.
3037 3051 */
3038 3052 error = strgeterr(stp, errs, (flag & STR_PEEK));
3039 3053 if (error != 0) {
3040 3054 *done = 1;
3041 3055 return (error);
3042 3056 }
3043 3057 }
3044 3058
3045 3059 /*
3046 3060 * If any module downstream has requested read notification
3047 3061 * by setting SNDMREAD flag using M_SETOPTS, send a message
3048 3062 * down stream.
3049 3063 */
3050 3064 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3051 3065 mutex_exit(&stp->sd_lock);
3052 3066 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3053 3067 (flag & STR_NOSIG), &error))) {
3054 3068 mutex_enter(&stp->sd_lock);
3055 3069 *done = 1;
3056 3070 return (error);
3057 3071 }
3058 3072 mp->b_datap->db_type = M_READ;
3059 3073 rd_count = (ssize_t *)mp->b_wptr;
3060 3074 *rd_count = count;
3061 3075 mp->b_wptr += sizeof (ssize_t);
3062 3076 /*
3063 3077 * Send the number of bytes requested by the
3064 3078 * read as the argument to M_READ.
3065 3079 */
3066 3080 stream_willservice(stp);
3067 3081 putnext(stp->sd_wrq, mp);
3068 3082 stream_runservice(stp);
3069 3083 mutex_enter(&stp->sd_lock);
3070 3084
3071 3085 /*
3072 3086 * If any data arrived due to inline processing
3073 3087 * of putnext(), don't sleep.
3074 3088 */
3075 3089 if (_RD(stp->sd_wrq)->q_first != NULL) {
3076 3090 *done = 0;
3077 3091 return (0);
3078 3092 }
3079 3093 }
3080 3094
3081 3095 if (fmode & (FNDELAY|FNONBLOCK)) {
3082 3096 if (!(flag & NOINTR))
3083 3097 error = EAGAIN;
3084 3098 else
3085 3099 error = 0;
3086 3100 *done = 1;
3087 3101 return (error);
3088 3102 }
3089 3103
3090 3104 stp->sd_flag |= slpflg;
3091 3105 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3092 3106 "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3093 3107 stp, flag, count, fmode, done);
3094 3108
3095 3109 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3096 3110 if (rval > 0) {
3097 3111 /* EMPTY */
3098 3112 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3099 3113 "strwaitq awakes(2):%X, %X, %X, %X, %X",
3100 3114 stp, flag, count, fmode, done);
3101 3115 } else if (rval == 0) {
3102 3116 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3103 3117 "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3104 3118 stp, flag, count, fmode, done);
3105 3119 stp->sd_flag &= ~slpflg;
3106 3120 cv_broadcast(sleepon);
3107 3121 if (!(flag & NOINTR))
3108 3122 error = EINTR;
3109 3123 else
3110 3124 error = 0;
3111 3125 *done = 1;
3112 3126 return (error);
3113 3127 } else {
3114 3128 /* timeout */
3115 3129 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3116 3130 "strwaitq timeout:%p, %X, %lX, %X, %p",
3117 3131 stp, flag, count, fmode, done);
3118 3132 *done = 1;
3119 3133 if (!(flag & NOINTR))
3120 3134 return (ETIME);
3121 3135 else
3122 3136 return (0);
3123 3137 }
3124 3138 /*
3125 3139 * If the caller implements delayed errors (i.e. queued after data)
3126 3140 * we can not check for errors here since data as well as an
3127 3141 * error might have arrived at the stream head. We return to
3128 3142 * have the caller check the read queue before checking for errors.
3129 3143 */
3130 3144 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3131 3145 error = strgeterr(stp, errs, (flag & STR_PEEK));
3132 3146 if (error != 0) {
3133 3147 *done = 1;
3134 3148 return (error);
3135 3149 }
3136 3150 }
3137 3151 *done = 0;
3138 3152 return (0);
3139 3153 }
3140 3154
3141 3155 /*
3142 3156 * Perform job control discipline access checks.
3143 3157 * Return 0 for success and the errno for failure.
3144 3158 */
3145 3159
3146 3160 #define cantsend(p, t, sig) \
3147 3161 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3148 3162
3149 3163 int
3150 3164 straccess(struct stdata *stp, enum jcaccess mode)
3151 3165 {
3152 3166 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */
3153 3167 kthread_t *t = curthread;
3154 3168 proc_t *p = ttoproc(t);
3155 3169 sess_t *sp;
3156 3170
3157 3171 ASSERT(mutex_owned(&stp->sd_lock));
3158 3172
3159 3173 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3160 3174 return (0);
3161 3175
3162 3176 mutex_enter(&p->p_lock); /* protects p_pgidp */
3163 3177
3164 3178 for (;;) {
3165 3179 mutex_enter(&p->p_splock); /* protects p->p_sessp */
3166 3180 sp = p->p_sessp;
3167 3181 mutex_enter(&sp->s_lock); /* protects sp->* */
3168 3182
3169 3183 /*
3170 3184 * If this is not the calling process's controlling terminal
3171 3185 * or if the calling process is already in the foreground
3172 3186 * then allow access.
3173 3187 */
3174 3188 if (sp->s_dev != stp->sd_vnode->v_rdev ||
3175 3189 p->p_pgidp == stp->sd_pgidp) {
3176 3190 mutex_exit(&sp->s_lock);
3177 3191 mutex_exit(&p->p_splock);
3178 3192 mutex_exit(&p->p_lock);
3179 3193 return (0);
3180 3194 }
3181 3195
3182 3196 /*
3183 3197 * Check to see if controlling terminal has been deallocated.
3184 3198 */
3185 3199 if (sp->s_vp == NULL) {
3186 3200 if (!cantsend(p, t, SIGHUP))
3187 3201 sigtoproc(p, t, SIGHUP);
3188 3202 mutex_exit(&sp->s_lock);
3189 3203 mutex_exit(&p->p_splock);
3190 3204 mutex_exit(&p->p_lock);
3191 3205 return (EIO);
3192 3206 }
3193 3207
3194 3208 mutex_exit(&sp->s_lock);
3195 3209 mutex_exit(&p->p_splock);
3196 3210
3197 3211 if (mode == JCGETP) {
3198 3212 mutex_exit(&p->p_lock);
3199 3213 return (0);
3200 3214 }
3201 3215
3202 3216 if (mode == JCREAD) {
3203 3217 if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3204 3218 mutex_exit(&p->p_lock);
3205 3219 return (EIO);
3206 3220 }
3207 3221 mutex_exit(&p->p_lock);
3208 3222 mutex_exit(&stp->sd_lock);
3209 3223 pgsignal(p->p_pgidp, SIGTTIN);
3210 3224 mutex_enter(&stp->sd_lock);
3211 3225 mutex_enter(&p->p_lock);
3212 3226 } else { /* mode == JCWRITE or JCSETP */
3213 3227 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3214 3228 cantsend(p, t, SIGTTOU)) {
3215 3229 mutex_exit(&p->p_lock);
3216 3230 return (0);
3217 3231 }
3218 3232 if (p->p_detached) {
3219 3233 mutex_exit(&p->p_lock);
3220 3234 return (EIO);
3221 3235 }
3222 3236 mutex_exit(&p->p_lock);
3223 3237 mutex_exit(&stp->sd_lock);
3224 3238 pgsignal(p->p_pgidp, SIGTTOU);
3225 3239 mutex_enter(&stp->sd_lock);
3226 3240 mutex_enter(&p->p_lock);
3227 3241 }
3228 3242
3229 3243 /*
3230 3244 * We call cv_wait_sig_swap() to cause the appropriate
3231 3245 * action for the jobcontrol signal to take place.
3232 3246 * If the signal is being caught, we will take the
3233 3247 * EINTR error return. Otherwise, the default action
3234 3248 * of causing the process to stop will take place.
3235 3249 * In this case, we rely on the periodic cv_broadcast() on
3236 3250 * &lbolt_cv to wake us up to loop around and test again.
3237 3251 * We can't get here if the signal is ignored or
3238 3252 * if the current thread is blocking the signal.
3239 3253 */
3240 3254 mutex_exit(&stp->sd_lock);
3241 3255 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3242 3256 mutex_exit(&p->p_lock);
3243 3257 mutex_enter(&stp->sd_lock);
3244 3258 return (EINTR);
3245 3259 }
3246 3260 mutex_exit(&p->p_lock);
3247 3261 mutex_enter(&stp->sd_lock);
3248 3262 mutex_enter(&p->p_lock);
3249 3263 }
3250 3264 }
3251 3265
3252 3266 /*
3253 3267 * Return size of message of block type (bp->b_datap->db_type)
3254 3268 */
3255 3269 size_t
3256 3270 xmsgsize(mblk_t *bp)
3257 3271 {
3258 3272 unsigned char type;
3259 3273 size_t count = 0;
3260 3274
3261 3275 type = bp->b_datap->db_type;
3262 3276
3263 3277 for (; bp; bp = bp->b_cont) {
3264 3278 if (type != bp->b_datap->db_type)
3265 3279 break;
3266 3280 ASSERT(bp->b_wptr >= bp->b_rptr);
3267 3281 count += bp->b_wptr - bp->b_rptr;
3268 3282 }
3269 3283 return (count);
3270 3284 }
3271 3285
3272 3286 /*
3273 3287 * Allocate a stream head.
3274 3288 */
3275 3289 struct stdata *
3276 3290 shalloc(queue_t *qp)
3277 3291 {
3278 3292 stdata_t *stp;
3279 3293
3280 3294 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3281 3295
3282 3296 stp->sd_wrq = _WR(qp);
3283 3297 stp->sd_strtab = NULL;
3284 3298 stp->sd_iocid = 0;
3285 3299 stp->sd_mate = NULL;
3286 3300 stp->sd_freezer = NULL;
3287 3301 stp->sd_refcnt = 0;
3288 3302 stp->sd_wakeq = 0;
3289 3303 stp->sd_anchor = 0;
3290 3304 stp->sd_struiowrq = NULL;
3291 3305 stp->sd_struiordq = NULL;
3292 3306 stp->sd_struiodnak = 0;
3293 3307 stp->sd_struionak = NULL;
3294 3308 stp->sd_t_audit_data = NULL;
3295 3309 stp->sd_rput_opt = 0;
3296 3310 stp->sd_wput_opt = 0;
3297 3311 stp->sd_read_opt = 0;
3298 3312 stp->sd_rprotofunc = strrput_proto;
3299 3313 stp->sd_rmiscfunc = strrput_misc;
3300 3314 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3301 3315 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3302 3316 stp->sd_ciputctrl = NULL;
3303 3317 stp->sd_nciputctrl = 0;
3304 3318 stp->sd_qhead = NULL;
3305 3319 stp->sd_qtail = NULL;
3306 3320 stp->sd_servid = NULL;
3307 3321 stp->sd_nqueues = 0;
3308 3322 stp->sd_svcflags = 0;
3309 3323 stp->sd_copyflag = 0;
3310 3324
3311 3325 return (stp);
3312 3326 }
3313 3327
3314 3328 /*
3315 3329 * Free a stream head.
3316 3330 */
3317 3331 void
3318 3332 shfree(stdata_t *stp)
3319 3333 {
3320 3334 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3321 3335
3322 3336 stp->sd_wrq = NULL;
3323 3337
3324 3338 mutex_enter(&stp->sd_qlock);
3325 3339 while (stp->sd_svcflags & STRS_SCHEDULED) {
3326 3340 STRSTAT(strwaits);
3327 3341 cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3328 3342 }
3329 3343 mutex_exit(&stp->sd_qlock);
3330 3344
3331 3345 if (stp->sd_ciputctrl != NULL) {
3332 3346 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3333 3347 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3334 3348 stp->sd_nciputctrl, 0);
3335 3349 ASSERT(ciputctrl_cache != NULL);
3336 3350 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3337 3351 stp->sd_ciputctrl = NULL;
3338 3352 stp->sd_nciputctrl = 0;
3339 3353 }
3340 3354 ASSERT(stp->sd_qhead == NULL);
3341 3355 ASSERT(stp->sd_qtail == NULL);
3342 3356 ASSERT(stp->sd_nqueues == 0);
3343 3357 kmem_cache_free(stream_head_cache, stp);
3344 3358 }
3345 3359
3346 3360 /*
3347 3361 * Allocate a pair of queues and a syncq for the pair
3348 3362 */
3349 3363 queue_t *
3350 3364 allocq(void)
3351 3365 {
3352 3366 queinfo_t *qip;
3353 3367 queue_t *qp, *wqp;
3354 3368 syncq_t *sq;
3355 3369
3356 3370 qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3357 3371
3358 3372 qp = &qip->qu_rqueue;
3359 3373 wqp = &qip->qu_wqueue;
3360 3374 sq = &qip->qu_syncq;
3361 3375
3362 3376 qp->q_last = NULL;
3363 3377 qp->q_next = NULL;
3364 3378 qp->q_ptr = NULL;
3365 3379 qp->q_flag = QUSE | QREADR;
3366 3380 qp->q_bandp = NULL;
3367 3381 qp->q_stream = NULL;
3368 3382 qp->q_syncq = sq;
3369 3383 qp->q_nband = 0;
3370 3384 qp->q_nfsrv = NULL;
3371 3385 qp->q_draining = 0;
3372 3386 qp->q_syncqmsgs = 0;
3373 3387 qp->q_spri = 0;
3374 3388 qp->q_qtstamp = 0;
3375 3389 qp->q_sqtstamp = 0;
3376 3390 qp->q_fp = NULL;
3377 3391
3378 3392 wqp->q_last = NULL;
3379 3393 wqp->q_next = NULL;
3380 3394 wqp->q_ptr = NULL;
3381 3395 wqp->q_flag = QUSE;
3382 3396 wqp->q_bandp = NULL;
3383 3397 wqp->q_stream = NULL;
3384 3398 wqp->q_syncq = sq;
3385 3399 wqp->q_nband = 0;
3386 3400 wqp->q_nfsrv = NULL;
3387 3401 wqp->q_draining = 0;
3388 3402 wqp->q_syncqmsgs = 0;
3389 3403 wqp->q_qtstamp = 0;
3390 3404 wqp->q_sqtstamp = 0;
3391 3405 wqp->q_spri = 0;
3392 3406
3393 3407 sq->sq_count = 0;
3394 3408 sq->sq_rmqcount = 0;
3395 3409 sq->sq_flags = 0;
3396 3410 sq->sq_type = 0;
3397 3411 sq->sq_callbflags = 0;
3398 3412 sq->sq_cancelid = 0;
3399 3413 sq->sq_ciputctrl = NULL;
3400 3414 sq->sq_nciputctrl = 0;
3401 3415 sq->sq_needexcl = 0;
3402 3416 sq->sq_svcflags = 0;
3403 3417
3404 3418 return (qp);
3405 3419 }
3406 3420
3407 3421 /*
3408 3422 * Free a pair of queues and the "attached" syncq.
3409 3423 * Discard any messages left on the syncq(s), remove the syncq(s) from the
3410 3424 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3411 3425 */
3412 3426 void
3413 3427 freeq(queue_t *qp)
3414 3428 {
3415 3429 qband_t *qbp, *nqbp;
3416 3430 syncq_t *sq, *outer;
3417 3431 queue_t *wqp = _WR(qp);
3418 3432
3419 3433 ASSERT(qp->q_flag & QREADR);
3420 3434
3421 3435 /*
3422 3436 * If a previously dispatched taskq job is scheduled to run
3423 3437 * sync_service() or a service routine is scheduled for the
3424 3438 * queues about to be freed, wait here until all service is
3425 3439 * done on the queue and all associated queues and syncqs.
3426 3440 */
3427 3441 wait_svc(qp);
3428 3442
3429 3443 (void) flush_syncq(qp->q_syncq, qp);
3430 3444 (void) flush_syncq(wqp->q_syncq, wqp);
3431 3445 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3432 3446
3433 3447 /*
3434 3448 * Flush the queues before q_next is set to NULL This is needed
3435 3449 * in order to backenable any downstream queue before we go away.
3436 3450 * Note: we are already removed from the stream so that the
3437 3451 * backenabling will not cause any messages to be delivered to our
3438 3452 * put procedures.
3439 3453 */
3440 3454 flushq(qp, FLUSHALL);
3441 3455 flushq(wqp, FLUSHALL);
3442 3456
3443 3457 /* Tidy up - removeq only does a half-remove from stream */
3444 3458 qp->q_next = wqp->q_next = NULL;
3445 3459 ASSERT(!(qp->q_flag & QENAB));
3446 3460 ASSERT(!(wqp->q_flag & QENAB));
3447 3461
3448 3462 outer = qp->q_syncq->sq_outer;
3449 3463 if (outer != NULL) {
3450 3464 outer_remove(outer, qp->q_syncq);
3451 3465 if (wqp->q_syncq != qp->q_syncq)
3452 3466 outer_remove(outer, wqp->q_syncq);
3453 3467 }
3454 3468 /*
3455 3469 * Free any syncqs that are outside what allocq returned.
3456 3470 */
3457 3471 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3458 3472 free_syncq(qp->q_syncq);
3459 3473 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3460 3474 free_syncq(wqp->q_syncq);
3461 3475
3462 3476 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3463 3477 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3464 3478 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3465 3479 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3466 3480 sq = SQ(qp);
3467 3481 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3468 3482 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3469 3483 ASSERT(sq->sq_outer == NULL);
3470 3484 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3471 3485 ASSERT(sq->sq_callbpend == NULL);
3472 3486 ASSERT(sq->sq_needexcl == 0);
3473 3487
3474 3488 if (sq->sq_ciputctrl != NULL) {
3475 3489 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3476 3490 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3477 3491 sq->sq_nciputctrl, 0);
3478 3492 ASSERT(ciputctrl_cache != NULL);
3479 3493 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3480 3494 sq->sq_ciputctrl = NULL;
3481 3495 sq->sq_nciputctrl = 0;
3482 3496 }
3483 3497
3484 3498 ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3485 3499 ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3486 3500 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3487 3501
3488 3502 qp->q_flag &= ~QUSE;
3489 3503 wqp->q_flag &= ~QUSE;
3490 3504
3491 3505 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3492 3506 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3493 3507
3494 3508 qbp = qp->q_bandp;
3495 3509 while (qbp) {
3496 3510 nqbp = qbp->qb_next;
3497 3511 freeband(qbp);
3498 3512 qbp = nqbp;
3499 3513 }
3500 3514 qbp = wqp->q_bandp;
3501 3515 while (qbp) {
3502 3516 nqbp = qbp->qb_next;
3503 3517 freeband(qbp);
3504 3518 qbp = nqbp;
3505 3519 }
3506 3520 kmem_cache_free(queue_cache, qp);
3507 3521 }
3508 3522
3509 3523 /*
3510 3524 * Allocate a qband structure.
3511 3525 */
3512 3526 qband_t *
3513 3527 allocband(void)
3514 3528 {
3515 3529 qband_t *qbp;
3516 3530
3517 3531 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3518 3532 if (qbp == NULL)
3519 3533 return (NULL);
3520 3534
3521 3535 qbp->qb_next = NULL;
3522 3536 qbp->qb_count = 0;
3523 3537 qbp->qb_mblkcnt = 0;
3524 3538 qbp->qb_first = NULL;
3525 3539 qbp->qb_last = NULL;
3526 3540 qbp->qb_flag = 0;
3527 3541
3528 3542 return (qbp);
3529 3543 }
3530 3544
3531 3545 /*
3532 3546 * Free a qband structure.
3533 3547 */
3534 3548 void
3535 3549 freeband(qband_t *qbp)
3536 3550 {
3537 3551 kmem_cache_free(qband_cache, qbp);
3538 3552 }
3539 3553
3540 3554 /*
3541 3555 * Just like putnextctl(9F), except that allocb_wait() is used.
3542 3556 *
3543 3557 * Consolidation Private, and of course only callable from the stream head or
3544 3558 * routines that may block.
3545 3559 */
3546 3560 int
3547 3561 putnextctl_wait(queue_t *q, int type)
3548 3562 {
3549 3563 mblk_t *bp;
3550 3564 int error;
3551 3565
3552 3566 if ((datamsg(type) && (type != M_DELAY)) ||
3553 3567 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3554 3568 return (0);
3555 3569
3556 3570 bp->b_datap->db_type = (unsigned char)type;
3557 3571 putnext(q, bp);
3558 3572 return (1);
3559 3573 }
3560 3574
3561 3575 /*
3562 3576 * Run any possible bufcalls.
3563 3577 */
3564 3578 void
3565 3579 runbufcalls(void)
3566 3580 {
3567 3581 strbufcall_t *bcp;
3568 3582
3569 3583 mutex_enter(&bcall_monitor);
3570 3584 mutex_enter(&strbcall_lock);
3571 3585
3572 3586 if (strbcalls.bc_head) {
3573 3587 size_t count;
3574 3588 int nevent;
3575 3589
3576 3590 /*
3577 3591 * count how many events are on the list
3578 3592 * now so we can check to avoid looping
3579 3593 * in low memory situations
3580 3594 */
3581 3595 nevent = 0;
3582 3596 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3583 3597 nevent++;
3584 3598
3585 3599 /*
3586 3600 * get estimate of available memory from kmem_avail().
3587 3601 * awake all bufcall functions waiting for
3588 3602 * memory whose request could be satisfied
3589 3603 * by 'count' memory and let 'em fight for it.
3590 3604 */
3591 3605 count = kmem_avail();
3592 3606 while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3593 3607 STRSTAT(bufcalls);
3594 3608 --nevent;
3595 3609 if (bcp->bc_size <= count) {
3596 3610 bcp->bc_executor = curthread;
3597 3611 mutex_exit(&strbcall_lock);
3598 3612 (*bcp->bc_func)(bcp->bc_arg);
3599 3613 mutex_enter(&strbcall_lock);
3600 3614 bcp->bc_executor = NULL;
3601 3615 cv_broadcast(&bcall_cv);
3602 3616 strbcalls.bc_head = bcp->bc_next;
3603 3617 kmem_free(bcp, sizeof (strbufcall_t));
3604 3618 } else {
3605 3619 /*
3606 3620 * too big, try again later - note
3607 3621 * that nevent was decremented above
3608 3622 * so we won't retry this one on this
3609 3623 * iteration of the loop
3610 3624 */
3611 3625 if (bcp->bc_next != NULL) {
3612 3626 strbcalls.bc_head = bcp->bc_next;
3613 3627 bcp->bc_next = NULL;
3614 3628 strbcalls.bc_tail->bc_next = bcp;
3615 3629 strbcalls.bc_tail = bcp;
3616 3630 }
3617 3631 }
3618 3632 }
3619 3633 if (strbcalls.bc_head == NULL)
3620 3634 strbcalls.bc_tail = NULL;
3621 3635 }
3622 3636
3623 3637 mutex_exit(&strbcall_lock);
3624 3638 mutex_exit(&bcall_monitor);
3625 3639 }
3626 3640
3627 3641
3628 3642 /*
3629 3643 * Actually run queue's service routine.
3630 3644 */
3631 3645 static void
3632 3646 runservice(queue_t *q)
3633 3647 {
3634 3648 qband_t *qbp;
3635 3649
3636 3650 ASSERT(q->q_qinfo->qi_srvp);
3637 3651 again:
3638 3652 entersq(q->q_syncq, SQ_SVC);
3639 3653 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3640 3654 "runservice starts:%p", q);
3641 3655
3642 3656 if (!(q->q_flag & QWCLOSE))
3643 3657 (*q->q_qinfo->qi_srvp)(q);
3644 3658
3645 3659 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3646 3660 "runservice ends:(%p)", q);
3647 3661
3648 3662 leavesq(q->q_syncq, SQ_SVC);
3649 3663
3650 3664 mutex_enter(QLOCK(q));
3651 3665 if (q->q_flag & QENAB) {
3652 3666 q->q_flag &= ~QENAB;
3653 3667 mutex_exit(QLOCK(q));
3654 3668 goto again;
3655 3669 }
3656 3670 q->q_flag &= ~QINSERVICE;
3657 3671 q->q_flag &= ~QBACK;
3658 3672 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3659 3673 qbp->qb_flag &= ~QB_BACK;
3660 3674 /*
3661 3675 * Wakeup thread waiting for the service procedure
3662 3676 * to be run (strclose and qdetach).
3663 3677 */
3664 3678 cv_broadcast(&q->q_wait);
3665 3679
3666 3680 mutex_exit(QLOCK(q));
3667 3681 }
3668 3682
3669 3683 /*
3670 3684 * Background processing of bufcalls.
3671 3685 */
3672 3686 void
3673 3687 streams_bufcall_service(void)
3674 3688 {
3675 3689 callb_cpr_t cprinfo;
3676 3690
3677 3691 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3678 3692 "streams_bufcall_service");
3679 3693
3680 3694 mutex_enter(&strbcall_lock);
3681 3695
3682 3696 for (;;) {
3683 3697 if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3684 3698 mutex_exit(&strbcall_lock);
3685 3699 runbufcalls();
3686 3700 mutex_enter(&strbcall_lock);
3687 3701 }
3688 3702 if (strbcalls.bc_head != NULL) {
3689 3703 STRSTAT(bcwaits);
3690 3704 /* Wait for memory to become available */
3691 3705 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3692 3706 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3693 3707 SEC_TO_TICK(60), TR_CLOCK_TICK);
3694 3708 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3695 3709 }
3696 3710
3697 3711 /* Wait for new work to arrive */
3698 3712 if (strbcalls.bc_head == NULL) {
3699 3713 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3700 3714 cv_wait(&strbcall_cv, &strbcall_lock);
3701 3715 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3702 3716 }
3703 3717 }
3704 3718 }
3705 3719
3706 3720 /*
3707 3721 * Background processing of streams background tasks which failed
3708 3722 * taskq_dispatch.
3709 3723 */
3710 3724 static void
3711 3725 streams_qbkgrnd_service(void)
3712 3726 {
3713 3727 callb_cpr_t cprinfo;
3714 3728 queue_t *q;
3715 3729
3716 3730 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3717 3731 "streams_bkgrnd_service");
3718 3732
3719 3733 mutex_enter(&service_queue);
3720 3734
3721 3735 for (;;) {
3722 3736 /*
3723 3737 * Wait for work to arrive.
3724 3738 */
3725 3739 while ((freebs_list == NULL) && (qhead == NULL)) {
3726 3740 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3727 3741 cv_wait(&services_to_run, &service_queue);
3728 3742 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3729 3743 }
3730 3744 /*
3731 3745 * Handle all pending freebs requests to free memory.
3732 3746 */
3733 3747 while (freebs_list != NULL) {
3734 3748 mblk_t *mp = freebs_list;
3735 3749 freebs_list = mp->b_next;
3736 3750 mutex_exit(&service_queue);
3737 3751 mblk_free(mp);
3738 3752 mutex_enter(&service_queue);
3739 3753 }
3740 3754 /*
3741 3755 * Run pending queues.
3742 3756 */
3743 3757 while (qhead != NULL) {
3744 3758 DQ(q, qhead, qtail, q_link);
3745 3759 ASSERT(q != NULL);
3746 3760 mutex_exit(&service_queue);
3747 3761 queue_service(q);
3748 3762 mutex_enter(&service_queue);
3749 3763 }
3750 3764 ASSERT(qhead == NULL && qtail == NULL);
3751 3765 }
3752 3766 }
3753 3767
3754 3768 /*
3755 3769 * Background processing of streams background tasks which failed
3756 3770 * taskq_dispatch.
3757 3771 */
3758 3772 static void
3759 3773 streams_sqbkgrnd_service(void)
3760 3774 {
3761 3775 callb_cpr_t cprinfo;
3762 3776 syncq_t *sq;
3763 3777
3764 3778 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3765 3779 "streams_sqbkgrnd_service");
3766 3780
3767 3781 mutex_enter(&service_queue);
3768 3782
3769 3783 for (;;) {
3770 3784 /*
3771 3785 * Wait for work to arrive.
3772 3786 */
3773 3787 while (sqhead == NULL) {
3774 3788 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3775 3789 cv_wait(&syncqs_to_run, &service_queue);
3776 3790 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3777 3791 }
3778 3792
3779 3793 /*
3780 3794 * Run pending syncqs.
3781 3795 */
3782 3796 while (sqhead != NULL) {
3783 3797 DQ(sq, sqhead, sqtail, sq_next);
3784 3798 ASSERT(sq != NULL);
3785 3799 ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3786 3800 mutex_exit(&service_queue);
3787 3801 syncq_service(sq);
3788 3802 mutex_enter(&service_queue);
3789 3803 }
3790 3804 }
3791 3805 }
3792 3806
3793 3807 /*
3794 3808 * Disable the syncq and wait for background syncq processing to complete.
3795 3809 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3796 3810 * list.
3797 3811 */
3798 3812 void
3799 3813 wait_sq_svc(syncq_t *sq)
3800 3814 {
3801 3815 mutex_enter(SQLOCK(sq));
3802 3816 sq->sq_svcflags |= SQ_DISABLED;
3803 3817 if (sq->sq_svcflags & SQ_BGTHREAD) {
3804 3818 syncq_t *sq_chase;
3805 3819 syncq_t *sq_curr;
3806 3820 int removed;
3807 3821
3808 3822 ASSERT(sq->sq_servcount == 1);
3809 3823 mutex_enter(&service_queue);
3810 3824 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3811 3825 mutex_exit(&service_queue);
3812 3826 if (removed) {
3813 3827 sq->sq_svcflags &= ~SQ_BGTHREAD;
3814 3828 sq->sq_servcount = 0;
3815 3829 STRSTAT(sqremoved);
3816 3830 goto done;
3817 3831 }
3818 3832 }
3819 3833 while (sq->sq_servcount != 0) {
3820 3834 sq->sq_flags |= SQ_WANTWAKEUP;
3821 3835 cv_wait(&sq->sq_wait, SQLOCK(sq));
3822 3836 }
3823 3837 done:
3824 3838 mutex_exit(SQLOCK(sq));
3825 3839 }
3826 3840
3827 3841 /*
3828 3842 * Put a syncq on the list of syncq's to be serviced by the sqthread.
3829 3843 * Add the argument to the end of the sqhead list and set the flag
3830 3844 * indicating this syncq has been enabled. If it has already been
3831 3845 * enabled, don't do anything.
3832 3846 * This routine assumes that SQLOCK is held.
3833 3847 * NOTE that the lock order is to have the SQLOCK first,
3834 3848 * so if the service_syncq lock is held, we need to release it
3835 3849 * before acquiring the SQLOCK (mostly relevant for the background
3836 3850 * thread, and this seems to be common among the STREAMS global locks).
3837 3851 * Note that the sq_svcflags are protected by the SQLOCK.
3838 3852 */
3839 3853 void
3840 3854 sqenable(syncq_t *sq)
3841 3855 {
3842 3856 /*
3843 3857 * This is probably not important except for where I believe it
3844 3858 * is being called. At that point, it should be held (and it
3845 3859 * is a pain to release it just for this routine, so don't do
3846 3860 * it).
3847 3861 */
3848 3862 ASSERT(MUTEX_HELD(SQLOCK(sq)));
3849 3863
3850 3864 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3851 3865 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3852 3866
3853 3867 /*
3854 3868 * Do not put on list if background thread is scheduled or
3855 3869 * syncq is disabled.
3856 3870 */
3857 3871 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3858 3872 return;
3859 3873
3860 3874 /*
3861 3875 * Check whether we should enable sq at all.
3862 3876 * Non PERMOD syncqs may be drained by at most one thread.
3863 3877 * PERMOD syncqs may be drained by several threads but we limit the
3864 3878 * total amount to the lesser of
3865 3879 * Number of queues on the squeue and
3866 3880 * Number of CPUs.
3867 3881 */
3868 3882 if (sq->sq_servcount != 0) {
3869 3883 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3870 3884 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3871 3885 STRSTAT(sqtoomany);
3872 3886 return;
3873 3887 }
3874 3888 }
3875 3889
3876 3890 sq->sq_tstamp = ddi_get_lbolt();
3877 3891 STRSTAT(sqenables);
3878 3892
3879 3893 /* Attempt a taskq dispatch */
3880 3894 sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3881 3895 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3882 3896 if (sq->sq_servid != NULL) {
3883 3897 sq->sq_servcount++;
3884 3898 return;
3885 3899 }
3886 3900
3887 3901 /*
3888 3902 * This taskq dispatch failed, but a previous one may have succeeded.
3889 3903 * Don't try to schedule on the background thread whilst there is
3890 3904 * outstanding taskq processing.
3891 3905 */
3892 3906 if (sq->sq_servcount != 0)
3893 3907 return;
3894 3908
3895 3909 /*
3896 3910 * System is low on resources and can't perform a non-sleeping
3897 3911 * dispatch. Schedule the syncq for a background thread and mark the
3898 3912 * syncq to avoid any further taskq dispatch attempts.
3899 3913 */
3900 3914 mutex_enter(&service_queue);
3901 3915 STRSTAT(taskqfails);
3902 3916 ENQUEUE(sq, sqhead, sqtail, sq_next);
3903 3917 sq->sq_svcflags |= SQ_BGTHREAD;
3904 3918 sq->sq_servcount = 1;
3905 3919 cv_signal(&syncqs_to_run);
3906 3920 mutex_exit(&service_queue);
3907 3921 }
3908 3922
3909 3923 /*
3910 3924 * Note: fifo_close() depends on the mblk_t on the queue being freed
3911 3925 * asynchronously. The asynchronous freeing of messages breaks the
3912 3926 * recursive call chain of fifo_close() while there are I_SENDFD type of
3913 3927 * messages referring to other file pointers on the queue. Then when
3914 3928 * closing pipes it can avoid stack overflow in case of daisy-chained
3915 3929 * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3916 3930 * share the same fifolock_t).
3917 3931 *
3918 3932 * No need to kpreempt_disable to access cpu_seqid. If we migrate and
3919 3933 * the esb queue does not match the new CPU, that is OK.
3920 3934 */
3921 3935 void
3922 3936 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3923 3937 {
3924 3938 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3925 3939 esb_queue_t *eqp;
3926 3940
3927 3941 ASSERT(dbp->db_mblk == mp);
3928 3942 ASSERT(qindex < esbq_nelem);
3929 3943
3930 3944 eqp = system_esbq_array;
3931 3945 if (eqp != NULL) {
3932 3946 eqp += qindex;
3933 3947 } else {
3934 3948 mutex_enter(&esbq_lock);
3935 3949 if (kmem_ready && system_esbq_array == NULL)
3936 3950 system_esbq_array = (esb_queue_t *)kmem_zalloc(
3937 3951 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3938 3952 mutex_exit(&esbq_lock);
3939 3953 eqp = system_esbq_array;
3940 3954 if (eqp != NULL)
3941 3955 eqp += qindex;
3942 3956 else
3943 3957 eqp = &system_esbq;
3944 3958 }
3945 3959
3946 3960 /*
3947 3961 * Check data sanity. The dblock should have non-empty free function.
3948 3962 * It is better to panic here then later when the dblock is freed
3949 3963 * asynchronously when the context is lost.
3950 3964 */
3951 3965 if (dbp->db_frtnp->free_func == NULL) {
3952 3966 panic("freebs_enqueue: dblock %p has a NULL free callback",
3953 3967 (void *)dbp);
3954 3968 }
3955 3969
3956 3970 mutex_enter(&eqp->eq_lock);
3957 3971 /* queue the new mblk on the esballoc queue */
3958 3972 if (eqp->eq_head == NULL) {
3959 3973 eqp->eq_head = eqp->eq_tail = mp;
3960 3974 } else {
3961 3975 eqp->eq_tail->b_next = mp;
3962 3976 eqp->eq_tail = mp;
3963 3977 }
3964 3978 eqp->eq_len++;
3965 3979
3966 3980 /* If we're the first thread to reach the threshold, process */
3967 3981 if (eqp->eq_len >= esbq_max_qlen &&
3968 3982 !(eqp->eq_flags & ESBQ_PROCESSING))
3969 3983 esballoc_process_queue(eqp);
3970 3984
3971 3985 esballoc_set_timer(eqp, esbq_timeout);
3972 3986 mutex_exit(&eqp->eq_lock);
3973 3987 }
3974 3988
3975 3989 static void
3976 3990 esballoc_process_queue(esb_queue_t *eqp)
3977 3991 {
3978 3992 mblk_t *mp;
3979 3993
3980 3994 ASSERT(MUTEX_HELD(&eqp->eq_lock));
3981 3995
3982 3996 eqp->eq_flags |= ESBQ_PROCESSING;
3983 3997
3984 3998 do {
3985 3999 /*
3986 4000 * Detach the message chain for processing.
3987 4001 */
3988 4002 mp = eqp->eq_head;
3989 4003 eqp->eq_tail->b_next = NULL;
3990 4004 eqp->eq_head = eqp->eq_tail = NULL;
3991 4005 eqp->eq_len = 0;
3992 4006 mutex_exit(&eqp->eq_lock);
3993 4007
3994 4008 /*
3995 4009 * Process the message chain.
3996 4010 */
3997 4011 esballoc_enqueue_mblk(mp);
3998 4012 mutex_enter(&eqp->eq_lock);
3999 4013 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
4000 4014
4001 4015 eqp->eq_flags &= ~ESBQ_PROCESSING;
4002 4016 }
4003 4017
4004 4018 /*
4005 4019 * taskq callback routine to free esballoced mblk's
4006 4020 */
4007 4021 static void
4008 4022 esballoc_mblk_free(mblk_t *mp)
4009 4023 {
4010 4024 mblk_t *nextmp;
4011 4025
4012 4026 for (; mp != NULL; mp = nextmp) {
4013 4027 nextmp = mp->b_next;
4014 4028 mp->b_next = NULL;
4015 4029 mblk_free(mp);
4016 4030 }
4017 4031 }
4018 4032
4019 4033 static void
4020 4034 esballoc_enqueue_mblk(mblk_t *mp)
4021 4035 {
4022 4036
4023 4037 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4024 4038 TQ_NOSLEEP) == NULL) {
4025 4039 mblk_t *first_mp = mp;
4026 4040 /*
4027 4041 * System is low on resources and can't perform a non-sleeping
4028 4042 * dispatch. Schedule for a background thread.
4029 4043 */
4030 4044 mutex_enter(&service_queue);
4031 4045 STRSTAT(taskqfails);
4032 4046
4033 4047 while (mp->b_next != NULL)
4034 4048 mp = mp->b_next;
4035 4049
4036 4050 mp->b_next = freebs_list;
4037 4051 freebs_list = first_mp;
4038 4052 cv_signal(&services_to_run);
4039 4053 mutex_exit(&service_queue);
4040 4054 }
4041 4055 }
4042 4056
4043 4057 static void
4044 4058 esballoc_timer(void *arg)
4045 4059 {
4046 4060 esb_queue_t *eqp = arg;
4047 4061
4048 4062 mutex_enter(&eqp->eq_lock);
4049 4063 eqp->eq_flags &= ~ESBQ_TIMER;
4050 4064
4051 4065 if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4052 4066 eqp->eq_len > 0)
4053 4067 esballoc_process_queue(eqp);
4054 4068
4055 4069 esballoc_set_timer(eqp, esbq_timeout);
4056 4070 mutex_exit(&eqp->eq_lock);
4057 4071 }
4058 4072
4059 4073 static void
4060 4074 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4061 4075 {
4062 4076 ASSERT(MUTEX_HELD(&eqp->eq_lock));
4063 4077
4064 4078 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4065 4079 (void) timeout(esballoc_timer, eqp, eq_timeout);
4066 4080 eqp->eq_flags |= ESBQ_TIMER;
4067 4081 }
4068 4082 }
4069 4083
4070 4084 /*
4071 4085 * Setup esbq array length based upon NCPU scaled by CPUs per
4072 4086 * queue. Use static system_esbq until kmem_ready and we can
4073 4087 * create an array in freebs_enqueue().
4074 4088 */
4075 4089 void
4076 4090 esballoc_queue_init(void)
4077 4091 {
4078 4092 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4079 4093 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4080 4094 esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4081 4095 system_esbq.eq_len = 0;
4082 4096 system_esbq.eq_head = system_esbq.eq_tail = NULL;
4083 4097 system_esbq.eq_flags = 0;
4084 4098 }
4085 4099
4086 4100 /*
4087 4101 * Set the QBACK or QB_BACK flag in the given queue for
4088 4102 * the given priority band.
4089 4103 */
4090 4104 void
4091 4105 setqback(queue_t *q, unsigned char pri)
4092 4106 {
4093 4107 int i;
4094 4108 qband_t *qbp;
4095 4109 qband_t **qbpp;
4096 4110
4097 4111 ASSERT(MUTEX_HELD(QLOCK(q)));
4098 4112 if (pri != 0) {
4099 4113 if (pri > q->q_nband) {
4100 4114 qbpp = &q->q_bandp;
4101 4115 while (*qbpp)
4102 4116 qbpp = &(*qbpp)->qb_next;
4103 4117 while (pri > q->q_nband) {
4104 4118 if ((*qbpp = allocband()) == NULL) {
4105 4119 cmn_err(CE_WARN,
4106 4120 "setqback: can't allocate qband\n");
4107 4121 return;
4108 4122 }
4109 4123 (*qbpp)->qb_hiwat = q->q_hiwat;
4110 4124 (*qbpp)->qb_lowat = q->q_lowat;
4111 4125 q->q_nband++;
4112 4126 qbpp = &(*qbpp)->qb_next;
4113 4127 }
4114 4128 }
4115 4129 qbp = q->q_bandp;
4116 4130 i = pri;
4117 4131 while (--i)
4118 4132 qbp = qbp->qb_next;
4119 4133 qbp->qb_flag |= QB_BACK;
4120 4134 } else {
4121 4135 q->q_flag |= QBACK;
4122 4136 }
4123 4137 }
4124 4138
4125 4139 int
4126 4140 strcopyin(void *from, void *to, size_t len, int copyflag)
4127 4141 {
4128 4142 if (copyflag & U_TO_K) {
4129 4143 ASSERT((copyflag & K_TO_K) == 0);
4130 4144 if (copyin(from, to, len))
4131 4145 return (EFAULT);
4132 4146 } else {
4133 4147 ASSERT(copyflag & K_TO_K);
4134 4148 bcopy(from, to, len);
4135 4149 }
4136 4150 return (0);
4137 4151 }
4138 4152
4139 4153 int
4140 4154 strcopyout(void *from, void *to, size_t len, int copyflag)
4141 4155 {
4142 4156 if (copyflag & U_TO_K) {
4143 4157 if (copyout(from, to, len))
4144 4158 return (EFAULT);
4145 4159 } else {
4146 4160 ASSERT(copyflag & K_TO_K);
4147 4161 bcopy(from, to, len);
4148 4162 }
4149 4163 return (0);
4150 4164 }
4151 4165
4152 4166 /*
4153 4167 * strsignal_nolock() posts a signal to the process(es) at the stream head.
4154 4168 * It assumes that the stream head lock is already held, whereas strsignal()
4155 4169 * acquires the lock first. This routine was created because a few callers
4156 4170 * release the stream head lock before calling only to re-acquire it after
4157 4171 * it returns.
4158 4172 */
4159 4173 void
4160 4174 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4161 4175 {
4162 4176 ASSERT(MUTEX_HELD(&stp->sd_lock));
4163 4177 switch (sig) {
4164 4178 case SIGPOLL:
4165 4179 if (stp->sd_sigflags & S_MSG)
4166 4180 strsendsig(stp->sd_siglist, S_MSG, band, 0);
4167 4181 break;
4168 4182 default:
4169 4183 if (stp->sd_pgidp)
4170 4184 pgsignal(stp->sd_pgidp, sig);
4171 4185 break;
4172 4186 }
4173 4187 }
4174 4188
4175 4189 void
4176 4190 strsignal(stdata_t *stp, int sig, int32_t band)
4177 4191 {
4178 4192 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4179 4193 "strsignal:%p, %X, %X", stp, sig, band);
4180 4194
4181 4195 mutex_enter(&stp->sd_lock);
4182 4196 switch (sig) {
4183 4197 case SIGPOLL:
4184 4198 if (stp->sd_sigflags & S_MSG)
4185 4199 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4186 4200 break;
4187 4201
4188 4202 default:
4189 4203 if (stp->sd_pgidp) {
4190 4204 pgsignal(stp->sd_pgidp, sig);
4191 4205 }
4192 4206 break;
4193 4207 }
4194 4208 mutex_exit(&stp->sd_lock);
4195 4209 }
4196 4210
4197 4211 void
4198 4212 strhup(stdata_t *stp)
4199 4213 {
4200 4214 ASSERT(mutex_owned(&stp->sd_lock));
4201 4215 pollwakeup(&stp->sd_pollist, POLLHUP);
4202 4216 if (stp->sd_sigflags & S_HANGUP)
4203 4217 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4204 4218 }
4205 4219
4206 4220 /*
4207 4221 * Backenable the first queue upstream from `q' with a service procedure.
4208 4222 */
4209 4223 void
4210 4224 backenable(queue_t *q, uchar_t pri)
4211 4225 {
4212 4226 queue_t *nq;
4213 4227
4214 4228 /*
4215 4229 * Our presence might not prevent other modules in our own
4216 4230 * stream from popping/pushing since the caller of getq might not
4217 4231 * have a claim on the queue (some drivers do a getq on somebody
4218 4232 * else's queue - they know that the queue itself is not going away
4219 4233 * but the framework has to guarantee q_next in that stream).
4220 4234 */
4221 4235 claimstr(q);
4222 4236
4223 4237 /* Find nearest back queue with service proc */
4224 4238 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4225 4239 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4226 4240 }
4227 4241
4228 4242 if (nq) {
4229 4243 kthread_t *freezer;
4230 4244 /*
4231 4245 * backenable can be called either with no locks held
4232 4246 * or with the stream frozen (the latter occurs when a module
4233 4247 * calls rmvq with the stream frozen). If the stream is frozen
4234 4248 * by the caller the caller will hold all qlocks in the stream.
4235 4249 * Note that a frozen stream doesn't freeze a mated stream,
4236 4250 * so we explicitly check for that.
4237 4251 */
4238 4252 freezer = STREAM(q)->sd_freezer;
4239 4253 if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4240 4254 mutex_enter(QLOCK(nq));
4241 4255 }
4242 4256 #ifdef DEBUG
4243 4257 else {
4244 4258 ASSERT(frozenstr(q));
4245 4259 ASSERT(MUTEX_HELD(QLOCK(q)));
4246 4260 ASSERT(MUTEX_HELD(QLOCK(nq)));
4247 4261 }
4248 4262 #endif
4249 4263 setqback(nq, pri);
4250 4264 qenable_locked(nq);
4251 4265 if (freezer != curthread || STREAM(q) != STREAM(nq))
4252 4266 mutex_exit(QLOCK(nq));
4253 4267 }
4254 4268 releasestr(q);
4255 4269 }
4256 4270
4257 4271 /*
4258 4272 * Return the appropriate errno when one of flags_to_check is set
4259 4273 * in sd_flags. Uses the exported error routines if they are set.
4260 4274 * Will return 0 if non error is set (or if the exported error routines
4261 4275 * do not return an error).
4262 4276 *
4263 4277 * If there is both a read and write error to check, we prefer the read error.
4264 4278 * Also, give preference to recorded errno's over the error functions.
4265 4279 * The flags that are handled are:
4266 4280 * STPLEX return EINVAL
4267 4281 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST)
4268 4282 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST)
4269 4283 * STRHUP return sd_werror
4270 4284 *
4271 4285 * If the caller indicates that the operation is a peek, a nonpersistent error
4272 4286 * is not cleared.
4273 4287 */
4274 4288 int
4275 4289 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4276 4290 {
4277 4291 int32_t sd_flag = stp->sd_flag & flags_to_check;
4278 4292 int error = 0;
4279 4293
4280 4294 ASSERT(MUTEX_HELD(&stp->sd_lock));
4281 4295 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4282 4296 if (sd_flag & STPLEX)
4283 4297 error = EINVAL;
4284 4298 else if (sd_flag & STRDERR) {
4285 4299 error = stp->sd_rerror;
4286 4300 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4287 4301 /*
4288 4302 * Read errors are non-persistent i.e. discarded once
4289 4303 * returned to a non-peeking caller,
4290 4304 */
4291 4305 stp->sd_rerror = 0;
4292 4306 stp->sd_flag &= ~STRDERR;
4293 4307 }
4294 4308 if (error == 0 && stp->sd_rderrfunc != NULL) {
4295 4309 int clearerr = 0;
4296 4310
4297 4311 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4298 4312 &clearerr);
4299 4313 if (clearerr) {
4300 4314 stp->sd_flag &= ~STRDERR;
4301 4315 stp->sd_rderrfunc = NULL;
4302 4316 }
4303 4317 }
4304 4318 } else if (sd_flag & STWRERR) {
4305 4319 error = stp->sd_werror;
4306 4320 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4307 4321 /*
4308 4322 * Write errors are non-persistent i.e. discarded once
4309 4323 * returned to a non-peeking caller,
4310 4324 */
4311 4325 stp->sd_werror = 0;
4312 4326 stp->sd_flag &= ~STWRERR;
4313 4327 }
4314 4328 if (error == 0 && stp->sd_wrerrfunc != NULL) {
4315 4329 int clearerr = 0;
4316 4330
4317 4331 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4318 4332 &clearerr);
4319 4333 if (clearerr) {
4320 4334 stp->sd_flag &= ~STWRERR;
4321 4335 stp->sd_wrerrfunc = NULL;
4322 4336 }
4323 4337 }
4324 4338 } else if (sd_flag & STRHUP) {
4325 4339 /* sd_werror set when STRHUP */
4326 4340 error = stp->sd_werror;
4327 4341 }
4328 4342 return (error);
4329 4343 }
4330 4344
4331 4345
4332 4346 /*
4333 4347 * Single-thread open/close/push/pop
4334 4348 * for twisted streams also
4335 4349 */
4336 4350 int
4337 4351 strstartplumb(stdata_t *stp, int flag, int cmd)
4338 4352 {
4339 4353 int waited = 1;
4340 4354 int error = 0;
4341 4355
4342 4356 if (STRMATED(stp)) {
4343 4357 struct stdata *stmatep = stp->sd_mate;
4344 4358
4345 4359 STRLOCKMATES(stp);
4346 4360 while (waited) {
4347 4361 waited = 0;
4348 4362 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4349 4363 if ((cmd == I_POP) &&
4350 4364 (flag & (FNDELAY|FNONBLOCK))) {
4351 4365 STRUNLOCKMATES(stp);
4352 4366 return (EAGAIN);
4353 4367 }
4354 4368 waited = 1;
4355 4369 mutex_exit(&stp->sd_lock);
4356 4370 if (!cv_wait_sig(&stmatep->sd_monitor,
4357 4371 &stmatep->sd_lock)) {
4358 4372 mutex_exit(&stmatep->sd_lock);
4359 4373 return (EINTR);
4360 4374 }
4361 4375 mutex_exit(&stmatep->sd_lock);
4362 4376 STRLOCKMATES(stp);
4363 4377 }
4364 4378 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4365 4379 if ((cmd == I_POP) &&
4366 4380 (flag & (FNDELAY|FNONBLOCK))) {
4367 4381 STRUNLOCKMATES(stp);
4368 4382 return (EAGAIN);
4369 4383 }
4370 4384 waited = 1;
4371 4385 mutex_exit(&stmatep->sd_lock);
4372 4386 if (!cv_wait_sig(&stp->sd_monitor,
4373 4387 &stp->sd_lock)) {
4374 4388 mutex_exit(&stp->sd_lock);
4375 4389 return (EINTR);
4376 4390 }
4377 4391 mutex_exit(&stp->sd_lock);
4378 4392 STRLOCKMATES(stp);
4379 4393 }
4380 4394 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4381 4395 error = strgeterr(stp,
4382 4396 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4383 4397 if (error != 0) {
4384 4398 STRUNLOCKMATES(stp);
4385 4399 return (error);
4386 4400 }
4387 4401 }
4388 4402 }
4389 4403 stp->sd_flag |= STRPLUMB;
4390 4404 STRUNLOCKMATES(stp);
4391 4405 } else {
4392 4406 mutex_enter(&stp->sd_lock);
4393 4407 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4394 4408 if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4395 4409 (flag & (FNDELAY|FNONBLOCK))) {
4396 4410 mutex_exit(&stp->sd_lock);
4397 4411 return (EAGAIN);
4398 4412 }
4399 4413 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4400 4414 mutex_exit(&stp->sd_lock);
4401 4415 return (EINTR);
4402 4416 }
4403 4417 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4404 4418 error = strgeterr(stp,
4405 4419 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4406 4420 if (error != 0) {
4407 4421 mutex_exit(&stp->sd_lock);
4408 4422 return (error);
4409 4423 }
4410 4424 }
4411 4425 }
4412 4426 stp->sd_flag |= STRPLUMB;
4413 4427 mutex_exit(&stp->sd_lock);
4414 4428 }
4415 4429 return (0);
4416 4430 }
4417 4431
4418 4432 /*
4419 4433 * Complete the plumbing operation associated with stream `stp'.
4420 4434 */
4421 4435 void
4422 4436 strendplumb(stdata_t *stp)
4423 4437 {
4424 4438 ASSERT(MUTEX_HELD(&stp->sd_lock));
4425 4439 ASSERT(stp->sd_flag & STRPLUMB);
4426 4440 stp->sd_flag &= ~STRPLUMB;
4427 4441 cv_broadcast(&stp->sd_monitor);
4428 4442 }
4429 4443
4430 4444 /*
4431 4445 * This describes how the STREAMS framework handles synchronization
4432 4446 * during open/push and close/pop.
4433 4447 * The key interfaces for open and close are qprocson and qprocsoff,
4434 4448 * respectively. While the close case in general is harder both open
4435 4449 * have close have significant similarities.
4436 4450 *
4437 4451 * During close the STREAMS framework has to both ensure that there
4438 4452 * are no stale references to the queue pair (and syncq) that
4439 4453 * are being closed and also provide the guarantees that are documented
4440 4454 * in qprocsoff(9F).
4441 4455 * If there are stale references to the queue that is closing it can
4442 4456 * result in kernel memory corruption or kernel panics.
4443 4457 *
4444 4458 * Note that is it up to the module/driver to ensure that it itself
4445 4459 * does not have any stale references to the closing queues once its close
4446 4460 * routine returns. This includes:
4447 4461 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4448 4462 * associated with the queues. For timeout and bufcall callbacks the
4449 4463 * module/driver also has to ensure (or wait for) any callbacks that
4450 4464 * are in progress.
4451 4465 * - If the module/driver is using esballoc it has to ensure that any
4452 4466 * esballoc free functions do not refer to a queue that has closed.
4453 4467 * (Note that in general the close routine can not wait for the esballoc'ed
4454 4468 * messages to be freed since that can cause a deadlock.)
4455 4469 * - Cancelling any interrupts that refer to the closing queues and
4456 4470 * also ensuring that there are no interrupts in progress that will
4457 4471 * refer to the closing queues once the close routine returns.
4458 4472 * - For multiplexors removing any driver global state that refers to
4459 4473 * the closing queue and also ensuring that there are no threads in
4460 4474 * the multiplexor that has picked up a queue pointer but not yet
4461 4475 * finished using it.
4462 4476 *
4463 4477 * In addition, a driver/module can only reference the q_next pointer
4464 4478 * in its open, close, put, or service procedures or in a
4465 4479 * qtimeout/qbufcall callback procedure executing "on" the correct
4466 4480 * stream. Thus it can not reference the q_next pointer in an interrupt
4467 4481 * routine or a timeout, bufcall or esballoc callback routine. Likewise
4468 4482 * it can not reference q_next of a different queue e.g. in a mux that
4469 4483 * passes messages from one queues put/service procedure to another queue.
4470 4484 * In all the cases when the driver/module can not access the q_next
4471 4485 * field it must use the *next* versions e.g. canputnext instead of
4472 4486 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4473 4487 *
4474 4488 *
4475 4489 * Assuming that the driver/module conforms to the above constraints
4476 4490 * the STREAMS framework has to avoid stale references to q_next for all
4477 4491 * the framework internal cases which include (but are not limited to):
4478 4492 * - Threads in canput/canputnext/backenable and elsewhere that are
4479 4493 * walking q_next.
4480 4494 * - Messages on a syncq that have a reference to the queue through b_queue.
4481 4495 * - Messages on an outer perimeter (syncq) that have a reference to the
4482 4496 * queue through b_queue.
4483 4497 * - Threads that use q_nfsrv (e.g. canput) to find a queue.
4484 4498 * Note that only canput and bcanput use q_nfsrv without any locking.
4485 4499 *
4486 4500 * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4487 4501 * after qprocsoff returns, the framework has to ensure that no threads can
4488 4502 * enter the put or service routines for the closing read or write-side queue.
4489 4503 * In addition to preventing "direct" entry into the put procedures
4490 4504 * the framework also has to prevent messages being drained from
4491 4505 * the syncq or the outer perimeter.
4492 4506 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4493 4507 * mechanism to prevent qwriter(PERIM_OUTER) from running after
4494 4508 * qprocsoff has returned.
4495 4509 * Note that if a module/driver uses put(9F) on one of its own queues
4496 4510 * it is up to the module/driver to ensure that the put() doesn't
4497 4511 * get called when the queue is closing.
4498 4512 *
4499 4513 *
4500 4514 * The framework aspects of the above "contract" is implemented by
4501 4515 * qprocsoff, removeq, and strlock:
4502 4516 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4503 4517 * entering the service procedures.
4504 4518 * - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4505 4519 * canputnext, backenable etc from dereferencing the q_next that will
4506 4520 * soon change.
4507 4521 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4508 4522 * or other q_next walker that uses claimstr/releasestr to finish.
4509 4523 * - optionally for every syncq in the stream strlock acquires all the
4510 4524 * sq_lock's and waits for all sq_counts to drop to a value that indicates
4511 4525 * that no thread executes in the put or service procedures and that no
4512 4526 * thread is draining into the module/driver. This ensures that no
4513 4527 * open, close, put, service, or qtimeout/qbufcall callback procedure is
4514 4528 * currently executing hence no such thread can end up with the old stale
4515 4529 * q_next value and no canput/backenable can have the old stale
4516 4530 * q_nfsrv/q_next.
4517 4531 * - qdetach (wait_svc) makes sure that any scheduled or running threads
4518 4532 * have either finished or observed the QWCLOSE flag and gone away.
4519 4533 */
4520 4534
4521 4535
4522 4536 /*
4523 4537 * Get all the locks necessary to change q_next.
4524 4538 *
4525 4539 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4526 4540 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4527 4541 * the only threads inside the syncq are threads currently calling removeq().
4528 4542 * Since threads calling removeq() are in the process of removing their queues
4529 4543 * from the stream, we do not need to worry about them accessing a stale q_next
4530 4544 * pointer and thus we do not need to wait for them to exit (in fact, waiting
4531 4545 * for them can cause deadlock).
4532 4546 *
4533 4547 * This routine is subject to starvation since it does not set any flag to
4534 4548 * prevent threads from entering a module in the stream (i.e. sq_count can
4535 4549 * increase on some syncq while it is waiting on some other syncq).
4536 4550 *
4537 4551 * Assumes that only one thread attempts to call strlock for a given
4538 4552 * stream. If this is not the case the two threads would deadlock.
4539 4553 * This assumption is guaranteed since strlock is only called by insertq
4540 4554 * and removeq and streams plumbing changes are single-threaded for
4541 4555 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4542 4556 *
4543 4557 * For pipes, it is not difficult to atomically designate a pair of streams
4544 4558 * to be mated. Once mated atomically by the framework the twisted pair remain
4545 4559 * configured that way until dismantled atomically by the framework.
4546 4560 * When plumbing takes place on a twisted stream it is necessary to ensure that
4547 4561 * this operation is done exclusively on the twisted stream since two such
4548 4562 * operations, each initiated on different ends of the pipe will deadlock
4549 4563 * waiting for each other to complete.
4550 4564 *
4551 4565 * On entry, no locks should be held.
4552 4566 * The locks acquired and held by strlock depends on a few factors.
4553 4567 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4554 4568 * and held on exit and all sq_count are at an acceptable level.
4555 4569 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4556 4570 * sd_refcnt being zero.
4557 4571 */
4558 4572
4559 4573 static void
4560 4574 strlock(struct stdata *stp, sqlist_t *sqlist)
4561 4575 {
4562 4576 syncql_t *sql, *sql2;
4563 4577 retry:
4564 4578 /*
4565 4579 * Wait for any claimstr to go away.
4566 4580 */
4567 4581 if (STRMATED(stp)) {
4568 4582 struct stdata *stp1, *stp2;
4569 4583
4570 4584 STRLOCKMATES(stp);
4571 4585 /*
4572 4586 * Note that the selection of locking order is not
4573 4587 * important, just that they are always acquired in
4574 4588 * the same order. To assure this, we choose this
4575 4589 * order based on the value of the pointer, and since
4576 4590 * the pointer will not change for the life of this
4577 4591 * pair, we will always grab the locks in the same
4578 4592 * order (and hence, prevent deadlocks).
4579 4593 */
4580 4594 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4581 4595 stp1 = stp;
4582 4596 stp2 = stp->sd_mate;
4583 4597 } else {
4584 4598 stp2 = stp;
4585 4599 stp1 = stp->sd_mate;
4586 4600 }
4587 4601 mutex_enter(&stp1->sd_reflock);
4588 4602 if (stp1->sd_refcnt > 0) {
4589 4603 STRUNLOCKMATES(stp);
4590 4604 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4591 4605 mutex_exit(&stp1->sd_reflock);
4592 4606 goto retry;
4593 4607 }
4594 4608 mutex_enter(&stp2->sd_reflock);
4595 4609 if (stp2->sd_refcnt > 0) {
4596 4610 STRUNLOCKMATES(stp);
4597 4611 mutex_exit(&stp1->sd_reflock);
4598 4612 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4599 4613 mutex_exit(&stp2->sd_reflock);
4600 4614 goto retry;
4601 4615 }
4602 4616 STREAM_PUTLOCKS_ENTER(stp1);
4603 4617 STREAM_PUTLOCKS_ENTER(stp2);
4604 4618 } else {
4605 4619 mutex_enter(&stp->sd_lock);
4606 4620 mutex_enter(&stp->sd_reflock);
4607 4621 while (stp->sd_refcnt > 0) {
4608 4622 mutex_exit(&stp->sd_lock);
4609 4623 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4610 4624 if (mutex_tryenter(&stp->sd_lock) == 0) {
4611 4625 mutex_exit(&stp->sd_reflock);
4612 4626 mutex_enter(&stp->sd_lock);
4613 4627 mutex_enter(&stp->sd_reflock);
4614 4628 }
4615 4629 }
4616 4630 STREAM_PUTLOCKS_ENTER(stp);
4617 4631 }
4618 4632
4619 4633 if (sqlist == NULL)
4620 4634 return;
4621 4635
4622 4636 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4623 4637 syncq_t *sq = sql->sql_sq;
4624 4638 uint16_t count;
4625 4639
4626 4640 mutex_enter(SQLOCK(sq));
4627 4641 count = sq->sq_count;
4628 4642 ASSERT(sq->sq_rmqcount <= count);
4629 4643 SQ_PUTLOCKS_ENTER(sq);
4630 4644 SUM_SQ_PUTCOUNTS(sq, count);
4631 4645 if (count == sq->sq_rmqcount)
4632 4646 continue;
4633 4647
4634 4648 /* Failed - drop all locks that we have acquired so far */
4635 4649 if (STRMATED(stp)) {
4636 4650 STREAM_PUTLOCKS_EXIT(stp);
4637 4651 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4638 4652 STRUNLOCKMATES(stp);
4639 4653 mutex_exit(&stp->sd_reflock);
4640 4654 mutex_exit(&stp->sd_mate->sd_reflock);
4641 4655 } else {
4642 4656 STREAM_PUTLOCKS_EXIT(stp);
4643 4657 mutex_exit(&stp->sd_lock);
4644 4658 mutex_exit(&stp->sd_reflock);
4645 4659 }
4646 4660 for (sql2 = sqlist->sqlist_head; sql2 != sql;
4647 4661 sql2 = sql2->sql_next) {
4648 4662 SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4649 4663 mutex_exit(SQLOCK(sql2->sql_sq));
4650 4664 }
4651 4665
4652 4666 /*
4653 4667 * The wait loop below may starve when there are many threads
4654 4668 * claiming the syncq. This is especially a problem with permod
4655 4669 * syncqs (IP). To lessen the impact of the problem we increment
4656 4670 * sq_needexcl and clear fastbits so that putnexts will slow
4657 4671 * down and call sqenable instead of draining right away.
4658 4672 */
4659 4673 sq->sq_needexcl++;
4660 4674 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4661 4675 while (count > sq->sq_rmqcount) {
4662 4676 sq->sq_flags |= SQ_WANTWAKEUP;
4663 4677 SQ_PUTLOCKS_EXIT(sq);
4664 4678 cv_wait(&sq->sq_wait, SQLOCK(sq));
4665 4679 count = sq->sq_count;
4666 4680 SQ_PUTLOCKS_ENTER(sq);
4667 4681 SUM_SQ_PUTCOUNTS(sq, count);
4668 4682 }
4669 4683 sq->sq_needexcl--;
4670 4684 if (sq->sq_needexcl == 0)
4671 4685 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4672 4686 SQ_PUTLOCKS_EXIT(sq);
4673 4687 ASSERT(count == sq->sq_rmqcount);
4674 4688 mutex_exit(SQLOCK(sq));
4675 4689 goto retry;
4676 4690 }
4677 4691 }
4678 4692
4679 4693 /*
4680 4694 * Drop all the locks that strlock acquired.
4681 4695 */
4682 4696 static void
4683 4697 strunlock(struct stdata *stp, sqlist_t *sqlist)
4684 4698 {
4685 4699 syncql_t *sql;
4686 4700
4687 4701 if (STRMATED(stp)) {
4688 4702 STREAM_PUTLOCKS_EXIT(stp);
4689 4703 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4690 4704 STRUNLOCKMATES(stp);
4691 4705 mutex_exit(&stp->sd_reflock);
4692 4706 mutex_exit(&stp->sd_mate->sd_reflock);
4693 4707 } else {
4694 4708 STREAM_PUTLOCKS_EXIT(stp);
4695 4709 mutex_exit(&stp->sd_lock);
4696 4710 mutex_exit(&stp->sd_reflock);
4697 4711 }
4698 4712
4699 4713 if (sqlist == NULL)
4700 4714 return;
4701 4715
4702 4716 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4703 4717 SQ_PUTLOCKS_EXIT(sql->sql_sq);
4704 4718 mutex_exit(SQLOCK(sql->sql_sq));
4705 4719 }
4706 4720 }
4707 4721
4708 4722 /*
4709 4723 * When the module has service procedure, we need check if the next
4710 4724 * module which has service procedure is in flow control to trigger
4711 4725 * the backenable.
4712 4726 */
4713 4727 static void
4714 4728 backenable_insertedq(queue_t *q)
4715 4729 {
4716 4730 qband_t *qbp;
4717 4731
4718 4732 claimstr(q);
4719 4733 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4720 4734 if (q->q_next->q_nfsrv->q_flag & QWANTW)
4721 4735 backenable(q, 0);
4722 4736
4723 4737 qbp = q->q_next->q_nfsrv->q_bandp;
4724 4738 for (; qbp != NULL; qbp = qbp->qb_next)
4725 4739 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4726 4740 backenable(q, qbp->qb_first->b_band);
4727 4741 }
4728 4742 releasestr(q);
4729 4743 }
4730 4744
4731 4745 /*
4732 4746 * Given two read queues, insert a new single one after another.
4733 4747 *
4734 4748 * This routine acquires all the necessary locks in order to change
4735 4749 * q_next and related pointer using strlock().
4736 4750 * It depends on the stream head ensuring that there are no concurrent
4737 4751 * insertq or removeq on the same stream. The stream head ensures this
4738 4752 * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4739 4753 *
4740 4754 * Note that no syncq locks are held during the q_next change. This is
4741 4755 * applied to all streams since, unlike removeq, there is no problem of stale
4742 4756 * pointers when adding a module to the stream. Thus drivers/modules that do a
4743 4757 * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4744 4758 * applied this optimization to all streams.
4745 4759 */
4746 4760 void
4747 4761 insertq(struct stdata *stp, queue_t *new)
4748 4762 {
4749 4763 queue_t *after;
4750 4764 queue_t *wafter;
4751 4765 queue_t *wnew = _WR(new);
4752 4766 boolean_t have_fifo = B_FALSE;
4753 4767
4754 4768 if (new->q_flag & _QINSERTING) {
4755 4769 ASSERT(stp->sd_vnode->v_type != VFIFO);
4756 4770 after = new->q_next;
4757 4771 wafter = _WR(new->q_next);
4758 4772 } else {
4759 4773 after = _RD(stp->sd_wrq);
4760 4774 wafter = stp->sd_wrq;
4761 4775 }
4762 4776
4763 4777 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4764 4778 "insertq:%p, %p", after, new);
4765 4779 ASSERT(after->q_flag & QREADR);
4766 4780 ASSERT(new->q_flag & QREADR);
4767 4781
4768 4782 strlock(stp, NULL);
4769 4783
4770 4784 /* Do we have a FIFO? */
4771 4785 if (wafter->q_next == after) {
4772 4786 have_fifo = B_TRUE;
4773 4787 wnew->q_next = new;
4774 4788 } else {
4775 4789 wnew->q_next = wafter->q_next;
4776 4790 }
4777 4791 new->q_next = after;
4778 4792
4779 4793 set_nfsrv_ptr(new, wnew, after, wafter);
4780 4794 /*
4781 4795 * set_nfsrv_ptr() needs to know if this is an insertion or not,
4782 4796 * so only reset this flag after calling it.
4783 4797 */
4784 4798 new->q_flag &= ~_QINSERTING;
4785 4799
4786 4800 if (have_fifo) {
4787 4801 wafter->q_next = wnew;
4788 4802 } else {
4789 4803 if (wafter->q_next)
4790 4804 _OTHERQ(wafter->q_next)->q_next = new;
4791 4805 wafter->q_next = wnew;
4792 4806 }
4793 4807
4794 4808 set_qend(new);
4795 4809 /* The QEND flag might have to be updated for the upstream guy */
4796 4810 set_qend(after);
4797 4811
4798 4812 ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4799 4813 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4800 4814 ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4801 4815 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4802 4816 strsetuio(stp);
4803 4817
4804 4818 /*
4805 4819 * If this was a module insertion, bump the push count.
4806 4820 */
4807 4821 if (!(new->q_flag & QISDRV))
4808 4822 stp->sd_pushcnt++;
4809 4823
4810 4824 strunlock(stp, NULL);
4811 4825
4812 4826 /* check if the write Q needs backenable */
4813 4827 backenable_insertedq(wnew);
4814 4828
4815 4829 /* check if the read Q needs backenable */
4816 4830 backenable_insertedq(new);
4817 4831 }
4818 4832
4819 4833 /*
4820 4834 * Given a read queue, unlink it from any neighbors.
4821 4835 *
4822 4836 * This routine acquires all the necessary locks in order to
4823 4837 * change q_next and related pointers and also guard against
4824 4838 * stale references (e.g. through q_next) to the queue that
4825 4839 * is being removed. It also plays part of the role in ensuring
4826 4840 * that the module's/driver's put procedure doesn't get called
4827 4841 * after qprocsoff returns.
4828 4842 *
4829 4843 * Removeq depends on the stream head ensuring that there are
4830 4844 * no concurrent insertq or removeq on the same stream. The
4831 4845 * stream head ensures this using the flags STWOPEN, STRCLOSE and
4832 4846 * STRPLUMB.
4833 4847 *
4834 4848 * The set of locks needed to remove the queue is different in
4835 4849 * different cases:
4836 4850 *
4837 4851 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4838 4852 * waiting for the syncq reference count to drop to 0 indicating that no
4839 4853 * non-close threads are present anywhere in the stream. This ensures that any
4840 4854 * module/driver can reference q_next in its open, close, put, or service
4841 4855 * procedures.
4842 4856 *
4843 4857 * The sq_rmqcount counter tracks the number of threads inside removeq().
4844 4858 * strlock() ensures that there is either no threads executing inside perimeter
4845 4859 * or there is only a thread calling qprocsoff().
4846 4860 *
4847 4861 * strlock() compares the value of sq_count with the number of threads inside
4848 4862 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4849 4863 * any threads waiting in strlock() when the sq_rmqcount increases.
4850 4864 */
4851 4865
4852 4866 void
4853 4867 removeq(queue_t *qp)
4854 4868 {
4855 4869 queue_t *wqp = _WR(qp);
4856 4870 struct stdata *stp = STREAM(qp);
4857 4871 sqlist_t *sqlist = NULL;
4858 4872 boolean_t isdriver;
4859 4873 int moved;
4860 4874 syncq_t *sq = qp->q_syncq;
4861 4875 syncq_t *wsq = wqp->q_syncq;
4862 4876
4863 4877 ASSERT(stp);
4864 4878
4865 4879 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4866 4880 "removeq:%p %p", qp, wqp);
4867 4881 ASSERT(qp->q_flag&QREADR);
4868 4882
4869 4883 /*
4870 4884 * For queues using Synchronous streams, we must wait for all threads in
4871 4885 * rwnext() to drain out before proceeding.
4872 4886 */
4873 4887 if (qp->q_flag & QSYNCSTR) {
4874 4888 /* First, we need wakeup any threads blocked in rwnext() */
4875 4889 mutex_enter(SQLOCK(sq));
4876 4890 if (sq->sq_flags & SQ_WANTWAKEUP) {
4877 4891 sq->sq_flags &= ~SQ_WANTWAKEUP;
4878 4892 cv_broadcast(&sq->sq_wait);
4879 4893 }
4880 4894 mutex_exit(SQLOCK(sq));
4881 4895
4882 4896 if (wsq != sq) {
4883 4897 mutex_enter(SQLOCK(wsq));
4884 4898 if (wsq->sq_flags & SQ_WANTWAKEUP) {
4885 4899 wsq->sq_flags &= ~SQ_WANTWAKEUP;
4886 4900 cv_broadcast(&wsq->sq_wait);
4887 4901 }
4888 4902 mutex_exit(SQLOCK(wsq));
4889 4903 }
4890 4904
4891 4905 mutex_enter(QLOCK(qp));
4892 4906 while (qp->q_rwcnt > 0) {
4893 4907 qp->q_flag |= QWANTRMQSYNC;
4894 4908 cv_wait(&qp->q_wait, QLOCK(qp));
4895 4909 }
4896 4910 mutex_exit(QLOCK(qp));
4897 4911
4898 4912 mutex_enter(QLOCK(wqp));
4899 4913 while (wqp->q_rwcnt > 0) {
4900 4914 wqp->q_flag |= QWANTRMQSYNC;
4901 4915 cv_wait(&wqp->q_wait, QLOCK(wqp));
4902 4916 }
4903 4917 mutex_exit(QLOCK(wqp));
4904 4918 }
4905 4919
4906 4920 mutex_enter(SQLOCK(sq));
4907 4921 sq->sq_rmqcount++;
4908 4922 if (sq->sq_flags & SQ_WANTWAKEUP) {
4909 4923 sq->sq_flags &= ~SQ_WANTWAKEUP;
4910 4924 cv_broadcast(&sq->sq_wait);
4911 4925 }
4912 4926 mutex_exit(SQLOCK(sq));
4913 4927
4914 4928 isdriver = (qp->q_flag & QISDRV);
4915 4929
4916 4930 sqlist = sqlist_build(qp, stp, STRMATED(stp));
4917 4931 strlock(stp, sqlist);
4918 4932
4919 4933 reset_nfsrv_ptr(qp, wqp);
4920 4934
4921 4935 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4922 4936 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4923 4937 /* Do we have a FIFO? */
4924 4938 if (wqp->q_next == qp) {
4925 4939 stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4926 4940 } else {
4927 4941 if (wqp->q_next)
4928 4942 backq(qp)->q_next = qp->q_next;
4929 4943 if (qp->q_next)
4930 4944 backq(wqp)->q_next = wqp->q_next;
4931 4945 }
4932 4946
4933 4947 /* The QEND flag might have to be updated for the upstream guy */
4934 4948 if (qp->q_next)
4935 4949 set_qend(qp->q_next);
4936 4950
4937 4951 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4938 4952 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4939 4953
4940 4954 /*
4941 4955 * Move any messages destined for the put procedures to the next
4942 4956 * syncq in line. Otherwise free them.
4943 4957 */
4944 4958 moved = 0;
4945 4959 /*
4946 4960 * Quick check to see whether there are any messages or events.
4947 4961 */
4948 4962 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4949 4963 moved += propagate_syncq(qp);
4950 4964 if (wqp->q_syncqmsgs != 0 ||
4951 4965 (wqp->q_syncq->sq_flags & SQ_EVENTS))
4952 4966 moved += propagate_syncq(wqp);
4953 4967
4954 4968 strsetuio(stp);
4955 4969
4956 4970 /*
4957 4971 * If this was a module removal, decrement the push count.
4958 4972 */
4959 4973 if (!isdriver)
4960 4974 stp->sd_pushcnt--;
4961 4975
4962 4976 strunlock(stp, sqlist);
4963 4977 sqlist_free(sqlist);
4964 4978
4965 4979 /*
4966 4980 * Make sure any messages that were propagated are drained.
4967 4981 * Also clear any QFULL bit caused by messages that were propagated.
4968 4982 */
4969 4983
4970 4984 if (qp->q_next != NULL) {
4971 4985 clr_qfull(qp);
4972 4986 /*
4973 4987 * For the driver calling qprocsoff, propagate_syncq
4974 4988 * frees all the messages instead of putting it in
4975 4989 * the stream head
4976 4990 */
4977 4991 if (!isdriver && (moved > 0))
4978 4992 emptysq(qp->q_next->q_syncq);
4979 4993 }
4980 4994 if (wqp->q_next != NULL) {
4981 4995 clr_qfull(wqp);
4982 4996 /*
4983 4997 * We come here for any pop of a module except for the
4984 4998 * case of driver being removed. We don't call emptysq
4985 4999 * if we did not move any messages. This will avoid holding
4986 5000 * PERMOD syncq locks in emptysq
4987 5001 */
4988 5002 if (moved > 0)
4989 5003 emptysq(wqp->q_next->q_syncq);
4990 5004 }
4991 5005
4992 5006 mutex_enter(SQLOCK(sq));
4993 5007 sq->sq_rmqcount--;
4994 5008 mutex_exit(SQLOCK(sq));
4995 5009 }
4996 5010
4997 5011 /*
4998 5012 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
4999 5013 * SQ_WRITER) on a syncq.
5000 5014 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5001 5015 * sync queue and waits until sq_count reaches maxcnt.
5002 5016 *
5003 5017 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5004 5018 * does not care about putnext threads that are in the middle of calling put
5005 5019 * entry points.
5006 5020 *
5007 5021 * This routine is used for both inner and outer syncqs.
5008 5022 */
5009 5023 static void
5010 5024 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5011 5025 {
5012 5026 uint16_t count = 0;
5013 5027
5014 5028 mutex_enter(SQLOCK(sq));
5015 5029 /*
5016 5030 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5017 5031 * SQ_FROZEN will be set if there is a frozen stream that has a
5018 5032 * queue which also refers to this "shared" syncq.
5019 5033 * SQ_BLOCKED will be set if there is "off" queue which also
5020 5034 * refers to this "shared" syncq.
5021 5035 */
5022 5036 if (maxcnt != -1) {
5023 5037 count = sq->sq_count;
5024 5038 SQ_PUTLOCKS_ENTER(sq);
5025 5039 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5026 5040 SUM_SQ_PUTCOUNTS(sq, count);
5027 5041 }
5028 5042 sq->sq_needexcl++;
5029 5043 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5030 5044
5031 5045 while ((sq->sq_flags & flag) ||
5032 5046 (maxcnt != -1 && count > (unsigned)maxcnt)) {
5033 5047 sq->sq_flags |= SQ_WANTWAKEUP;
5034 5048 if (maxcnt != -1) {
5035 5049 SQ_PUTLOCKS_EXIT(sq);
5036 5050 }
5037 5051 cv_wait(&sq->sq_wait, SQLOCK(sq));
5038 5052 if (maxcnt != -1) {
5039 5053 count = sq->sq_count;
5040 5054 SQ_PUTLOCKS_ENTER(sq);
5041 5055 SUM_SQ_PUTCOUNTS(sq, count);
5042 5056 }
5043 5057 }
5044 5058 sq->sq_needexcl--;
5045 5059 sq->sq_flags |= flag;
5046 5060 ASSERT(maxcnt == -1 || count == maxcnt);
5047 5061 if (maxcnt != -1) {
5048 5062 if (sq->sq_needexcl == 0) {
5049 5063 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5050 5064 }
5051 5065 SQ_PUTLOCKS_EXIT(sq);
5052 5066 } else if (sq->sq_needexcl == 0) {
5053 5067 SQ_PUTCOUNT_SETFAST(sq);
5054 5068 }
5055 5069
5056 5070 mutex_exit(SQLOCK(sq));
5057 5071 }
5058 5072
5059 5073 /*
5060 5074 * Reset a flag that was set with blocksq.
5061 5075 *
5062 5076 * Can not use this routine to reset SQ_WRITER.
5063 5077 *
5064 5078 * If "isouter" is set then the syncq is assumed to be an outer perimeter
5065 5079 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5066 5080 * to handle the queued qwriter operations.
5067 5081 *
5068 5082 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5069 5083 * sq_putlocks are used.
5070 5084 */
5071 5085 static void
5072 5086 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5073 5087 {
5074 5088 uint16_t flags;
5075 5089
5076 5090 mutex_enter(SQLOCK(sq));
5077 5091 ASSERT(resetflag != SQ_WRITER);
5078 5092 ASSERT(sq->sq_flags & resetflag);
5079 5093 flags = sq->sq_flags & ~resetflag;
5080 5094 sq->sq_flags = flags;
5081 5095 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5082 5096 if (flags & SQ_WANTWAKEUP) {
5083 5097 flags &= ~SQ_WANTWAKEUP;
5084 5098 cv_broadcast(&sq->sq_wait);
5085 5099 }
5086 5100 sq->sq_flags = flags;
5087 5101 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5088 5102 if (!isouter) {
5089 5103 /* drain_syncq drops SQLOCK */
5090 5104 drain_syncq(sq);
5091 5105 return;
5092 5106 }
5093 5107 }
5094 5108 }
5095 5109 mutex_exit(SQLOCK(sq));
5096 5110 }
5097 5111
5098 5112 /*
5099 5113 * Reset a flag that was set with blocksq.
5100 5114 * Does not drain the syncq. Use emptysq() for that.
5101 5115 * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5102 5116 *
5103 5117 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5104 5118 * sq_putlocks are used.
5105 5119 */
5106 5120 static int
5107 5121 dropsq(syncq_t *sq, uint16_t resetflag)
5108 5122 {
5109 5123 uint16_t flags;
5110 5124
5111 5125 mutex_enter(SQLOCK(sq));
5112 5126 ASSERT(sq->sq_flags & resetflag);
5113 5127 flags = sq->sq_flags & ~resetflag;
5114 5128 if (flags & SQ_WANTWAKEUP) {
5115 5129 flags &= ~SQ_WANTWAKEUP;
5116 5130 cv_broadcast(&sq->sq_wait);
5117 5131 }
5118 5132 sq->sq_flags = flags;
5119 5133 mutex_exit(SQLOCK(sq));
5120 5134 if (flags & SQ_QUEUED)
5121 5135 return (1);
5122 5136 return (0);
5123 5137 }
5124 5138
5125 5139 /*
5126 5140 * Empty all the messages on a syncq.
5127 5141 *
5128 5142 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5129 5143 * sq_putlocks are used.
5130 5144 */
5131 5145 static void
5132 5146 emptysq(syncq_t *sq)
5133 5147 {
5134 5148 uint16_t flags;
5135 5149
5136 5150 mutex_enter(SQLOCK(sq));
5137 5151 flags = sq->sq_flags;
5138 5152 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5139 5153 /*
5140 5154 * To prevent potential recursive invocation of drain_syncq we
5141 5155 * do not call drain_syncq if count is non-zero.
5142 5156 */
5143 5157 if (sq->sq_count == 0) {
5144 5158 /* drain_syncq() drops SQLOCK */
5145 5159 drain_syncq(sq);
5146 5160 return;
5147 5161 } else
5148 5162 sqenable(sq);
5149 5163 }
5150 5164 mutex_exit(SQLOCK(sq));
5151 5165 }
5152 5166
5153 5167 /*
5154 5168 * Ordered insert while removing duplicates.
5155 5169 */
5156 5170 static void
5157 5171 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5158 5172 {
5159 5173 syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5160 5174
5161 5175 prev_sqlpp = &sqlist->sqlist_head;
5162 5176 while ((sqlp = *prev_sqlpp) != NULL) {
5163 5177 if (sqlp->sql_sq >= sqp) {
5164 5178 if (sqlp->sql_sq == sqp) /* duplicate */
5165 5179 return;
5166 5180 break;
5167 5181 }
5168 5182 prev_sqlpp = &sqlp->sql_next;
5169 5183 }
5170 5184 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5171 5185 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5172 5186 new_sqlp->sql_next = sqlp;
5173 5187 new_sqlp->sql_sq = sqp;
5174 5188 *prev_sqlpp = new_sqlp;
5175 5189 }
5176 5190
5177 5191 /*
5178 5192 * Walk the write side queues until we hit either the driver
5179 5193 * or a twist in the stream (_SAMESTR will return false in both
5180 5194 * these cases) then turn around and walk the read side queues
5181 5195 * back up to the stream head.
5182 5196 */
5183 5197 static void
5184 5198 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5185 5199 {
5186 5200 while (q != NULL) {
5187 5201 sqlist_insert(sqlist, q->q_syncq);
5188 5202
5189 5203 if (_SAMESTR(q))
5190 5204 q = q->q_next;
5191 5205 else if (!(q->q_flag & QREADR))
5192 5206 q = _RD(q);
5193 5207 else
5194 5208 q = NULL;
5195 5209 }
5196 5210 }
5197 5211
5198 5212 /*
5199 5213 * Allocate and build a list of all syncqs in a stream and the syncq(s)
5200 5214 * associated with the "q" parameter. The resulting list is sorted in a
5201 5215 * canonical order and is free of duplicates.
5202 5216 * Assumes the passed queue is a _RD(q).
5203 5217 */
5204 5218 static sqlist_t *
5205 5219 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5206 5220 {
5207 5221 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5208 5222
5209 5223 /*
5210 5224 * start with the current queue/qpair
5211 5225 */
5212 5226 ASSERT(q->q_flag & QREADR);
5213 5227
5214 5228 sqlist_insert(sqlist, q->q_syncq);
5215 5229 sqlist_insert(sqlist, _WR(q)->q_syncq);
5216 5230
5217 5231 sqlist_insertall(sqlist, stp->sd_wrq);
5218 5232 if (do_twist)
5219 5233 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5220 5234
5221 5235 return (sqlist);
5222 5236 }
5223 5237
5224 5238 static sqlist_t *
5225 5239 sqlist_alloc(struct stdata *stp, int kmflag)
5226 5240 {
5227 5241 size_t sqlist_size;
5228 5242 sqlist_t *sqlist;
5229 5243
5230 5244 /*
5231 5245 * Allocate 2 syncql_t's for each pushed module. Note that
5232 5246 * the sqlist_t structure already has 4 syncql_t's built in:
5233 5247 * 2 for the stream head, and 2 for the driver/other stream head.
5234 5248 */
5235 5249 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5236 5250 sizeof (sqlist_t);
5237 5251 if (STRMATED(stp))
5238 5252 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5239 5253 sqlist = kmem_alloc(sqlist_size, kmflag);
5240 5254
5241 5255 sqlist->sqlist_head = NULL;
5242 5256 sqlist->sqlist_size = sqlist_size;
5243 5257 sqlist->sqlist_index = 0;
5244 5258
5245 5259 return (sqlist);
5246 5260 }
5247 5261
5248 5262 /*
5249 5263 * Free the list created by sqlist_alloc()
5250 5264 */
5251 5265 static void
5252 5266 sqlist_free(sqlist_t *sqlist)
5253 5267 {
5254 5268 kmem_free(sqlist, sqlist->sqlist_size);
5255 5269 }
5256 5270
5257 5271 /*
5258 5272 * Prevent any new entries into any syncq in this stream.
5259 5273 * Used by freezestr.
5260 5274 */
5261 5275 void
5262 5276 strblock(queue_t *q)
5263 5277 {
5264 5278 struct stdata *stp;
5265 5279 syncql_t *sql;
5266 5280 sqlist_t *sqlist;
5267 5281
5268 5282 q = _RD(q);
5269 5283
5270 5284 stp = STREAM(q);
5271 5285 ASSERT(stp != NULL);
5272 5286
5273 5287 /*
5274 5288 * Get a sorted list with all the duplicates removed containing
5275 5289 * all the syncqs referenced by this stream.
5276 5290 */
5277 5291 sqlist = sqlist_build(q, stp, B_FALSE);
5278 5292 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5279 5293 blocksq(sql->sql_sq, SQ_FROZEN, -1);
5280 5294 sqlist_free(sqlist);
5281 5295 }
5282 5296
5283 5297 /*
5284 5298 * Release the block on new entries into this stream
5285 5299 */
5286 5300 void
5287 5301 strunblock(queue_t *q)
5288 5302 {
5289 5303 struct stdata *stp;
5290 5304 syncql_t *sql;
5291 5305 sqlist_t *sqlist;
5292 5306 int drain_needed;
5293 5307
5294 5308 q = _RD(q);
5295 5309
5296 5310 /*
5297 5311 * Get a sorted list with all the duplicates removed containing
5298 5312 * all the syncqs referenced by this stream.
5299 5313 * Have to drop the SQ_FROZEN flag on all the syncqs before
5300 5314 * starting to drain them; otherwise the draining might
5301 5315 * cause a freezestr in some module on the stream (which
5302 5316 * would deadlock).
5303 5317 */
5304 5318 stp = STREAM(q);
5305 5319 ASSERT(stp != NULL);
5306 5320 sqlist = sqlist_build(q, stp, B_FALSE);
5307 5321 drain_needed = 0;
5308 5322 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5309 5323 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5310 5324 if (drain_needed) {
5311 5325 for (sql = sqlist->sqlist_head; sql != NULL;
5312 5326 sql = sql->sql_next)
5313 5327 emptysq(sql->sql_sq);
5314 5328 }
5315 5329 sqlist_free(sqlist);
5316 5330 }
5317 5331
5318 5332 #ifdef DEBUG
5319 5333 static int
5320 5334 qprocsareon(queue_t *rq)
5321 5335 {
5322 5336 if (rq->q_next == NULL)
5323 5337 return (0);
5324 5338 return (_WR(rq->q_next)->q_next == _WR(rq));
5325 5339 }
5326 5340
5327 5341 int
5328 5342 qclaimed(queue_t *q)
5329 5343 {
5330 5344 uint_t count;
5331 5345
5332 5346 count = q->q_syncq->sq_count;
5333 5347 SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5334 5348 return (count != 0);
5335 5349 }
5336 5350
5337 5351 /*
5338 5352 * Check if anyone has frozen this stream with freezestr
5339 5353 */
5340 5354 int
5341 5355 frozenstr(queue_t *q)
5342 5356 {
5343 5357 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5344 5358 }
5345 5359 #endif /* DEBUG */
5346 5360
5347 5361 /*
5348 5362 * Enter a queue.
5349 5363 * Obsoleted interface. Should not be used.
5350 5364 */
5351 5365 void
5352 5366 enterq(queue_t *q)
5353 5367 {
5354 5368 entersq(q->q_syncq, SQ_CALLBACK);
5355 5369 }
5356 5370
5357 5371 void
5358 5372 leaveq(queue_t *q)
5359 5373 {
5360 5374 leavesq(q->q_syncq, SQ_CALLBACK);
5361 5375 }
5362 5376
5363 5377 /*
5364 5378 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5365 5379 * to check.
5366 5380 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5367 5381 * calls and the running of open, close and service procedures.
5368 5382 *
5369 5383 * If c_inner bit is set no need to grab sq_putlocks since we don't care
5370 5384 * if other threads have entered or are entering put entry point.
5371 5385 *
5372 5386 * If c_inner bit is set it might have been possible to use
5373 5387 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5374 5388 * open/close path for IP) but since the count may need to be decremented in
5375 5389 * qwait() we wouldn't know which counter to decrement. Currently counter is
5376 5390 * selected by current cpu_seqid and current CPU can change at any moment. XXX
5377 5391 * in the future we might use curthread id bits to select the counter and this
5378 5392 * would stay constant across routine calls.
5379 5393 */
5380 5394 void
5381 5395 entersq(syncq_t *sq, int entrypoint)
5382 5396 {
5383 5397 uint16_t count = 0;
5384 5398 uint16_t flags;
5385 5399 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5386 5400 uint16_t type;
5387 5401 uint_t c_inner = entrypoint & SQ_CI;
5388 5402 uint_t c_outer = entrypoint & SQ_CO;
5389 5403
5390 5404 /*
5391 5405 * Increment ref count to keep closes out of this queue.
5392 5406 */
5393 5407 ASSERT(sq);
5394 5408 ASSERT(c_inner && c_outer);
5395 5409 mutex_enter(SQLOCK(sq));
5396 5410 flags = sq->sq_flags;
5397 5411 type = sq->sq_type;
5398 5412 if (!(type & c_inner)) {
5399 5413 /* Make sure all putcounts now use slowlock. */
5400 5414 count = sq->sq_count;
5401 5415 SQ_PUTLOCKS_ENTER(sq);
5402 5416 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5403 5417 SUM_SQ_PUTCOUNTS(sq, count);
5404 5418 sq->sq_needexcl++;
5405 5419 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5406 5420 waitflags |= SQ_MESSAGES;
5407 5421 }
5408 5422 /*
5409 5423 * Wait until we can enter the inner perimeter.
5410 5424 * If we want exclusive access we wait until sq_count is 0.
5411 5425 * We have to do this before entering the outer perimeter in order
5412 5426 * to preserve put/close message ordering.
5413 5427 */
5414 5428 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5415 5429 sq->sq_flags = flags | SQ_WANTWAKEUP;
5416 5430 if (!(type & c_inner)) {
5417 5431 SQ_PUTLOCKS_EXIT(sq);
5418 5432 }
5419 5433 cv_wait(&sq->sq_wait, SQLOCK(sq));
5420 5434 if (!(type & c_inner)) {
5421 5435 count = sq->sq_count;
5422 5436 SQ_PUTLOCKS_ENTER(sq);
5423 5437 SUM_SQ_PUTCOUNTS(sq, count);
5424 5438 }
5425 5439 flags = sq->sq_flags;
5426 5440 }
5427 5441
5428 5442 if (!(type & c_inner)) {
5429 5443 ASSERT(sq->sq_needexcl > 0);
5430 5444 sq->sq_needexcl--;
5431 5445 if (sq->sq_needexcl == 0) {
5432 5446 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5433 5447 }
5434 5448 }
5435 5449
5436 5450 /* Check if we need to enter the outer perimeter */
5437 5451 if (!(type & c_outer)) {
5438 5452 /*
5439 5453 * We have to enter the outer perimeter exclusively before
5440 5454 * we can increment sq_count to avoid deadlock. This implies
5441 5455 * that we have to re-check sq_flags and sq_count.
5442 5456 *
5443 5457 * is it possible to have c_inner set when c_outer is not set?
5444 5458 */
5445 5459 if (!(type & c_inner)) {
5446 5460 SQ_PUTLOCKS_EXIT(sq);
5447 5461 }
5448 5462 mutex_exit(SQLOCK(sq));
5449 5463 outer_enter(sq->sq_outer, SQ_GOAWAY);
5450 5464 mutex_enter(SQLOCK(sq));
5451 5465 flags = sq->sq_flags;
5452 5466 /*
5453 5467 * there should be no need to recheck sq_putcounts
5454 5468 * because outer_enter() has already waited for them to clear
5455 5469 * after setting SQ_WRITER.
5456 5470 */
5457 5471 count = sq->sq_count;
5458 5472 #ifdef DEBUG
5459 5473 /*
5460 5474 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5461 5475 * of doing an ASSERT internally. Others should do
5462 5476 * something like
5463 5477 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5464 5478 * without the need to #ifdef DEBUG it.
5465 5479 */
5466 5480 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5467 5481 #endif
5468 5482 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5469 5483 (!(type & c_inner) && count != 0)) {
5470 5484 sq->sq_flags = flags | SQ_WANTWAKEUP;
5471 5485 cv_wait(&sq->sq_wait, SQLOCK(sq));
5472 5486 count = sq->sq_count;
5473 5487 flags = sq->sq_flags;
5474 5488 }
5475 5489 }
5476 5490
5477 5491 sq->sq_count++;
5478 5492 ASSERT(sq->sq_count != 0); /* Wraparound */
5479 5493 if (!(type & c_inner)) {
5480 5494 /* Exclusive entry */
5481 5495 ASSERT(sq->sq_count == 1);
5482 5496 sq->sq_flags |= SQ_EXCL;
5483 5497 if (type & c_outer) {
5484 5498 SQ_PUTLOCKS_EXIT(sq);
5485 5499 }
5486 5500 }
5487 5501 mutex_exit(SQLOCK(sq));
5488 5502 }
5489 5503
5490 5504 /*
5491 5505 * Leave a syncq. Announce to framework that closes may proceed.
5492 5506 * c_inner and c_outer specify which concurrency bits to check.
5493 5507 *
5494 5508 * Must never be called from driver or module put entry point.
5495 5509 *
5496 5510 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5497 5511 * sq_putlocks are used.
5498 5512 */
5499 5513 void
5500 5514 leavesq(syncq_t *sq, int entrypoint)
5501 5515 {
5502 5516 uint16_t flags;
5503 5517 uint16_t type;
5504 5518 uint_t c_outer = entrypoint & SQ_CO;
5505 5519 #ifdef DEBUG
5506 5520 uint_t c_inner = entrypoint & SQ_CI;
5507 5521 #endif
5508 5522
5509 5523 /*
5510 5524 * Decrement ref count, drain the syncq if possible, and wake up
5511 5525 * any waiting close.
5512 5526 */
5513 5527 ASSERT(sq);
5514 5528 ASSERT(c_inner && c_outer);
5515 5529 mutex_enter(SQLOCK(sq));
5516 5530 flags = sq->sq_flags;
5517 5531 type = sq->sq_type;
5518 5532 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5519 5533
5520 5534 if (flags & SQ_WANTWAKEUP) {
5521 5535 flags &= ~SQ_WANTWAKEUP;
5522 5536 cv_broadcast(&sq->sq_wait);
5523 5537 }
5524 5538 if (flags & SQ_WANTEXWAKEUP) {
5525 5539 flags &= ~SQ_WANTEXWAKEUP;
5526 5540 cv_broadcast(&sq->sq_exitwait);
5527 5541 }
5528 5542
5529 5543 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5530 5544 /*
5531 5545 * The syncq needs to be drained. "Exit" the syncq
5532 5546 * before calling drain_syncq.
5533 5547 */
5534 5548 ASSERT(sq->sq_count != 0);
5535 5549 sq->sq_count--;
5536 5550 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5537 5551 sq->sq_flags = flags & ~SQ_EXCL;
5538 5552 drain_syncq(sq);
5539 5553 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5540 5554 /* Check if we need to exit the outer perimeter */
5541 5555 /* XXX will this ever be true? */
5542 5556 if (!(type & c_outer))
5543 5557 outer_exit(sq->sq_outer);
5544 5558 return;
5545 5559 }
5546 5560 }
5547 5561 ASSERT(sq->sq_count != 0);
5548 5562 sq->sq_count--;
5549 5563 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5550 5564 sq->sq_flags = flags & ~SQ_EXCL;
5551 5565 mutex_exit(SQLOCK(sq));
5552 5566
5553 5567 /* Check if we need to exit the outer perimeter */
5554 5568 if (!(sq->sq_type & c_outer))
5555 5569 outer_exit(sq->sq_outer);
5556 5570 }
5557 5571
5558 5572 /*
5559 5573 * Prevent q_next from changing in this stream by incrementing sq_count.
5560 5574 *
5561 5575 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5562 5576 * sq_putlocks are used.
5563 5577 */
5564 5578 void
5565 5579 claimq(queue_t *qp)
5566 5580 {
5567 5581 syncq_t *sq = qp->q_syncq;
5568 5582
5569 5583 mutex_enter(SQLOCK(sq));
5570 5584 sq->sq_count++;
5571 5585 ASSERT(sq->sq_count != 0); /* Wraparound */
5572 5586 mutex_exit(SQLOCK(sq));
5573 5587 }
5574 5588
5575 5589 /*
5576 5590 * Undo claimq.
5577 5591 *
5578 5592 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5579 5593 * sq_putlocks are used.
5580 5594 */
5581 5595 void
5582 5596 releaseq(queue_t *qp)
5583 5597 {
5584 5598 syncq_t *sq = qp->q_syncq;
5585 5599 uint16_t flags;
5586 5600
5587 5601 mutex_enter(SQLOCK(sq));
5588 5602 ASSERT(sq->sq_count > 0);
5589 5603 sq->sq_count--;
5590 5604
5591 5605 flags = sq->sq_flags;
5592 5606 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5593 5607 if (flags & SQ_WANTWAKEUP) {
5594 5608 flags &= ~SQ_WANTWAKEUP;
5595 5609 cv_broadcast(&sq->sq_wait);
5596 5610 }
5597 5611 sq->sq_flags = flags;
5598 5612 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5599 5613 /*
5600 5614 * To prevent potential recursive invocation of
5601 5615 * drain_syncq we do not call drain_syncq if count is
5602 5616 * non-zero.
5603 5617 */
5604 5618 if (sq->sq_count == 0) {
5605 5619 drain_syncq(sq);
5606 5620 return;
5607 5621 } else
5608 5622 sqenable(sq);
5609 5623 }
5610 5624 }
5611 5625 mutex_exit(SQLOCK(sq));
5612 5626 }
5613 5627
5614 5628 /*
5615 5629 * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5616 5630 */
5617 5631 void
5618 5632 claimstr(queue_t *qp)
5619 5633 {
5620 5634 struct stdata *stp = STREAM(qp);
5621 5635
5622 5636 mutex_enter(&stp->sd_reflock);
5623 5637 stp->sd_refcnt++;
5624 5638 ASSERT(stp->sd_refcnt != 0); /* Wraparound */
5625 5639 mutex_exit(&stp->sd_reflock);
5626 5640 }
5627 5641
5628 5642 /*
5629 5643 * Undo claimstr.
5630 5644 */
5631 5645 void
5632 5646 releasestr(queue_t *qp)
5633 5647 {
5634 5648 struct stdata *stp = STREAM(qp);
5635 5649
5636 5650 mutex_enter(&stp->sd_reflock);
5637 5651 ASSERT(stp->sd_refcnt != 0);
5638 5652 if (--stp->sd_refcnt == 0)
5639 5653 cv_broadcast(&stp->sd_refmonitor);
5640 5654 mutex_exit(&stp->sd_reflock);
5641 5655 }
5642 5656
5643 5657 static syncq_t *
5644 5658 new_syncq(void)
5645 5659 {
5646 5660 return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5647 5661 }
5648 5662
5649 5663 static void
5650 5664 free_syncq(syncq_t *sq)
5651 5665 {
5652 5666 ASSERT(sq->sq_head == NULL);
5653 5667 ASSERT(sq->sq_outer == NULL);
5654 5668 ASSERT(sq->sq_callbpend == NULL);
5655 5669 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5656 5670 (sq->sq_onext == sq && sq->sq_oprev == sq));
5657 5671
5658 5672 if (sq->sq_ciputctrl != NULL) {
5659 5673 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5660 5674 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5661 5675 sq->sq_nciputctrl, 0);
5662 5676 ASSERT(ciputctrl_cache != NULL);
5663 5677 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5664 5678 }
5665 5679
5666 5680 sq->sq_tail = NULL;
5667 5681 sq->sq_evhead = NULL;
5668 5682 sq->sq_evtail = NULL;
5669 5683 sq->sq_ciputctrl = NULL;
5670 5684 sq->sq_nciputctrl = 0;
5671 5685 sq->sq_count = 0;
5672 5686 sq->sq_rmqcount = 0;
5673 5687 sq->sq_callbflags = 0;
5674 5688 sq->sq_cancelid = 0;
5675 5689 sq->sq_next = NULL;
5676 5690 sq->sq_needexcl = 0;
5677 5691 sq->sq_svcflags = 0;
5678 5692 sq->sq_nqueues = 0;
5679 5693 sq->sq_pri = 0;
5680 5694 sq->sq_onext = NULL;
5681 5695 sq->sq_oprev = NULL;
5682 5696 sq->sq_flags = 0;
5683 5697 sq->sq_type = 0;
5684 5698 sq->sq_servcount = 0;
5685 5699
5686 5700 kmem_cache_free(syncq_cache, sq);
5687 5701 }
5688 5702
5689 5703 /* Outer perimeter code */
5690 5704
5691 5705 /*
5692 5706 * The outer syncq uses the fields and flags in the syncq slightly
5693 5707 * differently from the inner syncqs.
5694 5708 * sq_count Incremented when there are pending or running
5695 5709 * writers at the outer perimeter to prevent the set of
5696 5710 * inner syncqs that belong to the outer perimeter from
5697 5711 * changing.
5698 5712 * sq_head/tail List of deferred qwriter(OUTER) operations.
5699 5713 *
5700 5714 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while
5701 5715 * inner syncqs are added to or removed from the
5702 5716 * outer perimeter.
5703 5717 * SQ_QUEUED sq_head/tail has messages or events queued.
5704 5718 *
5705 5719 * SQ_WRITER A thread is currently traversing all the inner syncqs
5706 5720 * setting the SQ_WRITER flag.
5707 5721 */
5708 5722
5709 5723 /*
5710 5724 * Get write access at the outer perimeter.
5711 5725 * Note that read access is done by entersq, putnext, and put by simply
5712 5726 * incrementing sq_count in the inner syncq.
5713 5727 *
5714 5728 * Waits until "flags" is no longer set in the outer to prevent multiple
5715 5729 * threads from having write access at the same time. SQ_WRITER has to be part
5716 5730 * of "flags".
5717 5731 *
5718 5732 * Increases sq_count on the outer syncq to keep away outer_insert/remove
5719 5733 * until the outer_exit is finished.
5720 5734 *
5721 5735 * outer_enter is vulnerable to starvation since it does not prevent new
5722 5736 * threads from entering the inner syncqs while it is waiting for sq_count to
5723 5737 * go to zero.
5724 5738 */
5725 5739 void
5726 5740 outer_enter(syncq_t *outer, uint16_t flags)
5727 5741 {
5728 5742 syncq_t *sq;
5729 5743 int wait_needed;
5730 5744 uint16_t count;
5731 5745
5732 5746 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5733 5747 outer->sq_oprev != NULL);
5734 5748 ASSERT(flags & SQ_WRITER);
5735 5749
5736 5750 retry:
5737 5751 mutex_enter(SQLOCK(outer));
5738 5752 while (outer->sq_flags & flags) {
5739 5753 outer->sq_flags |= SQ_WANTWAKEUP;
5740 5754 cv_wait(&outer->sq_wait, SQLOCK(outer));
5741 5755 }
5742 5756
5743 5757 ASSERT(!(outer->sq_flags & SQ_WRITER));
5744 5758 outer->sq_flags |= SQ_WRITER;
5745 5759 outer->sq_count++;
5746 5760 ASSERT(outer->sq_count != 0); /* wraparound */
5747 5761 wait_needed = 0;
5748 5762 /*
5749 5763 * Set SQ_WRITER on all the inner syncqs while holding
5750 5764 * the SQLOCK on the outer syncq. This ensures that the changing
5751 5765 * of SQ_WRITER is atomic under the outer SQLOCK.
5752 5766 */
5753 5767 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5754 5768 mutex_enter(SQLOCK(sq));
5755 5769 count = sq->sq_count;
5756 5770 SQ_PUTLOCKS_ENTER(sq);
5757 5771 sq->sq_flags |= SQ_WRITER;
5758 5772 SUM_SQ_PUTCOUNTS(sq, count);
5759 5773 if (count != 0)
5760 5774 wait_needed = 1;
5761 5775 SQ_PUTLOCKS_EXIT(sq);
5762 5776 mutex_exit(SQLOCK(sq));
5763 5777 }
5764 5778 mutex_exit(SQLOCK(outer));
5765 5779
5766 5780 /*
5767 5781 * Get everybody out of the syncqs sequentially.
5768 5782 * Note that we don't actually need to acquire the PUTLOCKS, since
5769 5783 * we have already cleared the fastbit, and set QWRITER. By
5770 5784 * definition, the count can not increase since putnext will
5771 5785 * take the slowlock path (and the purpose of acquiring the
5772 5786 * putlocks was to make sure it didn't increase while we were
5773 5787 * waiting).
5774 5788 *
5775 5789 * Note that we still acquire the PUTLOCKS to be safe.
5776 5790 */
5777 5791 if (wait_needed) {
5778 5792 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5779 5793 mutex_enter(SQLOCK(sq));
5780 5794 count = sq->sq_count;
5781 5795 SQ_PUTLOCKS_ENTER(sq);
5782 5796 SUM_SQ_PUTCOUNTS(sq, count);
5783 5797 while (count != 0) {
5784 5798 sq->sq_flags |= SQ_WANTWAKEUP;
5785 5799 SQ_PUTLOCKS_EXIT(sq);
5786 5800 cv_wait(&sq->sq_wait, SQLOCK(sq));
5787 5801 count = sq->sq_count;
5788 5802 SQ_PUTLOCKS_ENTER(sq);
5789 5803 SUM_SQ_PUTCOUNTS(sq, count);
5790 5804 }
5791 5805 SQ_PUTLOCKS_EXIT(sq);
5792 5806 mutex_exit(SQLOCK(sq));
5793 5807 }
5794 5808 /*
5795 5809 * Verify that none of the flags got set while we
5796 5810 * were waiting for the sq_counts to drop.
5797 5811 * If this happens we exit and retry entering the
5798 5812 * outer perimeter.
5799 5813 */
5800 5814 mutex_enter(SQLOCK(outer));
5801 5815 if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5802 5816 mutex_exit(SQLOCK(outer));
5803 5817 outer_exit(outer);
5804 5818 goto retry;
5805 5819 }
5806 5820 mutex_exit(SQLOCK(outer));
5807 5821 }
5808 5822 }
5809 5823
5810 5824 /*
5811 5825 * Drop the write access at the outer perimeter.
5812 5826 * Read access is dropped implicitly (by putnext, put, and leavesq) by
5813 5827 * decrementing sq_count.
5814 5828 */
5815 5829 void
5816 5830 outer_exit(syncq_t *outer)
5817 5831 {
5818 5832 syncq_t *sq;
5819 5833 int drain_needed;
5820 5834 uint16_t flags;
5821 5835
5822 5836 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5823 5837 outer->sq_oprev != NULL);
5824 5838 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5825 5839
5826 5840 /*
5827 5841 * Atomically (from the perspective of threads calling become_writer)
5828 5842 * drop the write access at the outer perimeter by holding
5829 5843 * SQLOCK(outer) across all the dropsq calls and the resetting of
5830 5844 * SQ_WRITER.
5831 5845 * This defines a locking order between the outer perimeter
5832 5846 * SQLOCK and the inner perimeter SQLOCKs.
5833 5847 */
5834 5848 mutex_enter(SQLOCK(outer));
5835 5849 flags = outer->sq_flags;
5836 5850 ASSERT(outer->sq_flags & SQ_WRITER);
5837 5851 if (flags & SQ_QUEUED) {
5838 5852 write_now(outer);
5839 5853 flags = outer->sq_flags;
5840 5854 }
5841 5855
5842 5856 /*
5843 5857 * sq_onext is stable since sq_count has not yet been decreased.
5844 5858 * Reset the SQ_WRITER flags in all syncqs.
5845 5859 * After dropping SQ_WRITER on the outer syncq we empty all the
5846 5860 * inner syncqs.
5847 5861 */
5848 5862 drain_needed = 0;
5849 5863 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5850 5864 drain_needed += dropsq(sq, SQ_WRITER);
5851 5865 ASSERT(!(outer->sq_flags & SQ_QUEUED));
5852 5866 flags &= ~SQ_WRITER;
5853 5867 if (drain_needed) {
5854 5868 outer->sq_flags = flags;
5855 5869 mutex_exit(SQLOCK(outer));
5856 5870 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5857 5871 emptysq(sq);
5858 5872 mutex_enter(SQLOCK(outer));
5859 5873 flags = outer->sq_flags;
5860 5874 }
5861 5875 if (flags & SQ_WANTWAKEUP) {
5862 5876 flags &= ~SQ_WANTWAKEUP;
5863 5877 cv_broadcast(&outer->sq_wait);
5864 5878 }
5865 5879 outer->sq_flags = flags;
5866 5880 ASSERT(outer->sq_count > 0);
5867 5881 outer->sq_count--;
5868 5882 mutex_exit(SQLOCK(outer));
5869 5883 }
5870 5884
5871 5885 /*
5872 5886 * Add another syncq to an outer perimeter.
5873 5887 * Block out all other access to the outer perimeter while it is being
5874 5888 * changed using blocksq.
5875 5889 * Assumes that the caller has *not* done an outer_enter.
5876 5890 *
5877 5891 * Vulnerable to starvation in blocksq.
5878 5892 */
5879 5893 static void
5880 5894 outer_insert(syncq_t *outer, syncq_t *sq)
5881 5895 {
5882 5896 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5883 5897 outer->sq_oprev != NULL);
5884 5898 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5885 5899 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */
5886 5900
5887 5901 /* Get exclusive access to the outer perimeter list */
5888 5902 blocksq(outer, SQ_BLOCKED, 0);
5889 5903 ASSERT(outer->sq_flags & SQ_BLOCKED);
5890 5904 ASSERT(!(outer->sq_flags & SQ_WRITER));
5891 5905
5892 5906 mutex_enter(SQLOCK(sq));
5893 5907 sq->sq_outer = outer;
5894 5908 outer->sq_onext->sq_oprev = sq;
5895 5909 sq->sq_onext = outer->sq_onext;
5896 5910 outer->sq_onext = sq;
5897 5911 sq->sq_oprev = outer;
5898 5912 mutex_exit(SQLOCK(sq));
5899 5913 unblocksq(outer, SQ_BLOCKED, 1);
5900 5914 }
5901 5915
5902 5916 /*
5903 5917 * Remove a syncq from an outer perimeter.
5904 5918 * Block out all other access to the outer perimeter while it is being
5905 5919 * changed using blocksq.
5906 5920 * Assumes that the caller has *not* done an outer_enter.
5907 5921 *
5908 5922 * Vulnerable to starvation in blocksq.
5909 5923 */
5910 5924 static void
5911 5925 outer_remove(syncq_t *outer, syncq_t *sq)
5912 5926 {
5913 5927 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5914 5928 outer->sq_oprev != NULL);
5915 5929 ASSERT(sq->sq_outer == outer);
5916 5930
5917 5931 /* Get exclusive access to the outer perimeter list */
5918 5932 blocksq(outer, SQ_BLOCKED, 0);
5919 5933 ASSERT(outer->sq_flags & SQ_BLOCKED);
5920 5934 ASSERT(!(outer->sq_flags & SQ_WRITER));
5921 5935
5922 5936 mutex_enter(SQLOCK(sq));
5923 5937 sq->sq_outer = NULL;
5924 5938 sq->sq_onext->sq_oprev = sq->sq_oprev;
5925 5939 sq->sq_oprev->sq_onext = sq->sq_onext;
5926 5940 sq->sq_oprev = sq->sq_onext = NULL;
5927 5941 mutex_exit(SQLOCK(sq));
5928 5942 unblocksq(outer, SQ_BLOCKED, 1);
5929 5943 }
5930 5944
5931 5945 /*
5932 5946 * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5933 5947 * If this is the first callback for this outer perimeter then add
5934 5948 * this outer perimeter to the list of outer perimeters that
5935 5949 * the qwriter_outer_thread will process.
5936 5950 *
5937 5951 * Increments sq_count in the outer syncq to prevent the membership
5938 5952 * of the outer perimeter (in terms of inner syncqs) to change while
5939 5953 * the callback is pending.
5940 5954 */
5941 5955 static void
5942 5956 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5943 5957 {
5944 5958 ASSERT(MUTEX_HELD(SQLOCK(outer)));
5945 5959
5946 5960 mp->b_prev = (mblk_t *)func;
5947 5961 mp->b_queue = q;
5948 5962 mp->b_next = NULL;
5949 5963 outer->sq_count++; /* Decremented when dequeued */
5950 5964 ASSERT(outer->sq_count != 0); /* Wraparound */
5951 5965 if (outer->sq_evhead == NULL) {
5952 5966 /* First message. */
5953 5967 outer->sq_evhead = outer->sq_evtail = mp;
5954 5968 outer->sq_flags |= SQ_EVENTS;
5955 5969 mutex_exit(SQLOCK(outer));
5956 5970 STRSTAT(qwr_outer);
5957 5971 (void) taskq_dispatch(streams_taskq,
5958 5972 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5959 5973 } else {
5960 5974 ASSERT(outer->sq_flags & SQ_EVENTS);
5961 5975 outer->sq_evtail->b_next = mp;
5962 5976 outer->sq_evtail = mp;
5963 5977 mutex_exit(SQLOCK(outer));
5964 5978 }
5965 5979 }
5966 5980
5967 5981 /*
5968 5982 * Try and upgrade to write access at the outer perimeter. If this can
5969 5983 * not be done without blocking then queue the callback to be done
5970 5984 * by the qwriter_outer_thread.
5971 5985 *
5972 5986 * This routine can only be called from put or service procedures plus
5973 5987 * asynchronous callback routines that have properly entered the queue (with
5974 5988 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5975 5989 * associated with q.
5976 5990 */
5977 5991 void
5978 5992 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
5979 5993 {
5980 5994 syncq_t *osq, *sq, *outer;
5981 5995 int failed;
5982 5996 uint16_t flags;
5983 5997
5984 5998 osq = q->q_syncq;
5985 5999 outer = osq->sq_outer;
5986 6000 if (outer == NULL)
5987 6001 panic("qwriter(PERIM_OUTER): no outer perimeter");
5988 6002 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5989 6003 outer->sq_oprev != NULL);
5990 6004
5991 6005 mutex_enter(SQLOCK(outer));
5992 6006 flags = outer->sq_flags;
5993 6007 /*
5994 6008 * If some thread is traversing sq_next, or if we are blocked by
5995 6009 * outer_insert or outer_remove, or if the we already have queued
5996 6010 * callbacks, then queue this callback for later processing.
5997 6011 *
5998 6012 * Also queue the qwriter for an interrupt thread in order
5999 6013 * to reduce the time spent running at high IPL.
6000 6014 * to identify there are events.
6001 6015 */
6002 6016 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6003 6017 /*
6004 6018 * Queue the become_writer request.
6005 6019 * The queueing is atomic under SQLOCK(outer) in order
6006 6020 * to synchronize with outer_exit.
6007 6021 * queue_writer will drop the outer SQLOCK
6008 6022 */
6009 6023 if (flags & SQ_BLOCKED) {
6010 6024 /* Must set SQ_WRITER on inner perimeter */
6011 6025 mutex_enter(SQLOCK(osq));
6012 6026 osq->sq_flags |= SQ_WRITER;
6013 6027 mutex_exit(SQLOCK(osq));
6014 6028 } else {
6015 6029 if (!(flags & SQ_WRITER)) {
6016 6030 /*
6017 6031 * The outer could have been SQ_BLOCKED thus
6018 6032 * SQ_WRITER might not be set on the inner.
6019 6033 */
6020 6034 mutex_enter(SQLOCK(osq));
6021 6035 osq->sq_flags |= SQ_WRITER;
6022 6036 mutex_exit(SQLOCK(osq));
6023 6037 }
6024 6038 ASSERT(osq->sq_flags & SQ_WRITER);
6025 6039 }
6026 6040 queue_writer(outer, func, q, mp);
6027 6041 return;
6028 6042 }
6029 6043 /*
6030 6044 * We are half-way to exclusive access to the outer perimeter.
6031 6045 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6032 6046 * while the inner syncqs are traversed.
6033 6047 */
6034 6048 outer->sq_count++;
6035 6049 ASSERT(outer->sq_count != 0); /* wraparound */
6036 6050 flags |= SQ_WRITER;
6037 6051 /*
6038 6052 * Check if we can run the function immediately. Mark all
6039 6053 * syncqs with the writer flag to prevent new entries into
6040 6054 * put and service procedures.
6041 6055 *
6042 6056 * Set SQ_WRITER on all the inner syncqs while holding
6043 6057 * the SQLOCK on the outer syncq. This ensures that the changing
6044 6058 * of SQ_WRITER is atomic under the outer SQLOCK.
6045 6059 */
6046 6060 failed = 0;
6047 6061 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6048 6062 uint16_t count;
6049 6063 uint_t maxcnt = (sq == osq) ? 1 : 0;
6050 6064
6051 6065 mutex_enter(SQLOCK(sq));
6052 6066 count = sq->sq_count;
6053 6067 SQ_PUTLOCKS_ENTER(sq);
6054 6068 SUM_SQ_PUTCOUNTS(sq, count);
6055 6069 if (sq->sq_count > maxcnt)
6056 6070 failed = 1;
6057 6071 sq->sq_flags |= SQ_WRITER;
6058 6072 SQ_PUTLOCKS_EXIT(sq);
6059 6073 mutex_exit(SQLOCK(sq));
6060 6074 }
6061 6075 if (failed) {
6062 6076 /*
6063 6077 * Some other thread has a read claim on the outer perimeter.
6064 6078 * Queue the callback for deferred processing.
6065 6079 *
6066 6080 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6067 6081 * so that other qwriter(OUTER) calls will queue their
6068 6082 * callbacks as well. queue_writer increments sq_count so we
6069 6083 * decrement to compensate for the our increment.
6070 6084 *
6071 6085 * Dropping SQ_WRITER enables the writer thread to work
6072 6086 * on this outer perimeter.
6073 6087 */
6074 6088 outer->sq_flags = flags;
6075 6089 queue_writer(outer, func, q, mp);
6076 6090 /* queue_writer dropper the lock */
6077 6091 mutex_enter(SQLOCK(outer));
6078 6092 ASSERT(outer->sq_count > 0);
6079 6093 outer->sq_count--;
6080 6094 ASSERT(outer->sq_flags & SQ_WRITER);
6081 6095 flags = outer->sq_flags;
6082 6096 flags &= ~SQ_WRITER;
6083 6097 if (flags & SQ_WANTWAKEUP) {
6084 6098 flags &= ~SQ_WANTWAKEUP;
6085 6099 cv_broadcast(&outer->sq_wait);
6086 6100 }
6087 6101 outer->sq_flags = flags;
6088 6102 mutex_exit(SQLOCK(outer));
6089 6103 return;
6090 6104 } else {
6091 6105 outer->sq_flags = flags;
6092 6106 mutex_exit(SQLOCK(outer));
6093 6107 }
6094 6108
6095 6109 /* Can run it immediately */
6096 6110 (*func)(q, mp);
6097 6111
6098 6112 outer_exit(outer);
6099 6113 }
6100 6114
6101 6115 /*
6102 6116 * Dequeue all writer callbacks from the outer perimeter and run them.
6103 6117 */
6104 6118 static void
6105 6119 write_now(syncq_t *outer)
6106 6120 {
6107 6121 mblk_t *mp;
6108 6122 queue_t *q;
6109 6123 void (*func)();
6110 6124
6111 6125 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6112 6126 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6113 6127 outer->sq_oprev != NULL);
6114 6128 while ((mp = outer->sq_evhead) != NULL) {
6115 6129 /*
6116 6130 * queues cannot be placed on the queuelist on the outer
6117 6131 * perimeter.
6118 6132 */
6119 6133 ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6120 6134 ASSERT((outer->sq_flags & SQ_EVENTS));
6121 6135
6122 6136 outer->sq_evhead = mp->b_next;
6123 6137 if (outer->sq_evhead == NULL) {
6124 6138 outer->sq_evtail = NULL;
6125 6139 outer->sq_flags &= ~SQ_EVENTS;
6126 6140 }
6127 6141 ASSERT(outer->sq_count != 0);
6128 6142 outer->sq_count--; /* Incremented when enqueued. */
6129 6143 mutex_exit(SQLOCK(outer));
6130 6144 /*
6131 6145 * Drop the message if the queue is closing.
6132 6146 * Make sure that the queue is "claimed" when the callback
6133 6147 * is run in order to satisfy various ASSERTs.
6134 6148 */
6135 6149 q = mp->b_queue;
6136 6150 func = (void (*)())mp->b_prev;
6137 6151 ASSERT(func != NULL);
6138 6152 mp->b_next = mp->b_prev = NULL;
6139 6153 if (q->q_flag & QWCLOSE) {
6140 6154 freemsg(mp);
6141 6155 } else {
6142 6156 claimq(q);
6143 6157 (*func)(q, mp);
6144 6158 releaseq(q);
6145 6159 }
6146 6160 mutex_enter(SQLOCK(outer));
6147 6161 }
6148 6162 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6149 6163 }
6150 6164
6151 6165 /*
6152 6166 * The list of messages on the inner syncq is effectively hashed
6153 6167 * by destination queue. These destination queues are doubly
6154 6168 * linked lists (hopefully) in priority order. Messages are then
6155 6169 * put on the queue referenced by the q_sqhead/q_sqtail elements.
6156 6170 * Additional messages are linked together by the b_next/b_prev
6157 6171 * elements in the mblk, with (similar to putq()) the first message
6158 6172 * having a NULL b_prev and the last message having a NULL b_next.
6159 6173 *
6160 6174 * Events, such as qwriter callbacks, are put onto a list in FIFO
6161 6175 * order referenced by sq_evhead, and sq_evtail. This is a singly
6162 6176 * linked list, and messages here MUST be processed in the order queued.
6163 6177 */
6164 6178
6165 6179 /*
6166 6180 * Run the events on the syncq event list (sq_evhead).
6167 6181 * Assumes there is only one claim on the syncq, it is
6168 6182 * already exclusive (SQ_EXCL set), and the SQLOCK held.
6169 6183 * Messages here are processed in order, with the SQ_EXCL bit
6170 6184 * held all the way through till the last message is processed.
6171 6185 */
6172 6186 void
6173 6187 sq_run_events(syncq_t *sq)
6174 6188 {
6175 6189 mblk_t *bp;
6176 6190 queue_t *qp;
6177 6191 uint16_t flags = sq->sq_flags;
6178 6192 void (*func)();
6179 6193
6180 6194 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6181 6195 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6182 6196 sq->sq_oprev == NULL) ||
6183 6197 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6184 6198 sq->sq_oprev != NULL));
6185 6199
6186 6200 ASSERT(flags & SQ_EXCL);
6187 6201 ASSERT(sq->sq_count == 1);
6188 6202
6189 6203 /*
6190 6204 * We need to process all of the events on this list. It
6191 6205 * is possible that new events will be added while we are
6192 6206 * away processing a callback, so on every loop, we start
6193 6207 * back at the beginning of the list.
6194 6208 */
6195 6209 /*
6196 6210 * We have to reaccess sq_evhead since there is a
6197 6211 * possibility of a new entry while we were running
6198 6212 * the callback.
6199 6213 */
6200 6214 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6201 6215 ASSERT(bp->b_queue->q_syncq == sq);
6202 6216 ASSERT(sq->sq_flags & SQ_EVENTS);
6203 6217
6204 6218 qp = bp->b_queue;
6205 6219 func = (void (*)())bp->b_prev;
6206 6220 ASSERT(func != NULL);
6207 6221
6208 6222 /*
6209 6223 * Messages from the event queue must be taken off in
6210 6224 * FIFO order.
6211 6225 */
6212 6226 ASSERT(sq->sq_evhead == bp);
6213 6227 sq->sq_evhead = bp->b_next;
6214 6228
6215 6229 if (bp->b_next == NULL) {
6216 6230 /* Deleting last */
6217 6231 ASSERT(sq->sq_evtail == bp);
6218 6232 sq->sq_evtail = NULL;
6219 6233 sq->sq_flags &= ~SQ_EVENTS;
6220 6234 }
6221 6235 bp->b_prev = bp->b_next = NULL;
6222 6236 ASSERT(bp->b_datap->db_ref != 0);
6223 6237
6224 6238 mutex_exit(SQLOCK(sq));
6225 6239
6226 6240 (*func)(qp, bp);
6227 6241
6228 6242 mutex_enter(SQLOCK(sq));
6229 6243 /*
6230 6244 * re-read the flags, since they could have changed.
6231 6245 */
6232 6246 flags = sq->sq_flags;
6233 6247 ASSERT(flags & SQ_EXCL);
6234 6248 }
6235 6249 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6236 6250 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6237 6251
6238 6252 if (flags & SQ_WANTWAKEUP) {
6239 6253 flags &= ~SQ_WANTWAKEUP;
6240 6254 cv_broadcast(&sq->sq_wait);
6241 6255 }
6242 6256 if (flags & SQ_WANTEXWAKEUP) {
6243 6257 flags &= ~SQ_WANTEXWAKEUP;
6244 6258 cv_broadcast(&sq->sq_exitwait);
6245 6259 }
6246 6260 sq->sq_flags = flags;
6247 6261 }
6248 6262
6249 6263 /*
6250 6264 * Put messages on the event list.
6251 6265 * If we can go exclusive now, do so and process the event list, otherwise
6252 6266 * let the last claim service this list (or wake the sqthread).
6253 6267 * This procedure assumes SQLOCK is held. To run the event list, it
6254 6268 * must be called with no claims.
6255 6269 */
6256 6270 static void
6257 6271 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6258 6272 {
6259 6273 uint16_t count;
6260 6274
6261 6275 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6262 6276 ASSERT(func != NULL);
6263 6277
6264 6278 /*
6265 6279 * This is a callback. Add it to the list of callbacks
6266 6280 * and see about upgrading.
6267 6281 */
6268 6282 mp->b_prev = (mblk_t *)func;
6269 6283 mp->b_queue = q;
6270 6284 mp->b_next = NULL;
6271 6285 if (sq->sq_evhead == NULL) {
6272 6286 sq->sq_evhead = sq->sq_evtail = mp;
6273 6287 sq->sq_flags |= SQ_EVENTS;
6274 6288 } else {
6275 6289 ASSERT(sq->sq_evtail != NULL);
6276 6290 ASSERT(sq->sq_evtail->b_next == NULL);
6277 6291 ASSERT(sq->sq_flags & SQ_EVENTS);
6278 6292 sq->sq_evtail->b_next = mp;
6279 6293 sq->sq_evtail = mp;
6280 6294 }
6281 6295 /*
6282 6296 * We have set SQ_EVENTS, so threads will have to
6283 6297 * unwind out of the perimeter, and new entries will
6284 6298 * not grab a putlock. But we still need to know
6285 6299 * how many threads have already made a claim to the
6286 6300 * syncq, so grab the putlocks, and sum the counts.
6287 6301 * If there are no claims on the syncq, we can upgrade
6288 6302 * to exclusive, and run the event list.
6289 6303 * NOTE: We hold the SQLOCK, so we can just grab the
6290 6304 * putlocks.
6291 6305 */
6292 6306 count = sq->sq_count;
6293 6307 SQ_PUTLOCKS_ENTER(sq);
6294 6308 SUM_SQ_PUTCOUNTS(sq, count);
6295 6309 /*
6296 6310 * We have no claim, so we need to check if there
6297 6311 * are no others, then we can upgrade.
6298 6312 */
6299 6313 /*
6300 6314 * There are currently no claims on
6301 6315 * the syncq by this thread (at least on this entry). The thread who has
6302 6316 * the claim should drain syncq.
6303 6317 */
6304 6318 if (count > 0) {
6305 6319 /*
6306 6320 * Can't upgrade - other threads inside.
6307 6321 */
6308 6322 SQ_PUTLOCKS_EXIT(sq);
6309 6323 mutex_exit(SQLOCK(sq));
6310 6324 return;
6311 6325 }
6312 6326 /*
6313 6327 * Need to set SQ_EXCL and make a claim on the syncq.
6314 6328 */
6315 6329 ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6316 6330 sq->sq_flags |= SQ_EXCL;
6317 6331 ASSERT(sq->sq_count == 0);
6318 6332 sq->sq_count++;
6319 6333 SQ_PUTLOCKS_EXIT(sq);
6320 6334
6321 6335 /* Process the events list */
6322 6336 sq_run_events(sq);
6323 6337
6324 6338 /*
6325 6339 * Release our claim...
6326 6340 */
6327 6341 sq->sq_count--;
6328 6342
6329 6343 /*
6330 6344 * And release SQ_EXCL.
6331 6345 * We don't need to acquire the putlocks to release
6332 6346 * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6333 6347 */
6334 6348 sq->sq_flags &= ~SQ_EXCL;
6335 6349
6336 6350 /*
6337 6351 * sq_run_events should have released SQ_EXCL
6338 6352 */
6339 6353 ASSERT(!(sq->sq_flags & SQ_EXCL));
6340 6354
6341 6355 /*
6342 6356 * If anything happened while we were running the
6343 6357 * events (or was there before), we need to process
6344 6358 * them now. We shouldn't be exclusive sine we
6345 6359 * released the perimeter above (plus, we asserted
6346 6360 * for it).
6347 6361 */
6348 6362 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6349 6363 drain_syncq(sq);
6350 6364 else
6351 6365 mutex_exit(SQLOCK(sq));
6352 6366 }
6353 6367
6354 6368 /*
6355 6369 * Perform delayed processing. The caller has to make sure that it is safe
6356 6370 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6357 6371 * set).
6358 6372 *
6359 6373 * Assume that the caller has NO claims on the syncq. However, a claim
6360 6374 * on the syncq does not indicate that a thread is draining the syncq.
6361 6375 * There may be more claims on the syncq than there are threads draining
6362 6376 * (i.e. #_threads_draining <= sq_count)
6363 6377 *
6364 6378 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6365 6379 * in order to preserve qwriter(OUTER) ordering constraints.
6366 6380 *
6367 6381 * sq_putcount only needs to be checked when dispatching the queued
6368 6382 * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6369 6383 */
6370 6384 void
6371 6385 drain_syncq(syncq_t *sq)
6372 6386 {
6373 6387 queue_t *qp;
6374 6388 uint16_t count;
6375 6389 uint16_t type = sq->sq_type;
6376 6390 uint16_t flags = sq->sq_flags;
6377 6391 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE;
6378 6392
6379 6393 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6380 6394 "drain_syncq start:%p", sq);
6381 6395 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6382 6396 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6383 6397 sq->sq_oprev == NULL) ||
6384 6398 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6385 6399 sq->sq_oprev != NULL));
6386 6400
6387 6401 /*
6388 6402 * Drop SQ_SERVICE flag.
6389 6403 */
6390 6404 if (bg_service)
6391 6405 sq->sq_svcflags &= ~SQ_SERVICE;
6392 6406
6393 6407 /*
6394 6408 * If SQ_EXCL is set, someone else is processing this syncq - let them
6395 6409 * finish the job.
6396 6410 */
6397 6411 if (flags & SQ_EXCL) {
6398 6412 if (bg_service) {
6399 6413 ASSERT(sq->sq_servcount != 0);
6400 6414 sq->sq_servcount--;
6401 6415 }
6402 6416 mutex_exit(SQLOCK(sq));
6403 6417 return;
6404 6418 }
6405 6419
6406 6420 /*
6407 6421 * This routine can be called by a background thread if
6408 6422 * it was scheduled by a hi-priority thread. SO, if there are
6409 6423 * NOT messages queued, return (remember, we have the SQLOCK,
6410 6424 * and it cannot change until we release it). Wakeup any waiters also.
6411 6425 */
6412 6426 if (!(flags & SQ_QUEUED)) {
6413 6427 if (flags & SQ_WANTWAKEUP) {
6414 6428 flags &= ~SQ_WANTWAKEUP;
6415 6429 cv_broadcast(&sq->sq_wait);
6416 6430 }
6417 6431 if (flags & SQ_WANTEXWAKEUP) {
6418 6432 flags &= ~SQ_WANTEXWAKEUP;
6419 6433 cv_broadcast(&sq->sq_exitwait);
6420 6434 }
6421 6435 sq->sq_flags = flags;
6422 6436 if (bg_service) {
6423 6437 ASSERT(sq->sq_servcount != 0);
6424 6438 sq->sq_servcount--;
6425 6439 }
6426 6440 mutex_exit(SQLOCK(sq));
6427 6441 return;
6428 6442 }
6429 6443
6430 6444 /*
6431 6445 * If this is not a concurrent put perimeter, we need to
6432 6446 * become exclusive to drain. Also, if not CIPUT, we would
6433 6447 * not have acquired a putlock, so we don't need to check
6434 6448 * the putcounts. If not entering with a claim, we test
6435 6449 * for sq_count == 0.
6436 6450 */
6437 6451 type = sq->sq_type;
6438 6452 if (!(type & SQ_CIPUT)) {
6439 6453 if (sq->sq_count > 1) {
6440 6454 if (bg_service) {
6441 6455 ASSERT(sq->sq_servcount != 0);
6442 6456 sq->sq_servcount--;
6443 6457 }
6444 6458 mutex_exit(SQLOCK(sq));
6445 6459 return;
6446 6460 }
6447 6461 sq->sq_flags |= SQ_EXCL;
6448 6462 }
6449 6463
6450 6464 /*
6451 6465 * This is where we make a claim to the syncq.
6452 6466 * This can either be done by incrementing a putlock, or
6453 6467 * the sq_count. But since we already have the SQLOCK
6454 6468 * here, we just bump the sq_count.
6455 6469 *
6456 6470 * Note that after we make a claim, we need to let the code
6457 6471 * fall through to the end of this routine to clean itself
6458 6472 * up. A return in the while loop will put the syncq in a
6459 6473 * very bad state.
6460 6474 */
6461 6475 sq->sq_count++;
6462 6476 ASSERT(sq->sq_count != 0); /* wraparound */
6463 6477
6464 6478 while ((flags = sq->sq_flags) & SQ_QUEUED) {
6465 6479 /*
6466 6480 * If we are told to stayaway or went exclusive,
6467 6481 * we are done.
6468 6482 */
6469 6483 if (flags & (SQ_STAYAWAY)) {
6470 6484 break;
6471 6485 }
6472 6486
6473 6487 /*
6474 6488 * If there are events to run, do so.
6475 6489 * We have one claim to the syncq, so if there are
6476 6490 * more than one, other threads are running.
6477 6491 */
6478 6492 if (sq->sq_evhead != NULL) {
6479 6493 ASSERT(sq->sq_flags & SQ_EVENTS);
6480 6494
6481 6495 count = sq->sq_count;
6482 6496 SQ_PUTLOCKS_ENTER(sq);
6483 6497 SUM_SQ_PUTCOUNTS(sq, count);
6484 6498 if (count > 1) {
6485 6499 SQ_PUTLOCKS_EXIT(sq);
6486 6500 /* Can't upgrade - other threads inside */
6487 6501 break;
6488 6502 }
6489 6503 ASSERT((flags & SQ_EXCL) == 0);
6490 6504 sq->sq_flags = flags | SQ_EXCL;
6491 6505 SQ_PUTLOCKS_EXIT(sq);
6492 6506 /*
6493 6507 * we have the only claim, run the events,
6494 6508 * sq_run_events will clear the SQ_EXCL flag.
6495 6509 */
6496 6510 sq_run_events(sq);
6497 6511
6498 6512 /*
6499 6513 * If this is a CIPUT perimeter, we need
6500 6514 * to drop the SQ_EXCL flag so we can properly
6501 6515 * continue draining the syncq.
6502 6516 */
6503 6517 if (type & SQ_CIPUT) {
6504 6518 ASSERT(sq->sq_flags & SQ_EXCL);
6505 6519 sq->sq_flags &= ~SQ_EXCL;
6506 6520 }
6507 6521
6508 6522 /*
6509 6523 * And go back to the beginning just in case
6510 6524 * anything changed while we were away.
6511 6525 */
6512 6526 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6513 6527 continue;
6514 6528 }
6515 6529
6516 6530 ASSERT(sq->sq_evhead == NULL);
6517 6531 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6518 6532
6519 6533 /*
6520 6534 * Find the queue that is not draining.
6521 6535 *
6522 6536 * q_draining is protected by QLOCK which we do not hold.
6523 6537 * But if it was set, then a thread was draining, and if it gets
6524 6538 * cleared, then it was because the thread has successfully
6525 6539 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6526 6540 * state to happen, a thread needs the SQLOCK which we hold, and
6527 6541 * if there was such a flag, we would have already seen it.
6528 6542 */
6529 6543
6530 6544 for (qp = sq->sq_head;
6531 6545 qp != NULL && (qp->q_draining ||
6532 6546 (qp->q_sqflags & Q_SQDRAINING));
6533 6547 qp = qp->q_sqnext)
6534 6548 ;
6535 6549
6536 6550 if (qp == NULL)
6537 6551 break;
6538 6552
6539 6553 /*
6540 6554 * We have a queue to work on, and we hold the
6541 6555 * SQLOCK and one claim, call qdrain_syncq.
6542 6556 * This means we need to release the SQLOCK and
6543 6557 * acquire the QLOCK (OK since we have a claim).
6544 6558 * Note that qdrain_syncq will actually dequeue
6545 6559 * this queue from the sq_head list when it is
6546 6560 * convinced all the work is done and release
6547 6561 * the QLOCK before returning.
6548 6562 */
6549 6563 qp->q_sqflags |= Q_SQDRAINING;
6550 6564 mutex_exit(SQLOCK(sq));
6551 6565 mutex_enter(QLOCK(qp));
6552 6566 qdrain_syncq(sq, qp);
6553 6567 mutex_enter(SQLOCK(sq));
6554 6568
6555 6569 /* The queue is drained */
6556 6570 ASSERT(qp->q_sqflags & Q_SQDRAINING);
6557 6571 qp->q_sqflags &= ~Q_SQDRAINING;
6558 6572 /*
6559 6573 * NOTE: After this point qp should not be used since it may be
6560 6574 * closed.
6561 6575 */
6562 6576 }
6563 6577
6564 6578 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6565 6579 flags = sq->sq_flags;
6566 6580
6567 6581 /*
6568 6582 * sq->sq_head cannot change because we hold the
6569 6583 * sqlock. However, a thread CAN decide that it is no longer
6570 6584 * going to drain that queue. However, this should be due to
6571 6585 * a GOAWAY state, and we should see that here.
6572 6586 *
6573 6587 * This loop is not very efficient. One solution may be adding a second
6574 6588 * pointer to the "draining" queue, but it is difficult to do when
6575 6589 * queues are inserted in the middle due to priority ordering. Another
6576 6590 * possibility is to yank the queue out of the sq list and put it onto
6577 6591 * the "draining list" and then put it back if it can't be drained.
6578 6592 */
6579 6593
6580 6594 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6581 6595 (type & SQ_CI) || sq->sq_head->q_draining);
6582 6596
6583 6597 /* Drop SQ_EXCL for non-CIPUT perimeters */
6584 6598 if (!(type & SQ_CIPUT))
6585 6599 flags &= ~SQ_EXCL;
6586 6600 ASSERT((flags & SQ_EXCL) == 0);
6587 6601
6588 6602 /* Wake up any waiters. */
6589 6603 if (flags & SQ_WANTWAKEUP) {
6590 6604 flags &= ~SQ_WANTWAKEUP;
6591 6605 cv_broadcast(&sq->sq_wait);
6592 6606 }
6593 6607 if (flags & SQ_WANTEXWAKEUP) {
6594 6608 flags &= ~SQ_WANTEXWAKEUP;
6595 6609 cv_broadcast(&sq->sq_exitwait);
6596 6610 }
6597 6611 sq->sq_flags = flags;
6598 6612
6599 6613 ASSERT(sq->sq_count != 0);
6600 6614 /* Release our claim. */
6601 6615 sq->sq_count--;
6602 6616
6603 6617 if (bg_service) {
6604 6618 ASSERT(sq->sq_servcount != 0);
6605 6619 sq->sq_servcount--;
6606 6620 }
6607 6621
6608 6622 mutex_exit(SQLOCK(sq));
6609 6623
6610 6624 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6611 6625 "drain_syncq end:%p", sq);
6612 6626 }
6613 6627
6614 6628
6615 6629 /*
6616 6630 *
6617 6631 * qdrain_syncq can be called (currently) from only one of two places:
6618 6632 * drain_syncq
6619 6633 * putnext (or some variation of it).
6620 6634 * and eventually
6621 6635 * qwait(_sig)
6622 6636 *
6623 6637 * If called from drain_syncq, we found it in the list of queues needing
6624 6638 * service, so there is work to be done (or it wouldn't be in the list).
6625 6639 *
6626 6640 * If called from some putnext variation, it was because the
6627 6641 * perimeter is open, but messages are blocking a putnext and
6628 6642 * there is not a thread working on it. Now a thread could start
6629 6643 * working on it while we are getting ready to do so ourself, but
6630 6644 * the thread would set the q_draining flag, and we can spin out.
6631 6645 *
6632 6646 * As for qwait(_sig), I think I shall let it continue to call
6633 6647 * drain_syncq directly (after all, it will get here eventually).
6634 6648 *
6635 6649 * qdrain_syncq has to terminate when:
6636 6650 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6637 6651 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6638 6652 *
6639 6653 * ASSUMES:
6640 6654 * One claim
6641 6655 * QLOCK held
6642 6656 * SQLOCK not held
6643 6657 * Will release QLOCK before returning
6644 6658 */
6645 6659 void
6646 6660 qdrain_syncq(syncq_t *sq, queue_t *q)
6647 6661 {
6648 6662 mblk_t *bp;
6649 6663 #ifdef DEBUG
6650 6664 uint16_t count;
6651 6665 #endif
6652 6666
6653 6667 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6654 6668 "drain_syncq start:%p", sq);
6655 6669 ASSERT(q->q_syncq == sq);
6656 6670 ASSERT(MUTEX_HELD(QLOCK(q)));
6657 6671 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6658 6672 /*
6659 6673 * For non-CIPUT perimeters, we should be called with the exclusive bit
6660 6674 * set already. For CIPUT perimeters, we will be doing a concurrent
6661 6675 * drain, so it better not be set.
6662 6676 */
6663 6677 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6664 6678 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6665 6679 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6666 6680 /*
6667 6681 * All outer pointers are set, or none of them are
6668 6682 */
6669 6683 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6670 6684 sq->sq_oprev == NULL) ||
6671 6685 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6672 6686 sq->sq_oprev != NULL));
6673 6687 #ifdef DEBUG
6674 6688 count = sq->sq_count;
6675 6689 /*
6676 6690 * This is OK without the putlocks, because we have one
6677 6691 * claim either from the sq_count, or a putcount. We could
6678 6692 * get an erroneous value from other counts, but ours won't
6679 6693 * change, so one way or another, we will have at least a
6680 6694 * value of one.
6681 6695 */
6682 6696 SUM_SQ_PUTCOUNTS(sq, count);
6683 6697 ASSERT(count >= 1);
6684 6698 #endif /* DEBUG */
6685 6699
6686 6700 /*
6687 6701 * The first thing to do is find out if a thread is already draining
6688 6702 * this queue. If so, we are done, just return.
6689 6703 */
6690 6704 if (q->q_draining) {
6691 6705 mutex_exit(QLOCK(q));
6692 6706 return;
6693 6707 }
6694 6708
6695 6709 /*
6696 6710 * If the perimeter is exclusive, there is nothing we can do right now,
6697 6711 * go away. Note that there is nothing to prevent this case from
6698 6712 * changing right after this check, but the spin-out will catch it.
6699 6713 */
6700 6714
6701 6715 /* Tell other threads that we are draining this queue */
6702 6716 q->q_draining = 1; /* Protected by QLOCK */
6703 6717
6704 6718 /*
6705 6719 * If there is nothing to do, clear QFULL as necessary. This caters for
6706 6720 * the case where an empty queue was enqueued onto the syncq.
6707 6721 */
6708 6722 if (q->q_sqhead == NULL) {
6709 6723 ASSERT(q->q_syncqmsgs == 0);
6710 6724 mutex_exit(QLOCK(q));
6711 6725 clr_qfull(q);
6712 6726 mutex_enter(QLOCK(q));
6713 6727 }
6714 6728
6715 6729 /*
6716 6730 * Note that q_sqhead must be re-checked here in case another message
6717 6731 * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6718 6732 */
6719 6733 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6720 6734 /*
6721 6735 * Because we can enter this routine just because a putnext is
6722 6736 * blocked, we need to spin out if the perimeter wants to go
6723 6737 * exclusive as well as just blocked. We need to spin out also
6724 6738 * if events are queued on the syncq.
6725 6739 * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6726 6740 * set it, and it can't become exclusive while we hold a claim.
6727 6741 */
6728 6742 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6729 6743 break;
6730 6744 }
6731 6745
6732 6746 #ifdef DEBUG
6733 6747 /*
6734 6748 * Since we are in qdrain_syncq, we already know the queue,
6735 6749 * but for sanity, we want to check this against the qp that
6736 6750 * was passed in by bp->b_queue.
6737 6751 */
6738 6752
6739 6753 ASSERT(bp->b_queue == q);
6740 6754 ASSERT(bp->b_queue->q_syncq == sq);
6741 6755 bp->b_queue = NULL;
6742 6756
6743 6757 /*
6744 6758 * We would have the following check in the DEBUG code:
6745 6759 *
6746 6760 * if (bp->b_prev != NULL) {
6747 6761 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6748 6762 * }
6749 6763 *
6750 6764 * This can't be done, however, since IP modifies qinfo
6751 6765 * structure at run-time (switching between IPv4 qinfo and IPv6
6752 6766 * qinfo), invalidating the check.
6753 6767 * So the assignment to func is left here, but the ASSERT itself
6754 6768 * is removed until the whole issue is resolved.
6755 6769 */
6756 6770 #endif
6757 6771 ASSERT(q->q_sqhead == bp);
6758 6772 q->q_sqhead = bp->b_next;
6759 6773 bp->b_prev = bp->b_next = NULL;
6760 6774 ASSERT(q->q_syncqmsgs > 0);
6761 6775 mutex_exit(QLOCK(q));
6762 6776
6763 6777 ASSERT(bp->b_datap->db_ref != 0);
6764 6778
6765 6779 (void) (*q->q_qinfo->qi_putp)(q, bp);
6766 6780
6767 6781 mutex_enter(QLOCK(q));
6768 6782
6769 6783 /*
6770 6784 * q_syncqmsgs should only be decremented after executing the
6771 6785 * put procedure to avoid message re-ordering. This is due to an
6772 6786 * optimisation in putnext() which can call the put procedure
6773 6787 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6774 6788 * being set).
6775 6789 *
6776 6790 * We also need to clear QFULL in the next service procedure
6777 6791 * queue if this is the last message destined for that queue.
6778 6792 *
6779 6793 * It would make better sense to have some sort of tunable for
6780 6794 * the low water mark, but these semantics are not yet defined.
6781 6795 * So, alas, we use a constant.
6782 6796 */
6783 6797 if (--q->q_syncqmsgs == 0) {
6784 6798 mutex_exit(QLOCK(q));
6785 6799 clr_qfull(q);
6786 6800 mutex_enter(QLOCK(q));
6787 6801 }
6788 6802
6789 6803 /*
6790 6804 * Always clear SQ_EXCL when CIPUT in order to handle
6791 6805 * qwriter(INNER). The putp() can call qwriter and get exclusive
6792 6806 * access IFF this is the only claim. So, we need to test for
6793 6807 * this possibility, acquire the mutex and clear the bit.
6794 6808 */
6795 6809 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6796 6810 mutex_enter(SQLOCK(sq));
6797 6811 sq->sq_flags &= ~SQ_EXCL;
6798 6812 mutex_exit(SQLOCK(sq));
6799 6813 }
6800 6814 }
6801 6815
6802 6816 /*
6803 6817 * We should either have no messages on this queue, or we were told to
6804 6818 * goaway by a waiter (which we will wake up at the end of this
6805 6819 * function).
6806 6820 */
6807 6821 ASSERT((q->q_sqhead == NULL) ||
6808 6822 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6809 6823
6810 6824 ASSERT(MUTEX_HELD(QLOCK(q)));
6811 6825 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6812 6826
6813 6827 /* Remove the q from the syncq list if all the messages are drained. */
6814 6828 if (q->q_sqhead == NULL) {
6815 6829 ASSERT(q->q_syncqmsgs == 0);
6816 6830 mutex_enter(SQLOCK(sq));
6817 6831 if (q->q_sqflags & Q_SQQUEUED)
6818 6832 SQRM_Q(sq, q);
6819 6833 mutex_exit(SQLOCK(sq));
6820 6834 /*
6821 6835 * Since the queue is removed from the list, reset its priority.
6822 6836 */
6823 6837 q->q_spri = 0;
6824 6838 }
6825 6839
6826 6840 /*
6827 6841 * Remember, the q_draining flag is used to let another thread know
6828 6842 * that there is a thread currently draining the messages for a queue.
6829 6843 * Since we are now done with this queue (even if there may be messages
6830 6844 * still there), we need to clear this flag so some thread will work on
6831 6845 * it if needed.
6832 6846 */
6833 6847 ASSERT(q->q_draining);
6834 6848 q->q_draining = 0;
6835 6849
6836 6850 /* Called with a claim, so OK to drop all locks. */
6837 6851 mutex_exit(QLOCK(q));
6838 6852
6839 6853 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6840 6854 "drain_syncq end:%p", sq);
6841 6855 }
6842 6856 /* END OF QDRAIN_SYNCQ */
6843 6857
6844 6858
6845 6859 /*
6846 6860 * This is the mate to qdrain_syncq, except that it is putting the message onto
6847 6861 * the queue instead of draining. Since the message is destined for the queue
6848 6862 * that is selected, there is no need to identify the function because the
6849 6863 * message is intended for the put routine for the queue. For debug kernels,
6850 6864 * this routine will do it anyway just in case.
6851 6865 *
6852 6866 * After the message is enqueued on the syncq, it calls putnext_tail()
6853 6867 * which will schedule a background thread to actually process the message.
6854 6868 *
6855 6869 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6856 6870 * SQLOCK(sq) and QLOCK(q) are not held.
6857 6871 */
6858 6872 void
6859 6873 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6860 6874 {
6861 6875 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6862 6876 ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6863 6877 ASSERT(sq->sq_count > 0);
6864 6878 ASSERT(q->q_syncq == sq);
6865 6879 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6866 6880 sq->sq_oprev == NULL) ||
6867 6881 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6868 6882 sq->sq_oprev != NULL));
6869 6883
6870 6884 mutex_enter(QLOCK(q));
6871 6885
6872 6886 #ifdef DEBUG
6873 6887 /*
6874 6888 * This is used for debug in the qfill_syncq/qdrain_syncq case
6875 6889 * to trace the queue that the message is intended for. Note
6876 6890 * that the original use was to identify the queue and function
6877 6891 * to call on the drain. In the new syncq, we have the context
6878 6892 * of the queue that we are draining, so call it's putproc and
6879 6893 * don't rely on the saved values. But for debug this is still
6880 6894 * useful information.
6881 6895 */
6882 6896 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6883 6897 mp->b_queue = q;
6884 6898 mp->b_next = NULL;
6885 6899 #endif
6886 6900 ASSERT(q->q_syncq == sq);
6887 6901 /*
6888 6902 * Enqueue the message on the list.
6889 6903 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to
6890 6904 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP().
6891 6905 */
6892 6906 SQPUT_MP(q, mp);
6893 6907 mutex_enter(SQLOCK(sq));
6894 6908
6895 6909 /*
6896 6910 * And queue on syncq for scheduling, if not already queued.
6897 6911 * Note that we need the SQLOCK for this, and for testing flags
6898 6912 * at the end to see if we will drain. So grab it now, and
6899 6913 * release it before we call qdrain_syncq or return.
6900 6914 */
6901 6915 if (!(q->q_sqflags & Q_SQQUEUED)) {
6902 6916 q->q_spri = curthread->t_pri;
6903 6917 SQPUT_Q(sq, q);
6904 6918 }
6905 6919 #ifdef DEBUG
6906 6920 else {
6907 6921 /*
6908 6922 * All of these conditions MUST be true!
6909 6923 */
6910 6924 ASSERT(sq->sq_tail != NULL);
6911 6925 if (sq->sq_tail == sq->sq_head) {
6912 6926 ASSERT((q->q_sqprev == NULL) &&
6913 6927 (q->q_sqnext == NULL));
6914 6928 } else {
6915 6929 ASSERT((q->q_sqprev != NULL) ||
6916 6930 (q->q_sqnext != NULL));
6917 6931 }
6918 6932 ASSERT(sq->sq_flags & SQ_QUEUED);
6919 6933 ASSERT(q->q_syncqmsgs != 0);
6920 6934 ASSERT(q->q_sqflags & Q_SQQUEUED);
6921 6935 }
6922 6936 #endif
6923 6937 mutex_exit(QLOCK(q));
6924 6938 /*
6925 6939 * SQLOCK is still held, so sq_count can be safely decremented.
6926 6940 */
6927 6941 sq->sq_count--;
6928 6942
6929 6943 putnext_tail(sq, q, 0);
6930 6944 /* Should not reference sq or q after this point. */
6931 6945 }
6932 6946
6933 6947 /* End of qfill_syncq */
6934 6948
6935 6949 /*
6936 6950 * Remove all messages from a syncq (if qp is NULL) or remove all messages
6937 6951 * that would be put into qp by drain_syncq.
6938 6952 * Used when deleting the syncq (qp == NULL) or when detaching
6939 6953 * a queue (qp != NULL).
6940 6954 * Return non-zero if one or more messages were freed.
6941 6955 *
6942 6956 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6943 6957 * sq_putlocks are used.
6944 6958 *
6945 6959 * NOTE: This function assumes that it is called from the close() context and
6946 6960 * that all the queues in the syncq are going away. For this reason it doesn't
6947 6961 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6948 6962 * currently valid, but it is useful to rethink this function to behave properly
6949 6963 * in other cases.
6950 6964 */
6951 6965 int
6952 6966 flush_syncq(syncq_t *sq, queue_t *qp)
6953 6967 {
6954 6968 mblk_t *bp, *mp_head, *mp_next, *mp_prev;
6955 6969 queue_t *q;
6956 6970 int ret = 0;
6957 6971
6958 6972 mutex_enter(SQLOCK(sq));
6959 6973
6960 6974 /*
6961 6975 * Before we leave, we need to make sure there are no
6962 6976 * events listed for this queue. All events for this queue
6963 6977 * will just be freed.
6964 6978 */
6965 6979 if (qp != NULL && sq->sq_evhead != NULL) {
6966 6980 ASSERT(sq->sq_flags & SQ_EVENTS);
6967 6981
6968 6982 mp_prev = NULL;
6969 6983 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6970 6984 mp_next = bp->b_next;
6971 6985 if (bp->b_queue == qp) {
6972 6986 /* Delete this message */
6973 6987 if (mp_prev != NULL) {
6974 6988 mp_prev->b_next = mp_next;
6975 6989 /*
6976 6990 * Update sq_evtail if the last element
6977 6991 * is removed.
6978 6992 */
6979 6993 if (bp == sq->sq_evtail) {
6980 6994 ASSERT(mp_next == NULL);
6981 6995 sq->sq_evtail = mp_prev;
6982 6996 }
6983 6997 } else
6984 6998 sq->sq_evhead = mp_next;
6985 6999 if (sq->sq_evhead == NULL)
6986 7000 sq->sq_flags &= ~SQ_EVENTS;
6987 7001 bp->b_prev = bp->b_next = NULL;
6988 7002 freemsg(bp);
6989 7003 ret++;
6990 7004 } else {
6991 7005 mp_prev = bp;
6992 7006 }
6993 7007 }
6994 7008 }
6995 7009
6996 7010 /*
6997 7011 * Walk sq_head and:
6998 7012 * - match qp if qp is set, remove it's messages
6999 7013 * - all if qp is not set
7000 7014 */
7001 7015 q = sq->sq_head;
7002 7016 while (q != NULL) {
7003 7017 ASSERT(q->q_syncq == sq);
7004 7018 if ((qp == NULL) || (qp == q)) {
7005 7019 /*
7006 7020 * Yank the messages as a list off the queue
7007 7021 */
7008 7022 mp_head = q->q_sqhead;
7009 7023 /*
7010 7024 * We do not have QLOCK(q) here (which is safe due to
7011 7025 * assumptions mentioned above). To obtain the lock we
7012 7026 * need to release SQLOCK which may allow lots of things
7013 7027 * to change upon us. This place requires more analysis.
7014 7028 */
7015 7029 q->q_sqhead = q->q_sqtail = NULL;
7016 7030 ASSERT(mp_head->b_queue &&
7017 7031 mp_head->b_queue->q_syncq == sq);
7018 7032
7019 7033 /*
7020 7034 * Free each of the messages.
7021 7035 */
7022 7036 for (bp = mp_head; bp != NULL; bp = mp_next) {
7023 7037 mp_next = bp->b_next;
7024 7038 bp->b_prev = bp->b_next = NULL;
7025 7039 freemsg(bp);
7026 7040 ret++;
7027 7041 }
7028 7042 /*
7029 7043 * Now remove the queue from the syncq.
7030 7044 */
7031 7045 ASSERT(q->q_sqflags & Q_SQQUEUED);
7032 7046 SQRM_Q(sq, q);
7033 7047 q->q_spri = 0;
7034 7048 q->q_syncqmsgs = 0;
7035 7049
7036 7050 /*
7037 7051 * If qp was specified, we are done with it and are
7038 7052 * going to drop SQLOCK(sq) and return. We wakeup syncq
7039 7053 * waiters while we still have the SQLOCK.
7040 7054 */
7041 7055 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7042 7056 sq->sq_flags &= ~SQ_WANTWAKEUP;
7043 7057 cv_broadcast(&sq->sq_wait);
7044 7058 }
7045 7059 /* Drop SQLOCK across clr_qfull */
7046 7060 mutex_exit(SQLOCK(sq));
7047 7061
7048 7062 /*
7049 7063 * We avoid doing the test that drain_syncq does and
7050 7064 * unconditionally clear qfull for every flushed
7051 7065 * message. Since flush_syncq is only called during
7052 7066 * close this should not be a problem.
7053 7067 */
7054 7068 clr_qfull(q);
7055 7069 if (qp != NULL) {
7056 7070 return (ret);
7057 7071 } else {
7058 7072 mutex_enter(SQLOCK(sq));
7059 7073 /*
7060 7074 * The head was removed by SQRM_Q above.
7061 7075 * reread the new head and flush it.
7062 7076 */
7063 7077 q = sq->sq_head;
7064 7078 }
7065 7079 } else {
7066 7080 q = q->q_sqnext;
7067 7081 }
7068 7082 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7069 7083 }
7070 7084
7071 7085 if (sq->sq_flags & SQ_WANTWAKEUP) {
7072 7086 sq->sq_flags &= ~SQ_WANTWAKEUP;
7073 7087 cv_broadcast(&sq->sq_wait);
7074 7088 }
7075 7089
7076 7090 mutex_exit(SQLOCK(sq));
7077 7091 return (ret);
7078 7092 }
7079 7093
7080 7094 /*
7081 7095 * Propagate all messages from a syncq to the next syncq that are associated
7082 7096 * with the specified queue. If the queue is attached to a driver or if the
7083 7097 * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7084 7098 *
7085 7099 * Assumes that the stream is strlock()'ed. We don't come here if there
7086 7100 * are no messages to propagate.
7087 7101 *
7088 7102 * NOTE : If the queue is attached to a driver, all the messages are freed
7089 7103 * as there is no point in propagating the messages from the driver syncq
7090 7104 * to the closing stream head which will in turn get freed later.
7091 7105 */
7092 7106 static int
7093 7107 propagate_syncq(queue_t *qp)
7094 7108 {
7095 7109 mblk_t *bp, *head, *tail, *prev, *next;
7096 7110 syncq_t *sq;
7097 7111 queue_t *nqp;
7098 7112 syncq_t *nsq;
7099 7113 boolean_t isdriver;
7100 7114 int moved = 0;
7101 7115 uint16_t flags;
7102 7116 pri_t priority = curthread->t_pri;
7103 7117 #ifdef DEBUG
7104 7118 void (*func)();
7105 7119 #endif
7106 7120
7107 7121 sq = qp->q_syncq;
7108 7122 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7109 7123 /* debug macro */
7110 7124 SQ_PUTLOCKS_HELD(sq);
7111 7125 /*
7112 7126 * As entersq() does not increment the sq_count for
7113 7127 * the write side, check sq_count for non-QPERQ
7114 7128 * perimeters alone.
7115 7129 */
7116 7130 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7117 7131
7118 7132 /*
7119 7133 * propagate_syncq() can be called because of either messages on the
7120 7134 * queue syncq or because on events on the queue syncq. Do actual
7121 7135 * message propagations if there are any messages.
7122 7136 */
7123 7137 if (qp->q_syncqmsgs) {
7124 7138 isdriver = (qp->q_flag & QISDRV);
7125 7139
7126 7140 if (!isdriver) {
7127 7141 nqp = qp->q_next;
7128 7142 nsq = nqp->q_syncq;
7129 7143 ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7130 7144 /* debug macro */
7131 7145 SQ_PUTLOCKS_HELD(nsq);
7132 7146 #ifdef DEBUG
7133 7147 func = (void (*)())nqp->q_qinfo->qi_putp;
7134 7148 #endif
7135 7149 }
7136 7150
7137 7151 SQRM_Q(sq, qp);
7138 7152 priority = MAX(qp->q_spri, priority);
7139 7153 qp->q_spri = 0;
7140 7154 head = qp->q_sqhead;
7141 7155 tail = qp->q_sqtail;
7142 7156 qp->q_sqhead = qp->q_sqtail = NULL;
7143 7157 qp->q_syncqmsgs = 0;
7144 7158
7145 7159 /*
7146 7160 * Walk the list of messages, and free them if this is a driver,
7147 7161 * otherwise reset the b_prev and b_queue value to the new putp.
7148 7162 * Afterward, we will just add the head to the end of the next
7149 7163 * syncq, and point the tail to the end of this one.
7150 7164 */
7151 7165
7152 7166 for (bp = head; bp != NULL; bp = next) {
7153 7167 next = bp->b_next;
7154 7168 if (isdriver) {
7155 7169 bp->b_prev = bp->b_next = NULL;
7156 7170 freemsg(bp);
7157 7171 continue;
7158 7172 }
7159 7173 /* Change the q values for this message */
7160 7174 bp->b_queue = nqp;
7161 7175 #ifdef DEBUG
7162 7176 bp->b_prev = (mblk_t *)func;
7163 7177 #endif
7164 7178 moved++;
7165 7179 }
7166 7180 /*
7167 7181 * Attach list of messages to the end of the new queue (if there
7168 7182 * is a list of messages).
7169 7183 */
7170 7184
7171 7185 if (!isdriver && head != NULL) {
7172 7186 ASSERT(tail != NULL);
7173 7187 if (nqp->q_sqhead == NULL) {
7174 7188 nqp->q_sqhead = head;
7175 7189 } else {
7176 7190 ASSERT(nqp->q_sqtail != NULL);
7177 7191 nqp->q_sqtail->b_next = head;
7178 7192 }
7179 7193 nqp->q_sqtail = tail;
7180 7194 /*
7181 7195 * When messages are moved from high priority queue to
7182 7196 * another queue, the destination queue priority is
7183 7197 * upgraded.
7184 7198 */
7185 7199
7186 7200 if (priority > nqp->q_spri)
7187 7201 nqp->q_spri = priority;
7188 7202
7189 7203 SQPUT_Q(nsq, nqp);
7190 7204
7191 7205 nqp->q_syncqmsgs += moved;
7192 7206 ASSERT(nqp->q_syncqmsgs != 0);
7193 7207 }
7194 7208 }
7195 7209
7196 7210 /*
7197 7211 * Before we leave, we need to make sure there are no
7198 7212 * events listed for this queue. All events for this queue
7199 7213 * will just be freed.
7200 7214 */
7201 7215 if (sq->sq_evhead != NULL) {
7202 7216 ASSERT(sq->sq_flags & SQ_EVENTS);
7203 7217 prev = NULL;
7204 7218 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7205 7219 next = bp->b_next;
7206 7220 if (bp->b_queue == qp) {
7207 7221 /* Delete this message */
7208 7222 if (prev != NULL) {
7209 7223 prev->b_next = next;
7210 7224 /*
7211 7225 * Update sq_evtail if the last element
7212 7226 * is removed.
7213 7227 */
7214 7228 if (bp == sq->sq_evtail) {
7215 7229 ASSERT(next == NULL);
7216 7230 sq->sq_evtail = prev;
7217 7231 }
7218 7232 } else
7219 7233 sq->sq_evhead = next;
7220 7234 if (sq->sq_evhead == NULL)
7221 7235 sq->sq_flags &= ~SQ_EVENTS;
7222 7236 bp->b_prev = bp->b_next = NULL;
7223 7237 freemsg(bp);
7224 7238 } else {
7225 7239 prev = bp;
7226 7240 }
7227 7241 }
7228 7242 }
7229 7243
7230 7244 flags = sq->sq_flags;
7231 7245
7232 7246 /* Wake up any waiter before leaving. */
7233 7247 if (flags & SQ_WANTWAKEUP) {
7234 7248 flags &= ~SQ_WANTWAKEUP;
7235 7249 cv_broadcast(&sq->sq_wait);
7236 7250 }
7237 7251 sq->sq_flags = flags;
7238 7252
7239 7253 return (moved);
7240 7254 }
7241 7255
7242 7256 /*
7243 7257 * Try and upgrade to exclusive access at the inner perimeter. If this can
7244 7258 * not be done without blocking then request will be queued on the syncq
7245 7259 * and drain_syncq will run it later.
7246 7260 *
7247 7261 * This routine can only be called from put or service procedures plus
7248 7262 * asynchronous callback routines that have properly entered the queue (with
7249 7263 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7250 7264 * associated with q.
7251 7265 */
7252 7266 void
7253 7267 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7254 7268 {
7255 7269 syncq_t *sq = q->q_syncq;
7256 7270 uint16_t count;
7257 7271
7258 7272 mutex_enter(SQLOCK(sq));
7259 7273 count = sq->sq_count;
7260 7274 SQ_PUTLOCKS_ENTER(sq);
7261 7275 SUM_SQ_PUTCOUNTS(sq, count);
7262 7276 ASSERT(count >= 1);
7263 7277 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7264 7278
7265 7279 if (count == 1) {
7266 7280 /*
7267 7281 * Can upgrade. This case also handles nested qwriter calls
7268 7282 * (when the qwriter callback function calls qwriter). In that
7269 7283 * case SQ_EXCL is already set.
7270 7284 */
7271 7285 sq->sq_flags |= SQ_EXCL;
7272 7286 SQ_PUTLOCKS_EXIT(sq);
7273 7287 mutex_exit(SQLOCK(sq));
7274 7288 (*func)(q, mp);
7275 7289 /*
7276 7290 * Assumes that leavesq, putnext, and drain_syncq will reset
7277 7291 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7278 7292 * until putnext, leavesq, or drain_syncq drops it.
7279 7293 * That way we handle nested qwriter(INNER) without dropping
7280 7294 * SQ_EXCL until the outermost qwriter callback routine is
7281 7295 * done.
7282 7296 */
7283 7297 return;
7284 7298 }
7285 7299 SQ_PUTLOCKS_EXIT(sq);
7286 7300 sqfill_events(sq, q, mp, func);
7287 7301 }
7288 7302
7289 7303 /*
7290 7304 * Synchronous callback support functions
7291 7305 */
7292 7306
7293 7307 /*
7294 7308 * Allocate a callback parameter structure.
7295 7309 * Assumes that caller initializes the flags and the id.
7296 7310 * Acquires SQLOCK(sq) if non-NULL is returned.
7297 7311 */
7298 7312 callbparams_t *
7299 7313 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7300 7314 {
7301 7315 callbparams_t *cbp;
7302 7316 size_t size = sizeof (callbparams_t);
7303 7317
7304 7318 cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7305 7319
7306 7320 /*
7307 7321 * Only try tryhard allocation if the caller is ready to panic.
7308 7322 * Otherwise just fail.
7309 7323 */
7310 7324 if (cbp == NULL) {
7311 7325 if (kmflags & KM_PANIC)
7312 7326 cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7313 7327 &size, kmflags);
7314 7328 else
7315 7329 return (NULL);
7316 7330 }
7317 7331
7318 7332 ASSERT(size >= sizeof (callbparams_t));
7319 7333 cbp->cbp_size = size;
7320 7334 cbp->cbp_sq = sq;
7321 7335 cbp->cbp_func = func;
7322 7336 cbp->cbp_arg = arg;
7323 7337 mutex_enter(SQLOCK(sq));
7324 7338 cbp->cbp_next = sq->sq_callbpend;
7325 7339 sq->sq_callbpend = cbp;
7326 7340 return (cbp);
7327 7341 }
7328 7342
7329 7343 void
7330 7344 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7331 7345 {
7332 7346 callbparams_t **pp, *p;
7333 7347
7334 7348 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7335 7349
7336 7350 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7337 7351 if (p == cbp) {
7338 7352 *pp = p->cbp_next;
7339 7353 kmem_free(p, p->cbp_size);
7340 7354 return;
7341 7355 }
7342 7356 }
7343 7357 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7344 7358 "callbparams_free: not found\n"));
7345 7359 }
7346 7360
7347 7361 void
7348 7362 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7349 7363 {
7350 7364 callbparams_t **pp, *p;
7351 7365
7352 7366 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7353 7367
7354 7368 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7355 7369 if (p->cbp_id == id && p->cbp_flags == flag) {
7356 7370 *pp = p->cbp_next;
7357 7371 kmem_free(p, p->cbp_size);
7358 7372 return;
7359 7373 }
7360 7374 }
7361 7375 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7362 7376 "callbparams_free_id: not found\n"));
7363 7377 }
7364 7378
7365 7379 /*
7366 7380 * Callback wrapper function used by once-only callbacks that can be
7367 7381 * cancelled (qtimeout and qbufcall)
7368 7382 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7369 7383 * cancelled by the qun* functions.
7370 7384 */
7371 7385 void
7372 7386 qcallbwrapper(void *arg)
7373 7387 {
7374 7388 callbparams_t *cbp = arg;
7375 7389 syncq_t *sq;
7376 7390 uint16_t count = 0;
7377 7391 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7378 7392 uint16_t type;
7379 7393
7380 7394 sq = cbp->cbp_sq;
7381 7395 mutex_enter(SQLOCK(sq));
7382 7396 type = sq->sq_type;
7383 7397 if (!(type & SQ_CICB)) {
7384 7398 count = sq->sq_count;
7385 7399 SQ_PUTLOCKS_ENTER(sq);
7386 7400 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7387 7401 SUM_SQ_PUTCOUNTS(sq, count);
7388 7402 sq->sq_needexcl++;
7389 7403 ASSERT(sq->sq_needexcl != 0); /* wraparound */
7390 7404 waitflags |= SQ_MESSAGES;
7391 7405 }
7392 7406 /* Can not handle exclusive entry at outer perimeter */
7393 7407 ASSERT(type & SQ_COCB);
7394 7408
7395 7409 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7396 7410 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7397 7411 (sq->sq_cancelid == cbp->cbp_id)) {
7398 7412 /* timeout has been cancelled */
7399 7413 sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7400 7414 callbparams_free(sq, cbp);
7401 7415 if (!(type & SQ_CICB)) {
7402 7416 ASSERT(sq->sq_needexcl > 0);
7403 7417 sq->sq_needexcl--;
7404 7418 if (sq->sq_needexcl == 0) {
7405 7419 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7406 7420 }
7407 7421 SQ_PUTLOCKS_EXIT(sq);
7408 7422 }
7409 7423 mutex_exit(SQLOCK(sq));
7410 7424 return;
7411 7425 }
7412 7426 sq->sq_flags |= SQ_WANTWAKEUP;
7413 7427 if (!(type & SQ_CICB)) {
7414 7428 SQ_PUTLOCKS_EXIT(sq);
7415 7429 }
7416 7430 cv_wait(&sq->sq_wait, SQLOCK(sq));
7417 7431 if (!(type & SQ_CICB)) {
7418 7432 count = sq->sq_count;
7419 7433 SQ_PUTLOCKS_ENTER(sq);
7420 7434 SUM_SQ_PUTCOUNTS(sq, count);
7421 7435 }
7422 7436 }
7423 7437
7424 7438 sq->sq_count++;
7425 7439 ASSERT(sq->sq_count != 0); /* Wraparound */
7426 7440 if (!(type & SQ_CICB)) {
7427 7441 ASSERT(count == 0);
7428 7442 sq->sq_flags |= SQ_EXCL;
7429 7443 ASSERT(sq->sq_needexcl > 0);
7430 7444 sq->sq_needexcl--;
7431 7445 if (sq->sq_needexcl == 0) {
7432 7446 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7433 7447 }
7434 7448 SQ_PUTLOCKS_EXIT(sq);
7435 7449 }
7436 7450
7437 7451 mutex_exit(SQLOCK(sq));
7438 7452
7439 7453 cbp->cbp_func(cbp->cbp_arg);
7440 7454
7441 7455 /*
7442 7456 * We drop the lock only for leavesq to re-acquire it.
7443 7457 * Possible optimization is inline of leavesq.
7444 7458 */
7445 7459 mutex_enter(SQLOCK(sq));
7446 7460 callbparams_free(sq, cbp);
7447 7461 mutex_exit(SQLOCK(sq));
7448 7462 leavesq(sq, SQ_CALLBACK);
7449 7463 }
7450 7464
7451 7465 /*
7452 7466 * No need to grab sq_putlocks here. See comment in strsubr.h that
7453 7467 * explains when sq_putlocks are used.
7454 7468 *
7455 7469 * sq_count (or one of the sq_putcounts) has already been
7456 7470 * decremented by the caller, and if SQ_QUEUED, we need to call
7457 7471 * drain_syncq (the global syncq drain).
7458 7472 * If putnext_tail is called with the SQ_EXCL bit set, we are in
7459 7473 * one of two states, non-CIPUT perimeter, and we need to clear
7460 7474 * it, or we went exclusive in the put procedure. In any case,
7461 7475 * we want to clear the bit now, and it is probably easier to do
7462 7476 * this at the beginning of this function (remember, we hold
7463 7477 * the SQLOCK). Lastly, if there are other messages queued
7464 7478 * on the syncq (and not for our destination), enable the syncq
7465 7479 * for background work.
7466 7480 */
7467 7481
7468 7482 /* ARGSUSED */
7469 7483 void
7470 7484 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7471 7485 {
7472 7486 uint16_t flags = sq->sq_flags;
7473 7487
7474 7488 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7475 7489 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7476 7490
7477 7491 /* Clear SQ_EXCL if set in passflags */
7478 7492 if (passflags & SQ_EXCL) {
7479 7493 flags &= ~SQ_EXCL;
7480 7494 }
7481 7495 if (flags & SQ_WANTWAKEUP) {
7482 7496 flags &= ~SQ_WANTWAKEUP;
7483 7497 cv_broadcast(&sq->sq_wait);
7484 7498 }
7485 7499 if (flags & SQ_WANTEXWAKEUP) {
7486 7500 flags &= ~SQ_WANTEXWAKEUP;
7487 7501 cv_broadcast(&sq->sq_exitwait);
7488 7502 }
7489 7503 sq->sq_flags = flags;
7490 7504
7491 7505 /*
7492 7506 * We have cleared SQ_EXCL if we were asked to, and started
7493 7507 * the wakeup process for waiters. If there are no writers
7494 7508 * then we need to drain the syncq if we were told to, or
7495 7509 * enable the background thread to do it.
7496 7510 */
7497 7511 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7498 7512 if ((passflags & SQ_QUEUED) ||
7499 7513 (sq->sq_svcflags & SQ_DISABLED)) {
7500 7514 /* drain_syncq will take care of events in the list */
7501 7515 drain_syncq(sq);
7502 7516 return;
7503 7517 } else if (flags & SQ_QUEUED) {
7504 7518 sqenable(sq);
7505 7519 }
7506 7520 }
7507 7521 /* Drop the SQLOCK on exit */
7508 7522 mutex_exit(SQLOCK(sq));
7509 7523 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7510 7524 "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7511 7525 }
7512 7526
7513 7527 void
7514 7528 set_qend(queue_t *q)
7515 7529 {
7516 7530 mutex_enter(QLOCK(q));
7517 7531 if (!O_SAMESTR(q))
7518 7532 q->q_flag |= QEND;
7519 7533 else
7520 7534 q->q_flag &= ~QEND;
7521 7535 mutex_exit(QLOCK(q));
7522 7536 q = _OTHERQ(q);
7523 7537 mutex_enter(QLOCK(q));
7524 7538 if (!O_SAMESTR(q))
7525 7539 q->q_flag |= QEND;
7526 7540 else
7527 7541 q->q_flag &= ~QEND;
7528 7542 mutex_exit(QLOCK(q));
7529 7543 }
7530 7544
7531 7545 /*
7532 7546 * Set QFULL in next service procedure queue (that cares) if not already
7533 7547 * set and if there are already more messages on the syncq than
7534 7548 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on
7535 7549 * any syncq.
7536 7550 *
7537 7551 * The fq here is the next queue with a service procedure. This is where
7538 7552 * we would fail canputnext, so this is where we need to set QFULL.
7539 7553 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7540 7554 *
7541 7555 * We already have QLOCK at this point. To avoid cross-locks with
7542 7556 * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7543 7557 * SQLOCK and sd_reflock, we need to drop respective locks first.
7544 7558 */
7545 7559 void
7546 7560 set_qfull(queue_t *q)
7547 7561 {
7548 7562 queue_t *fq = NULL;
7549 7563
7550 7564 ASSERT(MUTEX_HELD(QLOCK(q)));
7551 7565 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7552 7566 (q->q_syncqmsgs > sq_max_size)) {
7553 7567 if ((fq = q->q_nfsrv) == q) {
7554 7568 fq->q_flag |= QFULL;
7555 7569 } else {
7556 7570 mutex_exit(QLOCK(q));
7557 7571 mutex_enter(QLOCK(fq));
7558 7572 fq->q_flag |= QFULL;
7559 7573 mutex_exit(QLOCK(fq));
7560 7574 mutex_enter(QLOCK(q));
7561 7575 }
7562 7576 }
7563 7577 }
7564 7578
7565 7579 void
7566 7580 clr_qfull(queue_t *q)
7567 7581 {
7568 7582 queue_t *oq = q;
7569 7583
7570 7584 q = q->q_nfsrv;
7571 7585 /* Fast check if there is any work to do before getting the lock. */
7572 7586 if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7573 7587 return;
7574 7588 }
7575 7589
7576 7590 /*
7577 7591 * Do not reset QFULL (and backenable) if the q_count is the reason
7578 7592 * for QFULL being set.
7579 7593 */
7580 7594 mutex_enter(QLOCK(q));
7581 7595 /*
7582 7596 * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7583 7597 * Hence clear the QFULL.
7584 7598 * If both q_count and q_mblkcnt are less than the hiwat mark,
7585 7599 * clear the QFULL.
7586 7600 */
7587 7601 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7588 7602 (q->q_mblkcnt < q->q_hiwat))) {
7589 7603 q->q_flag &= ~QFULL;
7590 7604 /*
7591 7605 * A little more confusing, how about this way:
7592 7606 * if someone wants to write,
7593 7607 * AND
7594 7608 * both counts are less than the lowat mark
7595 7609 * OR
7596 7610 * the lowat mark is zero
7597 7611 * THEN
7598 7612 * backenable
7599 7613 */
7600 7614 if ((q->q_flag & QWANTW) &&
7601 7615 (((q->q_count < q->q_lowat) &&
7602 7616 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7603 7617 q->q_flag &= ~QWANTW;
7604 7618 mutex_exit(QLOCK(q));
7605 7619 backenable(oq, 0);
7606 7620 } else
7607 7621 mutex_exit(QLOCK(q));
7608 7622 } else
7609 7623 mutex_exit(QLOCK(q));
7610 7624 }
7611 7625
7612 7626 /*
7613 7627 * Set the forward service procedure pointer.
7614 7628 *
7615 7629 * Called at insert-time to cache a queue's next forward service procedure in
7616 7630 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted
7617 7631 * has a service procedure then q_nfsrv points to itself. If the queue to be
7618 7632 * inserted does not have a service procedure, then q_nfsrv points to the next
7619 7633 * queue forward that has a service procedure. If the queue is at the logical
7620 7634 * end of the stream (driver for write side, stream head for the read side)
7621 7635 * and does not have a service procedure, then q_nfsrv also points to itself.
7622 7636 */
7623 7637 void
7624 7638 set_nfsrv_ptr(
7625 7639 queue_t *rnew, /* read queue pointer to new module */
7626 7640 queue_t *wnew, /* write queue pointer to new module */
7627 7641 queue_t *prev_rq, /* read queue pointer to the module above */
7628 7642 queue_t *prev_wq) /* write queue pointer to the module above */
7629 7643 {
7630 7644 queue_t *qp;
7631 7645
7632 7646 if (prev_wq->q_next == NULL) {
7633 7647 /*
7634 7648 * Insert the driver, initialize the driver and stream head.
7635 7649 * In this case, prev_rq/prev_wq should be the stream head.
7636 7650 * _I_INSERT does not allow inserting a driver. Make sure
7637 7651 * that it is not an insertion.
7638 7652 */
7639 7653 ASSERT(!(rnew->q_flag & _QINSERTING));
7640 7654 wnew->q_nfsrv = wnew;
7641 7655 if (rnew->q_qinfo->qi_srvp)
7642 7656 rnew->q_nfsrv = rnew;
7643 7657 else
7644 7658 rnew->q_nfsrv = prev_rq;
7645 7659 prev_rq->q_nfsrv = prev_rq;
7646 7660 prev_wq->q_nfsrv = prev_wq;
7647 7661 } else {
7648 7662 /*
7649 7663 * set up read side q_nfsrv pointer. This MUST be done
7650 7664 * before setting the write side, because the setting of
7651 7665 * the write side for a fifo may depend on it.
7652 7666 *
7653 7667 * Suppose we have a fifo that only has pipemod pushed.
7654 7668 * pipemod has no read or write service procedures, so
7655 7669 * nfsrv for both pipemod queues points to prev_rq (the
7656 7670 * stream read head). Now push bufmod (which has only a
7657 7671 * read service procedure). Doing the write side first,
7658 7672 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7659 7673 * is WRONG; the next queue forward from wnew with a
7660 7674 * service procedure will be rnew, not the stream read head.
7661 7675 * Since the downstream queue (which in the case of a fifo
7662 7676 * is the read queue rnew) can affect upstream queues, it
7663 7677 * needs to be done first. Setting up the read side first
7664 7678 * sets nfsrv for both pipemod queues to rnew and then
7665 7679 * when the write side is set up, wnew-q_nfsrv will also
7666 7680 * point to rnew.
7667 7681 */
7668 7682 if (rnew->q_qinfo->qi_srvp) {
7669 7683 /*
7670 7684 * use _OTHERQ() because, if this is a pipe, next
7671 7685 * module may have been pushed from other end and
7672 7686 * q_next could be a read queue.
7673 7687 */
7674 7688 qp = _OTHERQ(prev_wq->q_next);
7675 7689 while (qp && qp->q_nfsrv != qp) {
7676 7690 qp->q_nfsrv = rnew;
7677 7691 qp = backq(qp);
7678 7692 }
7679 7693 rnew->q_nfsrv = rnew;
7680 7694 } else
7681 7695 rnew->q_nfsrv = prev_rq->q_nfsrv;
7682 7696
7683 7697 /* set up write side q_nfsrv pointer */
7684 7698 if (wnew->q_qinfo->qi_srvp) {
7685 7699 wnew->q_nfsrv = wnew;
7686 7700
7687 7701 /*
7688 7702 * For insertion, need to update nfsrv of the modules
7689 7703 * above which do not have a service routine.
7690 7704 */
7691 7705 if (rnew->q_flag & _QINSERTING) {
7692 7706 for (qp = prev_wq;
7693 7707 qp != NULL && qp->q_nfsrv != qp;
7694 7708 qp = backq(qp)) {
7695 7709 qp->q_nfsrv = wnew->q_nfsrv;
7696 7710 }
7697 7711 }
7698 7712 } else {
7699 7713 if (prev_wq->q_next == prev_rq)
7700 7714 /*
7701 7715 * Since prev_wq/prev_rq are the middle of a
7702 7716 * fifo, wnew/rnew will also be the middle of
7703 7717 * a fifo and wnew's nfsrv is same as rnew's.
7704 7718 */
7705 7719 wnew->q_nfsrv = rnew->q_nfsrv;
7706 7720 else
7707 7721 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7708 7722 }
7709 7723 }
7710 7724 }
7711 7725
7712 7726 /*
7713 7727 * Reset the forward service procedure pointer; called at remove-time.
7714 7728 */
7715 7729 void
7716 7730 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7717 7731 {
7718 7732 queue_t *tmp_qp;
7719 7733
7720 7734 /* Reset the write side q_nfsrv pointer for _I_REMOVE */
7721 7735 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7722 7736 for (tmp_qp = backq(wqp);
7723 7737 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7724 7738 tmp_qp = backq(tmp_qp)) {
7725 7739 tmp_qp->q_nfsrv = wqp->q_nfsrv;
7726 7740 }
7727 7741 }
7728 7742
7729 7743 /* reset the read side q_nfsrv pointer */
7730 7744 if (rqp->q_qinfo->qi_srvp) {
7731 7745 if (wqp->q_next) { /* non-driver case */
7732 7746 tmp_qp = _OTHERQ(wqp->q_next);
7733 7747 while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7734 7748 /* Note that rqp->q_next cannot be NULL */
7735 7749 ASSERT(rqp->q_next != NULL);
7736 7750 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7737 7751 tmp_qp = backq(tmp_qp);
7738 7752 }
7739 7753 }
7740 7754 }
7741 7755 }
7742 7756
7743 7757 /*
7744 7758 * This routine should be called after all stream geometry changes to update
7745 7759 * the stream head cached struio() rd/wr queue pointers. Note must be called
7746 7760 * with the streamlock()ed.
7747 7761 *
7748 7762 * Note: only enables Synchronous STREAMS for a side of a Stream which has
7749 7763 * an explicit synchronous barrier module queue. That is, a queue that
7750 7764 * has specified a struio() type.
7751 7765 */
7752 7766 static void
7753 7767 strsetuio(stdata_t *stp)
7754 7768 {
7755 7769 queue_t *wrq;
7756 7770
7757 7771 if (stp->sd_flag & STPLEX) {
7758 7772 /*
7759 7773 * Not streamhead, but a mux, so no Synchronous STREAMS.
7760 7774 */
7761 7775 stp->sd_struiowrq = NULL;
7762 7776 stp->sd_struiordq = NULL;
7763 7777 return;
7764 7778 }
7765 7779 /*
7766 7780 * Scan the write queue(s) while synchronous
7767 7781 * until we find a qinfo uio type specified.
7768 7782 */
7769 7783 wrq = stp->sd_wrq->q_next;
7770 7784 while (wrq) {
7771 7785 if (wrq->q_struiot == STRUIOT_NONE) {
7772 7786 wrq = 0;
7773 7787 break;
7774 7788 }
7775 7789 if (wrq->q_struiot != STRUIOT_DONTCARE)
7776 7790 break;
7777 7791 if (! _SAMESTR(wrq)) {
7778 7792 wrq = 0;
7779 7793 break;
7780 7794 }
7781 7795 wrq = wrq->q_next;
7782 7796 }
7783 7797 stp->sd_struiowrq = wrq;
7784 7798 /*
7785 7799 * Scan the read queue(s) while synchronous
7786 7800 * until we find a qinfo uio type specified.
7787 7801 */
7788 7802 wrq = stp->sd_wrq->q_next;
7789 7803 while (wrq) {
7790 7804 if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7791 7805 wrq = 0;
7792 7806 break;
7793 7807 }
7794 7808 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7795 7809 break;
7796 7810 if (! _SAMESTR(wrq)) {
7797 7811 wrq = 0;
7798 7812 break;
7799 7813 }
7800 7814 wrq = wrq->q_next;
7801 7815 }
7802 7816 stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7803 7817 }
7804 7818
7805 7819 /*
7806 7820 * pass_wput, unblocks the passthru queues, so that
7807 7821 * messages can arrive at muxs lower read queue, before
7808 7822 * I_LINK/I_UNLINK is acked/nacked.
7809 7823 */
7810 7824 static void
7811 7825 pass_wput(queue_t *q, mblk_t *mp)
7812 7826 {
7813 7827 syncq_t *sq;
7814 7828
7815 7829 sq = _RD(q)->q_syncq;
7816 7830 if (sq->sq_flags & SQ_BLOCKED)
7817 7831 unblocksq(sq, SQ_BLOCKED, 0);
7818 7832 putnext(q, mp);
7819 7833 }
7820 7834
7821 7835 /*
7822 7836 * Set up queues for the link/unlink.
7823 7837 * Create a new queue and block it and then insert it
7824 7838 * below the stream head on the lower stream.
7825 7839 * This prevents any messages from arriving during the setq
7826 7840 * as well as while the mux is processing the LINK/I_UNLINK.
7827 7841 * The blocked passq is unblocked once the LINK/I_UNLINK has
7828 7842 * been acked or nacked or if a message is generated and sent
7829 7843 * down muxs write put procedure.
7830 7844 * See pass_wput().
7831 7845 *
7832 7846 * After the new queue is inserted, all messages coming from below are
7833 7847 * blocked. The call to strlock will ensure that all activity in the stream head
7834 7848 * read queue syncq is stopped (sq_count drops to zero).
7835 7849 */
7836 7850 static queue_t *
7837 7851 link_addpassthru(stdata_t *stpdown)
7838 7852 {
7839 7853 queue_t *passq;
7840 7854 sqlist_t sqlist;
7841 7855
7842 7856 passq = allocq();
7843 7857 STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7844 7858 /* setq might sleep in allocator - avoid holding locks. */
7845 7859 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7846 7860 SQ_CI|SQ_CO, B_FALSE);
7847 7861 claimq(passq);
7848 7862 blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7849 7863 insertq(STREAM(passq), passq);
7850 7864
7851 7865 /*
7852 7866 * Use strlock() to wait for the stream head sq_count to drop to zero
7853 7867 * since we are going to change q_ptr in the stream head. Note that
7854 7868 * insertq() doesn't wait for any syncq counts to drop to zero.
7855 7869 */
7856 7870 sqlist.sqlist_head = NULL;
7857 7871 sqlist.sqlist_index = 0;
7858 7872 sqlist.sqlist_size = sizeof (sqlist_t);
7859 7873 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7860 7874 strlock(stpdown, &sqlist);
7861 7875 strunlock(stpdown, &sqlist);
7862 7876
7863 7877 releaseq(passq);
7864 7878 return (passq);
7865 7879 }
7866 7880
7867 7881 /*
7868 7882 * Let messages flow up into the mux by removing
7869 7883 * the passq.
7870 7884 */
7871 7885 static void
7872 7886 link_rempassthru(queue_t *passq)
7873 7887 {
7874 7888 claimq(passq);
7875 7889 removeq(passq);
7876 7890 releaseq(passq);
7877 7891 freeq(passq);
7878 7892 }
7879 7893
7880 7894 /*
7881 7895 * Wait for the condition variable pointed to by `cvp' to be signaled,
7882 7896 * or for `tim' milliseconds to elapse, whichever comes first. If `tim'
7883 7897 * is negative, then there is no time limit. If `nosigs' is non-zero,
7884 7898 * then the wait will be non-interruptible.
7885 7899 *
7886 7900 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7887 7901 */
7888 7902 clock_t
7889 7903 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7890 7904 {
7891 7905 clock_t ret;
7892 7906
7893 7907 if (tim < 0) {
7894 7908 if (nosigs) {
7895 7909 cv_wait(cvp, mp);
7896 7910 ret = 1;
7897 7911 } else {
7898 7912 ret = cv_wait_sig(cvp, mp);
7899 7913 }
7900 7914 } else if (tim > 0) {
7901 7915 /*
7902 7916 * convert milliseconds to clock ticks
7903 7917 */
7904 7918 if (nosigs) {
7905 7919 ret = cv_reltimedwait(cvp, mp,
7906 7920 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7907 7921 } else {
7908 7922 ret = cv_reltimedwait_sig(cvp, mp,
7909 7923 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7910 7924 }
7911 7925 } else {
7912 7926 ret = -1;
7913 7927 }
7914 7928 return (ret);
7915 7929 }
7916 7930
7917 7931 /*
7918 7932 * Wait until the stream head can determine if it is at the mark but
7919 7933 * don't wait forever to prevent a race condition between the "mark" state
7920 7934 * in the stream head and any mark state in the caller/user of this routine.
7921 7935 *
7922 7936 * This is used by sockets and for a socket it would be incorrect
7923 7937 * to return a failure for SIOCATMARK when there is no data in the receive
7924 7938 * queue and the marked urgent data is traveling up the stream.
7925 7939 *
7926 7940 * This routine waits until the mark is known by waiting for one of these
7927 7941 * three events:
7928 7942 * The stream head read queue becoming non-empty (including an EOF).
7929 7943 * The STRATMARK flag being set (due to a MSGMARKNEXT message).
7930 7944 * The STRNOTATMARK flag being set (which indicates that the transport
7931 7945 * has sent a MSGNOTMARKNEXT message to indicate that it is not at
7932 7946 * the mark).
7933 7947 *
7934 7948 * The routine returns 1 if the stream is at the mark; 0 if it can
7935 7949 * be determined that the stream is not at the mark.
7936 7950 * If the wait times out and it can't determine
7937 7951 * whether or not the stream might be at the mark the routine will return -1.
7938 7952 *
7939 7953 * Note: This routine should only be used when a mark is pending i.e.,
7940 7954 * in the socket case the SIGURG has been posted.
7941 7955 * Note2: This can not wakeup just because synchronous streams indicate
7942 7956 * that data is available since it is not possible to use the synchronous
7943 7957 * streams interfaces to determine the b_flag value for the data queued below
7944 7958 * the stream head.
7945 7959 */
7946 7960 int
7947 7961 strwaitmark(vnode_t *vp)
7948 7962 {
7949 7963 struct stdata *stp = vp->v_stream;
7950 7964 queue_t *rq = _RD(stp->sd_wrq);
7951 7965 int mark;
7952 7966
7953 7967 mutex_enter(&stp->sd_lock);
7954 7968 while (rq->q_first == NULL &&
7955 7969 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7956 7970 stp->sd_flag |= RSLEEP;
7957 7971
7958 7972 /* Wait for 100 milliseconds for any state change. */
7959 7973 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7960 7974 mutex_exit(&stp->sd_lock);
7961 7975 return (-1);
7962 7976 }
7963 7977 }
7964 7978 if (stp->sd_flag & STRATMARK)
7965 7979 mark = 1;
7966 7980 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7967 7981 mark = 1;
7968 7982 else
7969 7983 mark = 0;
7970 7984
7971 7985 mutex_exit(&stp->sd_lock);
7972 7986 return (mark);
7973 7987 }
7974 7988
7975 7989 /*
7976 7990 * Set a read side error. If persist is set change the socket error
7977 7991 * to persistent. If errfunc is set install the function as the exported
7978 7992 * error handler.
7979 7993 */
7980 7994 void
7981 7995 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
7982 7996 {
7983 7997 struct stdata *stp = vp->v_stream;
7984 7998
7985 7999 mutex_enter(&stp->sd_lock);
7986 8000 stp->sd_rerror = error;
7987 8001 if (error == 0 && errfunc == NULL)
7988 8002 stp->sd_flag &= ~STRDERR;
7989 8003 else
7990 8004 stp->sd_flag |= STRDERR;
7991 8005 if (persist) {
7992 8006 stp->sd_flag &= ~STRDERRNONPERSIST;
7993 8007 } else {
7994 8008 stp->sd_flag |= STRDERRNONPERSIST;
7995 8009 }
7996 8010 stp->sd_rderrfunc = errfunc;
7997 8011 if (error != 0 || errfunc != NULL) {
7998 8012 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
7999 8013 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8000 8014 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8001 8015
8002 8016 mutex_exit(&stp->sd_lock);
8003 8017 pollwakeup(&stp->sd_pollist, POLLERR);
8004 8018 mutex_enter(&stp->sd_lock);
8005 8019
8006 8020 if (stp->sd_sigflags & S_ERROR)
8007 8021 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8008 8022 }
8009 8023 mutex_exit(&stp->sd_lock);
8010 8024 }
8011 8025
8012 8026 /*
8013 8027 * Set a write side error. If persist is set change the socket error
8014 8028 * to persistent.
8015 8029 */
8016 8030 void
8017 8031 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8018 8032 {
8019 8033 struct stdata *stp = vp->v_stream;
8020 8034
8021 8035 mutex_enter(&stp->sd_lock);
8022 8036 stp->sd_werror = error;
8023 8037 if (error == 0 && errfunc == NULL)
8024 8038 stp->sd_flag &= ~STWRERR;
8025 8039 else
8026 8040 stp->sd_flag |= STWRERR;
8027 8041 if (persist) {
8028 8042 stp->sd_flag &= ~STWRERRNONPERSIST;
8029 8043 } else {
8030 8044 stp->sd_flag |= STWRERRNONPERSIST;
8031 8045 }
8032 8046 stp->sd_wrerrfunc = errfunc;
8033 8047 if (error != 0 || errfunc != NULL) {
8034 8048 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
8035 8049 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8036 8050 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8037 8051
8038 8052 mutex_exit(&stp->sd_lock);
8039 8053 pollwakeup(&stp->sd_pollist, POLLERR);
8040 8054 mutex_enter(&stp->sd_lock);
8041 8055
8042 8056 if (stp->sd_sigflags & S_ERROR)
8043 8057 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8044 8058 }
8045 8059 mutex_exit(&stp->sd_lock);
8046 8060 }
8047 8061
8048 8062 /*
8049 8063 * Make the stream return 0 (EOF) when all data has been read.
8050 8064 * No effect on write side.
8051 8065 */
8052 8066 void
8053 8067 strseteof(vnode_t *vp, int eof)
8054 8068 {
8055 8069 struct stdata *stp = vp->v_stream;
8056 8070
8057 8071 mutex_enter(&stp->sd_lock);
8058 8072 if (!eof) {
8059 8073 stp->sd_flag &= ~STREOF;
8060 8074 mutex_exit(&stp->sd_lock);
8061 8075 return;
8062 8076 }
8063 8077 stp->sd_flag |= STREOF;
8064 8078 if (stp->sd_flag & RSLEEP) {
8065 8079 stp->sd_flag &= ~RSLEEP;
8066 8080 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8067 8081 }
8068 8082
8069 8083 mutex_exit(&stp->sd_lock);
8070 8084 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8071 8085 mutex_enter(&stp->sd_lock);
8072 8086
8073 8087 if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8074 8088 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8075 8089 mutex_exit(&stp->sd_lock);
8076 8090 }
8077 8091
8078 8092 void
8079 8093 strflushrq(vnode_t *vp, int flag)
|
↓ open down ↓ |
5603 lines elided |
↑ open up ↑ |
8080 8094 {
8081 8095 struct stdata *stp = vp->v_stream;
8082 8096
8083 8097 mutex_enter(&stp->sd_lock);
8084 8098 flushq(_RD(stp->sd_wrq), flag);
8085 8099 mutex_exit(&stp->sd_lock);
8086 8100 }
8087 8101
8088 8102 void
8089 8103 strsetrputhooks(vnode_t *vp, uint_t flags,
8090 - msgfunc_t protofunc, msgfunc_t miscfunc)
8104 + msgfunc_t protofunc, msgfunc_t miscfunc)
8091 8105 {
8092 8106 struct stdata *stp = vp->v_stream;
8093 8107
8094 8108 mutex_enter(&stp->sd_lock);
8095 8109
8096 8110 if (protofunc == NULL)
8097 8111 stp->sd_rprotofunc = strrput_proto;
8098 8112 else
8099 8113 stp->sd_rprotofunc = protofunc;
8100 8114
8101 8115 if (miscfunc == NULL)
8102 8116 stp->sd_rmiscfunc = strrput_misc;
8103 8117 else
8104 8118 stp->sd_rmiscfunc = miscfunc;
8105 8119
8106 8120 if (flags & SH_CONSOL_DATA)
8107 8121 stp->sd_rput_opt |= SR_CONSOL_DATA;
8108 8122 else
8109 8123 stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8110 8124
8111 8125 if (flags & SH_SIGALLDATA)
8112 8126 stp->sd_rput_opt |= SR_SIGALLDATA;
8113 8127 else
8114 8128 stp->sd_rput_opt &= ~SR_SIGALLDATA;
8115 8129
8116 8130 if (flags & SH_IGN_ZEROLEN)
8117 8131 stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8118 8132 else
8119 8133 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8120 8134
8121 8135 mutex_exit(&stp->sd_lock);
8122 8136 }
8123 8137
8124 8138 void
8125 8139 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8126 8140 {
8127 8141 struct stdata *stp = vp->v_stream;
8128 8142
8129 8143 mutex_enter(&stp->sd_lock);
8130 8144 stp->sd_closetime = closetime;
8131 8145
8132 8146 if (flags & SH_SIGPIPE)
8133 8147 stp->sd_wput_opt |= SW_SIGPIPE;
8134 8148 else
8135 8149 stp->sd_wput_opt &= ~SW_SIGPIPE;
8136 8150 if (flags & SH_RECHECK_ERR)
8137 8151 stp->sd_wput_opt |= SW_RECHECK_ERR;
8138 8152 else
8139 8153 stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8140 8154
8141 8155 mutex_exit(&stp->sd_lock);
8142 8156 }
8143 8157
8144 8158 void
8145 8159 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8146 8160 {
8147 8161 struct stdata *stp = vp->v_stream;
8148 8162
8149 8163 mutex_enter(&stp->sd_lock);
8150 8164
8151 8165 stp->sd_rputdatafunc = rdatafunc;
8152 8166 stp->sd_wputdatafunc = wdatafunc;
8153 8167
8154 8168 mutex_exit(&stp->sd_lock);
8155 8169 }
8156 8170
8157 8171 /* Used within framework when the queue is already locked */
8158 8172 void
8159 8173 qenable_locked(queue_t *q)
8160 8174 {
8161 8175 stdata_t *stp = STREAM(q);
8162 8176
8163 8177 ASSERT(MUTEX_HELD(QLOCK(q)));
8164 8178
8165 8179 if (!q->q_qinfo->qi_srvp)
8166 8180 return;
8167 8181
8168 8182 /*
8169 8183 * Do not place on run queue if already enabled or closing.
8170 8184 */
8171 8185 if (q->q_flag & (QWCLOSE|QENAB))
8172 8186 return;
8173 8187
8174 8188 /*
8175 8189 * mark queue enabled and place on run list if it is not already being
8176 8190 * serviced. If it is serviced, the runservice() function will detect
8177 8191 * that QENAB is set and call service procedure before clearing
8178 8192 * QINSERVICE flag.
8179 8193 */
8180 8194 q->q_flag |= QENAB;
8181 8195 if (q->q_flag & QINSERVICE)
8182 8196 return;
8183 8197
8184 8198 /* Record the time of qenable */
8185 8199 q->q_qtstamp = ddi_get_lbolt();
8186 8200
8187 8201 /*
8188 8202 * Put the queue in the stp list and schedule it for background
8189 8203 * processing if it is not already scheduled or if stream head does not
8190 8204 * intent to process it in the foreground later by setting
8191 8205 * STRS_WILLSERVICE flag.
8192 8206 */
8193 8207 mutex_enter(&stp->sd_qlock);
8194 8208 /*
8195 8209 * If there are already something on the list, stp flags should show
8196 8210 * intention to drain it.
8197 8211 */
8198 8212 IMPLY(STREAM_NEEDSERVICE(stp),
8199 8213 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8200 8214
8201 8215 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8202 8216 stp->sd_nqueues++;
8203 8217
8204 8218 /*
8205 8219 * If no one will drain this stream we are the first producer and
8206 8220 * need to schedule it for background thread.
8207 8221 */
8208 8222 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8209 8223 /*
8210 8224 * No one will service this stream later, so we have to
8211 8225 * schedule it now.
8212 8226 */
8213 8227 STRSTAT(stenables);
8214 8228 stp->sd_svcflags |= STRS_SCHEDULED;
8215 8229 stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8216 8230 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8217 8231
8218 8232 if (stp->sd_servid == NULL) {
8219 8233 /*
8220 8234 * Task queue failed so fail over to the backup
8221 8235 * servicing thread.
8222 8236 */
8223 8237 STRSTAT(taskqfails);
8224 8238 /*
8225 8239 * It is safe to clear STRS_SCHEDULED flag because it
8226 8240 * was set by this thread above.
8227 8241 */
8228 8242 stp->sd_svcflags &= ~STRS_SCHEDULED;
8229 8243
8230 8244 /*
8231 8245 * Failover scheduling is protected by service_queue
8232 8246 * lock.
8233 8247 */
8234 8248 mutex_enter(&service_queue);
8235 8249 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8236 8250 ASSERT(q->q_link == NULL);
8237 8251 /*
8238 8252 * Append the queue to qhead/qtail list.
8239 8253 */
8240 8254 if (qhead == NULL)
8241 8255 qhead = q;
8242 8256 else
8243 8257 qtail->q_link = q;
8244 8258 qtail = q;
8245 8259 /*
8246 8260 * Clear stp queue list.
8247 8261 */
8248 8262 stp->sd_qhead = stp->sd_qtail = NULL;
8249 8263 stp->sd_nqueues = 0;
8250 8264 /*
8251 8265 * Wakeup background queue processing thread.
8252 8266 */
8253 8267 cv_signal(&services_to_run);
8254 8268 mutex_exit(&service_queue);
8255 8269 }
8256 8270 }
8257 8271 mutex_exit(&stp->sd_qlock);
8258 8272 }
8259 8273
8260 8274 static void
8261 8275 queue_service(queue_t *q)
8262 8276 {
8263 8277 /*
8264 8278 * The queue in the list should have
8265 8279 * QENAB flag set and should not have
8266 8280 * QINSERVICE flag set. QINSERVICE is
8267 8281 * set when the queue is dequeued and
8268 8282 * qenable_locked doesn't enqueue a
8269 8283 * queue with QINSERVICE set.
8270 8284 */
8271 8285
8272 8286 ASSERT(!(q->q_flag & QINSERVICE));
8273 8287 ASSERT((q->q_flag & QENAB));
8274 8288 mutex_enter(QLOCK(q));
8275 8289 q->q_flag &= ~QENAB;
8276 8290 q->q_flag |= QINSERVICE;
8277 8291 mutex_exit(QLOCK(q));
8278 8292 runservice(q);
8279 8293 }
8280 8294
8281 8295 static void
8282 8296 syncq_service(syncq_t *sq)
8283 8297 {
8284 8298 STRSTAT(syncqservice);
8285 8299 mutex_enter(SQLOCK(sq));
8286 8300 ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8287 8301 ASSERT(sq->sq_servcount != 0);
8288 8302 ASSERT(sq->sq_next == NULL);
8289 8303
8290 8304 /* if we came here from the background thread, clear the flag */
8291 8305 if (sq->sq_svcflags & SQ_BGTHREAD)
8292 8306 sq->sq_svcflags &= ~SQ_BGTHREAD;
8293 8307
8294 8308 /* let drain_syncq know that it's being called in the background */
8295 8309 sq->sq_svcflags |= SQ_SERVICE;
8296 8310 drain_syncq(sq);
8297 8311 }
8298 8312
8299 8313 static void
8300 8314 qwriter_outer_service(syncq_t *outer)
8301 8315 {
8302 8316 /*
8303 8317 * Note that SQ_WRITER is used on the outer perimeter
8304 8318 * to signal that a qwriter(OUTER) is either investigating
8305 8319 * running or that it is actually running a function.
8306 8320 */
8307 8321 outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8308 8322
8309 8323 /*
8310 8324 * All inner syncq are empty and have SQ_WRITER set
8311 8325 * to block entering the outer perimeter.
8312 8326 *
8313 8327 * We do not need to explicitly call write_now since
8314 8328 * outer_exit does it for us.
8315 8329 */
8316 8330 outer_exit(outer);
8317 8331 }
8318 8332
8319 8333 static void
8320 8334 mblk_free(mblk_t *mp)
8321 8335 {
8322 8336 dblk_t *dbp = mp->b_datap;
8323 8337 frtn_t *frp = dbp->db_frtnp;
8324 8338
8325 8339 mp->b_next = NULL;
8326 8340 if (dbp->db_fthdr != NULL)
8327 8341 str_ftfree(dbp);
8328 8342
8329 8343 ASSERT(dbp->db_fthdr == NULL);
8330 8344 frp->free_func(frp->free_arg);
8331 8345 ASSERT(dbp->db_mblk == mp);
8332 8346
8333 8347 if (dbp->db_credp != NULL) {
8334 8348 crfree(dbp->db_credp);
8335 8349 dbp->db_credp = NULL;
8336 8350 }
8337 8351 dbp->db_cpid = -1;
8338 8352 dbp->db_struioflag = 0;
8339 8353 dbp->db_struioun.cksum.flags = 0;
8340 8354
8341 8355 kmem_cache_free(dbp->db_cache, dbp);
8342 8356 }
8343 8357
8344 8358 /*
8345 8359 * Background processing of the stream queue list.
8346 8360 */
8347 8361 static void
8348 8362 stream_service(stdata_t *stp)
8349 8363 {
8350 8364 queue_t *q;
8351 8365
8352 8366 mutex_enter(&stp->sd_qlock);
8353 8367
8354 8368 STR_SERVICE(stp, q);
8355 8369
8356 8370 stp->sd_svcflags &= ~STRS_SCHEDULED;
8357 8371 stp->sd_servid = NULL;
8358 8372 cv_signal(&stp->sd_qcv);
8359 8373 mutex_exit(&stp->sd_qlock);
8360 8374 }
8361 8375
8362 8376 /*
8363 8377 * Foreground processing of the stream queue list.
8364 8378 */
8365 8379 void
8366 8380 stream_runservice(stdata_t *stp)
8367 8381 {
8368 8382 queue_t *q;
8369 8383
8370 8384 mutex_enter(&stp->sd_qlock);
8371 8385 STRSTAT(rservice);
8372 8386 /*
8373 8387 * We are going to drain this stream queue list, so qenable_locked will
8374 8388 * not schedule it until we finish.
8375 8389 */
8376 8390 stp->sd_svcflags |= STRS_WILLSERVICE;
8377 8391
8378 8392 STR_SERVICE(stp, q);
8379 8393
8380 8394 stp->sd_svcflags &= ~STRS_WILLSERVICE;
8381 8395 mutex_exit(&stp->sd_qlock);
8382 8396 /*
8383 8397 * Help backup background thread to drain the qhead/qtail list.
8384 8398 */
8385 8399 while (qhead != NULL) {
8386 8400 STRSTAT(qhelps);
8387 8401 mutex_enter(&service_queue);
8388 8402 DQ(q, qhead, qtail, q_link);
8389 8403 mutex_exit(&service_queue);
8390 8404 if (q != NULL)
8391 8405 queue_service(q);
8392 8406 }
8393 8407 }
8394 8408
8395 8409 void
8396 8410 stream_willservice(stdata_t *stp)
8397 8411 {
8398 8412 mutex_enter(&stp->sd_qlock);
8399 8413 stp->sd_svcflags |= STRS_WILLSERVICE;
8400 8414 mutex_exit(&stp->sd_qlock);
8401 8415 }
8402 8416
8403 8417 /*
8404 8418 * Replace the cred currently in the mblk with a different one.
8405 8419 * Also update db_cpid.
8406 8420 */
8407 8421 void
8408 8422 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8409 8423 {
8410 8424 dblk_t *dbp = mp->b_datap;
8411 8425 cred_t *ocr = dbp->db_credp;
8412 8426
8413 8427 ASSERT(cr != NULL);
8414 8428
8415 8429 if (cr != ocr) {
8416 8430 crhold(dbp->db_credp = cr);
8417 8431 if (ocr != NULL)
8418 8432 crfree(ocr);
8419 8433 }
8420 8434 /* Don't overwrite with NOPID */
8421 8435 if (cpid != NOPID)
8422 8436 dbp->db_cpid = cpid;
8423 8437 }
8424 8438
8425 8439 /*
8426 8440 * If the src message has a cred, then replace the cred currently in the mblk
8427 8441 * with it.
8428 8442 * Also update db_cpid.
8429 8443 */
8430 8444 void
8431 8445 mblk_copycred(mblk_t *mp, const mblk_t *src)
8432 8446 {
8433 8447 dblk_t *dbp = mp->b_datap;
8434 8448 cred_t *cr, *ocr;
8435 8449 pid_t cpid;
8436 8450
8437 8451 cr = msg_getcred(src, &cpid);
8438 8452 if (cr == NULL)
8439 8453 return;
8440 8454
8441 8455 ocr = dbp->db_credp;
8442 8456 if (cr != ocr) {
8443 8457 crhold(dbp->db_credp = cr);
8444 8458 if (ocr != NULL)
8445 8459 crfree(ocr);
8446 8460 }
8447 8461 /* Don't overwrite with NOPID */
8448 8462 if (cpid != NOPID)
8449 8463 dbp->db_cpid = cpid;
8450 8464 }
8451 8465
8452 8466 int
8453 8467 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8454 8468 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8455 8469 uint32_t flags, int km_flags)
8456 8470 {
8457 8471 int rc = 0;
8458 8472
8459 8473 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8460 8474 if (mp->b_datap->db_type == M_DATA) {
8461 8475 /* Associate values for M_DATA type */
8462 8476 DB_CKSUMSTART(mp) = (intptr_t)start;
8463 8477 DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8464 8478 DB_CKSUMEND(mp) = (intptr_t)end;
8465 8479 DB_CKSUMFLAGS(mp) = flags;
8466 8480 DB_CKSUM16(mp) = (uint16_t)value;
8467 8481
8468 8482 } else {
8469 8483 pattrinfo_t pa_info;
8470 8484
8471 8485 ASSERT(mmd != NULL);
8472 8486
8473 8487 pa_info.type = PATTR_HCKSUM;
8474 8488 pa_info.len = sizeof (pattr_hcksum_t);
8475 8489
8476 8490 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8477 8491 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8478 8492
8479 8493 hck->hcksum_start_offset = start;
8480 8494 hck->hcksum_stuff_offset = stuff;
8481 8495 hck->hcksum_end_offset = end;
8482 8496 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8483 8497 hck->hcksum_flags = flags;
8484 8498 } else {
8485 8499 rc = -1;
8486 8500 }
8487 8501 }
8488 8502 return (rc);
8489 8503 }
8490 8504
8491 8505 void
8492 8506 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8493 8507 uint32_t *start, uint32_t *stuff, uint32_t *end,
8494 8508 uint32_t *value, uint32_t *flags)
8495 8509 {
8496 8510 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8497 8511 if (mp->b_datap->db_type == M_DATA) {
8498 8512 if (flags != NULL) {
8499 8513 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8500 8514 if ((*flags & (HCK_PARTIALCKSUM |
8501 8515 HCK_FULLCKSUM)) != 0) {
8502 8516 if (value != NULL)
8503 8517 *value = (uint32_t)DB_CKSUM16(mp);
8504 8518 if ((*flags & HCK_PARTIALCKSUM) != 0) {
8505 8519 if (start != NULL)
8506 8520 *start =
8507 8521 (uint32_t)DB_CKSUMSTART(mp);
8508 8522 if (stuff != NULL)
8509 8523 *stuff =
8510 8524 (uint32_t)DB_CKSUMSTUFF(mp);
8511 8525 if (end != NULL)
8512 8526 *end =
8513 8527 (uint32_t)DB_CKSUMEND(mp);
8514 8528 }
8515 8529 }
8516 8530 }
8517 8531 } else {
8518 8532 pattrinfo_t hck_attr = {PATTR_HCKSUM};
8519 8533
8520 8534 ASSERT(mmd != NULL);
8521 8535
8522 8536 /* get hardware checksum attribute */
8523 8537 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8524 8538 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8525 8539
8526 8540 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8527 8541 if (flags != NULL)
8528 8542 *flags = hck->hcksum_flags;
8529 8543 if (start != NULL)
8530 8544 *start = hck->hcksum_start_offset;
8531 8545 if (stuff != NULL)
8532 8546 *stuff = hck->hcksum_stuff_offset;
8533 8547 if (end != NULL)
8534 8548 *end = hck->hcksum_end_offset;
8535 8549 if (value != NULL)
8536 8550 *value = (uint32_t)
8537 8551 hck->hcksum_cksum_val.inet_cksum;
8538 8552 }
8539 8553 }
8540 8554 }
8541 8555
8542 8556 void
8543 8557 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8544 8558 {
8545 8559 ASSERT(DB_TYPE(mp) == M_DATA);
8546 8560 ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8547 8561
8548 8562 /* Set the flags */
8549 8563 DB_LSOFLAGS(mp) |= flags;
8550 8564 DB_LSOMSS(mp) = mss;
8551 8565 }
8552 8566
8553 8567 void
8554 8568 lso_info_cleanup(mblk_t *mp)
8555 8569 {
8556 8570 ASSERT(DB_TYPE(mp) == M_DATA);
8557 8571
8558 8572 /* Clear the flags */
8559 8573 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8560 8574 DB_LSOMSS(mp) = 0;
8561 8575 }
8562 8576
8563 8577 /*
8564 8578 * Checksum buffer *bp for len bytes with psum partial checksum,
8565 8579 * or 0 if none, and return the 16 bit partial checksum.
8566 8580 */
8567 8581 unsigned
8568 8582 bcksum(uchar_t *bp, int len, unsigned int psum)
8569 8583 {
8570 8584 int odd = len & 1;
8571 8585 extern unsigned int ip_ocsum();
8572 8586
8573 8587 if (((intptr_t)bp & 1) == 0 && !odd) {
8574 8588 /*
8575 8589 * Bp is 16 bit aligned and len is multiple of 16 bit word.
8576 8590 */
8577 8591 return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8578 8592 }
8579 8593 if (((intptr_t)bp & 1) != 0) {
8580 8594 /*
8581 8595 * Bp isn't 16 bit aligned.
8582 8596 */
8583 8597 unsigned int tsum;
8584 8598
8585 8599 #ifdef _LITTLE_ENDIAN
8586 8600 psum += *bp;
8587 8601 #else
8588 8602 psum += *bp << 8;
8589 8603 #endif
8590 8604 len--;
8591 8605 bp++;
8592 8606 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8593 8607 psum += (tsum << 8) & 0xffff | (tsum >> 8);
8594 8608 if (len & 1) {
8595 8609 bp += len - 1;
8596 8610 #ifdef _LITTLE_ENDIAN
8597 8611 psum += *bp << 8;
8598 8612 #else
8599 8613 psum += *bp;
8600 8614 #endif
8601 8615 }
8602 8616 } else {
8603 8617 /*
8604 8618 * Bp is 16 bit aligned.
8605 8619 */
8606 8620 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8607 8621 if (odd) {
8608 8622 bp += len - 1;
8609 8623 #ifdef _LITTLE_ENDIAN
8610 8624 psum += *bp;
8611 8625 #else
8612 8626 psum += *bp << 8;
8613 8627 #endif
8614 8628 }
8615 8629 }
8616 8630 /*
8617 8631 * Normalize psum to 16 bits before returning the new partial
8618 8632 * checksum. The max psum value before normalization is 0x3FDFE.
8619 8633 */
8620 8634 return ((psum >> 16) + (psum & 0xFFFF));
8621 8635 }
8622 8636
8623 8637 boolean_t
8624 8638 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8625 8639 {
8626 8640 boolean_t rc;
8627 8641
8628 8642 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8629 8643 if (DB_TYPE(mp) == M_DATA) {
8630 8644 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8631 8645 } else {
8632 8646 pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8633 8647
8634 8648 ASSERT(mmd != NULL);
8635 8649 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8636 8650 }
8637 8651 return (rc);
8638 8652 }
8639 8653
8640 8654 void
8641 8655 freemsgchain(mblk_t *mp)
8642 8656 {
8643 8657 mblk_t *next;
8644 8658
8645 8659 while (mp != NULL) {
8646 8660 next = mp->b_next;
8647 8661 mp->b_next = NULL;
8648 8662
8649 8663 freemsg(mp);
8650 8664 mp = next;
8651 8665 }
8652 8666 }
8653 8667
8654 8668 mblk_t *
8655 8669 copymsgchain(mblk_t *mp)
8656 8670 {
8657 8671 mblk_t *nmp = NULL;
8658 8672 mblk_t **nmpp = &nmp;
8659 8673
8660 8674 for (; mp != NULL; mp = mp->b_next) {
8661 8675 if ((*nmpp = copymsg(mp)) == NULL) {
8662 8676 freemsgchain(nmp);
8663 8677 return (NULL);
8664 8678 }
8665 8679
8666 8680 nmpp = &((*nmpp)->b_next);
8667 8681 }
8668 8682
8669 8683 return (nmp);
8670 8684 }
8671 8685
8672 8686 /* NOTE: Do not add code after this point. */
8673 8687 #undef QLOCK
8674 8688
8675 8689 /*
8676 8690 * Replacement for QLOCK macro for those that can't use it.
8677 8691 */
8678 8692 kmutex_t *
8679 8693 QLOCK(queue_t *q)
8680 8694 {
8681 8695 return (&(q)->q_lock);
8682 8696 }
8683 8697
8684 8698 /*
8685 8699 * Dummy runqueues/queuerun functions functions for backwards compatibility.
8686 8700 */
8687 8701 #undef runqueues
8688 8702 void
8689 8703 runqueues(void)
8690 8704 {
8691 8705 }
8692 8706
8693 8707 #undef queuerun
8694 8708 void
8695 8709 queuerun(void)
8696 8710 {
8697 8711 }
8698 8712
8699 8713 /*
8700 8714 * Initialize the STR stack instance, which tracks autopush and persistent
8701 8715 * links.
8702 8716 */
8703 8717 /* ARGSUSED */
8704 8718 static void *
8705 8719 str_stack_init(netstackid_t stackid, netstack_t *ns)
8706 8720 {
8707 8721 str_stack_t *ss;
8708 8722 int i;
8709 8723
8710 8724 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8711 8725 ss->ss_netstack = ns;
8712 8726
8713 8727 /*
8714 8728 * set up autopush
8715 8729 */
8716 8730 sad_initspace(ss);
8717 8731
8718 8732 /*
8719 8733 * set up mux_node structures.
8720 8734 */
8721 8735 ss->ss_devcnt = devcnt; /* In case it should change before free */
8722 8736 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8723 8737 ss->ss_devcnt), KM_SLEEP);
8724 8738 for (i = 0; i < ss->ss_devcnt; i++)
8725 8739 ss->ss_mux_nodes[i].mn_imaj = i;
8726 8740 return (ss);
8727 8741 }
8728 8742
8729 8743 /*
8730 8744 * Note: run at zone shutdown and not destroy so that the PLINKs are
8731 8745 * gone by the time other cleanup happens from the destroy callbacks.
8732 8746 */
8733 8747 static void
8734 8748 str_stack_shutdown(netstackid_t stackid, void *arg)
8735 8749 {
8736 8750 str_stack_t *ss = (str_stack_t *)arg;
8737 8751 int i;
8738 8752 cred_t *cr;
8739 8753
8740 8754 cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8741 8755 ASSERT(cr != NULL);
8742 8756
8743 8757 /* Undo all the I_PLINKs for this zone */
8744 8758 for (i = 0; i < ss->ss_devcnt; i++) {
8745 8759 struct mux_edge *ep;
8746 8760 ldi_handle_t lh;
8747 8761 ldi_ident_t li;
8748 8762 int ret;
8749 8763 int rval;
8750 8764 dev_t rdev;
8751 8765
8752 8766 ep = ss->ss_mux_nodes[i].mn_outp;
8753 8767 if (ep == NULL)
8754 8768 continue;
8755 8769 ret = ldi_ident_from_major((major_t)i, &li);
8756 8770 if (ret != 0) {
8757 8771 continue;
8758 8772 }
8759 8773 rdev = ep->me_dev;
8760 8774 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8761 8775 cr, &lh, li);
8762 8776 if (ret != 0) {
8763 8777 ldi_ident_release(li);
8764 8778 continue;
8765 8779 }
8766 8780
8767 8781 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8768 8782 cr, &rval);
8769 8783 if (ret) {
8770 8784 (void) ldi_close(lh, FREAD|FWRITE, cr);
8771 8785 ldi_ident_release(li);
8772 8786 continue;
8773 8787 }
8774 8788 (void) ldi_close(lh, FREAD|FWRITE, cr);
8775 8789
8776 8790 /* Close layered handles */
8777 8791 ldi_ident_release(li);
8778 8792 }
8779 8793 crfree(cr);
8780 8794
8781 8795 sad_freespace(ss);
8782 8796
8783 8797 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8784 8798 ss->ss_mux_nodes = NULL;
8785 8799 }
8786 8800
8787 8801 /*
8788 8802 * Free the structure; str_stack_shutdown did the other cleanup work.
8789 8803 */
8790 8804 /* ARGSUSED */
8791 8805 static void
8792 8806 str_stack_fini(netstackid_t stackid, void *arg)
8793 8807 {
8794 8808 str_stack_t *ss = (str_stack_t *)arg;
8795 8809
8796 8810 kmem_free(ss, sizeof (*ss));
8797 8811 }
|
↓ open down ↓ |
697 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX