1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 /* All Rights Reserved */
23
24
25 /*
26 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 * Copyright (c) 2016 by Delphix. All rights reserved.
29 */
30
31 #include <sys/types.h>
32 #include <sys/sysmacros.h>
33 #include <sys/param.h>
34 #include <sys/errno.h>
35 #include <sys/signal.h>
36 #include <sys/proc.h>
37 #include <sys/conf.h>
38 #include <sys/cred.h>
39 #include <sys/user.h>
40 #include <sys/vnode.h>
41 #include <sys/file.h>
42 #include <sys/session.h>
43 #include <sys/stream.h>
44 #include <sys/strsubr.h>
45 #include <sys/stropts.h>
46 #include <sys/poll.h>
47 #include <sys/systm.h>
48 #include <sys/cpuvar.h>
49 #include <sys/uio.h>
50 #include <sys/cmn_err.h>
51 #include <sys/priocntl.h>
52 #include <sys/procset.h>
53 #include <sys/vmem.h>
54 #include <sys/bitmap.h>
55 #include <sys/kmem.h>
56 #include <sys/siginfo.h>
57 #include <sys/vtrace.h>
58 #include <sys/callb.h>
59 #include <sys/debug.h>
60 #include <sys/modctl.h>
61 #include <sys/vmsystm.h>
62 #include <vm/page.h>
63 #include <sys/atomic.h>
64 #include <sys/suntpi.h>
65 #include <sys/strlog.h>
66 #include <sys/promif.h>
67 #include <sys/project.h>
68 #include <sys/vm.h>
69 #include <sys/taskq.h>
70 #include <sys/sunddi.h>
71 #include <sys/sunldi_impl.h>
72 #include <sys/strsun.h>
73 #include <sys/isa_defs.h>
74 #include <sys/multidata.h>
75 #include <sys/pattr.h>
76 #include <sys/strft.h>
77 #include <sys/fs/snode.h>
78 #include <sys/zone.h>
79 #include <sys/open.h>
80 #include <sys/sunldi.h>
81 #include <sys/sad.h>
82 #include <sys/netstack.h>
83
84 #define O_SAMESTR(q) (((q)->q_next) && \
85 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
86
87 /*
88 * WARNING:
89 * The variables and routines in this file are private, belonging
90 * to the STREAMS subsystem. These should not be used by modules
91 * or drivers. Compatibility will not be guaranteed.
92 */
93
94 /*
95 * Id value used to distinguish between different multiplexor links.
96 */
97 static int32_t lnk_id = 0;
98
99 #define STREAMS_LOPRI MINCLSYSPRI
100 static pri_t streams_lopri = STREAMS_LOPRI;
101
102 #define STRSTAT(x) (str_statistics.x.value.ui64++)
103 typedef struct str_stat {
104 kstat_named_t sqenables;
105 kstat_named_t stenables;
106 kstat_named_t syncqservice;
107 kstat_named_t freebs;
108 kstat_named_t qwr_outer;
109 kstat_named_t rservice;
110 kstat_named_t strwaits;
111 kstat_named_t taskqfails;
112 kstat_named_t bufcalls;
113 kstat_named_t qhelps;
114 kstat_named_t qremoved;
115 kstat_named_t sqremoved;
116 kstat_named_t bcwaits;
117 kstat_named_t sqtoomany;
118 } str_stat_t;
119
120 static str_stat_t str_statistics = {
121 { "sqenables", KSTAT_DATA_UINT64 },
122 { "stenables", KSTAT_DATA_UINT64 },
123 { "syncqservice", KSTAT_DATA_UINT64 },
124 { "freebs", KSTAT_DATA_UINT64 },
125 { "qwr_outer", KSTAT_DATA_UINT64 },
126 { "rservice", KSTAT_DATA_UINT64 },
127 { "strwaits", KSTAT_DATA_UINT64 },
128 { "taskqfails", KSTAT_DATA_UINT64 },
129 { "bufcalls", KSTAT_DATA_UINT64 },
130 { "qhelps", KSTAT_DATA_UINT64 },
131 { "qremoved", KSTAT_DATA_UINT64 },
132 { "sqremoved", KSTAT_DATA_UINT64 },
133 { "bcwaits", KSTAT_DATA_UINT64 },
134 { "sqtoomany", KSTAT_DATA_UINT64 },
135 };
136
137 static kstat_t *str_kstat;
138
139 /*
140 * qrunflag was used previously to control background scheduling of queues. It
141 * is not used anymore, but kept here in case some module still wants to access
142 * it via qready() and setqsched macros.
143 */
144 char qrunflag; /* Unused */
145
146 /*
147 * Most of the streams scheduling is done via task queues. Task queues may fail
148 * for non-sleep dispatches, so there are two backup threads servicing failed
149 * requests for queues and syncqs. Both of these threads also service failed
150 * dispatches freebs requests. Queues are put in the list specified by `qhead'
151 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
152 * requests are put into `freebs_list' which has no tail pointer. All three
153 * lists are protected by a single `service_queue' lock and use
154 * `services_to_run' condition variable for signaling background threads. Use of
155 * a single lock should not be a problem because it is only used under heavy
156 * loads when task queues start to fail and at that time it may be a good idea
157 * to throttle scheduling requests.
158 *
159 * NOTE: queues and syncqs should be scheduled by two separate threads because
160 * queue servicing may be blocked waiting for a syncq which may be also
161 * scheduled for background execution. This may create a deadlock when only one
162 * thread is used for both.
163 */
164
165 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */
166
167 static kmutex_t service_queue; /* protects all of servicing vars */
168 static kcondvar_t services_to_run; /* wake up background service thread */
169 static kcondvar_t syncqs_to_run; /* wake up background service thread */
170
171 /*
172 * List of queues scheduled for background processing due to lack of resources
173 * in the task queues. Protected by service_queue lock;
174 */
175 static struct queue *qhead;
176 static struct queue *qtail;
177
178 /*
179 * Same list for syncqs
180 */
181 static syncq_t *sqhead;
182 static syncq_t *sqtail;
183
184 static mblk_t *freebs_list; /* list of buffers to free */
185
186 /*
187 * Backup threads for servicing queues and syncqs
188 */
189 kthread_t *streams_qbkgrnd_thread;
190 kthread_t *streams_sqbkgrnd_thread;
191
192 /*
193 * Bufcalls related variables.
194 */
195 struct bclist strbcalls; /* list of waiting bufcalls */
196 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */
197 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */
198 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */
199 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */
200 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */
201
202 kmutex_t strresources; /* protects global resources */
203 kmutex_t muxifier; /* single-threads multiplexor creation */
204
205 static void *str_stack_init(netstackid_t stackid, netstack_t *ns);
206 static void str_stack_shutdown(netstackid_t stackid, void *arg);
207 static void str_stack_fini(netstackid_t stackid, void *arg);
208
209 /*
210 * run_queues is no longer used, but is kept in case some 3rd party
211 * module/driver decides to use it.
212 */
213 int run_queues = 0;
214
215 /*
216 * sq_max_size is the depth of the syncq (in number of messages) before
217 * qfill_syncq() starts QFULL'ing destination queues. As its primary
218 * consumer - IP is no longer D_MTPERMOD, but there may be other
219 * modules/drivers depend on this syncq flow control, we prefer to
220 * choose a large number as the default value. For potential
221 * performance gain, this value is tunable in /etc/system.
222 */
223 int sq_max_size = 10000;
224
225 /*
226 * The number of ciputctrl structures per syncq and stream we create when
227 * needed.
228 */
229 int n_ciputctrl;
230 int max_n_ciputctrl = 16;
231 /*
232 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
233 */
234 int min_n_ciputctrl = 2;
235
236 /*
237 * Per-driver/module syncqs
238 * ========================
239 *
240 * For drivers/modules that use PERMOD or outer syncqs we keep a list of
241 * perdm structures, new entries being added (and new syncqs allocated) when
242 * setq() encounters a module/driver with a streamtab that it hasn't seen
243 * before.
244 * The reason for this mechanism is that some modules and drivers share a
245 * common streamtab and it is necessary for those modules and drivers to also
246 * share a common PERMOD syncq.
247 *
248 * perdm_list --> dm_str == streamtab_1
249 * dm_sq == syncq_1
250 * dm_ref
251 * dm_next --> dm_str == streamtab_2
252 * dm_sq == syncq_2
253 * dm_ref
254 * dm_next --> ... NULL
255 *
256 * The dm_ref field is incremented for each new driver/module that takes
257 * a reference to the perdm structure and hence shares the syncq.
258 * References are held in the fmodsw_impl_t structure for each STREAMS module
259 * or the dev_impl array (indexed by device major number) for each driver.
260 *
261 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
262 * ^ ^ ^ ^
263 * | ______________/ | |
264 * | / | |
265 * dev_impl: ...|x|y|... module A module B
266 *
267 * When a module/driver is unloaded the reference count is decremented and,
268 * when it falls to zero, the perdm structure is removed from the list and
269 * the syncq is freed (see rele_dm()).
270 */
271 perdm_t *perdm_list = NULL;
272 static krwlock_t perdm_rwlock;
273 cdevsw_impl_t *devimpl;
274
275 extern struct qinit strdata;
276 extern struct qinit stwdata;
277
278 static void runservice(queue_t *);
279 static void streams_bufcall_service(void);
280 static void streams_qbkgrnd_service(void);
281 static void streams_sqbkgrnd_service(void);
282 static syncq_t *new_syncq(void);
283 static void free_syncq(syncq_t *);
284 static void outer_insert(syncq_t *, syncq_t *);
285 static void outer_remove(syncq_t *, syncq_t *);
286 static void write_now(syncq_t *);
287 static void clr_qfull(queue_t *);
288 static void runbufcalls(void);
289 static void sqenable(syncq_t *);
290 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
291 static void wait_q_syncq(queue_t *);
292 static void backenable_insertedq(queue_t *);
293
294 static void queue_service(queue_t *);
295 static void stream_service(stdata_t *);
296 static void syncq_service(syncq_t *);
297 static void qwriter_outer_service(syncq_t *);
298 static void mblk_free(mblk_t *);
299 #ifdef DEBUG
300 static int qprocsareon(queue_t *);
301 #endif
302
303 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
304 static void reset_nfsrv_ptr(queue_t *, queue_t *);
305 void set_qfull(queue_t *);
306
307 static void sq_run_events(syncq_t *);
308 static int propagate_syncq(queue_t *);
309
310 static void blocksq(syncq_t *, ushort_t, int);
311 static void unblocksq(syncq_t *, ushort_t, int);
312 static int dropsq(syncq_t *, uint16_t);
313 static void emptysq(syncq_t *);
314 static sqlist_t *sqlist_alloc(struct stdata *, int);
315 static void sqlist_free(sqlist_t *);
316 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t);
317 static void sqlist_insert(sqlist_t *, syncq_t *);
318 static void sqlist_insertall(sqlist_t *, queue_t *);
319
320 static void strsetuio(stdata_t *);
321
322 struct kmem_cache *stream_head_cache;
323 struct kmem_cache *queue_cache;
324 struct kmem_cache *syncq_cache;
325 struct kmem_cache *qband_cache;
326 struct kmem_cache *linkinfo_cache;
327 struct kmem_cache *ciputctrl_cache = NULL;
328
329 static linkinfo_t *linkinfo_list;
330
331 /* Global esballoc throttling queue */
332 static esb_queue_t system_esbq;
333
334 /* Array of esballoc throttling queues, of length esbq_nelem */
335 static esb_queue_t *volatile system_esbq_array;
336 static int esbq_nelem;
337 static kmutex_t esbq_lock;
338 static int esbq_log2_cpus_per_q = 0;
339
340 /* Scale the system_esbq length by setting number of CPUs per queue. */
341 uint_t esbq_cpus_per_q = 1;
342
343 /*
344 * esballoc tunable parameters.
345 */
346 int esbq_max_qlen = 0x16; /* throttled queue length */
347 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */
348
349 /*
350 * Routines to handle esballoc queueing.
351 */
352 static void esballoc_process_queue(esb_queue_t *);
353 static void esballoc_enqueue_mblk(mblk_t *);
354 static void esballoc_timer(void *);
355 static void esballoc_set_timer(esb_queue_t *, clock_t);
356 static void esballoc_mblk_free(mblk_t *);
357
358 /*
359 * Qinit structure and Module_info structures
360 * for passthru read and write queues
361 */
362
363 static void pass_wput(queue_t *, mblk_t *);
364 static queue_t *link_addpassthru(stdata_t *);
365 static void link_rempassthru(queue_t *);
366
367 struct module_info passthru_info = {
368 0,
369 "passthru",
370 0,
371 INFPSZ,
372 STRHIGH,
373 STRLOW
374 };
375
376 struct qinit passthru_rinit = {
377 (int (*)())putnext,
378 NULL,
379 NULL,
380 NULL,
381 NULL,
382 &passthru_info,
383 NULL
384 };
385
386 struct qinit passthru_winit = {
387 (int (*)()) pass_wput,
388 NULL,
389 NULL,
390 NULL,
391 NULL,
392 &passthru_info,
393 NULL
394 };
395
396 /*
397 * Verify correctness of list head/tail pointers.
398 */
399 #define LISTCHECK(head, tail, link) { \
400 EQUIV(head, tail); \
401 IMPLY(tail != NULL, tail->link == NULL); \
402 }
403
404 /*
405 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
406 * using a `link' field.
407 */
408 #define ENQUEUE(el, head, tail, link) { \
409 ASSERT(el->link == NULL); \
410 LISTCHECK(head, tail, link); \
411 if (head == NULL) \
412 head = el; \
413 else \
414 tail->link = el; \
415 tail = el; \
416 }
417
418 /*
419 * Dequeue the first element of the list denoted by `head' and `tail' pointers
420 * using a `link' field and put result into `el'.
421 */
422 #define DQ(el, head, tail, link) { \
423 LISTCHECK(head, tail, link); \
424 el = head; \
425 if (head != NULL) { \
426 head = head->link; \
427 if (head == NULL) \
428 tail = NULL; \
429 el->link = NULL; \
430 } \
431 }
432
433 /*
434 * Remove `el' from the list using `chase' and `curr' pointers and return result
435 * in `succeed'.
436 */
437 #define RMQ(el, head, tail, link, chase, curr, succeed) { \
438 LISTCHECK(head, tail, link); \
439 chase = NULL; \
440 succeed = 0; \
441 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
442 chase = curr; \
443 if (curr != NULL) { \
444 succeed = 1; \
445 ASSERT(curr == el); \
446 if (chase != NULL) \
447 chase->link = curr->link; \
448 else \
449 head = curr->link; \
450 curr->link = NULL; \
451 if (curr == tail) \
452 tail = chase; \
453 } \
454 LISTCHECK(head, tail, link); \
455 }
456
457 /* Handling of delayed messages on the inner syncq. */
458
459 /*
460 * DEBUG versions should use function versions (to simplify tracing) and
461 * non-DEBUG kernels should use macro versions.
462 */
463
464 /*
465 * Put a queue on the syncq list of queues.
466 * Assumes SQLOCK held.
467 */
468 #define SQPUT_Q(sq, qp) \
469 { \
470 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
471 if (!(qp->q_sqflags & Q_SQQUEUED)) { \
472 /* The queue should not be linked anywhere */ \
473 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
474 /* Head and tail may only be NULL simultaneously */ \
475 EQUIV(sq->sq_head, sq->sq_tail); \
476 /* Queue may be only enqueued on its syncq */ \
477 ASSERT(sq == qp->q_syncq); \
478 /* Check the correctness of SQ_MESSAGES flag */ \
479 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
480 /* Sanity check first/last elements of the list */ \
481 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
482 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
483 /* \
484 * Sanity check of priority field: empty queue should \
485 * have zero priority \
486 * and nqueues equal to zero. \
487 */ \
488 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
489 /* Sanity check of sq_nqueues field */ \
490 EQUIV(sq->sq_head, sq->sq_nqueues); \
491 if (sq->sq_head == NULL) { \
492 sq->sq_head = sq->sq_tail = qp; \
493 sq->sq_flags |= SQ_MESSAGES; \
494 } else if (qp->q_spri == 0) { \
495 qp->q_sqprev = sq->sq_tail; \
496 sq->sq_tail->q_sqnext = qp; \
497 sq->sq_tail = qp; \
498 } else { \
499 /* \
500 * Put this queue in priority order: higher \
501 * priority gets closer to the head. \
502 */ \
503 queue_t **qpp = &sq->sq_tail; \
504 queue_t *qnext = NULL; \
505 \
506 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
507 qnext = *qpp; \
508 qpp = &(*qpp)->q_sqprev; \
509 } \
510 qp->q_sqnext = qnext; \
511 qp->q_sqprev = *qpp; \
512 if (*qpp != NULL) { \
513 (*qpp)->q_sqnext = qp; \
514 } else { \
515 sq->sq_head = qp; \
516 sq->sq_pri = sq->sq_head->q_spri; \
517 } \
518 *qpp = qp; \
519 } \
520 qp->q_sqflags |= Q_SQQUEUED; \
521 qp->q_sqtstamp = ddi_get_lbolt(); \
522 sq->sq_nqueues++; \
523 } \
524 }
525
526 /*
527 * Remove a queue from the syncq list
528 * Assumes SQLOCK held.
529 */
530 #define SQRM_Q(sq, qp) \
531 { \
532 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
533 ASSERT(qp->q_sqflags & Q_SQQUEUED); \
534 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
535 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
536 /* Check that the queue is actually in the list */ \
537 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
538 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
539 ASSERT(sq->sq_nqueues != 0); \
540 if (qp->q_sqprev == NULL) { \
541 /* First queue on list, make head q_sqnext */ \
542 sq->sq_head = qp->q_sqnext; \
543 } else { \
544 /* Make prev->next == next */ \
545 qp->q_sqprev->q_sqnext = qp->q_sqnext; \
546 } \
547 if (qp->q_sqnext == NULL) { \
548 /* Last queue on list, make tail sqprev */ \
549 sq->sq_tail = qp->q_sqprev; \
550 } else { \
551 /* Make next->prev == prev */ \
552 qp->q_sqnext->q_sqprev = qp->q_sqprev; \
553 } \
554 /* clear out references on this queue */ \
555 qp->q_sqprev = qp->q_sqnext = NULL; \
556 qp->q_sqflags &= ~Q_SQQUEUED; \
557 /* If there is nothing queued, clear SQ_MESSAGES */ \
558 if (sq->sq_head != NULL) { \
559 sq->sq_pri = sq->sq_head->q_spri; \
560 } else { \
561 sq->sq_flags &= ~SQ_MESSAGES; \
562 sq->sq_pri = 0; \
563 } \
564 sq->sq_nqueues--; \
565 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
566 (sq->sq_flags & SQ_QUEUED) == 0); \
567 }
568
569 /* Hide the definition from the header file. */
570 #ifdef SQPUT_MP
571 #undef SQPUT_MP
572 #endif
573
574 /*
575 * Put a message on the queue syncq.
576 * Assumes QLOCK held.
577 */
578 #define SQPUT_MP(qp, mp) \
579 { \
580 ASSERT(MUTEX_HELD(QLOCK(qp))); \
581 ASSERT(qp->q_sqhead == NULL || \
582 (qp->q_sqtail != NULL && \
583 qp->q_sqtail->b_next == NULL)); \
584 qp->q_syncqmsgs++; \
585 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \
586 if (qp->q_sqhead == NULL) { \
587 qp->q_sqhead = qp->q_sqtail = mp; \
588 } else { \
589 qp->q_sqtail->b_next = mp; \
590 qp->q_sqtail = mp; \
591 } \
592 ASSERT(qp->q_syncqmsgs > 0); \
593 set_qfull(qp); \
594 }
595
596 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \
597 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
598 if ((sq)->sq_ciputctrl != NULL) { \
599 int i; \
600 int nlocks = (sq)->sq_nciputctrl; \
601 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
602 ASSERT((sq)->sq_type & SQ_CIPUT); \
603 for (i = 0; i <= nlocks; i++) { \
604 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
605 cip[i].ciputctrl_count |= SQ_FASTPUT; \
606 } \
607 } \
608 }
609
610
611 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \
612 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
613 if ((sq)->sq_ciputctrl != NULL) { \
614 int i; \
615 int nlocks = (sq)->sq_nciputctrl; \
616 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
617 ASSERT((sq)->sq_type & SQ_CIPUT); \
618 for (i = 0; i <= nlocks; i++) { \
619 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
620 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \
621 } \
622 } \
623 }
624
625 /*
626 * Run service procedures for all queues in the stream head.
627 */
628 #define STR_SERVICE(stp, q) { \
629 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \
630 while (stp->sd_qhead != NULL) { \
631 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \
632 ASSERT(stp->sd_nqueues > 0); \
633 stp->sd_nqueues--; \
634 ASSERT(!(q->q_flag & QINSERVICE)); \
635 mutex_exit(&stp->sd_qlock); \
636 queue_service(q); \
637 mutex_enter(&stp->sd_qlock); \
638 } \
639 ASSERT(stp->sd_nqueues == 0); \
640 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \
641 }
642
643 /*
644 * Constructor/destructor routines for the stream head cache
645 */
646 /* ARGSUSED */
647 static int
648 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
649 {
650 stdata_t *stp = buf;
651
652 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
653 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
654 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
655 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
656 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
657 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
658 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
659 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
660 stp->sd_wrq = NULL;
661
662 return (0);
663 }
664
665 /* ARGSUSED */
666 static void
667 stream_head_destructor(void *buf, void *cdrarg)
668 {
669 stdata_t *stp = buf;
670
671 mutex_destroy(&stp->sd_lock);
672 mutex_destroy(&stp->sd_reflock);
673 mutex_destroy(&stp->sd_qlock);
674 cv_destroy(&stp->sd_monitor);
675 cv_destroy(&stp->sd_iocmonitor);
676 cv_destroy(&stp->sd_refmonitor);
677 cv_destroy(&stp->sd_qcv);
678 cv_destroy(&stp->sd_zcopy_wait);
679 }
680
681 /*
682 * Constructor/destructor routines for the queue cache
683 */
684 /* ARGSUSED */
685 static int
686 queue_constructor(void *buf, void *cdrarg, int kmflags)
687 {
688 queinfo_t *qip = buf;
689 queue_t *qp = &qip->qu_rqueue;
690 queue_t *wqp = &qip->qu_wqueue;
691 syncq_t *sq = &qip->qu_syncq;
692
693 qp->q_first = NULL;
694 qp->q_link = NULL;
695 qp->q_count = 0;
696 qp->q_mblkcnt = 0;
697 qp->q_sqhead = NULL;
698 qp->q_sqtail = NULL;
699 qp->q_sqnext = NULL;
700 qp->q_sqprev = NULL;
701 qp->q_sqflags = 0;
702 qp->q_rwcnt = 0;
703 qp->q_spri = 0;
704
705 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
706 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
707
708 wqp->q_first = NULL;
709 wqp->q_link = NULL;
710 wqp->q_count = 0;
711 wqp->q_mblkcnt = 0;
712 wqp->q_sqhead = NULL;
713 wqp->q_sqtail = NULL;
714 wqp->q_sqnext = NULL;
715 wqp->q_sqprev = NULL;
716 wqp->q_sqflags = 0;
717 wqp->q_rwcnt = 0;
718 wqp->q_spri = 0;
719
720 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
721 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
722
723 sq->sq_head = NULL;
724 sq->sq_tail = NULL;
725 sq->sq_evhead = NULL;
726 sq->sq_evtail = NULL;
727 sq->sq_callbpend = NULL;
728 sq->sq_outer = NULL;
729 sq->sq_onext = NULL;
730 sq->sq_oprev = NULL;
731 sq->sq_next = NULL;
732 sq->sq_svcflags = 0;
733 sq->sq_servcount = 0;
734 sq->sq_needexcl = 0;
735 sq->sq_nqueues = 0;
736 sq->sq_pri = 0;
737
738 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
739 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
740 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
741
742 return (0);
743 }
744
745 /* ARGSUSED */
746 static void
747 queue_destructor(void *buf, void *cdrarg)
748 {
749 queinfo_t *qip = buf;
750 queue_t *qp = &qip->qu_rqueue;
751 queue_t *wqp = &qip->qu_wqueue;
752 syncq_t *sq = &qip->qu_syncq;
753
754 ASSERT(qp->q_sqhead == NULL);
755 ASSERT(wqp->q_sqhead == NULL);
756 ASSERT(qp->q_sqnext == NULL);
757 ASSERT(wqp->q_sqnext == NULL);
758 ASSERT(qp->q_rwcnt == 0);
759 ASSERT(wqp->q_rwcnt == 0);
760
761 mutex_destroy(&qp->q_lock);
762 cv_destroy(&qp->q_wait);
763
764 mutex_destroy(&wqp->q_lock);
765 cv_destroy(&wqp->q_wait);
766
767 mutex_destroy(&sq->sq_lock);
768 cv_destroy(&sq->sq_wait);
769 cv_destroy(&sq->sq_exitwait);
770 }
771
772 /*
773 * Constructor/destructor routines for the syncq cache
774 */
775 /* ARGSUSED */
776 static int
777 syncq_constructor(void *buf, void *cdrarg, int kmflags)
778 {
779 syncq_t *sq = buf;
780
781 bzero(buf, sizeof (syncq_t));
782
783 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
784 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
785 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
786
787 return (0);
788 }
789
790 /* ARGSUSED */
791 static void
792 syncq_destructor(void *buf, void *cdrarg)
793 {
794 syncq_t *sq = buf;
795
796 ASSERT(sq->sq_head == NULL);
797 ASSERT(sq->sq_tail == NULL);
798 ASSERT(sq->sq_evhead == NULL);
799 ASSERT(sq->sq_evtail == NULL);
800 ASSERT(sq->sq_callbpend == NULL);
801 ASSERT(sq->sq_callbflags == 0);
802 ASSERT(sq->sq_outer == NULL);
803 ASSERT(sq->sq_onext == NULL);
804 ASSERT(sq->sq_oprev == NULL);
805 ASSERT(sq->sq_next == NULL);
806 ASSERT(sq->sq_needexcl == 0);
807 ASSERT(sq->sq_svcflags == 0);
808 ASSERT(sq->sq_servcount == 0);
809 ASSERT(sq->sq_nqueues == 0);
810 ASSERT(sq->sq_pri == 0);
811 ASSERT(sq->sq_count == 0);
812 ASSERT(sq->sq_rmqcount == 0);
813 ASSERT(sq->sq_cancelid == 0);
814 ASSERT(sq->sq_ciputctrl == NULL);
815 ASSERT(sq->sq_nciputctrl == 0);
816 ASSERT(sq->sq_type == 0);
817 ASSERT(sq->sq_flags == 0);
818
819 mutex_destroy(&sq->sq_lock);
820 cv_destroy(&sq->sq_wait);
821 cv_destroy(&sq->sq_exitwait);
822 }
823
824 /* ARGSUSED */
825 static int
826 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
827 {
828 ciputctrl_t *cip = buf;
829 int i;
830
831 for (i = 0; i < n_ciputctrl; i++) {
832 cip[i].ciputctrl_count = SQ_FASTPUT;
833 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
834 }
835
836 return (0);
837 }
838
839 /* ARGSUSED */
840 static void
841 ciputctrl_destructor(void *buf, void *cdrarg)
842 {
843 ciputctrl_t *cip = buf;
844 int i;
845
846 for (i = 0; i < n_ciputctrl; i++) {
847 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
848 mutex_destroy(&cip[i].ciputctrl_lock);
849 }
850 }
851
852 /*
853 * Init routine run from main at boot time.
854 */
855 void
856 strinit(void)
857 {
858 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
859
860 stream_head_cache = kmem_cache_create("stream_head_cache",
861 sizeof (stdata_t), 0,
862 stream_head_constructor, stream_head_destructor, NULL,
863 NULL, NULL, 0);
864
865 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
866 queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
867
868 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
869 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
870
871 qband_cache = kmem_cache_create("qband_cache",
872 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
873
874 linkinfo_cache = kmem_cache_create("linkinfo_cache",
875 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
876
877 n_ciputctrl = ncpus;
878 n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
879 ASSERT(n_ciputctrl >= 1);
880 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
881 if (n_ciputctrl >= min_n_ciputctrl) {
882 ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
883 sizeof (ciputctrl_t) * n_ciputctrl,
884 sizeof (ciputctrl_t), ciputctrl_constructor,
885 ciputctrl_destructor, NULL, NULL, NULL, 0);
886 }
887
888 streams_taskq = system_taskq;
889
890 if (streams_taskq == NULL)
891 panic("strinit: no memory for streams taskq!");
892
893 bc_bkgrnd_thread = thread_create(NULL, 0,
894 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
895
896 streams_qbkgrnd_thread = thread_create(NULL, 0,
897 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
898
899 streams_sqbkgrnd_thread = thread_create(NULL, 0,
900 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
901
902 /*
903 * Create STREAMS kstats.
904 */
905 str_kstat = kstat_create("streams", 0, "strstat",
906 "net", KSTAT_TYPE_NAMED,
907 sizeof (str_statistics) / sizeof (kstat_named_t),
908 KSTAT_FLAG_VIRTUAL);
909
910 if (str_kstat != NULL) {
911 str_kstat->ks_data = &str_statistics;
912 kstat_install(str_kstat);
913 }
914
915 /*
916 * TPI support routine initialisation.
917 */
918 tpi_init();
919
920 /*
921 * Handle to have autopush and persistent link information per
922 * zone.
923 * Note: uses shutdown hook instead of destroy hook so that the
924 * persistent links can be torn down before the destroy hooks
925 * in the TCP/IP stack are called.
926 */
927 netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
928 str_stack_fini);
929 }
930
931 void
932 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
933 {
934 struct stdata *stp;
935
936 ASSERT(vp->v_stream);
937 stp = vp->v_stream;
938 /* Have to hold sd_lock to prevent siglist from changing */
939 mutex_enter(&stp->sd_lock);
940 if (stp->sd_sigflags & event)
941 strsendsig(stp->sd_siglist, event, band, error);
942 mutex_exit(&stp->sd_lock);
943 }
944
945 /*
946 * Send the "sevent" set of signals to a process.
947 * This might send more than one signal if the process is registered
948 * for multiple events. The caller should pass in an sevent that only
949 * includes the events for which the process has registered.
950 */
951 static void
952 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
953 uchar_t band, int error)
954 {
955 ASSERT(MUTEX_HELD(&proc->p_lock));
956
957 info->si_band = 0;
958 info->si_errno = 0;
959
960 if (sevent & S_ERROR) {
961 sevent &= ~S_ERROR;
962 info->si_code = POLL_ERR;
963 info->si_errno = error;
964 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
965 "strsendsig:proc %p info %p", proc, info);
966 sigaddq(proc, NULL, info, KM_NOSLEEP);
967 info->si_errno = 0;
968 }
969 if (sevent & S_HANGUP) {
970 sevent &= ~S_HANGUP;
971 info->si_code = POLL_HUP;
972 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
973 "strsendsig:proc %p info %p", proc, info);
974 sigaddq(proc, NULL, info, KM_NOSLEEP);
975 }
976 if (sevent & S_HIPRI) {
977 sevent &= ~S_HIPRI;
978 info->si_code = POLL_PRI;
979 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
980 "strsendsig:proc %p info %p", proc, info);
981 sigaddq(proc, NULL, info, KM_NOSLEEP);
982 }
983 if (sevent & S_RDBAND) {
984 sevent &= ~S_RDBAND;
985 if (events & S_BANDURG)
986 sigtoproc(proc, NULL, SIGURG);
987 else
988 sigtoproc(proc, NULL, SIGPOLL);
989 }
990 if (sevent & S_WRBAND) {
991 sevent &= ~S_WRBAND;
992 sigtoproc(proc, NULL, SIGPOLL);
993 }
994 if (sevent & S_INPUT) {
995 sevent &= ~S_INPUT;
996 info->si_code = POLL_IN;
997 info->si_band = band;
998 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
999 "strsendsig:proc %p info %p", proc, info);
1000 sigaddq(proc, NULL, info, KM_NOSLEEP);
1001 info->si_band = 0;
1002 }
1003 if (sevent & S_OUTPUT) {
1004 sevent &= ~S_OUTPUT;
1005 info->si_code = POLL_OUT;
1006 info->si_band = band;
1007 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1008 "strsendsig:proc %p info %p", proc, info);
1009 sigaddq(proc, NULL, info, KM_NOSLEEP);
1010 info->si_band = 0;
1011 }
1012 if (sevent & S_MSG) {
1013 sevent &= ~S_MSG;
1014 info->si_code = POLL_MSG;
1015 info->si_band = band;
1016 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1017 "strsendsig:proc %p info %p", proc, info);
1018 sigaddq(proc, NULL, info, KM_NOSLEEP);
1019 info->si_band = 0;
1020 }
1021 if (sevent & S_RDNORM) {
1022 sevent &= ~S_RDNORM;
1023 sigtoproc(proc, NULL, SIGPOLL);
1024 }
1025 if (sevent != 0) {
1026 panic("strsendsig: unknown event(s) %x", sevent);
1027 }
1028 }
1029
1030 /*
1031 * Send SIGPOLL/SIGURG signal to all processes and process groups
1032 * registered on the given signal list that want a signal for at
1033 * least one of the specified events.
1034 *
1035 * Must be called with exclusive access to siglist (caller holding sd_lock).
1036 *
1037 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1038 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1039 * while it is in the siglist.
1040 *
1041 * For performance reasons (MP scalability) the code drops pidlock
1042 * when sending signals to a single process.
1043 * When sending to a process group the code holds
1044 * pidlock to prevent the membership in the process group from changing
1045 * while walking the p_pglink list.
1046 */
1047 void
1048 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1049 {
1050 strsig_t *ssp;
1051 k_siginfo_t info;
1052 struct pid *pidp;
1053 proc_t *proc;
1054
1055 info.si_signo = SIGPOLL;
1056 info.si_errno = 0;
1057 for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1058 int sevent;
1059
1060 sevent = ssp->ss_events & event;
1061 if (sevent == 0)
1062 continue;
1063
1064 if ((pidp = ssp->ss_pidp) == NULL) {
1065 /* pid was released but still on event list */
1066 continue;
1067 }
1068
1069
1070 if (ssp->ss_pid > 0) {
1071 /*
1072 * XXX This unfortunately still generates
1073 * a signal when a fd is closed but
1074 * the proc is active.
1075 */
1076 ASSERT(ssp->ss_pid == pidp->pid_id);
1077
1078 mutex_enter(&pidlock);
1079 proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1080 if (proc == NULL) {
1081 mutex_exit(&pidlock);
1082 continue;
1083 }
1084 mutex_enter(&proc->p_lock);
1085 mutex_exit(&pidlock);
1086 dosendsig(proc, ssp->ss_events, sevent, &info,
1087 band, error);
1088 mutex_exit(&proc->p_lock);
1089 } else {
1090 /*
1091 * Send to process group. Hold pidlock across
1092 * calls to dosendsig().
1093 */
1094 pid_t pgrp = -ssp->ss_pid;
1095
1096 mutex_enter(&pidlock);
1097 proc = pgfind_zone(pgrp, ALL_ZONES);
1098 while (proc != NULL) {
1099 mutex_enter(&proc->p_lock);
1100 dosendsig(proc, ssp->ss_events, sevent,
1101 &info, band, error);
1102 mutex_exit(&proc->p_lock);
1103 proc = proc->p_pglink;
1104 }
1105 mutex_exit(&pidlock);
1106 }
1107 }
1108 }
1109
1110 /*
1111 * Attach a stream device or module.
1112 * qp is a read queue; the new queue goes in so its next
1113 * read ptr is the argument, and the write queue corresponding
1114 * to the argument points to this queue. Return 0 on success,
1115 * or a non-zero errno on failure.
1116 */
1117 int
1118 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1119 boolean_t is_insert)
1120 {
1121 major_t major;
1122 cdevsw_impl_t *dp;
1123 struct streamtab *str;
1124 queue_t *rq;
1125 queue_t *wrq;
1126 uint32_t qflag;
1127 uint32_t sqtype;
1128 perdm_t *dmp;
1129 int error;
1130 int sflag;
1131
1132 rq = allocq();
1133 wrq = _WR(rq);
1134 STREAM(rq) = STREAM(wrq) = STREAM(qp);
1135
1136 if (fp != NULL) {
1137 str = fp->f_str;
1138 qflag = fp->f_qflag;
1139 sqtype = fp->f_sqtype;
1140 dmp = fp->f_dmp;
1141 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1142 sflag = MODOPEN;
1143
1144 /*
1145 * stash away a pointer to the module structure so we can
1146 * unref it in qdetach.
1147 */
1148 rq->q_fp = fp;
1149 } else {
1150 ASSERT(!is_insert);
1151
1152 major = getmajor(*devp);
1153 dp = &devimpl[major];
1154
1155 str = dp->d_str;
1156 ASSERT(str == STREAMSTAB(major));
1157
1158 qflag = dp->d_qflag;
1159 ASSERT(qflag & QISDRV);
1160 sqtype = dp->d_sqtype;
1161
1162 /* create perdm_t if needed */
1163 if (NEED_DM(dp->d_dmp, qflag))
1164 dp->d_dmp = hold_dm(str, qflag, sqtype);
1165
1166 dmp = dp->d_dmp;
1167 sflag = 0;
1168 }
1169
1170 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1171 "qattach:qflag == %X(%X)", qflag, *devp);
1172
1173 /* setq might sleep in allocator - avoid holding locks. */
1174 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1175
1176 /*
1177 * Before calling the module's open routine, set up the q_next
1178 * pointer for inserting a module in the middle of a stream.
1179 *
1180 * Note that we can always set _QINSERTING and set up q_next
1181 * pointer for both inserting and pushing a module. Then there
1182 * is no need for the is_insert parameter. In insertq(), called
1183 * by qprocson(), assume that q_next of the new module always points
1184 * to the correct queue and use it for insertion. Everything should
1185 * work out fine. But in the first release of _I_INSERT, we
1186 * distinguish between inserting and pushing to make sure that
1187 * pushing a module follows the same code path as before.
1188 */
1189 if (is_insert) {
1190 rq->q_flag |= _QINSERTING;
1191 rq->q_next = qp;
1192 }
1193
1194 /*
1195 * If there is an outer perimeter get exclusive access during
1196 * the open procedure. Bump up the reference count on the queue.
1197 */
1198 entersq(rq->q_syncq, SQ_OPENCLOSE);
1199 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1200 if (error != 0)
1201 goto failed;
1202 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1203 ASSERT(qprocsareon(rq));
1204 return (0);
1205
1206 failed:
1207 rq->q_flag &= ~_QINSERTING;
1208 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1209 qprocsoff(rq);
1210 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1211 rq->q_next = wrq->q_next = NULL;
1212 qdetach(rq, 0, 0, crp, B_FALSE);
1213 return (error);
1214 }
1215
1216 /*
1217 * Handle second open of stream. For modules, set the
1218 * last argument to MODOPEN and do not pass any open flags.
1219 * Ignore dummydev since this is not the first open.
1220 */
1221 int
1222 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1223 {
1224 int error;
1225 dev_t dummydev;
1226 queue_t *wqp = _WR(qp);
1227
1228 ASSERT(qp->q_flag & QREADR);
1229 entersq(qp->q_syncq, SQ_OPENCLOSE);
1230
1231 dummydev = *devp;
1232 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1233 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1234 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1235 mutex_enter(&STREAM(qp)->sd_lock);
1236 qp->q_stream->sd_flag |= STREOPENFAIL;
1237 mutex_exit(&STREAM(qp)->sd_lock);
1238 return (error);
1239 }
1240 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1241
1242 /*
1243 * successful open should have done qprocson()
1244 */
1245 ASSERT(qprocsareon(_RD(qp)));
1246 return (0);
1247 }
1248
1249 /*
1250 * Detach a stream module or device.
1251 * If clmode == 1 then the module or driver was opened and its
1252 * close routine must be called. If clmode == 0, the module
1253 * or driver was never opened or the open failed, and so its close
1254 * should not be called.
1255 */
1256 void
1257 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1258 {
1259 queue_t *wqp = _WR(qp);
1260 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1261
1262 if (STREAM_NEEDSERVICE(STREAM(qp)))
1263 stream_runservice(STREAM(qp));
1264
1265 if (clmode) {
1266 /*
1267 * Make sure that all the messages on the write side syncq are
1268 * processed and nothing is left. Since we are closing, no new
1269 * messages may appear there.
1270 */
1271 wait_q_syncq(wqp);
1272
1273 entersq(qp->q_syncq, SQ_OPENCLOSE);
1274 if (is_remove) {
1275 mutex_enter(QLOCK(qp));
1276 qp->q_flag |= _QREMOVING;
1277 mutex_exit(QLOCK(qp));
1278 }
1279 (*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1280 /*
1281 * Check that qprocsoff() was actually called.
1282 */
1283 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1284
1285 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1286 } else {
1287 disable_svc(qp);
1288 }
1289
1290 /*
1291 * Allow any threads blocked in entersq to proceed and discover
1292 * the QWCLOSE is set.
1293 * Note: This assumes that all users of entersq check QWCLOSE.
1294 * Currently runservice is the only entersq that can happen
1295 * after removeq has finished.
1296 * Removeq will have discarded all messages destined to the closing
1297 * pair of queues from the syncq.
1298 * NOTE: Calling a function inside an assert is unconventional.
1299 * However, it does not cause any problem since flush_syncq() does
1300 * not change any state except when it returns non-zero i.e.
1301 * when the assert will trigger.
1302 */
1303 ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1304 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1305 ASSERT((qp->q_flag & QPERMOD) ||
1306 ((qp->q_syncq->sq_head == NULL) &&
1307 (wqp->q_syncq->sq_head == NULL)));
1308
1309 /* release any fmodsw_impl_t structure held on behalf of the queue */
1310 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1311 if (qp->q_fp != NULL)
1312 fmodsw_rele(qp->q_fp);
1313
1314 /* freeq removes us from the outer perimeter if any */
1315 freeq(qp);
1316 }
1317
1318 /* Prevent service procedures from being called */
1319 void
1320 disable_svc(queue_t *qp)
1321 {
1322 queue_t *wqp = _WR(qp);
1323
1324 ASSERT(qp->q_flag & QREADR);
1325 mutex_enter(QLOCK(qp));
1326 qp->q_flag |= QWCLOSE;
1327 mutex_exit(QLOCK(qp));
1328 mutex_enter(QLOCK(wqp));
1329 wqp->q_flag |= QWCLOSE;
1330 mutex_exit(QLOCK(wqp));
1331 }
1332
1333 /* Allow service procedures to be called again */
1334 void
1335 enable_svc(queue_t *qp)
1336 {
1337 queue_t *wqp = _WR(qp);
1338
1339 ASSERT(qp->q_flag & QREADR);
1340 mutex_enter(QLOCK(qp));
1341 qp->q_flag &= ~QWCLOSE;
1342 mutex_exit(QLOCK(qp));
1343 mutex_enter(QLOCK(wqp));
1344 wqp->q_flag &= ~QWCLOSE;
1345 mutex_exit(QLOCK(wqp));
1346 }
1347
1348 /*
1349 * Remove queue from qhead/qtail if it is enabled.
1350 * Only reset QENAB if the queue was removed from the runlist.
1351 * A queue goes through 3 stages:
1352 * It is on the service list and QENAB is set.
1353 * It is removed from the service list but QENAB is still set.
1354 * QENAB gets changed to QINSERVICE.
1355 * QINSERVICE is reset (when the service procedure is done)
1356 * Thus we can not reset QENAB unless we actually removed it from the service
1357 * queue.
1358 */
1359 void
1360 remove_runlist(queue_t *qp)
1361 {
1362 if (qp->q_flag & QENAB && qhead != NULL) {
1363 queue_t *q_chase;
1364 queue_t *q_curr;
1365 int removed;
1366
1367 mutex_enter(&service_queue);
1368 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1369 mutex_exit(&service_queue);
1370 if (removed) {
1371 STRSTAT(qremoved);
1372 qp->q_flag &= ~QENAB;
1373 }
1374 }
1375 }
1376
1377
1378 /*
1379 * Wait for any pending service processing to complete.
1380 * The removal of queues from the runlist is not atomic with the
1381 * clearing of the QENABLED flag and setting the INSERVICE flag.
1382 * consequently it is possible for remove_runlist in strclose
1383 * to not find the queue on the runlist but for it to be QENABLED
1384 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1385 * as well as INSERVICE.
1386 */
1387 void
1388 wait_svc(queue_t *qp)
1389 {
1390 queue_t *wqp = _WR(qp);
1391
1392 ASSERT(qp->q_flag & QREADR);
1393
1394 /*
1395 * Try to remove queues from qhead/qtail list.
1396 */
1397 if (qhead != NULL) {
1398 remove_runlist(qp);
1399 remove_runlist(wqp);
1400 }
1401 /*
1402 * Wait till the syncqs associated with the queue disappear from the
1403 * background processing list.
1404 * This only needs to be done for non-PERMOD perimeters since
1405 * for PERMOD perimeters the syncq may be shared and will only be freed
1406 * when the last module/driver is unloaded.
1407 * If for PERMOD perimeters queue was on the syncq list, removeq()
1408 * should call propagate_syncq() or drain_syncq() for it. Both of these
1409 * functions remove the queue from its syncq list, so sqthread will not
1410 * try to access the queue.
1411 */
1412 if (!(qp->q_flag & QPERMOD)) {
1413 syncq_t *rsq = qp->q_syncq;
1414 syncq_t *wsq = wqp->q_syncq;
1415
1416 /*
1417 * Disable rsq and wsq and wait for any background processing of
1418 * syncq to complete.
1419 */
1420 wait_sq_svc(rsq);
1421 if (wsq != rsq)
1422 wait_sq_svc(wsq);
1423 }
1424
1425 mutex_enter(QLOCK(qp));
1426 while (qp->q_flag & (QINSERVICE|QENAB))
1427 cv_wait(&qp->q_wait, QLOCK(qp));
1428 mutex_exit(QLOCK(qp));
1429 mutex_enter(QLOCK(wqp));
1430 while (wqp->q_flag & (QINSERVICE|QENAB))
1431 cv_wait(&wqp->q_wait, QLOCK(wqp));
1432 mutex_exit(QLOCK(wqp));
1433 }
1434
1435 /*
1436 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1437 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1438 * also be set, and is passed through to allocb_cred_wait().
1439 *
1440 * Returns errno on failure, zero on success.
1441 */
1442 int
1443 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1444 {
1445 mblk_t *tmp;
1446 ssize_t count;
1447 int error = 0;
1448
1449 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1450 (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1451
1452 if (bp->b_datap->db_type == M_IOCTL) {
1453 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1454 } else {
1455 ASSERT(bp->b_datap->db_type == M_COPYIN);
1456 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1457 }
1458 /*
1459 * strdoioctl validates ioc_count, so if this assert fails it
1460 * cannot be due to user error.
1461 */
1462 ASSERT(count >= 0);
1463
1464 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1465 curproc->p_pid)) == NULL) {
1466 return (error);
1467 }
1468 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1469 if (error != 0) {
1470 freeb(tmp);
1471 return (error);
1472 }
1473 DB_CPID(tmp) = curproc->p_pid;
1474 tmp->b_wptr += count;
1475 bp->b_cont = tmp;
1476
1477 return (0);
1478 }
1479
1480 /*
1481 * Copy ioctl data to user-land. Return non-zero errno on failure,
1482 * 0 for success.
1483 */
1484 int
1485 getiocd(mblk_t *bp, char *arg, int copymode)
1486 {
1487 ssize_t count;
1488 size_t n;
1489 int error;
1490
1491 if (bp->b_datap->db_type == M_IOCACK)
1492 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1493 else {
1494 ASSERT(bp->b_datap->db_type == M_COPYOUT);
1495 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1496 }
1497 ASSERT(count >= 0);
1498
1499 for (bp = bp->b_cont; bp && count;
1500 count -= n, bp = bp->b_cont, arg += n) {
1501 n = MIN(count, bp->b_wptr - bp->b_rptr);
1502 error = strcopyout(bp->b_rptr, arg, n, copymode);
1503 if (error)
1504 return (error);
1505 }
1506 ASSERT(count == 0);
1507 return (0);
1508 }
1509
1510 /*
1511 * Allocate a linkinfo entry given the write queue of the
1512 * bottom module of the top stream and the write queue of the
1513 * stream head of the bottom stream.
1514 */
1515 linkinfo_t *
1516 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1517 {
1518 linkinfo_t *linkp;
1519
1520 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1521
1522 linkp->li_lblk.l_qtop = qup;
1523 linkp->li_lblk.l_qbot = qdown;
1524 linkp->li_fpdown = fpdown;
1525
1526 mutex_enter(&strresources);
1527 linkp->li_next = linkinfo_list;
1528 linkp->li_prev = NULL;
1529 if (linkp->li_next)
1530 linkp->li_next->li_prev = linkp;
1531 linkinfo_list = linkp;
1532 linkp->li_lblk.l_index = ++lnk_id;
1533 ASSERT(lnk_id != 0); /* this should never wrap in practice */
1534 mutex_exit(&strresources);
1535
1536 return (linkp);
1537 }
1538
1539 /*
1540 * Free a linkinfo entry.
1541 */
1542 void
1543 lbfree(linkinfo_t *linkp)
1544 {
1545 mutex_enter(&strresources);
1546 if (linkp->li_next)
1547 linkp->li_next->li_prev = linkp->li_prev;
1548 if (linkp->li_prev)
1549 linkp->li_prev->li_next = linkp->li_next;
1550 else
1551 linkinfo_list = linkp->li_next;
1552 mutex_exit(&strresources);
1553
1554 kmem_cache_free(linkinfo_cache, linkp);
1555 }
1556
1557 /*
1558 * Check for a potential linking cycle.
1559 * Return 1 if a link will result in a cycle,
1560 * and 0 otherwise.
1561 */
1562 int
1563 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1564 {
1565 struct mux_node *np;
1566 struct mux_edge *ep;
1567 int i;
1568 major_t lomaj;
1569 major_t upmaj;
1570 /*
1571 * if the lower stream is a pipe/FIFO, return, since link
1572 * cycles can not happen on pipes/FIFOs
1573 */
1574 if (lostp->sd_vnode->v_type == VFIFO)
1575 return (0);
1576
1577 for (i = 0; i < ss->ss_devcnt; i++) {
1578 np = &ss->ss_mux_nodes[i];
1579 MUX_CLEAR(np);
1580 }
1581 lomaj = getmajor(lostp->sd_vnode->v_rdev);
1582 upmaj = getmajor(upstp->sd_vnode->v_rdev);
1583 np = &ss->ss_mux_nodes[lomaj];
1584 for (;;) {
1585 if (!MUX_DIDVISIT(np)) {
1586 if (np->mn_imaj == upmaj)
1587 return (1);
1588 if (np->mn_outp == NULL) {
1589 MUX_VISIT(np);
1590 if (np->mn_originp == NULL)
1591 return (0);
1592 np = np->mn_originp;
1593 continue;
1594 }
1595 MUX_VISIT(np);
1596 np->mn_startp = np->mn_outp;
1597 } else {
1598 if (np->mn_startp == NULL) {
1599 if (np->mn_originp == NULL)
1600 return (0);
1601 else {
1602 np = np->mn_originp;
1603 continue;
1604 }
1605 }
1606 /*
1607 * If ep->me_nodep is a FIFO (me_nodep == NULL),
1608 * ignore the edge and move on. ep->me_nodep gets
1609 * set to NULL in mux_addedge() if it is a FIFO.
1610 *
1611 */
1612 ep = np->mn_startp;
1613 np->mn_startp = ep->me_nextp;
1614 if (ep->me_nodep == NULL)
1615 continue;
1616 ep->me_nodep->mn_originp = np;
1617 np = ep->me_nodep;
1618 }
1619 }
1620 }
1621
1622 /*
1623 * Find linkinfo entry corresponding to the parameters.
1624 */
1625 linkinfo_t *
1626 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1627 {
1628 linkinfo_t *linkp;
1629 struct mux_edge *mep;
1630 struct mux_node *mnp;
1631 queue_t *qup;
1632
1633 mutex_enter(&strresources);
1634 if ((type & LINKTYPEMASK) == LINKNORMAL) {
1635 qup = getendq(stp->sd_wrq);
1636 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1637 if ((qup == linkp->li_lblk.l_qtop) &&
1638 (!index || (index == linkp->li_lblk.l_index))) {
1639 mutex_exit(&strresources);
1640 return (linkp);
1641 }
1642 }
1643 } else {
1644 ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1645 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1646 mep = mnp->mn_outp;
1647 while (mep) {
1648 if ((index == 0) || (index == mep->me_muxid))
1649 break;
1650 mep = mep->me_nextp;
1651 }
1652 if (!mep) {
1653 mutex_exit(&strresources);
1654 return (NULL);
1655 }
1656 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1657 if ((!linkp->li_lblk.l_qtop) &&
1658 (mep->me_muxid == linkp->li_lblk.l_index)) {
1659 mutex_exit(&strresources);
1660 return (linkp);
1661 }
1662 }
1663 }
1664 mutex_exit(&strresources);
1665 return (NULL);
1666 }
1667
1668 /*
1669 * Given a queue ptr, follow the chain of q_next pointers until you reach the
1670 * last queue on the chain and return it.
1671 */
1672 queue_t *
1673 getendq(queue_t *q)
1674 {
1675 ASSERT(q != NULL);
1676 while (_SAMESTR(q))
1677 q = q->q_next;
1678 return (q);
1679 }
1680
1681 /*
1682 * Wait for the syncq count to drop to zero.
1683 * sq could be either outer or inner.
1684 */
1685
1686 static void
1687 wait_syncq(syncq_t *sq)
1688 {
1689 uint16_t count;
1690
1691 mutex_enter(SQLOCK(sq));
1692 count = sq->sq_count;
1693 SQ_PUTLOCKS_ENTER(sq);
1694 SUM_SQ_PUTCOUNTS(sq, count);
1695 while (count != 0) {
1696 sq->sq_flags |= SQ_WANTWAKEUP;
1697 SQ_PUTLOCKS_EXIT(sq);
1698 cv_wait(&sq->sq_wait, SQLOCK(sq));
1699 count = sq->sq_count;
1700 SQ_PUTLOCKS_ENTER(sq);
1701 SUM_SQ_PUTCOUNTS(sq, count);
1702 }
1703 SQ_PUTLOCKS_EXIT(sq);
1704 mutex_exit(SQLOCK(sq));
1705 }
1706
1707 /*
1708 * Wait while there are any messages for the queue in its syncq.
1709 */
1710 static void
1711 wait_q_syncq(queue_t *q)
1712 {
1713 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1714 syncq_t *sq = q->q_syncq;
1715
1716 mutex_enter(SQLOCK(sq));
1717 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1718 sq->sq_flags |= SQ_WANTWAKEUP;
1719 cv_wait(&sq->sq_wait, SQLOCK(sq));
1720 }
1721 mutex_exit(SQLOCK(sq));
1722 }
1723 }
1724
1725
1726 int
1727 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1728 int lhlink)
1729 {
1730 struct stdata *stp;
1731 struct strioctl strioc;
1732 struct linkinfo *linkp;
1733 struct stdata *stpdown;
1734 struct streamtab *str;
1735 queue_t *passq;
1736 syncq_t *passyncq;
1737 queue_t *rq;
1738 cdevsw_impl_t *dp;
1739 uint32_t qflag;
1740 uint32_t sqtype;
1741 perdm_t *dmp;
1742 int error = 0;
1743 netstack_t *ns;
1744 str_stack_t *ss;
1745
1746 stp = vp->v_stream;
1747 TRACE_1(TR_FAC_STREAMS_FR,
1748 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1749 /*
1750 * Test for invalid upper stream
1751 */
1752 if (stp->sd_flag & STRHUP) {
1753 return (ENXIO);
1754 }
1755 if (vp->v_type == VFIFO) {
1756 return (EINVAL);
1757 }
1758 if (stp->sd_strtab == NULL) {
1759 return (EINVAL);
1760 }
1761 if (!stp->sd_strtab->st_muxwinit) {
1762 return (EINVAL);
1763 }
1764 if (fpdown == NULL) {
1765 return (EBADF);
1766 }
1767 ns = netstack_find_by_cred(crp);
1768 ASSERT(ns != NULL);
1769 ss = ns->netstack_str;
1770 ASSERT(ss != NULL);
1771
1772 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1773 netstack_rele(ss->ss_netstack);
1774 return (EINVAL);
1775 }
1776 mutex_enter(&muxifier);
1777 if (stp->sd_flag & STPLEX) {
1778 mutex_exit(&muxifier);
1779 netstack_rele(ss->ss_netstack);
1780 return (ENXIO);
1781 }
1782
1783 /*
1784 * Test for invalid lower stream.
1785 * The check for the v_type != VFIFO and having a major
1786 * number not >= devcnt is done to avoid problems with
1787 * adding mux_node entry past the end of mux_nodes[].
1788 * For FIFO's we don't add an entry so this isn't a
1789 * problem.
1790 */
1791 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1792 (stpdown == stp) || (stpdown->sd_flag &
1793 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1794 ((stpdown->sd_vnode->v_type != VFIFO) &&
1795 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1796 linkcycle(stp, stpdown, ss)) {
1797 mutex_exit(&muxifier);
1798 netstack_rele(ss->ss_netstack);
1799 return (EINVAL);
1800 }
1801 TRACE_1(TR_FAC_STREAMS_FR,
1802 TR_STPDOWN, "stpdown:%p", stpdown);
1803 rq = getendq(stp->sd_wrq);
1804 if (cmd == I_PLINK)
1805 rq = NULL;
1806
1807 linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1808
1809 strioc.ic_cmd = cmd;
1810 strioc.ic_timout = INFTIM;
1811 strioc.ic_len = sizeof (struct linkblk);
1812 strioc.ic_dp = (char *)&linkp->li_lblk;
1813
1814 /*
1815 * STRPLUMB protects plumbing changes and should be set before
1816 * link_addpassthru()/link_rempassthru() are called, so it is set here
1817 * and cleared in the end of mlink when passthru queue is removed.
1818 * Setting of STRPLUMB prevents reopens of the stream while passthru
1819 * queue is in-place (it is not a proper module and doesn't have open
1820 * entry point).
1821 *
1822 * STPLEX prevents any threads from entering the stream from above. It
1823 * can't be set before the call to link_addpassthru() because putnext
1824 * from below may cause stream head I/O routines to be called and these
1825 * routines assert that STPLEX is not set. After link_addpassthru()
1826 * nothing may come from below since the pass queue syncq is blocked.
1827 * Note also that STPLEX should be cleared before the call to
1828 * link_rempassthru() since when messages start flowing to the stream
1829 * head (e.g. because of message propagation from the pass queue) stream
1830 * head I/O routines may be called with STPLEX flag set.
1831 *
1832 * When STPLEX is set, nothing may come into the stream from above and
1833 * it is safe to do a setq which will change stream head. So, the
1834 * correct sequence of actions is:
1835 *
1836 * 1) Set STRPLUMB
1837 * 2) Call link_addpassthru()
1838 * 3) Set STPLEX
1839 * 4) Call setq and update the stream state
1840 * 5) Clear STPLEX
1841 * 6) Call link_rempassthru()
1842 * 7) Clear STRPLUMB
1843 *
1844 * The same sequence applies to munlink() code.
1845 */
1846 mutex_enter(&stpdown->sd_lock);
1847 stpdown->sd_flag |= STRPLUMB;
1848 mutex_exit(&stpdown->sd_lock);
1849 /*
1850 * Add passthru queue below lower mux. This will block
1851 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1852 */
1853 passq = link_addpassthru(stpdown);
1854
1855 mutex_enter(&stpdown->sd_lock);
1856 stpdown->sd_flag |= STPLEX;
1857 mutex_exit(&stpdown->sd_lock);
1858
1859 rq = _RD(stpdown->sd_wrq);
1860 /*
1861 * There may be messages in the streamhead's syncq due to messages
1862 * that arrived before link_addpassthru() was done. To avoid
1863 * background processing of the syncq happening simultaneous with
1864 * setq processing, we disable the streamhead syncq and wait until
1865 * existing background thread finishes working on it.
1866 */
1867 wait_sq_svc(rq->q_syncq);
1868 passyncq = passq->q_syncq;
1869 if (!(passyncq->sq_flags & SQ_BLOCKED))
1870 blocksq(passyncq, SQ_BLOCKED, 0);
1871
1872 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1873 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1874 rq->q_ptr = _WR(rq)->q_ptr = NULL;
1875
1876 /* setq might sleep in allocator - avoid holding locks. */
1877 /* Note: we are holding muxifier here. */
1878
1879 str = stp->sd_strtab;
1880 dp = &devimpl[getmajor(vp->v_rdev)];
1881 ASSERT(dp->d_str == str);
1882
1883 qflag = dp->d_qflag;
1884 sqtype = dp->d_sqtype;
1885
1886 /* create perdm_t if needed */
1887 if (NEED_DM(dp->d_dmp, qflag))
1888 dp->d_dmp = hold_dm(str, qflag, sqtype);
1889
1890 dmp = dp->d_dmp;
1891
1892 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1893 B_TRUE);
1894
1895 /*
1896 * XXX Remove any "odd" messages from the queue.
1897 * Keep only M_DATA, M_PROTO, M_PCPROTO.
1898 */
1899 error = strdoioctl(stp, &strioc, FNATIVE,
1900 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1901 if (error != 0) {
1902 lbfree(linkp);
1903
1904 if (!(passyncq->sq_flags & SQ_BLOCKED))
1905 blocksq(passyncq, SQ_BLOCKED, 0);
1906 /*
1907 * Restore the stream head queue and then remove
1908 * the passq. Turn off STPLEX before we turn on
1909 * the stream by removing the passq.
1910 */
1911 rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1912 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1913 B_TRUE);
1914
1915 mutex_enter(&stpdown->sd_lock);
1916 stpdown->sd_flag &= ~STPLEX;
1917 mutex_exit(&stpdown->sd_lock);
1918
1919 link_rempassthru(passq);
1920
1921 mutex_enter(&stpdown->sd_lock);
1922 stpdown->sd_flag &= ~STRPLUMB;
1923 /* Wakeup anyone waiting for STRPLUMB to clear. */
1924 cv_broadcast(&stpdown->sd_monitor);
1925 mutex_exit(&stpdown->sd_lock);
1926
1927 mutex_exit(&muxifier);
1928 netstack_rele(ss->ss_netstack);
1929 return (error);
1930 }
1931 mutex_enter(&fpdown->f_tlock);
1932 fpdown->f_count++;
1933 mutex_exit(&fpdown->f_tlock);
1934
1935 /*
1936 * if we've made it here the linkage is all set up so we should also
1937 * set up the layered driver linkages
1938 */
1939
1940 ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1941 if (cmd == I_LINK) {
1942 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1943 } else {
1944 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1945 }
1946
1947 link_rempassthru(passq);
1948
1949 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1950
1951 /*
1952 * Mark the upper stream as having dependent links
1953 * so that strclose can clean it up.
1954 */
1955 if (cmd == I_LINK) {
1956 mutex_enter(&stp->sd_lock);
1957 stp->sd_flag |= STRHASLINKS;
1958 mutex_exit(&stp->sd_lock);
1959 }
1960 /*
1961 * Wake up any other processes that may have been
1962 * waiting on the lower stream. These will all
1963 * error out.
1964 */
1965 mutex_enter(&stpdown->sd_lock);
1966 /* The passthru module is removed so we may release STRPLUMB */
1967 stpdown->sd_flag &= ~STRPLUMB;
1968 cv_broadcast(&rq->q_wait);
1969 cv_broadcast(&_WR(rq)->q_wait);
1970 cv_broadcast(&stpdown->sd_monitor);
1971 mutex_exit(&stpdown->sd_lock);
1972 mutex_exit(&muxifier);
1973 *rvalp = linkp->li_lblk.l_index;
1974 netstack_rele(ss->ss_netstack);
1975 return (0);
1976 }
1977
1978 int
1979 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1980 {
1981 int ret;
1982 struct file *fpdown;
1983
1984 fpdown = getf(arg);
1985 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1986 if (fpdown != NULL)
1987 releasef(arg);
1988 return (ret);
1989 }
1990
1991 /*
1992 * Unlink a multiplexor link. Stp is the controlling stream for the
1993 * link, and linkp points to the link's entry in the linkinfo list.
1994 * The muxifier lock must be held on entry and is dropped on exit.
1995 *
1996 * NOTE : Currently it is assumed that mux would process all the messages
1997 * sitting on it's queue before ACKing the UNLINK. It is the responsibility
1998 * of the mux to handle all the messages that arrive before UNLINK.
1999 * If the mux has to send down messages on its lower stream before
2000 * ACKing I_UNLINK, then it *should* know to handle messages even
2001 * after the UNLINK is acked (actually it should be able to handle till we
2002 * re-block the read side of the pass queue here). If the mux does not
2003 * open up the lower stream, any messages that arrive during UNLINK
2004 * will be put in the stream head. In the case of lower stream opening
2005 * up, some messages might land in the stream head depending on when
2006 * the message arrived and when the read side of the pass queue was
2007 * re-blocked.
2008 */
2009 int
2010 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2011 str_stack_t *ss)
2012 {
2013 struct strioctl strioc;
2014 struct stdata *stpdown;
2015 queue_t *rq, *wrq;
2016 queue_t *passq;
2017 syncq_t *passyncq;
2018 int error = 0;
2019 file_t *fpdown;
2020
2021 ASSERT(MUTEX_HELD(&muxifier));
2022
2023 stpdown = linkp->li_fpdown->f_vnode->v_stream;
2024
2025 /*
2026 * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2027 */
2028 mutex_enter(&stpdown->sd_lock);
2029 stpdown->sd_flag |= STRPLUMB;
2030 mutex_exit(&stpdown->sd_lock);
2031
2032 /*
2033 * Add passthru queue below lower mux. This will block
2034 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2035 */
2036 passq = link_addpassthru(stpdown);
2037
2038 if ((flag & LINKTYPEMASK) == LINKNORMAL)
2039 strioc.ic_cmd = I_UNLINK;
2040 else
2041 strioc.ic_cmd = I_PUNLINK;
2042 strioc.ic_timout = INFTIM;
2043 strioc.ic_len = sizeof (struct linkblk);
2044 strioc.ic_dp = (char *)&linkp->li_lblk;
2045
2046 error = strdoioctl(stp, &strioc, FNATIVE,
2047 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2048
2049 /*
2050 * If there was an error and this is not called via strclose,
2051 * return to the user. Otherwise, pretend there was no error
2052 * and close the link.
2053 */
2054 if (error) {
2055 if (flag & LINKCLOSE) {
2056 cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2057 "unlink ioctl, closing anyway (%d)\n", error);
2058 } else {
2059 link_rempassthru(passq);
2060 mutex_enter(&stpdown->sd_lock);
2061 stpdown->sd_flag &= ~STRPLUMB;
2062 cv_broadcast(&stpdown->sd_monitor);
2063 mutex_exit(&stpdown->sd_lock);
2064 mutex_exit(&muxifier);
2065 return (error);
2066 }
2067 }
2068
2069 mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2070 fpdown = linkp->li_fpdown;
2071 lbfree(linkp);
2072
2073 /*
2074 * We go ahead and drop muxifier here--it's a nasty global lock that
2075 * can slow others down. It's okay to since attempts to mlink() this
2076 * stream will be stopped because STPLEX is still set in the stdata
2077 * structure, and munlink() is stopped because mux_rmvedge() and
2078 * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2079 * respectively. Note that we defer the closef() of fpdown until
2080 * after we drop muxifier since strclose() can call munlinkall().
2081 */
2082 mutex_exit(&muxifier);
2083
2084 wrq = stpdown->sd_wrq;
2085 rq = _RD(wrq);
2086
2087 /*
2088 * Get rid of outstanding service procedure runs, before we make
2089 * it a stream head, since a stream head doesn't have any service
2090 * procedure.
2091 */
2092 disable_svc(rq);
2093 wait_svc(rq);
2094
2095 /*
2096 * Since we don't disable the syncq for QPERMOD, we wait for whatever
2097 * is queued up to be finished. mux should take care that nothing is
2098 * send down to this queue. We should do it now as we're going to block
2099 * passyncq if it was unblocked.
2100 */
2101 if (wrq->q_flag & QPERMOD) {
2102 syncq_t *sq = wrq->q_syncq;
2103
2104 mutex_enter(SQLOCK(sq));
2105 while (wrq->q_sqflags & Q_SQQUEUED) {
2106 sq->sq_flags |= SQ_WANTWAKEUP;
2107 cv_wait(&sq->sq_wait, SQLOCK(sq));
2108 }
2109 mutex_exit(SQLOCK(sq));
2110 }
2111 passyncq = passq->q_syncq;
2112 if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2113
2114 syncq_t *sq, *outer;
2115
2116 /*
2117 * Messages could be flowing from underneath. We will
2118 * block the read side of the passq. This would be
2119 * sufficient for QPAIR and QPERQ muxes to ensure
2120 * that no data is flowing up into this queue
2121 * and hence no thread active in this instance of
2122 * lower mux. But for QPERMOD and QMTOUTPERIM there
2123 * could be messages on the inner and outer/inner
2124 * syncqs respectively. We will wait for them to drain.
2125 * Because passq is blocked messages end up in the syncq
2126 * And qfill_syncq could possibly end up setting QFULL
2127 * which will access the rq->q_flag. Hence, we have to
2128 * acquire the QLOCK in setq.
2129 *
2130 * XXX Messages can also flow from top into this
2131 * queue though the unlink is over (Ex. some instance
2132 * in putnext() called from top that has still not
2133 * accessed this queue. And also putq(lowerq) ?).
2134 * Solution : How about blocking the l_qtop queue ?
2135 * Do we really care about such pure D_MP muxes ?
2136 */
2137
2138 blocksq(passyncq, SQ_BLOCKED, 0);
2139
2140 sq = rq->q_syncq;
2141 if ((outer = sq->sq_outer) != NULL) {
2142
2143 /*
2144 * We have to just wait for the outer sq_count
2145 * drop to zero. As this does not prevent new
2146 * messages to enter the outer perimeter, this
2147 * is subject to starvation.
2148 *
2149 * NOTE :Because of blocksq above, messages could
2150 * be in the inner syncq only because of some
2151 * thread holding the outer perimeter exclusively.
2152 * Hence it would be sufficient to wait for the
2153 * exclusive holder of the outer perimeter to drain
2154 * the inner and outer syncqs. But we will not depend
2155 * on this feature and hence check the inner syncqs
2156 * separately.
2157 */
2158 wait_syncq(outer);
2159 }
2160
2161
2162 /*
2163 * There could be messages destined for
2164 * this queue. Let the exclusive holder
2165 * drain it.
2166 */
2167
2168 wait_syncq(sq);
2169 ASSERT((rq->q_flag & QPERMOD) ||
2170 ((rq->q_syncq->sq_head == NULL) &&
2171 (_WR(rq)->q_syncq->sq_head == NULL)));
2172 }
2173
2174 /*
2175 * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2176 * case as we don't disable its syncq or remove it off the syncq
2177 * service list.
2178 */
2179 if (rq->q_flag & QPERMOD) {
2180 syncq_t *sq = rq->q_syncq;
2181
2182 mutex_enter(SQLOCK(sq));
2183 while (rq->q_sqflags & Q_SQQUEUED) {
2184 sq->sq_flags |= SQ_WANTWAKEUP;
2185 cv_wait(&sq->sq_wait, SQLOCK(sq));
2186 }
2187 mutex_exit(SQLOCK(sq));
2188 }
2189
2190 /*
2191 * flush_syncq changes states only when there are some messages to
2192 * free, i.e. when it returns non-zero value to return.
2193 */
2194 ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2195 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2196
2197 /*
2198 * Nobody else should know about this queue now.
2199 * If the mux did not process the messages before
2200 * acking the I_UNLINK, free them now.
2201 */
2202
2203 flushq(rq, FLUSHALL);
2204 flushq(_WR(rq), FLUSHALL);
2205
2206 /*
2207 * Convert the mux lower queue into a stream head queue.
2208 * Turn off STPLEX before we turn on the stream by removing the passq.
2209 */
2210 rq->q_ptr = wrq->q_ptr = stpdown;
2211 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2212
2213 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2214 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2215
2216 enable_svc(rq);
2217
2218 /*
2219 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2220 * needs to be set to prevent reopen() of the stream - such reopen may
2221 * try to call non-existent pass queue open routine and panic.
2222 */
2223 mutex_enter(&stpdown->sd_lock);
2224 stpdown->sd_flag &= ~STPLEX;
2225 mutex_exit(&stpdown->sd_lock);
2226
2227 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2228 ((flag & LINKTYPEMASK) == LINKPERSIST));
2229
2230 /* clean up the layered driver linkages */
2231 if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2232 ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2233 } else {
2234 ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2235 }
2236
2237 link_rempassthru(passq);
2238
2239 /*
2240 * Now all plumbing changes are finished and STRPLUMB is no
2241 * longer needed.
2242 */
2243 mutex_enter(&stpdown->sd_lock);
2244 stpdown->sd_flag &= ~STRPLUMB;
2245 cv_broadcast(&stpdown->sd_monitor);
2246 mutex_exit(&stpdown->sd_lock);
2247
2248 (void) closef(fpdown);
2249 return (0);
2250 }
2251
2252 /*
2253 * Unlink all multiplexor links for which stp is the controlling stream.
2254 * Return 0, or a non-zero errno on failure.
2255 */
2256 int
2257 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2258 {
2259 linkinfo_t *linkp;
2260 int error = 0;
2261
2262 mutex_enter(&muxifier);
2263 while (linkp = findlinks(stp, 0, flag, ss)) {
2264 /*
2265 * munlink() releases the muxifier lock.
2266 */
2267 if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2268 return (error);
2269 mutex_enter(&muxifier);
2270 }
2271 mutex_exit(&muxifier);
2272 return (0);
2273 }
2274
2275 /*
2276 * A multiplexor link has been made. Add an
2277 * edge to the directed graph.
2278 */
2279 void
2280 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2281 {
2282 struct mux_node *np;
2283 struct mux_edge *ep;
2284 major_t upmaj;
2285 major_t lomaj;
2286
2287 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2288 lomaj = getmajor(lostp->sd_vnode->v_rdev);
2289 np = &ss->ss_mux_nodes[upmaj];
2290 if (np->mn_outp) {
2291 ep = np->mn_outp;
2292 while (ep->me_nextp)
2293 ep = ep->me_nextp;
2294 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2295 ep = ep->me_nextp;
2296 } else {
2297 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2298 ep = np->mn_outp;
2299 }
2300 ep->me_nextp = NULL;
2301 ep->me_muxid = muxid;
2302 /*
2303 * Save the dev_t for the purposes of str_stack_shutdown.
2304 * str_stack_shutdown assumes that the device allows reopen, since
2305 * this dev_t is the one after any cloning by xx_open().
2306 * Would prefer finding the dev_t from before any cloning,
2307 * but specfs doesn't retain that.
2308 */
2309 ep->me_dev = upstp->sd_vnode->v_rdev;
2310 if (lostp->sd_vnode->v_type == VFIFO)
2311 ep->me_nodep = NULL;
2312 else
2313 ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2314 }
2315
2316 /*
2317 * A multiplexor link has been removed. Remove the
2318 * edge in the directed graph.
2319 */
2320 void
2321 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2322 {
2323 struct mux_node *np;
2324 struct mux_edge *ep;
2325 struct mux_edge *pep = NULL;
2326 major_t upmaj;
2327
2328 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2329 np = &ss->ss_mux_nodes[upmaj];
2330 ASSERT(np->mn_outp != NULL);
2331 ep = np->mn_outp;
2332 while (ep) {
2333 if (ep->me_muxid == muxid) {
2334 if (pep)
2335 pep->me_nextp = ep->me_nextp;
2336 else
2337 np->mn_outp = ep->me_nextp;
2338 kmem_free(ep, sizeof (struct mux_edge));
2339 return;
2340 }
2341 pep = ep;
2342 ep = ep->me_nextp;
2343 }
2344 ASSERT(0); /* should not reach here */
2345 }
2346
2347 /*
2348 * Translate the device flags (from conf.h) to the corresponding
2349 * qflag and sq_flag (type) values.
2350 */
2351 int
2352 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2353 uint32_t *sqtypep)
2354 {
2355 uint32_t qflag = 0;
2356 uint32_t sqtype = 0;
2357
2358 if (devflag & _D_OLD)
2359 goto bad;
2360
2361 /* Inner perimeter presence and scope */
2362 switch (devflag & D_MTINNER_MASK) {
2363 case D_MP:
2364 qflag |= QMTSAFE;
2365 sqtype |= SQ_CI;
2366 break;
2367 case D_MTPERQ|D_MP:
2368 qflag |= QPERQ;
2369 break;
2370 case D_MTQPAIR|D_MP:
2371 qflag |= QPAIR;
2372 break;
2373 case D_MTPERMOD|D_MP:
2374 qflag |= QPERMOD;
2375 break;
2376 default:
2377 goto bad;
2378 }
2379
2380 /* Outer perimeter */
2381 if (devflag & D_MTOUTPERIM) {
2382 switch (devflag & D_MTINNER_MASK) {
2383 case D_MP:
2384 case D_MTPERQ|D_MP:
2385 case D_MTQPAIR|D_MP:
2386 break;
2387 default:
2388 goto bad;
2389 }
2390 qflag |= QMTOUTPERIM;
2391 }
2392
2393 /* Inner perimeter modifiers */
2394 if (devflag & D_MTINNER_MOD) {
2395 switch (devflag & D_MTINNER_MASK) {
2396 case D_MP:
2397 goto bad;
2398 default:
2399 break;
2400 }
2401 if (devflag & D_MTPUTSHARED)
2402 sqtype |= SQ_CIPUT;
2403 if (devflag & _D_MTOCSHARED) {
2404 /*
2405 * The code in putnext assumes that it has the
2406 * highest concurrency by not checking sq_count.
2407 * Thus _D_MTOCSHARED can only be supported when
2408 * D_MTPUTSHARED is set.
2409 */
2410 if (!(devflag & D_MTPUTSHARED))
2411 goto bad;
2412 sqtype |= SQ_CIOC;
2413 }
2414 if (devflag & _D_MTCBSHARED) {
2415 /*
2416 * The code in putnext assumes that it has the
2417 * highest concurrency by not checking sq_count.
2418 * Thus _D_MTCBSHARED can only be supported when
2419 * D_MTPUTSHARED is set.
2420 */
2421 if (!(devflag & D_MTPUTSHARED))
2422 goto bad;
2423 sqtype |= SQ_CICB;
2424 }
2425 if (devflag & _D_MTSVCSHARED) {
2426 /*
2427 * The code in putnext assumes that it has the
2428 * highest concurrency by not checking sq_count.
2429 * Thus _D_MTSVCSHARED can only be supported when
2430 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2431 * supported only for QPERMOD.
2432 */
2433 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2434 goto bad;
2435 sqtype |= SQ_CISVC;
2436 }
2437 }
2438
2439 /* Default outer perimeter concurrency */
2440 sqtype |= SQ_CO;
2441
2442 /* Outer perimeter modifiers */
2443 if (devflag & D_MTOCEXCL) {
2444 if (!(devflag & D_MTOUTPERIM)) {
2445 /* No outer perimeter */
2446 goto bad;
2447 }
2448 sqtype &= ~SQ_COOC;
2449 }
2450
2451 /* Synchronous Streams extended qinit structure */
2452 if (devflag & D_SYNCSTR)
2453 qflag |= QSYNCSTR;
2454
2455 /*
2456 * Private flag used by a transport module to indicate
2457 * to sockfs that it supports direct-access mode without
2458 * having to go through STREAMS.
2459 */
2460 if (devflag & _D_DIRECT) {
2461 /* Reject unless the module is fully-MT (no perimeter) */
2462 if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2463 goto bad;
2464 qflag |= _QDIRECT;
2465 }
2466
2467 *qflagp = qflag;
2468 *sqtypep = sqtype;
2469 return (0);
2470
2471 bad:
2472 cmn_err(CE_WARN,
2473 "stropen: bad MT flags (0x%x) in driver '%s'",
2474 (int)(qflag & D_MTSAFETY_MASK),
2475 stp->st_rdinit->qi_minfo->mi_idname);
2476
2477 return (EINVAL);
2478 }
2479
2480 /*
2481 * Set the interface values for a pair of queues (qinit structure,
2482 * packet sizes, water marks).
2483 * setq assumes that the caller does not have a claim (entersq or claimq)
2484 * on the queue.
2485 */
2486 void
2487 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2488 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2489 {
2490 queue_t *wq;
2491 syncq_t *sq, *outer;
2492
2493 ASSERT(rq->q_flag & QREADR);
2494 ASSERT((qflag & QMT_TYPEMASK) != 0);
2495 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2496
2497 wq = _WR(rq);
2498 rq->q_qinfo = rinit;
2499 rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2500 rq->q_lowat = rinit->qi_minfo->mi_lowat;
2501 rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2502 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2503 wq->q_qinfo = winit;
2504 wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2505 wq->q_lowat = winit->qi_minfo->mi_lowat;
2506 wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2507 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2508
2509 /* Remove old syncqs */
2510 sq = rq->q_syncq;
2511 outer = sq->sq_outer;
2512 if (outer != NULL) {
2513 ASSERT(wq->q_syncq->sq_outer == outer);
2514 outer_remove(outer, rq->q_syncq);
2515 if (wq->q_syncq != rq->q_syncq)
2516 outer_remove(outer, wq->q_syncq);
2517 }
2518 ASSERT(sq->sq_outer == NULL);
2519 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2520
2521 if (sq != SQ(rq)) {
2522 if (!(rq->q_flag & QPERMOD))
2523 free_syncq(sq);
2524 if (wq->q_syncq == rq->q_syncq)
2525 wq->q_syncq = NULL;
2526 rq->q_syncq = NULL;
2527 }
2528 if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2529 wq->q_syncq != SQ(rq)) {
2530 free_syncq(wq->q_syncq);
2531 wq->q_syncq = NULL;
2532 }
2533 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2534 rq->q_syncq->sq_tail == NULL));
2535 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2536 wq->q_syncq->sq_tail == NULL));
2537
2538 if (!(rq->q_flag & QPERMOD) &&
2539 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2540 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2541 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2542 rq->q_syncq->sq_nciputctrl, 0);
2543 ASSERT(ciputctrl_cache != NULL);
2544 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2545 rq->q_syncq->sq_ciputctrl = NULL;
2546 rq->q_syncq->sq_nciputctrl = 0;
2547 }
2548
2549 if (!(wq->q_flag & QPERMOD) &&
2550 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2551 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2552 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2553 wq->q_syncq->sq_nciputctrl, 0);
2554 ASSERT(ciputctrl_cache != NULL);
2555 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2556 wq->q_syncq->sq_ciputctrl = NULL;
2557 wq->q_syncq->sq_nciputctrl = 0;
2558 }
2559
2560 sq = SQ(rq);
2561 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2562 ASSERT(sq->sq_outer == NULL);
2563 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2564
2565 /*
2566 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2567 * bits in sq_flag based on the sqtype.
2568 */
2569 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2570
2571 rq->q_syncq = wq->q_syncq = sq;
2572 sq->sq_type = sqtype;
2573 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2574
2575 /*
2576 * We are making sq_svcflags zero,
2577 * resetting SQ_DISABLED in case it was set by
2578 * wait_svc() in the munlink path.
2579 *
2580 */
2581 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2582 sq->sq_svcflags = 0;
2583
2584 /*
2585 * We need to acquire the lock here for the mlink and munlink case,
2586 * where canputnext, backenable, etc can access the q_flag.
2587 */
2588 if (lock_needed) {
2589 mutex_enter(QLOCK(rq));
2590 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2591 mutex_exit(QLOCK(rq));
2592 mutex_enter(QLOCK(wq));
2593 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2594 mutex_exit(QLOCK(wq));
2595 } else {
2596 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2597 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2598 }
2599
2600 if (qflag & QPERQ) {
2601 /* Allocate a separate syncq for the write side */
2602 sq = new_syncq();
2603 sq->sq_type = rq->q_syncq->sq_type;
2604 sq->sq_flags = rq->q_syncq->sq_flags;
2605 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2606 sq->sq_oprev == NULL);
2607 wq->q_syncq = sq;
2608 }
2609 if (qflag & QPERMOD) {
2610 sq = dmp->dm_sq;
2611
2612 /*
2613 * Assert that we do have an inner perimeter syncq and that it
2614 * does not have an outer perimeter associated with it.
2615 */
2616 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2617 sq->sq_oprev == NULL);
2618 rq->q_syncq = wq->q_syncq = sq;
2619 }
2620 if (qflag & QMTOUTPERIM) {
2621 outer = dmp->dm_sq;
2622
2623 ASSERT(outer->sq_outer == NULL);
2624 outer_insert(outer, rq->q_syncq);
2625 if (wq->q_syncq != rq->q_syncq)
2626 outer_insert(outer, wq->q_syncq);
2627 }
2628 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2629 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2630 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2631 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2632 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2633
2634 /*
2635 * Initialize struio() types.
2636 */
2637 rq->q_struiot =
2638 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2639 wq->q_struiot =
2640 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2641 }
2642
2643 perdm_t *
2644 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2645 {
2646 syncq_t *sq;
2647 perdm_t **pp;
2648 perdm_t *p;
2649 perdm_t *dmp;
2650
2651 ASSERT(str != NULL);
2652 ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2653
2654 rw_enter(&perdm_rwlock, RW_READER);
2655 for (p = perdm_list; p != NULL; p = p->dm_next) {
2656 if (p->dm_str == str) { /* found one */
2657 atomic_inc_32(&(p->dm_ref));
2658 rw_exit(&perdm_rwlock);
2659 return (p);
2660 }
2661 }
2662 rw_exit(&perdm_rwlock);
2663
2664 sq = new_syncq();
2665 if (qflag & QPERMOD) {
2666 sq->sq_type = sqtype | SQ_PERMOD;
2667 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2668 } else {
2669 ASSERT(qflag & QMTOUTPERIM);
2670 sq->sq_onext = sq->sq_oprev = sq;
2671 }
2672
2673 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2674 dmp->dm_sq = sq;
2675 dmp->dm_str = str;
2676 dmp->dm_ref = 1;
2677 dmp->dm_next = NULL;
2678
2679 rw_enter(&perdm_rwlock, RW_WRITER);
2680 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2681 if (p->dm_str == str) { /* already present */
2682 p->dm_ref++;
2683 rw_exit(&perdm_rwlock);
2684 free_syncq(sq);
2685 kmem_free(dmp, sizeof (perdm_t));
2686 return (p);
2687 }
2688 }
2689
2690 *pp = dmp;
2691 rw_exit(&perdm_rwlock);
2692 return (dmp);
2693 }
2694
2695 void
2696 rele_dm(perdm_t *dmp)
2697 {
2698 perdm_t **pp;
2699 perdm_t *p;
2700
2701 rw_enter(&perdm_rwlock, RW_WRITER);
2702 ASSERT(dmp->dm_ref > 0);
2703
2704 if (--dmp->dm_ref > 0) {
2705 rw_exit(&perdm_rwlock);
2706 return;
2707 }
2708
2709 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2710 if (p == dmp)
2711 break;
2712 ASSERT(p == dmp);
2713 *pp = p->dm_next;
2714 rw_exit(&perdm_rwlock);
2715
2716 /*
2717 * Wait for any background processing that relies on the
2718 * syncq to complete before it is freed.
2719 */
2720 wait_sq_svc(p->dm_sq);
2721 free_syncq(p->dm_sq);
2722 kmem_free(p, sizeof (perdm_t));
2723 }
2724
2725 /*
2726 * Make a protocol message given control and data buffers.
2727 * n.b., this can block; be careful of what locks you hold when calling it.
2728 *
2729 * If sd_maxblk is less than *iosize this routine can fail part way through
2730 * (due to an allocation failure). In this case on return *iosize will contain
2731 * the amount that was consumed. Otherwise *iosize will not be modified
2732 * i.e. it will contain the amount that was consumed.
2733 */
2734 int
2735 strmakemsg(
2736 struct strbuf *mctl,
2737 ssize_t *iosize,
2738 struct uio *uiop,
2739 stdata_t *stp,
2740 int32_t flag,
2741 mblk_t **mpp)
2742 {
2743 mblk_t *mpctl = NULL;
2744 mblk_t *mpdata = NULL;
2745 int error;
2746
2747 ASSERT(uiop != NULL);
2748
2749 *mpp = NULL;
2750 /* Create control part, if any */
2751 if ((mctl != NULL) && (mctl->len >= 0)) {
2752 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2753 if (error)
2754 return (error);
2755 }
2756 /* Create data part, if any */
2757 if (*iosize >= 0) {
2758 error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2759 if (error) {
2760 freemsg(mpctl);
2761 return (error);
2762 }
2763 }
2764 if (mpctl != NULL) {
2765 if (mpdata != NULL)
2766 linkb(mpctl, mpdata);
2767 *mpp = mpctl;
2768 } else {
2769 *mpp = mpdata;
2770 }
2771 return (0);
2772 }
2773
2774 /*
2775 * Make the control part of a protocol message given a control buffer.
2776 * n.b., this can block; be careful of what locks you hold when calling it.
2777 */
2778 int
2779 strmakectl(
2780 struct strbuf *mctl,
2781 int32_t flag,
2782 int32_t fflag,
2783 mblk_t **mpp)
2784 {
2785 mblk_t *bp = NULL;
2786 unsigned char msgtype;
2787 int error = 0;
2788 cred_t *cr = CRED();
2789
2790 /* We do not support interrupt threads using the stream head to send */
2791 ASSERT(cr != NULL);
2792
2793 *mpp = NULL;
2794 /*
2795 * Create control part of message, if any.
2796 */
2797 if ((mctl != NULL) && (mctl->len >= 0)) {
2798 caddr_t base;
2799 int ctlcount;
2800 int allocsz;
2801
2802 if (flag & RS_HIPRI)
2803 msgtype = M_PCPROTO;
2804 else
2805 msgtype = M_PROTO;
2806
2807 ctlcount = mctl->len;
2808 base = mctl->buf;
2809
2810 /*
2811 * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2812 * blocks by increasing the size to something more usable.
2813 */
2814 allocsz = MAX(ctlcount, 64);
2815
2816 /*
2817 * Range checking has already been done; simply try
2818 * to allocate a message block for the ctl part.
2819 */
2820 while ((bp = allocb_cred(allocsz, cr,
2821 curproc->p_pid)) == NULL) {
2822 if (fflag & (FNDELAY|FNONBLOCK))
2823 return (EAGAIN);
2824 if (error = strwaitbuf(allocsz, BPRI_MED))
2825 return (error);
2826 }
2827
2828 bp->b_datap->db_type = msgtype;
2829 if (copyin(base, bp->b_wptr, ctlcount)) {
2830 freeb(bp);
2831 return (EFAULT);
2832 }
2833 bp->b_wptr += ctlcount;
2834 }
2835 *mpp = bp;
2836 return (0);
2837 }
2838
2839 /*
2840 * Make a protocol message given data buffers.
2841 * n.b., this can block; be careful of what locks you hold when calling it.
2842 *
2843 * If sd_maxblk is less than *iosize this routine can fail part way through
2844 * (due to an allocation failure). In this case on return *iosize will contain
2845 * the amount that was consumed. Otherwise *iosize will not be modified
2846 * i.e. it will contain the amount that was consumed.
2847 */
2848 int
2849 strmakedata(
2850 ssize_t *iosize,
2851 struct uio *uiop,
2852 stdata_t *stp,
2853 int32_t flag,
2854 mblk_t **mpp)
2855 {
2856 mblk_t *mp = NULL;
2857 mblk_t *bp;
2858 int wroff = (int)stp->sd_wroff;
2859 int tail_len = (int)stp->sd_tail;
2860 int extra = wroff + tail_len;
2861 int error = 0;
2862 ssize_t maxblk;
2863 ssize_t count = *iosize;
2864 cred_t *cr;
2865
2866 *mpp = NULL;
2867 if (count < 0)
2868 return (0);
2869
2870 /* We do not support interrupt threads using the stream head to send */
2871 cr = CRED();
2872 ASSERT(cr != NULL);
2873
2874 maxblk = stp->sd_maxblk;
2875 if (maxblk == INFPSZ)
2876 maxblk = count;
2877
2878 /*
2879 * Create data part of message, if any.
2880 */
2881 do {
2882 ssize_t size;
2883 dblk_t *dp;
2884
2885 ASSERT(uiop);
2886
2887 size = MIN(count, maxblk);
2888
2889 while ((bp = allocb_cred(size + extra, cr,
2890 curproc->p_pid)) == NULL) {
2891 error = EAGAIN;
2892 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2893 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2894 if (count == *iosize) {
2895 freemsg(mp);
2896 return (error);
2897 } else {
2898 *iosize -= count;
2899 *mpp = mp;
2900 return (0);
2901 }
2902 }
2903 }
2904 dp = bp->b_datap;
2905 dp->db_cpid = curproc->p_pid;
2906 ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2907 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2908
2909 if (flag & STRUIO_POSTPONE) {
2910 /*
2911 * Setup the stream uio portion of the
2912 * dblk for subsequent use by struioget().
2913 */
2914 dp->db_struioflag = STRUIO_SPEC;
2915 dp->db_cksumstart = 0;
2916 dp->db_cksumstuff = 0;
2917 dp->db_cksumend = size;
2918 *(long long *)dp->db_struioun.data = 0ll;
2919 bp->b_wptr += size;
2920 } else {
2921 if (stp->sd_copyflag & STRCOPYCACHED)
2922 uiop->uio_extflg |= UIO_COPY_CACHED;
2923
2924 if (size != 0) {
2925 error = uiomove(bp->b_wptr, size, UIO_WRITE,
2926 uiop);
2927 if (error != 0) {
2928 freeb(bp);
2929 freemsg(mp);
2930 return (error);
2931 }
2932 }
2933 bp->b_wptr += size;
2934
2935 if (stp->sd_wputdatafunc != NULL) {
2936 mblk_t *newbp;
2937
2938 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2939 bp, NULL, NULL, NULL, NULL);
2940 if (newbp == NULL) {
2941 freeb(bp);
2942 freemsg(mp);
2943 return (ECOMM);
2944 }
2945 bp = newbp;
2946 }
2947 }
2948
2949 count -= size;
2950
2951 if (mp == NULL)
2952 mp = bp;
2953 else
2954 linkb(mp, bp);
2955 } while (count > 0);
2956
2957 *mpp = mp;
2958 return (0);
2959 }
2960
2961 /*
2962 * Wait for a buffer to become available. Return non-zero errno
2963 * if not able to wait, 0 if buffer is probably there.
2964 */
2965 int
2966 strwaitbuf(size_t size, int pri)
2967 {
2968 bufcall_id_t id;
2969
2970 mutex_enter(&bcall_monitor);
2971 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2972 &ttoproc(curthread)->p_flag_cv)) == 0) {
2973 mutex_exit(&bcall_monitor);
2974 return (ENOSR);
2975 }
2976 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2977 unbufcall(id);
2978 mutex_exit(&bcall_monitor);
2979 return (EINTR);
2980 }
2981 unbufcall(id);
2982 mutex_exit(&bcall_monitor);
2983 return (0);
2984 }
2985
2986 /*
2987 * This function waits for a read or write event to happen on a stream.
2988 * fmode can specify FNDELAY and/or FNONBLOCK.
2989 * The timeout is in ms with -1 meaning infinite.
2990 * The flag values work as follows:
2991 * READWAIT Check for read side errors, send M_READ
2992 * GETWAIT Check for read side errors, no M_READ
2993 * WRITEWAIT Check for write side errors.
2994 * NOINTR Do not return error if nonblocking or timeout.
2995 * STR_NOERROR Ignore all errors except STPLEX.
2996 * STR_NOSIG Ignore/hold signals during the duration of the call.
2997 * STR_PEEK Pass through the strgeterr().
2998 */
2999 int
3000 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3001 int *done)
3002 {
3003 int slpflg, errs;
3004 int error;
3005 kcondvar_t *sleepon;
3006 mblk_t *mp;
3007 ssize_t *rd_count;
3008 clock_t rval;
3009
3010 ASSERT(MUTEX_HELD(&stp->sd_lock));
3011 if ((flag & READWAIT) || (flag & GETWAIT)) {
3012 slpflg = RSLEEP;
3013 sleepon = &_RD(stp->sd_wrq)->q_wait;
3014 errs = STRDERR|STPLEX;
3015 } else {
3016 slpflg = WSLEEP;
3017 sleepon = &stp->sd_wrq->q_wait;
3018 errs = STWRERR|STRHUP|STPLEX;
3019 }
3020 if (flag & STR_NOERROR)
3021 errs = STPLEX;
3022
3023 if (stp->sd_wakeq & slpflg) {
3024 /*
3025 * A strwakeq() is pending, no need to sleep.
3026 */
3027 stp->sd_wakeq &= ~slpflg;
3028 *done = 0;
3029 return (0);
3030 }
3031
3032 if (stp->sd_flag & errs) {
3033 /*
3034 * Check for errors before going to sleep since the
3035 * caller might not have checked this while holding
3036 * sd_lock.
3037 */
3038 error = strgeterr(stp, errs, (flag & STR_PEEK));
3039 if (error != 0) {
3040 *done = 1;
3041 return (error);
3042 }
3043 }
3044
3045 /*
3046 * If any module downstream has requested read notification
3047 * by setting SNDMREAD flag using M_SETOPTS, send a message
3048 * down stream.
3049 */
3050 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3051 mutex_exit(&stp->sd_lock);
3052 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3053 (flag & STR_NOSIG), &error))) {
3054 mutex_enter(&stp->sd_lock);
3055 *done = 1;
3056 return (error);
3057 }
3058 mp->b_datap->db_type = M_READ;
3059 rd_count = (ssize_t *)mp->b_wptr;
3060 *rd_count = count;
3061 mp->b_wptr += sizeof (ssize_t);
3062 /*
3063 * Send the number of bytes requested by the
3064 * read as the argument to M_READ.
3065 */
3066 stream_willservice(stp);
3067 putnext(stp->sd_wrq, mp);
3068 stream_runservice(stp);
3069 mutex_enter(&stp->sd_lock);
3070
3071 /*
3072 * If any data arrived due to inline processing
3073 * of putnext(), don't sleep.
3074 */
3075 if (_RD(stp->sd_wrq)->q_first != NULL) {
3076 *done = 0;
3077 return (0);
3078 }
3079 }
3080
3081 if (fmode & (FNDELAY|FNONBLOCK)) {
3082 if (!(flag & NOINTR))
3083 error = EAGAIN;
3084 else
3085 error = 0;
3086 *done = 1;
3087 return (error);
3088 }
3089
3090 stp->sd_flag |= slpflg;
3091 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3092 "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3093 stp, flag, count, fmode, done);
3094
3095 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3096 if (rval > 0) {
3097 /* EMPTY */
3098 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3099 "strwaitq awakes(2):%X, %X, %X, %X, %X",
3100 stp, flag, count, fmode, done);
3101 } else if (rval == 0) {
3102 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3103 "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3104 stp, flag, count, fmode, done);
3105 stp->sd_flag &= ~slpflg;
3106 cv_broadcast(sleepon);
3107 if (!(flag & NOINTR))
3108 error = EINTR;
3109 else
3110 error = 0;
3111 *done = 1;
3112 return (error);
3113 } else {
3114 /* timeout */
3115 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3116 "strwaitq timeout:%p, %X, %lX, %X, %p",
3117 stp, flag, count, fmode, done);
3118 *done = 1;
3119 if (!(flag & NOINTR))
3120 return (ETIME);
3121 else
3122 return (0);
3123 }
3124 /*
3125 * If the caller implements delayed errors (i.e. queued after data)
3126 * we can not check for errors here since data as well as an
3127 * error might have arrived at the stream head. We return to
3128 * have the caller check the read queue before checking for errors.
3129 */
3130 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3131 error = strgeterr(stp, errs, (flag & STR_PEEK));
3132 if (error != 0) {
3133 *done = 1;
3134 return (error);
3135 }
3136 }
3137 *done = 0;
3138 return (0);
3139 }
3140
3141 /*
3142 * Perform job control discipline access checks.
3143 * Return 0 for success and the errno for failure.
3144 */
3145
3146 #define cantsend(p, t, sig) \
3147 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3148
3149 int
3150 straccess(struct stdata *stp, enum jcaccess mode)
3151 {
3152 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */
3153 kthread_t *t = curthread;
3154 proc_t *p = ttoproc(t);
3155 sess_t *sp;
3156
3157 ASSERT(mutex_owned(&stp->sd_lock));
3158
3159 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3160 return (0);
3161
3162 mutex_enter(&p->p_lock); /* protects p_pgidp */
3163
3164 for (;;) {
3165 mutex_enter(&p->p_splock); /* protects p->p_sessp */
3166 sp = p->p_sessp;
3167 mutex_enter(&sp->s_lock); /* protects sp->* */
3168
3169 /*
3170 * If this is not the calling process's controlling terminal
3171 * or if the calling process is already in the foreground
3172 * then allow access.
3173 */
3174 if (sp->s_dev != stp->sd_vnode->v_rdev ||
3175 p->p_pgidp == stp->sd_pgidp) {
3176 mutex_exit(&sp->s_lock);
3177 mutex_exit(&p->p_splock);
3178 mutex_exit(&p->p_lock);
3179 return (0);
3180 }
3181
3182 /*
3183 * Check to see if controlling terminal has been deallocated.
3184 */
3185 if (sp->s_vp == NULL) {
3186 if (!cantsend(p, t, SIGHUP))
3187 sigtoproc(p, t, SIGHUP);
3188 mutex_exit(&sp->s_lock);
3189 mutex_exit(&p->p_splock);
3190 mutex_exit(&p->p_lock);
3191 return (EIO);
3192 }
3193
3194 mutex_exit(&sp->s_lock);
3195 mutex_exit(&p->p_splock);
3196
3197 if (mode == JCGETP) {
3198 mutex_exit(&p->p_lock);
3199 return (0);
3200 }
3201
3202 if (mode == JCREAD) {
3203 if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3204 mutex_exit(&p->p_lock);
3205 return (EIO);
3206 }
3207 mutex_exit(&p->p_lock);
3208 mutex_exit(&stp->sd_lock);
3209 pgsignal(p->p_pgidp, SIGTTIN);
3210 mutex_enter(&stp->sd_lock);
3211 mutex_enter(&p->p_lock);
3212 } else { /* mode == JCWRITE or JCSETP */
3213 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3214 cantsend(p, t, SIGTTOU)) {
3215 mutex_exit(&p->p_lock);
3216 return (0);
3217 }
3218 if (p->p_detached) {
3219 mutex_exit(&p->p_lock);
3220 return (EIO);
3221 }
3222 mutex_exit(&p->p_lock);
3223 mutex_exit(&stp->sd_lock);
3224 pgsignal(p->p_pgidp, SIGTTOU);
3225 mutex_enter(&stp->sd_lock);
3226 mutex_enter(&p->p_lock);
3227 }
3228
3229 /*
3230 * We call cv_wait_sig_swap() to cause the appropriate
3231 * action for the jobcontrol signal to take place.
3232 * If the signal is being caught, we will take the
3233 * EINTR error return. Otherwise, the default action
3234 * of causing the process to stop will take place.
3235 * In this case, we rely on the periodic cv_broadcast() on
3236 * &lbolt_cv to wake us up to loop around and test again.
3237 * We can't get here if the signal is ignored or
3238 * if the current thread is blocking the signal.
3239 */
3240 mutex_exit(&stp->sd_lock);
3241 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3242 mutex_exit(&p->p_lock);
3243 mutex_enter(&stp->sd_lock);
3244 return (EINTR);
3245 }
3246 mutex_exit(&p->p_lock);
3247 mutex_enter(&stp->sd_lock);
3248 mutex_enter(&p->p_lock);
3249 }
3250 }
3251
3252 /*
3253 * Return size of message of block type (bp->b_datap->db_type)
3254 */
3255 size_t
3256 xmsgsize(mblk_t *bp)
3257 {
3258 unsigned char type;
3259 size_t count = 0;
3260
3261 type = bp->b_datap->db_type;
3262
3263 for (; bp; bp = bp->b_cont) {
3264 if (type != bp->b_datap->db_type)
3265 break;
3266 ASSERT(bp->b_wptr >= bp->b_rptr);
3267 count += bp->b_wptr - bp->b_rptr;
3268 }
3269 return (count);
3270 }
3271
3272 /*
3273 * Allocate a stream head.
3274 */
3275 struct stdata *
3276 shalloc(queue_t *qp)
3277 {
3278 stdata_t *stp;
3279
3280 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3281
3282 stp->sd_wrq = _WR(qp);
3283 stp->sd_strtab = NULL;
3284 stp->sd_iocid = 0;
3285 stp->sd_mate = NULL;
3286 stp->sd_freezer = NULL;
3287 stp->sd_refcnt = 0;
3288 stp->sd_wakeq = 0;
3289 stp->sd_anchor = 0;
3290 stp->sd_struiowrq = NULL;
3291 stp->sd_struiordq = NULL;
3292 stp->sd_struiodnak = 0;
3293 stp->sd_struionak = NULL;
3294 stp->sd_t_audit_data = NULL;
3295 stp->sd_rput_opt = 0;
3296 stp->sd_wput_opt = 0;
3297 stp->sd_read_opt = 0;
3298 stp->sd_rprotofunc = strrput_proto;
3299 stp->sd_rmiscfunc = strrput_misc;
3300 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3301 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3302 stp->sd_ciputctrl = NULL;
3303 stp->sd_nciputctrl = 0;
3304 stp->sd_qhead = NULL;
3305 stp->sd_qtail = NULL;
3306 stp->sd_servid = NULL;
3307 stp->sd_nqueues = 0;
3308 stp->sd_svcflags = 0;
3309 stp->sd_copyflag = 0;
3310
3311 return (stp);
3312 }
3313
3314 /*
3315 * Free a stream head.
3316 */
3317 void
3318 shfree(stdata_t *stp)
3319 {
3320 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3321
3322 stp->sd_wrq = NULL;
3323
3324 mutex_enter(&stp->sd_qlock);
3325 while (stp->sd_svcflags & STRS_SCHEDULED) {
3326 STRSTAT(strwaits);
3327 cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3328 }
3329 mutex_exit(&stp->sd_qlock);
3330
3331 if (stp->sd_ciputctrl != NULL) {
3332 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3333 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3334 stp->sd_nciputctrl, 0);
3335 ASSERT(ciputctrl_cache != NULL);
3336 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3337 stp->sd_ciputctrl = NULL;
3338 stp->sd_nciputctrl = 0;
3339 }
3340 ASSERT(stp->sd_qhead == NULL);
3341 ASSERT(stp->sd_qtail == NULL);
3342 ASSERT(stp->sd_nqueues == 0);
3343 kmem_cache_free(stream_head_cache, stp);
3344 }
3345
3346 /*
3347 * Allocate a pair of queues and a syncq for the pair
3348 */
3349 queue_t *
3350 allocq(void)
3351 {
3352 queinfo_t *qip;
3353 queue_t *qp, *wqp;
3354 syncq_t *sq;
3355
3356 qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3357
3358 qp = &qip->qu_rqueue;
3359 wqp = &qip->qu_wqueue;
3360 sq = &qip->qu_syncq;
3361
3362 qp->q_last = NULL;
3363 qp->q_next = NULL;
3364 qp->q_ptr = NULL;
3365 qp->q_flag = QUSE | QREADR;
3366 qp->q_bandp = NULL;
3367 qp->q_stream = NULL;
3368 qp->q_syncq = sq;
3369 qp->q_nband = 0;
3370 qp->q_nfsrv = NULL;
3371 qp->q_draining = 0;
3372 qp->q_syncqmsgs = 0;
3373 qp->q_spri = 0;
3374 qp->q_qtstamp = 0;
3375 qp->q_sqtstamp = 0;
3376 qp->q_fp = NULL;
3377
3378 wqp->q_last = NULL;
3379 wqp->q_next = NULL;
3380 wqp->q_ptr = NULL;
3381 wqp->q_flag = QUSE;
3382 wqp->q_bandp = NULL;
3383 wqp->q_stream = NULL;
3384 wqp->q_syncq = sq;
3385 wqp->q_nband = 0;
3386 wqp->q_nfsrv = NULL;
3387 wqp->q_draining = 0;
3388 wqp->q_syncqmsgs = 0;
3389 wqp->q_qtstamp = 0;
3390 wqp->q_sqtstamp = 0;
3391 wqp->q_spri = 0;
3392
3393 sq->sq_count = 0;
3394 sq->sq_rmqcount = 0;
3395 sq->sq_flags = 0;
3396 sq->sq_type = 0;
3397 sq->sq_callbflags = 0;
3398 sq->sq_cancelid = 0;
3399 sq->sq_ciputctrl = NULL;
3400 sq->sq_nciputctrl = 0;
3401 sq->sq_needexcl = 0;
3402 sq->sq_svcflags = 0;
3403
3404 return (qp);
3405 }
3406
3407 /*
3408 * Free a pair of queues and the "attached" syncq.
3409 * Discard any messages left on the syncq(s), remove the syncq(s) from the
3410 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3411 */
3412 void
3413 freeq(queue_t *qp)
3414 {
3415 qband_t *qbp, *nqbp;
3416 syncq_t *sq, *outer;
3417 queue_t *wqp = _WR(qp);
3418
3419 ASSERT(qp->q_flag & QREADR);
3420
3421 /*
3422 * If a previously dispatched taskq job is scheduled to run
3423 * sync_service() or a service routine is scheduled for the
3424 * queues about to be freed, wait here until all service is
3425 * done on the queue and all associated queues and syncqs.
3426 */
3427 wait_svc(qp);
3428
3429 (void) flush_syncq(qp->q_syncq, qp);
3430 (void) flush_syncq(wqp->q_syncq, wqp);
3431 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3432
3433 /*
3434 * Flush the queues before q_next is set to NULL This is needed
3435 * in order to backenable any downstream queue before we go away.
3436 * Note: we are already removed from the stream so that the
3437 * backenabling will not cause any messages to be delivered to our
3438 * put procedures.
3439 */
3440 flushq(qp, FLUSHALL);
3441 flushq(wqp, FLUSHALL);
3442
3443 /* Tidy up - removeq only does a half-remove from stream */
3444 qp->q_next = wqp->q_next = NULL;
3445 ASSERT(!(qp->q_flag & QENAB));
3446 ASSERT(!(wqp->q_flag & QENAB));
3447
3448 outer = qp->q_syncq->sq_outer;
3449 if (outer != NULL) {
3450 outer_remove(outer, qp->q_syncq);
3451 if (wqp->q_syncq != qp->q_syncq)
3452 outer_remove(outer, wqp->q_syncq);
3453 }
3454 /*
3455 * Free any syncqs that are outside what allocq returned.
3456 */
3457 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3458 free_syncq(qp->q_syncq);
3459 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3460 free_syncq(wqp->q_syncq);
3461
3462 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3463 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3464 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3465 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3466 sq = SQ(qp);
3467 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3468 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3469 ASSERT(sq->sq_outer == NULL);
3470 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3471 ASSERT(sq->sq_callbpend == NULL);
3472 ASSERT(sq->sq_needexcl == 0);
3473
3474 if (sq->sq_ciputctrl != NULL) {
3475 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3476 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3477 sq->sq_nciputctrl, 0);
3478 ASSERT(ciputctrl_cache != NULL);
3479 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3480 sq->sq_ciputctrl = NULL;
3481 sq->sq_nciputctrl = 0;
3482 }
3483
3484 ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3485 ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3486 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3487
3488 qp->q_flag &= ~QUSE;
3489 wqp->q_flag &= ~QUSE;
3490
3491 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3492 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3493
3494 qbp = qp->q_bandp;
3495 while (qbp) {
3496 nqbp = qbp->qb_next;
3497 freeband(qbp);
3498 qbp = nqbp;
3499 }
3500 qbp = wqp->q_bandp;
3501 while (qbp) {
3502 nqbp = qbp->qb_next;
3503 freeband(qbp);
3504 qbp = nqbp;
3505 }
3506 kmem_cache_free(queue_cache, qp);
3507 }
3508
3509 /*
3510 * Allocate a qband structure.
3511 */
3512 qband_t *
3513 allocband(void)
3514 {
3515 qband_t *qbp;
3516
3517 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3518 if (qbp == NULL)
3519 return (NULL);
3520
3521 qbp->qb_next = NULL;
3522 qbp->qb_count = 0;
3523 qbp->qb_mblkcnt = 0;
3524 qbp->qb_first = NULL;
3525 qbp->qb_last = NULL;
3526 qbp->qb_flag = 0;
3527
3528 return (qbp);
3529 }
3530
3531 /*
3532 * Free a qband structure.
3533 */
3534 void
3535 freeband(qband_t *qbp)
3536 {
3537 kmem_cache_free(qband_cache, qbp);
3538 }
3539
3540 /*
3541 * Just like putnextctl(9F), except that allocb_wait() is used.
3542 *
3543 * Consolidation Private, and of course only callable from the stream head or
3544 * routines that may block.
3545 */
3546 int
3547 putnextctl_wait(queue_t *q, int type)
3548 {
3549 mblk_t *bp;
3550 int error;
3551
3552 if ((datamsg(type) && (type != M_DELAY)) ||
3553 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3554 return (0);
3555
3556 bp->b_datap->db_type = (unsigned char)type;
3557 putnext(q, bp);
3558 return (1);
3559 }
3560
3561 /*
3562 * Run any possible bufcalls.
3563 */
3564 void
3565 runbufcalls(void)
3566 {
3567 strbufcall_t *bcp;
3568
3569 mutex_enter(&bcall_monitor);
3570 mutex_enter(&strbcall_lock);
3571
3572 if (strbcalls.bc_head) {
3573 size_t count;
3574 int nevent;
3575
3576 /*
3577 * count how many events are on the list
3578 * now so we can check to avoid looping
3579 * in low memory situations
3580 */
3581 nevent = 0;
3582 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3583 nevent++;
3584
3585 /*
3586 * get estimate of available memory from kmem_avail().
3587 * awake all bufcall functions waiting for
3588 * memory whose request could be satisfied
3589 * by 'count' memory and let 'em fight for it.
3590 */
3591 count = kmem_avail();
3592 while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3593 STRSTAT(bufcalls);
3594 --nevent;
3595 if (bcp->bc_size <= count) {
3596 bcp->bc_executor = curthread;
3597 mutex_exit(&strbcall_lock);
3598 (*bcp->bc_func)(bcp->bc_arg);
3599 mutex_enter(&strbcall_lock);
3600 bcp->bc_executor = NULL;
3601 cv_broadcast(&bcall_cv);
3602 strbcalls.bc_head = bcp->bc_next;
3603 kmem_free(bcp, sizeof (strbufcall_t));
3604 } else {
3605 /*
3606 * too big, try again later - note
3607 * that nevent was decremented above
3608 * so we won't retry this one on this
3609 * iteration of the loop
3610 */
3611 if (bcp->bc_next != NULL) {
3612 strbcalls.bc_head = bcp->bc_next;
3613 bcp->bc_next = NULL;
3614 strbcalls.bc_tail->bc_next = bcp;
3615 strbcalls.bc_tail = bcp;
3616 }
3617 }
3618 }
3619 if (strbcalls.bc_head == NULL)
3620 strbcalls.bc_tail = NULL;
3621 }
3622
3623 mutex_exit(&strbcall_lock);
3624 mutex_exit(&bcall_monitor);
3625 }
3626
3627
3628 /*
3629 * Actually run queue's service routine.
3630 */
3631 static void
3632 runservice(queue_t *q)
3633 {
3634 qband_t *qbp;
3635
3636 ASSERT(q->q_qinfo->qi_srvp);
3637 again:
3638 entersq(q->q_syncq, SQ_SVC);
3639 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3640 "runservice starts:%p", q);
3641
3642 if (!(q->q_flag & QWCLOSE))
3643 (*q->q_qinfo->qi_srvp)(q);
3644
3645 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3646 "runservice ends:(%p)", q);
3647
3648 leavesq(q->q_syncq, SQ_SVC);
3649
3650 mutex_enter(QLOCK(q));
3651 if (q->q_flag & QENAB) {
3652 q->q_flag &= ~QENAB;
3653 mutex_exit(QLOCK(q));
3654 goto again;
3655 }
3656 q->q_flag &= ~QINSERVICE;
3657 q->q_flag &= ~QBACK;
3658 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3659 qbp->qb_flag &= ~QB_BACK;
3660 /*
3661 * Wakeup thread waiting for the service procedure
3662 * to be run (strclose and qdetach).
3663 */
3664 cv_broadcast(&q->q_wait);
3665
3666 mutex_exit(QLOCK(q));
3667 }
3668
3669 /*
3670 * Background processing of bufcalls.
3671 */
3672 void
3673 streams_bufcall_service(void)
3674 {
3675 callb_cpr_t cprinfo;
3676
3677 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3678 "streams_bufcall_service");
3679
3680 mutex_enter(&strbcall_lock);
3681
3682 for (;;) {
3683 if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3684 mutex_exit(&strbcall_lock);
3685 runbufcalls();
3686 mutex_enter(&strbcall_lock);
3687 }
3688 if (strbcalls.bc_head != NULL) {
3689 STRSTAT(bcwaits);
3690 /* Wait for memory to become available */
3691 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3692 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3693 SEC_TO_TICK(60), TR_CLOCK_TICK);
3694 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3695 }
3696
3697 /* Wait for new work to arrive */
3698 if (strbcalls.bc_head == NULL) {
3699 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3700 cv_wait(&strbcall_cv, &strbcall_lock);
3701 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3702 }
3703 }
3704 }
3705
3706 /*
3707 * Background processing of streams background tasks which failed
3708 * taskq_dispatch.
3709 */
3710 static void
3711 streams_qbkgrnd_service(void)
3712 {
3713 callb_cpr_t cprinfo;
3714 queue_t *q;
3715
3716 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3717 "streams_bkgrnd_service");
3718
3719 mutex_enter(&service_queue);
3720
3721 for (;;) {
3722 /*
3723 * Wait for work to arrive.
3724 */
3725 while ((freebs_list == NULL) && (qhead == NULL)) {
3726 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3727 cv_wait(&services_to_run, &service_queue);
3728 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3729 }
3730 /*
3731 * Handle all pending freebs requests to free memory.
3732 */
3733 while (freebs_list != NULL) {
3734 mblk_t *mp = freebs_list;
3735 freebs_list = mp->b_next;
3736 mutex_exit(&service_queue);
3737 mblk_free(mp);
3738 mutex_enter(&service_queue);
3739 }
3740 /*
3741 * Run pending queues.
3742 */
3743 while (qhead != NULL) {
3744 DQ(q, qhead, qtail, q_link);
3745 ASSERT(q != NULL);
3746 mutex_exit(&service_queue);
3747 queue_service(q);
3748 mutex_enter(&service_queue);
3749 }
3750 ASSERT(qhead == NULL && qtail == NULL);
3751 }
3752 }
3753
3754 /*
3755 * Background processing of streams background tasks which failed
3756 * taskq_dispatch.
3757 */
3758 static void
3759 streams_sqbkgrnd_service(void)
3760 {
3761 callb_cpr_t cprinfo;
3762 syncq_t *sq;
3763
3764 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3765 "streams_sqbkgrnd_service");
3766
3767 mutex_enter(&service_queue);
3768
3769 for (;;) {
3770 /*
3771 * Wait for work to arrive.
3772 */
3773 while (sqhead == NULL) {
3774 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3775 cv_wait(&syncqs_to_run, &service_queue);
3776 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3777 }
3778
3779 /*
3780 * Run pending syncqs.
3781 */
3782 while (sqhead != NULL) {
3783 DQ(sq, sqhead, sqtail, sq_next);
3784 ASSERT(sq != NULL);
3785 ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3786 mutex_exit(&service_queue);
3787 syncq_service(sq);
3788 mutex_enter(&service_queue);
3789 }
3790 }
3791 }
3792
3793 /*
3794 * Disable the syncq and wait for background syncq processing to complete.
3795 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3796 * list.
3797 */
3798 void
3799 wait_sq_svc(syncq_t *sq)
3800 {
3801 mutex_enter(SQLOCK(sq));
3802 sq->sq_svcflags |= SQ_DISABLED;
3803 if (sq->sq_svcflags & SQ_BGTHREAD) {
3804 syncq_t *sq_chase;
3805 syncq_t *sq_curr;
3806 int removed;
3807
3808 ASSERT(sq->sq_servcount == 1);
3809 mutex_enter(&service_queue);
3810 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3811 mutex_exit(&service_queue);
3812 if (removed) {
3813 sq->sq_svcflags &= ~SQ_BGTHREAD;
3814 sq->sq_servcount = 0;
3815 STRSTAT(sqremoved);
3816 goto done;
3817 }
3818 }
3819 while (sq->sq_servcount != 0) {
3820 sq->sq_flags |= SQ_WANTWAKEUP;
3821 cv_wait(&sq->sq_wait, SQLOCK(sq));
3822 }
3823 done:
3824 mutex_exit(SQLOCK(sq));
3825 }
3826
3827 /*
3828 * Put a syncq on the list of syncq's to be serviced by the sqthread.
3829 * Add the argument to the end of the sqhead list and set the flag
3830 * indicating this syncq has been enabled. If it has already been
3831 * enabled, don't do anything.
3832 * This routine assumes that SQLOCK is held.
3833 * NOTE that the lock order is to have the SQLOCK first,
3834 * so if the service_syncq lock is held, we need to release it
3835 * before acquiring the SQLOCK (mostly relevant for the background
3836 * thread, and this seems to be common among the STREAMS global locks).
3837 * Note that the sq_svcflags are protected by the SQLOCK.
3838 */
3839 void
3840 sqenable(syncq_t *sq)
3841 {
3842 /*
3843 * This is probably not important except for where I believe it
3844 * is being called. At that point, it should be held (and it
3845 * is a pain to release it just for this routine, so don't do
3846 * it).
3847 */
3848 ASSERT(MUTEX_HELD(SQLOCK(sq)));
3849
3850 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3851 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3852
3853 /*
3854 * Do not put on list if background thread is scheduled or
3855 * syncq is disabled.
3856 */
3857 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3858 return;
3859
3860 /*
3861 * Check whether we should enable sq at all.
3862 * Non PERMOD syncqs may be drained by at most one thread.
3863 * PERMOD syncqs may be drained by several threads but we limit the
3864 * total amount to the lesser of
3865 * Number of queues on the squeue and
3866 * Number of CPUs.
3867 */
3868 if (sq->sq_servcount != 0) {
3869 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3870 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3871 STRSTAT(sqtoomany);
3872 return;
3873 }
3874 }
3875
3876 sq->sq_tstamp = ddi_get_lbolt();
3877 STRSTAT(sqenables);
3878
3879 /* Attempt a taskq dispatch */
3880 sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3881 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3882 if (sq->sq_servid != NULL) {
3883 sq->sq_servcount++;
3884 return;
3885 }
3886
3887 /*
3888 * This taskq dispatch failed, but a previous one may have succeeded.
3889 * Don't try to schedule on the background thread whilst there is
3890 * outstanding taskq processing.
3891 */
3892 if (sq->sq_servcount != 0)
3893 return;
3894
3895 /*
3896 * System is low on resources and can't perform a non-sleeping
3897 * dispatch. Schedule the syncq for a background thread and mark the
3898 * syncq to avoid any further taskq dispatch attempts.
3899 */
3900 mutex_enter(&service_queue);
3901 STRSTAT(taskqfails);
3902 ENQUEUE(sq, sqhead, sqtail, sq_next);
3903 sq->sq_svcflags |= SQ_BGTHREAD;
3904 sq->sq_servcount = 1;
3905 cv_signal(&syncqs_to_run);
3906 mutex_exit(&service_queue);
3907 }
3908
3909 /*
3910 * Note: fifo_close() depends on the mblk_t on the queue being freed
3911 * asynchronously. The asynchronous freeing of messages breaks the
3912 * recursive call chain of fifo_close() while there are I_SENDFD type of
3913 * messages referring to other file pointers on the queue. Then when
3914 * closing pipes it can avoid stack overflow in case of daisy-chained
3915 * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3916 * share the same fifolock_t).
3917 *
3918 * No need to kpreempt_disable to access cpu_seqid. If we migrate and
3919 * the esb queue does not match the new CPU, that is OK.
3920 */
3921 void
3922 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3923 {
3924 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3925 esb_queue_t *eqp;
3926
3927 ASSERT(dbp->db_mblk == mp);
3928 ASSERT(qindex < esbq_nelem);
3929
3930 eqp = system_esbq_array;
3931 if (eqp != NULL) {
3932 eqp += qindex;
3933 } else {
3934 mutex_enter(&esbq_lock);
3935 if (kmem_ready && system_esbq_array == NULL)
3936 system_esbq_array = (esb_queue_t *)kmem_zalloc(
3937 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3938 mutex_exit(&esbq_lock);
3939 eqp = system_esbq_array;
3940 if (eqp != NULL)
3941 eqp += qindex;
3942 else
3943 eqp = &system_esbq;
3944 }
3945
3946 /*
3947 * Check data sanity. The dblock should have non-empty free function.
3948 * It is better to panic here then later when the dblock is freed
3949 * asynchronously when the context is lost.
3950 */
3951 if (dbp->db_frtnp->free_func == NULL) {
3952 panic("freebs_enqueue: dblock %p has a NULL free callback",
3953 (void *)dbp);
3954 }
3955
3956 mutex_enter(&eqp->eq_lock);
3957 /* queue the new mblk on the esballoc queue */
3958 if (eqp->eq_head == NULL) {
3959 eqp->eq_head = eqp->eq_tail = mp;
3960 } else {
3961 eqp->eq_tail->b_next = mp;
3962 eqp->eq_tail = mp;
3963 }
3964 eqp->eq_len++;
3965
3966 /* If we're the first thread to reach the threshold, process */
3967 if (eqp->eq_len >= esbq_max_qlen &&
3968 !(eqp->eq_flags & ESBQ_PROCESSING))
3969 esballoc_process_queue(eqp);
3970
3971 esballoc_set_timer(eqp, esbq_timeout);
3972 mutex_exit(&eqp->eq_lock);
3973 }
3974
3975 static void
3976 esballoc_process_queue(esb_queue_t *eqp)
3977 {
3978 mblk_t *mp;
3979
3980 ASSERT(MUTEX_HELD(&eqp->eq_lock));
3981
3982 eqp->eq_flags |= ESBQ_PROCESSING;
3983
3984 do {
3985 /*
3986 * Detach the message chain for processing.
3987 */
3988 mp = eqp->eq_head;
3989 eqp->eq_tail->b_next = NULL;
3990 eqp->eq_head = eqp->eq_tail = NULL;
3991 eqp->eq_len = 0;
3992 mutex_exit(&eqp->eq_lock);
3993
3994 /*
3995 * Process the message chain.
3996 */
3997 esballoc_enqueue_mblk(mp);
3998 mutex_enter(&eqp->eq_lock);
3999 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
4000
4001 eqp->eq_flags &= ~ESBQ_PROCESSING;
4002 }
4003
4004 /*
4005 * taskq callback routine to free esballoced mblk's
4006 */
4007 static void
4008 esballoc_mblk_free(mblk_t *mp)
4009 {
4010 mblk_t *nextmp;
4011
4012 for (; mp != NULL; mp = nextmp) {
4013 nextmp = mp->b_next;
4014 mp->b_next = NULL;
4015 mblk_free(mp);
4016 }
4017 }
4018
4019 static void
4020 esballoc_enqueue_mblk(mblk_t *mp)
4021 {
4022
4023 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4024 TQ_NOSLEEP) == NULL) {
4025 mblk_t *first_mp = mp;
4026 /*
4027 * System is low on resources and can't perform a non-sleeping
4028 * dispatch. Schedule for a background thread.
4029 */
4030 mutex_enter(&service_queue);
4031 STRSTAT(taskqfails);
4032
4033 while (mp->b_next != NULL)
4034 mp = mp->b_next;
4035
4036 mp->b_next = freebs_list;
4037 freebs_list = first_mp;
4038 cv_signal(&services_to_run);
4039 mutex_exit(&service_queue);
4040 }
4041 }
4042
4043 static void
4044 esballoc_timer(void *arg)
4045 {
4046 esb_queue_t *eqp = arg;
4047
4048 mutex_enter(&eqp->eq_lock);
4049 eqp->eq_flags &= ~ESBQ_TIMER;
4050
4051 if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4052 eqp->eq_len > 0)
4053 esballoc_process_queue(eqp);
4054
4055 esballoc_set_timer(eqp, esbq_timeout);
4056 mutex_exit(&eqp->eq_lock);
4057 }
4058
4059 static void
4060 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4061 {
4062 ASSERT(MUTEX_HELD(&eqp->eq_lock));
4063
4064 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4065 (void) timeout(esballoc_timer, eqp, eq_timeout);
4066 eqp->eq_flags |= ESBQ_TIMER;
4067 }
4068 }
4069
4070 /*
4071 * Setup esbq array length based upon NCPU scaled by CPUs per
4072 * queue. Use static system_esbq until kmem_ready and we can
4073 * create an array in freebs_enqueue().
4074 */
4075 void
4076 esballoc_queue_init(void)
4077 {
4078 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4079 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4080 esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4081 system_esbq.eq_len = 0;
4082 system_esbq.eq_head = system_esbq.eq_tail = NULL;
4083 system_esbq.eq_flags = 0;
4084 }
4085
4086 /*
4087 * Set the QBACK or QB_BACK flag in the given queue for
4088 * the given priority band.
4089 */
4090 void
4091 setqback(queue_t *q, unsigned char pri)
4092 {
4093 int i;
4094 qband_t *qbp;
4095 qband_t **qbpp;
4096
4097 ASSERT(MUTEX_HELD(QLOCK(q)));
4098 if (pri != 0) {
4099 if (pri > q->q_nband) {
4100 qbpp = &q->q_bandp;
4101 while (*qbpp)
4102 qbpp = &(*qbpp)->qb_next;
4103 while (pri > q->q_nband) {
4104 if ((*qbpp = allocband()) == NULL) {
4105 cmn_err(CE_WARN,
4106 "setqback: can't allocate qband\n");
4107 return;
4108 }
4109 (*qbpp)->qb_hiwat = q->q_hiwat;
4110 (*qbpp)->qb_lowat = q->q_lowat;
4111 q->q_nband++;
4112 qbpp = &(*qbpp)->qb_next;
4113 }
4114 }
4115 qbp = q->q_bandp;
4116 i = pri;
4117 while (--i)
4118 qbp = qbp->qb_next;
4119 qbp->qb_flag |= QB_BACK;
4120 } else {
4121 q->q_flag |= QBACK;
4122 }
4123 }
4124
4125 int
4126 strcopyin(void *from, void *to, size_t len, int copyflag)
4127 {
4128 if (copyflag & U_TO_K) {
4129 ASSERT((copyflag & K_TO_K) == 0);
4130 if (copyin(from, to, len))
4131 return (EFAULT);
4132 } else {
4133 ASSERT(copyflag & K_TO_K);
4134 bcopy(from, to, len);
4135 }
4136 return (0);
4137 }
4138
4139 int
4140 strcopyout(void *from, void *to, size_t len, int copyflag)
4141 {
4142 if (copyflag & U_TO_K) {
4143 if (copyout(from, to, len))
4144 return (EFAULT);
4145 } else {
4146 ASSERT(copyflag & K_TO_K);
4147 bcopy(from, to, len);
4148 }
4149 return (0);
4150 }
4151
4152 /*
4153 * strsignal_nolock() posts a signal to the process(es) at the stream head.
4154 * It assumes that the stream head lock is already held, whereas strsignal()
4155 * acquires the lock first. This routine was created because a few callers
4156 * release the stream head lock before calling only to re-acquire it after
4157 * it returns.
4158 */
4159 void
4160 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4161 {
4162 ASSERT(MUTEX_HELD(&stp->sd_lock));
4163 switch (sig) {
4164 case SIGPOLL:
4165 if (stp->sd_sigflags & S_MSG)
4166 strsendsig(stp->sd_siglist, S_MSG, band, 0);
4167 break;
4168 default:
4169 if (stp->sd_pgidp)
4170 pgsignal(stp->sd_pgidp, sig);
4171 break;
4172 }
4173 }
4174
4175 void
4176 strsignal(stdata_t *stp, int sig, int32_t band)
4177 {
4178 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4179 "strsignal:%p, %X, %X", stp, sig, band);
4180
4181 mutex_enter(&stp->sd_lock);
4182 switch (sig) {
4183 case SIGPOLL:
4184 if (stp->sd_sigflags & S_MSG)
4185 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4186 break;
4187
4188 default:
4189 if (stp->sd_pgidp) {
4190 pgsignal(stp->sd_pgidp, sig);
4191 }
4192 break;
4193 }
4194 mutex_exit(&stp->sd_lock);
4195 }
4196
4197 void
4198 strhup(stdata_t *stp)
4199 {
4200 ASSERT(mutex_owned(&stp->sd_lock));
4201 pollwakeup(&stp->sd_pollist, POLLHUP);
4202 if (stp->sd_sigflags & S_HANGUP)
4203 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4204 }
4205
4206 /*
4207 * Backenable the first queue upstream from `q' with a service procedure.
4208 */
4209 void
4210 backenable(queue_t *q, uchar_t pri)
4211 {
4212 queue_t *nq;
4213
4214 /*
4215 * Our presence might not prevent other modules in our own
4216 * stream from popping/pushing since the caller of getq might not
4217 * have a claim on the queue (some drivers do a getq on somebody
4218 * else's queue - they know that the queue itself is not going away
4219 * but the framework has to guarantee q_next in that stream).
4220 */
4221 claimstr(q);
4222
4223 /* Find nearest back queue with service proc */
4224 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4225 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4226 }
4227
4228 if (nq) {
4229 kthread_t *freezer;
4230 /*
4231 * backenable can be called either with no locks held
4232 * or with the stream frozen (the latter occurs when a module
4233 * calls rmvq with the stream frozen). If the stream is frozen
4234 * by the caller the caller will hold all qlocks in the stream.
4235 * Note that a frozen stream doesn't freeze a mated stream,
4236 * so we explicitly check for that.
4237 */
4238 freezer = STREAM(q)->sd_freezer;
4239 if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4240 mutex_enter(QLOCK(nq));
4241 }
4242 #ifdef DEBUG
4243 else {
4244 ASSERT(frozenstr(q));
4245 ASSERT(MUTEX_HELD(QLOCK(q)));
4246 ASSERT(MUTEX_HELD(QLOCK(nq)));
4247 }
4248 #endif
4249 setqback(nq, pri);
4250 qenable_locked(nq);
4251 if (freezer != curthread || STREAM(q) != STREAM(nq))
4252 mutex_exit(QLOCK(nq));
4253 }
4254 releasestr(q);
4255 }
4256
4257 /*
4258 * Return the appropriate errno when one of flags_to_check is set
4259 * in sd_flags. Uses the exported error routines if they are set.
4260 * Will return 0 if non error is set (or if the exported error routines
4261 * do not return an error).
4262 *
4263 * If there is both a read and write error to check, we prefer the read error.
4264 * Also, give preference to recorded errno's over the error functions.
4265 * The flags that are handled are:
4266 * STPLEX return EINVAL
4267 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST)
4268 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST)
4269 * STRHUP return sd_werror
4270 *
4271 * If the caller indicates that the operation is a peek, a nonpersistent error
4272 * is not cleared.
4273 */
4274 int
4275 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4276 {
4277 int32_t sd_flag = stp->sd_flag & flags_to_check;
4278 int error = 0;
4279
4280 ASSERT(MUTEX_HELD(&stp->sd_lock));
4281 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4282 if (sd_flag & STPLEX)
4283 error = EINVAL;
4284 else if (sd_flag & STRDERR) {
4285 error = stp->sd_rerror;
4286 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4287 /*
4288 * Read errors are non-persistent i.e. discarded once
4289 * returned to a non-peeking caller,
4290 */
4291 stp->sd_rerror = 0;
4292 stp->sd_flag &= ~STRDERR;
4293 }
4294 if (error == 0 && stp->sd_rderrfunc != NULL) {
4295 int clearerr = 0;
4296
4297 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4298 &clearerr);
4299 if (clearerr) {
4300 stp->sd_flag &= ~STRDERR;
4301 stp->sd_rderrfunc = NULL;
4302 }
4303 }
4304 } else if (sd_flag & STWRERR) {
4305 error = stp->sd_werror;
4306 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4307 /*
4308 * Write errors are non-persistent i.e. discarded once
4309 * returned to a non-peeking caller,
4310 */
4311 stp->sd_werror = 0;
4312 stp->sd_flag &= ~STWRERR;
4313 }
4314 if (error == 0 && stp->sd_wrerrfunc != NULL) {
4315 int clearerr = 0;
4316
4317 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4318 &clearerr);
4319 if (clearerr) {
4320 stp->sd_flag &= ~STWRERR;
4321 stp->sd_wrerrfunc = NULL;
4322 }
4323 }
4324 } else if (sd_flag & STRHUP) {
4325 /* sd_werror set when STRHUP */
4326 error = stp->sd_werror;
4327 }
4328 return (error);
4329 }
4330
4331
4332 /*
4333 * Single-thread open/close/push/pop
4334 * for twisted streams also
4335 */
4336 int
4337 strstartplumb(stdata_t *stp, int flag, int cmd)
4338 {
4339 int waited = 1;
4340 int error = 0;
4341
4342 if (STRMATED(stp)) {
4343 struct stdata *stmatep = stp->sd_mate;
4344
4345 STRLOCKMATES(stp);
4346 while (waited) {
4347 waited = 0;
4348 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4349 if ((cmd == I_POP) &&
4350 (flag & (FNDELAY|FNONBLOCK))) {
4351 STRUNLOCKMATES(stp);
4352 return (EAGAIN);
4353 }
4354 waited = 1;
4355 mutex_exit(&stp->sd_lock);
4356 if (!cv_wait_sig(&stmatep->sd_monitor,
4357 &stmatep->sd_lock)) {
4358 mutex_exit(&stmatep->sd_lock);
4359 return (EINTR);
4360 }
4361 mutex_exit(&stmatep->sd_lock);
4362 STRLOCKMATES(stp);
4363 }
4364 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4365 if ((cmd == I_POP) &&
4366 (flag & (FNDELAY|FNONBLOCK))) {
4367 STRUNLOCKMATES(stp);
4368 return (EAGAIN);
4369 }
4370 waited = 1;
4371 mutex_exit(&stmatep->sd_lock);
4372 if (!cv_wait_sig(&stp->sd_monitor,
4373 &stp->sd_lock)) {
4374 mutex_exit(&stp->sd_lock);
4375 return (EINTR);
4376 }
4377 mutex_exit(&stp->sd_lock);
4378 STRLOCKMATES(stp);
4379 }
4380 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4381 error = strgeterr(stp,
4382 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4383 if (error != 0) {
4384 STRUNLOCKMATES(stp);
4385 return (error);
4386 }
4387 }
4388 }
4389 stp->sd_flag |= STRPLUMB;
4390 STRUNLOCKMATES(stp);
4391 } else {
4392 mutex_enter(&stp->sd_lock);
4393 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4394 if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4395 (flag & (FNDELAY|FNONBLOCK))) {
4396 mutex_exit(&stp->sd_lock);
4397 return (EAGAIN);
4398 }
4399 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4400 mutex_exit(&stp->sd_lock);
4401 return (EINTR);
4402 }
4403 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4404 error = strgeterr(stp,
4405 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4406 if (error != 0) {
4407 mutex_exit(&stp->sd_lock);
4408 return (error);
4409 }
4410 }
4411 }
4412 stp->sd_flag |= STRPLUMB;
4413 mutex_exit(&stp->sd_lock);
4414 }
4415 return (0);
4416 }
4417
4418 /*
4419 * Complete the plumbing operation associated with stream `stp'.
4420 */
4421 void
4422 strendplumb(stdata_t *stp)
4423 {
4424 ASSERT(MUTEX_HELD(&stp->sd_lock));
4425 ASSERT(stp->sd_flag & STRPLUMB);
4426 stp->sd_flag &= ~STRPLUMB;
4427 cv_broadcast(&stp->sd_monitor);
4428 }
4429
4430 /*
4431 * This describes how the STREAMS framework handles synchronization
4432 * during open/push and close/pop.
4433 * The key interfaces for open and close are qprocson and qprocsoff,
4434 * respectively. While the close case in general is harder both open
4435 * have close have significant similarities.
4436 *
4437 * During close the STREAMS framework has to both ensure that there
4438 * are no stale references to the queue pair (and syncq) that
4439 * are being closed and also provide the guarantees that are documented
4440 * in qprocsoff(9F).
4441 * If there are stale references to the queue that is closing it can
4442 * result in kernel memory corruption or kernel panics.
4443 *
4444 * Note that is it up to the module/driver to ensure that it itself
4445 * does not have any stale references to the closing queues once its close
4446 * routine returns. This includes:
4447 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4448 * associated with the queues. For timeout and bufcall callbacks the
4449 * module/driver also has to ensure (or wait for) any callbacks that
4450 * are in progress.
4451 * - If the module/driver is using esballoc it has to ensure that any
4452 * esballoc free functions do not refer to a queue that has closed.
4453 * (Note that in general the close routine can not wait for the esballoc'ed
4454 * messages to be freed since that can cause a deadlock.)
4455 * - Cancelling any interrupts that refer to the closing queues and
4456 * also ensuring that there are no interrupts in progress that will
4457 * refer to the closing queues once the close routine returns.
4458 * - For multiplexors removing any driver global state that refers to
4459 * the closing queue and also ensuring that there are no threads in
4460 * the multiplexor that has picked up a queue pointer but not yet
4461 * finished using it.
4462 *
4463 * In addition, a driver/module can only reference the q_next pointer
4464 * in its open, close, put, or service procedures or in a
4465 * qtimeout/qbufcall callback procedure executing "on" the correct
4466 * stream. Thus it can not reference the q_next pointer in an interrupt
4467 * routine or a timeout, bufcall or esballoc callback routine. Likewise
4468 * it can not reference q_next of a different queue e.g. in a mux that
4469 * passes messages from one queues put/service procedure to another queue.
4470 * In all the cases when the driver/module can not access the q_next
4471 * field it must use the *next* versions e.g. canputnext instead of
4472 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4473 *
4474 *
4475 * Assuming that the driver/module conforms to the above constraints
4476 * the STREAMS framework has to avoid stale references to q_next for all
4477 * the framework internal cases which include (but are not limited to):
4478 * - Threads in canput/canputnext/backenable and elsewhere that are
4479 * walking q_next.
4480 * - Messages on a syncq that have a reference to the queue through b_queue.
4481 * - Messages on an outer perimeter (syncq) that have a reference to the
4482 * queue through b_queue.
4483 * - Threads that use q_nfsrv (e.g. canput) to find a queue.
4484 * Note that only canput and bcanput use q_nfsrv without any locking.
4485 *
4486 * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4487 * after qprocsoff returns, the framework has to ensure that no threads can
4488 * enter the put or service routines for the closing read or write-side queue.
4489 * In addition to preventing "direct" entry into the put procedures
4490 * the framework also has to prevent messages being drained from
4491 * the syncq or the outer perimeter.
4492 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4493 * mechanism to prevent qwriter(PERIM_OUTER) from running after
4494 * qprocsoff has returned.
4495 * Note that if a module/driver uses put(9F) on one of its own queues
4496 * it is up to the module/driver to ensure that the put() doesn't
4497 * get called when the queue is closing.
4498 *
4499 *
4500 * The framework aspects of the above "contract" is implemented by
4501 * qprocsoff, removeq, and strlock:
4502 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4503 * entering the service procedures.
4504 * - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4505 * canputnext, backenable etc from dereferencing the q_next that will
4506 * soon change.
4507 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4508 * or other q_next walker that uses claimstr/releasestr to finish.
4509 * - optionally for every syncq in the stream strlock acquires all the
4510 * sq_lock's and waits for all sq_counts to drop to a value that indicates
4511 * that no thread executes in the put or service procedures and that no
4512 * thread is draining into the module/driver. This ensures that no
4513 * open, close, put, service, or qtimeout/qbufcall callback procedure is
4514 * currently executing hence no such thread can end up with the old stale
4515 * q_next value and no canput/backenable can have the old stale
4516 * q_nfsrv/q_next.
4517 * - qdetach (wait_svc) makes sure that any scheduled or running threads
4518 * have either finished or observed the QWCLOSE flag and gone away.
4519 */
4520
4521
4522 /*
4523 * Get all the locks necessary to change q_next.
4524 *
4525 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4526 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4527 * the only threads inside the syncq are threads currently calling removeq().
4528 * Since threads calling removeq() are in the process of removing their queues
4529 * from the stream, we do not need to worry about them accessing a stale q_next
4530 * pointer and thus we do not need to wait for them to exit (in fact, waiting
4531 * for them can cause deadlock).
4532 *
4533 * This routine is subject to starvation since it does not set any flag to
4534 * prevent threads from entering a module in the stream (i.e. sq_count can
4535 * increase on some syncq while it is waiting on some other syncq).
4536 *
4537 * Assumes that only one thread attempts to call strlock for a given
4538 * stream. If this is not the case the two threads would deadlock.
4539 * This assumption is guaranteed since strlock is only called by insertq
4540 * and removeq and streams plumbing changes are single-threaded for
4541 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4542 *
4543 * For pipes, it is not difficult to atomically designate a pair of streams
4544 * to be mated. Once mated atomically by the framework the twisted pair remain
4545 * configured that way until dismantled atomically by the framework.
4546 * When plumbing takes place on a twisted stream it is necessary to ensure that
4547 * this operation is done exclusively on the twisted stream since two such
4548 * operations, each initiated on different ends of the pipe will deadlock
4549 * waiting for each other to complete.
4550 *
4551 * On entry, no locks should be held.
4552 * The locks acquired and held by strlock depends on a few factors.
4553 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4554 * and held on exit and all sq_count are at an acceptable level.
4555 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4556 * sd_refcnt being zero.
4557 */
4558
4559 static void
4560 strlock(struct stdata *stp, sqlist_t *sqlist)
4561 {
4562 syncql_t *sql, *sql2;
4563 retry:
4564 /*
4565 * Wait for any claimstr to go away.
4566 */
4567 if (STRMATED(stp)) {
4568 struct stdata *stp1, *stp2;
4569
4570 STRLOCKMATES(stp);
4571 /*
4572 * Note that the selection of locking order is not
4573 * important, just that they are always acquired in
4574 * the same order. To assure this, we choose this
4575 * order based on the value of the pointer, and since
4576 * the pointer will not change for the life of this
4577 * pair, we will always grab the locks in the same
4578 * order (and hence, prevent deadlocks).
4579 */
4580 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4581 stp1 = stp;
4582 stp2 = stp->sd_mate;
4583 } else {
4584 stp2 = stp;
4585 stp1 = stp->sd_mate;
4586 }
4587 mutex_enter(&stp1->sd_reflock);
4588 if (stp1->sd_refcnt > 0) {
4589 STRUNLOCKMATES(stp);
4590 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4591 mutex_exit(&stp1->sd_reflock);
4592 goto retry;
4593 }
4594 mutex_enter(&stp2->sd_reflock);
4595 if (stp2->sd_refcnt > 0) {
4596 STRUNLOCKMATES(stp);
4597 mutex_exit(&stp1->sd_reflock);
4598 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4599 mutex_exit(&stp2->sd_reflock);
4600 goto retry;
4601 }
4602 STREAM_PUTLOCKS_ENTER(stp1);
4603 STREAM_PUTLOCKS_ENTER(stp2);
4604 } else {
4605 mutex_enter(&stp->sd_lock);
4606 mutex_enter(&stp->sd_reflock);
4607 while (stp->sd_refcnt > 0) {
4608 mutex_exit(&stp->sd_lock);
4609 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4610 if (mutex_tryenter(&stp->sd_lock) == 0) {
4611 mutex_exit(&stp->sd_reflock);
4612 mutex_enter(&stp->sd_lock);
4613 mutex_enter(&stp->sd_reflock);
4614 }
4615 }
4616 STREAM_PUTLOCKS_ENTER(stp);
4617 }
4618
4619 if (sqlist == NULL)
4620 return;
4621
4622 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4623 syncq_t *sq = sql->sql_sq;
4624 uint16_t count;
4625
4626 mutex_enter(SQLOCK(sq));
4627 count = sq->sq_count;
4628 ASSERT(sq->sq_rmqcount <= count);
4629 SQ_PUTLOCKS_ENTER(sq);
4630 SUM_SQ_PUTCOUNTS(sq, count);
4631 if (count == sq->sq_rmqcount)
4632 continue;
4633
4634 /* Failed - drop all locks that we have acquired so far */
4635 if (STRMATED(stp)) {
4636 STREAM_PUTLOCKS_EXIT(stp);
4637 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4638 STRUNLOCKMATES(stp);
4639 mutex_exit(&stp->sd_reflock);
4640 mutex_exit(&stp->sd_mate->sd_reflock);
4641 } else {
4642 STREAM_PUTLOCKS_EXIT(stp);
4643 mutex_exit(&stp->sd_lock);
4644 mutex_exit(&stp->sd_reflock);
4645 }
4646 for (sql2 = sqlist->sqlist_head; sql2 != sql;
4647 sql2 = sql2->sql_next) {
4648 SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4649 mutex_exit(SQLOCK(sql2->sql_sq));
4650 }
4651
4652 /*
4653 * The wait loop below may starve when there are many threads
4654 * claiming the syncq. This is especially a problem with permod
4655 * syncqs (IP). To lessen the impact of the problem we increment
4656 * sq_needexcl and clear fastbits so that putnexts will slow
4657 * down and call sqenable instead of draining right away.
4658 */
4659 sq->sq_needexcl++;
4660 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4661 while (count > sq->sq_rmqcount) {
4662 sq->sq_flags |= SQ_WANTWAKEUP;
4663 SQ_PUTLOCKS_EXIT(sq);
4664 cv_wait(&sq->sq_wait, SQLOCK(sq));
4665 count = sq->sq_count;
4666 SQ_PUTLOCKS_ENTER(sq);
4667 SUM_SQ_PUTCOUNTS(sq, count);
4668 }
4669 sq->sq_needexcl--;
4670 if (sq->sq_needexcl == 0)
4671 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4672 SQ_PUTLOCKS_EXIT(sq);
4673 ASSERT(count == sq->sq_rmqcount);
4674 mutex_exit(SQLOCK(sq));
4675 goto retry;
4676 }
4677 }
4678
4679 /*
4680 * Drop all the locks that strlock acquired.
4681 */
4682 static void
4683 strunlock(struct stdata *stp, sqlist_t *sqlist)
4684 {
4685 syncql_t *sql;
4686
4687 if (STRMATED(stp)) {
4688 STREAM_PUTLOCKS_EXIT(stp);
4689 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4690 STRUNLOCKMATES(stp);
4691 mutex_exit(&stp->sd_reflock);
4692 mutex_exit(&stp->sd_mate->sd_reflock);
4693 } else {
4694 STREAM_PUTLOCKS_EXIT(stp);
4695 mutex_exit(&stp->sd_lock);
4696 mutex_exit(&stp->sd_reflock);
4697 }
4698
4699 if (sqlist == NULL)
4700 return;
4701
4702 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4703 SQ_PUTLOCKS_EXIT(sql->sql_sq);
4704 mutex_exit(SQLOCK(sql->sql_sq));
4705 }
4706 }
4707
4708 /*
4709 * When the module has service procedure, we need check if the next
4710 * module which has service procedure is in flow control to trigger
4711 * the backenable.
4712 */
4713 static void
4714 backenable_insertedq(queue_t *q)
4715 {
4716 qband_t *qbp;
4717
4718 claimstr(q);
4719 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4720 if (q->q_next->q_nfsrv->q_flag & QWANTW)
4721 backenable(q, 0);
4722
4723 qbp = q->q_next->q_nfsrv->q_bandp;
4724 for (; qbp != NULL; qbp = qbp->qb_next)
4725 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4726 backenable(q, qbp->qb_first->b_band);
4727 }
4728 releasestr(q);
4729 }
4730
4731 /*
4732 * Given two read queues, insert a new single one after another.
4733 *
4734 * This routine acquires all the necessary locks in order to change
4735 * q_next and related pointer using strlock().
4736 * It depends on the stream head ensuring that there are no concurrent
4737 * insertq or removeq on the same stream. The stream head ensures this
4738 * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4739 *
4740 * Note that no syncq locks are held during the q_next change. This is
4741 * applied to all streams since, unlike removeq, there is no problem of stale
4742 * pointers when adding a module to the stream. Thus drivers/modules that do a
4743 * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4744 * applied this optimization to all streams.
4745 */
4746 void
4747 insertq(struct stdata *stp, queue_t *new)
4748 {
4749 queue_t *after;
4750 queue_t *wafter;
4751 queue_t *wnew = _WR(new);
4752 boolean_t have_fifo = B_FALSE;
4753
4754 if (new->q_flag & _QINSERTING) {
4755 ASSERT(stp->sd_vnode->v_type != VFIFO);
4756 after = new->q_next;
4757 wafter = _WR(new->q_next);
4758 } else {
4759 after = _RD(stp->sd_wrq);
4760 wafter = stp->sd_wrq;
4761 }
4762
4763 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4764 "insertq:%p, %p", after, new);
4765 ASSERT(after->q_flag & QREADR);
4766 ASSERT(new->q_flag & QREADR);
4767
4768 strlock(stp, NULL);
4769
4770 /* Do we have a FIFO? */
4771 if (wafter->q_next == after) {
4772 have_fifo = B_TRUE;
4773 wnew->q_next = new;
4774 } else {
4775 wnew->q_next = wafter->q_next;
4776 }
4777 new->q_next = after;
4778
4779 set_nfsrv_ptr(new, wnew, after, wafter);
4780 /*
4781 * set_nfsrv_ptr() needs to know if this is an insertion or not,
4782 * so only reset this flag after calling it.
4783 */
4784 new->q_flag &= ~_QINSERTING;
4785
4786 if (have_fifo) {
4787 wafter->q_next = wnew;
4788 } else {
4789 if (wafter->q_next)
4790 _OTHERQ(wafter->q_next)->q_next = new;
4791 wafter->q_next = wnew;
4792 }
4793
4794 set_qend(new);
4795 /* The QEND flag might have to be updated for the upstream guy */
4796 set_qend(after);
4797
4798 ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4799 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4800 ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4801 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4802 strsetuio(stp);
4803
4804 /*
4805 * If this was a module insertion, bump the push count.
4806 */
4807 if (!(new->q_flag & QISDRV))
4808 stp->sd_pushcnt++;
4809
4810 strunlock(stp, NULL);
4811
4812 /* check if the write Q needs backenable */
4813 backenable_insertedq(wnew);
4814
4815 /* check if the read Q needs backenable */
4816 backenable_insertedq(new);
4817 }
4818
4819 /*
4820 * Given a read queue, unlink it from any neighbors.
4821 *
4822 * This routine acquires all the necessary locks in order to
4823 * change q_next and related pointers and also guard against
4824 * stale references (e.g. through q_next) to the queue that
4825 * is being removed. It also plays part of the role in ensuring
4826 * that the module's/driver's put procedure doesn't get called
4827 * after qprocsoff returns.
4828 *
4829 * Removeq depends on the stream head ensuring that there are
4830 * no concurrent insertq or removeq on the same stream. The
4831 * stream head ensures this using the flags STWOPEN, STRCLOSE and
4832 * STRPLUMB.
4833 *
4834 * The set of locks needed to remove the queue is different in
4835 * different cases:
4836 *
4837 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4838 * waiting for the syncq reference count to drop to 0 indicating that no
4839 * non-close threads are present anywhere in the stream. This ensures that any
4840 * module/driver can reference q_next in its open, close, put, or service
4841 * procedures.
4842 *
4843 * The sq_rmqcount counter tracks the number of threads inside removeq().
4844 * strlock() ensures that there is either no threads executing inside perimeter
4845 * or there is only a thread calling qprocsoff().
4846 *
4847 * strlock() compares the value of sq_count with the number of threads inside
4848 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4849 * any threads waiting in strlock() when the sq_rmqcount increases.
4850 */
4851
4852 void
4853 removeq(queue_t *qp)
4854 {
4855 queue_t *wqp = _WR(qp);
4856 struct stdata *stp = STREAM(qp);
4857 sqlist_t *sqlist = NULL;
4858 boolean_t isdriver;
4859 int moved;
4860 syncq_t *sq = qp->q_syncq;
4861 syncq_t *wsq = wqp->q_syncq;
4862
4863 ASSERT(stp);
4864
4865 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4866 "removeq:%p %p", qp, wqp);
4867 ASSERT(qp->q_flag&QREADR);
4868
4869 /*
4870 * For queues using Synchronous streams, we must wait for all threads in
4871 * rwnext() to drain out before proceeding.
4872 */
4873 if (qp->q_flag & QSYNCSTR) {
4874 /* First, we need wakeup any threads blocked in rwnext() */
4875 mutex_enter(SQLOCK(sq));
4876 if (sq->sq_flags & SQ_WANTWAKEUP) {
4877 sq->sq_flags &= ~SQ_WANTWAKEUP;
4878 cv_broadcast(&sq->sq_wait);
4879 }
4880 mutex_exit(SQLOCK(sq));
4881
4882 if (wsq != sq) {
4883 mutex_enter(SQLOCK(wsq));
4884 if (wsq->sq_flags & SQ_WANTWAKEUP) {
4885 wsq->sq_flags &= ~SQ_WANTWAKEUP;
4886 cv_broadcast(&wsq->sq_wait);
4887 }
4888 mutex_exit(SQLOCK(wsq));
4889 }
4890
4891 mutex_enter(QLOCK(qp));
4892 while (qp->q_rwcnt > 0) {
4893 qp->q_flag |= QWANTRMQSYNC;
4894 cv_wait(&qp->q_wait, QLOCK(qp));
4895 }
4896 mutex_exit(QLOCK(qp));
4897
4898 mutex_enter(QLOCK(wqp));
4899 while (wqp->q_rwcnt > 0) {
4900 wqp->q_flag |= QWANTRMQSYNC;
4901 cv_wait(&wqp->q_wait, QLOCK(wqp));
4902 }
4903 mutex_exit(QLOCK(wqp));
4904 }
4905
4906 mutex_enter(SQLOCK(sq));
4907 sq->sq_rmqcount++;
4908 if (sq->sq_flags & SQ_WANTWAKEUP) {
4909 sq->sq_flags &= ~SQ_WANTWAKEUP;
4910 cv_broadcast(&sq->sq_wait);
4911 }
4912 mutex_exit(SQLOCK(sq));
4913
4914 isdriver = (qp->q_flag & QISDRV);
4915
4916 sqlist = sqlist_build(qp, stp, STRMATED(stp));
4917 strlock(stp, sqlist);
4918
4919 reset_nfsrv_ptr(qp, wqp);
4920
4921 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4922 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4923 /* Do we have a FIFO? */
4924 if (wqp->q_next == qp) {
4925 stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4926 } else {
4927 if (wqp->q_next)
4928 backq(qp)->q_next = qp->q_next;
4929 if (qp->q_next)
4930 backq(wqp)->q_next = wqp->q_next;
4931 }
4932
4933 /* The QEND flag might have to be updated for the upstream guy */
4934 if (qp->q_next)
4935 set_qend(qp->q_next);
4936
4937 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4938 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4939
4940 /*
4941 * Move any messages destined for the put procedures to the next
4942 * syncq in line. Otherwise free them.
4943 */
4944 moved = 0;
4945 /*
4946 * Quick check to see whether there are any messages or events.
4947 */
4948 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4949 moved += propagate_syncq(qp);
4950 if (wqp->q_syncqmsgs != 0 ||
4951 (wqp->q_syncq->sq_flags & SQ_EVENTS))
4952 moved += propagate_syncq(wqp);
4953
4954 strsetuio(stp);
4955
4956 /*
4957 * If this was a module removal, decrement the push count.
4958 */
4959 if (!isdriver)
4960 stp->sd_pushcnt--;
4961
4962 strunlock(stp, sqlist);
4963 sqlist_free(sqlist);
4964
4965 /*
4966 * Make sure any messages that were propagated are drained.
4967 * Also clear any QFULL bit caused by messages that were propagated.
4968 */
4969
4970 if (qp->q_next != NULL) {
4971 clr_qfull(qp);
4972 /*
4973 * For the driver calling qprocsoff, propagate_syncq
4974 * frees all the messages instead of putting it in
4975 * the stream head
4976 */
4977 if (!isdriver && (moved > 0))
4978 emptysq(qp->q_next->q_syncq);
4979 }
4980 if (wqp->q_next != NULL) {
4981 clr_qfull(wqp);
4982 /*
4983 * We come here for any pop of a module except for the
4984 * case of driver being removed. We don't call emptysq
4985 * if we did not move any messages. This will avoid holding
4986 * PERMOD syncq locks in emptysq
4987 */
4988 if (moved > 0)
4989 emptysq(wqp->q_next->q_syncq);
4990 }
4991
4992 mutex_enter(SQLOCK(sq));
4993 sq->sq_rmqcount--;
4994 mutex_exit(SQLOCK(sq));
4995 }
4996
4997 /*
4998 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
4999 * SQ_WRITER) on a syncq.
5000 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5001 * sync queue and waits until sq_count reaches maxcnt.
5002 *
5003 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5004 * does not care about putnext threads that are in the middle of calling put
5005 * entry points.
5006 *
5007 * This routine is used for both inner and outer syncqs.
5008 */
5009 static void
5010 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5011 {
5012 uint16_t count = 0;
5013
5014 mutex_enter(SQLOCK(sq));
5015 /*
5016 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5017 * SQ_FROZEN will be set if there is a frozen stream that has a
5018 * queue which also refers to this "shared" syncq.
5019 * SQ_BLOCKED will be set if there is "off" queue which also
5020 * refers to this "shared" syncq.
5021 */
5022 if (maxcnt != -1) {
5023 count = sq->sq_count;
5024 SQ_PUTLOCKS_ENTER(sq);
5025 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5026 SUM_SQ_PUTCOUNTS(sq, count);
5027 }
5028 sq->sq_needexcl++;
5029 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5030
5031 while ((sq->sq_flags & flag) ||
5032 (maxcnt != -1 && count > (unsigned)maxcnt)) {
5033 sq->sq_flags |= SQ_WANTWAKEUP;
5034 if (maxcnt != -1) {
5035 SQ_PUTLOCKS_EXIT(sq);
5036 }
5037 cv_wait(&sq->sq_wait, SQLOCK(sq));
5038 if (maxcnt != -1) {
5039 count = sq->sq_count;
5040 SQ_PUTLOCKS_ENTER(sq);
5041 SUM_SQ_PUTCOUNTS(sq, count);
5042 }
5043 }
5044 sq->sq_needexcl--;
5045 sq->sq_flags |= flag;
5046 ASSERT(maxcnt == -1 || count == maxcnt);
5047 if (maxcnt != -1) {
5048 if (sq->sq_needexcl == 0) {
5049 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5050 }
5051 SQ_PUTLOCKS_EXIT(sq);
5052 } else if (sq->sq_needexcl == 0) {
5053 SQ_PUTCOUNT_SETFAST(sq);
5054 }
5055
5056 mutex_exit(SQLOCK(sq));
5057 }
5058
5059 /*
5060 * Reset a flag that was set with blocksq.
5061 *
5062 * Can not use this routine to reset SQ_WRITER.
5063 *
5064 * If "isouter" is set then the syncq is assumed to be an outer perimeter
5065 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5066 * to handle the queued qwriter operations.
5067 *
5068 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5069 * sq_putlocks are used.
5070 */
5071 static void
5072 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5073 {
5074 uint16_t flags;
5075
5076 mutex_enter(SQLOCK(sq));
5077 ASSERT(resetflag != SQ_WRITER);
5078 ASSERT(sq->sq_flags & resetflag);
5079 flags = sq->sq_flags & ~resetflag;
5080 sq->sq_flags = flags;
5081 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5082 if (flags & SQ_WANTWAKEUP) {
5083 flags &= ~SQ_WANTWAKEUP;
5084 cv_broadcast(&sq->sq_wait);
5085 }
5086 sq->sq_flags = flags;
5087 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5088 if (!isouter) {
5089 /* drain_syncq drops SQLOCK */
5090 drain_syncq(sq);
5091 return;
5092 }
5093 }
5094 }
5095 mutex_exit(SQLOCK(sq));
5096 }
5097
5098 /*
5099 * Reset a flag that was set with blocksq.
5100 * Does not drain the syncq. Use emptysq() for that.
5101 * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5102 *
5103 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5104 * sq_putlocks are used.
5105 */
5106 static int
5107 dropsq(syncq_t *sq, uint16_t resetflag)
5108 {
5109 uint16_t flags;
5110
5111 mutex_enter(SQLOCK(sq));
5112 ASSERT(sq->sq_flags & resetflag);
5113 flags = sq->sq_flags & ~resetflag;
5114 if (flags & SQ_WANTWAKEUP) {
5115 flags &= ~SQ_WANTWAKEUP;
5116 cv_broadcast(&sq->sq_wait);
5117 }
5118 sq->sq_flags = flags;
5119 mutex_exit(SQLOCK(sq));
5120 if (flags & SQ_QUEUED)
5121 return (1);
5122 return (0);
5123 }
5124
5125 /*
5126 * Empty all the messages on a syncq.
5127 *
5128 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5129 * sq_putlocks are used.
5130 */
5131 static void
5132 emptysq(syncq_t *sq)
5133 {
5134 uint16_t flags;
5135
5136 mutex_enter(SQLOCK(sq));
5137 flags = sq->sq_flags;
5138 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5139 /*
5140 * To prevent potential recursive invocation of drain_syncq we
5141 * do not call drain_syncq if count is non-zero.
5142 */
5143 if (sq->sq_count == 0) {
5144 /* drain_syncq() drops SQLOCK */
5145 drain_syncq(sq);
5146 return;
5147 } else
5148 sqenable(sq);
5149 }
5150 mutex_exit(SQLOCK(sq));
5151 }
5152
5153 /*
5154 * Ordered insert while removing duplicates.
5155 */
5156 static void
5157 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5158 {
5159 syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5160
5161 prev_sqlpp = &sqlist->sqlist_head;
5162 while ((sqlp = *prev_sqlpp) != NULL) {
5163 if (sqlp->sql_sq >= sqp) {
5164 if (sqlp->sql_sq == sqp) /* duplicate */
5165 return;
5166 break;
5167 }
5168 prev_sqlpp = &sqlp->sql_next;
5169 }
5170 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5171 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5172 new_sqlp->sql_next = sqlp;
5173 new_sqlp->sql_sq = sqp;
5174 *prev_sqlpp = new_sqlp;
5175 }
5176
5177 /*
5178 * Walk the write side queues until we hit either the driver
5179 * or a twist in the stream (_SAMESTR will return false in both
5180 * these cases) then turn around and walk the read side queues
5181 * back up to the stream head.
5182 */
5183 static void
5184 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5185 {
5186 while (q != NULL) {
5187 sqlist_insert(sqlist, q->q_syncq);
5188
5189 if (_SAMESTR(q))
5190 q = q->q_next;
5191 else if (!(q->q_flag & QREADR))
5192 q = _RD(q);
5193 else
5194 q = NULL;
5195 }
5196 }
5197
5198 /*
5199 * Allocate and build a list of all syncqs in a stream and the syncq(s)
5200 * associated with the "q" parameter. The resulting list is sorted in a
5201 * canonical order and is free of duplicates.
5202 * Assumes the passed queue is a _RD(q).
5203 */
5204 static sqlist_t *
5205 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5206 {
5207 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5208
5209 /*
5210 * start with the current queue/qpair
5211 */
5212 ASSERT(q->q_flag & QREADR);
5213
5214 sqlist_insert(sqlist, q->q_syncq);
5215 sqlist_insert(sqlist, _WR(q)->q_syncq);
5216
5217 sqlist_insertall(sqlist, stp->sd_wrq);
5218 if (do_twist)
5219 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5220
5221 return (sqlist);
5222 }
5223
5224 static sqlist_t *
5225 sqlist_alloc(struct stdata *stp, int kmflag)
5226 {
5227 size_t sqlist_size;
5228 sqlist_t *sqlist;
5229
5230 /*
5231 * Allocate 2 syncql_t's for each pushed module. Note that
5232 * the sqlist_t structure already has 4 syncql_t's built in:
5233 * 2 for the stream head, and 2 for the driver/other stream head.
5234 */
5235 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5236 sizeof (sqlist_t);
5237 if (STRMATED(stp))
5238 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5239 sqlist = kmem_alloc(sqlist_size, kmflag);
5240
5241 sqlist->sqlist_head = NULL;
5242 sqlist->sqlist_size = sqlist_size;
5243 sqlist->sqlist_index = 0;
5244
5245 return (sqlist);
5246 }
5247
5248 /*
5249 * Free the list created by sqlist_alloc()
5250 */
5251 static void
5252 sqlist_free(sqlist_t *sqlist)
5253 {
5254 kmem_free(sqlist, sqlist->sqlist_size);
5255 }
5256
5257 /*
5258 * Prevent any new entries into any syncq in this stream.
5259 * Used by freezestr.
5260 */
5261 void
5262 strblock(queue_t *q)
5263 {
5264 struct stdata *stp;
5265 syncql_t *sql;
5266 sqlist_t *sqlist;
5267
5268 q = _RD(q);
5269
5270 stp = STREAM(q);
5271 ASSERT(stp != NULL);
5272
5273 /*
5274 * Get a sorted list with all the duplicates removed containing
5275 * all the syncqs referenced by this stream.
5276 */
5277 sqlist = sqlist_build(q, stp, B_FALSE);
5278 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5279 blocksq(sql->sql_sq, SQ_FROZEN, -1);
5280 sqlist_free(sqlist);
5281 }
5282
5283 /*
5284 * Release the block on new entries into this stream
5285 */
5286 void
5287 strunblock(queue_t *q)
5288 {
5289 struct stdata *stp;
5290 syncql_t *sql;
5291 sqlist_t *sqlist;
5292 int drain_needed;
5293
5294 q = _RD(q);
5295
5296 /*
5297 * Get a sorted list with all the duplicates removed containing
5298 * all the syncqs referenced by this stream.
5299 * Have to drop the SQ_FROZEN flag on all the syncqs before
5300 * starting to drain them; otherwise the draining might
5301 * cause a freezestr in some module on the stream (which
5302 * would deadlock).
5303 */
5304 stp = STREAM(q);
5305 ASSERT(stp != NULL);
5306 sqlist = sqlist_build(q, stp, B_FALSE);
5307 drain_needed = 0;
5308 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5309 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5310 if (drain_needed) {
5311 for (sql = sqlist->sqlist_head; sql != NULL;
5312 sql = sql->sql_next)
5313 emptysq(sql->sql_sq);
5314 }
5315 sqlist_free(sqlist);
5316 }
5317
5318 #ifdef DEBUG
5319 static int
5320 qprocsareon(queue_t *rq)
5321 {
5322 if (rq->q_next == NULL)
5323 return (0);
5324 return (_WR(rq->q_next)->q_next == _WR(rq));
5325 }
5326
5327 int
5328 qclaimed(queue_t *q)
5329 {
5330 uint_t count;
5331
5332 count = q->q_syncq->sq_count;
5333 SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5334 return (count != 0);
5335 }
5336
5337 /*
5338 * Check if anyone has frozen this stream with freezestr
5339 */
5340 int
5341 frozenstr(queue_t *q)
5342 {
5343 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5344 }
5345 #endif /* DEBUG */
5346
5347 /*
5348 * Enter a queue.
5349 * Obsoleted interface. Should not be used.
5350 */
5351 void
5352 enterq(queue_t *q)
5353 {
5354 entersq(q->q_syncq, SQ_CALLBACK);
5355 }
5356
5357 void
5358 leaveq(queue_t *q)
5359 {
5360 leavesq(q->q_syncq, SQ_CALLBACK);
5361 }
5362
5363 /*
5364 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5365 * to check.
5366 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5367 * calls and the running of open, close and service procedures.
5368 *
5369 * If c_inner bit is set no need to grab sq_putlocks since we don't care
5370 * if other threads have entered or are entering put entry point.
5371 *
5372 * If c_inner bit is set it might have been possible to use
5373 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5374 * open/close path for IP) but since the count may need to be decremented in
5375 * qwait() we wouldn't know which counter to decrement. Currently counter is
5376 * selected by current cpu_seqid and current CPU can change at any moment. XXX
5377 * in the future we might use curthread id bits to select the counter and this
5378 * would stay constant across routine calls.
5379 */
5380 void
5381 entersq(syncq_t *sq, int entrypoint)
5382 {
5383 uint16_t count = 0;
5384 uint16_t flags;
5385 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5386 uint16_t type;
5387 uint_t c_inner = entrypoint & SQ_CI;
5388 uint_t c_outer = entrypoint & SQ_CO;
5389
5390 /*
5391 * Increment ref count to keep closes out of this queue.
5392 */
5393 ASSERT(sq);
5394 ASSERT(c_inner && c_outer);
5395 mutex_enter(SQLOCK(sq));
5396 flags = sq->sq_flags;
5397 type = sq->sq_type;
5398 if (!(type & c_inner)) {
5399 /* Make sure all putcounts now use slowlock. */
5400 count = sq->sq_count;
5401 SQ_PUTLOCKS_ENTER(sq);
5402 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5403 SUM_SQ_PUTCOUNTS(sq, count);
5404 sq->sq_needexcl++;
5405 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5406 waitflags |= SQ_MESSAGES;
5407 }
5408 /*
5409 * Wait until we can enter the inner perimeter.
5410 * If we want exclusive access we wait until sq_count is 0.
5411 * We have to do this before entering the outer perimeter in order
5412 * to preserve put/close message ordering.
5413 */
5414 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5415 sq->sq_flags = flags | SQ_WANTWAKEUP;
5416 if (!(type & c_inner)) {
5417 SQ_PUTLOCKS_EXIT(sq);
5418 }
5419 cv_wait(&sq->sq_wait, SQLOCK(sq));
5420 if (!(type & c_inner)) {
5421 count = sq->sq_count;
5422 SQ_PUTLOCKS_ENTER(sq);
5423 SUM_SQ_PUTCOUNTS(sq, count);
5424 }
5425 flags = sq->sq_flags;
5426 }
5427
5428 if (!(type & c_inner)) {
5429 ASSERT(sq->sq_needexcl > 0);
5430 sq->sq_needexcl--;
5431 if (sq->sq_needexcl == 0) {
5432 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5433 }
5434 }
5435
5436 /* Check if we need to enter the outer perimeter */
5437 if (!(type & c_outer)) {
5438 /*
5439 * We have to enter the outer perimeter exclusively before
5440 * we can increment sq_count to avoid deadlock. This implies
5441 * that we have to re-check sq_flags and sq_count.
5442 *
5443 * is it possible to have c_inner set when c_outer is not set?
5444 */
5445 if (!(type & c_inner)) {
5446 SQ_PUTLOCKS_EXIT(sq);
5447 }
5448 mutex_exit(SQLOCK(sq));
5449 outer_enter(sq->sq_outer, SQ_GOAWAY);
5450 mutex_enter(SQLOCK(sq));
5451 flags = sq->sq_flags;
5452 /*
5453 * there should be no need to recheck sq_putcounts
5454 * because outer_enter() has already waited for them to clear
5455 * after setting SQ_WRITER.
5456 */
5457 count = sq->sq_count;
5458 #ifdef DEBUG
5459 /*
5460 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5461 * of doing an ASSERT internally. Others should do
5462 * something like
5463 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5464 * without the need to #ifdef DEBUG it.
5465 */
5466 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5467 #endif
5468 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5469 (!(type & c_inner) && count != 0)) {
5470 sq->sq_flags = flags | SQ_WANTWAKEUP;
5471 cv_wait(&sq->sq_wait, SQLOCK(sq));
5472 count = sq->sq_count;
5473 flags = sq->sq_flags;
5474 }
5475 }
5476
5477 sq->sq_count++;
5478 ASSERT(sq->sq_count != 0); /* Wraparound */
5479 if (!(type & c_inner)) {
5480 /* Exclusive entry */
5481 ASSERT(sq->sq_count == 1);
5482 sq->sq_flags |= SQ_EXCL;
5483 if (type & c_outer) {
5484 SQ_PUTLOCKS_EXIT(sq);
5485 }
5486 }
5487 mutex_exit(SQLOCK(sq));
5488 }
5489
5490 /*
5491 * Leave a syncq. Announce to framework that closes may proceed.
5492 * c_inner and c_outer specify which concurrency bits to check.
5493 *
5494 * Must never be called from driver or module put entry point.
5495 *
5496 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5497 * sq_putlocks are used.
5498 */
5499 void
5500 leavesq(syncq_t *sq, int entrypoint)
5501 {
5502 uint16_t flags;
5503 uint16_t type;
5504 uint_t c_outer = entrypoint & SQ_CO;
5505 #ifdef DEBUG
5506 uint_t c_inner = entrypoint & SQ_CI;
5507 #endif
5508
5509 /*
5510 * Decrement ref count, drain the syncq if possible, and wake up
5511 * any waiting close.
5512 */
5513 ASSERT(sq);
5514 ASSERT(c_inner && c_outer);
5515 mutex_enter(SQLOCK(sq));
5516 flags = sq->sq_flags;
5517 type = sq->sq_type;
5518 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5519
5520 if (flags & SQ_WANTWAKEUP) {
5521 flags &= ~SQ_WANTWAKEUP;
5522 cv_broadcast(&sq->sq_wait);
5523 }
5524 if (flags & SQ_WANTEXWAKEUP) {
5525 flags &= ~SQ_WANTEXWAKEUP;
5526 cv_broadcast(&sq->sq_exitwait);
5527 }
5528
5529 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5530 /*
5531 * The syncq needs to be drained. "Exit" the syncq
5532 * before calling drain_syncq.
5533 */
5534 ASSERT(sq->sq_count != 0);
5535 sq->sq_count--;
5536 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5537 sq->sq_flags = flags & ~SQ_EXCL;
5538 drain_syncq(sq);
5539 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5540 /* Check if we need to exit the outer perimeter */
5541 /* XXX will this ever be true? */
5542 if (!(type & c_outer))
5543 outer_exit(sq->sq_outer);
5544 return;
5545 }
5546 }
5547 ASSERT(sq->sq_count != 0);
5548 sq->sq_count--;
5549 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5550 sq->sq_flags = flags & ~SQ_EXCL;
5551 mutex_exit(SQLOCK(sq));
5552
5553 /* Check if we need to exit the outer perimeter */
5554 if (!(sq->sq_type & c_outer))
5555 outer_exit(sq->sq_outer);
5556 }
5557
5558 /*
5559 * Prevent q_next from changing in this stream by incrementing sq_count.
5560 *
5561 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5562 * sq_putlocks are used.
5563 */
5564 void
5565 claimq(queue_t *qp)
5566 {
5567 syncq_t *sq = qp->q_syncq;
5568
5569 mutex_enter(SQLOCK(sq));
5570 sq->sq_count++;
5571 ASSERT(sq->sq_count != 0); /* Wraparound */
5572 mutex_exit(SQLOCK(sq));
5573 }
5574
5575 /*
5576 * Undo claimq.
5577 *
5578 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5579 * sq_putlocks are used.
5580 */
5581 void
5582 releaseq(queue_t *qp)
5583 {
5584 syncq_t *sq = qp->q_syncq;
5585 uint16_t flags;
5586
5587 mutex_enter(SQLOCK(sq));
5588 ASSERT(sq->sq_count > 0);
5589 sq->sq_count--;
5590
5591 flags = sq->sq_flags;
5592 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5593 if (flags & SQ_WANTWAKEUP) {
5594 flags &= ~SQ_WANTWAKEUP;
5595 cv_broadcast(&sq->sq_wait);
5596 }
5597 sq->sq_flags = flags;
5598 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5599 /*
5600 * To prevent potential recursive invocation of
5601 * drain_syncq we do not call drain_syncq if count is
5602 * non-zero.
5603 */
5604 if (sq->sq_count == 0) {
5605 drain_syncq(sq);
5606 return;
5607 } else
5608 sqenable(sq);
5609 }
5610 }
5611 mutex_exit(SQLOCK(sq));
5612 }
5613
5614 /*
5615 * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5616 */
5617 void
5618 claimstr(queue_t *qp)
5619 {
5620 struct stdata *stp = STREAM(qp);
5621
5622 mutex_enter(&stp->sd_reflock);
5623 stp->sd_refcnt++;
5624 ASSERT(stp->sd_refcnt != 0); /* Wraparound */
5625 mutex_exit(&stp->sd_reflock);
5626 }
5627
5628 /*
5629 * Undo claimstr.
5630 */
5631 void
5632 releasestr(queue_t *qp)
5633 {
5634 struct stdata *stp = STREAM(qp);
5635
5636 mutex_enter(&stp->sd_reflock);
5637 ASSERT(stp->sd_refcnt != 0);
5638 if (--stp->sd_refcnt == 0)
5639 cv_broadcast(&stp->sd_refmonitor);
5640 mutex_exit(&stp->sd_reflock);
5641 }
5642
5643 static syncq_t *
5644 new_syncq(void)
5645 {
5646 return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5647 }
5648
5649 static void
5650 free_syncq(syncq_t *sq)
5651 {
5652 ASSERT(sq->sq_head == NULL);
5653 ASSERT(sq->sq_outer == NULL);
5654 ASSERT(sq->sq_callbpend == NULL);
5655 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5656 (sq->sq_onext == sq && sq->sq_oprev == sq));
5657
5658 if (sq->sq_ciputctrl != NULL) {
5659 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5660 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5661 sq->sq_nciputctrl, 0);
5662 ASSERT(ciputctrl_cache != NULL);
5663 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5664 }
5665
5666 sq->sq_tail = NULL;
5667 sq->sq_evhead = NULL;
5668 sq->sq_evtail = NULL;
5669 sq->sq_ciputctrl = NULL;
5670 sq->sq_nciputctrl = 0;
5671 sq->sq_count = 0;
5672 sq->sq_rmqcount = 0;
5673 sq->sq_callbflags = 0;
5674 sq->sq_cancelid = 0;
5675 sq->sq_next = NULL;
5676 sq->sq_needexcl = 0;
5677 sq->sq_svcflags = 0;
5678 sq->sq_nqueues = 0;
5679 sq->sq_pri = 0;
5680 sq->sq_onext = NULL;
5681 sq->sq_oprev = NULL;
5682 sq->sq_flags = 0;
5683 sq->sq_type = 0;
5684 sq->sq_servcount = 0;
5685
5686 kmem_cache_free(syncq_cache, sq);
5687 }
5688
5689 /* Outer perimeter code */
5690
5691 /*
5692 * The outer syncq uses the fields and flags in the syncq slightly
5693 * differently from the inner syncqs.
5694 * sq_count Incremented when there are pending or running
5695 * writers at the outer perimeter to prevent the set of
5696 * inner syncqs that belong to the outer perimeter from
5697 * changing.
5698 * sq_head/tail List of deferred qwriter(OUTER) operations.
5699 *
5700 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while
5701 * inner syncqs are added to or removed from the
5702 * outer perimeter.
5703 * SQ_QUEUED sq_head/tail has messages or events queued.
5704 *
5705 * SQ_WRITER A thread is currently traversing all the inner syncqs
5706 * setting the SQ_WRITER flag.
5707 */
5708
5709 /*
5710 * Get write access at the outer perimeter.
5711 * Note that read access is done by entersq, putnext, and put by simply
5712 * incrementing sq_count in the inner syncq.
5713 *
5714 * Waits until "flags" is no longer set in the outer to prevent multiple
5715 * threads from having write access at the same time. SQ_WRITER has to be part
5716 * of "flags".
5717 *
5718 * Increases sq_count on the outer syncq to keep away outer_insert/remove
5719 * until the outer_exit is finished.
5720 *
5721 * outer_enter is vulnerable to starvation since it does not prevent new
5722 * threads from entering the inner syncqs while it is waiting for sq_count to
5723 * go to zero.
5724 */
5725 void
5726 outer_enter(syncq_t *outer, uint16_t flags)
5727 {
5728 syncq_t *sq;
5729 int wait_needed;
5730 uint16_t count;
5731
5732 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5733 outer->sq_oprev != NULL);
5734 ASSERT(flags & SQ_WRITER);
5735
5736 retry:
5737 mutex_enter(SQLOCK(outer));
5738 while (outer->sq_flags & flags) {
5739 outer->sq_flags |= SQ_WANTWAKEUP;
5740 cv_wait(&outer->sq_wait, SQLOCK(outer));
5741 }
5742
5743 ASSERT(!(outer->sq_flags & SQ_WRITER));
5744 outer->sq_flags |= SQ_WRITER;
5745 outer->sq_count++;
5746 ASSERT(outer->sq_count != 0); /* wraparound */
5747 wait_needed = 0;
5748 /*
5749 * Set SQ_WRITER on all the inner syncqs while holding
5750 * the SQLOCK on the outer syncq. This ensures that the changing
5751 * of SQ_WRITER is atomic under the outer SQLOCK.
5752 */
5753 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5754 mutex_enter(SQLOCK(sq));
5755 count = sq->sq_count;
5756 SQ_PUTLOCKS_ENTER(sq);
5757 sq->sq_flags |= SQ_WRITER;
5758 SUM_SQ_PUTCOUNTS(sq, count);
5759 if (count != 0)
5760 wait_needed = 1;
5761 SQ_PUTLOCKS_EXIT(sq);
5762 mutex_exit(SQLOCK(sq));
5763 }
5764 mutex_exit(SQLOCK(outer));
5765
5766 /*
5767 * Get everybody out of the syncqs sequentially.
5768 * Note that we don't actually need to acquire the PUTLOCKS, since
5769 * we have already cleared the fastbit, and set QWRITER. By
5770 * definition, the count can not increase since putnext will
5771 * take the slowlock path (and the purpose of acquiring the
5772 * putlocks was to make sure it didn't increase while we were
5773 * waiting).
5774 *
5775 * Note that we still acquire the PUTLOCKS to be safe.
5776 */
5777 if (wait_needed) {
5778 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5779 mutex_enter(SQLOCK(sq));
5780 count = sq->sq_count;
5781 SQ_PUTLOCKS_ENTER(sq);
5782 SUM_SQ_PUTCOUNTS(sq, count);
5783 while (count != 0) {
5784 sq->sq_flags |= SQ_WANTWAKEUP;
5785 SQ_PUTLOCKS_EXIT(sq);
5786 cv_wait(&sq->sq_wait, SQLOCK(sq));
5787 count = sq->sq_count;
5788 SQ_PUTLOCKS_ENTER(sq);
5789 SUM_SQ_PUTCOUNTS(sq, count);
5790 }
5791 SQ_PUTLOCKS_EXIT(sq);
5792 mutex_exit(SQLOCK(sq));
5793 }
5794 /*
5795 * Verify that none of the flags got set while we
5796 * were waiting for the sq_counts to drop.
5797 * If this happens we exit and retry entering the
5798 * outer perimeter.
5799 */
5800 mutex_enter(SQLOCK(outer));
5801 if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5802 mutex_exit(SQLOCK(outer));
5803 outer_exit(outer);
5804 goto retry;
5805 }
5806 mutex_exit(SQLOCK(outer));
5807 }
5808 }
5809
5810 /*
5811 * Drop the write access at the outer perimeter.
5812 * Read access is dropped implicitly (by putnext, put, and leavesq) by
5813 * decrementing sq_count.
5814 */
5815 void
5816 outer_exit(syncq_t *outer)
5817 {
5818 syncq_t *sq;
5819 int drain_needed;
5820 uint16_t flags;
5821
5822 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5823 outer->sq_oprev != NULL);
5824 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5825
5826 /*
5827 * Atomically (from the perspective of threads calling become_writer)
5828 * drop the write access at the outer perimeter by holding
5829 * SQLOCK(outer) across all the dropsq calls and the resetting of
5830 * SQ_WRITER.
5831 * This defines a locking order between the outer perimeter
5832 * SQLOCK and the inner perimeter SQLOCKs.
5833 */
5834 mutex_enter(SQLOCK(outer));
5835 flags = outer->sq_flags;
5836 ASSERT(outer->sq_flags & SQ_WRITER);
5837 if (flags & SQ_QUEUED) {
5838 write_now(outer);
5839 flags = outer->sq_flags;
5840 }
5841
5842 /*
5843 * sq_onext is stable since sq_count has not yet been decreased.
5844 * Reset the SQ_WRITER flags in all syncqs.
5845 * After dropping SQ_WRITER on the outer syncq we empty all the
5846 * inner syncqs.
5847 */
5848 drain_needed = 0;
5849 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5850 drain_needed += dropsq(sq, SQ_WRITER);
5851 ASSERT(!(outer->sq_flags & SQ_QUEUED));
5852 flags &= ~SQ_WRITER;
5853 if (drain_needed) {
5854 outer->sq_flags = flags;
5855 mutex_exit(SQLOCK(outer));
5856 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5857 emptysq(sq);
5858 mutex_enter(SQLOCK(outer));
5859 flags = outer->sq_flags;
5860 }
5861 if (flags & SQ_WANTWAKEUP) {
5862 flags &= ~SQ_WANTWAKEUP;
5863 cv_broadcast(&outer->sq_wait);
5864 }
5865 outer->sq_flags = flags;
5866 ASSERT(outer->sq_count > 0);
5867 outer->sq_count--;
5868 mutex_exit(SQLOCK(outer));
5869 }
5870
5871 /*
5872 * Add another syncq to an outer perimeter.
5873 * Block out all other access to the outer perimeter while it is being
5874 * changed using blocksq.
5875 * Assumes that the caller has *not* done an outer_enter.
5876 *
5877 * Vulnerable to starvation in blocksq.
5878 */
5879 static void
5880 outer_insert(syncq_t *outer, syncq_t *sq)
5881 {
5882 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5883 outer->sq_oprev != NULL);
5884 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5885 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */
5886
5887 /* Get exclusive access to the outer perimeter list */
5888 blocksq(outer, SQ_BLOCKED, 0);
5889 ASSERT(outer->sq_flags & SQ_BLOCKED);
5890 ASSERT(!(outer->sq_flags & SQ_WRITER));
5891
5892 mutex_enter(SQLOCK(sq));
5893 sq->sq_outer = outer;
5894 outer->sq_onext->sq_oprev = sq;
5895 sq->sq_onext = outer->sq_onext;
5896 outer->sq_onext = sq;
5897 sq->sq_oprev = outer;
5898 mutex_exit(SQLOCK(sq));
5899 unblocksq(outer, SQ_BLOCKED, 1);
5900 }
5901
5902 /*
5903 * Remove a syncq from an outer perimeter.
5904 * Block out all other access to the outer perimeter while it is being
5905 * changed using blocksq.
5906 * Assumes that the caller has *not* done an outer_enter.
5907 *
5908 * Vulnerable to starvation in blocksq.
5909 */
5910 static void
5911 outer_remove(syncq_t *outer, syncq_t *sq)
5912 {
5913 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5914 outer->sq_oprev != NULL);
5915 ASSERT(sq->sq_outer == outer);
5916
5917 /* Get exclusive access to the outer perimeter list */
5918 blocksq(outer, SQ_BLOCKED, 0);
5919 ASSERT(outer->sq_flags & SQ_BLOCKED);
5920 ASSERT(!(outer->sq_flags & SQ_WRITER));
5921
5922 mutex_enter(SQLOCK(sq));
5923 sq->sq_outer = NULL;
5924 sq->sq_onext->sq_oprev = sq->sq_oprev;
5925 sq->sq_oprev->sq_onext = sq->sq_onext;
5926 sq->sq_oprev = sq->sq_onext = NULL;
5927 mutex_exit(SQLOCK(sq));
5928 unblocksq(outer, SQ_BLOCKED, 1);
5929 }
5930
5931 /*
5932 * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5933 * If this is the first callback for this outer perimeter then add
5934 * this outer perimeter to the list of outer perimeters that
5935 * the qwriter_outer_thread will process.
5936 *
5937 * Increments sq_count in the outer syncq to prevent the membership
5938 * of the outer perimeter (in terms of inner syncqs) to change while
5939 * the callback is pending.
5940 */
5941 static void
5942 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5943 {
5944 ASSERT(MUTEX_HELD(SQLOCK(outer)));
5945
5946 mp->b_prev = (mblk_t *)func;
5947 mp->b_queue = q;
5948 mp->b_next = NULL;
5949 outer->sq_count++; /* Decremented when dequeued */
5950 ASSERT(outer->sq_count != 0); /* Wraparound */
5951 if (outer->sq_evhead == NULL) {
5952 /* First message. */
5953 outer->sq_evhead = outer->sq_evtail = mp;
5954 outer->sq_flags |= SQ_EVENTS;
5955 mutex_exit(SQLOCK(outer));
5956 STRSTAT(qwr_outer);
5957 (void) taskq_dispatch(streams_taskq,
5958 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5959 } else {
5960 ASSERT(outer->sq_flags & SQ_EVENTS);
5961 outer->sq_evtail->b_next = mp;
5962 outer->sq_evtail = mp;
5963 mutex_exit(SQLOCK(outer));
5964 }
5965 }
5966
5967 /*
5968 * Try and upgrade to write access at the outer perimeter. If this can
5969 * not be done without blocking then queue the callback to be done
5970 * by the qwriter_outer_thread.
5971 *
5972 * This routine can only be called from put or service procedures plus
5973 * asynchronous callback routines that have properly entered the queue (with
5974 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5975 * associated with q.
5976 */
5977 void
5978 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
5979 {
5980 syncq_t *osq, *sq, *outer;
5981 int failed;
5982 uint16_t flags;
5983
5984 osq = q->q_syncq;
5985 outer = osq->sq_outer;
5986 if (outer == NULL)
5987 panic("qwriter(PERIM_OUTER): no outer perimeter");
5988 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5989 outer->sq_oprev != NULL);
5990
5991 mutex_enter(SQLOCK(outer));
5992 flags = outer->sq_flags;
5993 /*
5994 * If some thread is traversing sq_next, or if we are blocked by
5995 * outer_insert or outer_remove, or if the we already have queued
5996 * callbacks, then queue this callback for later processing.
5997 *
5998 * Also queue the qwriter for an interrupt thread in order
5999 * to reduce the time spent running at high IPL.
6000 * to identify there are events.
6001 */
6002 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6003 /*
6004 * Queue the become_writer request.
6005 * The queueing is atomic under SQLOCK(outer) in order
6006 * to synchronize with outer_exit.
6007 * queue_writer will drop the outer SQLOCK
6008 */
6009 if (flags & SQ_BLOCKED) {
6010 /* Must set SQ_WRITER on inner perimeter */
6011 mutex_enter(SQLOCK(osq));
6012 osq->sq_flags |= SQ_WRITER;
6013 mutex_exit(SQLOCK(osq));
6014 } else {
6015 if (!(flags & SQ_WRITER)) {
6016 /*
6017 * The outer could have been SQ_BLOCKED thus
6018 * SQ_WRITER might not be set on the inner.
6019 */
6020 mutex_enter(SQLOCK(osq));
6021 osq->sq_flags |= SQ_WRITER;
6022 mutex_exit(SQLOCK(osq));
6023 }
6024 ASSERT(osq->sq_flags & SQ_WRITER);
6025 }
6026 queue_writer(outer, func, q, mp);
6027 return;
6028 }
6029 /*
6030 * We are half-way to exclusive access to the outer perimeter.
6031 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6032 * while the inner syncqs are traversed.
6033 */
6034 outer->sq_count++;
6035 ASSERT(outer->sq_count != 0); /* wraparound */
6036 flags |= SQ_WRITER;
6037 /*
6038 * Check if we can run the function immediately. Mark all
6039 * syncqs with the writer flag to prevent new entries into
6040 * put and service procedures.
6041 *
6042 * Set SQ_WRITER on all the inner syncqs while holding
6043 * the SQLOCK on the outer syncq. This ensures that the changing
6044 * of SQ_WRITER is atomic under the outer SQLOCK.
6045 */
6046 failed = 0;
6047 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6048 uint16_t count;
6049 uint_t maxcnt = (sq == osq) ? 1 : 0;
6050
6051 mutex_enter(SQLOCK(sq));
6052 count = sq->sq_count;
6053 SQ_PUTLOCKS_ENTER(sq);
6054 SUM_SQ_PUTCOUNTS(sq, count);
6055 if (sq->sq_count > maxcnt)
6056 failed = 1;
6057 sq->sq_flags |= SQ_WRITER;
6058 SQ_PUTLOCKS_EXIT(sq);
6059 mutex_exit(SQLOCK(sq));
6060 }
6061 if (failed) {
6062 /*
6063 * Some other thread has a read claim on the outer perimeter.
6064 * Queue the callback for deferred processing.
6065 *
6066 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6067 * so that other qwriter(OUTER) calls will queue their
6068 * callbacks as well. queue_writer increments sq_count so we
6069 * decrement to compensate for the our increment.
6070 *
6071 * Dropping SQ_WRITER enables the writer thread to work
6072 * on this outer perimeter.
6073 */
6074 outer->sq_flags = flags;
6075 queue_writer(outer, func, q, mp);
6076 /* queue_writer dropper the lock */
6077 mutex_enter(SQLOCK(outer));
6078 ASSERT(outer->sq_count > 0);
6079 outer->sq_count--;
6080 ASSERT(outer->sq_flags & SQ_WRITER);
6081 flags = outer->sq_flags;
6082 flags &= ~SQ_WRITER;
6083 if (flags & SQ_WANTWAKEUP) {
6084 flags &= ~SQ_WANTWAKEUP;
6085 cv_broadcast(&outer->sq_wait);
6086 }
6087 outer->sq_flags = flags;
6088 mutex_exit(SQLOCK(outer));
6089 return;
6090 } else {
6091 outer->sq_flags = flags;
6092 mutex_exit(SQLOCK(outer));
6093 }
6094
6095 /* Can run it immediately */
6096 (*func)(q, mp);
6097
6098 outer_exit(outer);
6099 }
6100
6101 /*
6102 * Dequeue all writer callbacks from the outer perimeter and run them.
6103 */
6104 static void
6105 write_now(syncq_t *outer)
6106 {
6107 mblk_t *mp;
6108 queue_t *q;
6109 void (*func)();
6110
6111 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6112 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6113 outer->sq_oprev != NULL);
6114 while ((mp = outer->sq_evhead) != NULL) {
6115 /*
6116 * queues cannot be placed on the queuelist on the outer
6117 * perimeter.
6118 */
6119 ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6120 ASSERT((outer->sq_flags & SQ_EVENTS));
6121
6122 outer->sq_evhead = mp->b_next;
6123 if (outer->sq_evhead == NULL) {
6124 outer->sq_evtail = NULL;
6125 outer->sq_flags &= ~SQ_EVENTS;
6126 }
6127 ASSERT(outer->sq_count != 0);
6128 outer->sq_count--; /* Incremented when enqueued. */
6129 mutex_exit(SQLOCK(outer));
6130 /*
6131 * Drop the message if the queue is closing.
6132 * Make sure that the queue is "claimed" when the callback
6133 * is run in order to satisfy various ASSERTs.
6134 */
6135 q = mp->b_queue;
6136 func = (void (*)())mp->b_prev;
6137 ASSERT(func != NULL);
6138 mp->b_next = mp->b_prev = NULL;
6139 if (q->q_flag & QWCLOSE) {
6140 freemsg(mp);
6141 } else {
6142 claimq(q);
6143 (*func)(q, mp);
6144 releaseq(q);
6145 }
6146 mutex_enter(SQLOCK(outer));
6147 }
6148 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6149 }
6150
6151 /*
6152 * The list of messages on the inner syncq is effectively hashed
6153 * by destination queue. These destination queues are doubly
6154 * linked lists (hopefully) in priority order. Messages are then
6155 * put on the queue referenced by the q_sqhead/q_sqtail elements.
6156 * Additional messages are linked together by the b_next/b_prev
6157 * elements in the mblk, with (similar to putq()) the first message
6158 * having a NULL b_prev and the last message having a NULL b_next.
6159 *
6160 * Events, such as qwriter callbacks, are put onto a list in FIFO
6161 * order referenced by sq_evhead, and sq_evtail. This is a singly
6162 * linked list, and messages here MUST be processed in the order queued.
6163 */
6164
6165 /*
6166 * Run the events on the syncq event list (sq_evhead).
6167 * Assumes there is only one claim on the syncq, it is
6168 * already exclusive (SQ_EXCL set), and the SQLOCK held.
6169 * Messages here are processed in order, with the SQ_EXCL bit
6170 * held all the way through till the last message is processed.
6171 */
6172 void
6173 sq_run_events(syncq_t *sq)
6174 {
6175 mblk_t *bp;
6176 queue_t *qp;
6177 uint16_t flags = sq->sq_flags;
6178 void (*func)();
6179
6180 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6181 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6182 sq->sq_oprev == NULL) ||
6183 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6184 sq->sq_oprev != NULL));
6185
6186 ASSERT(flags & SQ_EXCL);
6187 ASSERT(sq->sq_count == 1);
6188
6189 /*
6190 * We need to process all of the events on this list. It
6191 * is possible that new events will be added while we are
6192 * away processing a callback, so on every loop, we start
6193 * back at the beginning of the list.
6194 */
6195 /*
6196 * We have to reaccess sq_evhead since there is a
6197 * possibility of a new entry while we were running
6198 * the callback.
6199 */
6200 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6201 ASSERT(bp->b_queue->q_syncq == sq);
6202 ASSERT(sq->sq_flags & SQ_EVENTS);
6203
6204 qp = bp->b_queue;
6205 func = (void (*)())bp->b_prev;
6206 ASSERT(func != NULL);
6207
6208 /*
6209 * Messages from the event queue must be taken off in
6210 * FIFO order.
6211 */
6212 ASSERT(sq->sq_evhead == bp);
6213 sq->sq_evhead = bp->b_next;
6214
6215 if (bp->b_next == NULL) {
6216 /* Deleting last */
6217 ASSERT(sq->sq_evtail == bp);
6218 sq->sq_evtail = NULL;
6219 sq->sq_flags &= ~SQ_EVENTS;
6220 }
6221 bp->b_prev = bp->b_next = NULL;
6222 ASSERT(bp->b_datap->db_ref != 0);
6223
6224 mutex_exit(SQLOCK(sq));
6225
6226 (*func)(qp, bp);
6227
6228 mutex_enter(SQLOCK(sq));
6229 /*
6230 * re-read the flags, since they could have changed.
6231 */
6232 flags = sq->sq_flags;
6233 ASSERT(flags & SQ_EXCL);
6234 }
6235 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6236 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6237
6238 if (flags & SQ_WANTWAKEUP) {
6239 flags &= ~SQ_WANTWAKEUP;
6240 cv_broadcast(&sq->sq_wait);
6241 }
6242 if (flags & SQ_WANTEXWAKEUP) {
6243 flags &= ~SQ_WANTEXWAKEUP;
6244 cv_broadcast(&sq->sq_exitwait);
6245 }
6246 sq->sq_flags = flags;
6247 }
6248
6249 /*
6250 * Put messages on the event list.
6251 * If we can go exclusive now, do so and process the event list, otherwise
6252 * let the last claim service this list (or wake the sqthread).
6253 * This procedure assumes SQLOCK is held. To run the event list, it
6254 * must be called with no claims.
6255 */
6256 static void
6257 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6258 {
6259 uint16_t count;
6260
6261 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6262 ASSERT(func != NULL);
6263
6264 /*
6265 * This is a callback. Add it to the list of callbacks
6266 * and see about upgrading.
6267 */
6268 mp->b_prev = (mblk_t *)func;
6269 mp->b_queue = q;
6270 mp->b_next = NULL;
6271 if (sq->sq_evhead == NULL) {
6272 sq->sq_evhead = sq->sq_evtail = mp;
6273 sq->sq_flags |= SQ_EVENTS;
6274 } else {
6275 ASSERT(sq->sq_evtail != NULL);
6276 ASSERT(sq->sq_evtail->b_next == NULL);
6277 ASSERT(sq->sq_flags & SQ_EVENTS);
6278 sq->sq_evtail->b_next = mp;
6279 sq->sq_evtail = mp;
6280 }
6281 /*
6282 * We have set SQ_EVENTS, so threads will have to
6283 * unwind out of the perimeter, and new entries will
6284 * not grab a putlock. But we still need to know
6285 * how many threads have already made a claim to the
6286 * syncq, so grab the putlocks, and sum the counts.
6287 * If there are no claims on the syncq, we can upgrade
6288 * to exclusive, and run the event list.
6289 * NOTE: We hold the SQLOCK, so we can just grab the
6290 * putlocks.
6291 */
6292 count = sq->sq_count;
6293 SQ_PUTLOCKS_ENTER(sq);
6294 SUM_SQ_PUTCOUNTS(sq, count);
6295 /*
6296 * We have no claim, so we need to check if there
6297 * are no others, then we can upgrade.
6298 */
6299 /*
6300 * There are currently no claims on
6301 * the syncq by this thread (at least on this entry). The thread who has
6302 * the claim should drain syncq.
6303 */
6304 if (count > 0) {
6305 /*
6306 * Can't upgrade - other threads inside.
6307 */
6308 SQ_PUTLOCKS_EXIT(sq);
6309 mutex_exit(SQLOCK(sq));
6310 return;
6311 }
6312 /*
6313 * Need to set SQ_EXCL and make a claim on the syncq.
6314 */
6315 ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6316 sq->sq_flags |= SQ_EXCL;
6317 ASSERT(sq->sq_count == 0);
6318 sq->sq_count++;
6319 SQ_PUTLOCKS_EXIT(sq);
6320
6321 /* Process the events list */
6322 sq_run_events(sq);
6323
6324 /*
6325 * Release our claim...
6326 */
6327 sq->sq_count--;
6328
6329 /*
6330 * And release SQ_EXCL.
6331 * We don't need to acquire the putlocks to release
6332 * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6333 */
6334 sq->sq_flags &= ~SQ_EXCL;
6335
6336 /*
6337 * sq_run_events should have released SQ_EXCL
6338 */
6339 ASSERT(!(sq->sq_flags & SQ_EXCL));
6340
6341 /*
6342 * If anything happened while we were running the
6343 * events (or was there before), we need to process
6344 * them now. We shouldn't be exclusive sine we
6345 * released the perimeter above (plus, we asserted
6346 * for it).
6347 */
6348 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6349 drain_syncq(sq);
6350 else
6351 mutex_exit(SQLOCK(sq));
6352 }
6353
6354 /*
6355 * Perform delayed processing. The caller has to make sure that it is safe
6356 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6357 * set).
6358 *
6359 * Assume that the caller has NO claims on the syncq. However, a claim
6360 * on the syncq does not indicate that a thread is draining the syncq.
6361 * There may be more claims on the syncq than there are threads draining
6362 * (i.e. #_threads_draining <= sq_count)
6363 *
6364 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6365 * in order to preserve qwriter(OUTER) ordering constraints.
6366 *
6367 * sq_putcount only needs to be checked when dispatching the queued
6368 * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6369 */
6370 void
6371 drain_syncq(syncq_t *sq)
6372 {
6373 queue_t *qp;
6374 uint16_t count;
6375 uint16_t type = sq->sq_type;
6376 uint16_t flags = sq->sq_flags;
6377 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE;
6378
6379 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6380 "drain_syncq start:%p", sq);
6381 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6382 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6383 sq->sq_oprev == NULL) ||
6384 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6385 sq->sq_oprev != NULL));
6386
6387 /*
6388 * Drop SQ_SERVICE flag.
6389 */
6390 if (bg_service)
6391 sq->sq_svcflags &= ~SQ_SERVICE;
6392
6393 /*
6394 * If SQ_EXCL is set, someone else is processing this syncq - let them
6395 * finish the job.
6396 */
6397 if (flags & SQ_EXCL) {
6398 if (bg_service) {
6399 ASSERT(sq->sq_servcount != 0);
6400 sq->sq_servcount--;
6401 }
6402 mutex_exit(SQLOCK(sq));
6403 return;
6404 }
6405
6406 /*
6407 * This routine can be called by a background thread if
6408 * it was scheduled by a hi-priority thread. SO, if there are
6409 * NOT messages queued, return (remember, we have the SQLOCK,
6410 * and it cannot change until we release it). Wakeup any waiters also.
6411 */
6412 if (!(flags & SQ_QUEUED)) {
6413 if (flags & SQ_WANTWAKEUP) {
6414 flags &= ~SQ_WANTWAKEUP;
6415 cv_broadcast(&sq->sq_wait);
6416 }
6417 if (flags & SQ_WANTEXWAKEUP) {
6418 flags &= ~SQ_WANTEXWAKEUP;
6419 cv_broadcast(&sq->sq_exitwait);
6420 }
6421 sq->sq_flags = flags;
6422 if (bg_service) {
6423 ASSERT(sq->sq_servcount != 0);
6424 sq->sq_servcount--;
6425 }
6426 mutex_exit(SQLOCK(sq));
6427 return;
6428 }
6429
6430 /*
6431 * If this is not a concurrent put perimeter, we need to
6432 * become exclusive to drain. Also, if not CIPUT, we would
6433 * not have acquired a putlock, so we don't need to check
6434 * the putcounts. If not entering with a claim, we test
6435 * for sq_count == 0.
6436 */
6437 type = sq->sq_type;
6438 if (!(type & SQ_CIPUT)) {
6439 if (sq->sq_count > 1) {
6440 if (bg_service) {
6441 ASSERT(sq->sq_servcount != 0);
6442 sq->sq_servcount--;
6443 }
6444 mutex_exit(SQLOCK(sq));
6445 return;
6446 }
6447 sq->sq_flags |= SQ_EXCL;
6448 }
6449
6450 /*
6451 * This is where we make a claim to the syncq.
6452 * This can either be done by incrementing a putlock, or
6453 * the sq_count. But since we already have the SQLOCK
6454 * here, we just bump the sq_count.
6455 *
6456 * Note that after we make a claim, we need to let the code
6457 * fall through to the end of this routine to clean itself
6458 * up. A return in the while loop will put the syncq in a
6459 * very bad state.
6460 */
6461 sq->sq_count++;
6462 ASSERT(sq->sq_count != 0); /* wraparound */
6463
6464 while ((flags = sq->sq_flags) & SQ_QUEUED) {
6465 /*
6466 * If we are told to stayaway or went exclusive,
6467 * we are done.
6468 */
6469 if (flags & (SQ_STAYAWAY)) {
6470 break;
6471 }
6472
6473 /*
6474 * If there are events to run, do so.
6475 * We have one claim to the syncq, so if there are
6476 * more than one, other threads are running.
6477 */
6478 if (sq->sq_evhead != NULL) {
6479 ASSERT(sq->sq_flags & SQ_EVENTS);
6480
6481 count = sq->sq_count;
6482 SQ_PUTLOCKS_ENTER(sq);
6483 SUM_SQ_PUTCOUNTS(sq, count);
6484 if (count > 1) {
6485 SQ_PUTLOCKS_EXIT(sq);
6486 /* Can't upgrade - other threads inside */
6487 break;
6488 }
6489 ASSERT((flags & SQ_EXCL) == 0);
6490 sq->sq_flags = flags | SQ_EXCL;
6491 SQ_PUTLOCKS_EXIT(sq);
6492 /*
6493 * we have the only claim, run the events,
6494 * sq_run_events will clear the SQ_EXCL flag.
6495 */
6496 sq_run_events(sq);
6497
6498 /*
6499 * If this is a CIPUT perimeter, we need
6500 * to drop the SQ_EXCL flag so we can properly
6501 * continue draining the syncq.
6502 */
6503 if (type & SQ_CIPUT) {
6504 ASSERT(sq->sq_flags & SQ_EXCL);
6505 sq->sq_flags &= ~SQ_EXCL;
6506 }
6507
6508 /*
6509 * And go back to the beginning just in case
6510 * anything changed while we were away.
6511 */
6512 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6513 continue;
6514 }
6515
6516 ASSERT(sq->sq_evhead == NULL);
6517 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6518
6519 /*
6520 * Find the queue that is not draining.
6521 *
6522 * q_draining is protected by QLOCK which we do not hold.
6523 * But if it was set, then a thread was draining, and if it gets
6524 * cleared, then it was because the thread has successfully
6525 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6526 * state to happen, a thread needs the SQLOCK which we hold, and
6527 * if there was such a flag, we would have already seen it.
6528 */
6529
6530 for (qp = sq->sq_head;
6531 qp != NULL && (qp->q_draining ||
6532 (qp->q_sqflags & Q_SQDRAINING));
6533 qp = qp->q_sqnext)
6534 ;
6535
6536 if (qp == NULL)
6537 break;
6538
6539 /*
6540 * We have a queue to work on, and we hold the
6541 * SQLOCK and one claim, call qdrain_syncq.
6542 * This means we need to release the SQLOCK and
6543 * acquire the QLOCK (OK since we have a claim).
6544 * Note that qdrain_syncq will actually dequeue
6545 * this queue from the sq_head list when it is
6546 * convinced all the work is done and release
6547 * the QLOCK before returning.
6548 */
6549 qp->q_sqflags |= Q_SQDRAINING;
6550 mutex_exit(SQLOCK(sq));
6551 mutex_enter(QLOCK(qp));
6552 qdrain_syncq(sq, qp);
6553 mutex_enter(SQLOCK(sq));
6554
6555 /* The queue is drained */
6556 ASSERT(qp->q_sqflags & Q_SQDRAINING);
6557 qp->q_sqflags &= ~Q_SQDRAINING;
6558 /*
6559 * NOTE: After this point qp should not be used since it may be
6560 * closed.
6561 */
6562 }
6563
6564 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6565 flags = sq->sq_flags;
6566
6567 /*
6568 * sq->sq_head cannot change because we hold the
6569 * sqlock. However, a thread CAN decide that it is no longer
6570 * going to drain that queue. However, this should be due to
6571 * a GOAWAY state, and we should see that here.
6572 *
6573 * This loop is not very efficient. One solution may be adding a second
6574 * pointer to the "draining" queue, but it is difficult to do when
6575 * queues are inserted in the middle due to priority ordering. Another
6576 * possibility is to yank the queue out of the sq list and put it onto
6577 * the "draining list" and then put it back if it can't be drained.
6578 */
6579
6580 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6581 (type & SQ_CI) || sq->sq_head->q_draining);
6582
6583 /* Drop SQ_EXCL for non-CIPUT perimeters */
6584 if (!(type & SQ_CIPUT))
6585 flags &= ~SQ_EXCL;
6586 ASSERT((flags & SQ_EXCL) == 0);
6587
6588 /* Wake up any waiters. */
6589 if (flags & SQ_WANTWAKEUP) {
6590 flags &= ~SQ_WANTWAKEUP;
6591 cv_broadcast(&sq->sq_wait);
6592 }
6593 if (flags & SQ_WANTEXWAKEUP) {
6594 flags &= ~SQ_WANTEXWAKEUP;
6595 cv_broadcast(&sq->sq_exitwait);
6596 }
6597 sq->sq_flags = flags;
6598
6599 ASSERT(sq->sq_count != 0);
6600 /* Release our claim. */
6601 sq->sq_count--;
6602
6603 if (bg_service) {
6604 ASSERT(sq->sq_servcount != 0);
6605 sq->sq_servcount--;
6606 }
6607
6608 mutex_exit(SQLOCK(sq));
6609
6610 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6611 "drain_syncq end:%p", sq);
6612 }
6613
6614
6615 /*
6616 *
6617 * qdrain_syncq can be called (currently) from only one of two places:
6618 * drain_syncq
6619 * putnext (or some variation of it).
6620 * and eventually
6621 * qwait(_sig)
6622 *
6623 * If called from drain_syncq, we found it in the list of queues needing
6624 * service, so there is work to be done (or it wouldn't be in the list).
6625 *
6626 * If called from some putnext variation, it was because the
6627 * perimeter is open, but messages are blocking a putnext and
6628 * there is not a thread working on it. Now a thread could start
6629 * working on it while we are getting ready to do so ourself, but
6630 * the thread would set the q_draining flag, and we can spin out.
6631 *
6632 * As for qwait(_sig), I think I shall let it continue to call
6633 * drain_syncq directly (after all, it will get here eventually).
6634 *
6635 * qdrain_syncq has to terminate when:
6636 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6637 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6638 *
6639 * ASSUMES:
6640 * One claim
6641 * QLOCK held
6642 * SQLOCK not held
6643 * Will release QLOCK before returning
6644 */
6645 void
6646 qdrain_syncq(syncq_t *sq, queue_t *q)
6647 {
6648 mblk_t *bp;
6649 #ifdef DEBUG
6650 uint16_t count;
6651 #endif
6652
6653 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6654 "drain_syncq start:%p", sq);
6655 ASSERT(q->q_syncq == sq);
6656 ASSERT(MUTEX_HELD(QLOCK(q)));
6657 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6658 /*
6659 * For non-CIPUT perimeters, we should be called with the exclusive bit
6660 * set already. For CIPUT perimeters, we will be doing a concurrent
6661 * drain, so it better not be set.
6662 */
6663 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6664 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6665 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6666 /*
6667 * All outer pointers are set, or none of them are
6668 */
6669 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6670 sq->sq_oprev == NULL) ||
6671 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6672 sq->sq_oprev != NULL));
6673 #ifdef DEBUG
6674 count = sq->sq_count;
6675 /*
6676 * This is OK without the putlocks, because we have one
6677 * claim either from the sq_count, or a putcount. We could
6678 * get an erroneous value from other counts, but ours won't
6679 * change, so one way or another, we will have at least a
6680 * value of one.
6681 */
6682 SUM_SQ_PUTCOUNTS(sq, count);
6683 ASSERT(count >= 1);
6684 #endif /* DEBUG */
6685
6686 /*
6687 * The first thing to do is find out if a thread is already draining
6688 * this queue. If so, we are done, just return.
6689 */
6690 if (q->q_draining) {
6691 mutex_exit(QLOCK(q));
6692 return;
6693 }
6694
6695 /*
6696 * If the perimeter is exclusive, there is nothing we can do right now,
6697 * go away. Note that there is nothing to prevent this case from
6698 * changing right after this check, but the spin-out will catch it.
6699 */
6700
6701 /* Tell other threads that we are draining this queue */
6702 q->q_draining = 1; /* Protected by QLOCK */
6703
6704 /*
6705 * If there is nothing to do, clear QFULL as necessary. This caters for
6706 * the case where an empty queue was enqueued onto the syncq.
6707 */
6708 if (q->q_sqhead == NULL) {
6709 ASSERT(q->q_syncqmsgs == 0);
6710 mutex_exit(QLOCK(q));
6711 clr_qfull(q);
6712 mutex_enter(QLOCK(q));
6713 }
6714
6715 /*
6716 * Note that q_sqhead must be re-checked here in case another message
6717 * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6718 */
6719 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6720 /*
6721 * Because we can enter this routine just because a putnext is
6722 * blocked, we need to spin out if the perimeter wants to go
6723 * exclusive as well as just blocked. We need to spin out also
6724 * if events are queued on the syncq.
6725 * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6726 * set it, and it can't become exclusive while we hold a claim.
6727 */
6728 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6729 break;
6730 }
6731
6732 #ifdef DEBUG
6733 /*
6734 * Since we are in qdrain_syncq, we already know the queue,
6735 * but for sanity, we want to check this against the qp that
6736 * was passed in by bp->b_queue.
6737 */
6738
6739 ASSERT(bp->b_queue == q);
6740 ASSERT(bp->b_queue->q_syncq == sq);
6741 bp->b_queue = NULL;
6742
6743 /*
6744 * We would have the following check in the DEBUG code:
6745 *
6746 * if (bp->b_prev != NULL) {
6747 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6748 * }
6749 *
6750 * This can't be done, however, since IP modifies qinfo
6751 * structure at run-time (switching between IPv4 qinfo and IPv6
6752 * qinfo), invalidating the check.
6753 * So the assignment to func is left here, but the ASSERT itself
6754 * is removed until the whole issue is resolved.
6755 */
6756 #endif
6757 ASSERT(q->q_sqhead == bp);
6758 q->q_sqhead = bp->b_next;
6759 bp->b_prev = bp->b_next = NULL;
6760 ASSERT(q->q_syncqmsgs > 0);
6761 mutex_exit(QLOCK(q));
6762
6763 ASSERT(bp->b_datap->db_ref != 0);
6764
6765 (void) (*q->q_qinfo->qi_putp)(q, bp);
6766
6767 mutex_enter(QLOCK(q));
6768
6769 /*
6770 * q_syncqmsgs should only be decremented after executing the
6771 * put procedure to avoid message re-ordering. This is due to an
6772 * optimisation in putnext() which can call the put procedure
6773 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6774 * being set).
6775 *
6776 * We also need to clear QFULL in the next service procedure
6777 * queue if this is the last message destined for that queue.
6778 *
6779 * It would make better sense to have some sort of tunable for
6780 * the low water mark, but these semantics are not yet defined.
6781 * So, alas, we use a constant.
6782 */
6783 if (--q->q_syncqmsgs == 0) {
6784 mutex_exit(QLOCK(q));
6785 clr_qfull(q);
6786 mutex_enter(QLOCK(q));
6787 }
6788
6789 /*
6790 * Always clear SQ_EXCL when CIPUT in order to handle
6791 * qwriter(INNER). The putp() can call qwriter and get exclusive
6792 * access IFF this is the only claim. So, we need to test for
6793 * this possibility, acquire the mutex and clear the bit.
6794 */
6795 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6796 mutex_enter(SQLOCK(sq));
6797 sq->sq_flags &= ~SQ_EXCL;
6798 mutex_exit(SQLOCK(sq));
6799 }
6800 }
6801
6802 /*
6803 * We should either have no messages on this queue, or we were told to
6804 * goaway by a waiter (which we will wake up at the end of this
6805 * function).
6806 */
6807 ASSERT((q->q_sqhead == NULL) ||
6808 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6809
6810 ASSERT(MUTEX_HELD(QLOCK(q)));
6811 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6812
6813 /* Remove the q from the syncq list if all the messages are drained. */
6814 if (q->q_sqhead == NULL) {
6815 ASSERT(q->q_syncqmsgs == 0);
6816 mutex_enter(SQLOCK(sq));
6817 if (q->q_sqflags & Q_SQQUEUED)
6818 SQRM_Q(sq, q);
6819 mutex_exit(SQLOCK(sq));
6820 /*
6821 * Since the queue is removed from the list, reset its priority.
6822 */
6823 q->q_spri = 0;
6824 }
6825
6826 /*
6827 * Remember, the q_draining flag is used to let another thread know
6828 * that there is a thread currently draining the messages for a queue.
6829 * Since we are now done with this queue (even if there may be messages
6830 * still there), we need to clear this flag so some thread will work on
6831 * it if needed.
6832 */
6833 ASSERT(q->q_draining);
6834 q->q_draining = 0;
6835
6836 /* Called with a claim, so OK to drop all locks. */
6837 mutex_exit(QLOCK(q));
6838
6839 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6840 "drain_syncq end:%p", sq);
6841 }
6842 /* END OF QDRAIN_SYNCQ */
6843
6844
6845 /*
6846 * This is the mate to qdrain_syncq, except that it is putting the message onto
6847 * the queue instead of draining. Since the message is destined for the queue
6848 * that is selected, there is no need to identify the function because the
6849 * message is intended for the put routine for the queue. For debug kernels,
6850 * this routine will do it anyway just in case.
6851 *
6852 * After the message is enqueued on the syncq, it calls putnext_tail()
6853 * which will schedule a background thread to actually process the message.
6854 *
6855 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6856 * SQLOCK(sq) and QLOCK(q) are not held.
6857 */
6858 void
6859 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6860 {
6861 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6862 ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6863 ASSERT(sq->sq_count > 0);
6864 ASSERT(q->q_syncq == sq);
6865 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6866 sq->sq_oprev == NULL) ||
6867 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6868 sq->sq_oprev != NULL));
6869
6870 mutex_enter(QLOCK(q));
6871
6872 #ifdef DEBUG
6873 /*
6874 * This is used for debug in the qfill_syncq/qdrain_syncq case
6875 * to trace the queue that the message is intended for. Note
6876 * that the original use was to identify the queue and function
6877 * to call on the drain. In the new syncq, we have the context
6878 * of the queue that we are draining, so call it's putproc and
6879 * don't rely on the saved values. But for debug this is still
6880 * useful information.
6881 */
6882 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6883 mp->b_queue = q;
6884 mp->b_next = NULL;
6885 #endif
6886 ASSERT(q->q_syncq == sq);
6887 /*
6888 * Enqueue the message on the list.
6889 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to
6890 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP().
6891 */
6892 SQPUT_MP(q, mp);
6893 mutex_enter(SQLOCK(sq));
6894
6895 /*
6896 * And queue on syncq for scheduling, if not already queued.
6897 * Note that we need the SQLOCK for this, and for testing flags
6898 * at the end to see if we will drain. So grab it now, and
6899 * release it before we call qdrain_syncq or return.
6900 */
6901 if (!(q->q_sqflags & Q_SQQUEUED)) {
6902 q->q_spri = curthread->t_pri;
6903 SQPUT_Q(sq, q);
6904 }
6905 #ifdef DEBUG
6906 else {
6907 /*
6908 * All of these conditions MUST be true!
6909 */
6910 ASSERT(sq->sq_tail != NULL);
6911 if (sq->sq_tail == sq->sq_head) {
6912 ASSERT((q->q_sqprev == NULL) &&
6913 (q->q_sqnext == NULL));
6914 } else {
6915 ASSERT((q->q_sqprev != NULL) ||
6916 (q->q_sqnext != NULL));
6917 }
6918 ASSERT(sq->sq_flags & SQ_QUEUED);
6919 ASSERT(q->q_syncqmsgs != 0);
6920 ASSERT(q->q_sqflags & Q_SQQUEUED);
6921 }
6922 #endif
6923 mutex_exit(QLOCK(q));
6924 /*
6925 * SQLOCK is still held, so sq_count can be safely decremented.
6926 */
6927 sq->sq_count--;
6928
6929 putnext_tail(sq, q, 0);
6930 /* Should not reference sq or q after this point. */
6931 }
6932
6933 /* End of qfill_syncq */
6934
6935 /*
6936 * Remove all messages from a syncq (if qp is NULL) or remove all messages
6937 * that would be put into qp by drain_syncq.
6938 * Used when deleting the syncq (qp == NULL) or when detaching
6939 * a queue (qp != NULL).
6940 * Return non-zero if one or more messages were freed.
6941 *
6942 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6943 * sq_putlocks are used.
6944 *
6945 * NOTE: This function assumes that it is called from the close() context and
6946 * that all the queues in the syncq are going away. For this reason it doesn't
6947 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6948 * currently valid, but it is useful to rethink this function to behave properly
6949 * in other cases.
6950 */
6951 int
6952 flush_syncq(syncq_t *sq, queue_t *qp)
6953 {
6954 mblk_t *bp, *mp_head, *mp_next, *mp_prev;
6955 queue_t *q;
6956 int ret = 0;
6957
6958 mutex_enter(SQLOCK(sq));
6959
6960 /*
6961 * Before we leave, we need to make sure there are no
6962 * events listed for this queue. All events for this queue
6963 * will just be freed.
6964 */
6965 if (qp != NULL && sq->sq_evhead != NULL) {
6966 ASSERT(sq->sq_flags & SQ_EVENTS);
6967
6968 mp_prev = NULL;
6969 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6970 mp_next = bp->b_next;
6971 if (bp->b_queue == qp) {
6972 /* Delete this message */
6973 if (mp_prev != NULL) {
6974 mp_prev->b_next = mp_next;
6975 /*
6976 * Update sq_evtail if the last element
6977 * is removed.
6978 */
6979 if (bp == sq->sq_evtail) {
6980 ASSERT(mp_next == NULL);
6981 sq->sq_evtail = mp_prev;
6982 }
6983 } else
6984 sq->sq_evhead = mp_next;
6985 if (sq->sq_evhead == NULL)
6986 sq->sq_flags &= ~SQ_EVENTS;
6987 bp->b_prev = bp->b_next = NULL;
6988 freemsg(bp);
6989 ret++;
6990 } else {
6991 mp_prev = bp;
6992 }
6993 }
6994 }
6995
6996 /*
6997 * Walk sq_head and:
6998 * - match qp if qp is set, remove it's messages
6999 * - all if qp is not set
7000 */
7001 q = sq->sq_head;
7002 while (q != NULL) {
7003 ASSERT(q->q_syncq == sq);
7004 if ((qp == NULL) || (qp == q)) {
7005 /*
7006 * Yank the messages as a list off the queue
7007 */
7008 mp_head = q->q_sqhead;
7009 /*
7010 * We do not have QLOCK(q) here (which is safe due to
7011 * assumptions mentioned above). To obtain the lock we
7012 * need to release SQLOCK which may allow lots of things
7013 * to change upon us. This place requires more analysis.
7014 */
7015 q->q_sqhead = q->q_sqtail = NULL;
7016 ASSERT(mp_head->b_queue &&
7017 mp_head->b_queue->q_syncq == sq);
7018
7019 /*
7020 * Free each of the messages.
7021 */
7022 for (bp = mp_head; bp != NULL; bp = mp_next) {
7023 mp_next = bp->b_next;
7024 bp->b_prev = bp->b_next = NULL;
7025 freemsg(bp);
7026 ret++;
7027 }
7028 /*
7029 * Now remove the queue from the syncq.
7030 */
7031 ASSERT(q->q_sqflags & Q_SQQUEUED);
7032 SQRM_Q(sq, q);
7033 q->q_spri = 0;
7034 q->q_syncqmsgs = 0;
7035
7036 /*
7037 * If qp was specified, we are done with it and are
7038 * going to drop SQLOCK(sq) and return. We wakeup syncq
7039 * waiters while we still have the SQLOCK.
7040 */
7041 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7042 sq->sq_flags &= ~SQ_WANTWAKEUP;
7043 cv_broadcast(&sq->sq_wait);
7044 }
7045 /* Drop SQLOCK across clr_qfull */
7046 mutex_exit(SQLOCK(sq));
7047
7048 /*
7049 * We avoid doing the test that drain_syncq does and
7050 * unconditionally clear qfull for every flushed
7051 * message. Since flush_syncq is only called during
7052 * close this should not be a problem.
7053 */
7054 clr_qfull(q);
7055 if (qp != NULL) {
7056 return (ret);
7057 } else {
7058 mutex_enter(SQLOCK(sq));
7059 /*
7060 * The head was removed by SQRM_Q above.
7061 * reread the new head and flush it.
7062 */
7063 q = sq->sq_head;
7064 }
7065 } else {
7066 q = q->q_sqnext;
7067 }
7068 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7069 }
7070
7071 if (sq->sq_flags & SQ_WANTWAKEUP) {
7072 sq->sq_flags &= ~SQ_WANTWAKEUP;
7073 cv_broadcast(&sq->sq_wait);
7074 }
7075
7076 mutex_exit(SQLOCK(sq));
7077 return (ret);
7078 }
7079
7080 /*
7081 * Propagate all messages from a syncq to the next syncq that are associated
7082 * with the specified queue. If the queue is attached to a driver or if the
7083 * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7084 *
7085 * Assumes that the stream is strlock()'ed. We don't come here if there
7086 * are no messages to propagate.
7087 *
7088 * NOTE : If the queue is attached to a driver, all the messages are freed
7089 * as there is no point in propagating the messages from the driver syncq
7090 * to the closing stream head which will in turn get freed later.
7091 */
7092 static int
7093 propagate_syncq(queue_t *qp)
7094 {
7095 mblk_t *bp, *head, *tail, *prev, *next;
7096 syncq_t *sq;
7097 queue_t *nqp;
7098 syncq_t *nsq;
7099 boolean_t isdriver;
7100 int moved = 0;
7101 uint16_t flags;
7102 pri_t priority = curthread->t_pri;
7103 #ifdef DEBUG
7104 void (*func)();
7105 #endif
7106
7107 sq = qp->q_syncq;
7108 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7109 /* debug macro */
7110 SQ_PUTLOCKS_HELD(sq);
7111 /*
7112 * As entersq() does not increment the sq_count for
7113 * the write side, check sq_count for non-QPERQ
7114 * perimeters alone.
7115 */
7116 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7117
7118 /*
7119 * propagate_syncq() can be called because of either messages on the
7120 * queue syncq or because on events on the queue syncq. Do actual
7121 * message propagations if there are any messages.
7122 */
7123 if (qp->q_syncqmsgs) {
7124 isdriver = (qp->q_flag & QISDRV);
7125
7126 if (!isdriver) {
7127 nqp = qp->q_next;
7128 nsq = nqp->q_syncq;
7129 ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7130 /* debug macro */
7131 SQ_PUTLOCKS_HELD(nsq);
7132 #ifdef DEBUG
7133 func = (void (*)())nqp->q_qinfo->qi_putp;
7134 #endif
7135 }
7136
7137 SQRM_Q(sq, qp);
7138 priority = MAX(qp->q_spri, priority);
7139 qp->q_spri = 0;
7140 head = qp->q_sqhead;
7141 tail = qp->q_sqtail;
7142 qp->q_sqhead = qp->q_sqtail = NULL;
7143 qp->q_syncqmsgs = 0;
7144
7145 /*
7146 * Walk the list of messages, and free them if this is a driver,
7147 * otherwise reset the b_prev and b_queue value to the new putp.
7148 * Afterward, we will just add the head to the end of the next
7149 * syncq, and point the tail to the end of this one.
7150 */
7151
7152 for (bp = head; bp != NULL; bp = next) {
7153 next = bp->b_next;
7154 if (isdriver) {
7155 bp->b_prev = bp->b_next = NULL;
7156 freemsg(bp);
7157 continue;
7158 }
7159 /* Change the q values for this message */
7160 bp->b_queue = nqp;
7161 #ifdef DEBUG
7162 bp->b_prev = (mblk_t *)func;
7163 #endif
7164 moved++;
7165 }
7166 /*
7167 * Attach list of messages to the end of the new queue (if there
7168 * is a list of messages).
7169 */
7170
7171 if (!isdriver && head != NULL) {
7172 ASSERT(tail != NULL);
7173 if (nqp->q_sqhead == NULL) {
7174 nqp->q_sqhead = head;
7175 } else {
7176 ASSERT(nqp->q_sqtail != NULL);
7177 nqp->q_sqtail->b_next = head;
7178 }
7179 nqp->q_sqtail = tail;
7180 /*
7181 * When messages are moved from high priority queue to
7182 * another queue, the destination queue priority is
7183 * upgraded.
7184 */
7185
7186 if (priority > nqp->q_spri)
7187 nqp->q_spri = priority;
7188
7189 SQPUT_Q(nsq, nqp);
7190
7191 nqp->q_syncqmsgs += moved;
7192 ASSERT(nqp->q_syncqmsgs != 0);
7193 }
7194 }
7195
7196 /*
7197 * Before we leave, we need to make sure there are no
7198 * events listed for this queue. All events for this queue
7199 * will just be freed.
7200 */
7201 if (sq->sq_evhead != NULL) {
7202 ASSERT(sq->sq_flags & SQ_EVENTS);
7203 prev = NULL;
7204 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7205 next = bp->b_next;
7206 if (bp->b_queue == qp) {
7207 /* Delete this message */
7208 if (prev != NULL) {
7209 prev->b_next = next;
7210 /*
7211 * Update sq_evtail if the last element
7212 * is removed.
7213 */
7214 if (bp == sq->sq_evtail) {
7215 ASSERT(next == NULL);
7216 sq->sq_evtail = prev;
7217 }
7218 } else
7219 sq->sq_evhead = next;
7220 if (sq->sq_evhead == NULL)
7221 sq->sq_flags &= ~SQ_EVENTS;
7222 bp->b_prev = bp->b_next = NULL;
7223 freemsg(bp);
7224 } else {
7225 prev = bp;
7226 }
7227 }
7228 }
7229
7230 flags = sq->sq_flags;
7231
7232 /* Wake up any waiter before leaving. */
7233 if (flags & SQ_WANTWAKEUP) {
7234 flags &= ~SQ_WANTWAKEUP;
7235 cv_broadcast(&sq->sq_wait);
7236 }
7237 sq->sq_flags = flags;
7238
7239 return (moved);
7240 }
7241
7242 /*
7243 * Try and upgrade to exclusive access at the inner perimeter. If this can
7244 * not be done without blocking then request will be queued on the syncq
7245 * and drain_syncq will run it later.
7246 *
7247 * This routine can only be called from put or service procedures plus
7248 * asynchronous callback routines that have properly entered the queue (with
7249 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7250 * associated with q.
7251 */
7252 void
7253 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7254 {
7255 syncq_t *sq = q->q_syncq;
7256 uint16_t count;
7257
7258 mutex_enter(SQLOCK(sq));
7259 count = sq->sq_count;
7260 SQ_PUTLOCKS_ENTER(sq);
7261 SUM_SQ_PUTCOUNTS(sq, count);
7262 ASSERT(count >= 1);
7263 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7264
7265 if (count == 1) {
7266 /*
7267 * Can upgrade. This case also handles nested qwriter calls
7268 * (when the qwriter callback function calls qwriter). In that
7269 * case SQ_EXCL is already set.
7270 */
7271 sq->sq_flags |= SQ_EXCL;
7272 SQ_PUTLOCKS_EXIT(sq);
7273 mutex_exit(SQLOCK(sq));
7274 (*func)(q, mp);
7275 /*
7276 * Assumes that leavesq, putnext, and drain_syncq will reset
7277 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7278 * until putnext, leavesq, or drain_syncq drops it.
7279 * That way we handle nested qwriter(INNER) without dropping
7280 * SQ_EXCL until the outermost qwriter callback routine is
7281 * done.
7282 */
7283 return;
7284 }
7285 SQ_PUTLOCKS_EXIT(sq);
7286 sqfill_events(sq, q, mp, func);
7287 }
7288
7289 /*
7290 * Synchronous callback support functions
7291 */
7292
7293 /*
7294 * Allocate a callback parameter structure.
7295 * Assumes that caller initializes the flags and the id.
7296 * Acquires SQLOCK(sq) if non-NULL is returned.
7297 */
7298 callbparams_t *
7299 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7300 {
7301 callbparams_t *cbp;
7302 size_t size = sizeof (callbparams_t);
7303
7304 cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7305
7306 /*
7307 * Only try tryhard allocation if the caller is ready to panic.
7308 * Otherwise just fail.
7309 */
7310 if (cbp == NULL) {
7311 if (kmflags & KM_PANIC)
7312 cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7313 &size, kmflags);
7314 else
7315 return (NULL);
7316 }
7317
7318 ASSERT(size >= sizeof (callbparams_t));
7319 cbp->cbp_size = size;
7320 cbp->cbp_sq = sq;
7321 cbp->cbp_func = func;
7322 cbp->cbp_arg = arg;
7323 mutex_enter(SQLOCK(sq));
7324 cbp->cbp_next = sq->sq_callbpend;
7325 sq->sq_callbpend = cbp;
7326 return (cbp);
7327 }
7328
7329 void
7330 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7331 {
7332 callbparams_t **pp, *p;
7333
7334 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7335
7336 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7337 if (p == cbp) {
7338 *pp = p->cbp_next;
7339 kmem_free(p, p->cbp_size);
7340 return;
7341 }
7342 }
7343 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7344 "callbparams_free: not found\n"));
7345 }
7346
7347 void
7348 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7349 {
7350 callbparams_t **pp, *p;
7351
7352 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7353
7354 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7355 if (p->cbp_id == id && p->cbp_flags == flag) {
7356 *pp = p->cbp_next;
7357 kmem_free(p, p->cbp_size);
7358 return;
7359 }
7360 }
7361 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7362 "callbparams_free_id: not found\n"));
7363 }
7364
7365 /*
7366 * Callback wrapper function used by once-only callbacks that can be
7367 * cancelled (qtimeout and qbufcall)
7368 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7369 * cancelled by the qun* functions.
7370 */
7371 void
7372 qcallbwrapper(void *arg)
7373 {
7374 callbparams_t *cbp = arg;
7375 syncq_t *sq;
7376 uint16_t count = 0;
7377 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7378 uint16_t type;
7379
7380 sq = cbp->cbp_sq;
7381 mutex_enter(SQLOCK(sq));
7382 type = sq->sq_type;
7383 if (!(type & SQ_CICB)) {
7384 count = sq->sq_count;
7385 SQ_PUTLOCKS_ENTER(sq);
7386 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7387 SUM_SQ_PUTCOUNTS(sq, count);
7388 sq->sq_needexcl++;
7389 ASSERT(sq->sq_needexcl != 0); /* wraparound */
7390 waitflags |= SQ_MESSAGES;
7391 }
7392 /* Can not handle exclusive entry at outer perimeter */
7393 ASSERT(type & SQ_COCB);
7394
7395 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7396 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7397 (sq->sq_cancelid == cbp->cbp_id)) {
7398 /* timeout has been cancelled */
7399 sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7400 callbparams_free(sq, cbp);
7401 if (!(type & SQ_CICB)) {
7402 ASSERT(sq->sq_needexcl > 0);
7403 sq->sq_needexcl--;
7404 if (sq->sq_needexcl == 0) {
7405 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7406 }
7407 SQ_PUTLOCKS_EXIT(sq);
7408 }
7409 mutex_exit(SQLOCK(sq));
7410 return;
7411 }
7412 sq->sq_flags |= SQ_WANTWAKEUP;
7413 if (!(type & SQ_CICB)) {
7414 SQ_PUTLOCKS_EXIT(sq);
7415 }
7416 cv_wait(&sq->sq_wait, SQLOCK(sq));
7417 if (!(type & SQ_CICB)) {
7418 count = sq->sq_count;
7419 SQ_PUTLOCKS_ENTER(sq);
7420 SUM_SQ_PUTCOUNTS(sq, count);
7421 }
7422 }
7423
7424 sq->sq_count++;
7425 ASSERT(sq->sq_count != 0); /* Wraparound */
7426 if (!(type & SQ_CICB)) {
7427 ASSERT(count == 0);
7428 sq->sq_flags |= SQ_EXCL;
7429 ASSERT(sq->sq_needexcl > 0);
7430 sq->sq_needexcl--;
7431 if (sq->sq_needexcl == 0) {
7432 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7433 }
7434 SQ_PUTLOCKS_EXIT(sq);
7435 }
7436
7437 mutex_exit(SQLOCK(sq));
7438
7439 cbp->cbp_func(cbp->cbp_arg);
7440
7441 /*
7442 * We drop the lock only for leavesq to re-acquire it.
7443 * Possible optimization is inline of leavesq.
7444 */
7445 mutex_enter(SQLOCK(sq));
7446 callbparams_free(sq, cbp);
7447 mutex_exit(SQLOCK(sq));
7448 leavesq(sq, SQ_CALLBACK);
7449 }
7450
7451 /*
7452 * No need to grab sq_putlocks here. See comment in strsubr.h that
7453 * explains when sq_putlocks are used.
7454 *
7455 * sq_count (or one of the sq_putcounts) has already been
7456 * decremented by the caller, and if SQ_QUEUED, we need to call
7457 * drain_syncq (the global syncq drain).
7458 * If putnext_tail is called with the SQ_EXCL bit set, we are in
7459 * one of two states, non-CIPUT perimeter, and we need to clear
7460 * it, or we went exclusive in the put procedure. In any case,
7461 * we want to clear the bit now, and it is probably easier to do
7462 * this at the beginning of this function (remember, we hold
7463 * the SQLOCK). Lastly, if there are other messages queued
7464 * on the syncq (and not for our destination), enable the syncq
7465 * for background work.
7466 */
7467
7468 /* ARGSUSED */
7469 void
7470 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7471 {
7472 uint16_t flags = sq->sq_flags;
7473
7474 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7475 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7476
7477 /* Clear SQ_EXCL if set in passflags */
7478 if (passflags & SQ_EXCL) {
7479 flags &= ~SQ_EXCL;
7480 }
7481 if (flags & SQ_WANTWAKEUP) {
7482 flags &= ~SQ_WANTWAKEUP;
7483 cv_broadcast(&sq->sq_wait);
7484 }
7485 if (flags & SQ_WANTEXWAKEUP) {
7486 flags &= ~SQ_WANTEXWAKEUP;
7487 cv_broadcast(&sq->sq_exitwait);
7488 }
7489 sq->sq_flags = flags;
7490
7491 /*
7492 * We have cleared SQ_EXCL if we were asked to, and started
7493 * the wakeup process for waiters. If there are no writers
7494 * then we need to drain the syncq if we were told to, or
7495 * enable the background thread to do it.
7496 */
7497 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7498 if ((passflags & SQ_QUEUED) ||
7499 (sq->sq_svcflags & SQ_DISABLED)) {
7500 /* drain_syncq will take care of events in the list */
7501 drain_syncq(sq);
7502 return;
7503 } else if (flags & SQ_QUEUED) {
7504 sqenable(sq);
7505 }
7506 }
7507 /* Drop the SQLOCK on exit */
7508 mutex_exit(SQLOCK(sq));
7509 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7510 "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7511 }
7512
7513 void
7514 set_qend(queue_t *q)
7515 {
7516 mutex_enter(QLOCK(q));
7517 if (!O_SAMESTR(q))
7518 q->q_flag |= QEND;
7519 else
7520 q->q_flag &= ~QEND;
7521 mutex_exit(QLOCK(q));
7522 q = _OTHERQ(q);
7523 mutex_enter(QLOCK(q));
7524 if (!O_SAMESTR(q))
7525 q->q_flag |= QEND;
7526 else
7527 q->q_flag &= ~QEND;
7528 mutex_exit(QLOCK(q));
7529 }
7530
7531 /*
7532 * Set QFULL in next service procedure queue (that cares) if not already
7533 * set and if there are already more messages on the syncq than
7534 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on
7535 * any syncq.
7536 *
7537 * The fq here is the next queue with a service procedure. This is where
7538 * we would fail canputnext, so this is where we need to set QFULL.
7539 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7540 *
7541 * We already have QLOCK at this point. To avoid cross-locks with
7542 * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7543 * SQLOCK and sd_reflock, we need to drop respective locks first.
7544 */
7545 void
7546 set_qfull(queue_t *q)
7547 {
7548 queue_t *fq = NULL;
7549
7550 ASSERT(MUTEX_HELD(QLOCK(q)));
7551 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7552 (q->q_syncqmsgs > sq_max_size)) {
7553 if ((fq = q->q_nfsrv) == q) {
7554 fq->q_flag |= QFULL;
7555 } else {
7556 mutex_exit(QLOCK(q));
7557 mutex_enter(QLOCK(fq));
7558 fq->q_flag |= QFULL;
7559 mutex_exit(QLOCK(fq));
7560 mutex_enter(QLOCK(q));
7561 }
7562 }
7563 }
7564
7565 void
7566 clr_qfull(queue_t *q)
7567 {
7568 queue_t *oq = q;
7569
7570 q = q->q_nfsrv;
7571 /* Fast check if there is any work to do before getting the lock. */
7572 if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7573 return;
7574 }
7575
7576 /*
7577 * Do not reset QFULL (and backenable) if the q_count is the reason
7578 * for QFULL being set.
7579 */
7580 mutex_enter(QLOCK(q));
7581 /*
7582 * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7583 * Hence clear the QFULL.
7584 * If both q_count and q_mblkcnt are less than the hiwat mark,
7585 * clear the QFULL.
7586 */
7587 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7588 (q->q_mblkcnt < q->q_hiwat))) {
7589 q->q_flag &= ~QFULL;
7590 /*
7591 * A little more confusing, how about this way:
7592 * if someone wants to write,
7593 * AND
7594 * both counts are less than the lowat mark
7595 * OR
7596 * the lowat mark is zero
7597 * THEN
7598 * backenable
7599 */
7600 if ((q->q_flag & QWANTW) &&
7601 (((q->q_count < q->q_lowat) &&
7602 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7603 q->q_flag &= ~QWANTW;
7604 mutex_exit(QLOCK(q));
7605 backenable(oq, 0);
7606 } else
7607 mutex_exit(QLOCK(q));
7608 } else
7609 mutex_exit(QLOCK(q));
7610 }
7611
7612 /*
7613 * Set the forward service procedure pointer.
7614 *
7615 * Called at insert-time to cache a queue's next forward service procedure in
7616 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted
7617 * has a service procedure then q_nfsrv points to itself. If the queue to be
7618 * inserted does not have a service procedure, then q_nfsrv points to the next
7619 * queue forward that has a service procedure. If the queue is at the logical
7620 * end of the stream (driver for write side, stream head for the read side)
7621 * and does not have a service procedure, then q_nfsrv also points to itself.
7622 */
7623 void
7624 set_nfsrv_ptr(
7625 queue_t *rnew, /* read queue pointer to new module */
7626 queue_t *wnew, /* write queue pointer to new module */
7627 queue_t *prev_rq, /* read queue pointer to the module above */
7628 queue_t *prev_wq) /* write queue pointer to the module above */
7629 {
7630 queue_t *qp;
7631
7632 if (prev_wq->q_next == NULL) {
7633 /*
7634 * Insert the driver, initialize the driver and stream head.
7635 * In this case, prev_rq/prev_wq should be the stream head.
7636 * _I_INSERT does not allow inserting a driver. Make sure
7637 * that it is not an insertion.
7638 */
7639 ASSERT(!(rnew->q_flag & _QINSERTING));
7640 wnew->q_nfsrv = wnew;
7641 if (rnew->q_qinfo->qi_srvp)
7642 rnew->q_nfsrv = rnew;
7643 else
7644 rnew->q_nfsrv = prev_rq;
7645 prev_rq->q_nfsrv = prev_rq;
7646 prev_wq->q_nfsrv = prev_wq;
7647 } else {
7648 /*
7649 * set up read side q_nfsrv pointer. This MUST be done
7650 * before setting the write side, because the setting of
7651 * the write side for a fifo may depend on it.
7652 *
7653 * Suppose we have a fifo that only has pipemod pushed.
7654 * pipemod has no read or write service procedures, so
7655 * nfsrv for both pipemod queues points to prev_rq (the
7656 * stream read head). Now push bufmod (which has only a
7657 * read service procedure). Doing the write side first,
7658 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7659 * is WRONG; the next queue forward from wnew with a
7660 * service procedure will be rnew, not the stream read head.
7661 * Since the downstream queue (which in the case of a fifo
7662 * is the read queue rnew) can affect upstream queues, it
7663 * needs to be done first. Setting up the read side first
7664 * sets nfsrv for both pipemod queues to rnew and then
7665 * when the write side is set up, wnew-q_nfsrv will also
7666 * point to rnew.
7667 */
7668 if (rnew->q_qinfo->qi_srvp) {
7669 /*
7670 * use _OTHERQ() because, if this is a pipe, next
7671 * module may have been pushed from other end and
7672 * q_next could be a read queue.
7673 */
7674 qp = _OTHERQ(prev_wq->q_next);
7675 while (qp && qp->q_nfsrv != qp) {
7676 qp->q_nfsrv = rnew;
7677 qp = backq(qp);
7678 }
7679 rnew->q_nfsrv = rnew;
7680 } else
7681 rnew->q_nfsrv = prev_rq->q_nfsrv;
7682
7683 /* set up write side q_nfsrv pointer */
7684 if (wnew->q_qinfo->qi_srvp) {
7685 wnew->q_nfsrv = wnew;
7686
7687 /*
7688 * For insertion, need to update nfsrv of the modules
7689 * above which do not have a service routine.
7690 */
7691 if (rnew->q_flag & _QINSERTING) {
7692 for (qp = prev_wq;
7693 qp != NULL && qp->q_nfsrv != qp;
7694 qp = backq(qp)) {
7695 qp->q_nfsrv = wnew->q_nfsrv;
7696 }
7697 }
7698 } else {
7699 if (prev_wq->q_next == prev_rq)
7700 /*
7701 * Since prev_wq/prev_rq are the middle of a
7702 * fifo, wnew/rnew will also be the middle of
7703 * a fifo and wnew's nfsrv is same as rnew's.
7704 */
7705 wnew->q_nfsrv = rnew->q_nfsrv;
7706 else
7707 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7708 }
7709 }
7710 }
7711
7712 /*
7713 * Reset the forward service procedure pointer; called at remove-time.
7714 */
7715 void
7716 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7717 {
7718 queue_t *tmp_qp;
7719
7720 /* Reset the write side q_nfsrv pointer for _I_REMOVE */
7721 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7722 for (tmp_qp = backq(wqp);
7723 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7724 tmp_qp = backq(tmp_qp)) {
7725 tmp_qp->q_nfsrv = wqp->q_nfsrv;
7726 }
7727 }
7728
7729 /* reset the read side q_nfsrv pointer */
7730 if (rqp->q_qinfo->qi_srvp) {
7731 if (wqp->q_next) { /* non-driver case */
7732 tmp_qp = _OTHERQ(wqp->q_next);
7733 while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7734 /* Note that rqp->q_next cannot be NULL */
7735 ASSERT(rqp->q_next != NULL);
7736 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7737 tmp_qp = backq(tmp_qp);
7738 }
7739 }
7740 }
7741 }
7742
7743 /*
7744 * This routine should be called after all stream geometry changes to update
7745 * the stream head cached struio() rd/wr queue pointers. Note must be called
7746 * with the streamlock()ed.
7747 *
7748 * Note: only enables Synchronous STREAMS for a side of a Stream which has
7749 * an explicit synchronous barrier module queue. That is, a queue that
7750 * has specified a struio() type.
7751 */
7752 static void
7753 strsetuio(stdata_t *stp)
7754 {
7755 queue_t *wrq;
7756
7757 if (stp->sd_flag & STPLEX) {
7758 /*
7759 * Not streamhead, but a mux, so no Synchronous STREAMS.
7760 */
7761 stp->sd_struiowrq = NULL;
7762 stp->sd_struiordq = NULL;
7763 return;
7764 }
7765 /*
7766 * Scan the write queue(s) while synchronous
7767 * until we find a qinfo uio type specified.
7768 */
7769 wrq = stp->sd_wrq->q_next;
7770 while (wrq) {
7771 if (wrq->q_struiot == STRUIOT_NONE) {
7772 wrq = 0;
7773 break;
7774 }
7775 if (wrq->q_struiot != STRUIOT_DONTCARE)
7776 break;
7777 if (! _SAMESTR(wrq)) {
7778 wrq = 0;
7779 break;
7780 }
7781 wrq = wrq->q_next;
7782 }
7783 stp->sd_struiowrq = wrq;
7784 /*
7785 * Scan the read queue(s) while synchronous
7786 * until we find a qinfo uio type specified.
7787 */
7788 wrq = stp->sd_wrq->q_next;
7789 while (wrq) {
7790 if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7791 wrq = 0;
7792 break;
7793 }
7794 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7795 break;
7796 if (! _SAMESTR(wrq)) {
7797 wrq = 0;
7798 break;
7799 }
7800 wrq = wrq->q_next;
7801 }
7802 stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7803 }
7804
7805 /*
7806 * pass_wput, unblocks the passthru queues, so that
7807 * messages can arrive at muxs lower read queue, before
7808 * I_LINK/I_UNLINK is acked/nacked.
7809 */
7810 static void
7811 pass_wput(queue_t *q, mblk_t *mp)
7812 {
7813 syncq_t *sq;
7814
7815 sq = _RD(q)->q_syncq;
7816 if (sq->sq_flags & SQ_BLOCKED)
7817 unblocksq(sq, SQ_BLOCKED, 0);
7818 putnext(q, mp);
7819 }
7820
7821 /*
7822 * Set up queues for the link/unlink.
7823 * Create a new queue and block it and then insert it
7824 * below the stream head on the lower stream.
7825 * This prevents any messages from arriving during the setq
7826 * as well as while the mux is processing the LINK/I_UNLINK.
7827 * The blocked passq is unblocked once the LINK/I_UNLINK has
7828 * been acked or nacked or if a message is generated and sent
7829 * down muxs write put procedure.
7830 * See pass_wput().
7831 *
7832 * After the new queue is inserted, all messages coming from below are
7833 * blocked. The call to strlock will ensure that all activity in the stream head
7834 * read queue syncq is stopped (sq_count drops to zero).
7835 */
7836 static queue_t *
7837 link_addpassthru(stdata_t *stpdown)
7838 {
7839 queue_t *passq;
7840 sqlist_t sqlist;
7841
7842 passq = allocq();
7843 STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7844 /* setq might sleep in allocator - avoid holding locks. */
7845 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7846 SQ_CI|SQ_CO, B_FALSE);
7847 claimq(passq);
7848 blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7849 insertq(STREAM(passq), passq);
7850
7851 /*
7852 * Use strlock() to wait for the stream head sq_count to drop to zero
7853 * since we are going to change q_ptr in the stream head. Note that
7854 * insertq() doesn't wait for any syncq counts to drop to zero.
7855 */
7856 sqlist.sqlist_head = NULL;
7857 sqlist.sqlist_index = 0;
7858 sqlist.sqlist_size = sizeof (sqlist_t);
7859 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7860 strlock(stpdown, &sqlist);
7861 strunlock(stpdown, &sqlist);
7862
7863 releaseq(passq);
7864 return (passq);
7865 }
7866
7867 /*
7868 * Let messages flow up into the mux by removing
7869 * the passq.
7870 */
7871 static void
7872 link_rempassthru(queue_t *passq)
7873 {
7874 claimq(passq);
7875 removeq(passq);
7876 releaseq(passq);
7877 freeq(passq);
7878 }
7879
7880 /*
7881 * Wait for the condition variable pointed to by `cvp' to be signaled,
7882 * or for `tim' milliseconds to elapse, whichever comes first. If `tim'
7883 * is negative, then there is no time limit. If `nosigs' is non-zero,
7884 * then the wait will be non-interruptible.
7885 *
7886 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7887 */
7888 clock_t
7889 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7890 {
7891 clock_t ret;
7892
7893 if (tim < 0) {
7894 if (nosigs) {
7895 cv_wait(cvp, mp);
7896 ret = 1;
7897 } else {
7898 ret = cv_wait_sig(cvp, mp);
7899 }
7900 } else if (tim > 0) {
7901 /*
7902 * convert milliseconds to clock ticks
7903 */
7904 if (nosigs) {
7905 ret = cv_reltimedwait(cvp, mp,
7906 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7907 } else {
7908 ret = cv_reltimedwait_sig(cvp, mp,
7909 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7910 }
7911 } else {
7912 ret = -1;
7913 }
7914 return (ret);
7915 }
7916
7917 /*
7918 * Wait until the stream head can determine if it is at the mark but
7919 * don't wait forever to prevent a race condition between the "mark" state
7920 * in the stream head and any mark state in the caller/user of this routine.
7921 *
7922 * This is used by sockets and for a socket it would be incorrect
7923 * to return a failure for SIOCATMARK when there is no data in the receive
7924 * queue and the marked urgent data is traveling up the stream.
7925 *
7926 * This routine waits until the mark is known by waiting for one of these
7927 * three events:
7928 * The stream head read queue becoming non-empty (including an EOF).
7929 * The STRATMARK flag being set (due to a MSGMARKNEXT message).
7930 * The STRNOTATMARK flag being set (which indicates that the transport
7931 * has sent a MSGNOTMARKNEXT message to indicate that it is not at
7932 * the mark).
7933 *
7934 * The routine returns 1 if the stream is at the mark; 0 if it can
7935 * be determined that the stream is not at the mark.
7936 * If the wait times out and it can't determine
7937 * whether or not the stream might be at the mark the routine will return -1.
7938 *
7939 * Note: This routine should only be used when a mark is pending i.e.,
7940 * in the socket case the SIGURG has been posted.
7941 * Note2: This can not wakeup just because synchronous streams indicate
7942 * that data is available since it is not possible to use the synchronous
7943 * streams interfaces to determine the b_flag value for the data queued below
7944 * the stream head.
7945 */
7946 int
7947 strwaitmark(vnode_t *vp)
7948 {
7949 struct stdata *stp = vp->v_stream;
7950 queue_t *rq = _RD(stp->sd_wrq);
7951 int mark;
7952
7953 mutex_enter(&stp->sd_lock);
7954 while (rq->q_first == NULL &&
7955 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7956 stp->sd_flag |= RSLEEP;
7957
7958 /* Wait for 100 milliseconds for any state change. */
7959 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7960 mutex_exit(&stp->sd_lock);
7961 return (-1);
7962 }
7963 }
7964 if (stp->sd_flag & STRATMARK)
7965 mark = 1;
7966 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7967 mark = 1;
7968 else
7969 mark = 0;
7970
7971 mutex_exit(&stp->sd_lock);
7972 return (mark);
7973 }
7974
7975 /*
7976 * Set a read side error. If persist is set change the socket error
7977 * to persistent. If errfunc is set install the function as the exported
7978 * error handler.
7979 */
7980 void
7981 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
7982 {
7983 struct stdata *stp = vp->v_stream;
7984
7985 mutex_enter(&stp->sd_lock);
7986 stp->sd_rerror = error;
7987 if (error == 0 && errfunc == NULL)
7988 stp->sd_flag &= ~STRDERR;
7989 else
7990 stp->sd_flag |= STRDERR;
7991 if (persist) {
7992 stp->sd_flag &= ~STRDERRNONPERSIST;
7993 } else {
7994 stp->sd_flag |= STRDERRNONPERSIST;
7995 }
7996 stp->sd_rderrfunc = errfunc;
7997 if (error != 0 || errfunc != NULL) {
7998 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
7999 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8000 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8001
8002 mutex_exit(&stp->sd_lock);
8003 pollwakeup(&stp->sd_pollist, POLLERR);
8004 mutex_enter(&stp->sd_lock);
8005
8006 if (stp->sd_sigflags & S_ERROR)
8007 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8008 }
8009 mutex_exit(&stp->sd_lock);
8010 }
8011
8012 /*
8013 * Set a write side error. If persist is set change the socket error
8014 * to persistent.
8015 */
8016 void
8017 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8018 {
8019 struct stdata *stp = vp->v_stream;
8020
8021 mutex_enter(&stp->sd_lock);
8022 stp->sd_werror = error;
8023 if (error == 0 && errfunc == NULL)
8024 stp->sd_flag &= ~STWRERR;
8025 else
8026 stp->sd_flag |= STWRERR;
8027 if (persist) {
8028 stp->sd_flag &= ~STWRERRNONPERSIST;
8029 } else {
8030 stp->sd_flag |= STWRERRNONPERSIST;
8031 }
8032 stp->sd_wrerrfunc = errfunc;
8033 if (error != 0 || errfunc != NULL) {
8034 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
8035 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8036 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8037
8038 mutex_exit(&stp->sd_lock);
8039 pollwakeup(&stp->sd_pollist, POLLERR);
8040 mutex_enter(&stp->sd_lock);
8041
8042 if (stp->sd_sigflags & S_ERROR)
8043 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8044 }
8045 mutex_exit(&stp->sd_lock);
8046 }
8047
8048 /*
8049 * Make the stream return 0 (EOF) when all data has been read.
8050 * No effect on write side.
8051 */
8052 void
8053 strseteof(vnode_t *vp, int eof)
8054 {
8055 struct stdata *stp = vp->v_stream;
8056
8057 mutex_enter(&stp->sd_lock);
8058 if (!eof) {
8059 stp->sd_flag &= ~STREOF;
8060 mutex_exit(&stp->sd_lock);
8061 return;
8062 }
8063 stp->sd_flag |= STREOF;
8064 if (stp->sd_flag & RSLEEP) {
8065 stp->sd_flag &= ~RSLEEP;
8066 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8067 }
8068
8069 mutex_exit(&stp->sd_lock);
8070 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8071 mutex_enter(&stp->sd_lock);
8072
8073 if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8074 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8075 mutex_exit(&stp->sd_lock);
8076 }
8077
8078 void
8079 strflushrq(vnode_t *vp, int flag)
8080 {
8081 struct stdata *stp = vp->v_stream;
8082
8083 mutex_enter(&stp->sd_lock);
8084 flushq(_RD(stp->sd_wrq), flag);
8085 mutex_exit(&stp->sd_lock);
8086 }
8087
8088 void
8089 strsetrputhooks(vnode_t *vp, uint_t flags,
8090 msgfunc_t protofunc, msgfunc_t miscfunc)
8091 {
8092 struct stdata *stp = vp->v_stream;
8093
8094 mutex_enter(&stp->sd_lock);
8095
8096 if (protofunc == NULL)
8097 stp->sd_rprotofunc = strrput_proto;
8098 else
8099 stp->sd_rprotofunc = protofunc;
8100
8101 if (miscfunc == NULL)
8102 stp->sd_rmiscfunc = strrput_misc;
8103 else
8104 stp->sd_rmiscfunc = miscfunc;
8105
8106 if (flags & SH_CONSOL_DATA)
8107 stp->sd_rput_opt |= SR_CONSOL_DATA;
8108 else
8109 stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8110
8111 if (flags & SH_SIGALLDATA)
8112 stp->sd_rput_opt |= SR_SIGALLDATA;
8113 else
8114 stp->sd_rput_opt &= ~SR_SIGALLDATA;
8115
8116 if (flags & SH_IGN_ZEROLEN)
8117 stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8118 else
8119 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8120
8121 mutex_exit(&stp->sd_lock);
8122 }
8123
8124 void
8125 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8126 {
8127 struct stdata *stp = vp->v_stream;
8128
8129 mutex_enter(&stp->sd_lock);
8130 stp->sd_closetime = closetime;
8131
8132 if (flags & SH_SIGPIPE)
8133 stp->sd_wput_opt |= SW_SIGPIPE;
8134 else
8135 stp->sd_wput_opt &= ~SW_SIGPIPE;
8136 if (flags & SH_RECHECK_ERR)
8137 stp->sd_wput_opt |= SW_RECHECK_ERR;
8138 else
8139 stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8140
8141 mutex_exit(&stp->sd_lock);
8142 }
8143
8144 void
8145 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8146 {
8147 struct stdata *stp = vp->v_stream;
8148
8149 mutex_enter(&stp->sd_lock);
8150
8151 stp->sd_rputdatafunc = rdatafunc;
8152 stp->sd_wputdatafunc = wdatafunc;
8153
8154 mutex_exit(&stp->sd_lock);
8155 }
8156
8157 /* Used within framework when the queue is already locked */
8158 void
8159 qenable_locked(queue_t *q)
8160 {
8161 stdata_t *stp = STREAM(q);
8162
8163 ASSERT(MUTEX_HELD(QLOCK(q)));
8164
8165 if (!q->q_qinfo->qi_srvp)
8166 return;
8167
8168 /*
8169 * Do not place on run queue if already enabled or closing.
8170 */
8171 if (q->q_flag & (QWCLOSE|QENAB))
8172 return;
8173
8174 /*
8175 * mark queue enabled and place on run list if it is not already being
8176 * serviced. If it is serviced, the runservice() function will detect
8177 * that QENAB is set and call service procedure before clearing
8178 * QINSERVICE flag.
8179 */
8180 q->q_flag |= QENAB;
8181 if (q->q_flag & QINSERVICE)
8182 return;
8183
8184 /* Record the time of qenable */
8185 q->q_qtstamp = ddi_get_lbolt();
8186
8187 /*
8188 * Put the queue in the stp list and schedule it for background
8189 * processing if it is not already scheduled or if stream head does not
8190 * intent to process it in the foreground later by setting
8191 * STRS_WILLSERVICE flag.
8192 */
8193 mutex_enter(&stp->sd_qlock);
8194 /*
8195 * If there are already something on the list, stp flags should show
8196 * intention to drain it.
8197 */
8198 IMPLY(STREAM_NEEDSERVICE(stp),
8199 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8200
8201 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8202 stp->sd_nqueues++;
8203
8204 /*
8205 * If no one will drain this stream we are the first producer and
8206 * need to schedule it for background thread.
8207 */
8208 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8209 /*
8210 * No one will service this stream later, so we have to
8211 * schedule it now.
8212 */
8213 STRSTAT(stenables);
8214 stp->sd_svcflags |= STRS_SCHEDULED;
8215 stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8216 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8217
8218 if (stp->sd_servid == NULL) {
8219 /*
8220 * Task queue failed so fail over to the backup
8221 * servicing thread.
8222 */
8223 STRSTAT(taskqfails);
8224 /*
8225 * It is safe to clear STRS_SCHEDULED flag because it
8226 * was set by this thread above.
8227 */
8228 stp->sd_svcflags &= ~STRS_SCHEDULED;
8229
8230 /*
8231 * Failover scheduling is protected by service_queue
8232 * lock.
8233 */
8234 mutex_enter(&service_queue);
8235 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8236 ASSERT(q->q_link == NULL);
8237 /*
8238 * Append the queue to qhead/qtail list.
8239 */
8240 if (qhead == NULL)
8241 qhead = q;
8242 else
8243 qtail->q_link = q;
8244 qtail = q;
8245 /*
8246 * Clear stp queue list.
8247 */
8248 stp->sd_qhead = stp->sd_qtail = NULL;
8249 stp->sd_nqueues = 0;
8250 /*
8251 * Wakeup background queue processing thread.
8252 */
8253 cv_signal(&services_to_run);
8254 mutex_exit(&service_queue);
8255 }
8256 }
8257 mutex_exit(&stp->sd_qlock);
8258 }
8259
8260 static void
8261 queue_service(queue_t *q)
8262 {
8263 /*
8264 * The queue in the list should have
8265 * QENAB flag set and should not have
8266 * QINSERVICE flag set. QINSERVICE is
8267 * set when the queue is dequeued and
8268 * qenable_locked doesn't enqueue a
8269 * queue with QINSERVICE set.
8270 */
8271
8272 ASSERT(!(q->q_flag & QINSERVICE));
8273 ASSERT((q->q_flag & QENAB));
8274 mutex_enter(QLOCK(q));
8275 q->q_flag &= ~QENAB;
8276 q->q_flag |= QINSERVICE;
8277 mutex_exit(QLOCK(q));
8278 runservice(q);
8279 }
8280
8281 static void
8282 syncq_service(syncq_t *sq)
8283 {
8284 STRSTAT(syncqservice);
8285 mutex_enter(SQLOCK(sq));
8286 ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8287 ASSERT(sq->sq_servcount != 0);
8288 ASSERT(sq->sq_next == NULL);
8289
8290 /* if we came here from the background thread, clear the flag */
8291 if (sq->sq_svcflags & SQ_BGTHREAD)
8292 sq->sq_svcflags &= ~SQ_BGTHREAD;
8293
8294 /* let drain_syncq know that it's being called in the background */
8295 sq->sq_svcflags |= SQ_SERVICE;
8296 drain_syncq(sq);
8297 }
8298
8299 static void
8300 qwriter_outer_service(syncq_t *outer)
8301 {
8302 /*
8303 * Note that SQ_WRITER is used on the outer perimeter
8304 * to signal that a qwriter(OUTER) is either investigating
8305 * running or that it is actually running a function.
8306 */
8307 outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8308
8309 /*
8310 * All inner syncq are empty and have SQ_WRITER set
8311 * to block entering the outer perimeter.
8312 *
8313 * We do not need to explicitly call write_now since
8314 * outer_exit does it for us.
8315 */
8316 outer_exit(outer);
8317 }
8318
8319 static void
8320 mblk_free(mblk_t *mp)
8321 {
8322 dblk_t *dbp = mp->b_datap;
8323 frtn_t *frp = dbp->db_frtnp;
8324
8325 mp->b_next = NULL;
8326 if (dbp->db_fthdr != NULL)
8327 str_ftfree(dbp);
8328
8329 ASSERT(dbp->db_fthdr == NULL);
8330 frp->free_func(frp->free_arg);
8331 ASSERT(dbp->db_mblk == mp);
8332
8333 if (dbp->db_credp != NULL) {
8334 crfree(dbp->db_credp);
8335 dbp->db_credp = NULL;
8336 }
8337 dbp->db_cpid = -1;
8338 dbp->db_struioflag = 0;
8339 dbp->db_struioun.cksum.flags = 0;
8340
8341 kmem_cache_free(dbp->db_cache, dbp);
8342 }
8343
8344 /*
8345 * Background processing of the stream queue list.
8346 */
8347 static void
8348 stream_service(stdata_t *stp)
8349 {
8350 queue_t *q;
8351
8352 mutex_enter(&stp->sd_qlock);
8353
8354 STR_SERVICE(stp, q);
8355
8356 stp->sd_svcflags &= ~STRS_SCHEDULED;
8357 stp->sd_servid = NULL;
8358 cv_signal(&stp->sd_qcv);
8359 mutex_exit(&stp->sd_qlock);
8360 }
8361
8362 /*
8363 * Foreground processing of the stream queue list.
8364 */
8365 void
8366 stream_runservice(stdata_t *stp)
8367 {
8368 queue_t *q;
8369
8370 mutex_enter(&stp->sd_qlock);
8371 STRSTAT(rservice);
8372 /*
8373 * We are going to drain this stream queue list, so qenable_locked will
8374 * not schedule it until we finish.
8375 */
8376 stp->sd_svcflags |= STRS_WILLSERVICE;
8377
8378 STR_SERVICE(stp, q);
8379
8380 stp->sd_svcflags &= ~STRS_WILLSERVICE;
8381 mutex_exit(&stp->sd_qlock);
8382 /*
8383 * Help backup background thread to drain the qhead/qtail list.
8384 */
8385 while (qhead != NULL) {
8386 STRSTAT(qhelps);
8387 mutex_enter(&service_queue);
8388 DQ(q, qhead, qtail, q_link);
8389 mutex_exit(&service_queue);
8390 if (q != NULL)
8391 queue_service(q);
8392 }
8393 }
8394
8395 void
8396 stream_willservice(stdata_t *stp)
8397 {
8398 mutex_enter(&stp->sd_qlock);
8399 stp->sd_svcflags |= STRS_WILLSERVICE;
8400 mutex_exit(&stp->sd_qlock);
8401 }
8402
8403 /*
8404 * Replace the cred currently in the mblk with a different one.
8405 * Also update db_cpid.
8406 */
8407 void
8408 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8409 {
8410 dblk_t *dbp = mp->b_datap;
8411 cred_t *ocr = dbp->db_credp;
8412
8413 ASSERT(cr != NULL);
8414
8415 if (cr != ocr) {
8416 crhold(dbp->db_credp = cr);
8417 if (ocr != NULL)
8418 crfree(ocr);
8419 }
8420 /* Don't overwrite with NOPID */
8421 if (cpid != NOPID)
8422 dbp->db_cpid = cpid;
8423 }
8424
8425 /*
8426 * If the src message has a cred, then replace the cred currently in the mblk
8427 * with it.
8428 * Also update db_cpid.
8429 */
8430 void
8431 mblk_copycred(mblk_t *mp, const mblk_t *src)
8432 {
8433 dblk_t *dbp = mp->b_datap;
8434 cred_t *cr, *ocr;
8435 pid_t cpid;
8436
8437 cr = msg_getcred(src, &cpid);
8438 if (cr == NULL)
8439 return;
8440
8441 ocr = dbp->db_credp;
8442 if (cr != ocr) {
8443 crhold(dbp->db_credp = cr);
8444 if (ocr != NULL)
8445 crfree(ocr);
8446 }
8447 /* Don't overwrite with NOPID */
8448 if (cpid != NOPID)
8449 dbp->db_cpid = cpid;
8450 }
8451
8452 int
8453 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8454 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8455 uint32_t flags, int km_flags)
8456 {
8457 int rc = 0;
8458
8459 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8460 if (mp->b_datap->db_type == M_DATA) {
8461 /* Associate values for M_DATA type */
8462 DB_CKSUMSTART(mp) = (intptr_t)start;
8463 DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8464 DB_CKSUMEND(mp) = (intptr_t)end;
8465 DB_CKSUMFLAGS(mp) = flags;
8466 DB_CKSUM16(mp) = (uint16_t)value;
8467
8468 } else {
8469 pattrinfo_t pa_info;
8470
8471 ASSERT(mmd != NULL);
8472
8473 pa_info.type = PATTR_HCKSUM;
8474 pa_info.len = sizeof (pattr_hcksum_t);
8475
8476 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8477 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8478
8479 hck->hcksum_start_offset = start;
8480 hck->hcksum_stuff_offset = stuff;
8481 hck->hcksum_end_offset = end;
8482 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8483 hck->hcksum_flags = flags;
8484 } else {
8485 rc = -1;
8486 }
8487 }
8488 return (rc);
8489 }
8490
8491 void
8492 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8493 uint32_t *start, uint32_t *stuff, uint32_t *end,
8494 uint32_t *value, uint32_t *flags)
8495 {
8496 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8497 if (mp->b_datap->db_type == M_DATA) {
8498 if (flags != NULL) {
8499 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8500 if ((*flags & (HCK_PARTIALCKSUM |
8501 HCK_FULLCKSUM)) != 0) {
8502 if (value != NULL)
8503 *value = (uint32_t)DB_CKSUM16(mp);
8504 if ((*flags & HCK_PARTIALCKSUM) != 0) {
8505 if (start != NULL)
8506 *start =
8507 (uint32_t)DB_CKSUMSTART(mp);
8508 if (stuff != NULL)
8509 *stuff =
8510 (uint32_t)DB_CKSUMSTUFF(mp);
8511 if (end != NULL)
8512 *end =
8513 (uint32_t)DB_CKSUMEND(mp);
8514 }
8515 }
8516 }
8517 } else {
8518 pattrinfo_t hck_attr = {PATTR_HCKSUM};
8519
8520 ASSERT(mmd != NULL);
8521
8522 /* get hardware checksum attribute */
8523 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8524 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8525
8526 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8527 if (flags != NULL)
8528 *flags = hck->hcksum_flags;
8529 if (start != NULL)
8530 *start = hck->hcksum_start_offset;
8531 if (stuff != NULL)
8532 *stuff = hck->hcksum_stuff_offset;
8533 if (end != NULL)
8534 *end = hck->hcksum_end_offset;
8535 if (value != NULL)
8536 *value = (uint32_t)
8537 hck->hcksum_cksum_val.inet_cksum;
8538 }
8539 }
8540 }
8541
8542 void
8543 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8544 {
8545 ASSERT(DB_TYPE(mp) == M_DATA);
8546 ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8547
8548 /* Set the flags */
8549 DB_LSOFLAGS(mp) |= flags;
8550 DB_LSOMSS(mp) = mss;
8551 }
8552
8553 void
8554 lso_info_cleanup(mblk_t *mp)
8555 {
8556 ASSERT(DB_TYPE(mp) == M_DATA);
8557
8558 /* Clear the flags */
8559 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8560 DB_LSOMSS(mp) = 0;
8561 }
8562
8563 /*
8564 * Checksum buffer *bp for len bytes with psum partial checksum,
8565 * or 0 if none, and return the 16 bit partial checksum.
8566 */
8567 unsigned
8568 bcksum(uchar_t *bp, int len, unsigned int psum)
8569 {
8570 int odd = len & 1;
8571 extern unsigned int ip_ocsum();
8572
8573 if (((intptr_t)bp & 1) == 0 && !odd) {
8574 /*
8575 * Bp is 16 bit aligned and len is multiple of 16 bit word.
8576 */
8577 return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8578 }
8579 if (((intptr_t)bp & 1) != 0) {
8580 /*
8581 * Bp isn't 16 bit aligned.
8582 */
8583 unsigned int tsum;
8584
8585 #ifdef _LITTLE_ENDIAN
8586 psum += *bp;
8587 #else
8588 psum += *bp << 8;
8589 #endif
8590 len--;
8591 bp++;
8592 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8593 psum += (tsum << 8) & 0xffff | (tsum >> 8);
8594 if (len & 1) {
8595 bp += len - 1;
8596 #ifdef _LITTLE_ENDIAN
8597 psum += *bp << 8;
8598 #else
8599 psum += *bp;
8600 #endif
8601 }
8602 } else {
8603 /*
8604 * Bp is 16 bit aligned.
8605 */
8606 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8607 if (odd) {
8608 bp += len - 1;
8609 #ifdef _LITTLE_ENDIAN
8610 psum += *bp;
8611 #else
8612 psum += *bp << 8;
8613 #endif
8614 }
8615 }
8616 /*
8617 * Normalize psum to 16 bits before returning the new partial
8618 * checksum. The max psum value before normalization is 0x3FDFE.
8619 */
8620 return ((psum >> 16) + (psum & 0xFFFF));
8621 }
8622
8623 boolean_t
8624 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8625 {
8626 boolean_t rc;
8627
8628 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8629 if (DB_TYPE(mp) == M_DATA) {
8630 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8631 } else {
8632 pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8633
8634 ASSERT(mmd != NULL);
8635 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8636 }
8637 return (rc);
8638 }
8639
8640 void
8641 freemsgchain(mblk_t *mp)
8642 {
8643 mblk_t *next;
8644
8645 while (mp != NULL) {
8646 next = mp->b_next;
8647 mp->b_next = NULL;
8648
8649 freemsg(mp);
8650 mp = next;
8651 }
8652 }
8653
8654 mblk_t *
8655 copymsgchain(mblk_t *mp)
8656 {
8657 mblk_t *nmp = NULL;
8658 mblk_t **nmpp = &nmp;
8659
8660 for (; mp != NULL; mp = mp->b_next) {
8661 if ((*nmpp = copymsg(mp)) == NULL) {
8662 freemsgchain(nmp);
8663 return (NULL);
8664 }
8665
8666 nmpp = &((*nmpp)->b_next);
8667 }
8668
8669 return (nmp);
8670 }
8671
8672 /* NOTE: Do not add code after this point. */
8673 #undef QLOCK
8674
8675 /*
8676 * Replacement for QLOCK macro for those that can't use it.
8677 */
8678 kmutex_t *
8679 QLOCK(queue_t *q)
8680 {
8681 return (&(q)->q_lock);
8682 }
8683
8684 /*
8685 * Dummy runqueues/queuerun functions functions for backwards compatibility.
8686 */
8687 #undef runqueues
8688 void
8689 runqueues(void)
8690 {
8691 }
8692
8693 #undef queuerun
8694 void
8695 queuerun(void)
8696 {
8697 }
8698
8699 /*
8700 * Initialize the STR stack instance, which tracks autopush and persistent
8701 * links.
8702 */
8703 /* ARGSUSED */
8704 static void *
8705 str_stack_init(netstackid_t stackid, netstack_t *ns)
8706 {
8707 str_stack_t *ss;
8708 int i;
8709
8710 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8711 ss->ss_netstack = ns;
8712
8713 /*
8714 * set up autopush
8715 */
8716 sad_initspace(ss);
8717
8718 /*
8719 * set up mux_node structures.
8720 */
8721 ss->ss_devcnt = devcnt; /* In case it should change before free */
8722 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8723 ss->ss_devcnt), KM_SLEEP);
8724 for (i = 0; i < ss->ss_devcnt; i++)
8725 ss->ss_mux_nodes[i].mn_imaj = i;
8726 return (ss);
8727 }
8728
8729 /*
8730 * Note: run at zone shutdown and not destroy so that the PLINKs are
8731 * gone by the time other cleanup happens from the destroy callbacks.
8732 */
8733 static void
8734 str_stack_shutdown(netstackid_t stackid, void *arg)
8735 {
8736 str_stack_t *ss = (str_stack_t *)arg;
8737 int i;
8738 cred_t *cr;
8739
8740 cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8741 ASSERT(cr != NULL);
8742
8743 /* Undo all the I_PLINKs for this zone */
8744 for (i = 0; i < ss->ss_devcnt; i++) {
8745 struct mux_edge *ep;
8746 ldi_handle_t lh;
8747 ldi_ident_t li;
8748 int ret;
8749 int rval;
8750 dev_t rdev;
8751
8752 ep = ss->ss_mux_nodes[i].mn_outp;
8753 if (ep == NULL)
8754 continue;
8755 ret = ldi_ident_from_major((major_t)i, &li);
8756 if (ret != 0) {
8757 continue;
8758 }
8759 rdev = ep->me_dev;
8760 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8761 cr, &lh, li);
8762 if (ret != 0) {
8763 ldi_ident_release(li);
8764 continue;
8765 }
8766
8767 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8768 cr, &rval);
8769 if (ret) {
8770 (void) ldi_close(lh, FREAD|FWRITE, cr);
8771 ldi_ident_release(li);
8772 continue;
8773 }
8774 (void) ldi_close(lh, FREAD|FWRITE, cr);
8775
8776 /* Close layered handles */
8777 ldi_ident_release(li);
8778 }
8779 crfree(cr);
8780
8781 sad_freespace(ss);
8782
8783 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8784 ss->ss_mux_nodes = NULL;
8785 }
8786
8787 /*
8788 * Free the structure; str_stack_shutdown did the other cleanup work.
8789 */
8790 /* ARGSUSED */
8791 static void
8792 str_stack_fini(netstackid_t stackid, void *arg)
8793 {
8794 str_stack_t *ss = (str_stack_t *)arg;
8795
8796 kmem_free(ss, sizeof (*ss));
8797 }