1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 /* All Rights Reserved */
23
24 /*
25 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
26 * Use is subject to license terms.
27 * Copyright (c) 2016 by Delphix. All rights reserved.
28 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
29 */
30 /*
31 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
32 */
33
34 #include <sys/types.h>
35 #include <sys/sysmacros.h>
36 #include <sys/param.h>
37 #include <sys/errno.h>
38 #include <sys/signal.h>
39 #include <sys/proc.h>
40 #include <sys/conf.h>
41 #include <sys/cred.h>
42 #include <sys/user.h>
43 #include <sys/vnode.h>
44 #include <sys/file.h>
45 #include <sys/session.h>
46 #include <sys/stream.h>
47 #include <sys/strsubr.h>
48 #include <sys/stropts.h>
49 #include <sys/poll.h>
50 #include <sys/systm.h>
51 #include <sys/cpuvar.h>
52 #include <sys/uio.h>
53 #include <sys/cmn_err.h>
54 #include <sys/priocntl.h>
55 #include <sys/procset.h>
56 #include <sys/vmem.h>
57 #include <sys/bitmap.h>
58 #include <sys/kmem.h>
59 #include <sys/siginfo.h>
60 #include <sys/vtrace.h>
61 #include <sys/callb.h>
62 #include <sys/debug.h>
63 #include <sys/modctl.h>
64 #include <sys/vmsystm.h>
65 #include <vm/page.h>
66 #include <sys/atomic.h>
67 #include <sys/suntpi.h>
68 #include <sys/strlog.h>
69 #include <sys/promif.h>
70 #include <sys/project.h>
71 #include <sys/vm.h>
72 #include <sys/taskq.h>
73 #include <sys/sunddi.h>
74 #include <sys/sunldi_impl.h>
75 #include <sys/strsun.h>
76 #include <sys/isa_defs.h>
77 #include <sys/multidata.h>
78 #include <sys/pattr.h>
79 #include <sys/strft.h>
80 #include <sys/fs/snode.h>
81 #include <sys/zone.h>
82 #include <sys/open.h>
83 #include <sys/sunldi.h>
84 #include <sys/sad.h>
85 #include <sys/netstack.h>
86
87 #define O_SAMESTR(q) (((q)->q_next) && \
88 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
89
90 /*
91 * WARNING:
92 * The variables and routines in this file are private, belonging
93 * to the STREAMS subsystem. These should not be used by modules
94 * or drivers. Compatibility will not be guaranteed.
95 */
96
97 /*
98 * Id value used to distinguish between different multiplexor links.
99 */
100 static int32_t lnk_id = 0;
101
102 #define STREAMS_LOPRI MINCLSYSPRI
103 static pri_t streams_lopri = STREAMS_LOPRI;
104
105 #define STRSTAT(x) (str_statistics.x.value.ui64++)
106 typedef struct str_stat {
107 kstat_named_t sqenables;
108 kstat_named_t stenables;
109 kstat_named_t syncqservice;
110 kstat_named_t freebs;
111 kstat_named_t qwr_outer;
112 kstat_named_t rservice;
113 kstat_named_t strwaits;
114 kstat_named_t taskqfails;
115 kstat_named_t bufcalls;
116 kstat_named_t qhelps;
117 kstat_named_t qremoved;
118 kstat_named_t sqremoved;
119 kstat_named_t bcwaits;
120 kstat_named_t sqtoomany;
121 } str_stat_t;
122
123 static str_stat_t str_statistics = {
124 { "sqenables", KSTAT_DATA_UINT64 },
125 { "stenables", KSTAT_DATA_UINT64 },
126 { "syncqservice", KSTAT_DATA_UINT64 },
127 { "freebs", KSTAT_DATA_UINT64 },
128 { "qwr_outer", KSTAT_DATA_UINT64 },
129 { "rservice", KSTAT_DATA_UINT64 },
130 { "strwaits", KSTAT_DATA_UINT64 },
131 { "taskqfails", KSTAT_DATA_UINT64 },
132 { "bufcalls", KSTAT_DATA_UINT64 },
133 { "qhelps", KSTAT_DATA_UINT64 },
134 { "qremoved", KSTAT_DATA_UINT64 },
135 { "sqremoved", KSTAT_DATA_UINT64 },
136 { "bcwaits", KSTAT_DATA_UINT64 },
137 { "sqtoomany", KSTAT_DATA_UINT64 },
138 };
139
140 static kstat_t *str_kstat;
141
142 /*
143 * qrunflag was used previously to control background scheduling of queues. It
144 * is not used anymore, but kept here in case some module still wants to access
145 * it via qready() and setqsched macros.
146 */
147 char qrunflag; /* Unused */
148
149 /*
150 * Most of the streams scheduling is done via task queues. Task queues may fail
151 * for non-sleep dispatches, so there are two backup threads servicing failed
152 * requests for queues and syncqs. Both of these threads also service failed
153 * dispatches freebs requests. Queues are put in the list specified by `qhead'
154 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
155 * requests are put into `freebs_list' which has no tail pointer. All three
156 * lists are protected by a single `service_queue' lock and use
157 * `services_to_run' condition variable for signaling background threads. Use of
158 * a single lock should not be a problem because it is only used under heavy
159 * loads when task queues start to fail and at that time it may be a good idea
160 * to throttle scheduling requests.
161 *
162 * NOTE: queues and syncqs should be scheduled by two separate threads because
163 * queue servicing may be blocked waiting for a syncq which may be also
164 * scheduled for background execution. This may create a deadlock when only one
165 * thread is used for both.
166 */
167
168 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */
169
170 static kmutex_t service_queue; /* protects all of servicing vars */
171 static kcondvar_t services_to_run; /* wake up background service thread */
172 static kcondvar_t syncqs_to_run; /* wake up background service thread */
173
174 /*
175 * List of queues scheduled for background processing due to lack of resources
176 * in the task queues. Protected by service_queue lock;
177 */
178 static struct queue *qhead;
179 static struct queue *qtail;
180
181 /*
182 * Same list for syncqs
183 */
184 static syncq_t *sqhead;
185 static syncq_t *sqtail;
186
187 static mblk_t *freebs_list; /* list of buffers to free */
188
189 /*
190 * Backup threads for servicing queues and syncqs
191 */
192 kthread_t *streams_qbkgrnd_thread;
193 kthread_t *streams_sqbkgrnd_thread;
194
195 /*
196 * Bufcalls related variables.
197 */
198 struct bclist strbcalls; /* list of waiting bufcalls */
199 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */
200 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */
201 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */
202 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */
203 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */
204
205 kmutex_t strresources; /* protects global resources */
206 kmutex_t muxifier; /* single-threads multiplexor creation */
207
208 static void *str_stack_init(netstackid_t stackid, netstack_t *ns);
209 static void str_stack_shutdown(netstackid_t stackid, void *arg);
210 static void str_stack_fini(netstackid_t stackid, void *arg);
211
212 /*
213 * run_queues is no longer used, but is kept in case some 3rd party
214 * module/driver decides to use it.
215 */
216 int run_queues = 0;
217
218 /*
219 * sq_max_size is the depth of the syncq (in number of messages) before
220 * qfill_syncq() starts QFULL'ing destination queues. As its primary
221 * consumer - IP is no longer D_MTPERMOD, but there may be other
222 * modules/drivers depend on this syncq flow control, we prefer to
223 * choose a large number as the default value. For potential
224 * performance gain, this value is tunable in /etc/system.
225 */
226 volatile int sq_max_size = 10000;
227
228 /*
229 * The number of ciputctrl structures per syncq and stream we create when
230 * needed.
231 */
232 int n_ciputctrl;
233 int max_n_ciputctrl = 16;
234 /*
235 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
236 */
237 int min_n_ciputctrl = 2;
238
239 /*
240 * Per-driver/module syncqs
241 * ========================
242 *
243 * For drivers/modules that use PERMOD or outer syncqs we keep a list of
244 * perdm structures, new entries being added (and new syncqs allocated) when
245 * setq() encounters a module/driver with a streamtab that it hasn't seen
246 * before.
247 * The reason for this mechanism is that some modules and drivers share a
248 * common streamtab and it is necessary for those modules and drivers to also
249 * share a common PERMOD syncq.
250 *
251 * perdm_list --> dm_str == streamtab_1
252 * dm_sq == syncq_1
253 * dm_ref
254 * dm_next --> dm_str == streamtab_2
255 * dm_sq == syncq_2
256 * dm_ref
257 * dm_next --> ... NULL
258 *
259 * The dm_ref field is incremented for each new driver/module that takes
260 * a reference to the perdm structure and hence shares the syncq.
261 * References are held in the fmodsw_impl_t structure for each STREAMS module
262 * or the dev_impl array (indexed by device major number) for each driver.
263 *
264 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
265 * ^ ^ ^ ^
266 * | ______________/ | |
267 * | / | |
268 * dev_impl: ...|x|y|... module A module B
269 *
270 * When a module/driver is unloaded the reference count is decremented and,
271 * when it falls to zero, the perdm structure is removed from the list and
272 * the syncq is freed (see rele_dm()).
273 */
274 perdm_t *perdm_list = NULL;
275 static krwlock_t perdm_rwlock;
276 cdevsw_impl_t *devimpl;
277
278 extern struct qinit strdata;
279 extern struct qinit stwdata;
280
281 static void runservice(queue_t *);
282 static void streams_bufcall_service(void);
283 static void streams_qbkgrnd_service(void);
284 static void streams_sqbkgrnd_service(void);
285 static syncq_t *new_syncq(void);
286 static void free_syncq(syncq_t *);
287 static void outer_insert(syncq_t *, syncq_t *);
288 static void outer_remove(syncq_t *, syncq_t *);
289 static void write_now(syncq_t *);
290 static void clr_qfull(queue_t *);
291 static void runbufcalls(void);
292 static void sqenable(syncq_t *);
293 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
294 static void wait_q_syncq(queue_t *);
295 static void backenable_insertedq(queue_t *);
296
297 static void queue_service(queue_t *);
298 static void stream_service(stdata_t *);
299 static void syncq_service(syncq_t *);
300 static void qwriter_outer_service(syncq_t *);
301 static void mblk_free(mblk_t *);
302 #ifdef DEBUG
303 static int qprocsareon(queue_t *);
304 #endif
305
306 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
307 static void reset_nfsrv_ptr(queue_t *, queue_t *);
308 void set_qfull(queue_t *);
309
310 static void sq_run_events(syncq_t *);
311 static int propagate_syncq(queue_t *);
312
313 static void blocksq(syncq_t *, ushort_t, int);
314 static void unblocksq(syncq_t *, ushort_t, int);
315 static int dropsq(syncq_t *, uint16_t);
316 static void emptysq(syncq_t *);
317 static sqlist_t *sqlist_alloc(struct stdata *, int);
318 static void sqlist_free(sqlist_t *);
319 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t);
320 static void sqlist_insert(sqlist_t *, syncq_t *);
321 static void sqlist_insertall(sqlist_t *, queue_t *);
322
323 static void strsetuio(stdata_t *);
324
325 struct kmem_cache *stream_head_cache;
326 struct kmem_cache *queue_cache;
327 struct kmem_cache *syncq_cache;
328 struct kmem_cache *qband_cache;
329 struct kmem_cache *linkinfo_cache;
330 struct kmem_cache *ciputctrl_cache = NULL;
331
332 static linkinfo_t *linkinfo_list;
333
334 /* Global esballoc throttling queue */
335 static esb_queue_t system_esbq;
336
337 /* Array of esballoc throttling queues, of length esbq_nelem */
338 static esb_queue_t *volatile system_esbq_array;
339 static int esbq_nelem;
340 static kmutex_t esbq_lock;
341 static int esbq_log2_cpus_per_q = 0;
342
343 /* Scale the system_esbq length by setting number of CPUs per queue. */
344 uint_t esbq_cpus_per_q = 1;
345
346 /*
347 * esballoc tunable parameters.
348 */
349 int esbq_max_qlen = 0x16; /* throttled queue length */
350 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */
351
352 /*
353 * Routines to handle esballoc queueing.
354 */
355 static void esballoc_process_queue(esb_queue_t *);
356 static void esballoc_enqueue_mblk(mblk_t *);
357 static void esballoc_timer(void *);
358 static void esballoc_set_timer(esb_queue_t *, clock_t);
359 static void esballoc_mblk_free(mblk_t *);
360
361 /*
362 * Qinit structure and Module_info structures
363 * for passthru read and write queues
364 */
365
366 static void pass_wput(queue_t *, mblk_t *);
367 static queue_t *link_addpassthru(stdata_t *);
368 static void link_rempassthru(queue_t *);
369
370 struct module_info passthru_info = {
371 0,
372 "passthru",
373 0,
374 INFPSZ,
375 STRHIGH,
376 STRLOW
377 };
378
379 struct qinit passthru_rinit = {
380 (int (*)())putnext,
381 NULL,
382 NULL,
383 NULL,
384 NULL,
385 &passthru_info,
386 NULL
387 };
388
389 struct qinit passthru_winit = {
390 (int (*)()) pass_wput,
391 NULL,
392 NULL,
393 NULL,
394 NULL,
395 &passthru_info,
396 NULL
397 };
398
399 /*
400 * Verify correctness of list head/tail pointers.
401 */
402 #define LISTCHECK(head, tail, link) { \
403 EQUIV(head, tail); \
404 IMPLY(tail != NULL, tail->link == NULL); \
405 }
406
407 /*
408 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
409 * using a `link' field.
410 */
411 #define ENQUEUE(el, head, tail, link) { \
412 ASSERT(el->link == NULL); \
413 LISTCHECK(head, tail, link); \
414 if (head == NULL) \
415 head = el; \
416 else \
417 tail->link = el; \
418 tail = el; \
419 }
420
421 /*
422 * Dequeue the first element of the list denoted by `head' and `tail' pointers
423 * using a `link' field and put result into `el'.
424 */
425 #define DQ(el, head, tail, link) { \
426 LISTCHECK(head, tail, link); \
427 el = head; \
428 if (head != NULL) { \
429 head = head->link; \
430 if (head == NULL) \
431 tail = NULL; \
432 el->link = NULL; \
433 } \
434 }
435
436 /*
437 * Remove `el' from the list using `chase' and `curr' pointers and return result
438 * in `succeed'.
439 */
440 #define RMQ(el, head, tail, link, chase, curr, succeed) { \
441 LISTCHECK(head, tail, link); \
442 chase = NULL; \
443 succeed = 0; \
444 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
445 chase = curr; \
446 if (curr != NULL) { \
447 succeed = 1; \
448 ASSERT(curr == el); \
449 if (chase != NULL) \
450 chase->link = curr->link; \
451 else \
452 head = curr->link; \
453 curr->link = NULL; \
454 if (curr == tail) \
455 tail = chase; \
456 } \
457 LISTCHECK(head, tail, link); \
458 }
459
460 /* Handling of delayed messages on the inner syncq. */
461
462 /*
463 * DEBUG versions should use function versions (to simplify tracing) and
464 * non-DEBUG kernels should use macro versions.
465 */
466
467 /*
468 * Put a queue on the syncq list of queues.
469 * Assumes SQLOCK held.
470 */
471 #define SQPUT_Q(sq, qp) \
472 { \
473 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
474 if (!(qp->q_sqflags & Q_SQQUEUED)) { \
475 /* The queue should not be linked anywhere */ \
476 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
477 /* Head and tail may only be NULL simultaneously */ \
478 EQUIV(sq->sq_head, sq->sq_tail); \
479 /* Queue may be only enqueued on its syncq */ \
480 ASSERT(sq == qp->q_syncq); \
481 /* Check the correctness of SQ_MESSAGES flag */ \
482 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
483 /* Sanity check first/last elements of the list */ \
484 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
485 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
486 /* \
487 * Sanity check of priority field: empty queue should \
488 * have zero priority \
489 * and nqueues equal to zero. \
490 */ \
491 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
492 /* Sanity check of sq_nqueues field */ \
493 EQUIV(sq->sq_head, sq->sq_nqueues); \
494 if (sq->sq_head == NULL) { \
495 sq->sq_head = sq->sq_tail = qp; \
496 sq->sq_flags |= SQ_MESSAGES; \
497 } else if (qp->q_spri == 0) { \
498 qp->q_sqprev = sq->sq_tail; \
499 sq->sq_tail->q_sqnext = qp; \
500 sq->sq_tail = qp; \
501 } else { \
502 /* \
503 * Put this queue in priority order: higher \
504 * priority gets closer to the head. \
505 */ \
506 queue_t **qpp = &sq->sq_tail; \
507 queue_t *qnext = NULL; \
508 \
509 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
510 qnext = *qpp; \
511 qpp = &(*qpp)->q_sqprev; \
512 } \
513 qp->q_sqnext = qnext; \
514 qp->q_sqprev = *qpp; \
515 if (*qpp != NULL) { \
516 (*qpp)->q_sqnext = qp; \
517 } else { \
518 sq->sq_head = qp; \
519 sq->sq_pri = sq->sq_head->q_spri; \
520 } \
521 *qpp = qp; \
522 } \
523 qp->q_sqflags |= Q_SQQUEUED; \
524 qp->q_sqtstamp = ddi_get_lbolt(); \
525 sq->sq_nqueues++; \
526 } \
527 }
528
529 /*
530 * Remove a queue from the syncq list
531 * Assumes SQLOCK held.
532 */
533 #define SQRM_Q(sq, qp) \
534 { \
535 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
536 ASSERT(qp->q_sqflags & Q_SQQUEUED); \
537 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
538 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
539 /* Check that the queue is actually in the list */ \
540 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
541 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
542 ASSERT(sq->sq_nqueues != 0); \
543 if (qp->q_sqprev == NULL) { \
544 /* First queue on list, make head q_sqnext */ \
545 sq->sq_head = qp->q_sqnext; \
546 } else { \
547 /* Make prev->next == next */ \
548 qp->q_sqprev->q_sqnext = qp->q_sqnext; \
549 } \
550 if (qp->q_sqnext == NULL) { \
551 /* Last queue on list, make tail sqprev */ \
552 sq->sq_tail = qp->q_sqprev; \
553 } else { \
554 /* Make next->prev == prev */ \
555 qp->q_sqnext->q_sqprev = qp->q_sqprev; \
556 } \
557 /* clear out references on this queue */ \
558 qp->q_sqprev = qp->q_sqnext = NULL; \
559 qp->q_sqflags &= ~Q_SQQUEUED; \
560 /* If there is nothing queued, clear SQ_MESSAGES */ \
561 if (sq->sq_head != NULL) { \
562 sq->sq_pri = sq->sq_head->q_spri; \
563 } else { \
564 sq->sq_flags &= ~SQ_MESSAGES; \
565 sq->sq_pri = 0; \
566 } \
567 sq->sq_nqueues--; \
568 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
569 (sq->sq_flags & SQ_QUEUED) == 0); \
570 }
571
572 /* Hide the definition from the header file. */
573 #ifdef SQPUT_MP
574 #undef SQPUT_MP
575 #endif
576
577 /*
578 * Put a message on the queue syncq.
579 * Assumes QLOCK held.
580 */
581 #define SQPUT_MP(qp, mp) \
582 { \
583 ASSERT(MUTEX_HELD(QLOCK(qp))); \
584 ASSERT(qp->q_sqhead == NULL || \
585 (qp->q_sqtail != NULL && \
586 qp->q_sqtail->b_next == NULL)); \
587 qp->q_syncqmsgs++; \
588 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \
589 if (qp->q_sqhead == NULL) { \
590 qp->q_sqhead = qp->q_sqtail = mp; \
591 } else { \
592 qp->q_sqtail->b_next = mp; \
593 qp->q_sqtail = mp; \
594 } \
595 ASSERT(qp->q_syncqmsgs > 0); \
596 set_qfull(qp); \
597 }
598
599 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \
600 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
601 if ((sq)->sq_ciputctrl != NULL) { \
602 int i; \
603 int nlocks = (sq)->sq_nciputctrl; \
604 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
605 ASSERT((sq)->sq_type & SQ_CIPUT); \
606 for (i = 0; i <= nlocks; i++) { \
607 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
608 cip[i].ciputctrl_count |= SQ_FASTPUT; \
609 } \
610 } \
611 }
612
613
614 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \
615 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
616 if ((sq)->sq_ciputctrl != NULL) { \
617 int i; \
618 int nlocks = (sq)->sq_nciputctrl; \
619 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
620 ASSERT((sq)->sq_type & SQ_CIPUT); \
621 for (i = 0; i <= nlocks; i++) { \
622 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
623 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \
624 } \
625 } \
626 }
627
628 /*
629 * Run service procedures for all queues in the stream head.
630 */
631 #define STR_SERVICE(stp, q) { \
632 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \
633 while (stp->sd_qhead != NULL) { \
634 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \
635 ASSERT(stp->sd_nqueues > 0); \
636 stp->sd_nqueues--; \
637 ASSERT(!(q->q_flag & QINSERVICE)); \
638 mutex_exit(&stp->sd_qlock); \
639 queue_service(q); \
640 mutex_enter(&stp->sd_qlock); \
641 } \
642 ASSERT(stp->sd_nqueues == 0); \
643 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \
644 }
645
646 /*
647 * Constructor/destructor routines for the stream head cache
648 */
649 /* ARGSUSED */
650 static int
651 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
652 {
653 stdata_t *stp = buf;
654
655 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
656 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
657 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
658 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
659 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
660 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
661 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
662 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
663 stp->sd_wrq = NULL;
664
665 return (0);
666 }
667
668 /* ARGSUSED */
669 static void
670 stream_head_destructor(void *buf, void *cdrarg)
671 {
672 stdata_t *stp = buf;
673
674 mutex_destroy(&stp->sd_lock);
675 mutex_destroy(&stp->sd_reflock);
676 mutex_destroy(&stp->sd_qlock);
677 cv_destroy(&stp->sd_monitor);
678 cv_destroy(&stp->sd_iocmonitor);
679 cv_destroy(&stp->sd_refmonitor);
680 cv_destroy(&stp->sd_qcv);
681 cv_destroy(&stp->sd_zcopy_wait);
682 }
683
684 /*
685 * Constructor/destructor routines for the queue cache
686 */
687 /* ARGSUSED */
688 static int
689 queue_constructor(void *buf, void *cdrarg, int kmflags)
690 {
691 queinfo_t *qip = buf;
692 queue_t *qp = &qip->qu_rqueue;
693 queue_t *wqp = &qip->qu_wqueue;
694 syncq_t *sq = &qip->qu_syncq;
695
696 qp->q_first = NULL;
697 qp->q_link = NULL;
698 qp->q_count = 0;
699 qp->q_mblkcnt = 0;
700 qp->q_sqhead = NULL;
701 qp->q_sqtail = NULL;
702 qp->q_sqnext = NULL;
703 qp->q_sqprev = NULL;
704 qp->q_sqflags = 0;
705 qp->q_rwcnt = 0;
706 qp->q_spri = 0;
707
708 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
709 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
710
711 wqp->q_first = NULL;
712 wqp->q_link = NULL;
713 wqp->q_count = 0;
714 wqp->q_mblkcnt = 0;
715 wqp->q_sqhead = NULL;
716 wqp->q_sqtail = NULL;
717 wqp->q_sqnext = NULL;
718 wqp->q_sqprev = NULL;
719 wqp->q_sqflags = 0;
720 wqp->q_rwcnt = 0;
721 wqp->q_spri = 0;
722
723 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
724 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
725
726 sq->sq_head = NULL;
727 sq->sq_tail = NULL;
728 sq->sq_evhead = NULL;
729 sq->sq_evtail = NULL;
730 sq->sq_callbpend = NULL;
731 sq->sq_outer = NULL;
732 sq->sq_onext = NULL;
733 sq->sq_oprev = NULL;
734 sq->sq_next = NULL;
735 sq->sq_svcflags = 0;
736 sq->sq_servcount = 0;
737 sq->sq_needexcl = 0;
738 sq->sq_nqueues = 0;
739 sq->sq_pri = 0;
740
741 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
742 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
743 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
744
745 return (0);
746 }
747
748 /* ARGSUSED */
749 static void
750 queue_destructor(void *buf, void *cdrarg)
751 {
752 queinfo_t *qip = buf;
753 queue_t *qp = &qip->qu_rqueue;
754 queue_t *wqp = &qip->qu_wqueue;
755 syncq_t *sq = &qip->qu_syncq;
756
757 ASSERT(qp->q_sqhead == NULL);
758 ASSERT(wqp->q_sqhead == NULL);
759 ASSERT(qp->q_sqnext == NULL);
760 ASSERT(wqp->q_sqnext == NULL);
761 ASSERT(qp->q_rwcnt == 0);
762 ASSERT(wqp->q_rwcnt == 0);
763
764 mutex_destroy(&qp->q_lock);
765 cv_destroy(&qp->q_wait);
766
767 mutex_destroy(&wqp->q_lock);
768 cv_destroy(&wqp->q_wait);
769
770 mutex_destroy(&sq->sq_lock);
771 cv_destroy(&sq->sq_wait);
772 cv_destroy(&sq->sq_exitwait);
773 }
774
775 /*
776 * Constructor/destructor routines for the syncq cache
777 */
778 /* ARGSUSED */
779 static int
780 syncq_constructor(void *buf, void *cdrarg, int kmflags)
781 {
782 syncq_t *sq = buf;
783
784 bzero(buf, sizeof (syncq_t));
785
786 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
787 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
788 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
789
790 return (0);
791 }
792
793 /* ARGSUSED */
794 static void
795 syncq_destructor(void *buf, void *cdrarg)
796 {
797 syncq_t *sq = buf;
798
799 ASSERT(sq->sq_head == NULL);
800 ASSERT(sq->sq_tail == NULL);
801 ASSERT(sq->sq_evhead == NULL);
802 ASSERT(sq->sq_evtail == NULL);
803 ASSERT(sq->sq_callbpend == NULL);
804 ASSERT(sq->sq_callbflags == 0);
805 ASSERT(sq->sq_outer == NULL);
806 ASSERT(sq->sq_onext == NULL);
807 ASSERT(sq->sq_oprev == NULL);
808 ASSERT(sq->sq_next == NULL);
809 ASSERT(sq->sq_needexcl == 0);
810 ASSERT(sq->sq_svcflags == 0);
811 ASSERT(sq->sq_servcount == 0);
812 ASSERT(sq->sq_nqueues == 0);
813 ASSERT(sq->sq_pri == 0);
814 ASSERT(sq->sq_count == 0);
815 ASSERT(sq->sq_rmqcount == 0);
816 ASSERT(sq->sq_cancelid == 0);
817 ASSERT(sq->sq_ciputctrl == NULL);
818 ASSERT(sq->sq_nciputctrl == 0);
819 ASSERT(sq->sq_type == 0);
820 ASSERT(sq->sq_flags == 0);
821
822 mutex_destroy(&sq->sq_lock);
823 cv_destroy(&sq->sq_wait);
824 cv_destroy(&sq->sq_exitwait);
825 }
826
827 /* ARGSUSED */
828 static int
829 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
830 {
831 ciputctrl_t *cip = buf;
832 int i;
833
834 for (i = 0; i < n_ciputctrl; i++) {
835 cip[i].ciputctrl_count = SQ_FASTPUT;
836 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
837 }
838
839 return (0);
840 }
841
842 /* ARGSUSED */
843 static void
844 ciputctrl_destructor(void *buf, void *cdrarg)
845 {
846 ciputctrl_t *cip = buf;
847 int i;
848
849 for (i = 0; i < n_ciputctrl; i++) {
850 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
851 mutex_destroy(&cip[i].ciputctrl_lock);
852 }
853 }
854
855 /*
856 * Init routine run from main at boot time.
857 */
858 void
859 strinit(void)
860 {
861 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
862
863 stream_head_cache = kmem_cache_create("stream_head_cache",
864 sizeof (stdata_t), 0,
865 stream_head_constructor, stream_head_destructor, NULL,
866 NULL, NULL, 0);
867
868 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
869 queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
870
871 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
872 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
873
874 qband_cache = kmem_cache_create("qband_cache",
875 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
876
877 linkinfo_cache = kmem_cache_create("linkinfo_cache",
878 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
879
880 n_ciputctrl = ncpus;
881 n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
882 ASSERT(n_ciputctrl >= 1);
883 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
884 if (n_ciputctrl >= min_n_ciputctrl) {
885 ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
886 sizeof (ciputctrl_t) * n_ciputctrl,
887 sizeof (ciputctrl_t), ciputctrl_constructor,
888 ciputctrl_destructor, NULL, NULL, NULL, 0);
889 }
890
891 streams_taskq = system_taskq;
892
893 if (streams_taskq == NULL)
894 panic("strinit: no memory for streams taskq!");
895
896 bc_bkgrnd_thread = thread_create(NULL, 0,
897 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
898
899 streams_qbkgrnd_thread = thread_create(NULL, 0,
900 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
901
902 streams_sqbkgrnd_thread = thread_create(NULL, 0,
903 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
904
905 /*
906 * Create STREAMS kstats.
907 */
908 str_kstat = kstat_create("streams", 0, "strstat",
909 "net", KSTAT_TYPE_NAMED,
910 sizeof (str_statistics) / sizeof (kstat_named_t),
911 KSTAT_FLAG_VIRTUAL);
912
913 if (str_kstat != NULL) {
914 str_kstat->ks_data = &str_statistics;
915 kstat_install(str_kstat);
916 }
917
918 /*
919 * TPI support routine initialisation.
920 */
921 tpi_init();
922
923 /*
924 * Handle to have autopush and persistent link information per
925 * zone.
926 * Note: uses shutdown hook instead of destroy hook so that the
927 * persistent links can be torn down before the destroy hooks
928 * in the TCP/IP stack are called.
929 */
930 netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
931 str_stack_fini);
932 }
933
934 void
935 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
936 {
937 struct stdata *stp;
938
939 ASSERT(vp->v_stream);
940 stp = vp->v_stream;
941 /* Have to hold sd_lock to prevent siglist from changing */
942 mutex_enter(&stp->sd_lock);
943 if (stp->sd_sigflags & event)
944 strsendsig(stp->sd_siglist, event, band, error);
945 mutex_exit(&stp->sd_lock);
946 }
947
948 /*
949 * Send the "sevent" set of signals to a process.
950 * This might send more than one signal if the process is registered
951 * for multiple events. The caller should pass in an sevent that only
952 * includes the events for which the process has registered.
953 */
954 static void
955 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
956 uchar_t band, int error)
957 {
958 ASSERT(MUTEX_HELD(&proc->p_lock));
959
960 info->si_band = 0;
961 info->si_errno = 0;
962
963 if (sevent & S_ERROR) {
964 sevent &= ~S_ERROR;
965 info->si_code = POLL_ERR;
966 info->si_errno = error;
967 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
968 "strsendsig:proc %p info %p", proc, info);
969 sigaddq(proc, NULL, info, KM_NOSLEEP);
970 info->si_errno = 0;
971 }
972 if (sevent & S_HANGUP) {
973 sevent &= ~S_HANGUP;
974 info->si_code = POLL_HUP;
975 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
976 "strsendsig:proc %p info %p", proc, info);
977 sigaddq(proc, NULL, info, KM_NOSLEEP);
978 }
979 if (sevent & S_HIPRI) {
980 sevent &= ~S_HIPRI;
981 info->si_code = POLL_PRI;
982 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
983 "strsendsig:proc %p info %p", proc, info);
984 sigaddq(proc, NULL, info, KM_NOSLEEP);
985 }
986 if (sevent & S_RDBAND) {
987 sevent &= ~S_RDBAND;
988 if (events & S_BANDURG)
989 sigtoproc(proc, NULL, SIGURG);
990 else
991 sigtoproc(proc, NULL, SIGPOLL);
992 }
993 if (sevent & S_WRBAND) {
994 sevent &= ~S_WRBAND;
995 sigtoproc(proc, NULL, SIGPOLL);
996 }
997 if (sevent & S_INPUT) {
998 sevent &= ~S_INPUT;
999 info->si_code = POLL_IN;
1000 info->si_band = band;
1001 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1002 "strsendsig:proc %p info %p", proc, info);
1003 sigaddq(proc, NULL, info, KM_NOSLEEP);
1004 info->si_band = 0;
1005 }
1006 if (sevent & S_OUTPUT) {
1007 sevent &= ~S_OUTPUT;
1008 info->si_code = POLL_OUT;
1009 info->si_band = band;
1010 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1011 "strsendsig:proc %p info %p", proc, info);
1012 sigaddq(proc, NULL, info, KM_NOSLEEP);
1013 info->si_band = 0;
1014 }
1015 if (sevent & S_MSG) {
1016 sevent &= ~S_MSG;
1017 info->si_code = POLL_MSG;
1018 info->si_band = band;
1019 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1020 "strsendsig:proc %p info %p", proc, info);
1021 sigaddq(proc, NULL, info, KM_NOSLEEP);
1022 info->si_band = 0;
1023 }
1024 if (sevent & S_RDNORM) {
1025 sevent &= ~S_RDNORM;
1026 sigtoproc(proc, NULL, SIGPOLL);
1027 }
1028 if (sevent != 0) {
1029 panic("strsendsig: unknown event(s) %x", sevent);
1030 }
1031 }
1032
1033 /*
1034 * Send SIGPOLL/SIGURG signal to all processes and process groups
1035 * registered on the given signal list that want a signal for at
1036 * least one of the specified events.
1037 *
1038 * Must be called with exclusive access to siglist (caller holding sd_lock).
1039 *
1040 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1041 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1042 * while it is in the siglist.
1043 *
1044 * For performance reasons (MP scalability) the code drops pidlock
1045 * when sending signals to a single process.
1046 * When sending to a process group the code holds
1047 * pidlock to prevent the membership in the process group from changing
1048 * while walking the p_pglink list.
1049 */
1050 void
1051 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1052 {
1053 strsig_t *ssp;
1054 k_siginfo_t info;
1055 struct pid *pidp;
1056 proc_t *proc;
1057
1058 info.si_signo = SIGPOLL;
1059 info.si_errno = 0;
1060 for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1061 int sevent;
1062
1063 sevent = ssp->ss_events & event;
1064 if (sevent == 0)
1065 continue;
1066
1067 if ((pidp = ssp->ss_pidp) == NULL) {
1068 /* pid was released but still on event list */
1069 continue;
1070 }
1071
1072
1073 if (ssp->ss_pid > 0) {
1074 /*
1075 * XXX This unfortunately still generates
1076 * a signal when a fd is closed but
1077 * the proc is active.
1078 */
1079 ASSERT(ssp->ss_pid == pidp->pid_id);
1080
1081 mutex_enter(&pidlock);
1082 proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1083 if (proc == NULL) {
1084 mutex_exit(&pidlock);
1085 continue;
1086 }
1087 mutex_enter(&proc->p_lock);
1088 mutex_exit(&pidlock);
1089 dosendsig(proc, ssp->ss_events, sevent, &info,
1090 band, error);
1091 mutex_exit(&proc->p_lock);
1092 } else {
1093 /*
1094 * Send to process group. Hold pidlock across
1095 * calls to dosendsig().
1096 */
1097 pid_t pgrp = -ssp->ss_pid;
1098
1099 mutex_enter(&pidlock);
1100 proc = pgfind_zone(pgrp, ALL_ZONES);
1101 while (proc != NULL) {
1102 mutex_enter(&proc->p_lock);
1103 dosendsig(proc, ssp->ss_events, sevent,
1104 &info, band, error);
1105 mutex_exit(&proc->p_lock);
1106 proc = proc->p_pglink;
1107 }
1108 mutex_exit(&pidlock);
1109 }
1110 }
1111 }
1112
1113 /*
1114 * Attach a stream device or module.
1115 * qp is a read queue; the new queue goes in so its next
1116 * read ptr is the argument, and the write queue corresponding
1117 * to the argument points to this queue. Return 0 on success,
1118 * or a non-zero errno on failure.
1119 */
1120 int
1121 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1122 boolean_t is_insert)
1123 {
1124 major_t major;
1125 cdevsw_impl_t *dp;
1126 struct streamtab *str;
1127 queue_t *rq;
1128 queue_t *wrq;
1129 uint32_t qflag;
1130 uint32_t sqtype;
1131 perdm_t *dmp;
1132 int error;
1133 int sflag;
1134
1135 rq = allocq();
1136 wrq = _WR(rq);
1137 STREAM(rq) = STREAM(wrq) = STREAM(qp);
1138
1139 if (fp != NULL) {
1140 str = fp->f_str;
1141 qflag = fp->f_qflag;
1142 sqtype = fp->f_sqtype;
1143 dmp = fp->f_dmp;
1144 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1145 sflag = MODOPEN;
1146
1147 /*
1148 * stash away a pointer to the module structure so we can
1149 * unref it in qdetach.
1150 */
1151 rq->q_fp = fp;
1152 } else {
1153 ASSERT(!is_insert);
1154
1155 major = getmajor(*devp);
1156 dp = &devimpl[major];
1157
1158 str = dp->d_str;
1159 ASSERT(str == STREAMSTAB(major));
1160
1161 qflag = dp->d_qflag;
1162 ASSERT(qflag & QISDRV);
1163 sqtype = dp->d_sqtype;
1164
1165 /* create perdm_t if needed */
1166 if (NEED_DM(dp->d_dmp, qflag))
1167 dp->d_dmp = hold_dm(str, qflag, sqtype);
1168
1169 dmp = dp->d_dmp;
1170 sflag = 0;
1171 }
1172
1173 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1174 "qattach:qflag == %X(%X)", qflag, *devp);
1175
1176 /* setq might sleep in allocator - avoid holding locks. */
1177 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1178
1179 /*
1180 * Before calling the module's open routine, set up the q_next
1181 * pointer for inserting a module in the middle of a stream.
1182 *
1183 * Note that we can always set _QINSERTING and set up q_next
1184 * pointer for both inserting and pushing a module. Then there
1185 * is no need for the is_insert parameter. In insertq(), called
1186 * by qprocson(), assume that q_next of the new module always points
1187 * to the correct queue and use it for insertion. Everything should
1188 * work out fine. But in the first release of _I_INSERT, we
1189 * distinguish between inserting and pushing to make sure that
1190 * pushing a module follows the same code path as before.
1191 */
1192 if (is_insert) {
1193 rq->q_flag |= _QINSERTING;
1194 rq->q_next = qp;
1195 }
1196
1197 /*
1198 * If there is an outer perimeter get exclusive access during
1199 * the open procedure. Bump up the reference count on the queue.
1200 */
1201 entersq(rq->q_syncq, SQ_OPENCLOSE);
1202 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1203 if (error != 0)
1204 goto failed;
1205 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1206 ASSERT(qprocsareon(rq));
1207 return (0);
1208
1209 failed:
1210 rq->q_flag &= ~_QINSERTING;
1211 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1212 qprocsoff(rq);
1213 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1214 rq->q_next = wrq->q_next = NULL;
1215 qdetach(rq, 0, 0, crp, B_FALSE);
1216 return (error);
1217 }
1218
1219 /*
1220 * Handle second open of stream. For modules, set the
1221 * last argument to MODOPEN and do not pass any open flags.
1222 * Ignore dummydev since this is not the first open.
1223 */
1224 int
1225 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1226 {
1227 int error;
1228 dev_t dummydev;
1229 queue_t *wqp = _WR(qp);
1230
1231 ASSERT(qp->q_flag & QREADR);
1232 entersq(qp->q_syncq, SQ_OPENCLOSE);
1233
1234 dummydev = *devp;
1235 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1236 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1237 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1238 mutex_enter(&STREAM(qp)->sd_lock);
1239 qp->q_stream->sd_flag |= STREOPENFAIL;
1240 mutex_exit(&STREAM(qp)->sd_lock);
1241 return (error);
1242 }
1243 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1244
1245 /*
1246 * successful open should have done qprocson()
1247 */
1248 ASSERT(qprocsareon(_RD(qp)));
1249 return (0);
1250 }
1251
1252 /*
1253 * Detach a stream module or device.
1254 * If clmode == 1 then the module or driver was opened and its
1255 * close routine must be called. If clmode == 0, the module
1256 * or driver was never opened or the open failed, and so its close
1257 * should not be called.
1258 */
1259 void
1260 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1261 {
1262 queue_t *wqp = _WR(qp);
1263 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1264
1265 if (STREAM_NEEDSERVICE(STREAM(qp)))
1266 stream_runservice(STREAM(qp));
1267
1268 if (clmode) {
1269 /*
1270 * Make sure that all the messages on the write side syncq are
1271 * processed and nothing is left. Since we are closing, no new
1272 * messages may appear there.
1273 */
1274 wait_q_syncq(wqp);
1275
1276 entersq(qp->q_syncq, SQ_OPENCLOSE);
1277 if (is_remove) {
1278 mutex_enter(QLOCK(qp));
1279 qp->q_flag |= _QREMOVING;
1280 mutex_exit(QLOCK(qp));
1281 }
1282 (*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1283 /*
1284 * Check that qprocsoff() was actually called.
1285 */
1286 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1287
1288 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1289 } else {
1290 disable_svc(qp);
1291 }
1292
1293 /*
1294 * Allow any threads blocked in entersq to proceed and discover
1295 * the QWCLOSE is set.
1296 * Note: This assumes that all users of entersq check QWCLOSE.
1297 * Currently runservice is the only entersq that can happen
1298 * after removeq has finished.
1299 * Removeq will have discarded all messages destined to the closing
1300 * pair of queues from the syncq.
1301 * NOTE: Calling a function inside an assert is unconventional.
1302 * However, it does not cause any problem since flush_syncq() does
1303 * not change any state except when it returns non-zero i.e.
1304 * when the assert will trigger.
1305 */
1306 ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1307 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1308 ASSERT((qp->q_flag & QPERMOD) ||
1309 ((qp->q_syncq->sq_head == NULL) &&
1310 (wqp->q_syncq->sq_head == NULL)));
1311
1312 /* release any fmodsw_impl_t structure held on behalf of the queue */
1313 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1314 if (qp->q_fp != NULL)
1315 fmodsw_rele(qp->q_fp);
1316
1317 /* freeq removes us from the outer perimeter if any */
1318 freeq(qp);
1319 }
1320
1321 /* Prevent service procedures from being called */
1322 void
1323 disable_svc(queue_t *qp)
1324 {
1325 queue_t *wqp = _WR(qp);
1326
1327 ASSERT(qp->q_flag & QREADR);
1328 mutex_enter(QLOCK(qp));
1329 qp->q_flag |= QWCLOSE;
1330 mutex_exit(QLOCK(qp));
1331 mutex_enter(QLOCK(wqp));
1332 wqp->q_flag |= QWCLOSE;
1333 mutex_exit(QLOCK(wqp));
1334 }
1335
1336 /* Allow service procedures to be called again */
1337 void
1338 enable_svc(queue_t *qp)
1339 {
1340 queue_t *wqp = _WR(qp);
1341
1342 ASSERT(qp->q_flag & QREADR);
1343 mutex_enter(QLOCK(qp));
1344 qp->q_flag &= ~QWCLOSE;
1345 mutex_exit(QLOCK(qp));
1346 mutex_enter(QLOCK(wqp));
1347 wqp->q_flag &= ~QWCLOSE;
1348 mutex_exit(QLOCK(wqp));
1349 }
1350
1351 /*
1352 * Remove queue from qhead/qtail if it is enabled.
1353 * Only reset QENAB if the queue was removed from the runlist.
1354 * A queue goes through 3 stages:
1355 * It is on the service list and QENAB is set.
1356 * It is removed from the service list but QENAB is still set.
1357 * QENAB gets changed to QINSERVICE.
1358 * QINSERVICE is reset (when the service procedure is done)
1359 * Thus we can not reset QENAB unless we actually removed it from the service
1360 * queue.
1361 */
1362 void
1363 remove_runlist(queue_t *qp)
1364 {
1365 if (qp->q_flag & QENAB && qhead != NULL) {
1366 queue_t *q_chase;
1367 queue_t *q_curr;
1368 int removed;
1369
1370 mutex_enter(&service_queue);
1371 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1372 mutex_exit(&service_queue);
1373 if (removed) {
1374 STRSTAT(qremoved);
1375 qp->q_flag &= ~QENAB;
1376 }
1377 }
1378 }
1379
1380
1381 /*
1382 * Wait for any pending service processing to complete.
1383 * The removal of queues from the runlist is not atomic with the
1384 * clearing of the QENABLED flag and setting the INSERVICE flag.
1385 * consequently it is possible for remove_runlist in strclose
1386 * to not find the queue on the runlist but for it to be QENABLED
1387 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1388 * as well as INSERVICE.
1389 */
1390 void
1391 wait_svc(queue_t *qp)
1392 {
1393 queue_t *wqp = _WR(qp);
1394
1395 ASSERT(qp->q_flag & QREADR);
1396
1397 /*
1398 * Try to remove queues from qhead/qtail list.
1399 */
1400 if (qhead != NULL) {
1401 remove_runlist(qp);
1402 remove_runlist(wqp);
1403 }
1404 /*
1405 * Wait till the syncqs associated with the queue disappear from the
1406 * background processing list.
1407 * This only needs to be done for non-PERMOD perimeters since
1408 * for PERMOD perimeters the syncq may be shared and will only be freed
1409 * when the last module/driver is unloaded.
1410 * If for PERMOD perimeters queue was on the syncq list, removeq()
1411 * should call propagate_syncq() or drain_syncq() for it. Both of these
1412 * functions remove the queue from its syncq list, so sqthread will not
1413 * try to access the queue.
1414 */
1415 if (!(qp->q_flag & QPERMOD)) {
1416 syncq_t *rsq = qp->q_syncq;
1417 syncq_t *wsq = wqp->q_syncq;
1418
1419 /*
1420 * Disable rsq and wsq and wait for any background processing of
1421 * syncq to complete.
1422 */
1423 wait_sq_svc(rsq);
1424 if (wsq != rsq)
1425 wait_sq_svc(wsq);
1426 }
1427
1428 mutex_enter(QLOCK(qp));
1429 while (qp->q_flag & (QINSERVICE|QENAB))
1430 cv_wait(&qp->q_wait, QLOCK(qp));
1431 mutex_exit(QLOCK(qp));
1432 mutex_enter(QLOCK(wqp));
1433 while (wqp->q_flag & (QINSERVICE|QENAB))
1434 cv_wait(&wqp->q_wait, QLOCK(wqp));
1435 mutex_exit(QLOCK(wqp));
1436 }
1437
1438 /*
1439 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1440 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1441 * also be set, and is passed through to allocb_cred_wait().
1442 *
1443 * Returns errno on failure, zero on success.
1444 */
1445 int
1446 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1447 {
1448 mblk_t *tmp;
1449 ssize_t count;
1450 int error = 0;
1451
1452 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1453 (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1454
1455 if (bp->b_datap->db_type == M_IOCTL) {
1456 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1457 } else {
1458 ASSERT(bp->b_datap->db_type == M_COPYIN);
1459 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1460 }
1461 /*
1462 * strdoioctl validates ioc_count, so if this assert fails it
1463 * cannot be due to user error.
1464 */
1465 ASSERT(count >= 0);
1466
1467 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1468 curproc->p_pid)) == NULL) {
1469 return (error);
1470 }
1471 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1472 if (error != 0) {
1473 freeb(tmp);
1474 return (error);
1475 }
1476 DB_CPID(tmp) = curproc->p_pid;
1477 tmp->b_wptr += count;
1478 bp->b_cont = tmp;
1479
1480 return (0);
1481 }
1482
1483 /*
1484 * Copy ioctl data to user-land. Return non-zero errno on failure,
1485 * 0 for success.
1486 */
1487 int
1488 getiocd(mblk_t *bp, char *arg, int copymode)
1489 {
1490 ssize_t count;
1491 size_t n;
1492 int error;
1493
1494 if (bp->b_datap->db_type == M_IOCACK)
1495 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1496 else {
1497 ASSERT(bp->b_datap->db_type == M_COPYOUT);
1498 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1499 }
1500 ASSERT(count >= 0);
1501
1502 for (bp = bp->b_cont; bp && count;
1503 count -= n, bp = bp->b_cont, arg += n) {
1504 n = MIN(count, bp->b_wptr - bp->b_rptr);
1505 error = strcopyout(bp->b_rptr, arg, n, copymode);
1506 if (error)
1507 return (error);
1508 }
1509 ASSERT(count == 0);
1510 return (0);
1511 }
1512
1513 /*
1514 * Allocate a linkinfo entry given the write queue of the
1515 * bottom module of the top stream and the write queue of the
1516 * stream head of the bottom stream.
1517 */
1518 linkinfo_t *
1519 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1520 {
1521 linkinfo_t *linkp;
1522
1523 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1524
1525 linkp->li_lblk.l_qtop = qup;
1526 linkp->li_lblk.l_qbot = qdown;
1527 linkp->li_fpdown = fpdown;
1528
1529 mutex_enter(&strresources);
1530 linkp->li_next = linkinfo_list;
1531 linkp->li_prev = NULL;
1532 if (linkp->li_next)
1533 linkp->li_next->li_prev = linkp;
1534 linkinfo_list = linkp;
1535 linkp->li_lblk.l_index = ++lnk_id;
1536 ASSERT(lnk_id != 0); /* this should never wrap in practice */
1537 mutex_exit(&strresources);
1538
1539 return (linkp);
1540 }
1541
1542 /*
1543 * Free a linkinfo entry.
1544 */
1545 void
1546 lbfree(linkinfo_t *linkp)
1547 {
1548 mutex_enter(&strresources);
1549 if (linkp->li_next)
1550 linkp->li_next->li_prev = linkp->li_prev;
1551 if (linkp->li_prev)
1552 linkp->li_prev->li_next = linkp->li_next;
1553 else
1554 linkinfo_list = linkp->li_next;
1555 mutex_exit(&strresources);
1556
1557 kmem_cache_free(linkinfo_cache, linkp);
1558 }
1559
1560 /*
1561 * Check for a potential linking cycle.
1562 * Return 1 if a link will result in a cycle,
1563 * and 0 otherwise.
1564 */
1565 int
1566 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1567 {
1568 struct mux_node *np;
1569 struct mux_edge *ep;
1570 int i;
1571 major_t lomaj;
1572 major_t upmaj;
1573 /*
1574 * if the lower stream is a pipe/FIFO, return, since link
1575 * cycles can not happen on pipes/FIFOs
1576 */
1577 if (lostp->sd_vnode->v_type == VFIFO)
1578 return (0);
1579
1580 for (i = 0; i < ss->ss_devcnt; i++) {
1581 np = &ss->ss_mux_nodes[i];
1582 MUX_CLEAR(np);
1583 }
1584 lomaj = getmajor(lostp->sd_vnode->v_rdev);
1585 upmaj = getmajor(upstp->sd_vnode->v_rdev);
1586 np = &ss->ss_mux_nodes[lomaj];
1587 for (;;) {
1588 if (!MUX_DIDVISIT(np)) {
1589 if (np->mn_imaj == upmaj)
1590 return (1);
1591 if (np->mn_outp == NULL) {
1592 MUX_VISIT(np);
1593 if (np->mn_originp == NULL)
1594 return (0);
1595 np = np->mn_originp;
1596 continue;
1597 }
1598 MUX_VISIT(np);
1599 np->mn_startp = np->mn_outp;
1600 } else {
1601 if (np->mn_startp == NULL) {
1602 if (np->mn_originp == NULL)
1603 return (0);
1604 else {
1605 np = np->mn_originp;
1606 continue;
1607 }
1608 }
1609 /*
1610 * If ep->me_nodep is a FIFO (me_nodep == NULL),
1611 * ignore the edge and move on. ep->me_nodep gets
1612 * set to NULL in mux_addedge() if it is a FIFO.
1613 *
1614 */
1615 ep = np->mn_startp;
1616 np->mn_startp = ep->me_nextp;
1617 if (ep->me_nodep == NULL)
1618 continue;
1619 ep->me_nodep->mn_originp = np;
1620 np = ep->me_nodep;
1621 }
1622 }
1623 }
1624
1625 /*
1626 * Find linkinfo entry corresponding to the parameters.
1627 */
1628 linkinfo_t *
1629 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1630 {
1631 linkinfo_t *linkp;
1632 struct mux_edge *mep;
1633 struct mux_node *mnp;
1634 queue_t *qup;
1635
1636 mutex_enter(&strresources);
1637 if ((type & LINKTYPEMASK) == LINKNORMAL) {
1638 qup = getendq(stp->sd_wrq);
1639 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1640 if ((qup == linkp->li_lblk.l_qtop) &&
1641 (!index || (index == linkp->li_lblk.l_index))) {
1642 mutex_exit(&strresources);
1643 return (linkp);
1644 }
1645 }
1646 } else {
1647 ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1648 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1649 mep = mnp->mn_outp;
1650 while (mep) {
1651 if ((index == 0) || (index == mep->me_muxid))
1652 break;
1653 mep = mep->me_nextp;
1654 }
1655 if (!mep) {
1656 mutex_exit(&strresources);
1657 return (NULL);
1658 }
1659 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1660 if ((!linkp->li_lblk.l_qtop) &&
1661 (mep->me_muxid == linkp->li_lblk.l_index)) {
1662 mutex_exit(&strresources);
1663 return (linkp);
1664 }
1665 }
1666 }
1667 mutex_exit(&strresources);
1668 return (NULL);
1669 }
1670
1671 /*
1672 * Given a queue ptr, follow the chain of q_next pointers until you reach the
1673 * last queue on the chain and return it.
1674 */
1675 queue_t *
1676 getendq(queue_t *q)
1677 {
1678 ASSERT(q != NULL);
1679 while (_SAMESTR(q))
1680 q = q->q_next;
1681 return (q);
1682 }
1683
1684 /*
1685 * Wait for the syncq count to drop to zero.
1686 * sq could be either outer or inner.
1687 */
1688
1689 static void
1690 wait_syncq(syncq_t *sq)
1691 {
1692 uint16_t count;
1693
1694 mutex_enter(SQLOCK(sq));
1695 count = sq->sq_count;
1696 SQ_PUTLOCKS_ENTER(sq);
1697 SUM_SQ_PUTCOUNTS(sq, count);
1698 while (count != 0) {
1699 sq->sq_flags |= SQ_WANTWAKEUP;
1700 SQ_PUTLOCKS_EXIT(sq);
1701 cv_wait(&sq->sq_wait, SQLOCK(sq));
1702 count = sq->sq_count;
1703 SQ_PUTLOCKS_ENTER(sq);
1704 SUM_SQ_PUTCOUNTS(sq, count);
1705 }
1706 SQ_PUTLOCKS_EXIT(sq);
1707 mutex_exit(SQLOCK(sq));
1708 }
1709
1710 /*
1711 * Wait while there are any messages for the queue in its syncq.
1712 */
1713 static void
1714 wait_q_syncq(queue_t *q)
1715 {
1716 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1717 syncq_t *sq = q->q_syncq;
1718
1719 mutex_enter(SQLOCK(sq));
1720 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1721 sq->sq_flags |= SQ_WANTWAKEUP;
1722 cv_wait(&sq->sq_wait, SQLOCK(sq));
1723 }
1724 mutex_exit(SQLOCK(sq));
1725 }
1726 }
1727
1728
1729 int
1730 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1731 int lhlink)
1732 {
1733 struct stdata *stp;
1734 struct strioctl strioc;
1735 struct linkinfo *linkp;
1736 struct stdata *stpdown;
1737 struct streamtab *str;
1738 queue_t *passq;
1739 syncq_t *passyncq;
1740 queue_t *rq;
1741 cdevsw_impl_t *dp;
1742 uint32_t qflag;
1743 uint32_t sqtype;
1744 perdm_t *dmp;
1745 int error = 0;
1746 netstack_t *ns;
1747 str_stack_t *ss;
1748
1749 stp = vp->v_stream;
1750 TRACE_1(TR_FAC_STREAMS_FR,
1751 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1752 /*
1753 * Test for invalid upper stream
1754 */
1755 if (stp->sd_flag & STRHUP) {
1756 return (ENXIO);
1757 }
1758 if (vp->v_type == VFIFO) {
1759 return (EINVAL);
1760 }
1761 if (stp->sd_strtab == NULL) {
1762 return (EINVAL);
1763 }
1764 if (!stp->sd_strtab->st_muxwinit) {
1765 return (EINVAL);
1766 }
1767 if (fpdown == NULL) {
1768 return (EBADF);
1769 }
1770 ns = netstack_find_by_cred(crp);
1771 ASSERT(ns != NULL);
1772 ss = ns->netstack_str;
1773 ASSERT(ss != NULL);
1774
1775 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1776 netstack_rele(ss->ss_netstack);
1777 return (EINVAL);
1778 }
1779 mutex_enter(&muxifier);
1780 if (stp->sd_flag & STPLEX) {
1781 mutex_exit(&muxifier);
1782 netstack_rele(ss->ss_netstack);
1783 return (ENXIO);
1784 }
1785
1786 /*
1787 * Test for invalid lower stream.
1788 * The check for the v_type != VFIFO and having a major
1789 * number not >= devcnt is done to avoid problems with
1790 * adding mux_node entry past the end of mux_nodes[].
1791 * For FIFO's we don't add an entry so this isn't a
1792 * problem.
1793 */
1794 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1795 (stpdown == stp) || (stpdown->sd_flag &
1796 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1797 ((stpdown->sd_vnode->v_type != VFIFO) &&
1798 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1799 linkcycle(stp, stpdown, ss)) {
1800 mutex_exit(&muxifier);
1801 netstack_rele(ss->ss_netstack);
1802 return (EINVAL);
1803 }
1804 TRACE_1(TR_FAC_STREAMS_FR,
1805 TR_STPDOWN, "stpdown:%p", stpdown);
1806 rq = getendq(stp->sd_wrq);
1807 if (cmd == I_PLINK)
1808 rq = NULL;
1809
1810 linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1811
1812 strioc.ic_cmd = cmd;
1813 strioc.ic_timout = INFTIM;
1814 strioc.ic_len = sizeof (struct linkblk);
1815 strioc.ic_dp = (char *)&linkp->li_lblk;
1816
1817 /*
1818 * STRPLUMB protects plumbing changes and should be set before
1819 * link_addpassthru()/link_rempassthru() are called, so it is set here
1820 * and cleared in the end of mlink when passthru queue is removed.
1821 * Setting of STRPLUMB prevents reopens of the stream while passthru
1822 * queue is in-place (it is not a proper module and doesn't have open
1823 * entry point).
1824 *
1825 * STPLEX prevents any threads from entering the stream from above. It
1826 * can't be set before the call to link_addpassthru() because putnext
1827 * from below may cause stream head I/O routines to be called and these
1828 * routines assert that STPLEX is not set. After link_addpassthru()
1829 * nothing may come from below since the pass queue syncq is blocked.
1830 * Note also that STPLEX should be cleared before the call to
1831 * link_rempassthru() since when messages start flowing to the stream
1832 * head (e.g. because of message propagation from the pass queue) stream
1833 * head I/O routines may be called with STPLEX flag set.
1834 *
1835 * When STPLEX is set, nothing may come into the stream from above and
1836 * it is safe to do a setq which will change stream head. So, the
1837 * correct sequence of actions is:
1838 *
1839 * 1) Set STRPLUMB
1840 * 2) Call link_addpassthru()
1841 * 3) Set STPLEX
1842 * 4) Call setq and update the stream state
1843 * 5) Clear STPLEX
1844 * 6) Call link_rempassthru()
1845 * 7) Clear STRPLUMB
1846 *
1847 * The same sequence applies to munlink() code.
1848 */
1849 mutex_enter(&stpdown->sd_lock);
1850 stpdown->sd_flag |= STRPLUMB;
1851 mutex_exit(&stpdown->sd_lock);
1852 /*
1853 * Add passthru queue below lower mux. This will block
1854 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1855 */
1856 passq = link_addpassthru(stpdown);
1857
1858 mutex_enter(&stpdown->sd_lock);
1859 stpdown->sd_flag |= STPLEX;
1860 mutex_exit(&stpdown->sd_lock);
1861
1862 rq = _RD(stpdown->sd_wrq);
1863 /*
1864 * There may be messages in the streamhead's syncq due to messages
1865 * that arrived before link_addpassthru() was done. To avoid
1866 * background processing of the syncq happening simultaneous with
1867 * setq processing, we disable the streamhead syncq and wait until
1868 * existing background thread finishes working on it.
1869 */
1870 wait_sq_svc(rq->q_syncq);
1871 passyncq = passq->q_syncq;
1872 if (!(passyncq->sq_flags & SQ_BLOCKED))
1873 blocksq(passyncq, SQ_BLOCKED, 0);
1874
1875 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1876 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1877 rq->q_ptr = _WR(rq)->q_ptr = NULL;
1878
1879 /* setq might sleep in allocator - avoid holding locks. */
1880 /* Note: we are holding muxifier here. */
1881
1882 str = stp->sd_strtab;
1883 dp = &devimpl[getmajor(vp->v_rdev)];
1884 ASSERT(dp->d_str == str);
1885
1886 qflag = dp->d_qflag;
1887 sqtype = dp->d_sqtype;
1888
1889 /* create perdm_t if needed */
1890 if (NEED_DM(dp->d_dmp, qflag))
1891 dp->d_dmp = hold_dm(str, qflag, sqtype);
1892
1893 dmp = dp->d_dmp;
1894
1895 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1896 B_TRUE);
1897
1898 /*
1899 * XXX Remove any "odd" messages from the queue.
1900 * Keep only M_DATA, M_PROTO, M_PCPROTO.
1901 */
1902 error = strdoioctl(stp, &strioc, FNATIVE,
1903 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1904 if (error != 0) {
1905 lbfree(linkp);
1906
1907 if (!(passyncq->sq_flags & SQ_BLOCKED))
1908 blocksq(passyncq, SQ_BLOCKED, 0);
1909 /*
1910 * Restore the stream head queue and then remove
1911 * the passq. Turn off STPLEX before we turn on
1912 * the stream by removing the passq.
1913 */
1914 rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1915 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1916 B_TRUE);
1917
1918 mutex_enter(&stpdown->sd_lock);
1919 stpdown->sd_flag &= ~STPLEX;
1920 mutex_exit(&stpdown->sd_lock);
1921
1922 link_rempassthru(passq);
1923
1924 mutex_enter(&stpdown->sd_lock);
1925 stpdown->sd_flag &= ~STRPLUMB;
1926 /* Wakeup anyone waiting for STRPLUMB to clear. */
1927 cv_broadcast(&stpdown->sd_monitor);
1928 mutex_exit(&stpdown->sd_lock);
1929
1930 mutex_exit(&muxifier);
1931 netstack_rele(ss->ss_netstack);
1932 return (error);
1933 }
1934 mutex_enter(&fpdown->f_tlock);
1935 fpdown->f_count++;
1936 mutex_exit(&fpdown->f_tlock);
1937
1938 /*
1939 * if we've made it here the linkage is all set up so we should also
1940 * set up the layered driver linkages
1941 */
1942
1943 ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1944 if (cmd == I_LINK) {
1945 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1946 } else {
1947 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1948 }
1949
1950 link_rempassthru(passq);
1951
1952 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1953
1954 /*
1955 * Mark the upper stream as having dependent links
1956 * so that strclose can clean it up.
1957 */
1958 if (cmd == I_LINK) {
1959 mutex_enter(&stp->sd_lock);
1960 stp->sd_flag |= STRHASLINKS;
1961 mutex_exit(&stp->sd_lock);
1962 }
1963 /*
1964 * Wake up any other processes that may have been
1965 * waiting on the lower stream. These will all
1966 * error out.
1967 */
1968 mutex_enter(&stpdown->sd_lock);
1969 /* The passthru module is removed so we may release STRPLUMB */
1970 stpdown->sd_flag &= ~STRPLUMB;
1971 cv_broadcast(&rq->q_wait);
1972 cv_broadcast(&_WR(rq)->q_wait);
1973 cv_broadcast(&stpdown->sd_monitor);
1974 mutex_exit(&stpdown->sd_lock);
1975 mutex_exit(&muxifier);
1976 *rvalp = linkp->li_lblk.l_index;
1977 netstack_rele(ss->ss_netstack);
1978 return (0);
1979 }
1980
1981 int
1982 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1983 {
1984 int ret;
1985 struct file *fpdown;
1986
1987 fpdown = getf(arg);
1988 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1989 if (fpdown != NULL)
1990 releasef(arg);
1991 return (ret);
1992 }
1993
1994 /*
1995 * Unlink a multiplexor link. Stp is the controlling stream for the
1996 * link, and linkp points to the link's entry in the linkinfo list.
1997 * The muxifier lock must be held on entry and is dropped on exit.
1998 *
1999 * NOTE : Currently it is assumed that mux would process all the messages
2000 * sitting on it's queue before ACKing the UNLINK. It is the responsibility
2001 * of the mux to handle all the messages that arrive before UNLINK.
2002 * If the mux has to send down messages on its lower stream before
2003 * ACKing I_UNLINK, then it *should* know to handle messages even
2004 * after the UNLINK is acked (actually it should be able to handle till we
2005 * re-block the read side of the pass queue here). If the mux does not
2006 * open up the lower stream, any messages that arrive during UNLINK
2007 * will be put in the stream head. In the case of lower stream opening
2008 * up, some messages might land in the stream head depending on when
2009 * the message arrived and when the read side of the pass queue was
2010 * re-blocked.
2011 */
2012 int
2013 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2014 str_stack_t *ss)
2015 {
2016 struct strioctl strioc;
2017 struct stdata *stpdown;
2018 queue_t *rq, *wrq;
2019 queue_t *passq;
2020 syncq_t *passyncq;
2021 int error = 0;
2022 file_t *fpdown;
2023
2024 ASSERT(MUTEX_HELD(&muxifier));
2025
2026 stpdown = linkp->li_fpdown->f_vnode->v_stream;
2027
2028 /*
2029 * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2030 */
2031 mutex_enter(&stpdown->sd_lock);
2032 stpdown->sd_flag |= STRPLUMB;
2033 mutex_exit(&stpdown->sd_lock);
2034
2035 /*
2036 * Add passthru queue below lower mux. This will block
2037 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2038 */
2039 passq = link_addpassthru(stpdown);
2040
2041 if ((flag & LINKTYPEMASK) == LINKNORMAL)
2042 strioc.ic_cmd = I_UNLINK;
2043 else
2044 strioc.ic_cmd = I_PUNLINK;
2045 strioc.ic_timout = INFTIM;
2046 strioc.ic_len = sizeof (struct linkblk);
2047 strioc.ic_dp = (char *)&linkp->li_lblk;
2048
2049 error = strdoioctl(stp, &strioc, FNATIVE,
2050 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2051
2052 /*
2053 * If there was an error and this is not called via strclose,
2054 * return to the user. Otherwise, pretend there was no error
2055 * and close the link.
2056 */
2057 if (error) {
2058 if (flag & LINKCLOSE) {
2059 cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2060 "unlink ioctl, closing anyway (%d)\n", error);
2061 } else {
2062 link_rempassthru(passq);
2063 mutex_enter(&stpdown->sd_lock);
2064 stpdown->sd_flag &= ~STRPLUMB;
2065 cv_broadcast(&stpdown->sd_monitor);
2066 mutex_exit(&stpdown->sd_lock);
2067 mutex_exit(&muxifier);
2068 return (error);
2069 }
2070 }
2071
2072 mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2073 fpdown = linkp->li_fpdown;
2074 lbfree(linkp);
2075
2076 /*
2077 * We go ahead and drop muxifier here--it's a nasty global lock that
2078 * can slow others down. It's okay to since attempts to mlink() this
2079 * stream will be stopped because STPLEX is still set in the stdata
2080 * structure, and munlink() is stopped because mux_rmvedge() and
2081 * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2082 * respectively. Note that we defer the closef() of fpdown until
2083 * after we drop muxifier since strclose() can call munlinkall().
2084 */
2085 mutex_exit(&muxifier);
2086
2087 wrq = stpdown->sd_wrq;
2088 rq = _RD(wrq);
2089
2090 /*
2091 * Get rid of outstanding service procedure runs, before we make
2092 * it a stream head, since a stream head doesn't have any service
2093 * procedure.
2094 */
2095 disable_svc(rq);
2096 wait_svc(rq);
2097
2098 /*
2099 * Since we don't disable the syncq for QPERMOD, we wait for whatever
2100 * is queued up to be finished. mux should take care that nothing is
2101 * send down to this queue. We should do it now as we're going to block
2102 * passyncq if it was unblocked.
2103 */
2104 if (wrq->q_flag & QPERMOD) {
2105 syncq_t *sq = wrq->q_syncq;
2106
2107 mutex_enter(SQLOCK(sq));
2108 while (wrq->q_sqflags & Q_SQQUEUED) {
2109 sq->sq_flags |= SQ_WANTWAKEUP;
2110 cv_wait(&sq->sq_wait, SQLOCK(sq));
2111 }
2112 mutex_exit(SQLOCK(sq));
2113 }
2114 passyncq = passq->q_syncq;
2115 if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2116
2117 syncq_t *sq, *outer;
2118
2119 /*
2120 * Messages could be flowing from underneath. We will
2121 * block the read side of the passq. This would be
2122 * sufficient for QPAIR and QPERQ muxes to ensure
2123 * that no data is flowing up into this queue
2124 * and hence no thread active in this instance of
2125 * lower mux. But for QPERMOD and QMTOUTPERIM there
2126 * could be messages on the inner and outer/inner
2127 * syncqs respectively. We will wait for them to drain.
2128 * Because passq is blocked messages end up in the syncq
2129 * And qfill_syncq could possibly end up setting QFULL
2130 * which will access the rq->q_flag. Hence, we have to
2131 * acquire the QLOCK in setq.
2132 *
2133 * XXX Messages can also flow from top into this
2134 * queue though the unlink is over (Ex. some instance
2135 * in putnext() called from top that has still not
2136 * accessed this queue. And also putq(lowerq) ?).
2137 * Solution : How about blocking the l_qtop queue ?
2138 * Do we really care about such pure D_MP muxes ?
2139 */
2140
2141 blocksq(passyncq, SQ_BLOCKED, 0);
2142
2143 sq = rq->q_syncq;
2144 if ((outer = sq->sq_outer) != NULL) {
2145
2146 /*
2147 * We have to just wait for the outer sq_count
2148 * drop to zero. As this does not prevent new
2149 * messages to enter the outer perimeter, this
2150 * is subject to starvation.
2151 *
2152 * NOTE :Because of blocksq above, messages could
2153 * be in the inner syncq only because of some
2154 * thread holding the outer perimeter exclusively.
2155 * Hence it would be sufficient to wait for the
2156 * exclusive holder of the outer perimeter to drain
2157 * the inner and outer syncqs. But we will not depend
2158 * on this feature and hence check the inner syncqs
2159 * separately.
2160 */
2161 wait_syncq(outer);
2162 }
2163
2164
2165 /*
2166 * There could be messages destined for
2167 * this queue. Let the exclusive holder
2168 * drain it.
2169 */
2170
2171 wait_syncq(sq);
2172 ASSERT((rq->q_flag & QPERMOD) ||
2173 ((rq->q_syncq->sq_head == NULL) &&
2174 (_WR(rq)->q_syncq->sq_head == NULL)));
2175 }
2176
2177 /*
2178 * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2179 * case as we don't disable its syncq or remove it off the syncq
2180 * service list.
2181 */
2182 if (rq->q_flag & QPERMOD) {
2183 syncq_t *sq = rq->q_syncq;
2184
2185 mutex_enter(SQLOCK(sq));
2186 while (rq->q_sqflags & Q_SQQUEUED) {
2187 sq->sq_flags |= SQ_WANTWAKEUP;
2188 cv_wait(&sq->sq_wait, SQLOCK(sq));
2189 }
2190 mutex_exit(SQLOCK(sq));
2191 }
2192
2193 /*
2194 * flush_syncq changes states only when there are some messages to
2195 * free, i.e. when it returns non-zero value to return.
2196 */
2197 ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2198 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2199
2200 /*
2201 * Nobody else should know about this queue now.
2202 * If the mux did not process the messages before
2203 * acking the I_UNLINK, free them now.
2204 */
2205
2206 flushq(rq, FLUSHALL);
2207 flushq(_WR(rq), FLUSHALL);
2208
2209 /*
2210 * Convert the mux lower queue into a stream head queue.
2211 * Turn off STPLEX before we turn on the stream by removing the passq.
2212 */
2213 rq->q_ptr = wrq->q_ptr = stpdown;
2214 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2215
2216 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2217 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2218
2219 enable_svc(rq);
2220
2221 /*
2222 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2223 * needs to be set to prevent reopen() of the stream - such reopen may
2224 * try to call non-existent pass queue open routine and panic.
2225 */
2226 mutex_enter(&stpdown->sd_lock);
2227 stpdown->sd_flag &= ~STPLEX;
2228 mutex_exit(&stpdown->sd_lock);
2229
2230 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2231 ((flag & LINKTYPEMASK) == LINKPERSIST));
2232
2233 /* clean up the layered driver linkages */
2234 if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2235 ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2236 } else {
2237 ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2238 }
2239
2240 link_rempassthru(passq);
2241
2242 /*
2243 * Now all plumbing changes are finished and STRPLUMB is no
2244 * longer needed.
2245 */
2246 mutex_enter(&stpdown->sd_lock);
2247 stpdown->sd_flag &= ~STRPLUMB;
2248 cv_broadcast(&stpdown->sd_monitor);
2249 mutex_exit(&stpdown->sd_lock);
2250
2251 (void) closef(fpdown);
2252 return (0);
2253 }
2254
2255 /*
2256 * Unlink all multiplexor links for which stp is the controlling stream.
2257 * Return 0, or a non-zero errno on failure.
2258 */
2259 int
2260 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2261 {
2262 linkinfo_t *linkp;
2263 int error = 0;
2264
2265 mutex_enter(&muxifier);
2266 while (linkp = findlinks(stp, 0, flag, ss)) {
2267 /*
2268 * munlink() releases the muxifier lock.
2269 */
2270 if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2271 return (error);
2272 mutex_enter(&muxifier);
2273 }
2274 mutex_exit(&muxifier);
2275 return (0);
2276 }
2277
2278 /*
2279 * A multiplexor link has been made. Add an
2280 * edge to the directed graph.
2281 */
2282 void
2283 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2284 {
2285 struct mux_node *np;
2286 struct mux_edge *ep;
2287 major_t upmaj;
2288 major_t lomaj;
2289
2290 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2291 lomaj = getmajor(lostp->sd_vnode->v_rdev);
2292 np = &ss->ss_mux_nodes[upmaj];
2293 if (np->mn_outp) {
2294 ep = np->mn_outp;
2295 while (ep->me_nextp)
2296 ep = ep->me_nextp;
2297 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2298 ep = ep->me_nextp;
2299 } else {
2300 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2301 ep = np->mn_outp;
2302 }
2303 ep->me_nextp = NULL;
2304 ep->me_muxid = muxid;
2305 /*
2306 * Save the dev_t for the purposes of str_stack_shutdown.
2307 * str_stack_shutdown assumes that the device allows reopen, since
2308 * this dev_t is the one after any cloning by xx_open().
2309 * Would prefer finding the dev_t from before any cloning,
2310 * but specfs doesn't retain that.
2311 */
2312 ep->me_dev = upstp->sd_vnode->v_rdev;
2313 if (lostp->sd_vnode->v_type == VFIFO)
2314 ep->me_nodep = NULL;
2315 else
2316 ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2317 }
2318
2319 /*
2320 * A multiplexor link has been removed. Remove the
2321 * edge in the directed graph.
2322 */
2323 void
2324 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2325 {
2326 struct mux_node *np;
2327 struct mux_edge *ep;
2328 struct mux_edge *pep = NULL;
2329 major_t upmaj;
2330
2331 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2332 np = &ss->ss_mux_nodes[upmaj];
2333 ASSERT(np->mn_outp != NULL);
2334 ep = np->mn_outp;
2335 while (ep) {
2336 if (ep->me_muxid == muxid) {
2337 if (pep)
2338 pep->me_nextp = ep->me_nextp;
2339 else
2340 np->mn_outp = ep->me_nextp;
2341 kmem_free(ep, sizeof (struct mux_edge));
2342 return;
2343 }
2344 pep = ep;
2345 ep = ep->me_nextp;
2346 }
2347 ASSERT(0); /* should not reach here */
2348 }
2349
2350 /*
2351 * Translate the device flags (from conf.h) to the corresponding
2352 * qflag and sq_flag (type) values.
2353 */
2354 int
2355 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2356 uint32_t *sqtypep)
2357 {
2358 uint32_t qflag = 0;
2359 uint32_t sqtype = 0;
2360
2361 if (devflag & _D_OLD)
2362 goto bad;
2363
2364 /* Inner perimeter presence and scope */
2365 switch (devflag & D_MTINNER_MASK) {
2366 case D_MP:
2367 qflag |= QMTSAFE;
2368 sqtype |= SQ_CI;
2369 break;
2370 case D_MTPERQ|D_MP:
2371 qflag |= QPERQ;
2372 break;
2373 case D_MTQPAIR|D_MP:
2374 qflag |= QPAIR;
2375 break;
2376 case D_MTPERMOD|D_MP:
2377 qflag |= QPERMOD;
2378 break;
2379 default:
2380 goto bad;
2381 }
2382
2383 /* Outer perimeter */
2384 if (devflag & D_MTOUTPERIM) {
2385 switch (devflag & D_MTINNER_MASK) {
2386 case D_MP:
2387 case D_MTPERQ|D_MP:
2388 case D_MTQPAIR|D_MP:
2389 break;
2390 default:
2391 goto bad;
2392 }
2393 qflag |= QMTOUTPERIM;
2394 }
2395
2396 /* Inner perimeter modifiers */
2397 if (devflag & D_MTINNER_MOD) {
2398 switch (devflag & D_MTINNER_MASK) {
2399 case D_MP:
2400 goto bad;
2401 default:
2402 break;
2403 }
2404 if (devflag & D_MTPUTSHARED)
2405 sqtype |= SQ_CIPUT;
2406 if (devflag & _D_MTOCSHARED) {
2407 /*
2408 * The code in putnext assumes that it has the
2409 * highest concurrency by not checking sq_count.
2410 * Thus _D_MTOCSHARED can only be supported when
2411 * D_MTPUTSHARED is set.
2412 */
2413 if (!(devflag & D_MTPUTSHARED))
2414 goto bad;
2415 sqtype |= SQ_CIOC;
2416 }
2417 if (devflag & _D_MTCBSHARED) {
2418 /*
2419 * The code in putnext assumes that it has the
2420 * highest concurrency by not checking sq_count.
2421 * Thus _D_MTCBSHARED can only be supported when
2422 * D_MTPUTSHARED is set.
2423 */
2424 if (!(devflag & D_MTPUTSHARED))
2425 goto bad;
2426 sqtype |= SQ_CICB;
2427 }
2428 if (devflag & _D_MTSVCSHARED) {
2429 /*
2430 * The code in putnext assumes that it has the
2431 * highest concurrency by not checking sq_count.
2432 * Thus _D_MTSVCSHARED can only be supported when
2433 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2434 * supported only for QPERMOD.
2435 */
2436 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2437 goto bad;
2438 sqtype |= SQ_CISVC;
2439 }
2440 }
2441
2442 /* Default outer perimeter concurrency */
2443 sqtype |= SQ_CO;
2444
2445 /* Outer perimeter modifiers */
2446 if (devflag & D_MTOCEXCL) {
2447 if (!(devflag & D_MTOUTPERIM)) {
2448 /* No outer perimeter */
2449 goto bad;
2450 }
2451 sqtype &= ~SQ_COOC;
2452 }
2453
2454 /* Synchronous Streams extended qinit structure */
2455 if (devflag & D_SYNCSTR)
2456 qflag |= QSYNCSTR;
2457
2458 /*
2459 * Private flag used by a transport module to indicate
2460 * to sockfs that it supports direct-access mode without
2461 * having to go through STREAMS.
2462 */
2463 if (devflag & _D_DIRECT) {
2464 /* Reject unless the module is fully-MT (no perimeter) */
2465 if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2466 goto bad;
2467 qflag |= _QDIRECT;
2468 }
2469
2470 /*
2471 * Private flag used to indicate that a streams module should only
2472 * be pushed once. The TTY streams modules have this flag since if
2473 * libc believes itself to be an xpg4 process then it will
2474 * automatically and unconditionally push them when a PTS device is
2475 * opened. If an application is not aware of this then without this
2476 * flag we would end up with duplicate modules.
2477 */
2478 if (devflag & _D_SINGLE_INSTANCE)
2479 qflag |= _QSINGLE_INSTANCE;
2480
2481 *qflagp = qflag;
2482 *sqtypep = sqtype;
2483 return (0);
2484
2485 bad:
2486 cmn_err(CE_WARN,
2487 "stropen: bad MT flags (0x%x) in driver '%s'",
2488 (int)(qflag & D_MTSAFETY_MASK),
2489 stp->st_rdinit->qi_minfo->mi_idname);
2490
2491 return (EINVAL);
2492 }
2493
2494 /*
2495 * Set the interface values for a pair of queues (qinit structure,
2496 * packet sizes, water marks).
2497 * setq assumes that the caller does not have a claim (entersq or claimq)
2498 * on the queue.
2499 */
2500 void
2501 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2502 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2503 {
2504 queue_t *wq;
2505 syncq_t *sq, *outer;
2506
2507 ASSERT(rq->q_flag & QREADR);
2508 ASSERT((qflag & QMT_TYPEMASK) != 0);
2509 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2510
2511 wq = _WR(rq);
2512 rq->q_qinfo = rinit;
2513 rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2514 rq->q_lowat = rinit->qi_minfo->mi_lowat;
2515 rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2516 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2517 wq->q_qinfo = winit;
2518 wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2519 wq->q_lowat = winit->qi_minfo->mi_lowat;
2520 wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2521 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2522
2523 /* Remove old syncqs */
2524 sq = rq->q_syncq;
2525 outer = sq->sq_outer;
2526 if (outer != NULL) {
2527 ASSERT(wq->q_syncq->sq_outer == outer);
2528 outer_remove(outer, rq->q_syncq);
2529 if (wq->q_syncq != rq->q_syncq)
2530 outer_remove(outer, wq->q_syncq);
2531 }
2532 ASSERT(sq->sq_outer == NULL);
2533 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2534
2535 if (sq != SQ(rq)) {
2536 if (!(rq->q_flag & QPERMOD))
2537 free_syncq(sq);
2538 if (wq->q_syncq == rq->q_syncq)
2539 wq->q_syncq = NULL;
2540 rq->q_syncq = NULL;
2541 }
2542 if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2543 wq->q_syncq != SQ(rq)) {
2544 free_syncq(wq->q_syncq);
2545 wq->q_syncq = NULL;
2546 }
2547 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2548 rq->q_syncq->sq_tail == NULL));
2549 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2550 wq->q_syncq->sq_tail == NULL));
2551
2552 if (!(rq->q_flag & QPERMOD) &&
2553 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2554 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2555 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2556 rq->q_syncq->sq_nciputctrl, 0);
2557 ASSERT(ciputctrl_cache != NULL);
2558 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2559 rq->q_syncq->sq_ciputctrl = NULL;
2560 rq->q_syncq->sq_nciputctrl = 0;
2561 }
2562
2563 if (!(wq->q_flag & QPERMOD) &&
2564 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2565 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2566 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2567 wq->q_syncq->sq_nciputctrl, 0);
2568 ASSERT(ciputctrl_cache != NULL);
2569 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2570 wq->q_syncq->sq_ciputctrl = NULL;
2571 wq->q_syncq->sq_nciputctrl = 0;
2572 }
2573
2574 sq = SQ(rq);
2575 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2576 ASSERT(sq->sq_outer == NULL);
2577 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2578
2579 /*
2580 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2581 * bits in sq_flag based on the sqtype.
2582 */
2583 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2584
2585 rq->q_syncq = wq->q_syncq = sq;
2586 sq->sq_type = sqtype;
2587 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2588
2589 /*
2590 * We are making sq_svcflags zero,
2591 * resetting SQ_DISABLED in case it was set by
2592 * wait_svc() in the munlink path.
2593 *
2594 */
2595 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2596 sq->sq_svcflags = 0;
2597
2598 /*
2599 * We need to acquire the lock here for the mlink and munlink case,
2600 * where canputnext, backenable, etc can access the q_flag.
2601 */
2602 if (lock_needed) {
2603 mutex_enter(QLOCK(rq));
2604 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2605 mutex_exit(QLOCK(rq));
2606 mutex_enter(QLOCK(wq));
2607 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2608 mutex_exit(QLOCK(wq));
2609 } else {
2610 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2611 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2612 }
2613
2614 if (qflag & QPERQ) {
2615 /* Allocate a separate syncq for the write side */
2616 sq = new_syncq();
2617 sq->sq_type = rq->q_syncq->sq_type;
2618 sq->sq_flags = rq->q_syncq->sq_flags;
2619 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2620 sq->sq_oprev == NULL);
2621 wq->q_syncq = sq;
2622 }
2623 if (qflag & QPERMOD) {
2624 sq = dmp->dm_sq;
2625
2626 /*
2627 * Assert that we do have an inner perimeter syncq and that it
2628 * does not have an outer perimeter associated with it.
2629 */
2630 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2631 sq->sq_oprev == NULL);
2632 rq->q_syncq = wq->q_syncq = sq;
2633 }
2634 if (qflag & QMTOUTPERIM) {
2635 outer = dmp->dm_sq;
2636
2637 ASSERT(outer->sq_outer == NULL);
2638 outer_insert(outer, rq->q_syncq);
2639 if (wq->q_syncq != rq->q_syncq)
2640 outer_insert(outer, wq->q_syncq);
2641 }
2642 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2643 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2644 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2645 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2646 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2647
2648 /*
2649 * Initialize struio() types.
2650 */
2651 rq->q_struiot =
2652 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2653 wq->q_struiot =
2654 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2655 }
2656
2657 perdm_t *
2658 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2659 {
2660 syncq_t *sq;
2661 perdm_t **pp;
2662 perdm_t *p;
2663 perdm_t *dmp;
2664
2665 ASSERT(str != NULL);
2666 ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2667
2668 rw_enter(&perdm_rwlock, RW_READER);
2669 for (p = perdm_list; p != NULL; p = p->dm_next) {
2670 if (p->dm_str == str) { /* found one */
2671 atomic_inc_32(&(p->dm_ref));
2672 rw_exit(&perdm_rwlock);
2673 return (p);
2674 }
2675 }
2676 rw_exit(&perdm_rwlock);
2677
2678 sq = new_syncq();
2679 if (qflag & QPERMOD) {
2680 sq->sq_type = sqtype | SQ_PERMOD;
2681 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2682 } else {
2683 ASSERT(qflag & QMTOUTPERIM);
2684 sq->sq_onext = sq->sq_oprev = sq;
2685 }
2686
2687 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2688 dmp->dm_sq = sq;
2689 dmp->dm_str = str;
2690 dmp->dm_ref = 1;
2691 dmp->dm_next = NULL;
2692
2693 rw_enter(&perdm_rwlock, RW_WRITER);
2694 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2695 if (p->dm_str == str) { /* already present */
2696 p->dm_ref++;
2697 rw_exit(&perdm_rwlock);
2698 free_syncq(sq);
2699 kmem_free(dmp, sizeof (perdm_t));
2700 return (p);
2701 }
2702 }
2703
2704 *pp = dmp;
2705 rw_exit(&perdm_rwlock);
2706 return (dmp);
2707 }
2708
2709 void
2710 rele_dm(perdm_t *dmp)
2711 {
2712 perdm_t **pp;
2713 perdm_t *p;
2714
2715 rw_enter(&perdm_rwlock, RW_WRITER);
2716 ASSERT(dmp->dm_ref > 0);
2717
2718 if (--dmp->dm_ref > 0) {
2719 rw_exit(&perdm_rwlock);
2720 return;
2721 }
2722
2723 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2724 if (p == dmp)
2725 break;
2726 ASSERT(p == dmp);
2727 *pp = p->dm_next;
2728 rw_exit(&perdm_rwlock);
2729
2730 /*
2731 * Wait for any background processing that relies on the
2732 * syncq to complete before it is freed.
2733 */
2734 wait_sq_svc(p->dm_sq);
2735 free_syncq(p->dm_sq);
2736 kmem_free(p, sizeof (perdm_t));
2737 }
2738
2739 /*
2740 * Make a protocol message given control and data buffers.
2741 * n.b., this can block; be careful of what locks you hold when calling it.
2742 *
2743 * If sd_maxblk is less than *iosize this routine can fail part way through
2744 * (due to an allocation failure). In this case on return *iosize will contain
2745 * the amount that was consumed. Otherwise *iosize will not be modified
2746 * i.e. it will contain the amount that was consumed.
2747 */
2748 int
2749 strmakemsg(
2750 struct strbuf *mctl,
2751 ssize_t *iosize,
2752 struct uio *uiop,
2753 stdata_t *stp,
2754 int32_t flag,
2755 mblk_t **mpp)
2756 {
2757 mblk_t *mpctl = NULL;
2758 mblk_t *mpdata = NULL;
2759 int error;
2760
2761 ASSERT(uiop != NULL);
2762
2763 *mpp = NULL;
2764 /* Create control part, if any */
2765 if ((mctl != NULL) && (mctl->len >= 0)) {
2766 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2767 if (error)
2768 return (error);
2769 }
2770 /* Create data part, if any */
2771 if (*iosize >= 0) {
2772 error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2773 if (error) {
2774 freemsg(mpctl);
2775 return (error);
2776 }
2777 }
2778 if (mpctl != NULL) {
2779 if (mpdata != NULL)
2780 linkb(mpctl, mpdata);
2781 *mpp = mpctl;
2782 } else {
2783 *mpp = mpdata;
2784 }
2785 return (0);
2786 }
2787
2788 /*
2789 * Make the control part of a protocol message given a control buffer.
2790 * n.b., this can block; be careful of what locks you hold when calling it.
2791 */
2792 int
2793 strmakectl(
2794 struct strbuf *mctl,
2795 int32_t flag,
2796 int32_t fflag,
2797 mblk_t **mpp)
2798 {
2799 mblk_t *bp = NULL;
2800 unsigned char msgtype;
2801 int error = 0;
2802 cred_t *cr = CRED();
2803
2804 /* We do not support interrupt threads using the stream head to send */
2805 ASSERT(cr != NULL);
2806
2807 *mpp = NULL;
2808 /*
2809 * Create control part of message, if any.
2810 */
2811 if ((mctl != NULL) && (mctl->len >= 0)) {
2812 caddr_t base;
2813 int ctlcount;
2814 int allocsz;
2815
2816 if (flag & RS_HIPRI)
2817 msgtype = M_PCPROTO;
2818 else
2819 msgtype = M_PROTO;
2820
2821 ctlcount = mctl->len;
2822 base = mctl->buf;
2823
2824 /*
2825 * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2826 * blocks by increasing the size to something more usable.
2827 */
2828 allocsz = MAX(ctlcount, 64);
2829
2830 /*
2831 * Range checking has already been done; simply try
2832 * to allocate a message block for the ctl part.
2833 */
2834 while ((bp = allocb_cred(allocsz, cr,
2835 curproc->p_pid)) == NULL) {
2836 if (fflag & (FNDELAY|FNONBLOCK))
2837 return (EAGAIN);
2838 if (error = strwaitbuf(allocsz, BPRI_MED))
2839 return (error);
2840 }
2841
2842 bp->b_datap->db_type = msgtype;
2843 if (copyin(base, bp->b_wptr, ctlcount)) {
2844 freeb(bp);
2845 return (EFAULT);
2846 }
2847 bp->b_wptr += ctlcount;
2848 }
2849 *mpp = bp;
2850 return (0);
2851 }
2852
2853 /*
2854 * Make a protocol message given data buffers.
2855 * n.b., this can block; be careful of what locks you hold when calling it.
2856 *
2857 * If sd_maxblk is less than *iosize this routine can fail part way through
2858 * (due to an allocation failure). In this case on return *iosize will contain
2859 * the amount that was consumed. Otherwise *iosize will not be modified
2860 * i.e. it will contain the amount that was consumed.
2861 */
2862 int
2863 strmakedata(
2864 ssize_t *iosize,
2865 struct uio *uiop,
2866 stdata_t *stp,
2867 int32_t flag,
2868 mblk_t **mpp)
2869 {
2870 mblk_t *mp = NULL;
2871 mblk_t *bp;
2872 int wroff = (int)stp->sd_wroff;
2873 int tail_len = (int)stp->sd_tail;
2874 int extra = wroff + tail_len;
2875 int error = 0;
2876 ssize_t maxblk;
2877 ssize_t count = *iosize;
2878 cred_t *cr;
2879
2880 *mpp = NULL;
2881 if (count < 0)
2882 return (0);
2883
2884 /* We do not support interrupt threads using the stream head to send */
2885 cr = CRED();
2886 ASSERT(cr != NULL);
2887
2888 maxblk = stp->sd_maxblk;
2889 if (maxblk == INFPSZ)
2890 maxblk = count;
2891
2892 /*
2893 * Create data part of message, if any.
2894 */
2895 do {
2896 ssize_t size;
2897 dblk_t *dp;
2898
2899 ASSERT(uiop);
2900
2901 size = MIN(count, maxblk);
2902
2903 while ((bp = allocb_cred(size + extra, cr,
2904 curproc->p_pid)) == NULL) {
2905 error = EAGAIN;
2906 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2907 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2908 if (count == *iosize) {
2909 freemsg(mp);
2910 return (error);
2911 } else {
2912 *iosize -= count;
2913 *mpp = mp;
2914 return (0);
2915 }
2916 }
2917 }
2918 dp = bp->b_datap;
2919 dp->db_cpid = curproc->p_pid;
2920 ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2921 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2922
2923 if (flag & STRUIO_POSTPONE) {
2924 /*
2925 * Setup the stream uio portion of the
2926 * dblk for subsequent use by struioget().
2927 */
2928 dp->db_struioflag = STRUIO_SPEC;
2929 dp->db_cksumstart = 0;
2930 dp->db_cksumstuff = 0;
2931 dp->db_cksumend = size;
2932 *(long long *)dp->db_struioun.data = 0ll;
2933 bp->b_wptr += size;
2934 } else {
2935 if (stp->sd_copyflag & STRCOPYCACHED)
2936 uiop->uio_extflg |= UIO_COPY_CACHED;
2937
2938 if (size != 0) {
2939 error = uiomove(bp->b_wptr, size, UIO_WRITE,
2940 uiop);
2941 if (error != 0) {
2942 freeb(bp);
2943 freemsg(mp);
2944 return (error);
2945 }
2946 }
2947 bp->b_wptr += size;
2948
2949 if (stp->sd_wputdatafunc != NULL) {
2950 mblk_t *newbp;
2951
2952 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2953 bp, NULL, NULL, NULL, NULL);
2954 if (newbp == NULL) {
2955 freeb(bp);
2956 freemsg(mp);
2957 return (ECOMM);
2958 }
2959 bp = newbp;
2960 }
2961 }
2962
2963 count -= size;
2964
2965 if (mp == NULL)
2966 mp = bp;
2967 else
2968 linkb(mp, bp);
2969 } while (count > 0);
2970
2971 *mpp = mp;
2972 return (0);
2973 }
2974
2975 /*
2976 * Wait for a buffer to become available. Return non-zero errno
2977 * if not able to wait, 0 if buffer is probably there.
2978 */
2979 int
2980 strwaitbuf(size_t size, int pri)
2981 {
2982 bufcall_id_t id;
2983
2984 mutex_enter(&bcall_monitor);
2985 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2986 &ttoproc(curthread)->p_flag_cv)) == 0) {
2987 mutex_exit(&bcall_monitor);
2988 return (ENOSR);
2989 }
2990 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2991 unbufcall(id);
2992 mutex_exit(&bcall_monitor);
2993 return (EINTR);
2994 }
2995 unbufcall(id);
2996 mutex_exit(&bcall_monitor);
2997 return (0);
2998 }
2999
3000 /*
3001 * This function waits for a read or write event to happen on a stream.
3002 * fmode can specify FNDELAY and/or FNONBLOCK.
3003 * The timeout is in ms with -1 meaning infinite.
3004 * The flag values work as follows:
3005 * READWAIT Check for read side errors, send M_READ
3006 * GETWAIT Check for read side errors, no M_READ
3007 * WRITEWAIT Check for write side errors.
3008 * NOINTR Do not return error if nonblocking or timeout.
3009 * STR_NOERROR Ignore all errors except STPLEX.
3010 * STR_NOSIG Ignore/hold signals during the duration of the call.
3011 * STR_PEEK Pass through the strgeterr().
3012 */
3013 int
3014 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3015 int *done)
3016 {
3017 int slpflg, errs;
3018 int error;
3019 kcondvar_t *sleepon;
3020 mblk_t *mp;
3021 ssize_t *rd_count;
3022 clock_t rval;
3023
3024 ASSERT(MUTEX_HELD(&stp->sd_lock));
3025 if ((flag & READWAIT) || (flag & GETWAIT)) {
3026 slpflg = RSLEEP;
3027 sleepon = &_RD(stp->sd_wrq)->q_wait;
3028 errs = STRDERR|STPLEX;
3029 } else {
3030 slpflg = WSLEEP;
3031 sleepon = &stp->sd_wrq->q_wait;
3032 errs = STWRERR|STRHUP|STPLEX;
3033 }
3034 if (flag & STR_NOERROR)
3035 errs = STPLEX;
3036
3037 if (stp->sd_wakeq & slpflg) {
3038 /*
3039 * A strwakeq() is pending, no need to sleep.
3040 */
3041 stp->sd_wakeq &= ~slpflg;
3042 *done = 0;
3043 return (0);
3044 }
3045
3046 if (stp->sd_flag & errs) {
3047 /*
3048 * Check for errors before going to sleep since the
3049 * caller might not have checked this while holding
3050 * sd_lock.
3051 */
3052 error = strgeterr(stp, errs, (flag & STR_PEEK));
3053 if (error != 0) {
3054 *done = 1;
3055 return (error);
3056 }
3057 }
3058
3059 /*
3060 * If any module downstream has requested read notification
3061 * by setting SNDMREAD flag using M_SETOPTS, send a message
3062 * down stream.
3063 */
3064 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3065 mutex_exit(&stp->sd_lock);
3066 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3067 (flag & STR_NOSIG), &error))) {
3068 mutex_enter(&stp->sd_lock);
3069 *done = 1;
3070 return (error);
3071 }
3072 mp->b_datap->db_type = M_READ;
3073 rd_count = (ssize_t *)mp->b_wptr;
3074 *rd_count = count;
3075 mp->b_wptr += sizeof (ssize_t);
3076 /*
3077 * Send the number of bytes requested by the
3078 * read as the argument to M_READ.
3079 */
3080 stream_willservice(stp);
3081 putnext(stp->sd_wrq, mp);
3082 stream_runservice(stp);
3083 mutex_enter(&stp->sd_lock);
3084
3085 /*
3086 * If any data arrived due to inline processing
3087 * of putnext(), don't sleep.
3088 */
3089 if (_RD(stp->sd_wrq)->q_first != NULL) {
3090 *done = 0;
3091 return (0);
3092 }
3093 }
3094
3095 if (fmode & (FNDELAY|FNONBLOCK)) {
3096 if (!(flag & NOINTR))
3097 error = EAGAIN;
3098 else
3099 error = 0;
3100 *done = 1;
3101 return (error);
3102 }
3103
3104 stp->sd_flag |= slpflg;
3105 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3106 "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3107 stp, flag, count, fmode, done);
3108
3109 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3110 if (rval > 0) {
3111 /* EMPTY */
3112 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3113 "strwaitq awakes(2):%X, %X, %X, %X, %X",
3114 stp, flag, count, fmode, done);
3115 } else if (rval == 0) {
3116 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3117 "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3118 stp, flag, count, fmode, done);
3119 stp->sd_flag &= ~slpflg;
3120 cv_broadcast(sleepon);
3121 if (!(flag & NOINTR))
3122 error = EINTR;
3123 else
3124 error = 0;
3125 *done = 1;
3126 return (error);
3127 } else {
3128 /* timeout */
3129 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3130 "strwaitq timeout:%p, %X, %lX, %X, %p",
3131 stp, flag, count, fmode, done);
3132 *done = 1;
3133 if (!(flag & NOINTR))
3134 return (ETIME);
3135 else
3136 return (0);
3137 }
3138 /*
3139 * If the caller implements delayed errors (i.e. queued after data)
3140 * we can not check for errors here since data as well as an
3141 * error might have arrived at the stream head. We return to
3142 * have the caller check the read queue before checking for errors.
3143 */
3144 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3145 error = strgeterr(stp, errs, (flag & STR_PEEK));
3146 if (error != 0) {
3147 *done = 1;
3148 return (error);
3149 }
3150 }
3151 *done = 0;
3152 return (0);
3153 }
3154
3155 /*
3156 * Perform job control discipline access checks.
3157 * Return 0 for success and the errno for failure.
3158 */
3159
3160 #define cantsend(p, t, sig) \
3161 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3162
3163 int
3164 straccess(struct stdata *stp, enum jcaccess mode)
3165 {
3166 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */
3167 kthread_t *t = curthread;
3168 proc_t *p = ttoproc(t);
3169 sess_t *sp;
3170
3171 ASSERT(mutex_owned(&stp->sd_lock));
3172
3173 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3174 return (0);
3175
3176 mutex_enter(&p->p_lock); /* protects p_pgidp */
3177
3178 for (;;) {
3179 mutex_enter(&p->p_splock); /* protects p->p_sessp */
3180 sp = p->p_sessp;
3181 mutex_enter(&sp->s_lock); /* protects sp->* */
3182
3183 /*
3184 * If this is not the calling process's controlling terminal
3185 * or if the calling process is already in the foreground
3186 * then allow access.
3187 */
3188 if (sp->s_dev != stp->sd_vnode->v_rdev ||
3189 p->p_pgidp == stp->sd_pgidp) {
3190 mutex_exit(&sp->s_lock);
3191 mutex_exit(&p->p_splock);
3192 mutex_exit(&p->p_lock);
3193 return (0);
3194 }
3195
3196 /*
3197 * Check to see if controlling terminal has been deallocated.
3198 */
3199 if (sp->s_vp == NULL) {
3200 if (!cantsend(p, t, SIGHUP))
3201 sigtoproc(p, t, SIGHUP);
3202 mutex_exit(&sp->s_lock);
3203 mutex_exit(&p->p_splock);
3204 mutex_exit(&p->p_lock);
3205 return (EIO);
3206 }
3207
3208 mutex_exit(&sp->s_lock);
3209 mutex_exit(&p->p_splock);
3210
3211 if (mode == JCGETP) {
3212 mutex_exit(&p->p_lock);
3213 return (0);
3214 }
3215
3216 if (mode == JCREAD) {
3217 if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3218 mutex_exit(&p->p_lock);
3219 return (EIO);
3220 }
3221 mutex_exit(&p->p_lock);
3222 mutex_exit(&stp->sd_lock);
3223 pgsignal(p->p_pgidp, SIGTTIN);
3224 mutex_enter(&stp->sd_lock);
3225 mutex_enter(&p->p_lock);
3226 } else { /* mode == JCWRITE or JCSETP */
3227 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3228 cantsend(p, t, SIGTTOU)) {
3229 mutex_exit(&p->p_lock);
3230 return (0);
3231 }
3232 if (p->p_detached) {
3233 mutex_exit(&p->p_lock);
3234 return (EIO);
3235 }
3236 mutex_exit(&p->p_lock);
3237 mutex_exit(&stp->sd_lock);
3238 pgsignal(p->p_pgidp, SIGTTOU);
3239 mutex_enter(&stp->sd_lock);
3240 mutex_enter(&p->p_lock);
3241 }
3242
3243 /*
3244 * We call cv_wait_sig_swap() to cause the appropriate
3245 * action for the jobcontrol signal to take place.
3246 * If the signal is being caught, we will take the
3247 * EINTR error return. Otherwise, the default action
3248 * of causing the process to stop will take place.
3249 * In this case, we rely on the periodic cv_broadcast() on
3250 * &lbolt_cv to wake us up to loop around and test again.
3251 * We can't get here if the signal is ignored or
3252 * if the current thread is blocking the signal.
3253 */
3254 mutex_exit(&stp->sd_lock);
3255 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3256 mutex_exit(&p->p_lock);
3257 mutex_enter(&stp->sd_lock);
3258 return (EINTR);
3259 }
3260 mutex_exit(&p->p_lock);
3261 mutex_enter(&stp->sd_lock);
3262 mutex_enter(&p->p_lock);
3263 }
3264 }
3265
3266 /*
3267 * Return size of message of block type (bp->b_datap->db_type)
3268 */
3269 size_t
3270 xmsgsize(mblk_t *bp)
3271 {
3272 unsigned char type;
3273 size_t count = 0;
3274
3275 type = bp->b_datap->db_type;
3276
3277 for (; bp; bp = bp->b_cont) {
3278 if (type != bp->b_datap->db_type)
3279 break;
3280 ASSERT(bp->b_wptr >= bp->b_rptr);
3281 count += bp->b_wptr - bp->b_rptr;
3282 }
3283 return (count);
3284 }
3285
3286 /*
3287 * Allocate a stream head.
3288 */
3289 struct stdata *
3290 shalloc(queue_t *qp)
3291 {
3292 stdata_t *stp;
3293
3294 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3295
3296 stp->sd_wrq = _WR(qp);
3297 stp->sd_strtab = NULL;
3298 stp->sd_iocid = 0;
3299 stp->sd_mate = NULL;
3300 stp->sd_freezer = NULL;
3301 stp->sd_refcnt = 0;
3302 stp->sd_wakeq = 0;
3303 stp->sd_anchor = 0;
3304 stp->sd_struiowrq = NULL;
3305 stp->sd_struiordq = NULL;
3306 stp->sd_struiodnak = 0;
3307 stp->sd_struionak = NULL;
3308 stp->sd_t_audit_data = NULL;
3309 stp->sd_rput_opt = 0;
3310 stp->sd_wput_opt = 0;
3311 stp->sd_read_opt = 0;
3312 stp->sd_rprotofunc = strrput_proto;
3313 stp->sd_rmiscfunc = strrput_misc;
3314 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3315 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3316 stp->sd_ciputctrl = NULL;
3317 stp->sd_nciputctrl = 0;
3318 stp->sd_qhead = NULL;
3319 stp->sd_qtail = NULL;
3320 stp->sd_servid = NULL;
3321 stp->sd_nqueues = 0;
3322 stp->sd_svcflags = 0;
3323 stp->sd_copyflag = 0;
3324
3325 return (stp);
3326 }
3327
3328 /*
3329 * Free a stream head.
3330 */
3331 void
3332 shfree(stdata_t *stp)
3333 {
3334 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3335
3336 stp->sd_wrq = NULL;
3337
3338 mutex_enter(&stp->sd_qlock);
3339 while (stp->sd_svcflags & STRS_SCHEDULED) {
3340 STRSTAT(strwaits);
3341 cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3342 }
3343 mutex_exit(&stp->sd_qlock);
3344
3345 if (stp->sd_ciputctrl != NULL) {
3346 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3347 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3348 stp->sd_nciputctrl, 0);
3349 ASSERT(ciputctrl_cache != NULL);
3350 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3351 stp->sd_ciputctrl = NULL;
3352 stp->sd_nciputctrl = 0;
3353 }
3354 ASSERT(stp->sd_qhead == NULL);
3355 ASSERT(stp->sd_qtail == NULL);
3356 ASSERT(stp->sd_nqueues == 0);
3357 kmem_cache_free(stream_head_cache, stp);
3358 }
3359
3360 /*
3361 * Allocate a pair of queues and a syncq for the pair
3362 */
3363 queue_t *
3364 allocq(void)
3365 {
3366 queinfo_t *qip;
3367 queue_t *qp, *wqp;
3368 syncq_t *sq;
3369
3370 qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3371
3372 qp = &qip->qu_rqueue;
3373 wqp = &qip->qu_wqueue;
3374 sq = &qip->qu_syncq;
3375
3376 qp->q_last = NULL;
3377 qp->q_next = NULL;
3378 qp->q_ptr = NULL;
3379 qp->q_flag = QUSE | QREADR;
3380 qp->q_bandp = NULL;
3381 qp->q_stream = NULL;
3382 qp->q_syncq = sq;
3383 qp->q_nband = 0;
3384 qp->q_nfsrv = NULL;
3385 qp->q_draining = 0;
3386 qp->q_syncqmsgs = 0;
3387 qp->q_spri = 0;
3388 qp->q_qtstamp = 0;
3389 qp->q_sqtstamp = 0;
3390 qp->q_fp = NULL;
3391
3392 wqp->q_last = NULL;
3393 wqp->q_next = NULL;
3394 wqp->q_ptr = NULL;
3395 wqp->q_flag = QUSE;
3396 wqp->q_bandp = NULL;
3397 wqp->q_stream = NULL;
3398 wqp->q_syncq = sq;
3399 wqp->q_nband = 0;
3400 wqp->q_nfsrv = NULL;
3401 wqp->q_draining = 0;
3402 wqp->q_syncqmsgs = 0;
3403 wqp->q_qtstamp = 0;
3404 wqp->q_sqtstamp = 0;
3405 wqp->q_spri = 0;
3406
3407 sq->sq_count = 0;
3408 sq->sq_rmqcount = 0;
3409 sq->sq_flags = 0;
3410 sq->sq_type = 0;
3411 sq->sq_callbflags = 0;
3412 sq->sq_cancelid = 0;
3413 sq->sq_ciputctrl = NULL;
3414 sq->sq_nciputctrl = 0;
3415 sq->sq_needexcl = 0;
3416 sq->sq_svcflags = 0;
3417
3418 return (qp);
3419 }
3420
3421 /*
3422 * Free a pair of queues and the "attached" syncq.
3423 * Discard any messages left on the syncq(s), remove the syncq(s) from the
3424 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3425 */
3426 void
3427 freeq(queue_t *qp)
3428 {
3429 qband_t *qbp, *nqbp;
3430 syncq_t *sq, *outer;
3431 queue_t *wqp = _WR(qp);
3432
3433 ASSERT(qp->q_flag & QREADR);
3434
3435 /*
3436 * If a previously dispatched taskq job is scheduled to run
3437 * sync_service() or a service routine is scheduled for the
3438 * queues about to be freed, wait here until all service is
3439 * done on the queue and all associated queues and syncqs.
3440 */
3441 wait_svc(qp);
3442
3443 (void) flush_syncq(qp->q_syncq, qp);
3444 (void) flush_syncq(wqp->q_syncq, wqp);
3445 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3446
3447 /*
3448 * Flush the queues before q_next is set to NULL This is needed
3449 * in order to backenable any downstream queue before we go away.
3450 * Note: we are already removed from the stream so that the
3451 * backenabling will not cause any messages to be delivered to our
3452 * put procedures.
3453 */
3454 flushq(qp, FLUSHALL);
3455 flushq(wqp, FLUSHALL);
3456
3457 /* Tidy up - removeq only does a half-remove from stream */
3458 qp->q_next = wqp->q_next = NULL;
3459 ASSERT(!(qp->q_flag & QENAB));
3460 ASSERT(!(wqp->q_flag & QENAB));
3461
3462 outer = qp->q_syncq->sq_outer;
3463 if (outer != NULL) {
3464 outer_remove(outer, qp->q_syncq);
3465 if (wqp->q_syncq != qp->q_syncq)
3466 outer_remove(outer, wqp->q_syncq);
3467 }
3468 /*
3469 * Free any syncqs that are outside what allocq returned.
3470 */
3471 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3472 free_syncq(qp->q_syncq);
3473 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3474 free_syncq(wqp->q_syncq);
3475
3476 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3477 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3478 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3479 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3480 sq = SQ(qp);
3481 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3482 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3483 ASSERT(sq->sq_outer == NULL);
3484 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3485 ASSERT(sq->sq_callbpend == NULL);
3486 ASSERT(sq->sq_needexcl == 0);
3487
3488 if (sq->sq_ciputctrl != NULL) {
3489 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3490 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3491 sq->sq_nciputctrl, 0);
3492 ASSERT(ciputctrl_cache != NULL);
3493 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3494 sq->sq_ciputctrl = NULL;
3495 sq->sq_nciputctrl = 0;
3496 }
3497
3498 ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3499 ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3500 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3501
3502 qp->q_flag &= ~QUSE;
3503 wqp->q_flag &= ~QUSE;
3504
3505 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3506 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3507
3508 qbp = qp->q_bandp;
3509 while (qbp) {
3510 nqbp = qbp->qb_next;
3511 freeband(qbp);
3512 qbp = nqbp;
3513 }
3514 qbp = wqp->q_bandp;
3515 while (qbp) {
3516 nqbp = qbp->qb_next;
3517 freeband(qbp);
3518 qbp = nqbp;
3519 }
3520 kmem_cache_free(queue_cache, qp);
3521 }
3522
3523 /*
3524 * Allocate a qband structure.
3525 */
3526 qband_t *
3527 allocband(void)
3528 {
3529 qband_t *qbp;
3530
3531 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3532 if (qbp == NULL)
3533 return (NULL);
3534
3535 qbp->qb_next = NULL;
3536 qbp->qb_count = 0;
3537 qbp->qb_mblkcnt = 0;
3538 qbp->qb_first = NULL;
3539 qbp->qb_last = NULL;
3540 qbp->qb_flag = 0;
3541
3542 return (qbp);
3543 }
3544
3545 /*
3546 * Free a qband structure.
3547 */
3548 void
3549 freeband(qband_t *qbp)
3550 {
3551 kmem_cache_free(qband_cache, qbp);
3552 }
3553
3554 /*
3555 * Just like putnextctl(9F), except that allocb_wait() is used.
3556 *
3557 * Consolidation Private, and of course only callable from the stream head or
3558 * routines that may block.
3559 */
3560 int
3561 putnextctl_wait(queue_t *q, int type)
3562 {
3563 mblk_t *bp;
3564 int error;
3565
3566 if ((datamsg(type) && (type != M_DELAY)) ||
3567 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3568 return (0);
3569
3570 bp->b_datap->db_type = (unsigned char)type;
3571 putnext(q, bp);
3572 return (1);
3573 }
3574
3575 /*
3576 * Run any possible bufcalls.
3577 */
3578 void
3579 runbufcalls(void)
3580 {
3581 strbufcall_t *bcp;
3582
3583 mutex_enter(&bcall_monitor);
3584 mutex_enter(&strbcall_lock);
3585
3586 if (strbcalls.bc_head) {
3587 size_t count;
3588 int nevent;
3589
3590 /*
3591 * count how many events are on the list
3592 * now so we can check to avoid looping
3593 * in low memory situations
3594 */
3595 nevent = 0;
3596 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3597 nevent++;
3598
3599 /*
3600 * get estimate of available memory from kmem_avail().
3601 * awake all bufcall functions waiting for
3602 * memory whose request could be satisfied
3603 * by 'count' memory and let 'em fight for it.
3604 */
3605 count = kmem_avail();
3606 while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3607 STRSTAT(bufcalls);
3608 --nevent;
3609 if (bcp->bc_size <= count) {
3610 bcp->bc_executor = curthread;
3611 mutex_exit(&strbcall_lock);
3612 (*bcp->bc_func)(bcp->bc_arg);
3613 mutex_enter(&strbcall_lock);
3614 bcp->bc_executor = NULL;
3615 cv_broadcast(&bcall_cv);
3616 strbcalls.bc_head = bcp->bc_next;
3617 kmem_free(bcp, sizeof (strbufcall_t));
3618 } else {
3619 /*
3620 * too big, try again later - note
3621 * that nevent was decremented above
3622 * so we won't retry this one on this
3623 * iteration of the loop
3624 */
3625 if (bcp->bc_next != NULL) {
3626 strbcalls.bc_head = bcp->bc_next;
3627 bcp->bc_next = NULL;
3628 strbcalls.bc_tail->bc_next = bcp;
3629 strbcalls.bc_tail = bcp;
3630 }
3631 }
3632 }
3633 if (strbcalls.bc_head == NULL)
3634 strbcalls.bc_tail = NULL;
3635 }
3636
3637 mutex_exit(&strbcall_lock);
3638 mutex_exit(&bcall_monitor);
3639 }
3640
3641
3642 /*
3643 * Actually run queue's service routine.
3644 */
3645 static void
3646 runservice(queue_t *q)
3647 {
3648 qband_t *qbp;
3649
3650 ASSERT(q->q_qinfo->qi_srvp);
3651 again:
3652 entersq(q->q_syncq, SQ_SVC);
3653 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3654 "runservice starts:%p", q);
3655
3656 if (!(q->q_flag & QWCLOSE))
3657 (*q->q_qinfo->qi_srvp)(q);
3658
3659 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3660 "runservice ends:(%p)", q);
3661
3662 leavesq(q->q_syncq, SQ_SVC);
3663
3664 mutex_enter(QLOCK(q));
3665 if (q->q_flag & QENAB) {
3666 q->q_flag &= ~QENAB;
3667 mutex_exit(QLOCK(q));
3668 goto again;
3669 }
3670 q->q_flag &= ~QINSERVICE;
3671 q->q_flag &= ~QBACK;
3672 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3673 qbp->qb_flag &= ~QB_BACK;
3674 /*
3675 * Wakeup thread waiting for the service procedure
3676 * to be run (strclose and qdetach).
3677 */
3678 cv_broadcast(&q->q_wait);
3679
3680 mutex_exit(QLOCK(q));
3681 }
3682
3683 /*
3684 * Background processing of bufcalls.
3685 */
3686 void
3687 streams_bufcall_service(void)
3688 {
3689 callb_cpr_t cprinfo;
3690
3691 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3692 "streams_bufcall_service");
3693
3694 mutex_enter(&strbcall_lock);
3695
3696 for (;;) {
3697 if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3698 mutex_exit(&strbcall_lock);
3699 runbufcalls();
3700 mutex_enter(&strbcall_lock);
3701 }
3702 if (strbcalls.bc_head != NULL) {
3703 STRSTAT(bcwaits);
3704 /* Wait for memory to become available */
3705 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3706 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3707 SEC_TO_TICK(60), TR_CLOCK_TICK);
3708 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3709 }
3710
3711 /* Wait for new work to arrive */
3712 if (strbcalls.bc_head == NULL) {
3713 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3714 cv_wait(&strbcall_cv, &strbcall_lock);
3715 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3716 }
3717 }
3718 }
3719
3720 /*
3721 * Background processing of streams background tasks which failed
3722 * taskq_dispatch.
3723 */
3724 static void
3725 streams_qbkgrnd_service(void)
3726 {
3727 callb_cpr_t cprinfo;
3728 queue_t *q;
3729
3730 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3731 "streams_bkgrnd_service");
3732
3733 mutex_enter(&service_queue);
3734
3735 for (;;) {
3736 /*
3737 * Wait for work to arrive.
3738 */
3739 while ((freebs_list == NULL) && (qhead == NULL)) {
3740 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3741 cv_wait(&services_to_run, &service_queue);
3742 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3743 }
3744 /*
3745 * Handle all pending freebs requests to free memory.
3746 */
3747 while (freebs_list != NULL) {
3748 mblk_t *mp = freebs_list;
3749 freebs_list = mp->b_next;
3750 mutex_exit(&service_queue);
3751 mblk_free(mp);
3752 mutex_enter(&service_queue);
3753 }
3754 /*
3755 * Run pending queues.
3756 */
3757 while (qhead != NULL) {
3758 DQ(q, qhead, qtail, q_link);
3759 ASSERT(q != NULL);
3760 mutex_exit(&service_queue);
3761 queue_service(q);
3762 mutex_enter(&service_queue);
3763 }
3764 ASSERT(qhead == NULL && qtail == NULL);
3765 }
3766 }
3767
3768 /*
3769 * Background processing of streams background tasks which failed
3770 * taskq_dispatch.
3771 */
3772 static void
3773 streams_sqbkgrnd_service(void)
3774 {
3775 callb_cpr_t cprinfo;
3776 syncq_t *sq;
3777
3778 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3779 "streams_sqbkgrnd_service");
3780
3781 mutex_enter(&service_queue);
3782
3783 for (;;) {
3784 /*
3785 * Wait for work to arrive.
3786 */
3787 while (sqhead == NULL) {
3788 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3789 cv_wait(&syncqs_to_run, &service_queue);
3790 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3791 }
3792
3793 /*
3794 * Run pending syncqs.
3795 */
3796 while (sqhead != NULL) {
3797 DQ(sq, sqhead, sqtail, sq_next);
3798 ASSERT(sq != NULL);
3799 ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3800 mutex_exit(&service_queue);
3801 syncq_service(sq);
3802 mutex_enter(&service_queue);
3803 }
3804 }
3805 }
3806
3807 /*
3808 * Disable the syncq and wait for background syncq processing to complete.
3809 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3810 * list.
3811 */
3812 void
3813 wait_sq_svc(syncq_t *sq)
3814 {
3815 mutex_enter(SQLOCK(sq));
3816 sq->sq_svcflags |= SQ_DISABLED;
3817 if (sq->sq_svcflags & SQ_BGTHREAD) {
3818 syncq_t *sq_chase;
3819 syncq_t *sq_curr;
3820 int removed;
3821
3822 ASSERT(sq->sq_servcount == 1);
3823 mutex_enter(&service_queue);
3824 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3825 mutex_exit(&service_queue);
3826 if (removed) {
3827 sq->sq_svcflags &= ~SQ_BGTHREAD;
3828 sq->sq_servcount = 0;
3829 STRSTAT(sqremoved);
3830 goto done;
3831 }
3832 }
3833 while (sq->sq_servcount != 0) {
3834 sq->sq_flags |= SQ_WANTWAKEUP;
3835 cv_wait(&sq->sq_wait, SQLOCK(sq));
3836 }
3837 done:
3838 mutex_exit(SQLOCK(sq));
3839 }
3840
3841 /*
3842 * Put a syncq on the list of syncq's to be serviced by the sqthread.
3843 * Add the argument to the end of the sqhead list and set the flag
3844 * indicating this syncq has been enabled. If it has already been
3845 * enabled, don't do anything.
3846 * This routine assumes that SQLOCK is held.
3847 * NOTE that the lock order is to have the SQLOCK first,
3848 * so if the service_syncq lock is held, we need to release it
3849 * before acquiring the SQLOCK (mostly relevant for the background
3850 * thread, and this seems to be common among the STREAMS global locks).
3851 * Note that the sq_svcflags are protected by the SQLOCK.
3852 */
3853 void
3854 sqenable(syncq_t *sq)
3855 {
3856 /*
3857 * This is probably not important except for where I believe it
3858 * is being called. At that point, it should be held (and it
3859 * is a pain to release it just for this routine, so don't do
3860 * it).
3861 */
3862 ASSERT(MUTEX_HELD(SQLOCK(sq)));
3863
3864 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3865 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3866
3867 /*
3868 * Do not put on list if background thread is scheduled or
3869 * syncq is disabled.
3870 */
3871 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3872 return;
3873
3874 /*
3875 * Check whether we should enable sq at all.
3876 * Non PERMOD syncqs may be drained by at most one thread.
3877 * PERMOD syncqs may be drained by several threads but we limit the
3878 * total amount to the lesser of
3879 * Number of queues on the squeue and
3880 * Number of CPUs.
3881 */
3882 if (sq->sq_servcount != 0) {
3883 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3884 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3885 STRSTAT(sqtoomany);
3886 return;
3887 }
3888 }
3889
3890 sq->sq_tstamp = ddi_get_lbolt();
3891 STRSTAT(sqenables);
3892
3893 /* Attempt a taskq dispatch */
3894 sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3895 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3896 if (sq->sq_servid != NULL) {
3897 sq->sq_servcount++;
3898 return;
3899 }
3900
3901 /*
3902 * This taskq dispatch failed, but a previous one may have succeeded.
3903 * Don't try to schedule on the background thread whilst there is
3904 * outstanding taskq processing.
3905 */
3906 if (sq->sq_servcount != 0)
3907 return;
3908
3909 /*
3910 * System is low on resources and can't perform a non-sleeping
3911 * dispatch. Schedule the syncq for a background thread and mark the
3912 * syncq to avoid any further taskq dispatch attempts.
3913 */
3914 mutex_enter(&service_queue);
3915 STRSTAT(taskqfails);
3916 ENQUEUE(sq, sqhead, sqtail, sq_next);
3917 sq->sq_svcflags |= SQ_BGTHREAD;
3918 sq->sq_servcount = 1;
3919 cv_signal(&syncqs_to_run);
3920 mutex_exit(&service_queue);
3921 }
3922
3923 /*
3924 * Note: fifo_close() depends on the mblk_t on the queue being freed
3925 * asynchronously. The asynchronous freeing of messages breaks the
3926 * recursive call chain of fifo_close() while there are I_SENDFD type of
3927 * messages referring to other file pointers on the queue. Then when
3928 * closing pipes it can avoid stack overflow in case of daisy-chained
3929 * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3930 * share the same fifolock_t).
3931 *
3932 * No need to kpreempt_disable to access cpu_seqid. If we migrate and
3933 * the esb queue does not match the new CPU, that is OK.
3934 */
3935 void
3936 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3937 {
3938 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3939 esb_queue_t *eqp;
3940
3941 ASSERT(dbp->db_mblk == mp);
3942 ASSERT(qindex < esbq_nelem);
3943
3944 eqp = system_esbq_array;
3945 if (eqp != NULL) {
3946 eqp += qindex;
3947 } else {
3948 mutex_enter(&esbq_lock);
3949 if (kmem_ready && system_esbq_array == NULL)
3950 system_esbq_array = (esb_queue_t *)kmem_zalloc(
3951 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3952 mutex_exit(&esbq_lock);
3953 eqp = system_esbq_array;
3954 if (eqp != NULL)
3955 eqp += qindex;
3956 else
3957 eqp = &system_esbq;
3958 }
3959
3960 /*
3961 * Check data sanity. The dblock should have non-empty free function.
3962 * It is better to panic here then later when the dblock is freed
3963 * asynchronously when the context is lost.
3964 */
3965 if (dbp->db_frtnp->free_func == NULL) {
3966 panic("freebs_enqueue: dblock %p has a NULL free callback",
3967 (void *)dbp);
3968 }
3969
3970 mutex_enter(&eqp->eq_lock);
3971 /* queue the new mblk on the esballoc queue */
3972 if (eqp->eq_head == NULL) {
3973 eqp->eq_head = eqp->eq_tail = mp;
3974 } else {
3975 eqp->eq_tail->b_next = mp;
3976 eqp->eq_tail = mp;
3977 }
3978 eqp->eq_len++;
3979
3980 /* If we're the first thread to reach the threshold, process */
3981 if (eqp->eq_len >= esbq_max_qlen &&
3982 !(eqp->eq_flags & ESBQ_PROCESSING))
3983 esballoc_process_queue(eqp);
3984
3985 esballoc_set_timer(eqp, esbq_timeout);
3986 mutex_exit(&eqp->eq_lock);
3987 }
3988
3989 static void
3990 esballoc_process_queue(esb_queue_t *eqp)
3991 {
3992 mblk_t *mp;
3993
3994 ASSERT(MUTEX_HELD(&eqp->eq_lock));
3995
3996 eqp->eq_flags |= ESBQ_PROCESSING;
3997
3998 do {
3999 /*
4000 * Detach the message chain for processing.
4001 */
4002 mp = eqp->eq_head;
4003 eqp->eq_tail->b_next = NULL;
4004 eqp->eq_head = eqp->eq_tail = NULL;
4005 eqp->eq_len = 0;
4006 mutex_exit(&eqp->eq_lock);
4007
4008 /*
4009 * Process the message chain.
4010 */
4011 esballoc_enqueue_mblk(mp);
4012 mutex_enter(&eqp->eq_lock);
4013 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
4014
4015 eqp->eq_flags &= ~ESBQ_PROCESSING;
4016 }
4017
4018 /*
4019 * taskq callback routine to free esballoced mblk's
4020 */
4021 static void
4022 esballoc_mblk_free(mblk_t *mp)
4023 {
4024 mblk_t *nextmp;
4025
4026 for (; mp != NULL; mp = nextmp) {
4027 nextmp = mp->b_next;
4028 mp->b_next = NULL;
4029 mblk_free(mp);
4030 }
4031 }
4032
4033 static void
4034 esballoc_enqueue_mblk(mblk_t *mp)
4035 {
4036
4037 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4038 TQ_NOSLEEP) == NULL) {
4039 mblk_t *first_mp = mp;
4040 /*
4041 * System is low on resources and can't perform a non-sleeping
4042 * dispatch. Schedule for a background thread.
4043 */
4044 mutex_enter(&service_queue);
4045 STRSTAT(taskqfails);
4046
4047 while (mp->b_next != NULL)
4048 mp = mp->b_next;
4049
4050 mp->b_next = freebs_list;
4051 freebs_list = first_mp;
4052 cv_signal(&services_to_run);
4053 mutex_exit(&service_queue);
4054 }
4055 }
4056
4057 static void
4058 esballoc_timer(void *arg)
4059 {
4060 esb_queue_t *eqp = arg;
4061
4062 mutex_enter(&eqp->eq_lock);
4063 eqp->eq_flags &= ~ESBQ_TIMER;
4064
4065 if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4066 eqp->eq_len > 0)
4067 esballoc_process_queue(eqp);
4068
4069 esballoc_set_timer(eqp, esbq_timeout);
4070 mutex_exit(&eqp->eq_lock);
4071 }
4072
4073 static void
4074 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4075 {
4076 ASSERT(MUTEX_HELD(&eqp->eq_lock));
4077
4078 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4079 (void) timeout(esballoc_timer, eqp, eq_timeout);
4080 eqp->eq_flags |= ESBQ_TIMER;
4081 }
4082 }
4083
4084 /*
4085 * Setup esbq array length based upon NCPU scaled by CPUs per
4086 * queue. Use static system_esbq until kmem_ready and we can
4087 * create an array in freebs_enqueue().
4088 */
4089 void
4090 esballoc_queue_init(void)
4091 {
4092 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4093 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4094 esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4095 system_esbq.eq_len = 0;
4096 system_esbq.eq_head = system_esbq.eq_tail = NULL;
4097 system_esbq.eq_flags = 0;
4098 }
4099
4100 /*
4101 * Set the QBACK or QB_BACK flag in the given queue for
4102 * the given priority band.
4103 */
4104 void
4105 setqback(queue_t *q, unsigned char pri)
4106 {
4107 int i;
4108 qband_t *qbp;
4109 qband_t **qbpp;
4110
4111 ASSERT(MUTEX_HELD(QLOCK(q)));
4112 if (pri != 0) {
4113 if (pri > q->q_nband) {
4114 qbpp = &q->q_bandp;
4115 while (*qbpp)
4116 qbpp = &(*qbpp)->qb_next;
4117 while (pri > q->q_nband) {
4118 if ((*qbpp = allocband()) == NULL) {
4119 cmn_err(CE_WARN,
4120 "setqback: can't allocate qband\n");
4121 return;
4122 }
4123 (*qbpp)->qb_hiwat = q->q_hiwat;
4124 (*qbpp)->qb_lowat = q->q_lowat;
4125 q->q_nband++;
4126 qbpp = &(*qbpp)->qb_next;
4127 }
4128 }
4129 qbp = q->q_bandp;
4130 i = pri;
4131 while (--i)
4132 qbp = qbp->qb_next;
4133 qbp->qb_flag |= QB_BACK;
4134 } else {
4135 q->q_flag |= QBACK;
4136 }
4137 }
4138
4139 int
4140 strcopyin(void *from, void *to, size_t len, int copyflag)
4141 {
4142 if (copyflag & U_TO_K) {
4143 ASSERT((copyflag & K_TO_K) == 0);
4144 if (copyin(from, to, len))
4145 return (EFAULT);
4146 } else {
4147 ASSERT(copyflag & K_TO_K);
4148 bcopy(from, to, len);
4149 }
4150 return (0);
4151 }
4152
4153 int
4154 strcopyout(void *from, void *to, size_t len, int copyflag)
4155 {
4156 if (copyflag & U_TO_K) {
4157 if (copyout(from, to, len))
4158 return (EFAULT);
4159 } else {
4160 ASSERT(copyflag & K_TO_K);
4161 bcopy(from, to, len);
4162 }
4163 return (0);
4164 }
4165
4166 /*
4167 * strsignal_nolock() posts a signal to the process(es) at the stream head.
4168 * It assumes that the stream head lock is already held, whereas strsignal()
4169 * acquires the lock first. This routine was created because a few callers
4170 * release the stream head lock before calling only to re-acquire it after
4171 * it returns.
4172 */
4173 void
4174 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4175 {
4176 ASSERT(MUTEX_HELD(&stp->sd_lock));
4177 switch (sig) {
4178 case SIGPOLL:
4179 if (stp->sd_sigflags & S_MSG)
4180 strsendsig(stp->sd_siglist, S_MSG, band, 0);
4181 break;
4182 default:
4183 if (stp->sd_pgidp)
4184 pgsignal(stp->sd_pgidp, sig);
4185 break;
4186 }
4187 }
4188
4189 void
4190 strsignal(stdata_t *stp, int sig, int32_t band)
4191 {
4192 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4193 "strsignal:%p, %X, %X", stp, sig, band);
4194
4195 mutex_enter(&stp->sd_lock);
4196 switch (sig) {
4197 case SIGPOLL:
4198 if (stp->sd_sigflags & S_MSG)
4199 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4200 break;
4201
4202 default:
4203 if (stp->sd_pgidp) {
4204 pgsignal(stp->sd_pgidp, sig);
4205 }
4206 break;
4207 }
4208 mutex_exit(&stp->sd_lock);
4209 }
4210
4211 void
4212 strhup(stdata_t *stp)
4213 {
4214 ASSERT(mutex_owned(&stp->sd_lock));
4215 pollwakeup(&stp->sd_pollist, POLLHUP);
4216 if (stp->sd_sigflags & S_HANGUP)
4217 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4218 }
4219
4220 /*
4221 * Backenable the first queue upstream from `q' with a service procedure.
4222 */
4223 void
4224 backenable(queue_t *q, uchar_t pri)
4225 {
4226 queue_t *nq;
4227
4228 /*
4229 * Our presence might not prevent other modules in our own
4230 * stream from popping/pushing since the caller of getq might not
4231 * have a claim on the queue (some drivers do a getq on somebody
4232 * else's queue - they know that the queue itself is not going away
4233 * but the framework has to guarantee q_next in that stream).
4234 */
4235 claimstr(q);
4236
4237 /* Find nearest back queue with service proc */
4238 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4239 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4240 }
4241
4242 if (nq) {
4243 kthread_t *freezer;
4244 /*
4245 * backenable can be called either with no locks held
4246 * or with the stream frozen (the latter occurs when a module
4247 * calls rmvq with the stream frozen). If the stream is frozen
4248 * by the caller the caller will hold all qlocks in the stream.
4249 * Note that a frozen stream doesn't freeze a mated stream,
4250 * so we explicitly check for that.
4251 */
4252 freezer = STREAM(q)->sd_freezer;
4253 if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4254 mutex_enter(QLOCK(nq));
4255 }
4256 #ifdef DEBUG
4257 else {
4258 ASSERT(frozenstr(q));
4259 ASSERT(MUTEX_HELD(QLOCK(q)));
4260 ASSERT(MUTEX_HELD(QLOCK(nq)));
4261 }
4262 #endif
4263 setqback(nq, pri);
4264 qenable_locked(nq);
4265 if (freezer != curthread || STREAM(q) != STREAM(nq))
4266 mutex_exit(QLOCK(nq));
4267 }
4268 releasestr(q);
4269 }
4270
4271 /*
4272 * Return the appropriate errno when one of flags_to_check is set
4273 * in sd_flags. Uses the exported error routines if they are set.
4274 * Will return 0 if non error is set (or if the exported error routines
4275 * do not return an error).
4276 *
4277 * If there is both a read and write error to check, we prefer the read error.
4278 * Also, give preference to recorded errno's over the error functions.
4279 * The flags that are handled are:
4280 * STPLEX return EINVAL
4281 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST)
4282 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST)
4283 * STRHUP return sd_werror
4284 *
4285 * If the caller indicates that the operation is a peek, a nonpersistent error
4286 * is not cleared.
4287 */
4288 int
4289 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4290 {
4291 int32_t sd_flag = stp->sd_flag & flags_to_check;
4292 int error = 0;
4293
4294 ASSERT(MUTEX_HELD(&stp->sd_lock));
4295 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4296 if (sd_flag & STPLEX)
4297 error = EINVAL;
4298 else if (sd_flag & STRDERR) {
4299 error = stp->sd_rerror;
4300 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4301 /*
4302 * Read errors are non-persistent i.e. discarded once
4303 * returned to a non-peeking caller,
4304 */
4305 stp->sd_rerror = 0;
4306 stp->sd_flag &= ~STRDERR;
4307 }
4308 if (error == 0 && stp->sd_rderrfunc != NULL) {
4309 int clearerr = 0;
4310
4311 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4312 &clearerr);
4313 if (clearerr) {
4314 stp->sd_flag &= ~STRDERR;
4315 stp->sd_rderrfunc = NULL;
4316 }
4317 }
4318 } else if (sd_flag & STWRERR) {
4319 error = stp->sd_werror;
4320 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4321 /*
4322 * Write errors are non-persistent i.e. discarded once
4323 * returned to a non-peeking caller,
4324 */
4325 stp->sd_werror = 0;
4326 stp->sd_flag &= ~STWRERR;
4327 }
4328 if (error == 0 && stp->sd_wrerrfunc != NULL) {
4329 int clearerr = 0;
4330
4331 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4332 &clearerr);
4333 if (clearerr) {
4334 stp->sd_flag &= ~STWRERR;
4335 stp->sd_wrerrfunc = NULL;
4336 }
4337 }
4338 } else if (sd_flag & STRHUP) {
4339 /* sd_werror set when STRHUP */
4340 error = stp->sd_werror;
4341 }
4342 return (error);
4343 }
4344
4345
4346 /*
4347 * Single-thread open/close/push/pop
4348 * for twisted streams also
4349 */
4350 int
4351 strstartplumb(stdata_t *stp, int flag, int cmd)
4352 {
4353 int waited = 1;
4354 int error = 0;
4355
4356 if (STRMATED(stp)) {
4357 struct stdata *stmatep = stp->sd_mate;
4358
4359 STRLOCKMATES(stp);
4360 while (waited) {
4361 waited = 0;
4362 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4363 if ((cmd == I_POP) &&
4364 (flag & (FNDELAY|FNONBLOCK))) {
4365 STRUNLOCKMATES(stp);
4366 return (EAGAIN);
4367 }
4368 waited = 1;
4369 mutex_exit(&stp->sd_lock);
4370 if (!cv_wait_sig(&stmatep->sd_monitor,
4371 &stmatep->sd_lock)) {
4372 mutex_exit(&stmatep->sd_lock);
4373 return (EINTR);
4374 }
4375 mutex_exit(&stmatep->sd_lock);
4376 STRLOCKMATES(stp);
4377 }
4378 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4379 if ((cmd == I_POP) &&
4380 (flag & (FNDELAY|FNONBLOCK))) {
4381 STRUNLOCKMATES(stp);
4382 return (EAGAIN);
4383 }
4384 waited = 1;
4385 mutex_exit(&stmatep->sd_lock);
4386 if (!cv_wait_sig(&stp->sd_monitor,
4387 &stp->sd_lock)) {
4388 mutex_exit(&stp->sd_lock);
4389 return (EINTR);
4390 }
4391 mutex_exit(&stp->sd_lock);
4392 STRLOCKMATES(stp);
4393 }
4394 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4395 error = strgeterr(stp,
4396 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4397 if (error != 0) {
4398 STRUNLOCKMATES(stp);
4399 return (error);
4400 }
4401 }
4402 }
4403 stp->sd_flag |= STRPLUMB;
4404 STRUNLOCKMATES(stp);
4405 } else {
4406 mutex_enter(&stp->sd_lock);
4407 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4408 if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4409 (flag & (FNDELAY|FNONBLOCK))) {
4410 mutex_exit(&stp->sd_lock);
4411 return (EAGAIN);
4412 }
4413 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4414 mutex_exit(&stp->sd_lock);
4415 return (EINTR);
4416 }
4417 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4418 error = strgeterr(stp,
4419 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4420 if (error != 0) {
4421 mutex_exit(&stp->sd_lock);
4422 return (error);
4423 }
4424 }
4425 }
4426 stp->sd_flag |= STRPLUMB;
4427 mutex_exit(&stp->sd_lock);
4428 }
4429 return (0);
4430 }
4431
4432 /*
4433 * Complete the plumbing operation associated with stream `stp'.
4434 */
4435 void
4436 strendplumb(stdata_t *stp)
4437 {
4438 ASSERT(MUTEX_HELD(&stp->sd_lock));
4439 ASSERT(stp->sd_flag & STRPLUMB);
4440 stp->sd_flag &= ~STRPLUMB;
4441 cv_broadcast(&stp->sd_monitor);
4442 }
4443
4444 /*
4445 * This describes how the STREAMS framework handles synchronization
4446 * during open/push and close/pop.
4447 * The key interfaces for open and close are qprocson and qprocsoff,
4448 * respectively. While the close case in general is harder both open
4449 * have close have significant similarities.
4450 *
4451 * During close the STREAMS framework has to both ensure that there
4452 * are no stale references to the queue pair (and syncq) that
4453 * are being closed and also provide the guarantees that are documented
4454 * in qprocsoff(9F).
4455 * If there are stale references to the queue that is closing it can
4456 * result in kernel memory corruption or kernel panics.
4457 *
4458 * Note that is it up to the module/driver to ensure that it itself
4459 * does not have any stale references to the closing queues once its close
4460 * routine returns. This includes:
4461 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4462 * associated with the queues. For timeout and bufcall callbacks the
4463 * module/driver also has to ensure (or wait for) any callbacks that
4464 * are in progress.
4465 * - If the module/driver is using esballoc it has to ensure that any
4466 * esballoc free functions do not refer to a queue that has closed.
4467 * (Note that in general the close routine can not wait for the esballoc'ed
4468 * messages to be freed since that can cause a deadlock.)
4469 * - Cancelling any interrupts that refer to the closing queues and
4470 * also ensuring that there are no interrupts in progress that will
4471 * refer to the closing queues once the close routine returns.
4472 * - For multiplexors removing any driver global state that refers to
4473 * the closing queue and also ensuring that there are no threads in
4474 * the multiplexor that has picked up a queue pointer but not yet
4475 * finished using it.
4476 *
4477 * In addition, a driver/module can only reference the q_next pointer
4478 * in its open, close, put, or service procedures or in a
4479 * qtimeout/qbufcall callback procedure executing "on" the correct
4480 * stream. Thus it can not reference the q_next pointer in an interrupt
4481 * routine or a timeout, bufcall or esballoc callback routine. Likewise
4482 * it can not reference q_next of a different queue e.g. in a mux that
4483 * passes messages from one queues put/service procedure to another queue.
4484 * In all the cases when the driver/module can not access the q_next
4485 * field it must use the *next* versions e.g. canputnext instead of
4486 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4487 *
4488 *
4489 * Assuming that the driver/module conforms to the above constraints
4490 * the STREAMS framework has to avoid stale references to q_next for all
4491 * the framework internal cases which include (but are not limited to):
4492 * - Threads in canput/canputnext/backenable and elsewhere that are
4493 * walking q_next.
4494 * - Messages on a syncq that have a reference to the queue through b_queue.
4495 * - Messages on an outer perimeter (syncq) that have a reference to the
4496 * queue through b_queue.
4497 * - Threads that use q_nfsrv (e.g. canput) to find a queue.
4498 * Note that only canput and bcanput use q_nfsrv without any locking.
4499 *
4500 * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4501 * after qprocsoff returns, the framework has to ensure that no threads can
4502 * enter the put or service routines for the closing read or write-side queue.
4503 * In addition to preventing "direct" entry into the put procedures
4504 * the framework also has to prevent messages being drained from
4505 * the syncq or the outer perimeter.
4506 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4507 * mechanism to prevent qwriter(PERIM_OUTER) from running after
4508 * qprocsoff has returned.
4509 * Note that if a module/driver uses put(9F) on one of its own queues
4510 * it is up to the module/driver to ensure that the put() doesn't
4511 * get called when the queue is closing.
4512 *
4513 *
4514 * The framework aspects of the above "contract" is implemented by
4515 * qprocsoff, removeq, and strlock:
4516 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4517 * entering the service procedures.
4518 * - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4519 * canputnext, backenable etc from dereferencing the q_next that will
4520 * soon change.
4521 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4522 * or other q_next walker that uses claimstr/releasestr to finish.
4523 * - optionally for every syncq in the stream strlock acquires all the
4524 * sq_lock's and waits for all sq_counts to drop to a value that indicates
4525 * that no thread executes in the put or service procedures and that no
4526 * thread is draining into the module/driver. This ensures that no
4527 * open, close, put, service, or qtimeout/qbufcall callback procedure is
4528 * currently executing hence no such thread can end up with the old stale
4529 * q_next value and no canput/backenable can have the old stale
4530 * q_nfsrv/q_next.
4531 * - qdetach (wait_svc) makes sure that any scheduled or running threads
4532 * have either finished or observed the QWCLOSE flag and gone away.
4533 */
4534
4535
4536 /*
4537 * Get all the locks necessary to change q_next.
4538 *
4539 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4540 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4541 * the only threads inside the syncq are threads currently calling removeq().
4542 * Since threads calling removeq() are in the process of removing their queues
4543 * from the stream, we do not need to worry about them accessing a stale q_next
4544 * pointer and thus we do not need to wait for them to exit (in fact, waiting
4545 * for them can cause deadlock).
4546 *
4547 * This routine is subject to starvation since it does not set any flag to
4548 * prevent threads from entering a module in the stream (i.e. sq_count can
4549 * increase on some syncq while it is waiting on some other syncq).
4550 *
4551 * Assumes that only one thread attempts to call strlock for a given
4552 * stream. If this is not the case the two threads would deadlock.
4553 * This assumption is guaranteed since strlock is only called by insertq
4554 * and removeq and streams plumbing changes are single-threaded for
4555 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4556 *
4557 * For pipes, it is not difficult to atomically designate a pair of streams
4558 * to be mated. Once mated atomically by the framework the twisted pair remain
4559 * configured that way until dismantled atomically by the framework.
4560 * When plumbing takes place on a twisted stream it is necessary to ensure that
4561 * this operation is done exclusively on the twisted stream since two such
4562 * operations, each initiated on different ends of the pipe will deadlock
4563 * waiting for each other to complete.
4564 *
4565 * On entry, no locks should be held.
4566 * The locks acquired and held by strlock depends on a few factors.
4567 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4568 * and held on exit and all sq_count are at an acceptable level.
4569 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4570 * sd_refcnt being zero.
4571 */
4572
4573 static void
4574 strlock(struct stdata *stp, sqlist_t *sqlist)
4575 {
4576 syncql_t *sql, *sql2;
4577 retry:
4578 /*
4579 * Wait for any claimstr to go away.
4580 */
4581 if (STRMATED(stp)) {
4582 struct stdata *stp1, *stp2;
4583
4584 STRLOCKMATES(stp);
4585 /*
4586 * Note that the selection of locking order is not
4587 * important, just that they are always acquired in
4588 * the same order. To assure this, we choose this
4589 * order based on the value of the pointer, and since
4590 * the pointer will not change for the life of this
4591 * pair, we will always grab the locks in the same
4592 * order (and hence, prevent deadlocks).
4593 */
4594 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4595 stp1 = stp;
4596 stp2 = stp->sd_mate;
4597 } else {
4598 stp2 = stp;
4599 stp1 = stp->sd_mate;
4600 }
4601 mutex_enter(&stp1->sd_reflock);
4602 if (stp1->sd_refcnt > 0) {
4603 STRUNLOCKMATES(stp);
4604 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4605 mutex_exit(&stp1->sd_reflock);
4606 goto retry;
4607 }
4608 mutex_enter(&stp2->sd_reflock);
4609 if (stp2->sd_refcnt > 0) {
4610 STRUNLOCKMATES(stp);
4611 mutex_exit(&stp1->sd_reflock);
4612 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4613 mutex_exit(&stp2->sd_reflock);
4614 goto retry;
4615 }
4616 STREAM_PUTLOCKS_ENTER(stp1);
4617 STREAM_PUTLOCKS_ENTER(stp2);
4618 } else {
4619 mutex_enter(&stp->sd_lock);
4620 mutex_enter(&stp->sd_reflock);
4621 while (stp->sd_refcnt > 0) {
4622 mutex_exit(&stp->sd_lock);
4623 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4624 if (mutex_tryenter(&stp->sd_lock) == 0) {
4625 mutex_exit(&stp->sd_reflock);
4626 mutex_enter(&stp->sd_lock);
4627 mutex_enter(&stp->sd_reflock);
4628 }
4629 }
4630 STREAM_PUTLOCKS_ENTER(stp);
4631 }
4632
4633 if (sqlist == NULL)
4634 return;
4635
4636 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4637 syncq_t *sq = sql->sql_sq;
4638 uint16_t count;
4639
4640 mutex_enter(SQLOCK(sq));
4641 count = sq->sq_count;
4642 ASSERT(sq->sq_rmqcount <= count);
4643 SQ_PUTLOCKS_ENTER(sq);
4644 SUM_SQ_PUTCOUNTS(sq, count);
4645 if (count == sq->sq_rmqcount)
4646 continue;
4647
4648 /* Failed - drop all locks that we have acquired so far */
4649 if (STRMATED(stp)) {
4650 STREAM_PUTLOCKS_EXIT(stp);
4651 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4652 STRUNLOCKMATES(stp);
4653 mutex_exit(&stp->sd_reflock);
4654 mutex_exit(&stp->sd_mate->sd_reflock);
4655 } else {
4656 STREAM_PUTLOCKS_EXIT(stp);
4657 mutex_exit(&stp->sd_lock);
4658 mutex_exit(&stp->sd_reflock);
4659 }
4660 for (sql2 = sqlist->sqlist_head; sql2 != sql;
4661 sql2 = sql2->sql_next) {
4662 SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4663 mutex_exit(SQLOCK(sql2->sql_sq));
4664 }
4665
4666 /*
4667 * The wait loop below may starve when there are many threads
4668 * claiming the syncq. This is especially a problem with permod
4669 * syncqs (IP). To lessen the impact of the problem we increment
4670 * sq_needexcl and clear fastbits so that putnexts will slow
4671 * down and call sqenable instead of draining right away.
4672 */
4673 sq->sq_needexcl++;
4674 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4675 while (count > sq->sq_rmqcount) {
4676 sq->sq_flags |= SQ_WANTWAKEUP;
4677 SQ_PUTLOCKS_EXIT(sq);
4678 cv_wait(&sq->sq_wait, SQLOCK(sq));
4679 count = sq->sq_count;
4680 SQ_PUTLOCKS_ENTER(sq);
4681 SUM_SQ_PUTCOUNTS(sq, count);
4682 }
4683 sq->sq_needexcl--;
4684 if (sq->sq_needexcl == 0)
4685 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4686 SQ_PUTLOCKS_EXIT(sq);
4687 ASSERT(count == sq->sq_rmqcount);
4688 mutex_exit(SQLOCK(sq));
4689 goto retry;
4690 }
4691 }
4692
4693 /*
4694 * Drop all the locks that strlock acquired.
4695 */
4696 static void
4697 strunlock(struct stdata *stp, sqlist_t *sqlist)
4698 {
4699 syncql_t *sql;
4700
4701 if (STRMATED(stp)) {
4702 STREAM_PUTLOCKS_EXIT(stp);
4703 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4704 STRUNLOCKMATES(stp);
4705 mutex_exit(&stp->sd_reflock);
4706 mutex_exit(&stp->sd_mate->sd_reflock);
4707 } else {
4708 STREAM_PUTLOCKS_EXIT(stp);
4709 mutex_exit(&stp->sd_lock);
4710 mutex_exit(&stp->sd_reflock);
4711 }
4712
4713 if (sqlist == NULL)
4714 return;
4715
4716 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4717 SQ_PUTLOCKS_EXIT(sql->sql_sq);
4718 mutex_exit(SQLOCK(sql->sql_sq));
4719 }
4720 }
4721
4722 /*
4723 * When the module has service procedure, we need check if the next
4724 * module which has service procedure is in flow control to trigger
4725 * the backenable.
4726 */
4727 static void
4728 backenable_insertedq(queue_t *q)
4729 {
4730 qband_t *qbp;
4731
4732 claimstr(q);
4733 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4734 if (q->q_next->q_nfsrv->q_flag & QWANTW)
4735 backenable(q, 0);
4736
4737 qbp = q->q_next->q_nfsrv->q_bandp;
4738 for (; qbp != NULL; qbp = qbp->qb_next)
4739 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4740 backenable(q, qbp->qb_first->b_band);
4741 }
4742 releasestr(q);
4743 }
4744
4745 /*
4746 * Given two read queues, insert a new single one after another.
4747 *
4748 * This routine acquires all the necessary locks in order to change
4749 * q_next and related pointer using strlock().
4750 * It depends on the stream head ensuring that there are no concurrent
4751 * insertq or removeq on the same stream. The stream head ensures this
4752 * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4753 *
4754 * Note that no syncq locks are held during the q_next change. This is
4755 * applied to all streams since, unlike removeq, there is no problem of stale
4756 * pointers when adding a module to the stream. Thus drivers/modules that do a
4757 * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4758 * applied this optimization to all streams.
4759 */
4760 void
4761 insertq(struct stdata *stp, queue_t *new)
4762 {
4763 queue_t *after;
4764 queue_t *wafter;
4765 queue_t *wnew = _WR(new);
4766 boolean_t have_fifo = B_FALSE;
4767
4768 if (new->q_flag & _QINSERTING) {
4769 ASSERT(stp->sd_vnode->v_type != VFIFO);
4770 after = new->q_next;
4771 wafter = _WR(new->q_next);
4772 } else {
4773 after = _RD(stp->sd_wrq);
4774 wafter = stp->sd_wrq;
4775 }
4776
4777 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4778 "insertq:%p, %p", after, new);
4779 ASSERT(after->q_flag & QREADR);
4780 ASSERT(new->q_flag & QREADR);
4781
4782 strlock(stp, NULL);
4783
4784 /* Do we have a FIFO? */
4785 if (wafter->q_next == after) {
4786 have_fifo = B_TRUE;
4787 wnew->q_next = new;
4788 } else {
4789 wnew->q_next = wafter->q_next;
4790 }
4791 new->q_next = after;
4792
4793 set_nfsrv_ptr(new, wnew, after, wafter);
4794 /*
4795 * set_nfsrv_ptr() needs to know if this is an insertion or not,
4796 * so only reset this flag after calling it.
4797 */
4798 new->q_flag &= ~_QINSERTING;
4799
4800 if (have_fifo) {
4801 wafter->q_next = wnew;
4802 } else {
4803 if (wafter->q_next)
4804 _OTHERQ(wafter->q_next)->q_next = new;
4805 wafter->q_next = wnew;
4806 }
4807
4808 set_qend(new);
4809 /* The QEND flag might have to be updated for the upstream guy */
4810 set_qend(after);
4811
4812 ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4813 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4814 ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4815 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4816 strsetuio(stp);
4817
4818 /*
4819 * If this was a module insertion, bump the push count.
4820 */
4821 if (!(new->q_flag & QISDRV))
4822 stp->sd_pushcnt++;
4823
4824 strunlock(stp, NULL);
4825
4826 /* check if the write Q needs backenable */
4827 backenable_insertedq(wnew);
4828
4829 /* check if the read Q needs backenable */
4830 backenable_insertedq(new);
4831 }
4832
4833 /*
4834 * Given a read queue, unlink it from any neighbors.
4835 *
4836 * This routine acquires all the necessary locks in order to
4837 * change q_next and related pointers and also guard against
4838 * stale references (e.g. through q_next) to the queue that
4839 * is being removed. It also plays part of the role in ensuring
4840 * that the module's/driver's put procedure doesn't get called
4841 * after qprocsoff returns.
4842 *
4843 * Removeq depends on the stream head ensuring that there are
4844 * no concurrent insertq or removeq on the same stream. The
4845 * stream head ensures this using the flags STWOPEN, STRCLOSE and
4846 * STRPLUMB.
4847 *
4848 * The set of locks needed to remove the queue is different in
4849 * different cases:
4850 *
4851 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4852 * waiting for the syncq reference count to drop to 0 indicating that no
4853 * non-close threads are present anywhere in the stream. This ensures that any
4854 * module/driver can reference q_next in its open, close, put, or service
4855 * procedures.
4856 *
4857 * The sq_rmqcount counter tracks the number of threads inside removeq().
4858 * strlock() ensures that there is either no threads executing inside perimeter
4859 * or there is only a thread calling qprocsoff().
4860 *
4861 * strlock() compares the value of sq_count with the number of threads inside
4862 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4863 * any threads waiting in strlock() when the sq_rmqcount increases.
4864 */
4865
4866 void
4867 removeq(queue_t *qp)
4868 {
4869 queue_t *wqp = _WR(qp);
4870 struct stdata *stp = STREAM(qp);
4871 sqlist_t *sqlist = NULL;
4872 boolean_t isdriver;
4873 int moved;
4874 syncq_t *sq = qp->q_syncq;
4875 syncq_t *wsq = wqp->q_syncq;
4876
4877 ASSERT(stp);
4878
4879 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4880 "removeq:%p %p", qp, wqp);
4881 ASSERT(qp->q_flag&QREADR);
4882
4883 /*
4884 * For queues using Synchronous streams, we must wait for all threads in
4885 * rwnext() to drain out before proceeding.
4886 */
4887 if (qp->q_flag & QSYNCSTR) {
4888 /* First, we need wakeup any threads blocked in rwnext() */
4889 mutex_enter(SQLOCK(sq));
4890 if (sq->sq_flags & SQ_WANTWAKEUP) {
4891 sq->sq_flags &= ~SQ_WANTWAKEUP;
4892 cv_broadcast(&sq->sq_wait);
4893 }
4894 mutex_exit(SQLOCK(sq));
4895
4896 if (wsq != sq) {
4897 mutex_enter(SQLOCK(wsq));
4898 if (wsq->sq_flags & SQ_WANTWAKEUP) {
4899 wsq->sq_flags &= ~SQ_WANTWAKEUP;
4900 cv_broadcast(&wsq->sq_wait);
4901 }
4902 mutex_exit(SQLOCK(wsq));
4903 }
4904
4905 mutex_enter(QLOCK(qp));
4906 while (qp->q_rwcnt > 0) {
4907 qp->q_flag |= QWANTRMQSYNC;
4908 cv_wait(&qp->q_wait, QLOCK(qp));
4909 }
4910 mutex_exit(QLOCK(qp));
4911
4912 mutex_enter(QLOCK(wqp));
4913 while (wqp->q_rwcnt > 0) {
4914 wqp->q_flag |= QWANTRMQSYNC;
4915 cv_wait(&wqp->q_wait, QLOCK(wqp));
4916 }
4917 mutex_exit(QLOCK(wqp));
4918 }
4919
4920 mutex_enter(SQLOCK(sq));
4921 sq->sq_rmqcount++;
4922 if (sq->sq_flags & SQ_WANTWAKEUP) {
4923 sq->sq_flags &= ~SQ_WANTWAKEUP;
4924 cv_broadcast(&sq->sq_wait);
4925 }
4926 mutex_exit(SQLOCK(sq));
4927
4928 isdriver = (qp->q_flag & QISDRV);
4929
4930 sqlist = sqlist_build(qp, stp, STRMATED(stp));
4931 strlock(stp, sqlist);
4932
4933 reset_nfsrv_ptr(qp, wqp);
4934
4935 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4936 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4937 /* Do we have a FIFO? */
4938 if (wqp->q_next == qp) {
4939 stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4940 } else {
4941 if (wqp->q_next)
4942 backq(qp)->q_next = qp->q_next;
4943 if (qp->q_next)
4944 backq(wqp)->q_next = wqp->q_next;
4945 }
4946
4947 /* The QEND flag might have to be updated for the upstream guy */
4948 if (qp->q_next)
4949 set_qend(qp->q_next);
4950
4951 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4952 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4953
4954 /*
4955 * Move any messages destined for the put procedures to the next
4956 * syncq in line. Otherwise free them.
4957 */
4958 moved = 0;
4959 /*
4960 * Quick check to see whether there are any messages or events.
4961 */
4962 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4963 moved += propagate_syncq(qp);
4964 if (wqp->q_syncqmsgs != 0 ||
4965 (wqp->q_syncq->sq_flags & SQ_EVENTS))
4966 moved += propagate_syncq(wqp);
4967
4968 strsetuio(stp);
4969
4970 /*
4971 * If this was a module removal, decrement the push count.
4972 */
4973 if (!isdriver)
4974 stp->sd_pushcnt--;
4975
4976 strunlock(stp, sqlist);
4977 sqlist_free(sqlist);
4978
4979 /*
4980 * Make sure any messages that were propagated are drained.
4981 * Also clear any QFULL bit caused by messages that were propagated.
4982 */
4983
4984 if (qp->q_next != NULL) {
4985 clr_qfull(qp);
4986 /*
4987 * For the driver calling qprocsoff, propagate_syncq
4988 * frees all the messages instead of putting it in
4989 * the stream head
4990 */
4991 if (!isdriver && (moved > 0))
4992 emptysq(qp->q_next->q_syncq);
4993 }
4994 if (wqp->q_next != NULL) {
4995 clr_qfull(wqp);
4996 /*
4997 * We come here for any pop of a module except for the
4998 * case of driver being removed. We don't call emptysq
4999 * if we did not move any messages. This will avoid holding
5000 * PERMOD syncq locks in emptysq
5001 */
5002 if (moved > 0)
5003 emptysq(wqp->q_next->q_syncq);
5004 }
5005
5006 mutex_enter(SQLOCK(sq));
5007 sq->sq_rmqcount--;
5008 mutex_exit(SQLOCK(sq));
5009 }
5010
5011 /*
5012 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
5013 * SQ_WRITER) on a syncq.
5014 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5015 * sync queue and waits until sq_count reaches maxcnt.
5016 *
5017 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5018 * does not care about putnext threads that are in the middle of calling put
5019 * entry points.
5020 *
5021 * This routine is used for both inner and outer syncqs.
5022 */
5023 static void
5024 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5025 {
5026 uint16_t count = 0;
5027
5028 mutex_enter(SQLOCK(sq));
5029 /*
5030 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5031 * SQ_FROZEN will be set if there is a frozen stream that has a
5032 * queue which also refers to this "shared" syncq.
5033 * SQ_BLOCKED will be set if there is "off" queue which also
5034 * refers to this "shared" syncq.
5035 */
5036 if (maxcnt != -1) {
5037 count = sq->sq_count;
5038 SQ_PUTLOCKS_ENTER(sq);
5039 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5040 SUM_SQ_PUTCOUNTS(sq, count);
5041 }
5042 sq->sq_needexcl++;
5043 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5044
5045 while ((sq->sq_flags & flag) ||
5046 (maxcnt != -1 && count > (unsigned)maxcnt)) {
5047 sq->sq_flags |= SQ_WANTWAKEUP;
5048 if (maxcnt != -1) {
5049 SQ_PUTLOCKS_EXIT(sq);
5050 }
5051 cv_wait(&sq->sq_wait, SQLOCK(sq));
5052 if (maxcnt != -1) {
5053 count = sq->sq_count;
5054 SQ_PUTLOCKS_ENTER(sq);
5055 SUM_SQ_PUTCOUNTS(sq, count);
5056 }
5057 }
5058 sq->sq_needexcl--;
5059 sq->sq_flags |= flag;
5060 ASSERT(maxcnt == -1 || count == maxcnt);
5061 if (maxcnt != -1) {
5062 if (sq->sq_needexcl == 0) {
5063 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5064 }
5065 SQ_PUTLOCKS_EXIT(sq);
5066 } else if (sq->sq_needexcl == 0) {
5067 SQ_PUTCOUNT_SETFAST(sq);
5068 }
5069
5070 mutex_exit(SQLOCK(sq));
5071 }
5072
5073 /*
5074 * Reset a flag that was set with blocksq.
5075 *
5076 * Can not use this routine to reset SQ_WRITER.
5077 *
5078 * If "isouter" is set then the syncq is assumed to be an outer perimeter
5079 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5080 * to handle the queued qwriter operations.
5081 *
5082 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5083 * sq_putlocks are used.
5084 */
5085 static void
5086 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5087 {
5088 uint16_t flags;
5089
5090 mutex_enter(SQLOCK(sq));
5091 ASSERT(resetflag != SQ_WRITER);
5092 ASSERT(sq->sq_flags & resetflag);
5093 flags = sq->sq_flags & ~resetflag;
5094 sq->sq_flags = flags;
5095 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5096 if (flags & SQ_WANTWAKEUP) {
5097 flags &= ~SQ_WANTWAKEUP;
5098 cv_broadcast(&sq->sq_wait);
5099 }
5100 sq->sq_flags = flags;
5101 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5102 if (!isouter) {
5103 /* drain_syncq drops SQLOCK */
5104 drain_syncq(sq);
5105 return;
5106 }
5107 }
5108 }
5109 mutex_exit(SQLOCK(sq));
5110 }
5111
5112 /*
5113 * Reset a flag that was set with blocksq.
5114 * Does not drain the syncq. Use emptysq() for that.
5115 * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5116 *
5117 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5118 * sq_putlocks are used.
5119 */
5120 static int
5121 dropsq(syncq_t *sq, uint16_t resetflag)
5122 {
5123 uint16_t flags;
5124
5125 mutex_enter(SQLOCK(sq));
5126 ASSERT(sq->sq_flags & resetflag);
5127 flags = sq->sq_flags & ~resetflag;
5128 if (flags & SQ_WANTWAKEUP) {
5129 flags &= ~SQ_WANTWAKEUP;
5130 cv_broadcast(&sq->sq_wait);
5131 }
5132 sq->sq_flags = flags;
5133 mutex_exit(SQLOCK(sq));
5134 if (flags & SQ_QUEUED)
5135 return (1);
5136 return (0);
5137 }
5138
5139 /*
5140 * Empty all the messages on a syncq.
5141 *
5142 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5143 * sq_putlocks are used.
5144 */
5145 static void
5146 emptysq(syncq_t *sq)
5147 {
5148 uint16_t flags;
5149
5150 mutex_enter(SQLOCK(sq));
5151 flags = sq->sq_flags;
5152 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5153 /*
5154 * To prevent potential recursive invocation of drain_syncq we
5155 * do not call drain_syncq if count is non-zero.
5156 */
5157 if (sq->sq_count == 0) {
5158 /* drain_syncq() drops SQLOCK */
5159 drain_syncq(sq);
5160 return;
5161 } else
5162 sqenable(sq);
5163 }
5164 mutex_exit(SQLOCK(sq));
5165 }
5166
5167 /*
5168 * Ordered insert while removing duplicates.
5169 */
5170 static void
5171 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5172 {
5173 syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5174
5175 prev_sqlpp = &sqlist->sqlist_head;
5176 while ((sqlp = *prev_sqlpp) != NULL) {
5177 if (sqlp->sql_sq >= sqp) {
5178 if (sqlp->sql_sq == sqp) /* duplicate */
5179 return;
5180 break;
5181 }
5182 prev_sqlpp = &sqlp->sql_next;
5183 }
5184 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5185 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5186 new_sqlp->sql_next = sqlp;
5187 new_sqlp->sql_sq = sqp;
5188 *prev_sqlpp = new_sqlp;
5189 }
5190
5191 /*
5192 * Walk the write side queues until we hit either the driver
5193 * or a twist in the stream (_SAMESTR will return false in both
5194 * these cases) then turn around and walk the read side queues
5195 * back up to the stream head.
5196 */
5197 static void
5198 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5199 {
5200 while (q != NULL) {
5201 sqlist_insert(sqlist, q->q_syncq);
5202
5203 if (_SAMESTR(q))
5204 q = q->q_next;
5205 else if (!(q->q_flag & QREADR))
5206 q = _RD(q);
5207 else
5208 q = NULL;
5209 }
5210 }
5211
5212 /*
5213 * Allocate and build a list of all syncqs in a stream and the syncq(s)
5214 * associated with the "q" parameter. The resulting list is sorted in a
5215 * canonical order and is free of duplicates.
5216 * Assumes the passed queue is a _RD(q).
5217 */
5218 static sqlist_t *
5219 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5220 {
5221 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5222
5223 /*
5224 * start with the current queue/qpair
5225 */
5226 ASSERT(q->q_flag & QREADR);
5227
5228 sqlist_insert(sqlist, q->q_syncq);
5229 sqlist_insert(sqlist, _WR(q)->q_syncq);
5230
5231 sqlist_insertall(sqlist, stp->sd_wrq);
5232 if (do_twist)
5233 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5234
5235 return (sqlist);
5236 }
5237
5238 static sqlist_t *
5239 sqlist_alloc(struct stdata *stp, int kmflag)
5240 {
5241 size_t sqlist_size;
5242 sqlist_t *sqlist;
5243
5244 /*
5245 * Allocate 2 syncql_t's for each pushed module. Note that
5246 * the sqlist_t structure already has 4 syncql_t's built in:
5247 * 2 for the stream head, and 2 for the driver/other stream head.
5248 */
5249 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5250 sizeof (sqlist_t);
5251 if (STRMATED(stp))
5252 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5253 sqlist = kmem_alloc(sqlist_size, kmflag);
5254
5255 sqlist->sqlist_head = NULL;
5256 sqlist->sqlist_size = sqlist_size;
5257 sqlist->sqlist_index = 0;
5258
5259 return (sqlist);
5260 }
5261
5262 /*
5263 * Free the list created by sqlist_alloc()
5264 */
5265 static void
5266 sqlist_free(sqlist_t *sqlist)
5267 {
5268 kmem_free(sqlist, sqlist->sqlist_size);
5269 }
5270
5271 /*
5272 * Prevent any new entries into any syncq in this stream.
5273 * Used by freezestr.
5274 */
5275 void
5276 strblock(queue_t *q)
5277 {
5278 struct stdata *stp;
5279 syncql_t *sql;
5280 sqlist_t *sqlist;
5281
5282 q = _RD(q);
5283
5284 stp = STREAM(q);
5285 ASSERT(stp != NULL);
5286
5287 /*
5288 * Get a sorted list with all the duplicates removed containing
5289 * all the syncqs referenced by this stream.
5290 */
5291 sqlist = sqlist_build(q, stp, B_FALSE);
5292 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5293 blocksq(sql->sql_sq, SQ_FROZEN, -1);
5294 sqlist_free(sqlist);
5295 }
5296
5297 /*
5298 * Release the block on new entries into this stream
5299 */
5300 void
5301 strunblock(queue_t *q)
5302 {
5303 struct stdata *stp;
5304 syncql_t *sql;
5305 sqlist_t *sqlist;
5306 int drain_needed;
5307
5308 q = _RD(q);
5309
5310 /*
5311 * Get a sorted list with all the duplicates removed containing
5312 * all the syncqs referenced by this stream.
5313 * Have to drop the SQ_FROZEN flag on all the syncqs before
5314 * starting to drain them; otherwise the draining might
5315 * cause a freezestr in some module on the stream (which
5316 * would deadlock).
5317 */
5318 stp = STREAM(q);
5319 ASSERT(stp != NULL);
5320 sqlist = sqlist_build(q, stp, B_FALSE);
5321 drain_needed = 0;
5322 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5323 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5324 if (drain_needed) {
5325 for (sql = sqlist->sqlist_head; sql != NULL;
5326 sql = sql->sql_next)
5327 emptysq(sql->sql_sq);
5328 }
5329 sqlist_free(sqlist);
5330 }
5331
5332 #ifdef DEBUG
5333 static int
5334 qprocsareon(queue_t *rq)
5335 {
5336 if (rq->q_next == NULL)
5337 return (0);
5338 return (_WR(rq->q_next)->q_next == _WR(rq));
5339 }
5340
5341 int
5342 qclaimed(queue_t *q)
5343 {
5344 uint_t count;
5345
5346 count = q->q_syncq->sq_count;
5347 SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5348 return (count != 0);
5349 }
5350
5351 /*
5352 * Check if anyone has frozen this stream with freezestr
5353 */
5354 int
5355 frozenstr(queue_t *q)
5356 {
5357 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5358 }
5359 #endif /* DEBUG */
5360
5361 /*
5362 * Enter a queue.
5363 * Obsoleted interface. Should not be used.
5364 */
5365 void
5366 enterq(queue_t *q)
5367 {
5368 entersq(q->q_syncq, SQ_CALLBACK);
5369 }
5370
5371 void
5372 leaveq(queue_t *q)
5373 {
5374 leavesq(q->q_syncq, SQ_CALLBACK);
5375 }
5376
5377 /*
5378 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5379 * to check.
5380 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5381 * calls and the running of open, close and service procedures.
5382 *
5383 * If c_inner bit is set no need to grab sq_putlocks since we don't care
5384 * if other threads have entered or are entering put entry point.
5385 *
5386 * If c_inner bit is set it might have been possible to use
5387 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5388 * open/close path for IP) but since the count may need to be decremented in
5389 * qwait() we wouldn't know which counter to decrement. Currently counter is
5390 * selected by current cpu_seqid and current CPU can change at any moment. XXX
5391 * in the future we might use curthread id bits to select the counter and this
5392 * would stay constant across routine calls.
5393 */
5394 void
5395 entersq(syncq_t *sq, int entrypoint)
5396 {
5397 uint16_t count = 0;
5398 uint16_t flags;
5399 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5400 uint16_t type;
5401 uint_t c_inner = entrypoint & SQ_CI;
5402 uint_t c_outer = entrypoint & SQ_CO;
5403
5404 /*
5405 * Increment ref count to keep closes out of this queue.
5406 */
5407 ASSERT(sq);
5408 ASSERT(c_inner && c_outer);
5409 mutex_enter(SQLOCK(sq));
5410 flags = sq->sq_flags;
5411 type = sq->sq_type;
5412 if (!(type & c_inner)) {
5413 /* Make sure all putcounts now use slowlock. */
5414 count = sq->sq_count;
5415 SQ_PUTLOCKS_ENTER(sq);
5416 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5417 SUM_SQ_PUTCOUNTS(sq, count);
5418 sq->sq_needexcl++;
5419 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5420 waitflags |= SQ_MESSAGES;
5421 }
5422 /*
5423 * Wait until we can enter the inner perimeter.
5424 * If we want exclusive access we wait until sq_count is 0.
5425 * We have to do this before entering the outer perimeter in order
5426 * to preserve put/close message ordering.
5427 */
5428 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5429 sq->sq_flags = flags | SQ_WANTWAKEUP;
5430 if (!(type & c_inner)) {
5431 SQ_PUTLOCKS_EXIT(sq);
5432 }
5433 cv_wait(&sq->sq_wait, SQLOCK(sq));
5434 if (!(type & c_inner)) {
5435 count = sq->sq_count;
5436 SQ_PUTLOCKS_ENTER(sq);
5437 SUM_SQ_PUTCOUNTS(sq, count);
5438 }
5439 flags = sq->sq_flags;
5440 }
5441
5442 if (!(type & c_inner)) {
5443 ASSERT(sq->sq_needexcl > 0);
5444 sq->sq_needexcl--;
5445 if (sq->sq_needexcl == 0) {
5446 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5447 }
5448 }
5449
5450 /* Check if we need to enter the outer perimeter */
5451 if (!(type & c_outer)) {
5452 /*
5453 * We have to enter the outer perimeter exclusively before
5454 * we can increment sq_count to avoid deadlock. This implies
5455 * that we have to re-check sq_flags and sq_count.
5456 *
5457 * is it possible to have c_inner set when c_outer is not set?
5458 */
5459 if (!(type & c_inner)) {
5460 SQ_PUTLOCKS_EXIT(sq);
5461 }
5462 mutex_exit(SQLOCK(sq));
5463 outer_enter(sq->sq_outer, SQ_GOAWAY);
5464 mutex_enter(SQLOCK(sq));
5465 flags = sq->sq_flags;
5466 /*
5467 * there should be no need to recheck sq_putcounts
5468 * because outer_enter() has already waited for them to clear
5469 * after setting SQ_WRITER.
5470 */
5471 count = sq->sq_count;
5472 #ifdef DEBUG
5473 /*
5474 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5475 * of doing an ASSERT internally. Others should do
5476 * something like
5477 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5478 * without the need to #ifdef DEBUG it.
5479 */
5480 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5481 #endif
5482 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5483 (!(type & c_inner) && count != 0)) {
5484 sq->sq_flags = flags | SQ_WANTWAKEUP;
5485 cv_wait(&sq->sq_wait, SQLOCK(sq));
5486 count = sq->sq_count;
5487 flags = sq->sq_flags;
5488 }
5489 }
5490
5491 sq->sq_count++;
5492 ASSERT(sq->sq_count != 0); /* Wraparound */
5493 if (!(type & c_inner)) {
5494 /* Exclusive entry */
5495 ASSERT(sq->sq_count == 1);
5496 sq->sq_flags |= SQ_EXCL;
5497 if (type & c_outer) {
5498 SQ_PUTLOCKS_EXIT(sq);
5499 }
5500 }
5501 mutex_exit(SQLOCK(sq));
5502 }
5503
5504 /*
5505 * Leave a syncq. Announce to framework that closes may proceed.
5506 * c_inner and c_outer specify which concurrency bits to check.
5507 *
5508 * Must never be called from driver or module put entry point.
5509 *
5510 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5511 * sq_putlocks are used.
5512 */
5513 void
5514 leavesq(syncq_t *sq, int entrypoint)
5515 {
5516 uint16_t flags;
5517 uint16_t type;
5518 uint_t c_outer = entrypoint & SQ_CO;
5519 #ifdef DEBUG
5520 uint_t c_inner = entrypoint & SQ_CI;
5521 #endif
5522
5523 /*
5524 * Decrement ref count, drain the syncq if possible, and wake up
5525 * any waiting close.
5526 */
5527 ASSERT(sq);
5528 ASSERT(c_inner && c_outer);
5529 mutex_enter(SQLOCK(sq));
5530 flags = sq->sq_flags;
5531 type = sq->sq_type;
5532 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5533
5534 if (flags & SQ_WANTWAKEUP) {
5535 flags &= ~SQ_WANTWAKEUP;
5536 cv_broadcast(&sq->sq_wait);
5537 }
5538 if (flags & SQ_WANTEXWAKEUP) {
5539 flags &= ~SQ_WANTEXWAKEUP;
5540 cv_broadcast(&sq->sq_exitwait);
5541 }
5542
5543 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5544 /*
5545 * The syncq needs to be drained. "Exit" the syncq
5546 * before calling drain_syncq.
5547 */
5548 ASSERT(sq->sq_count != 0);
5549 sq->sq_count--;
5550 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5551 sq->sq_flags = flags & ~SQ_EXCL;
5552 drain_syncq(sq);
5553 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5554 /* Check if we need to exit the outer perimeter */
5555 /* XXX will this ever be true? */
5556 if (!(type & c_outer))
5557 outer_exit(sq->sq_outer);
5558 return;
5559 }
5560 }
5561 ASSERT(sq->sq_count != 0);
5562 sq->sq_count--;
5563 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5564 sq->sq_flags = flags & ~SQ_EXCL;
5565 mutex_exit(SQLOCK(sq));
5566
5567 /* Check if we need to exit the outer perimeter */
5568 if (!(sq->sq_type & c_outer))
5569 outer_exit(sq->sq_outer);
5570 }
5571
5572 /*
5573 * Prevent q_next from changing in this stream by incrementing sq_count.
5574 *
5575 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5576 * sq_putlocks are used.
5577 */
5578 void
5579 claimq(queue_t *qp)
5580 {
5581 syncq_t *sq = qp->q_syncq;
5582
5583 mutex_enter(SQLOCK(sq));
5584 sq->sq_count++;
5585 ASSERT(sq->sq_count != 0); /* Wraparound */
5586 mutex_exit(SQLOCK(sq));
5587 }
5588
5589 /*
5590 * Undo claimq.
5591 *
5592 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5593 * sq_putlocks are used.
5594 */
5595 void
5596 releaseq(queue_t *qp)
5597 {
5598 syncq_t *sq = qp->q_syncq;
5599 uint16_t flags;
5600
5601 mutex_enter(SQLOCK(sq));
5602 ASSERT(sq->sq_count > 0);
5603 sq->sq_count--;
5604
5605 flags = sq->sq_flags;
5606 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5607 if (flags & SQ_WANTWAKEUP) {
5608 flags &= ~SQ_WANTWAKEUP;
5609 cv_broadcast(&sq->sq_wait);
5610 }
5611 sq->sq_flags = flags;
5612 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5613 /*
5614 * To prevent potential recursive invocation of
5615 * drain_syncq we do not call drain_syncq if count is
5616 * non-zero.
5617 */
5618 if (sq->sq_count == 0) {
5619 drain_syncq(sq);
5620 return;
5621 } else
5622 sqenable(sq);
5623 }
5624 }
5625 mutex_exit(SQLOCK(sq));
5626 }
5627
5628 /*
5629 * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5630 */
5631 void
5632 claimstr(queue_t *qp)
5633 {
5634 struct stdata *stp = STREAM(qp);
5635
5636 mutex_enter(&stp->sd_reflock);
5637 stp->sd_refcnt++;
5638 ASSERT(stp->sd_refcnt != 0); /* Wraparound */
5639 mutex_exit(&stp->sd_reflock);
5640 }
5641
5642 /*
5643 * Undo claimstr.
5644 */
5645 void
5646 releasestr(queue_t *qp)
5647 {
5648 struct stdata *stp = STREAM(qp);
5649
5650 mutex_enter(&stp->sd_reflock);
5651 ASSERT(stp->sd_refcnt != 0);
5652 if (--stp->sd_refcnt == 0)
5653 cv_broadcast(&stp->sd_refmonitor);
5654 mutex_exit(&stp->sd_reflock);
5655 }
5656
5657 static syncq_t *
5658 new_syncq(void)
5659 {
5660 return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5661 }
5662
5663 static void
5664 free_syncq(syncq_t *sq)
5665 {
5666 ASSERT(sq->sq_head == NULL);
5667 ASSERT(sq->sq_outer == NULL);
5668 ASSERT(sq->sq_callbpend == NULL);
5669 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5670 (sq->sq_onext == sq && sq->sq_oprev == sq));
5671
5672 if (sq->sq_ciputctrl != NULL) {
5673 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5674 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5675 sq->sq_nciputctrl, 0);
5676 ASSERT(ciputctrl_cache != NULL);
5677 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5678 }
5679
5680 sq->sq_tail = NULL;
5681 sq->sq_evhead = NULL;
5682 sq->sq_evtail = NULL;
5683 sq->sq_ciputctrl = NULL;
5684 sq->sq_nciputctrl = 0;
5685 sq->sq_count = 0;
5686 sq->sq_rmqcount = 0;
5687 sq->sq_callbflags = 0;
5688 sq->sq_cancelid = 0;
5689 sq->sq_next = NULL;
5690 sq->sq_needexcl = 0;
5691 sq->sq_svcflags = 0;
5692 sq->sq_nqueues = 0;
5693 sq->sq_pri = 0;
5694 sq->sq_onext = NULL;
5695 sq->sq_oprev = NULL;
5696 sq->sq_flags = 0;
5697 sq->sq_type = 0;
5698 sq->sq_servcount = 0;
5699
5700 kmem_cache_free(syncq_cache, sq);
5701 }
5702
5703 /* Outer perimeter code */
5704
5705 /*
5706 * The outer syncq uses the fields and flags in the syncq slightly
5707 * differently from the inner syncqs.
5708 * sq_count Incremented when there are pending or running
5709 * writers at the outer perimeter to prevent the set of
5710 * inner syncqs that belong to the outer perimeter from
5711 * changing.
5712 * sq_head/tail List of deferred qwriter(OUTER) operations.
5713 *
5714 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while
5715 * inner syncqs are added to or removed from the
5716 * outer perimeter.
5717 * SQ_QUEUED sq_head/tail has messages or events queued.
5718 *
5719 * SQ_WRITER A thread is currently traversing all the inner syncqs
5720 * setting the SQ_WRITER flag.
5721 */
5722
5723 /*
5724 * Get write access at the outer perimeter.
5725 * Note that read access is done by entersq, putnext, and put by simply
5726 * incrementing sq_count in the inner syncq.
5727 *
5728 * Waits until "flags" is no longer set in the outer to prevent multiple
5729 * threads from having write access at the same time. SQ_WRITER has to be part
5730 * of "flags".
5731 *
5732 * Increases sq_count on the outer syncq to keep away outer_insert/remove
5733 * until the outer_exit is finished.
5734 *
5735 * outer_enter is vulnerable to starvation since it does not prevent new
5736 * threads from entering the inner syncqs while it is waiting for sq_count to
5737 * go to zero.
5738 */
5739 void
5740 outer_enter(syncq_t *outer, uint16_t flags)
5741 {
5742 syncq_t *sq;
5743 int wait_needed;
5744 uint16_t count;
5745
5746 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5747 outer->sq_oprev != NULL);
5748 ASSERT(flags & SQ_WRITER);
5749
5750 retry:
5751 mutex_enter(SQLOCK(outer));
5752 while (outer->sq_flags & flags) {
5753 outer->sq_flags |= SQ_WANTWAKEUP;
5754 cv_wait(&outer->sq_wait, SQLOCK(outer));
5755 }
5756
5757 ASSERT(!(outer->sq_flags & SQ_WRITER));
5758 outer->sq_flags |= SQ_WRITER;
5759 outer->sq_count++;
5760 ASSERT(outer->sq_count != 0); /* wraparound */
5761 wait_needed = 0;
5762 /*
5763 * Set SQ_WRITER on all the inner syncqs while holding
5764 * the SQLOCK on the outer syncq. This ensures that the changing
5765 * of SQ_WRITER is atomic under the outer SQLOCK.
5766 */
5767 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5768 mutex_enter(SQLOCK(sq));
5769 count = sq->sq_count;
5770 SQ_PUTLOCKS_ENTER(sq);
5771 sq->sq_flags |= SQ_WRITER;
5772 SUM_SQ_PUTCOUNTS(sq, count);
5773 if (count != 0)
5774 wait_needed = 1;
5775 SQ_PUTLOCKS_EXIT(sq);
5776 mutex_exit(SQLOCK(sq));
5777 }
5778 mutex_exit(SQLOCK(outer));
5779
5780 /*
5781 * Get everybody out of the syncqs sequentially.
5782 * Note that we don't actually need to acquire the PUTLOCKS, since
5783 * we have already cleared the fastbit, and set QWRITER. By
5784 * definition, the count can not increase since putnext will
5785 * take the slowlock path (and the purpose of acquiring the
5786 * putlocks was to make sure it didn't increase while we were
5787 * waiting).
5788 *
5789 * Note that we still acquire the PUTLOCKS to be safe.
5790 */
5791 if (wait_needed) {
5792 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5793 mutex_enter(SQLOCK(sq));
5794 count = sq->sq_count;
5795 SQ_PUTLOCKS_ENTER(sq);
5796 SUM_SQ_PUTCOUNTS(sq, count);
5797 while (count != 0) {
5798 sq->sq_flags |= SQ_WANTWAKEUP;
5799 SQ_PUTLOCKS_EXIT(sq);
5800 cv_wait(&sq->sq_wait, SQLOCK(sq));
5801 count = sq->sq_count;
5802 SQ_PUTLOCKS_ENTER(sq);
5803 SUM_SQ_PUTCOUNTS(sq, count);
5804 }
5805 SQ_PUTLOCKS_EXIT(sq);
5806 mutex_exit(SQLOCK(sq));
5807 }
5808 /*
5809 * Verify that none of the flags got set while we
5810 * were waiting for the sq_counts to drop.
5811 * If this happens we exit and retry entering the
5812 * outer perimeter.
5813 */
5814 mutex_enter(SQLOCK(outer));
5815 if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5816 mutex_exit(SQLOCK(outer));
5817 outer_exit(outer);
5818 goto retry;
5819 }
5820 mutex_exit(SQLOCK(outer));
5821 }
5822 }
5823
5824 /*
5825 * Drop the write access at the outer perimeter.
5826 * Read access is dropped implicitly (by putnext, put, and leavesq) by
5827 * decrementing sq_count.
5828 */
5829 void
5830 outer_exit(syncq_t *outer)
5831 {
5832 syncq_t *sq;
5833 int drain_needed;
5834 uint16_t flags;
5835
5836 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5837 outer->sq_oprev != NULL);
5838 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5839
5840 /*
5841 * Atomically (from the perspective of threads calling become_writer)
5842 * drop the write access at the outer perimeter by holding
5843 * SQLOCK(outer) across all the dropsq calls and the resetting of
5844 * SQ_WRITER.
5845 * This defines a locking order between the outer perimeter
5846 * SQLOCK and the inner perimeter SQLOCKs.
5847 */
5848 mutex_enter(SQLOCK(outer));
5849 flags = outer->sq_flags;
5850 ASSERT(outer->sq_flags & SQ_WRITER);
5851 if (flags & SQ_QUEUED) {
5852 write_now(outer);
5853 flags = outer->sq_flags;
5854 }
5855
5856 /*
5857 * sq_onext is stable since sq_count has not yet been decreased.
5858 * Reset the SQ_WRITER flags in all syncqs.
5859 * After dropping SQ_WRITER on the outer syncq we empty all the
5860 * inner syncqs.
5861 */
5862 drain_needed = 0;
5863 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5864 drain_needed += dropsq(sq, SQ_WRITER);
5865 ASSERT(!(outer->sq_flags & SQ_QUEUED));
5866 flags &= ~SQ_WRITER;
5867 if (drain_needed) {
5868 outer->sq_flags = flags;
5869 mutex_exit(SQLOCK(outer));
5870 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5871 emptysq(sq);
5872 mutex_enter(SQLOCK(outer));
5873 flags = outer->sq_flags;
5874 }
5875 if (flags & SQ_WANTWAKEUP) {
5876 flags &= ~SQ_WANTWAKEUP;
5877 cv_broadcast(&outer->sq_wait);
5878 }
5879 outer->sq_flags = flags;
5880 ASSERT(outer->sq_count > 0);
5881 outer->sq_count--;
5882 mutex_exit(SQLOCK(outer));
5883 }
5884
5885 /*
5886 * Add another syncq to an outer perimeter.
5887 * Block out all other access to the outer perimeter while it is being
5888 * changed using blocksq.
5889 * Assumes that the caller has *not* done an outer_enter.
5890 *
5891 * Vulnerable to starvation in blocksq.
5892 */
5893 static void
5894 outer_insert(syncq_t *outer, syncq_t *sq)
5895 {
5896 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5897 outer->sq_oprev != NULL);
5898 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5899 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */
5900
5901 /* Get exclusive access to the outer perimeter list */
5902 blocksq(outer, SQ_BLOCKED, 0);
5903 ASSERT(outer->sq_flags & SQ_BLOCKED);
5904 ASSERT(!(outer->sq_flags & SQ_WRITER));
5905
5906 mutex_enter(SQLOCK(sq));
5907 sq->sq_outer = outer;
5908 outer->sq_onext->sq_oprev = sq;
5909 sq->sq_onext = outer->sq_onext;
5910 outer->sq_onext = sq;
5911 sq->sq_oprev = outer;
5912 mutex_exit(SQLOCK(sq));
5913 unblocksq(outer, SQ_BLOCKED, 1);
5914 }
5915
5916 /*
5917 * Remove a syncq from an outer perimeter.
5918 * Block out all other access to the outer perimeter while it is being
5919 * changed using blocksq.
5920 * Assumes that the caller has *not* done an outer_enter.
5921 *
5922 * Vulnerable to starvation in blocksq.
5923 */
5924 static void
5925 outer_remove(syncq_t *outer, syncq_t *sq)
5926 {
5927 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5928 outer->sq_oprev != NULL);
5929 ASSERT(sq->sq_outer == outer);
5930
5931 /* Get exclusive access to the outer perimeter list */
5932 blocksq(outer, SQ_BLOCKED, 0);
5933 ASSERT(outer->sq_flags & SQ_BLOCKED);
5934 ASSERT(!(outer->sq_flags & SQ_WRITER));
5935
5936 mutex_enter(SQLOCK(sq));
5937 sq->sq_outer = NULL;
5938 sq->sq_onext->sq_oprev = sq->sq_oprev;
5939 sq->sq_oprev->sq_onext = sq->sq_onext;
5940 sq->sq_oprev = sq->sq_onext = NULL;
5941 mutex_exit(SQLOCK(sq));
5942 unblocksq(outer, SQ_BLOCKED, 1);
5943 }
5944
5945 /*
5946 * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5947 * If this is the first callback for this outer perimeter then add
5948 * this outer perimeter to the list of outer perimeters that
5949 * the qwriter_outer_thread will process.
5950 *
5951 * Increments sq_count in the outer syncq to prevent the membership
5952 * of the outer perimeter (in terms of inner syncqs) to change while
5953 * the callback is pending.
5954 */
5955 static void
5956 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5957 {
5958 ASSERT(MUTEX_HELD(SQLOCK(outer)));
5959
5960 mp->b_prev = (mblk_t *)func;
5961 mp->b_queue = q;
5962 mp->b_next = NULL;
5963 outer->sq_count++; /* Decremented when dequeued */
5964 ASSERT(outer->sq_count != 0); /* Wraparound */
5965 if (outer->sq_evhead == NULL) {
5966 /* First message. */
5967 outer->sq_evhead = outer->sq_evtail = mp;
5968 outer->sq_flags |= SQ_EVENTS;
5969 mutex_exit(SQLOCK(outer));
5970 STRSTAT(qwr_outer);
5971 (void) taskq_dispatch(streams_taskq,
5972 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5973 } else {
5974 ASSERT(outer->sq_flags & SQ_EVENTS);
5975 outer->sq_evtail->b_next = mp;
5976 outer->sq_evtail = mp;
5977 mutex_exit(SQLOCK(outer));
5978 }
5979 }
5980
5981 /*
5982 * Try and upgrade to write access at the outer perimeter. If this can
5983 * not be done without blocking then queue the callback to be done
5984 * by the qwriter_outer_thread.
5985 *
5986 * This routine can only be called from put or service procedures plus
5987 * asynchronous callback routines that have properly entered the queue (with
5988 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5989 * associated with q.
5990 */
5991 void
5992 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
5993 {
5994 syncq_t *osq, *sq, *outer;
5995 int failed;
5996 uint16_t flags;
5997
5998 osq = q->q_syncq;
5999 outer = osq->sq_outer;
6000 if (outer == NULL)
6001 panic("qwriter(PERIM_OUTER): no outer perimeter");
6002 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6003 outer->sq_oprev != NULL);
6004
6005 mutex_enter(SQLOCK(outer));
6006 flags = outer->sq_flags;
6007 /*
6008 * If some thread is traversing sq_next, or if we are blocked by
6009 * outer_insert or outer_remove, or if the we already have queued
6010 * callbacks, then queue this callback for later processing.
6011 *
6012 * Also queue the qwriter for an interrupt thread in order
6013 * to reduce the time spent running at high IPL.
6014 * to identify there are events.
6015 */
6016 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6017 /*
6018 * Queue the become_writer request.
6019 * The queueing is atomic under SQLOCK(outer) in order
6020 * to synchronize with outer_exit.
6021 * queue_writer will drop the outer SQLOCK
6022 */
6023 if (flags & SQ_BLOCKED) {
6024 /* Must set SQ_WRITER on inner perimeter */
6025 mutex_enter(SQLOCK(osq));
6026 osq->sq_flags |= SQ_WRITER;
6027 mutex_exit(SQLOCK(osq));
6028 } else {
6029 if (!(flags & SQ_WRITER)) {
6030 /*
6031 * The outer could have been SQ_BLOCKED thus
6032 * SQ_WRITER might not be set on the inner.
6033 */
6034 mutex_enter(SQLOCK(osq));
6035 osq->sq_flags |= SQ_WRITER;
6036 mutex_exit(SQLOCK(osq));
6037 }
6038 ASSERT(osq->sq_flags & SQ_WRITER);
6039 }
6040 queue_writer(outer, func, q, mp);
6041 return;
6042 }
6043 /*
6044 * We are half-way to exclusive access to the outer perimeter.
6045 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6046 * while the inner syncqs are traversed.
6047 */
6048 outer->sq_count++;
6049 ASSERT(outer->sq_count != 0); /* wraparound */
6050 flags |= SQ_WRITER;
6051 /*
6052 * Check if we can run the function immediately. Mark all
6053 * syncqs with the writer flag to prevent new entries into
6054 * put and service procedures.
6055 *
6056 * Set SQ_WRITER on all the inner syncqs while holding
6057 * the SQLOCK on the outer syncq. This ensures that the changing
6058 * of SQ_WRITER is atomic under the outer SQLOCK.
6059 */
6060 failed = 0;
6061 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6062 uint16_t count;
6063 uint_t maxcnt = (sq == osq) ? 1 : 0;
6064
6065 mutex_enter(SQLOCK(sq));
6066 count = sq->sq_count;
6067 SQ_PUTLOCKS_ENTER(sq);
6068 SUM_SQ_PUTCOUNTS(sq, count);
6069 if (sq->sq_count > maxcnt)
6070 failed = 1;
6071 sq->sq_flags |= SQ_WRITER;
6072 SQ_PUTLOCKS_EXIT(sq);
6073 mutex_exit(SQLOCK(sq));
6074 }
6075 if (failed) {
6076 /*
6077 * Some other thread has a read claim on the outer perimeter.
6078 * Queue the callback for deferred processing.
6079 *
6080 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6081 * so that other qwriter(OUTER) calls will queue their
6082 * callbacks as well. queue_writer increments sq_count so we
6083 * decrement to compensate for the our increment.
6084 *
6085 * Dropping SQ_WRITER enables the writer thread to work
6086 * on this outer perimeter.
6087 */
6088 outer->sq_flags = flags;
6089 queue_writer(outer, func, q, mp);
6090 /* queue_writer dropper the lock */
6091 mutex_enter(SQLOCK(outer));
6092 ASSERT(outer->sq_count > 0);
6093 outer->sq_count--;
6094 ASSERT(outer->sq_flags & SQ_WRITER);
6095 flags = outer->sq_flags;
6096 flags &= ~SQ_WRITER;
6097 if (flags & SQ_WANTWAKEUP) {
6098 flags &= ~SQ_WANTWAKEUP;
6099 cv_broadcast(&outer->sq_wait);
6100 }
6101 outer->sq_flags = flags;
6102 mutex_exit(SQLOCK(outer));
6103 return;
6104 } else {
6105 outer->sq_flags = flags;
6106 mutex_exit(SQLOCK(outer));
6107 }
6108
6109 /* Can run it immediately */
6110 (*func)(q, mp);
6111
6112 outer_exit(outer);
6113 }
6114
6115 /*
6116 * Dequeue all writer callbacks from the outer perimeter and run them.
6117 */
6118 static void
6119 write_now(syncq_t *outer)
6120 {
6121 mblk_t *mp;
6122 queue_t *q;
6123 void (*func)();
6124
6125 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6126 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6127 outer->sq_oprev != NULL);
6128 while ((mp = outer->sq_evhead) != NULL) {
6129 /*
6130 * queues cannot be placed on the queuelist on the outer
6131 * perimeter.
6132 */
6133 ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6134 ASSERT((outer->sq_flags & SQ_EVENTS));
6135
6136 outer->sq_evhead = mp->b_next;
6137 if (outer->sq_evhead == NULL) {
6138 outer->sq_evtail = NULL;
6139 outer->sq_flags &= ~SQ_EVENTS;
6140 }
6141 ASSERT(outer->sq_count != 0);
6142 outer->sq_count--; /* Incremented when enqueued. */
6143 mutex_exit(SQLOCK(outer));
6144 /*
6145 * Drop the message if the queue is closing.
6146 * Make sure that the queue is "claimed" when the callback
6147 * is run in order to satisfy various ASSERTs.
6148 */
6149 q = mp->b_queue;
6150 func = (void (*)())mp->b_prev;
6151 ASSERT(func != NULL);
6152 mp->b_next = mp->b_prev = NULL;
6153 if (q->q_flag & QWCLOSE) {
6154 freemsg(mp);
6155 } else {
6156 claimq(q);
6157 (*func)(q, mp);
6158 releaseq(q);
6159 }
6160 mutex_enter(SQLOCK(outer));
6161 }
6162 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6163 }
6164
6165 /*
6166 * The list of messages on the inner syncq is effectively hashed
6167 * by destination queue. These destination queues are doubly
6168 * linked lists (hopefully) in priority order. Messages are then
6169 * put on the queue referenced by the q_sqhead/q_sqtail elements.
6170 * Additional messages are linked together by the b_next/b_prev
6171 * elements in the mblk, with (similar to putq()) the first message
6172 * having a NULL b_prev and the last message having a NULL b_next.
6173 *
6174 * Events, such as qwriter callbacks, are put onto a list in FIFO
6175 * order referenced by sq_evhead, and sq_evtail. This is a singly
6176 * linked list, and messages here MUST be processed in the order queued.
6177 */
6178
6179 /*
6180 * Run the events on the syncq event list (sq_evhead).
6181 * Assumes there is only one claim on the syncq, it is
6182 * already exclusive (SQ_EXCL set), and the SQLOCK held.
6183 * Messages here are processed in order, with the SQ_EXCL bit
6184 * held all the way through till the last message is processed.
6185 */
6186 void
6187 sq_run_events(syncq_t *sq)
6188 {
6189 mblk_t *bp;
6190 queue_t *qp;
6191 uint16_t flags = sq->sq_flags;
6192 void (*func)();
6193
6194 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6195 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6196 sq->sq_oprev == NULL) ||
6197 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6198 sq->sq_oprev != NULL));
6199
6200 ASSERT(flags & SQ_EXCL);
6201 ASSERT(sq->sq_count == 1);
6202
6203 /*
6204 * We need to process all of the events on this list. It
6205 * is possible that new events will be added while we are
6206 * away processing a callback, so on every loop, we start
6207 * back at the beginning of the list.
6208 */
6209 /*
6210 * We have to reaccess sq_evhead since there is a
6211 * possibility of a new entry while we were running
6212 * the callback.
6213 */
6214 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6215 ASSERT(bp->b_queue->q_syncq == sq);
6216 ASSERT(sq->sq_flags & SQ_EVENTS);
6217
6218 qp = bp->b_queue;
6219 func = (void (*)())bp->b_prev;
6220 ASSERT(func != NULL);
6221
6222 /*
6223 * Messages from the event queue must be taken off in
6224 * FIFO order.
6225 */
6226 ASSERT(sq->sq_evhead == bp);
6227 sq->sq_evhead = bp->b_next;
6228
6229 if (bp->b_next == NULL) {
6230 /* Deleting last */
6231 ASSERT(sq->sq_evtail == bp);
6232 sq->sq_evtail = NULL;
6233 sq->sq_flags &= ~SQ_EVENTS;
6234 }
6235 bp->b_prev = bp->b_next = NULL;
6236 ASSERT(bp->b_datap->db_ref != 0);
6237
6238 mutex_exit(SQLOCK(sq));
6239
6240 (*func)(qp, bp);
6241
6242 mutex_enter(SQLOCK(sq));
6243 /*
6244 * re-read the flags, since they could have changed.
6245 */
6246 flags = sq->sq_flags;
6247 ASSERT(flags & SQ_EXCL);
6248 }
6249 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6250 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6251
6252 if (flags & SQ_WANTWAKEUP) {
6253 flags &= ~SQ_WANTWAKEUP;
6254 cv_broadcast(&sq->sq_wait);
6255 }
6256 if (flags & SQ_WANTEXWAKEUP) {
6257 flags &= ~SQ_WANTEXWAKEUP;
6258 cv_broadcast(&sq->sq_exitwait);
6259 }
6260 sq->sq_flags = flags;
6261 }
6262
6263 /*
6264 * Put messages on the event list.
6265 * If we can go exclusive now, do so and process the event list, otherwise
6266 * let the last claim service this list (or wake the sqthread).
6267 * This procedure assumes SQLOCK is held. To run the event list, it
6268 * must be called with no claims.
6269 */
6270 static void
6271 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6272 {
6273 uint16_t count;
6274
6275 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6276 ASSERT(func != NULL);
6277
6278 /*
6279 * This is a callback. Add it to the list of callbacks
6280 * and see about upgrading.
6281 */
6282 mp->b_prev = (mblk_t *)func;
6283 mp->b_queue = q;
6284 mp->b_next = NULL;
6285 if (sq->sq_evhead == NULL) {
6286 sq->sq_evhead = sq->sq_evtail = mp;
6287 sq->sq_flags |= SQ_EVENTS;
6288 } else {
6289 ASSERT(sq->sq_evtail != NULL);
6290 ASSERT(sq->sq_evtail->b_next == NULL);
6291 ASSERT(sq->sq_flags & SQ_EVENTS);
6292 sq->sq_evtail->b_next = mp;
6293 sq->sq_evtail = mp;
6294 }
6295 /*
6296 * We have set SQ_EVENTS, so threads will have to
6297 * unwind out of the perimeter, and new entries will
6298 * not grab a putlock. But we still need to know
6299 * how many threads have already made a claim to the
6300 * syncq, so grab the putlocks, and sum the counts.
6301 * If there are no claims on the syncq, we can upgrade
6302 * to exclusive, and run the event list.
6303 * NOTE: We hold the SQLOCK, so we can just grab the
6304 * putlocks.
6305 */
6306 count = sq->sq_count;
6307 SQ_PUTLOCKS_ENTER(sq);
6308 SUM_SQ_PUTCOUNTS(sq, count);
6309 /*
6310 * We have no claim, so we need to check if there
6311 * are no others, then we can upgrade.
6312 */
6313 /*
6314 * There are currently no claims on
6315 * the syncq by this thread (at least on this entry). The thread who has
6316 * the claim should drain syncq.
6317 */
6318 if (count > 0) {
6319 /*
6320 * Can't upgrade - other threads inside.
6321 */
6322 SQ_PUTLOCKS_EXIT(sq);
6323 mutex_exit(SQLOCK(sq));
6324 return;
6325 }
6326 /*
6327 * Need to set SQ_EXCL and make a claim on the syncq.
6328 */
6329 ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6330 sq->sq_flags |= SQ_EXCL;
6331 ASSERT(sq->sq_count == 0);
6332 sq->sq_count++;
6333 SQ_PUTLOCKS_EXIT(sq);
6334
6335 /* Process the events list */
6336 sq_run_events(sq);
6337
6338 /*
6339 * Release our claim...
6340 */
6341 sq->sq_count--;
6342
6343 /*
6344 * And release SQ_EXCL.
6345 * We don't need to acquire the putlocks to release
6346 * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6347 */
6348 sq->sq_flags &= ~SQ_EXCL;
6349
6350 /*
6351 * sq_run_events should have released SQ_EXCL
6352 */
6353 ASSERT(!(sq->sq_flags & SQ_EXCL));
6354
6355 /*
6356 * If anything happened while we were running the
6357 * events (or was there before), we need to process
6358 * them now. We shouldn't be exclusive sine we
6359 * released the perimeter above (plus, we asserted
6360 * for it).
6361 */
6362 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6363 drain_syncq(sq);
6364 else
6365 mutex_exit(SQLOCK(sq));
6366 }
6367
6368 /*
6369 * Perform delayed processing. The caller has to make sure that it is safe
6370 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6371 * set).
6372 *
6373 * Assume that the caller has NO claims on the syncq. However, a claim
6374 * on the syncq does not indicate that a thread is draining the syncq.
6375 * There may be more claims on the syncq than there are threads draining
6376 * (i.e. #_threads_draining <= sq_count)
6377 *
6378 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6379 * in order to preserve qwriter(OUTER) ordering constraints.
6380 *
6381 * sq_putcount only needs to be checked when dispatching the queued
6382 * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6383 */
6384 void
6385 drain_syncq(syncq_t *sq)
6386 {
6387 queue_t *qp;
6388 uint16_t count;
6389 uint16_t type = sq->sq_type;
6390 uint16_t flags = sq->sq_flags;
6391 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE;
6392
6393 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6394 "drain_syncq start:%p", sq);
6395 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6396 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6397 sq->sq_oprev == NULL) ||
6398 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6399 sq->sq_oprev != NULL));
6400
6401 /*
6402 * Drop SQ_SERVICE flag.
6403 */
6404 if (bg_service)
6405 sq->sq_svcflags &= ~SQ_SERVICE;
6406
6407 /*
6408 * If SQ_EXCL is set, someone else is processing this syncq - let them
6409 * finish the job.
6410 */
6411 if (flags & SQ_EXCL) {
6412 if (bg_service) {
6413 ASSERT(sq->sq_servcount != 0);
6414 sq->sq_servcount--;
6415 }
6416 mutex_exit(SQLOCK(sq));
6417 return;
6418 }
6419
6420 /*
6421 * This routine can be called by a background thread if
6422 * it was scheduled by a hi-priority thread. SO, if there are
6423 * NOT messages queued, return (remember, we have the SQLOCK,
6424 * and it cannot change until we release it). Wakeup any waiters also.
6425 */
6426 if (!(flags & SQ_QUEUED)) {
6427 if (flags & SQ_WANTWAKEUP) {
6428 flags &= ~SQ_WANTWAKEUP;
6429 cv_broadcast(&sq->sq_wait);
6430 }
6431 if (flags & SQ_WANTEXWAKEUP) {
6432 flags &= ~SQ_WANTEXWAKEUP;
6433 cv_broadcast(&sq->sq_exitwait);
6434 }
6435 sq->sq_flags = flags;
6436 if (bg_service) {
6437 ASSERT(sq->sq_servcount != 0);
6438 sq->sq_servcount--;
6439 }
6440 mutex_exit(SQLOCK(sq));
6441 return;
6442 }
6443
6444 /*
6445 * If this is not a concurrent put perimeter, we need to
6446 * become exclusive to drain. Also, if not CIPUT, we would
6447 * not have acquired a putlock, so we don't need to check
6448 * the putcounts. If not entering with a claim, we test
6449 * for sq_count == 0.
6450 */
6451 type = sq->sq_type;
6452 if (!(type & SQ_CIPUT)) {
6453 if (sq->sq_count > 1) {
6454 if (bg_service) {
6455 ASSERT(sq->sq_servcount != 0);
6456 sq->sq_servcount--;
6457 }
6458 mutex_exit(SQLOCK(sq));
6459 return;
6460 }
6461 sq->sq_flags |= SQ_EXCL;
6462 }
6463
6464 /*
6465 * This is where we make a claim to the syncq.
6466 * This can either be done by incrementing a putlock, or
6467 * the sq_count. But since we already have the SQLOCK
6468 * here, we just bump the sq_count.
6469 *
6470 * Note that after we make a claim, we need to let the code
6471 * fall through to the end of this routine to clean itself
6472 * up. A return in the while loop will put the syncq in a
6473 * very bad state.
6474 */
6475 sq->sq_count++;
6476 ASSERT(sq->sq_count != 0); /* wraparound */
6477
6478 while ((flags = sq->sq_flags) & SQ_QUEUED) {
6479 /*
6480 * If we are told to stayaway or went exclusive,
6481 * we are done.
6482 */
6483 if (flags & (SQ_STAYAWAY)) {
6484 break;
6485 }
6486
6487 /*
6488 * If there are events to run, do so.
6489 * We have one claim to the syncq, so if there are
6490 * more than one, other threads are running.
6491 */
6492 if (sq->sq_evhead != NULL) {
6493 ASSERT(sq->sq_flags & SQ_EVENTS);
6494
6495 count = sq->sq_count;
6496 SQ_PUTLOCKS_ENTER(sq);
6497 SUM_SQ_PUTCOUNTS(sq, count);
6498 if (count > 1) {
6499 SQ_PUTLOCKS_EXIT(sq);
6500 /* Can't upgrade - other threads inside */
6501 break;
6502 }
6503 ASSERT((flags & SQ_EXCL) == 0);
6504 sq->sq_flags = flags | SQ_EXCL;
6505 SQ_PUTLOCKS_EXIT(sq);
6506 /*
6507 * we have the only claim, run the events,
6508 * sq_run_events will clear the SQ_EXCL flag.
6509 */
6510 sq_run_events(sq);
6511
6512 /*
6513 * If this is a CIPUT perimeter, we need
6514 * to drop the SQ_EXCL flag so we can properly
6515 * continue draining the syncq.
6516 */
6517 if (type & SQ_CIPUT) {
6518 ASSERT(sq->sq_flags & SQ_EXCL);
6519 sq->sq_flags &= ~SQ_EXCL;
6520 }
6521
6522 /*
6523 * And go back to the beginning just in case
6524 * anything changed while we were away.
6525 */
6526 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6527 continue;
6528 }
6529
6530 ASSERT(sq->sq_evhead == NULL);
6531 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6532
6533 /*
6534 * Find the queue that is not draining.
6535 *
6536 * q_draining is protected by QLOCK which we do not hold.
6537 * But if it was set, then a thread was draining, and if it gets
6538 * cleared, then it was because the thread has successfully
6539 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6540 * state to happen, a thread needs the SQLOCK which we hold, and
6541 * if there was such a flag, we would have already seen it.
6542 */
6543
6544 for (qp = sq->sq_head;
6545 qp != NULL && (qp->q_draining ||
6546 (qp->q_sqflags & Q_SQDRAINING));
6547 qp = qp->q_sqnext)
6548 ;
6549
6550 if (qp == NULL)
6551 break;
6552
6553 /*
6554 * We have a queue to work on, and we hold the
6555 * SQLOCK and one claim, call qdrain_syncq.
6556 * This means we need to release the SQLOCK and
6557 * acquire the QLOCK (OK since we have a claim).
6558 * Note that qdrain_syncq will actually dequeue
6559 * this queue from the sq_head list when it is
6560 * convinced all the work is done and release
6561 * the QLOCK before returning.
6562 */
6563 qp->q_sqflags |= Q_SQDRAINING;
6564 mutex_exit(SQLOCK(sq));
6565 mutex_enter(QLOCK(qp));
6566 qdrain_syncq(sq, qp);
6567 mutex_enter(SQLOCK(sq));
6568
6569 /* The queue is drained */
6570 ASSERT(qp->q_sqflags & Q_SQDRAINING);
6571 qp->q_sqflags &= ~Q_SQDRAINING;
6572 /*
6573 * NOTE: After this point qp should not be used since it may be
6574 * closed.
6575 */
6576 }
6577
6578 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6579 flags = sq->sq_flags;
6580
6581 /*
6582 * sq->sq_head cannot change because we hold the
6583 * sqlock. However, a thread CAN decide that it is no longer
6584 * going to drain that queue. However, this should be due to
6585 * a GOAWAY state, and we should see that here.
6586 *
6587 * This loop is not very efficient. One solution may be adding a second
6588 * pointer to the "draining" queue, but it is difficult to do when
6589 * queues are inserted in the middle due to priority ordering. Another
6590 * possibility is to yank the queue out of the sq list and put it onto
6591 * the "draining list" and then put it back if it can't be drained.
6592 */
6593
6594 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6595 (type & SQ_CI) || sq->sq_head->q_draining);
6596
6597 /* Drop SQ_EXCL for non-CIPUT perimeters */
6598 if (!(type & SQ_CIPUT))
6599 flags &= ~SQ_EXCL;
6600 ASSERT((flags & SQ_EXCL) == 0);
6601
6602 /* Wake up any waiters. */
6603 if (flags & SQ_WANTWAKEUP) {
6604 flags &= ~SQ_WANTWAKEUP;
6605 cv_broadcast(&sq->sq_wait);
6606 }
6607 if (flags & SQ_WANTEXWAKEUP) {
6608 flags &= ~SQ_WANTEXWAKEUP;
6609 cv_broadcast(&sq->sq_exitwait);
6610 }
6611 sq->sq_flags = flags;
6612
6613 ASSERT(sq->sq_count != 0);
6614 /* Release our claim. */
6615 sq->sq_count--;
6616
6617 if (bg_service) {
6618 ASSERT(sq->sq_servcount != 0);
6619 sq->sq_servcount--;
6620 }
6621
6622 mutex_exit(SQLOCK(sq));
6623
6624 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6625 "drain_syncq end:%p", sq);
6626 }
6627
6628
6629 /*
6630 *
6631 * qdrain_syncq can be called (currently) from only one of two places:
6632 * drain_syncq
6633 * putnext (or some variation of it).
6634 * and eventually
6635 * qwait(_sig)
6636 *
6637 * If called from drain_syncq, we found it in the list of queues needing
6638 * service, so there is work to be done (or it wouldn't be in the list).
6639 *
6640 * If called from some putnext variation, it was because the
6641 * perimeter is open, but messages are blocking a putnext and
6642 * there is not a thread working on it. Now a thread could start
6643 * working on it while we are getting ready to do so ourself, but
6644 * the thread would set the q_draining flag, and we can spin out.
6645 *
6646 * As for qwait(_sig), I think I shall let it continue to call
6647 * drain_syncq directly (after all, it will get here eventually).
6648 *
6649 * qdrain_syncq has to terminate when:
6650 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6651 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6652 *
6653 * ASSUMES:
6654 * One claim
6655 * QLOCK held
6656 * SQLOCK not held
6657 * Will release QLOCK before returning
6658 */
6659 void
6660 qdrain_syncq(syncq_t *sq, queue_t *q)
6661 {
6662 mblk_t *bp;
6663 #ifdef DEBUG
6664 uint16_t count;
6665 #endif
6666
6667 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6668 "drain_syncq start:%p", sq);
6669 ASSERT(q->q_syncq == sq);
6670 ASSERT(MUTEX_HELD(QLOCK(q)));
6671 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6672 /*
6673 * For non-CIPUT perimeters, we should be called with the exclusive bit
6674 * set already. For CIPUT perimeters, we will be doing a concurrent
6675 * drain, so it better not be set.
6676 */
6677 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6678 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6679 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6680 /*
6681 * All outer pointers are set, or none of them are
6682 */
6683 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6684 sq->sq_oprev == NULL) ||
6685 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6686 sq->sq_oprev != NULL));
6687 #ifdef DEBUG
6688 count = sq->sq_count;
6689 /*
6690 * This is OK without the putlocks, because we have one
6691 * claim either from the sq_count, or a putcount. We could
6692 * get an erroneous value from other counts, but ours won't
6693 * change, so one way or another, we will have at least a
6694 * value of one.
6695 */
6696 SUM_SQ_PUTCOUNTS(sq, count);
6697 ASSERT(count >= 1);
6698 #endif /* DEBUG */
6699
6700 /*
6701 * The first thing to do is find out if a thread is already draining
6702 * this queue. If so, we are done, just return.
6703 */
6704 if (q->q_draining) {
6705 mutex_exit(QLOCK(q));
6706 return;
6707 }
6708
6709 /*
6710 * If the perimeter is exclusive, there is nothing we can do right now,
6711 * go away. Note that there is nothing to prevent this case from
6712 * changing right after this check, but the spin-out will catch it.
6713 */
6714
6715 /* Tell other threads that we are draining this queue */
6716 q->q_draining = 1; /* Protected by QLOCK */
6717
6718 /*
6719 * If there is nothing to do, clear QFULL as necessary. This caters for
6720 * the case where an empty queue was enqueued onto the syncq.
6721 */
6722 if (q->q_sqhead == NULL) {
6723 ASSERT(q->q_syncqmsgs == 0);
6724 mutex_exit(QLOCK(q));
6725 clr_qfull(q);
6726 mutex_enter(QLOCK(q));
6727 }
6728
6729 /*
6730 * Note that q_sqhead must be re-checked here in case another message
6731 * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6732 */
6733 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6734 /*
6735 * Because we can enter this routine just because a putnext is
6736 * blocked, we need to spin out if the perimeter wants to go
6737 * exclusive as well as just blocked. We need to spin out also
6738 * if events are queued on the syncq.
6739 * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6740 * set it, and it can't become exclusive while we hold a claim.
6741 */
6742 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6743 break;
6744 }
6745
6746 #ifdef DEBUG
6747 /*
6748 * Since we are in qdrain_syncq, we already know the queue,
6749 * but for sanity, we want to check this against the qp that
6750 * was passed in by bp->b_queue.
6751 */
6752
6753 ASSERT(bp->b_queue == q);
6754 ASSERT(bp->b_queue->q_syncq == sq);
6755 bp->b_queue = NULL;
6756
6757 /*
6758 * We would have the following check in the DEBUG code:
6759 *
6760 * if (bp->b_prev != NULL) {
6761 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6762 * }
6763 *
6764 * This can't be done, however, since IP modifies qinfo
6765 * structure at run-time (switching between IPv4 qinfo and IPv6
6766 * qinfo), invalidating the check.
6767 * So the assignment to func is left here, but the ASSERT itself
6768 * is removed until the whole issue is resolved.
6769 */
6770 #endif
6771 ASSERT(q->q_sqhead == bp);
6772 q->q_sqhead = bp->b_next;
6773 bp->b_prev = bp->b_next = NULL;
6774 ASSERT(q->q_syncqmsgs > 0);
6775 mutex_exit(QLOCK(q));
6776
6777 ASSERT(bp->b_datap->db_ref != 0);
6778
6779 (void) (*q->q_qinfo->qi_putp)(q, bp);
6780
6781 mutex_enter(QLOCK(q));
6782
6783 /*
6784 * q_syncqmsgs should only be decremented after executing the
6785 * put procedure to avoid message re-ordering. This is due to an
6786 * optimisation in putnext() which can call the put procedure
6787 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6788 * being set).
6789 *
6790 * We also need to clear QFULL in the next service procedure
6791 * queue if this is the last message destined for that queue.
6792 *
6793 * It would make better sense to have some sort of tunable for
6794 * the low water mark, but these semantics are not yet defined.
6795 * So, alas, we use a constant.
6796 */
6797 if (--q->q_syncqmsgs == 0) {
6798 mutex_exit(QLOCK(q));
6799 clr_qfull(q);
6800 mutex_enter(QLOCK(q));
6801 }
6802
6803 /*
6804 * Always clear SQ_EXCL when CIPUT in order to handle
6805 * qwriter(INNER). The putp() can call qwriter and get exclusive
6806 * access IFF this is the only claim. So, we need to test for
6807 * this possibility, acquire the mutex and clear the bit.
6808 */
6809 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6810 mutex_enter(SQLOCK(sq));
6811 sq->sq_flags &= ~SQ_EXCL;
6812 mutex_exit(SQLOCK(sq));
6813 }
6814 }
6815
6816 /*
6817 * We should either have no messages on this queue, or we were told to
6818 * goaway by a waiter (which we will wake up at the end of this
6819 * function).
6820 */
6821 ASSERT((q->q_sqhead == NULL) ||
6822 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6823
6824 ASSERT(MUTEX_HELD(QLOCK(q)));
6825 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6826
6827 /* Remove the q from the syncq list if all the messages are drained. */
6828 if (q->q_sqhead == NULL) {
6829 ASSERT(q->q_syncqmsgs == 0);
6830 mutex_enter(SQLOCK(sq));
6831 if (q->q_sqflags & Q_SQQUEUED)
6832 SQRM_Q(sq, q);
6833 mutex_exit(SQLOCK(sq));
6834 /*
6835 * Since the queue is removed from the list, reset its priority.
6836 */
6837 q->q_spri = 0;
6838 }
6839
6840 /*
6841 * Remember, the q_draining flag is used to let another thread know
6842 * that there is a thread currently draining the messages for a queue.
6843 * Since we are now done with this queue (even if there may be messages
6844 * still there), we need to clear this flag so some thread will work on
6845 * it if needed.
6846 */
6847 ASSERT(q->q_draining);
6848 q->q_draining = 0;
6849
6850 /* Called with a claim, so OK to drop all locks. */
6851 mutex_exit(QLOCK(q));
6852
6853 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6854 "drain_syncq end:%p", sq);
6855 }
6856 /* END OF QDRAIN_SYNCQ */
6857
6858
6859 /*
6860 * This is the mate to qdrain_syncq, except that it is putting the message onto
6861 * the queue instead of draining. Since the message is destined for the queue
6862 * that is selected, there is no need to identify the function because the
6863 * message is intended for the put routine for the queue. For debug kernels,
6864 * this routine will do it anyway just in case.
6865 *
6866 * After the message is enqueued on the syncq, it calls putnext_tail()
6867 * which will schedule a background thread to actually process the message.
6868 *
6869 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6870 * SQLOCK(sq) and QLOCK(q) are not held.
6871 */
6872 void
6873 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6874 {
6875 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6876 ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6877 ASSERT(sq->sq_count > 0);
6878 ASSERT(q->q_syncq == sq);
6879 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6880 sq->sq_oprev == NULL) ||
6881 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6882 sq->sq_oprev != NULL));
6883
6884 mutex_enter(QLOCK(q));
6885
6886 #ifdef DEBUG
6887 /*
6888 * This is used for debug in the qfill_syncq/qdrain_syncq case
6889 * to trace the queue that the message is intended for. Note
6890 * that the original use was to identify the queue and function
6891 * to call on the drain. In the new syncq, we have the context
6892 * of the queue that we are draining, so call it's putproc and
6893 * don't rely on the saved values. But for debug this is still
6894 * useful information.
6895 */
6896 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6897 mp->b_queue = q;
6898 mp->b_next = NULL;
6899 #endif
6900 ASSERT(q->q_syncq == sq);
6901 /*
6902 * Enqueue the message on the list.
6903 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to
6904 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP().
6905 */
6906 SQPUT_MP(q, mp);
6907 mutex_enter(SQLOCK(sq));
6908
6909 /*
6910 * And queue on syncq for scheduling, if not already queued.
6911 * Note that we need the SQLOCK for this, and for testing flags
6912 * at the end to see if we will drain. So grab it now, and
6913 * release it before we call qdrain_syncq or return.
6914 */
6915 if (!(q->q_sqflags & Q_SQQUEUED)) {
6916 q->q_spri = curthread->t_pri;
6917 SQPUT_Q(sq, q);
6918 }
6919 #ifdef DEBUG
6920 else {
6921 /*
6922 * All of these conditions MUST be true!
6923 */
6924 ASSERT(sq->sq_tail != NULL);
6925 if (sq->sq_tail == sq->sq_head) {
6926 ASSERT((q->q_sqprev == NULL) &&
6927 (q->q_sqnext == NULL));
6928 } else {
6929 ASSERT((q->q_sqprev != NULL) ||
6930 (q->q_sqnext != NULL));
6931 }
6932 ASSERT(sq->sq_flags & SQ_QUEUED);
6933 ASSERT(q->q_syncqmsgs != 0);
6934 ASSERT(q->q_sqflags & Q_SQQUEUED);
6935 }
6936 #endif
6937 mutex_exit(QLOCK(q));
6938 /*
6939 * SQLOCK is still held, so sq_count can be safely decremented.
6940 */
6941 sq->sq_count--;
6942
6943 putnext_tail(sq, q, 0);
6944 /* Should not reference sq or q after this point. */
6945 }
6946
6947 /* End of qfill_syncq */
6948
6949 /*
6950 * Remove all messages from a syncq (if qp is NULL) or remove all messages
6951 * that would be put into qp by drain_syncq.
6952 * Used when deleting the syncq (qp == NULL) or when detaching
6953 * a queue (qp != NULL).
6954 * Return non-zero if one or more messages were freed.
6955 *
6956 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6957 * sq_putlocks are used.
6958 *
6959 * NOTE: This function assumes that it is called from the close() context and
6960 * that all the queues in the syncq are going away. For this reason it doesn't
6961 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6962 * currently valid, but it is useful to rethink this function to behave properly
6963 * in other cases.
6964 */
6965 int
6966 flush_syncq(syncq_t *sq, queue_t *qp)
6967 {
6968 mblk_t *bp, *mp_head, *mp_next, *mp_prev;
6969 queue_t *q;
6970 int ret = 0;
6971
6972 mutex_enter(SQLOCK(sq));
6973
6974 /*
6975 * Before we leave, we need to make sure there are no
6976 * events listed for this queue. All events for this queue
6977 * will just be freed.
6978 */
6979 if (qp != NULL && sq->sq_evhead != NULL) {
6980 ASSERT(sq->sq_flags & SQ_EVENTS);
6981
6982 mp_prev = NULL;
6983 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6984 mp_next = bp->b_next;
6985 if (bp->b_queue == qp) {
6986 /* Delete this message */
6987 if (mp_prev != NULL) {
6988 mp_prev->b_next = mp_next;
6989 /*
6990 * Update sq_evtail if the last element
6991 * is removed.
6992 */
6993 if (bp == sq->sq_evtail) {
6994 ASSERT(mp_next == NULL);
6995 sq->sq_evtail = mp_prev;
6996 }
6997 } else
6998 sq->sq_evhead = mp_next;
6999 if (sq->sq_evhead == NULL)
7000 sq->sq_flags &= ~SQ_EVENTS;
7001 bp->b_prev = bp->b_next = NULL;
7002 freemsg(bp);
7003 ret++;
7004 } else {
7005 mp_prev = bp;
7006 }
7007 }
7008 }
7009
7010 /*
7011 * Walk sq_head and:
7012 * - match qp if qp is set, remove it's messages
7013 * - all if qp is not set
7014 */
7015 q = sq->sq_head;
7016 while (q != NULL) {
7017 ASSERT(q->q_syncq == sq);
7018 if ((qp == NULL) || (qp == q)) {
7019 /*
7020 * Yank the messages as a list off the queue
7021 */
7022 mp_head = q->q_sqhead;
7023 /*
7024 * We do not have QLOCK(q) here (which is safe due to
7025 * assumptions mentioned above). To obtain the lock we
7026 * need to release SQLOCK which may allow lots of things
7027 * to change upon us. This place requires more analysis.
7028 */
7029 q->q_sqhead = q->q_sqtail = NULL;
7030 ASSERT(mp_head->b_queue &&
7031 mp_head->b_queue->q_syncq == sq);
7032
7033 /*
7034 * Free each of the messages.
7035 */
7036 for (bp = mp_head; bp != NULL; bp = mp_next) {
7037 mp_next = bp->b_next;
7038 bp->b_prev = bp->b_next = NULL;
7039 freemsg(bp);
7040 ret++;
7041 }
7042 /*
7043 * Now remove the queue from the syncq.
7044 */
7045 ASSERT(q->q_sqflags & Q_SQQUEUED);
7046 SQRM_Q(sq, q);
7047 q->q_spri = 0;
7048 q->q_syncqmsgs = 0;
7049
7050 /*
7051 * If qp was specified, we are done with it and are
7052 * going to drop SQLOCK(sq) and return. We wakeup syncq
7053 * waiters while we still have the SQLOCK.
7054 */
7055 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7056 sq->sq_flags &= ~SQ_WANTWAKEUP;
7057 cv_broadcast(&sq->sq_wait);
7058 }
7059 /* Drop SQLOCK across clr_qfull */
7060 mutex_exit(SQLOCK(sq));
7061
7062 /*
7063 * We avoid doing the test that drain_syncq does and
7064 * unconditionally clear qfull for every flushed
7065 * message. Since flush_syncq is only called during
7066 * close this should not be a problem.
7067 */
7068 clr_qfull(q);
7069 if (qp != NULL) {
7070 return (ret);
7071 } else {
7072 mutex_enter(SQLOCK(sq));
7073 /*
7074 * The head was removed by SQRM_Q above.
7075 * reread the new head and flush it.
7076 */
7077 q = sq->sq_head;
7078 }
7079 } else {
7080 q = q->q_sqnext;
7081 }
7082 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7083 }
7084
7085 if (sq->sq_flags & SQ_WANTWAKEUP) {
7086 sq->sq_flags &= ~SQ_WANTWAKEUP;
7087 cv_broadcast(&sq->sq_wait);
7088 }
7089
7090 mutex_exit(SQLOCK(sq));
7091 return (ret);
7092 }
7093
7094 /*
7095 * Propagate all messages from a syncq to the next syncq that are associated
7096 * with the specified queue. If the queue is attached to a driver or if the
7097 * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7098 *
7099 * Assumes that the stream is strlock()'ed. We don't come here if there
7100 * are no messages to propagate.
7101 *
7102 * NOTE : If the queue is attached to a driver, all the messages are freed
7103 * as there is no point in propagating the messages from the driver syncq
7104 * to the closing stream head which will in turn get freed later.
7105 */
7106 static int
7107 propagate_syncq(queue_t *qp)
7108 {
7109 mblk_t *bp, *head, *tail, *prev, *next;
7110 syncq_t *sq;
7111 queue_t *nqp;
7112 syncq_t *nsq;
7113 boolean_t isdriver;
7114 int moved = 0;
7115 uint16_t flags;
7116 pri_t priority = curthread->t_pri;
7117 #ifdef DEBUG
7118 void (*func)();
7119 #endif
7120
7121 sq = qp->q_syncq;
7122 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7123 /* debug macro */
7124 SQ_PUTLOCKS_HELD(sq);
7125 /*
7126 * As entersq() does not increment the sq_count for
7127 * the write side, check sq_count for non-QPERQ
7128 * perimeters alone.
7129 */
7130 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7131
7132 /*
7133 * propagate_syncq() can be called because of either messages on the
7134 * queue syncq or because on events on the queue syncq. Do actual
7135 * message propagations if there are any messages.
7136 */
7137 if (qp->q_syncqmsgs) {
7138 isdriver = (qp->q_flag & QISDRV);
7139
7140 if (!isdriver) {
7141 nqp = qp->q_next;
7142 nsq = nqp->q_syncq;
7143 ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7144 /* debug macro */
7145 SQ_PUTLOCKS_HELD(nsq);
7146 #ifdef DEBUG
7147 func = (void (*)())nqp->q_qinfo->qi_putp;
7148 #endif
7149 }
7150
7151 SQRM_Q(sq, qp);
7152 priority = MAX(qp->q_spri, priority);
7153 qp->q_spri = 0;
7154 head = qp->q_sqhead;
7155 tail = qp->q_sqtail;
7156 qp->q_sqhead = qp->q_sqtail = NULL;
7157 qp->q_syncqmsgs = 0;
7158
7159 /*
7160 * Walk the list of messages, and free them if this is a driver,
7161 * otherwise reset the b_prev and b_queue value to the new putp.
7162 * Afterward, we will just add the head to the end of the next
7163 * syncq, and point the tail to the end of this one.
7164 */
7165
7166 for (bp = head; bp != NULL; bp = next) {
7167 next = bp->b_next;
7168 if (isdriver) {
7169 bp->b_prev = bp->b_next = NULL;
7170 freemsg(bp);
7171 continue;
7172 }
7173 /* Change the q values for this message */
7174 bp->b_queue = nqp;
7175 #ifdef DEBUG
7176 bp->b_prev = (mblk_t *)func;
7177 #endif
7178 moved++;
7179 }
7180 /*
7181 * Attach list of messages to the end of the new queue (if there
7182 * is a list of messages).
7183 */
7184
7185 if (!isdriver && head != NULL) {
7186 ASSERT(tail != NULL);
7187 if (nqp->q_sqhead == NULL) {
7188 nqp->q_sqhead = head;
7189 } else {
7190 ASSERT(nqp->q_sqtail != NULL);
7191 nqp->q_sqtail->b_next = head;
7192 }
7193 nqp->q_sqtail = tail;
7194 /*
7195 * When messages are moved from high priority queue to
7196 * another queue, the destination queue priority is
7197 * upgraded.
7198 */
7199
7200 if (priority > nqp->q_spri)
7201 nqp->q_spri = priority;
7202
7203 SQPUT_Q(nsq, nqp);
7204
7205 nqp->q_syncqmsgs += moved;
7206 ASSERT(nqp->q_syncqmsgs != 0);
7207 }
7208 }
7209
7210 /*
7211 * Before we leave, we need to make sure there are no
7212 * events listed for this queue. All events for this queue
7213 * will just be freed.
7214 */
7215 if (sq->sq_evhead != NULL) {
7216 ASSERT(sq->sq_flags & SQ_EVENTS);
7217 prev = NULL;
7218 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7219 next = bp->b_next;
7220 if (bp->b_queue == qp) {
7221 /* Delete this message */
7222 if (prev != NULL) {
7223 prev->b_next = next;
7224 /*
7225 * Update sq_evtail if the last element
7226 * is removed.
7227 */
7228 if (bp == sq->sq_evtail) {
7229 ASSERT(next == NULL);
7230 sq->sq_evtail = prev;
7231 }
7232 } else
7233 sq->sq_evhead = next;
7234 if (sq->sq_evhead == NULL)
7235 sq->sq_flags &= ~SQ_EVENTS;
7236 bp->b_prev = bp->b_next = NULL;
7237 freemsg(bp);
7238 } else {
7239 prev = bp;
7240 }
7241 }
7242 }
7243
7244 flags = sq->sq_flags;
7245
7246 /* Wake up any waiter before leaving. */
7247 if (flags & SQ_WANTWAKEUP) {
7248 flags &= ~SQ_WANTWAKEUP;
7249 cv_broadcast(&sq->sq_wait);
7250 }
7251 sq->sq_flags = flags;
7252
7253 return (moved);
7254 }
7255
7256 /*
7257 * Try and upgrade to exclusive access at the inner perimeter. If this can
7258 * not be done without blocking then request will be queued on the syncq
7259 * and drain_syncq will run it later.
7260 *
7261 * This routine can only be called from put or service procedures plus
7262 * asynchronous callback routines that have properly entered the queue (with
7263 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7264 * associated with q.
7265 */
7266 void
7267 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7268 {
7269 syncq_t *sq = q->q_syncq;
7270 uint16_t count;
7271
7272 mutex_enter(SQLOCK(sq));
7273 count = sq->sq_count;
7274 SQ_PUTLOCKS_ENTER(sq);
7275 SUM_SQ_PUTCOUNTS(sq, count);
7276 ASSERT(count >= 1);
7277 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7278
7279 if (count == 1) {
7280 /*
7281 * Can upgrade. This case also handles nested qwriter calls
7282 * (when the qwriter callback function calls qwriter). In that
7283 * case SQ_EXCL is already set.
7284 */
7285 sq->sq_flags |= SQ_EXCL;
7286 SQ_PUTLOCKS_EXIT(sq);
7287 mutex_exit(SQLOCK(sq));
7288 (*func)(q, mp);
7289 /*
7290 * Assumes that leavesq, putnext, and drain_syncq will reset
7291 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7292 * until putnext, leavesq, or drain_syncq drops it.
7293 * That way we handle nested qwriter(INNER) without dropping
7294 * SQ_EXCL until the outermost qwriter callback routine is
7295 * done.
7296 */
7297 return;
7298 }
7299 SQ_PUTLOCKS_EXIT(sq);
7300 sqfill_events(sq, q, mp, func);
7301 }
7302
7303 /*
7304 * Synchronous callback support functions
7305 */
7306
7307 /*
7308 * Allocate a callback parameter structure.
7309 * Assumes that caller initializes the flags and the id.
7310 * Acquires SQLOCK(sq) if non-NULL is returned.
7311 */
7312 callbparams_t *
7313 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7314 {
7315 callbparams_t *cbp;
7316 size_t size = sizeof (callbparams_t);
7317
7318 cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7319
7320 /*
7321 * Only try tryhard allocation if the caller is ready to panic.
7322 * Otherwise just fail.
7323 */
7324 if (cbp == NULL) {
7325 if (kmflags & KM_PANIC)
7326 cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7327 &size, kmflags);
7328 else
7329 return (NULL);
7330 }
7331
7332 ASSERT(size >= sizeof (callbparams_t));
7333 cbp->cbp_size = size;
7334 cbp->cbp_sq = sq;
7335 cbp->cbp_func = func;
7336 cbp->cbp_arg = arg;
7337 mutex_enter(SQLOCK(sq));
7338 cbp->cbp_next = sq->sq_callbpend;
7339 sq->sq_callbpend = cbp;
7340 return (cbp);
7341 }
7342
7343 void
7344 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7345 {
7346 callbparams_t **pp, *p;
7347
7348 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7349
7350 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7351 if (p == cbp) {
7352 *pp = p->cbp_next;
7353 kmem_free(p, p->cbp_size);
7354 return;
7355 }
7356 }
7357 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7358 "callbparams_free: not found\n"));
7359 }
7360
7361 void
7362 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7363 {
7364 callbparams_t **pp, *p;
7365
7366 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7367
7368 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7369 if (p->cbp_id == id && p->cbp_flags == flag) {
7370 *pp = p->cbp_next;
7371 kmem_free(p, p->cbp_size);
7372 return;
7373 }
7374 }
7375 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7376 "callbparams_free_id: not found\n"));
7377 }
7378
7379 /*
7380 * Callback wrapper function used by once-only callbacks that can be
7381 * cancelled (qtimeout and qbufcall)
7382 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7383 * cancelled by the qun* functions.
7384 */
7385 void
7386 qcallbwrapper(void *arg)
7387 {
7388 callbparams_t *cbp = arg;
7389 syncq_t *sq;
7390 uint16_t count = 0;
7391 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7392 uint16_t type;
7393
7394 sq = cbp->cbp_sq;
7395 mutex_enter(SQLOCK(sq));
7396 type = sq->sq_type;
7397 if (!(type & SQ_CICB)) {
7398 count = sq->sq_count;
7399 SQ_PUTLOCKS_ENTER(sq);
7400 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7401 SUM_SQ_PUTCOUNTS(sq, count);
7402 sq->sq_needexcl++;
7403 ASSERT(sq->sq_needexcl != 0); /* wraparound */
7404 waitflags |= SQ_MESSAGES;
7405 }
7406 /* Can not handle exclusive entry at outer perimeter */
7407 ASSERT(type & SQ_COCB);
7408
7409 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7410 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7411 (sq->sq_cancelid == cbp->cbp_id)) {
7412 /* timeout has been cancelled */
7413 sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7414 callbparams_free(sq, cbp);
7415 if (!(type & SQ_CICB)) {
7416 ASSERT(sq->sq_needexcl > 0);
7417 sq->sq_needexcl--;
7418 if (sq->sq_needexcl == 0) {
7419 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7420 }
7421 SQ_PUTLOCKS_EXIT(sq);
7422 }
7423 mutex_exit(SQLOCK(sq));
7424 return;
7425 }
7426 sq->sq_flags |= SQ_WANTWAKEUP;
7427 if (!(type & SQ_CICB)) {
7428 SQ_PUTLOCKS_EXIT(sq);
7429 }
7430 cv_wait(&sq->sq_wait, SQLOCK(sq));
7431 if (!(type & SQ_CICB)) {
7432 count = sq->sq_count;
7433 SQ_PUTLOCKS_ENTER(sq);
7434 SUM_SQ_PUTCOUNTS(sq, count);
7435 }
7436 }
7437
7438 sq->sq_count++;
7439 ASSERT(sq->sq_count != 0); /* Wraparound */
7440 if (!(type & SQ_CICB)) {
7441 ASSERT(count == 0);
7442 sq->sq_flags |= SQ_EXCL;
7443 ASSERT(sq->sq_needexcl > 0);
7444 sq->sq_needexcl--;
7445 if (sq->sq_needexcl == 0) {
7446 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7447 }
7448 SQ_PUTLOCKS_EXIT(sq);
7449 }
7450
7451 mutex_exit(SQLOCK(sq));
7452
7453 cbp->cbp_func(cbp->cbp_arg);
7454
7455 /*
7456 * We drop the lock only for leavesq to re-acquire it.
7457 * Possible optimization is inline of leavesq.
7458 */
7459 mutex_enter(SQLOCK(sq));
7460 callbparams_free(sq, cbp);
7461 mutex_exit(SQLOCK(sq));
7462 leavesq(sq, SQ_CALLBACK);
7463 }
7464
7465 /*
7466 * No need to grab sq_putlocks here. See comment in strsubr.h that
7467 * explains when sq_putlocks are used.
7468 *
7469 * sq_count (or one of the sq_putcounts) has already been
7470 * decremented by the caller, and if SQ_QUEUED, we need to call
7471 * drain_syncq (the global syncq drain).
7472 * If putnext_tail is called with the SQ_EXCL bit set, we are in
7473 * one of two states, non-CIPUT perimeter, and we need to clear
7474 * it, or we went exclusive in the put procedure. In any case,
7475 * we want to clear the bit now, and it is probably easier to do
7476 * this at the beginning of this function (remember, we hold
7477 * the SQLOCK). Lastly, if there are other messages queued
7478 * on the syncq (and not for our destination), enable the syncq
7479 * for background work.
7480 */
7481
7482 /* ARGSUSED */
7483 void
7484 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7485 {
7486 uint16_t flags = sq->sq_flags;
7487
7488 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7489 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7490
7491 /* Clear SQ_EXCL if set in passflags */
7492 if (passflags & SQ_EXCL) {
7493 flags &= ~SQ_EXCL;
7494 }
7495 if (flags & SQ_WANTWAKEUP) {
7496 flags &= ~SQ_WANTWAKEUP;
7497 cv_broadcast(&sq->sq_wait);
7498 }
7499 if (flags & SQ_WANTEXWAKEUP) {
7500 flags &= ~SQ_WANTEXWAKEUP;
7501 cv_broadcast(&sq->sq_exitwait);
7502 }
7503 sq->sq_flags = flags;
7504
7505 /*
7506 * We have cleared SQ_EXCL if we were asked to, and started
7507 * the wakeup process for waiters. If there are no writers
7508 * then we need to drain the syncq if we were told to, or
7509 * enable the background thread to do it.
7510 */
7511 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7512 if ((passflags & SQ_QUEUED) ||
7513 (sq->sq_svcflags & SQ_DISABLED)) {
7514 /* drain_syncq will take care of events in the list */
7515 drain_syncq(sq);
7516 return;
7517 } else if (flags & SQ_QUEUED) {
7518 sqenable(sq);
7519 }
7520 }
7521 /* Drop the SQLOCK on exit */
7522 mutex_exit(SQLOCK(sq));
7523 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7524 "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7525 }
7526
7527 void
7528 set_qend(queue_t *q)
7529 {
7530 mutex_enter(QLOCK(q));
7531 if (!O_SAMESTR(q))
7532 q->q_flag |= QEND;
7533 else
7534 q->q_flag &= ~QEND;
7535 mutex_exit(QLOCK(q));
7536 q = _OTHERQ(q);
7537 mutex_enter(QLOCK(q));
7538 if (!O_SAMESTR(q))
7539 q->q_flag |= QEND;
7540 else
7541 q->q_flag &= ~QEND;
7542 mutex_exit(QLOCK(q));
7543 }
7544
7545 /*
7546 * Set QFULL in next service procedure queue (that cares) if not already
7547 * set and if there are already more messages on the syncq than
7548 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on
7549 * any syncq.
7550 *
7551 * The fq here is the next queue with a service procedure. This is where
7552 * we would fail canputnext, so this is where we need to set QFULL.
7553 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7554 *
7555 * We already have QLOCK at this point. To avoid cross-locks with
7556 * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7557 * SQLOCK and sd_reflock, we need to drop respective locks first.
7558 */
7559 void
7560 set_qfull(queue_t *q)
7561 {
7562 queue_t *fq = NULL;
7563
7564 ASSERT(MUTEX_HELD(QLOCK(q)));
7565 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7566 (q->q_syncqmsgs > sq_max_size)) {
7567 if ((fq = q->q_nfsrv) == q) {
7568 fq->q_flag |= QFULL;
7569 } else {
7570 mutex_exit(QLOCK(q));
7571 mutex_enter(QLOCK(fq));
7572 fq->q_flag |= QFULL;
7573 mutex_exit(QLOCK(fq));
7574 mutex_enter(QLOCK(q));
7575 }
7576 }
7577 }
7578
7579 void
7580 clr_qfull(queue_t *q)
7581 {
7582 queue_t *oq = q;
7583
7584 q = q->q_nfsrv;
7585 /* Fast check if there is any work to do before getting the lock. */
7586 if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7587 return;
7588 }
7589
7590 /*
7591 * Do not reset QFULL (and backenable) if the q_count is the reason
7592 * for QFULL being set.
7593 */
7594 mutex_enter(QLOCK(q));
7595 /*
7596 * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7597 * Hence clear the QFULL.
7598 * If both q_count and q_mblkcnt are less than the hiwat mark,
7599 * clear the QFULL.
7600 */
7601 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7602 (q->q_mblkcnt < q->q_hiwat))) {
7603 q->q_flag &= ~QFULL;
7604 /*
7605 * A little more confusing, how about this way:
7606 * if someone wants to write,
7607 * AND
7608 * both counts are less than the lowat mark
7609 * OR
7610 * the lowat mark is zero
7611 * THEN
7612 * backenable
7613 */
7614 if ((q->q_flag & QWANTW) &&
7615 (((q->q_count < q->q_lowat) &&
7616 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7617 q->q_flag &= ~QWANTW;
7618 mutex_exit(QLOCK(q));
7619 backenable(oq, 0);
7620 } else
7621 mutex_exit(QLOCK(q));
7622 } else
7623 mutex_exit(QLOCK(q));
7624 }
7625
7626 /*
7627 * Set the forward service procedure pointer.
7628 *
7629 * Called at insert-time to cache a queue's next forward service procedure in
7630 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted
7631 * has a service procedure then q_nfsrv points to itself. If the queue to be
7632 * inserted does not have a service procedure, then q_nfsrv points to the next
7633 * queue forward that has a service procedure. If the queue is at the logical
7634 * end of the stream (driver for write side, stream head for the read side)
7635 * and does not have a service procedure, then q_nfsrv also points to itself.
7636 */
7637 void
7638 set_nfsrv_ptr(
7639 queue_t *rnew, /* read queue pointer to new module */
7640 queue_t *wnew, /* write queue pointer to new module */
7641 queue_t *prev_rq, /* read queue pointer to the module above */
7642 queue_t *prev_wq) /* write queue pointer to the module above */
7643 {
7644 queue_t *qp;
7645
7646 if (prev_wq->q_next == NULL) {
7647 /*
7648 * Insert the driver, initialize the driver and stream head.
7649 * In this case, prev_rq/prev_wq should be the stream head.
7650 * _I_INSERT does not allow inserting a driver. Make sure
7651 * that it is not an insertion.
7652 */
7653 ASSERT(!(rnew->q_flag & _QINSERTING));
7654 wnew->q_nfsrv = wnew;
7655 if (rnew->q_qinfo->qi_srvp)
7656 rnew->q_nfsrv = rnew;
7657 else
7658 rnew->q_nfsrv = prev_rq;
7659 prev_rq->q_nfsrv = prev_rq;
7660 prev_wq->q_nfsrv = prev_wq;
7661 } else {
7662 /*
7663 * set up read side q_nfsrv pointer. This MUST be done
7664 * before setting the write side, because the setting of
7665 * the write side for a fifo may depend on it.
7666 *
7667 * Suppose we have a fifo that only has pipemod pushed.
7668 * pipemod has no read or write service procedures, so
7669 * nfsrv for both pipemod queues points to prev_rq (the
7670 * stream read head). Now push bufmod (which has only a
7671 * read service procedure). Doing the write side first,
7672 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7673 * is WRONG; the next queue forward from wnew with a
7674 * service procedure will be rnew, not the stream read head.
7675 * Since the downstream queue (which in the case of a fifo
7676 * is the read queue rnew) can affect upstream queues, it
7677 * needs to be done first. Setting up the read side first
7678 * sets nfsrv for both pipemod queues to rnew and then
7679 * when the write side is set up, wnew-q_nfsrv will also
7680 * point to rnew.
7681 */
7682 if (rnew->q_qinfo->qi_srvp) {
7683 /*
7684 * use _OTHERQ() because, if this is a pipe, next
7685 * module may have been pushed from other end and
7686 * q_next could be a read queue.
7687 */
7688 qp = _OTHERQ(prev_wq->q_next);
7689 while (qp && qp->q_nfsrv != qp) {
7690 qp->q_nfsrv = rnew;
7691 qp = backq(qp);
7692 }
7693 rnew->q_nfsrv = rnew;
7694 } else
7695 rnew->q_nfsrv = prev_rq->q_nfsrv;
7696
7697 /* set up write side q_nfsrv pointer */
7698 if (wnew->q_qinfo->qi_srvp) {
7699 wnew->q_nfsrv = wnew;
7700
7701 /*
7702 * For insertion, need to update nfsrv of the modules
7703 * above which do not have a service routine.
7704 */
7705 if (rnew->q_flag & _QINSERTING) {
7706 for (qp = prev_wq;
7707 qp != NULL && qp->q_nfsrv != qp;
7708 qp = backq(qp)) {
7709 qp->q_nfsrv = wnew->q_nfsrv;
7710 }
7711 }
7712 } else {
7713 if (prev_wq->q_next == prev_rq)
7714 /*
7715 * Since prev_wq/prev_rq are the middle of a
7716 * fifo, wnew/rnew will also be the middle of
7717 * a fifo and wnew's nfsrv is same as rnew's.
7718 */
7719 wnew->q_nfsrv = rnew->q_nfsrv;
7720 else
7721 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7722 }
7723 }
7724 }
7725
7726 /*
7727 * Reset the forward service procedure pointer; called at remove-time.
7728 */
7729 void
7730 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7731 {
7732 queue_t *tmp_qp;
7733
7734 /* Reset the write side q_nfsrv pointer for _I_REMOVE */
7735 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7736 for (tmp_qp = backq(wqp);
7737 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7738 tmp_qp = backq(tmp_qp)) {
7739 tmp_qp->q_nfsrv = wqp->q_nfsrv;
7740 }
7741 }
7742
7743 /* reset the read side q_nfsrv pointer */
7744 if (rqp->q_qinfo->qi_srvp) {
7745 if (wqp->q_next) { /* non-driver case */
7746 tmp_qp = _OTHERQ(wqp->q_next);
7747 while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7748 /* Note that rqp->q_next cannot be NULL */
7749 ASSERT(rqp->q_next != NULL);
7750 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7751 tmp_qp = backq(tmp_qp);
7752 }
7753 }
7754 }
7755 }
7756
7757 /*
7758 * This routine should be called after all stream geometry changes to update
7759 * the stream head cached struio() rd/wr queue pointers. Note must be called
7760 * with the streamlock()ed.
7761 *
7762 * Note: only enables Synchronous STREAMS for a side of a Stream which has
7763 * an explicit synchronous barrier module queue. That is, a queue that
7764 * has specified a struio() type.
7765 */
7766 static void
7767 strsetuio(stdata_t *stp)
7768 {
7769 queue_t *wrq;
7770
7771 if (stp->sd_flag & STPLEX) {
7772 /*
7773 * Not streamhead, but a mux, so no Synchronous STREAMS.
7774 */
7775 stp->sd_struiowrq = NULL;
7776 stp->sd_struiordq = NULL;
7777 return;
7778 }
7779 /*
7780 * Scan the write queue(s) while synchronous
7781 * until we find a qinfo uio type specified.
7782 */
7783 wrq = stp->sd_wrq->q_next;
7784 while (wrq) {
7785 if (wrq->q_struiot == STRUIOT_NONE) {
7786 wrq = 0;
7787 break;
7788 }
7789 if (wrq->q_struiot != STRUIOT_DONTCARE)
7790 break;
7791 if (! _SAMESTR(wrq)) {
7792 wrq = 0;
7793 break;
7794 }
7795 wrq = wrq->q_next;
7796 }
7797 stp->sd_struiowrq = wrq;
7798 /*
7799 * Scan the read queue(s) while synchronous
7800 * until we find a qinfo uio type specified.
7801 */
7802 wrq = stp->sd_wrq->q_next;
7803 while (wrq) {
7804 if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7805 wrq = 0;
7806 break;
7807 }
7808 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7809 break;
7810 if (! _SAMESTR(wrq)) {
7811 wrq = 0;
7812 break;
7813 }
7814 wrq = wrq->q_next;
7815 }
7816 stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7817 }
7818
7819 /*
7820 * pass_wput, unblocks the passthru queues, so that
7821 * messages can arrive at muxs lower read queue, before
7822 * I_LINK/I_UNLINK is acked/nacked.
7823 */
7824 static void
7825 pass_wput(queue_t *q, mblk_t *mp)
7826 {
7827 syncq_t *sq;
7828
7829 sq = _RD(q)->q_syncq;
7830 if (sq->sq_flags & SQ_BLOCKED)
7831 unblocksq(sq, SQ_BLOCKED, 0);
7832 putnext(q, mp);
7833 }
7834
7835 /*
7836 * Set up queues for the link/unlink.
7837 * Create a new queue and block it and then insert it
7838 * below the stream head on the lower stream.
7839 * This prevents any messages from arriving during the setq
7840 * as well as while the mux is processing the LINK/I_UNLINK.
7841 * The blocked passq is unblocked once the LINK/I_UNLINK has
7842 * been acked or nacked or if a message is generated and sent
7843 * down muxs write put procedure.
7844 * See pass_wput().
7845 *
7846 * After the new queue is inserted, all messages coming from below are
7847 * blocked. The call to strlock will ensure that all activity in the stream head
7848 * read queue syncq is stopped (sq_count drops to zero).
7849 */
7850 static queue_t *
7851 link_addpassthru(stdata_t *stpdown)
7852 {
7853 queue_t *passq;
7854 sqlist_t sqlist;
7855
7856 passq = allocq();
7857 STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7858 /* setq might sleep in allocator - avoid holding locks. */
7859 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7860 SQ_CI|SQ_CO, B_FALSE);
7861 claimq(passq);
7862 blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7863 insertq(STREAM(passq), passq);
7864
7865 /*
7866 * Use strlock() to wait for the stream head sq_count to drop to zero
7867 * since we are going to change q_ptr in the stream head. Note that
7868 * insertq() doesn't wait for any syncq counts to drop to zero.
7869 */
7870 sqlist.sqlist_head = NULL;
7871 sqlist.sqlist_index = 0;
7872 sqlist.sqlist_size = sizeof (sqlist_t);
7873 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7874 strlock(stpdown, &sqlist);
7875 strunlock(stpdown, &sqlist);
7876
7877 releaseq(passq);
7878 return (passq);
7879 }
7880
7881 /*
7882 * Let messages flow up into the mux by removing
7883 * the passq.
7884 */
7885 static void
7886 link_rempassthru(queue_t *passq)
7887 {
7888 claimq(passq);
7889 removeq(passq);
7890 releaseq(passq);
7891 freeq(passq);
7892 }
7893
7894 /*
7895 * Wait for the condition variable pointed to by `cvp' to be signaled,
7896 * or for `tim' milliseconds to elapse, whichever comes first. If `tim'
7897 * is negative, then there is no time limit. If `nosigs' is non-zero,
7898 * then the wait will be non-interruptible.
7899 *
7900 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7901 */
7902 clock_t
7903 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7904 {
7905 clock_t ret;
7906
7907 if (tim < 0) {
7908 if (nosigs) {
7909 cv_wait(cvp, mp);
7910 ret = 1;
7911 } else {
7912 ret = cv_wait_sig(cvp, mp);
7913 }
7914 } else if (tim > 0) {
7915 /*
7916 * convert milliseconds to clock ticks
7917 */
7918 if (nosigs) {
7919 ret = cv_reltimedwait(cvp, mp,
7920 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7921 } else {
7922 ret = cv_reltimedwait_sig(cvp, mp,
7923 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7924 }
7925 } else {
7926 ret = -1;
7927 }
7928 return (ret);
7929 }
7930
7931 /*
7932 * Wait until the stream head can determine if it is at the mark but
7933 * don't wait forever to prevent a race condition between the "mark" state
7934 * in the stream head and any mark state in the caller/user of this routine.
7935 *
7936 * This is used by sockets and for a socket it would be incorrect
7937 * to return a failure for SIOCATMARK when there is no data in the receive
7938 * queue and the marked urgent data is traveling up the stream.
7939 *
7940 * This routine waits until the mark is known by waiting for one of these
7941 * three events:
7942 * The stream head read queue becoming non-empty (including an EOF).
7943 * The STRATMARK flag being set (due to a MSGMARKNEXT message).
7944 * The STRNOTATMARK flag being set (which indicates that the transport
7945 * has sent a MSGNOTMARKNEXT message to indicate that it is not at
7946 * the mark).
7947 *
7948 * The routine returns 1 if the stream is at the mark; 0 if it can
7949 * be determined that the stream is not at the mark.
7950 * If the wait times out and it can't determine
7951 * whether or not the stream might be at the mark the routine will return -1.
7952 *
7953 * Note: This routine should only be used when a mark is pending i.e.,
7954 * in the socket case the SIGURG has been posted.
7955 * Note2: This can not wakeup just because synchronous streams indicate
7956 * that data is available since it is not possible to use the synchronous
7957 * streams interfaces to determine the b_flag value for the data queued below
7958 * the stream head.
7959 */
7960 int
7961 strwaitmark(vnode_t *vp)
7962 {
7963 struct stdata *stp = vp->v_stream;
7964 queue_t *rq = _RD(stp->sd_wrq);
7965 int mark;
7966
7967 mutex_enter(&stp->sd_lock);
7968 while (rq->q_first == NULL &&
7969 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7970 stp->sd_flag |= RSLEEP;
7971
7972 /* Wait for 100 milliseconds for any state change. */
7973 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7974 mutex_exit(&stp->sd_lock);
7975 return (-1);
7976 }
7977 }
7978 if (stp->sd_flag & STRATMARK)
7979 mark = 1;
7980 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7981 mark = 1;
7982 else
7983 mark = 0;
7984
7985 mutex_exit(&stp->sd_lock);
7986 return (mark);
7987 }
7988
7989 /*
7990 * Set a read side error. If persist is set change the socket error
7991 * to persistent. If errfunc is set install the function as the exported
7992 * error handler.
7993 */
7994 void
7995 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
7996 {
7997 struct stdata *stp = vp->v_stream;
7998
7999 mutex_enter(&stp->sd_lock);
8000 stp->sd_rerror = error;
8001 if (error == 0 && errfunc == NULL)
8002 stp->sd_flag &= ~STRDERR;
8003 else
8004 stp->sd_flag |= STRDERR;
8005 if (persist) {
8006 stp->sd_flag &= ~STRDERRNONPERSIST;
8007 } else {
8008 stp->sd_flag |= STRDERRNONPERSIST;
8009 }
8010 stp->sd_rderrfunc = errfunc;
8011 if (error != 0 || errfunc != NULL) {
8012 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
8013 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8014 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8015
8016 mutex_exit(&stp->sd_lock);
8017 pollwakeup(&stp->sd_pollist, POLLERR);
8018 mutex_enter(&stp->sd_lock);
8019
8020 if (stp->sd_sigflags & S_ERROR)
8021 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8022 }
8023 mutex_exit(&stp->sd_lock);
8024 }
8025
8026 /*
8027 * Set a write side error. If persist is set change the socket error
8028 * to persistent.
8029 */
8030 void
8031 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8032 {
8033 struct stdata *stp = vp->v_stream;
8034
8035 mutex_enter(&stp->sd_lock);
8036 stp->sd_werror = error;
8037 if (error == 0 && errfunc == NULL)
8038 stp->sd_flag &= ~STWRERR;
8039 else
8040 stp->sd_flag |= STWRERR;
8041 if (persist) {
8042 stp->sd_flag &= ~STWRERRNONPERSIST;
8043 } else {
8044 stp->sd_flag |= STWRERRNONPERSIST;
8045 }
8046 stp->sd_wrerrfunc = errfunc;
8047 if (error != 0 || errfunc != NULL) {
8048 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
8049 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8050 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8051
8052 mutex_exit(&stp->sd_lock);
8053 pollwakeup(&stp->sd_pollist, POLLERR);
8054 mutex_enter(&stp->sd_lock);
8055
8056 if (stp->sd_sigflags & S_ERROR)
8057 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8058 }
8059 mutex_exit(&stp->sd_lock);
8060 }
8061
8062 /*
8063 * Make the stream return 0 (EOF) when all data has been read.
8064 * No effect on write side.
8065 */
8066 void
8067 strseteof(vnode_t *vp, int eof)
8068 {
8069 struct stdata *stp = vp->v_stream;
8070
8071 mutex_enter(&stp->sd_lock);
8072 if (!eof) {
8073 stp->sd_flag &= ~STREOF;
8074 mutex_exit(&stp->sd_lock);
8075 return;
8076 }
8077 stp->sd_flag |= STREOF;
8078 if (stp->sd_flag & RSLEEP) {
8079 stp->sd_flag &= ~RSLEEP;
8080 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8081 }
8082
8083 mutex_exit(&stp->sd_lock);
8084 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8085 mutex_enter(&stp->sd_lock);
8086
8087 if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8088 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8089 mutex_exit(&stp->sd_lock);
8090 }
8091
8092 void
8093 strflushrq(vnode_t *vp, int flag)
8094 {
8095 struct stdata *stp = vp->v_stream;
8096
8097 mutex_enter(&stp->sd_lock);
8098 flushq(_RD(stp->sd_wrq), flag);
8099 mutex_exit(&stp->sd_lock);
8100 }
8101
8102 void
8103 strsetrputhooks(vnode_t *vp, uint_t flags,
8104 msgfunc_t protofunc, msgfunc_t miscfunc)
8105 {
8106 struct stdata *stp = vp->v_stream;
8107
8108 mutex_enter(&stp->sd_lock);
8109
8110 if (protofunc == NULL)
8111 stp->sd_rprotofunc = strrput_proto;
8112 else
8113 stp->sd_rprotofunc = protofunc;
8114
8115 if (miscfunc == NULL)
8116 stp->sd_rmiscfunc = strrput_misc;
8117 else
8118 stp->sd_rmiscfunc = miscfunc;
8119
8120 if (flags & SH_CONSOL_DATA)
8121 stp->sd_rput_opt |= SR_CONSOL_DATA;
8122 else
8123 stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8124
8125 if (flags & SH_SIGALLDATA)
8126 stp->sd_rput_opt |= SR_SIGALLDATA;
8127 else
8128 stp->sd_rput_opt &= ~SR_SIGALLDATA;
8129
8130 if (flags & SH_IGN_ZEROLEN)
8131 stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8132 else
8133 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8134
8135 mutex_exit(&stp->sd_lock);
8136 }
8137
8138 void
8139 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8140 {
8141 struct stdata *stp = vp->v_stream;
8142
8143 mutex_enter(&stp->sd_lock);
8144 stp->sd_closetime = closetime;
8145
8146 if (flags & SH_SIGPIPE)
8147 stp->sd_wput_opt |= SW_SIGPIPE;
8148 else
8149 stp->sd_wput_opt &= ~SW_SIGPIPE;
8150 if (flags & SH_RECHECK_ERR)
8151 stp->sd_wput_opt |= SW_RECHECK_ERR;
8152 else
8153 stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8154
8155 mutex_exit(&stp->sd_lock);
8156 }
8157
8158 void
8159 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8160 {
8161 struct stdata *stp = vp->v_stream;
8162
8163 mutex_enter(&stp->sd_lock);
8164
8165 stp->sd_rputdatafunc = rdatafunc;
8166 stp->sd_wputdatafunc = wdatafunc;
8167
8168 mutex_exit(&stp->sd_lock);
8169 }
8170
8171 /* Used within framework when the queue is already locked */
8172 void
8173 qenable_locked(queue_t *q)
8174 {
8175 stdata_t *stp = STREAM(q);
8176
8177 ASSERT(MUTEX_HELD(QLOCK(q)));
8178
8179 if (!q->q_qinfo->qi_srvp)
8180 return;
8181
8182 /*
8183 * Do not place on run queue if already enabled or closing.
8184 */
8185 if (q->q_flag & (QWCLOSE|QENAB))
8186 return;
8187
8188 /*
8189 * mark queue enabled and place on run list if it is not already being
8190 * serviced. If it is serviced, the runservice() function will detect
8191 * that QENAB is set and call service procedure before clearing
8192 * QINSERVICE flag.
8193 */
8194 q->q_flag |= QENAB;
8195 if (q->q_flag & QINSERVICE)
8196 return;
8197
8198 /* Record the time of qenable */
8199 q->q_qtstamp = ddi_get_lbolt();
8200
8201 /*
8202 * Put the queue in the stp list and schedule it for background
8203 * processing if it is not already scheduled or if stream head does not
8204 * intent to process it in the foreground later by setting
8205 * STRS_WILLSERVICE flag.
8206 */
8207 mutex_enter(&stp->sd_qlock);
8208 /*
8209 * If there are already something on the list, stp flags should show
8210 * intention to drain it.
8211 */
8212 IMPLY(STREAM_NEEDSERVICE(stp),
8213 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8214
8215 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8216 stp->sd_nqueues++;
8217
8218 /*
8219 * If no one will drain this stream we are the first producer and
8220 * need to schedule it for background thread.
8221 */
8222 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8223 /*
8224 * No one will service this stream later, so we have to
8225 * schedule it now.
8226 */
8227 STRSTAT(stenables);
8228 stp->sd_svcflags |= STRS_SCHEDULED;
8229 stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8230 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8231
8232 if (stp->sd_servid == NULL) {
8233 /*
8234 * Task queue failed so fail over to the backup
8235 * servicing thread.
8236 */
8237 STRSTAT(taskqfails);
8238 /*
8239 * It is safe to clear STRS_SCHEDULED flag because it
8240 * was set by this thread above.
8241 */
8242 stp->sd_svcflags &= ~STRS_SCHEDULED;
8243
8244 /*
8245 * Failover scheduling is protected by service_queue
8246 * lock.
8247 */
8248 mutex_enter(&service_queue);
8249 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8250 ASSERT(q->q_link == NULL);
8251 /*
8252 * Append the queue to qhead/qtail list.
8253 */
8254 if (qhead == NULL)
8255 qhead = q;
8256 else
8257 qtail->q_link = q;
8258 qtail = q;
8259 /*
8260 * Clear stp queue list.
8261 */
8262 stp->sd_qhead = stp->sd_qtail = NULL;
8263 stp->sd_nqueues = 0;
8264 /*
8265 * Wakeup background queue processing thread.
8266 */
8267 cv_signal(&services_to_run);
8268 mutex_exit(&service_queue);
8269 }
8270 }
8271 mutex_exit(&stp->sd_qlock);
8272 }
8273
8274 static void
8275 queue_service(queue_t *q)
8276 {
8277 /*
8278 * The queue in the list should have
8279 * QENAB flag set and should not have
8280 * QINSERVICE flag set. QINSERVICE is
8281 * set when the queue is dequeued and
8282 * qenable_locked doesn't enqueue a
8283 * queue with QINSERVICE set.
8284 */
8285
8286 ASSERT(!(q->q_flag & QINSERVICE));
8287 ASSERT((q->q_flag & QENAB));
8288 mutex_enter(QLOCK(q));
8289 q->q_flag &= ~QENAB;
8290 q->q_flag |= QINSERVICE;
8291 mutex_exit(QLOCK(q));
8292 runservice(q);
8293 }
8294
8295 static void
8296 syncq_service(syncq_t *sq)
8297 {
8298 STRSTAT(syncqservice);
8299 mutex_enter(SQLOCK(sq));
8300 ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8301 ASSERT(sq->sq_servcount != 0);
8302 ASSERT(sq->sq_next == NULL);
8303
8304 /* if we came here from the background thread, clear the flag */
8305 if (sq->sq_svcflags & SQ_BGTHREAD)
8306 sq->sq_svcflags &= ~SQ_BGTHREAD;
8307
8308 /* let drain_syncq know that it's being called in the background */
8309 sq->sq_svcflags |= SQ_SERVICE;
8310 drain_syncq(sq);
8311 }
8312
8313 static void
8314 qwriter_outer_service(syncq_t *outer)
8315 {
8316 /*
8317 * Note that SQ_WRITER is used on the outer perimeter
8318 * to signal that a qwriter(OUTER) is either investigating
8319 * running or that it is actually running a function.
8320 */
8321 outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8322
8323 /*
8324 * All inner syncq are empty and have SQ_WRITER set
8325 * to block entering the outer perimeter.
8326 *
8327 * We do not need to explicitly call write_now since
8328 * outer_exit does it for us.
8329 */
8330 outer_exit(outer);
8331 }
8332
8333 static void
8334 mblk_free(mblk_t *mp)
8335 {
8336 dblk_t *dbp = mp->b_datap;
8337 frtn_t *frp = dbp->db_frtnp;
8338
8339 mp->b_next = NULL;
8340 if (dbp->db_fthdr != NULL)
8341 str_ftfree(dbp);
8342
8343 ASSERT(dbp->db_fthdr == NULL);
8344 frp->free_func(frp->free_arg);
8345 ASSERT(dbp->db_mblk == mp);
8346
8347 if (dbp->db_credp != NULL) {
8348 crfree(dbp->db_credp);
8349 dbp->db_credp = NULL;
8350 }
8351 dbp->db_cpid = -1;
8352 dbp->db_struioflag = 0;
8353 dbp->db_struioun.cksum.flags = 0;
8354
8355 kmem_cache_free(dbp->db_cache, dbp);
8356 }
8357
8358 /*
8359 * Background processing of the stream queue list.
8360 */
8361 static void
8362 stream_service(stdata_t *stp)
8363 {
8364 queue_t *q;
8365
8366 mutex_enter(&stp->sd_qlock);
8367
8368 STR_SERVICE(stp, q);
8369
8370 stp->sd_svcflags &= ~STRS_SCHEDULED;
8371 stp->sd_servid = NULL;
8372 cv_signal(&stp->sd_qcv);
8373 mutex_exit(&stp->sd_qlock);
8374 }
8375
8376 /*
8377 * Foreground processing of the stream queue list.
8378 */
8379 void
8380 stream_runservice(stdata_t *stp)
8381 {
8382 queue_t *q;
8383
8384 mutex_enter(&stp->sd_qlock);
8385 STRSTAT(rservice);
8386 /*
8387 * We are going to drain this stream queue list, so qenable_locked will
8388 * not schedule it until we finish.
8389 */
8390 stp->sd_svcflags |= STRS_WILLSERVICE;
8391
8392 STR_SERVICE(stp, q);
8393
8394 stp->sd_svcflags &= ~STRS_WILLSERVICE;
8395 mutex_exit(&stp->sd_qlock);
8396 /*
8397 * Help backup background thread to drain the qhead/qtail list.
8398 */
8399 while (qhead != NULL) {
8400 STRSTAT(qhelps);
8401 mutex_enter(&service_queue);
8402 DQ(q, qhead, qtail, q_link);
8403 mutex_exit(&service_queue);
8404 if (q != NULL)
8405 queue_service(q);
8406 }
8407 }
8408
8409 void
8410 stream_willservice(stdata_t *stp)
8411 {
8412 mutex_enter(&stp->sd_qlock);
8413 stp->sd_svcflags |= STRS_WILLSERVICE;
8414 mutex_exit(&stp->sd_qlock);
8415 }
8416
8417 /*
8418 * Replace the cred currently in the mblk with a different one.
8419 * Also update db_cpid.
8420 */
8421 void
8422 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8423 {
8424 dblk_t *dbp = mp->b_datap;
8425 cred_t *ocr = dbp->db_credp;
8426
8427 ASSERT(cr != NULL);
8428
8429 if (cr != ocr) {
8430 crhold(dbp->db_credp = cr);
8431 if (ocr != NULL)
8432 crfree(ocr);
8433 }
8434 /* Don't overwrite with NOPID */
8435 if (cpid != NOPID)
8436 dbp->db_cpid = cpid;
8437 }
8438
8439 /*
8440 * If the src message has a cred, then replace the cred currently in the mblk
8441 * with it.
8442 * Also update db_cpid.
8443 */
8444 void
8445 mblk_copycred(mblk_t *mp, const mblk_t *src)
8446 {
8447 dblk_t *dbp = mp->b_datap;
8448 cred_t *cr, *ocr;
8449 pid_t cpid;
8450
8451 cr = msg_getcred(src, &cpid);
8452 if (cr == NULL)
8453 return;
8454
8455 ocr = dbp->db_credp;
8456 if (cr != ocr) {
8457 crhold(dbp->db_credp = cr);
8458 if (ocr != NULL)
8459 crfree(ocr);
8460 }
8461 /* Don't overwrite with NOPID */
8462 if (cpid != NOPID)
8463 dbp->db_cpid = cpid;
8464 }
8465
8466 int
8467 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8468 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8469 uint32_t flags, int km_flags)
8470 {
8471 int rc = 0;
8472
8473 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8474 if (mp->b_datap->db_type == M_DATA) {
8475 /* Associate values for M_DATA type */
8476 DB_CKSUMSTART(mp) = (intptr_t)start;
8477 DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8478 DB_CKSUMEND(mp) = (intptr_t)end;
8479 DB_CKSUMFLAGS(mp) = flags;
8480 DB_CKSUM16(mp) = (uint16_t)value;
8481
8482 } else {
8483 pattrinfo_t pa_info;
8484
8485 ASSERT(mmd != NULL);
8486
8487 pa_info.type = PATTR_HCKSUM;
8488 pa_info.len = sizeof (pattr_hcksum_t);
8489
8490 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8491 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8492
8493 hck->hcksum_start_offset = start;
8494 hck->hcksum_stuff_offset = stuff;
8495 hck->hcksum_end_offset = end;
8496 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8497 hck->hcksum_flags = flags;
8498 } else {
8499 rc = -1;
8500 }
8501 }
8502 return (rc);
8503 }
8504
8505 void
8506 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8507 uint32_t *start, uint32_t *stuff, uint32_t *end,
8508 uint32_t *value, uint32_t *flags)
8509 {
8510 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8511 if (mp->b_datap->db_type == M_DATA) {
8512 if (flags != NULL) {
8513 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8514 if ((*flags & (HCK_PARTIALCKSUM |
8515 HCK_FULLCKSUM)) != 0) {
8516 if (value != NULL)
8517 *value = (uint32_t)DB_CKSUM16(mp);
8518 if ((*flags & HCK_PARTIALCKSUM) != 0) {
8519 if (start != NULL)
8520 *start =
8521 (uint32_t)DB_CKSUMSTART(mp);
8522 if (stuff != NULL)
8523 *stuff =
8524 (uint32_t)DB_CKSUMSTUFF(mp);
8525 if (end != NULL)
8526 *end =
8527 (uint32_t)DB_CKSUMEND(mp);
8528 }
8529 }
8530 }
8531 } else {
8532 pattrinfo_t hck_attr = {PATTR_HCKSUM};
8533
8534 ASSERT(mmd != NULL);
8535
8536 /* get hardware checksum attribute */
8537 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8538 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8539
8540 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8541 if (flags != NULL)
8542 *flags = hck->hcksum_flags;
8543 if (start != NULL)
8544 *start = hck->hcksum_start_offset;
8545 if (stuff != NULL)
8546 *stuff = hck->hcksum_stuff_offset;
8547 if (end != NULL)
8548 *end = hck->hcksum_end_offset;
8549 if (value != NULL)
8550 *value = (uint32_t)
8551 hck->hcksum_cksum_val.inet_cksum;
8552 }
8553 }
8554 }
8555
8556 void
8557 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8558 {
8559 ASSERT(DB_TYPE(mp) == M_DATA);
8560 ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8561
8562 /* Set the flags */
8563 DB_LSOFLAGS(mp) |= flags;
8564 DB_LSOMSS(mp) = mss;
8565 }
8566
8567 void
8568 lso_info_cleanup(mblk_t *mp)
8569 {
8570 ASSERT(DB_TYPE(mp) == M_DATA);
8571
8572 /* Clear the flags */
8573 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8574 DB_LSOMSS(mp) = 0;
8575 }
8576
8577 /*
8578 * Checksum buffer *bp for len bytes with psum partial checksum,
8579 * or 0 if none, and return the 16 bit partial checksum.
8580 */
8581 unsigned
8582 bcksum(uchar_t *bp, int len, unsigned int psum)
8583 {
8584 int odd = len & 1;
8585 extern unsigned int ip_ocsum();
8586
8587 if (((intptr_t)bp & 1) == 0 && !odd) {
8588 /*
8589 * Bp is 16 bit aligned and len is multiple of 16 bit word.
8590 */
8591 return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8592 }
8593 if (((intptr_t)bp & 1) != 0) {
8594 /*
8595 * Bp isn't 16 bit aligned.
8596 */
8597 unsigned int tsum;
8598
8599 #ifdef _LITTLE_ENDIAN
8600 psum += *bp;
8601 #else
8602 psum += *bp << 8;
8603 #endif
8604 len--;
8605 bp++;
8606 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8607 psum += (tsum << 8) & 0xffff | (tsum >> 8);
8608 if (len & 1) {
8609 bp += len - 1;
8610 #ifdef _LITTLE_ENDIAN
8611 psum += *bp << 8;
8612 #else
8613 psum += *bp;
8614 #endif
8615 }
8616 } else {
8617 /*
8618 * Bp is 16 bit aligned.
8619 */
8620 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8621 if (odd) {
8622 bp += len - 1;
8623 #ifdef _LITTLE_ENDIAN
8624 psum += *bp;
8625 #else
8626 psum += *bp << 8;
8627 #endif
8628 }
8629 }
8630 /*
8631 * Normalize psum to 16 bits before returning the new partial
8632 * checksum. The max psum value before normalization is 0x3FDFE.
8633 */
8634 return ((psum >> 16) + (psum & 0xFFFF));
8635 }
8636
8637 boolean_t
8638 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8639 {
8640 boolean_t rc;
8641
8642 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8643 if (DB_TYPE(mp) == M_DATA) {
8644 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8645 } else {
8646 pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8647
8648 ASSERT(mmd != NULL);
8649 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8650 }
8651 return (rc);
8652 }
8653
8654 void
8655 freemsgchain(mblk_t *mp)
8656 {
8657 mblk_t *next;
8658
8659 while (mp != NULL) {
8660 next = mp->b_next;
8661 mp->b_next = NULL;
8662
8663 freemsg(mp);
8664 mp = next;
8665 }
8666 }
8667
8668 mblk_t *
8669 copymsgchain(mblk_t *mp)
8670 {
8671 mblk_t *nmp = NULL;
8672 mblk_t **nmpp = &nmp;
8673
8674 for (; mp != NULL; mp = mp->b_next) {
8675 if ((*nmpp = copymsg(mp)) == NULL) {
8676 freemsgchain(nmp);
8677 return (NULL);
8678 }
8679
8680 nmpp = &((*nmpp)->b_next);
8681 }
8682
8683 return (nmp);
8684 }
8685
8686 /* NOTE: Do not add code after this point. */
8687 #undef QLOCK
8688
8689 /*
8690 * Replacement for QLOCK macro for those that can't use it.
8691 */
8692 kmutex_t *
8693 QLOCK(queue_t *q)
8694 {
8695 return (&(q)->q_lock);
8696 }
8697
8698 /*
8699 * Dummy runqueues/queuerun functions functions for backwards compatibility.
8700 */
8701 #undef runqueues
8702 void
8703 runqueues(void)
8704 {
8705 }
8706
8707 #undef queuerun
8708 void
8709 queuerun(void)
8710 {
8711 }
8712
8713 /*
8714 * Initialize the STR stack instance, which tracks autopush and persistent
8715 * links.
8716 */
8717 /* ARGSUSED */
8718 static void *
8719 str_stack_init(netstackid_t stackid, netstack_t *ns)
8720 {
8721 str_stack_t *ss;
8722 int i;
8723
8724 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8725 ss->ss_netstack = ns;
8726
8727 /*
8728 * set up autopush
8729 */
8730 sad_initspace(ss);
8731
8732 /*
8733 * set up mux_node structures.
8734 */
8735 ss->ss_devcnt = devcnt; /* In case it should change before free */
8736 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8737 ss->ss_devcnt), KM_SLEEP);
8738 for (i = 0; i < ss->ss_devcnt; i++)
8739 ss->ss_mux_nodes[i].mn_imaj = i;
8740 return (ss);
8741 }
8742
8743 /*
8744 * Note: run at zone shutdown and not destroy so that the PLINKs are
8745 * gone by the time other cleanup happens from the destroy callbacks.
8746 */
8747 static void
8748 str_stack_shutdown(netstackid_t stackid, void *arg)
8749 {
8750 str_stack_t *ss = (str_stack_t *)arg;
8751 int i;
8752 cred_t *cr;
8753
8754 cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8755 ASSERT(cr != NULL);
8756
8757 /* Undo all the I_PLINKs for this zone */
8758 for (i = 0; i < ss->ss_devcnt; i++) {
8759 struct mux_edge *ep;
8760 ldi_handle_t lh;
8761 ldi_ident_t li;
8762 int ret;
8763 int rval;
8764 dev_t rdev;
8765
8766 ep = ss->ss_mux_nodes[i].mn_outp;
8767 if (ep == NULL)
8768 continue;
8769 ret = ldi_ident_from_major((major_t)i, &li);
8770 if (ret != 0) {
8771 continue;
8772 }
8773 rdev = ep->me_dev;
8774 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8775 cr, &lh, li);
8776 if (ret != 0) {
8777 ldi_ident_release(li);
8778 continue;
8779 }
8780
8781 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8782 cr, &rval);
8783 if (ret) {
8784 (void) ldi_close(lh, FREAD|FWRITE, cr);
8785 ldi_ident_release(li);
8786 continue;
8787 }
8788 (void) ldi_close(lh, FREAD|FWRITE, cr);
8789
8790 /* Close layered handles */
8791 ldi_ident_release(li);
8792 }
8793 crfree(cr);
8794
8795 sad_freespace(ss);
8796
8797 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8798 ss->ss_mux_nodes = NULL;
8799 }
8800
8801 /*
8802 * Free the structure; str_stack_shutdown did the other cleanup work.
8803 */
8804 /* ARGSUSED */
8805 static void
8806 str_stack_fini(netstackid_t stackid, void *arg)
8807 {
8808 str_stack_t *ss = (str_stack_t *)arg;
8809
8810 kmem_free(ss, sizeof (*ss));
8811 }