1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2012 Milan Jurik. All rights reserved.
25 * Copyright (c) 2016 by Delphix. All rights reserved.
26 * Copyright 2017 Joyent, Inc.
27 */
28
29
30 /*
31 * Overview of the RSM Kernel Agent:
32 * ---------------------------------
33 *
34 * rsm.c constitutes the implementation of the RSM kernel agent. The RSM
35 * kernel agent is a pseudo device driver which makes use of the RSMPI
36 * interface on behalf of the RSMAPI user library.
37 *
38 * The kernel agent functionality can be categorized into the following
39 * components:
40 * 1. Driver Infrastructure
41 * 2. Export/Import Segment Management
42 * 3. Internal resource allocation/deallocation
43 *
44 * The driver infrastructure includes the basic module loading entry points
45 * like _init, _info, _fini to load, unload and report information about
46 * the driver module. The driver infrastructure also includes the
47 * autoconfiguration entry points namely, attach, detach and getinfo for
48 * the device autoconfiguration.
49 *
50 * The kernel agent is a pseudo character device driver and exports
51 * a cb_ops structure which defines the driver entry points for character
52 * device access. This includes the open and close entry points. The
53 * other entry points provided include ioctl, devmap and segmap and chpoll.
54 * read and write entry points are not used since the device is memory
55 * mapped. Also ddi_prop_op is used for the prop_op entry point.
56 *
57 * The ioctl entry point supports a number of commands, which are used by
58 * the RSMAPI library in order to export and import segments. These
59 * commands include commands for binding and rebinding the physical pages
60 * allocated to the virtual address range, publishing the export segment,
61 * unpublishing and republishing an export segment, creating an
62 * import segment and a virtual connection from this import segment to
63 * an export segment, performing scatter-gather data transfer, barrier
64 * operations.
65 *
66 *
67 * Export and Import segments:
68 * ---------------------------
69 *
70 * In order to create an RSM export segment a process allocates a range in its
71 * virtual address space for the segment using standard Solaris interfaces.
72 * The process then calls RSMAPI, which in turn makes an ioctl call to the
73 * RSM kernel agent for an allocation of physical memory pages and for
74 * creation of the export segment by binding these pages to the virtual
75 * address range. These pages are locked in memory so that remote accesses
76 * are always applied to the correct page. Then the RSM segment is published,
77 * again via RSMAPI making an ioctl to the RSM kernel agent, and a segment id
78 * is assigned to it.
79 *
80 * In order to import a published RSM segment, RSMAPI creates an import
81 * segment and forms a virtual connection across the interconnect to the
82 * export segment, via an ioctl into the kernel agent with the connect
83 * command. The import segment setup is completed by mapping the
84 * local device memory into the importers virtual address space. The
85 * mapping of the import segment is handled by the segmap/devmap
86 * infrastructure described as follows.
87 *
88 * Segmap and Devmap interfaces:
89 *
90 * The RSM kernel agent allows device memory to be directly accessed by user
91 * threads via memory mapping. In order to do so, the RSM kernel agent
92 * supports the devmap and segmap entry points.
93 *
94 * The segmap entry point(rsm_segmap) is responsible for setting up a memory
95 * mapping as requested by mmap. The devmap entry point(rsm_devmap) is
96 * responsible for exporting the device memory to the user applications.
97 * rsm_segmap calls RSMPI rsm_map to allocate device memory. Then the
98 * control is transfered to the devmap_setup call which calls rsm_devmap.
99 *
100 * rsm_devmap validates the user mapping to the device or kernel memory
101 * and passes the information to the system for setting up the mapping. The
102 * actual setting up of the mapping is done by devmap_devmem_setup(for
103 * device memory) or devmap_umem_setup(for kernel memory). Callbacks are
104 * registered for device context management via the devmap_devmem_setup
105 * or devmap_umem_setup calls. The callbacks are rsmmap_map, rsmmap_unmap,
106 * rsmmap_access, rsmmap_dup. The callbacks are called when a new mapping
107 * is created, a mapping is freed, a mapping is accessed or an existing
108 * mapping is duplicated respectively. These callbacks allow the RSM kernel
109 * agent to maintain state information associated with the mappings.
110 * The state information is mainly in the form of a cookie list for the import
111 * segment for which mapping has been done.
112 *
113 * Forced disconnect of import segments:
114 *
115 * When an exported segment is unpublished, the exporter sends a forced
116 * disconnect message to all its importers. The importer segments are
117 * unloaded and disconnected. This involves unloading the original
118 * mappings and remapping to a preallocated kernel trash page. This is
119 * done by devmap_umem_remap. The trash/dummy page is a kernel page,
120 * preallocated by the kernel agent during attach using ddi_umem_alloc with
121 * the DDI_UMEM_TRASH flag set. This avoids a core dump in the application
122 * due to unloading of the original mappings.
123 *
124 * Additionally every segment has a mapping generation number associated
125 * with it. This is an entry in the barrier generation page, created
126 * during attach time. This mapping generation number for the import
127 * segments is incremented on a force disconnect to notify the application
128 * of the force disconnect. On this notification, the application needs
129 * to reconnect the segment to establish a new legitimate mapping.
130 *
131 *
132 * Locks used in the kernel agent:
133 * -------------------------------
134 *
135 * The kernel agent uses a variety of mutexes and condition variables for
136 * mutual exclusion of the shared data structures and for synchronization
137 * between the various threads. Some of the locks are described as follows.
138 *
139 * Each resource structure, which represents either an export/import segment
140 * has a lock associated with it. The lock is the resource mutex, rsmrc_lock.
141 * This is used directly by RSMRC_LOCK and RSMRC_UNLOCK macros and in the
142 * rsmseglock_acquire and rsmseglock_release macros. An additional
143 * lock called the rsmsi_lock is used for the shared import data structure
144 * that is relevant for resources representing import segments. There is
145 * also a condition variable associated with the resource called s_cv. This
146 * is used to wait for events like the segment state change etc.
147 *
148 * The resource structures are allocated from a pool of resource structures,
149 * called rsm_resource. This pool is protected via a reader-writer lock,
150 * called rsmrc_lock.
151 *
152 * There are two separate hash tables, one for the export segments and
153 * one for the import segments. The export segments are inserted into the
154 * export segment hash table only after they have been published and the
155 * import segments are inserted in the import segments list only after they
156 * have successfully connected to an exported segment. These tables are
157 * protected via reader-writer locks.
158 *
159 * Debug Support in the kernel agent:
160 * ----------------------------------
161 *
162 * Debugging support in the kernel agent is provided by the following
163 * macros.
164 *
165 * DBG_PRINTF((category, level, message)) is a macro which logs a debug
166 * message to the kernel agents debug buffer, rsmka_dbg. This debug buffer
167 * can be viewed in kmdb as *rsmka_dbg/s. The message is logged based
168 * on the definition of the category and level. All messages that belong to
169 * the specified category(rsmdbg_category) and are of an equal or greater
170 * severity than the specified level(rsmdbg_level) are logged. The message
171 * is a string which uses the same formatting rules as the strings used in
172 * printf.
173 *
174 * The category defines which component of the kernel agent has logged this
175 * message. There are a number of categories that have been defined such as
176 * RSM_KERNEL_AGENT, RSM_OPS, RSM_IMPORT, RSM_EXPORT etc. A macro,
177 * DBG_ADDCATEGORY is used to add in another category to the currently
178 * specified category value so that the component using this new category
179 * can also effectively log debug messages. Thus, the category of a specific
180 * message is some combination of the available categories and we can define
181 * sub-categories if we want a finer level of granularity.
182 *
183 * The level defines the severity of the message. Different level values are
184 * defined, with RSM_ERR being the most severe and RSM_DEBUG_VERBOSE being
185 * the least severe(debug level is 0).
186 *
187 * DBG_DEFINE and DBG_DEFINE_STR are macros provided to declare a debug
188 * variable or a string respectively.
189 *
190 *
191 * NOTES:
192 *
193 * Special Fork and Exec Handling:
194 * -------------------------------
195 *
196 * The backing physical pages of an exported segment are always locked down.
197 * Thus, there are two cases in which a process having exported segments
198 * will cause a cpu to hang: (1) the process invokes exec; (2) a process
199 * forks and invokes exit before the duped file descriptors for the export
200 * segments are closed in the child process. The hang is caused because the
201 * address space release algorithm in Solaris VM subsystem is based on a
202 * non-blocking loop which does not terminate while segments are locked
203 * down. In addition to this, Solaris VM subsystem lacks a callback
204 * mechanism to the rsm kernel agent to allow unlocking these export
205 * segment pages.
206 *
207 * In order to circumvent this problem, the kernel agent does the following.
208 * The Solaris VM subsystem keeps memory segments in increasing order of
209 * virtual addressses. Thus a special page(special_exit_offset) is allocated
210 * by the kernel agent and is mmapped into the heap area of the process address
211 * space(the mmap is done by the RSMAPI library). During the mmap processing
212 * of this special page by the devmap infrastructure, a callback(the same
213 * devmap context management callbacks discussed above) is registered for an
214 * unmap.
215 *
216 * As discussed above, this page is processed by the Solaris address space
217 * release code before any of the exported segments pages(which are allocated
218 * from high memory). It is during this processing that the unmap callback gets
219 * called and this callback is responsible for force destroying the exported
220 * segments and thus eliminating the problem of locked pages.
221 *
222 * Flow-control:
223 * ------------
224 *
225 * A credit based flow control algorithm is used for messages whose
226 * processing cannot be done in the interrupt context because it might
227 * involve invoking rsmpi calls, or might take a long time to complete
228 * or might need to allocate resources. The algorithm operates on a per
229 * path basis. To send a message the pathend needs to have a credit and
230 * it consumes one for every message that is flow controlled. On the
231 * receiving pathend the message is put on a msgbuf_queue and a task is
232 * dispatched on the worker thread - recv_taskq where it is processed.
233 * After processing the message, the receiving pathend dequeues the message,
234 * and if it has processed > RSMIPC_LOTSFREE_MSGBUFS messages sends
235 * credits to the sender pathend.
236 *
237 * RSM_DRTEST:
238 * -----------
239 *
240 * This is used to enable the DR testing using a test driver on test
241 * platforms which do not supported DR.
242 *
243 */
244
245 #include <sys/types.h>
246 #include <sys/param.h>
247 #include <sys/user.h>
248 #include <sys/buf.h>
249 #include <sys/systm.h>
250 #include <sys/cred.h>
251 #include <sys/vm.h>
252 #include <sys/uio.h>
253 #include <vm/seg.h>
254 #include <vm/page.h>
255 #include <sys/stat.h>
256
257 #include <sys/time.h>
258 #include <sys/errno.h>
259
260 #include <sys/file.h>
261 #include <sys/uio.h>
262 #include <sys/proc.h>
263 #include <sys/mman.h>
264 #include <sys/open.h>
265 #include <sys/atomic.h>
266 #include <sys/mem_config.h>
267
268
269 #include <sys/ddi.h>
270 #include <sys/devops.h>
271 #include <sys/ddidevmap.h>
272 #include <sys/sunddi.h>
273 #include <sys/esunddi.h>
274 #include <sys/ddi_impldefs.h>
275
276 #include <sys/kmem.h>
277 #include <sys/conf.h>
278 #include <sys/devops.h>
279 #include <sys/ddi_impldefs.h>
280
281 #include <sys/modctl.h>
282
283 #include <sys/policy.h>
284 #include <sys/types.h>
285 #include <sys/conf.h>
286 #include <sys/param.h>
287
288 #include <sys/taskq.h>
289
290 #include <sys/rsm/rsm_common.h>
291 #include <sys/rsm/rsmapi_common.h>
292 #include <sys/rsm/rsm.h>
293 #include <rsm_in.h>
294 #include <sys/rsm/rsmka_path_int.h>
295 #include <sys/rsm/rsmpi.h>
296
297 #include <sys/modctl.h>
298 #include <sys/debug.h>
299
300 #include <sys/tuneable.h>
301
302 #ifdef RSM_DRTEST
303 extern int rsm_kphysm_setup_func_register(kphysm_setup_vector_t *vec,
304 void *arg);
305 extern void rsm_kphysm_setup_func_unregister(kphysm_setup_vector_t *vec,
306 void *arg);
307 #endif
308
309 extern void dbg_printf(int category, int level, char *fmt, ...);
310 extern void rsmka_pathmanager_init();
311 extern void rsmka_pathmanager_cleanup();
312 extern void rele_sendq_token(sendq_token_t *);
313 extern rsm_addr_t get_remote_hwaddr(adapter_t *, rsm_node_id_t);
314 extern rsm_node_id_t get_remote_nodeid(adapter_t *, rsm_addr_t);
315 extern int rsmka_topology_ioctl(caddr_t, int, int);
316
317 extern pri_t maxclsyspri;
318 extern work_queue_t work_queue;
319 extern kmutex_t ipc_info_lock;
320 extern kmutex_t ipc_info_cvlock;
321 extern kcondvar_t ipc_info_cv;
322 extern kmutex_t path_hold_cvlock;
323 extern kcondvar_t path_hold_cv;
324
325 extern kmutex_t rsmka_buf_lock;
326
327 extern path_t *rsm_find_path(char *, int, rsm_addr_t);
328 extern adapter_t *rsmka_lookup_adapter(char *, int);
329 extern sendq_token_t *rsmka_get_sendq_token(rsm_node_id_t, sendq_token_t *);
330 extern boolean_t rsmka_do_path_active(path_t *, int);
331 extern boolean_t rsmka_check_node_alive(rsm_node_id_t);
332 extern void rsmka_release_adapter(adapter_t *);
333 extern void rsmka_enqueue_msgbuf(path_t *path, void *data);
334 extern void rsmka_dequeue_msgbuf(path_t *path);
335 extern msgbuf_elem_t *rsmka_gethead_msgbuf(path_t *path);
336 /* lint -w2 */
337
338 static int rsm_open(dev_t *, int, int, cred_t *);
339 static int rsm_close(dev_t, int, int, cred_t *);
340 static int rsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
341 cred_t *credp, int *rvalp);
342 static int rsm_devmap(dev_t, devmap_cookie_t, offset_t, size_t, size_t *,
343 uint_t);
344 static int rsm_segmap(dev_t, off_t, struct as *, caddr_t *, off_t, uint_t,
345 uint_t, uint_t, cred_t *);
346 static int rsm_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
347 struct pollhead **phpp);
348
349 static int rsm_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
350 static int rsm_attach(dev_info_t *, ddi_attach_cmd_t);
351 static int rsm_detach(dev_info_t *, ddi_detach_cmd_t);
352
353 static int rsmipc_send(rsm_node_id_t, rsmipc_request_t *, rsmipc_reply_t *);
354 static void rsm_force_unload(rsm_node_id_t, rsm_memseg_id_t, boolean_t);
355 static void rsm_send_importer_disconnects(rsm_memseg_id_t, rsm_node_id_t);
356 static void rsm_send_republish(rsm_memseg_id_t, rsmapi_access_entry_t *, int,
357 rsm_permission_t);
358 static void rsm_export_force_destroy(ddi_umem_cookie_t *);
359 static void rsmacl_free(rsmapi_access_entry_t *, int);
360 static void rsmpiacl_free(rsm_access_entry_t *, int);
361
362 static int rsm_inc_pgcnt(pgcnt_t);
363 static void rsm_dec_pgcnt(pgcnt_t);
364 static void rsm_free_mapinfo(rsm_mapinfo_t *mapinfop);
365 static rsm_mapinfo_t *rsm_get_mapinfo(rsmseg_t *, off_t, size_t, off_t *,
366 size_t *);
367 static void exporter_quiesce();
368 static void rsmseg_suspend(rsmseg_t *, int *);
369 static void rsmsegshare_suspend(rsmseg_t *);
370 static int rsmseg_resume(rsmseg_t *, void **);
371 static int rsmsegshare_resume(rsmseg_t *);
372
373 static struct cb_ops rsm_cb_ops = {
374 rsm_open, /* open */
375 rsm_close, /* close */
376 nodev, /* strategy */
377 nodev, /* print */
378 nodev, /* dump */
379 nodev, /* read */
380 nodev, /* write */
381 rsm_ioctl, /* ioctl */
382 rsm_devmap, /* devmap */
383 NULL, /* mmap */
384 rsm_segmap, /* segmap */
385 rsm_chpoll, /* poll */
386 ddi_prop_op, /* cb_prop_op */
387 0, /* streamtab */
388 D_NEW|D_MP|D_DEVMAP, /* Driver compatibility flag */
389 0,
390 0,
391 0
392 };
393
394 static struct dev_ops rsm_ops = {
395 DEVO_REV, /* devo_rev, */
396 0, /* refcnt */
397 rsm_info, /* get_dev_info */
398 nulldev, /* identify */
399 nulldev, /* probe */
400 rsm_attach, /* attach */
401 rsm_detach, /* detach */
402 nodev, /* reset */
403 &rsm_cb_ops, /* driver operations */
404 (struct bus_ops *)0, /* bus operations */
405 0,
406 ddi_quiesce_not_needed, /* quiesce */
407 };
408
409 /*
410 * Module linkage information for the kernel.
411 */
412
413 static struct modldrv modldrv = {
414 &mod_driverops, /* Type of module. This one is a pseudo driver */
415 "Remote Shared Memory Driver",
416 &rsm_ops, /* driver ops */
417 };
418
419 static struct modlinkage modlinkage = {
420 MODREV_1,
421 (void *)&modldrv,
422 0,
423 0,
424 0
425 };
426
427 static void rsm_dr_callback_post_add(void *arg, pgcnt_t delta);
428 static int rsm_dr_callback_pre_del(void *arg, pgcnt_t delta);
429 static void rsm_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled);
430
431 static kphysm_setup_vector_t rsm_dr_callback_vec = {
432 KPHYSM_SETUP_VECTOR_VERSION,
433 rsm_dr_callback_post_add,
434 rsm_dr_callback_pre_del,
435 rsm_dr_callback_post_del
436 };
437
438 /* This flag can be changed to 0 to help with PIT testing */
439 int rsmka_modunloadok = 1;
440 int no_reply_cnt = 0;
441
442 uint64_t rsm_ctrlmsg_errcnt = 0;
443 uint64_t rsm_ipcsend_errcnt = 0;
444
445 #define MAX_NODES 64
446
447 static struct rsm_driver_data rsm_drv_data;
448 static struct rsmresource_table rsm_resource;
449
450 static void rsmresource_insert(minor_t, rsmresource_t *, rsm_resource_type_t);
451 static void rsmresource_destroy(void);
452 static int rsmresource_alloc(minor_t *);
453 static rsmresource_t *rsmresource_free(minor_t rnum);
454 static int rsm_closeconnection(rsmseg_t *seg, void **cookie);
455 static int rsm_unpublish(rsmseg_t *seg, int mode);
456 static int rsm_unbind(rsmseg_t *seg);
457 static uint_t rsmhash(rsm_memseg_id_t key);
458 static void rsmhash_alloc(rsmhash_table_t *rhash, int size);
459 static void rsmhash_free(rsmhash_table_t *rhash, int size);
460 static void *rsmhash_getbkt(rsmhash_table_t *rhash, uint_t hashval);
461 static void **rsmhash_bktaddr(rsmhash_table_t *rhash, uint_t hashval);
462 static int rsm_send_notimporting(rsm_node_id_t dest, rsm_memseg_id_t segid,
463 void *cookie);
464 int rsm_disconnect(rsmseg_t *seg);
465 void rsmseg_unload(rsmseg_t *);
466 void rsm_suspend_complete(rsm_node_id_t src_node, int flag);
467
468 rsm_intr_hand_ret_t rsm_srv_func(rsm_controller_object_t *chd,
469 rsm_intr_q_op_t opcode, rsm_addr_t src,
470 void *data, size_t size, rsm_intr_hand_arg_t arg);
471
472 static void rsm_intr_callback(void *, rsm_addr_t, rsm_intr_hand_arg_t);
473
474 rsm_node_id_t my_nodeid;
475
476 /* cookie, va, offsets and length for the barrier */
477 static rsm_gnum_t *bar_va;
478 static ddi_umem_cookie_t bar_cookie;
479 static off_t barrier_offset;
480 static size_t barrier_size;
481 static int max_segs;
482
483 /* cookie for the trash memory */
484 static ddi_umem_cookie_t remap_cookie;
485
486 static rsm_memseg_id_t rsm_nextavail_segmentid;
487
488 extern taskq_t *work_taskq;
489 extern char *taskq_name;
490
491 static dev_info_t *rsm_dip; /* private copy of devinfo pointer */
492
493 static rsmhash_table_t rsm_export_segs; /* list of exported segs */
494 rsmhash_table_t rsm_import_segs; /* list of imported segs */
495 static rsmhash_table_t rsm_event_queues; /* list of event queues */
496
497 static rsm_ipc_t rsm_ipc; /* ipc info */
498
499 /* list of nodes to which RSMIPC_MSG_SUSPEND has been sent */
500 static list_head_t rsm_suspend_list;
501
502 /* list of descriptors for remote importers */
503 static importers_table_t importer_list;
504
505 kmutex_t rsm_suspend_cvlock;
506 kcondvar_t rsm_suspend_cv;
507
508 static kmutex_t rsm_lock;
509
510 adapter_t loopback_adapter;
511 rsm_controller_attr_t loopback_attr;
512
513 int rsmipc_send_controlmsg(path_t *path, int msgtype);
514
515 void rsmka_init_loopback();
516
517 int rsmka_null_seg_create(
518 rsm_controller_handle_t,
519 rsm_memseg_export_handle_t *,
520 size_t,
521 uint_t,
522 rsm_memory_local_t *,
523 rsm_resource_callback_t,
524 rsm_resource_callback_arg_t);
525
526 int rsmka_null_seg_destroy(
527 rsm_memseg_export_handle_t);
528
529 int rsmka_null_bind(
530 rsm_memseg_export_handle_t,
531 off_t,
532 rsm_memory_local_t *,
533 rsm_resource_callback_t,
534 rsm_resource_callback_arg_t);
535
536 int rsmka_null_unbind(
537 rsm_memseg_export_handle_t,
538 off_t,
539 size_t);
540
541 int rsmka_null_rebind(
542 rsm_memseg_export_handle_t,
543 off_t,
544 rsm_memory_local_t *,
545 rsm_resource_callback_t,
546 rsm_resource_callback_arg_t);
547
548 int rsmka_null_publish(
549 rsm_memseg_export_handle_t,
550 rsm_access_entry_t [],
551 uint_t,
552 rsm_memseg_id_t,
553 rsm_resource_callback_t,
554 rsm_resource_callback_arg_t);
555
556
557 int rsmka_null_republish(
558 rsm_memseg_export_handle_t,
559 rsm_access_entry_t [],
560 uint_t,
561 rsm_resource_callback_t,
562 rsm_resource_callback_arg_t);
563
564 int rsmka_null_unpublish(
565 rsm_memseg_export_handle_t);
566
567 rsm_ops_t null_rsmpi_ops;
568
569 /*
570 * data and locks to keep track of total amount of exported memory
571 */
572 static pgcnt_t rsm_pgcnt;
573 static pgcnt_t rsm_pgcnt_max; /* max allowed */
574 static kmutex_t rsm_pgcnt_lock;
575
576 static int rsm_enable_dr;
577
578 static char loopback_str[] = "loopback";
579
580 int rsm_hash_size;
581
582 /*
583 * The locking model is as follows:
584 *
585 * Local operations:
586 * find resource - grab reader lock on resouce list
587 * insert rc - grab writer lock
588 * delete rc - grab writer lock and resource mutex
589 * read/write - no lock
590 *
591 * Remote invocations:
592 * find resource - grab read lock and resource mutex
593 *
594 * State:
595 * resource state - grab resource mutex
596 */
597
598 int
599 _init(void)
600 {
601 int e;
602
603 e = mod_install(&modlinkage);
604 if (e != 0) {
605 return (e);
606 }
607
608 mutex_init(&rsm_lock, NULL, MUTEX_DRIVER, NULL);
609
610 mutex_init(&rsmka_buf_lock, NULL, MUTEX_DEFAULT, NULL);
611
612
613 rw_init(&rsm_resource.rsmrc_lock, NULL, RW_DRIVER, NULL);
614
615 rsm_hash_size = RSM_HASHSZ;
616
617 rw_init(&rsm_export_segs.rsmhash_rw, NULL, RW_DRIVER, NULL);
618
619 rw_init(&rsm_import_segs.rsmhash_rw, NULL, RW_DRIVER, NULL);
620
621 mutex_init(&importer_list.lock, NULL, MUTEX_DRIVER, NULL);
622
623 mutex_init(&rsm_ipc.lock, NULL, MUTEX_DRIVER, NULL);
624 cv_init(&rsm_ipc.cv, NULL, CV_DRIVER, 0);
625
626 mutex_init(&rsm_suspend_cvlock, NULL, MUTEX_DRIVER, NULL);
627 cv_init(&rsm_suspend_cv, NULL, CV_DRIVER, 0);
628
629 mutex_init(&rsm_drv_data.drv_lock, NULL, MUTEX_DRIVER, NULL);
630 cv_init(&rsm_drv_data.drv_cv, NULL, CV_DRIVER, 0);
631
632 rsm_ipc.count = RSMIPC_SZ;
633 rsm_ipc.wanted = 0;
634 rsm_ipc.sequence = 0;
635
636 (void) mutex_init(&rsm_pgcnt_lock, NULL, MUTEX_DRIVER, NULL);
637
638 for (e = 0; e < RSMIPC_SZ; e++) {
639 rsmipc_slot_t *slot = &rsm_ipc.slots[e];
640
641 RSMIPC_SET(slot, RSMIPC_FREE);
642 mutex_init(&slot->rsmipc_lock, NULL, MUTEX_DRIVER, NULL);
643 cv_init(&slot->rsmipc_cv, NULL, CV_DRIVER, 0);
644 }
645
646 /*
647 * Initialize the suspend message list
648 */
649 rsm_suspend_list.list_head = NULL;
650 mutex_init(&rsm_suspend_list.list_lock, NULL, MUTEX_DRIVER, NULL);
651
652 /*
653 * It is assumed here that configuration data is available
654 * during system boot since _init may be called at that time.
655 */
656
657 rsmka_pathmanager_init();
658
659 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE,
660 "rsm: _init done\n"));
661
662 return (DDI_SUCCESS);
663
664 }
665
666 int
667 _info(struct modinfo *modinfop)
668 {
669
670 return (mod_info(&modlinkage, modinfop));
671 }
672
673 int
674 _fini(void)
675 {
676 int e;
677
678 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE,
679 "rsm: _fini enter\n"));
680
681 /*
682 * The rsmka_modunloadok flag is simply used to help with
683 * the PIT testing. Make this flag 0 to disallow modunload.
684 */
685 if (rsmka_modunloadok == 0)
686 return (EBUSY);
687
688 /* rsm_detach will be called as a result of mod_remove */
689 e = mod_remove(&modlinkage);
690 if (e) {
691 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_ERR,
692 "Unable to fini RSM %x\n", e));
693 return (e);
694 }
695
696 rsmka_pathmanager_cleanup();
697
698 rw_destroy(&rsm_resource.rsmrc_lock);
699
700 rw_destroy(&rsm_export_segs.rsmhash_rw);
701 rw_destroy(&rsm_import_segs.rsmhash_rw);
702 rw_destroy(&rsm_event_queues.rsmhash_rw);
703
704 mutex_destroy(&importer_list.lock);
705
706 mutex_destroy(&rsm_ipc.lock);
707 cv_destroy(&rsm_ipc.cv);
708
709 (void) mutex_destroy(&rsm_suspend_list.list_lock);
710
711 (void) mutex_destroy(&rsm_pgcnt_lock);
712
713 DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE, "_fini done\n"));
714
715 return (DDI_SUCCESS);
716
717 }
718
719 /*ARGSUSED1*/
720 static int
721 rsm_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
722 {
723 minor_t rnum;
724 int percent;
725 int ret;
726 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI);
727
728 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_attach enter\n"));
729
730 switch (cmd) {
731 case DDI_ATTACH:
732 break;
733 case DDI_RESUME:
734 default:
735 DBG_PRINTF((category, RSM_ERR,
736 "rsm:rsm_attach - cmd not supported\n"));
737 return (DDI_FAILURE);
738 }
739
740 if (rsm_dip != NULL) {
741 DBG_PRINTF((category, RSM_ERR,
742 "rsm:rsm_attach - supports only "
743 "one instance\n"));
744 return (DDI_FAILURE);
745 }
746
747 rsm_enable_dr = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
748 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
749 "enable-dynamic-reconfiguration", 1);
750
751 mutex_enter(&rsm_drv_data.drv_lock);
752 rsm_drv_data.drv_state = RSM_DRV_REG_PROCESSING;
753 mutex_exit(&rsm_drv_data.drv_lock);
754
755 if (rsm_enable_dr) {
756 #ifdef RSM_DRTEST
757 ret = rsm_kphysm_setup_func_register(&rsm_dr_callback_vec,
758 (void *)NULL);
759 #else
760 ret = kphysm_setup_func_register(&rsm_dr_callback_vec,
761 (void *)NULL);
762 #endif
763 if (ret != 0) {
764 mutex_exit(&rsm_drv_data.drv_lock);
765 cmn_err(CE_CONT, "rsm:rsm_attach - Dynamic "
766 "reconfiguration setup failed\n");
767 return (DDI_FAILURE);
768 }
769 }
770
771 mutex_enter(&rsm_drv_data.drv_lock);
772 ASSERT(rsm_drv_data.drv_state == RSM_DRV_REG_PROCESSING);
773 rsm_drv_data.drv_state = RSM_DRV_OK;
774 cv_broadcast(&rsm_drv_data.drv_cv);
775 mutex_exit(&rsm_drv_data.drv_lock);
776
777 /*
778 * page_list_read_lock();
779 * xx_setup();
780 * page_list_read_unlock();
781 */
782
783 rsm_hash_size = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
784 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
785 "segment-hashtable-size", RSM_HASHSZ);
786 if (rsm_hash_size == 0) {
787 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
788 "rsm: segment-hashtable-size in rsm.conf "
789 "must be greater than 0, defaulting to 128\n"));
790 rsm_hash_size = RSM_HASHSZ;
791 }
792
793 DBG_PRINTF((category, RSM_DEBUG, "rsm_attach rsm_hash_size: %d\n",
794 rsm_hash_size));
795
796 rsm_pgcnt = 0;
797
798 percent = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
799 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
800 "max-exported-memory", 0);
801 if (percent < 0) {
802 DBG_PRINTF((category, RSM_ERR,
803 "rsm:rsm_attach not enough memory available to "
804 "export, or max-exported-memory set incorrectly.\n"));
805 return (DDI_FAILURE);
806 }
807 /* 0 indicates no fixed upper limit. maxmem is the max */
808 /* available pageable physical mem */
809 rsm_pgcnt_max = (percent*maxmem)/100;
810
811 if (rsm_pgcnt_max > 0) {
812 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
813 "rsm: Available physical memory = %lu pages, "
814 "Max exportable memory = %lu pages",
815 maxmem, rsm_pgcnt_max));
816 }
817
818 /*
819 * Create minor number
820 */
821 if (rsmresource_alloc(&rnum) != RSM_SUCCESS) {
822 DBG_PRINTF((category, RSM_ERR,
823 "rsm: rsm_attach - Unable to get "
824 "minor number\n"));
825 return (DDI_FAILURE);
826 }
827
828 ASSERT(rnum == RSM_DRIVER_MINOR);
829
830 if (ddi_create_minor_node(devi, DRIVER_NAME, S_IFCHR,
831 rnum, DDI_PSEUDO, NULL) == DDI_FAILURE) {
832 DBG_PRINTF((category, RSM_ERR,
833 "rsm: rsm_attach - unable to allocate "
834 "minor #\n"));
835 return (DDI_FAILURE);
836 }
837
838 rsm_dip = devi;
839 /*
840 * Allocate the hashtables
841 */
842 rsmhash_alloc(&rsm_export_segs, rsm_hash_size);
843 rsmhash_alloc(&rsm_import_segs, rsm_hash_size);
844
845 importer_list.bucket = (importing_token_t **)
846 kmem_zalloc(rsm_hash_size * sizeof (importing_token_t *), KM_SLEEP);
847
848 /*
849 * Allocate a resource struct
850 */
851 {
852 rsmresource_t *p;
853
854 p = (rsmresource_t *)kmem_zalloc(sizeof (*p), KM_SLEEP);
855
856 mutex_init(&p->rsmrc_lock, NULL, MUTEX_DRIVER, (void *) NULL);
857
858 rsmresource_insert(rnum, p, RSM_RESOURCE_BAR);
859 }
860
861 /*
862 * Based on the rsm.conf property max-segments, determine the maximum
863 * number of segments that can be exported/imported. This is then used
864 * to determine the size for barrier failure pages.
865 */
866
867 /* First get the max number of segments from the rsm.conf file */
868 max_segs = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
869 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
870 "max-segments", 0);
871 if (max_segs == 0) {
872 /* Use default number of segments */
873 max_segs = RSM_MAX_NUM_SEG;
874 }
875
876 /*
877 * Based on the max number of segments allowed, determine the barrier
878 * page size. add 1 to max_segs since the barrier page itself uses
879 * a slot
880 */
881 barrier_size = roundup((max_segs + 1) * sizeof (rsm_gnum_t),
882 PAGESIZE);
883
884 /*
885 * allocation of the barrier failure page
886 */
887 bar_va = (rsm_gnum_t *)ddi_umem_alloc(barrier_size,
888 DDI_UMEM_SLEEP, &bar_cookie);
889
890 /*
891 * Set the barrier_offset
892 */
893 barrier_offset = 0;
894
895 /*
896 * Allocate a trash memory and get a cookie for it. This will be used
897 * when remapping segments during force disconnects. Allocate the
898 * trash memory with a large size which is page aligned.
899 */
900 (void) ddi_umem_alloc((size_t)TRASHSIZE,
901 DDI_UMEM_TRASH, &remap_cookie);
902
903 /* initialize user segment id allocation variable */
904 rsm_nextavail_segmentid = (rsm_memseg_id_t)RSM_USER_APP_ID_BASE;
905
906 /*
907 * initialize the null_rsmpi_ops vector and the loopback adapter
908 */
909 rsmka_init_loopback();
910
911
912 ddi_report_dev(devi);
913
914 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_attach done\n"));
915
916 return (DDI_SUCCESS);
917 }
918
919 /*
920 * The call to mod_remove in the _fine routine will cause the system
921 * to call rsm_detach
922 */
923 /*ARGSUSED*/
924 static int
925 rsm_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
926 {
927 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI);
928
929 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_detach enter\n"));
930
931 switch (cmd) {
932 case DDI_DETACH:
933 break;
934 default:
935 DBG_PRINTF((category, RSM_ERR,
936 "rsm:rsm_detach - cmd %x not supported\n",
937 cmd));
938 return (DDI_FAILURE);
939 }
940
941 mutex_enter(&rsm_drv_data.drv_lock);
942 while (rsm_drv_data.drv_state != RSM_DRV_OK)
943 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
944 rsm_drv_data.drv_state = RSM_DRV_UNREG_PROCESSING;
945 mutex_exit(&rsm_drv_data.drv_lock);
946
947 /*
948 * Unregister the DR callback functions
949 */
950 if (rsm_enable_dr) {
951 #ifdef RSM_DRTEST
952 rsm_kphysm_setup_func_unregister(&rsm_dr_callback_vec,
953 (void *)NULL);
954 #else
955 kphysm_setup_func_unregister(&rsm_dr_callback_vec,
956 (void *)NULL);
957 #endif
958 }
959
960 mutex_enter(&rsm_drv_data.drv_lock);
961 ASSERT(rsm_drv_data.drv_state == RSM_DRV_UNREG_PROCESSING);
962 rsm_drv_data.drv_state = RSM_DRV_NEW;
963 mutex_exit(&rsm_drv_data.drv_lock);
964
965 ASSERT(rsm_suspend_list.list_head == NULL);
966
967 /*
968 * Release all resources, seglist, controller, ...
969 */
970
971 /* remove intersend queues */
972 /* remove registered services */
973
974
975 ddi_remove_minor_node(dip, DRIVER_NAME);
976 rsm_dip = NULL;
977
978 /*
979 * Free minor zero resource
980 */
981 {
982 rsmresource_t *p;
983
984 p = rsmresource_free(RSM_DRIVER_MINOR);
985 if (p) {
986 mutex_destroy(&p->rsmrc_lock);
987 kmem_free((void *)p, sizeof (*p));
988 }
989 }
990
991 /*
992 * Free resource table
993 */
994
995 rsmresource_destroy();
996
997 /*
998 * Free the hash tables
999 */
1000 rsmhash_free(&rsm_export_segs, rsm_hash_size);
1001 rsmhash_free(&rsm_import_segs, rsm_hash_size);
1002
1003 kmem_free((void *)importer_list.bucket,
1004 rsm_hash_size * sizeof (importing_token_t *));
1005 importer_list.bucket = NULL;
1006
1007
1008 /* free barrier page */
1009 if (bar_cookie != NULL) {
1010 ddi_umem_free(bar_cookie);
1011 }
1012 bar_va = NULL;
1013 bar_cookie = NULL;
1014
1015 /*
1016 * Free the memory allocated for the trash
1017 */
1018 if (remap_cookie != NULL) {
1019 ddi_umem_free(remap_cookie);
1020 }
1021 remap_cookie = NULL;
1022
1023 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_detach done\n"));
1024
1025 return (DDI_SUCCESS);
1026 }
1027
1028 /*ARGSUSED*/
1029 static int
1030 rsm_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1031 {
1032 register int error;
1033 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI);
1034
1035 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_info enter\n"));
1036
1037 switch (infocmd) {
1038 case DDI_INFO_DEVT2DEVINFO:
1039 if (rsm_dip == NULL)
1040 error = DDI_FAILURE;
1041 else {
1042 *result = (void *)rsm_dip;
1043 error = DDI_SUCCESS;
1044 }
1045 break;
1046 case DDI_INFO_DEVT2INSTANCE:
1047 *result = (void *)0;
1048 error = DDI_SUCCESS;
1049 break;
1050 default:
1051 error = DDI_FAILURE;
1052 }
1053
1054 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_info done\n"));
1055 return (error);
1056 }
1057
1058 adapter_t *
1059 rsm_getadapter(rsm_ioctlmsg_t *msg, int mode)
1060 {
1061 adapter_t *adapter;
1062 char adapter_devname[MAXNAMELEN];
1063 int instance;
1064 DBG_DEFINE(category,
1065 RSM_KERNEL_AGENT | RSM_IMPORT | RSM_EXPORT | RSM_IOCTL);
1066
1067 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_getadapter enter\n"));
1068
1069 instance = msg->cnum;
1070
1071 if ((msg->cname_len <= 0) || (msg->cname_len > MAXNAMELEN)) {
1072 return (NULL);
1073 }
1074
1075 if (ddi_copyin(msg->cname, adapter_devname, msg->cname_len, mode))
1076 return (NULL);
1077
1078 if (strcmp(adapter_devname, "loopback") == 0)
1079 return (&loopback_adapter);
1080
1081 adapter = rsmka_lookup_adapter(adapter_devname, instance);
1082
1083 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_getadapter done\n"));
1084
1085 return (adapter);
1086 }
1087
1088
1089 /*
1090 * *********************** Resource Number Management ********************
1091 * All resources are stored in a simple hash table. The table is an array
1092 * of pointers to resource blks. Each blk contains:
1093 * base - base number of this blk
1094 * used - number of used slots in this blk.
1095 * blks - array of pointers to resource items.
1096 * An entry in a resource blk is empty if it's NULL.
1097 *
1098 * We start with no resource array. Each time we run out of slots, we
1099 * reallocate a new larger array and copy the pointer to the new array and
1100 * a new resource blk is allocated and added to the hash table.
1101 *
1102 * The resource control block contains:
1103 * root - array of pointer of resource blks
1104 * sz - current size of array.
1105 * len - last valid entry in array.
1106 *
1107 * A search operation based on a resource number is as follows:
1108 * index = rnum / RESOURCE_BLKSZ;
1109 * ASSERT(index < resource_block.len);
1110 * ASSERT(index < resource_block.sz);
1111 * offset = rnum % RESOURCE_BLKSZ;
1112 * ASSERT(offset >= resource_block.root[index]->base);
1113 * ASSERT(offset < resource_block.root[index]->base + RESOURCE_BLKSZ);
1114 * return resource_block.root[index]->blks[offset];
1115 *
1116 * A resource blk is freed with its used count reachs zero.
1117 */
1118 static int
1119 rsmresource_alloc(minor_t *rnum)
1120 {
1121
1122 /* search for available resource slot */
1123 int i, j, empty = -1;
1124 rsmresource_blk_t *blk;
1125
1126 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1127 "rsmresource_alloc enter\n"));
1128
1129 rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER);
1130
1131 /* Try to find an empty slot */
1132 for (i = 0; i < rsm_resource.rsmrc_len; i++) {
1133 blk = rsm_resource.rsmrc_root[i];
1134 if (blk != NULL && blk->rsmrcblk_avail > 0) {
1135 /* found an empty slot in this blk */
1136 for (j = 0; j < RSMRC_BLKSZ; j++) {
1137 if (blk->rsmrcblk_blks[j] == NULL) {
1138 *rnum = (minor_t)
1139 (j + (i * RSMRC_BLKSZ));
1140 /*
1141 * obey gen page limits
1142 */
1143 if (*rnum >= max_segs + 1) {
1144 if (empty < 0) {
1145 rw_exit(&rsm_resource.
1146 rsmrc_lock);
1147 DBG_PRINTF((
1148 RSM_KERNEL_ALL,
1149 RSM_ERR,
1150 "rsmresource"
1151 "_alloc failed:"
1152 "not enough res"
1153 "%d\n", *rnum));
1154 return (RSMERR_INSUFFICIENT_RESOURCES);
1155 } else {
1156 /* use empty slot */
1157 break;
1158 }
1159
1160 }
1161
1162 blk->rsmrcblk_blks[j] = RSMRC_RESERVED;
1163 blk->rsmrcblk_avail--;
1164 rw_exit(&rsm_resource.rsmrc_lock);
1165 DBG_PRINTF((RSM_KERNEL_ALL,
1166 RSM_DEBUG_VERBOSE,
1167 "rsmresource_alloc done\n"));
1168 return (RSM_SUCCESS);
1169 }
1170 }
1171 } else if (blk == NULL && empty < 0) {
1172 /* remember first empty slot */
1173 empty = i;
1174 }
1175 }
1176
1177 /* Couldn't find anything, allocate a new blk */
1178 /*
1179 * Do we need to reallocate the root array
1180 */
1181 if (empty < 0) {
1182 if (rsm_resource.rsmrc_len == rsm_resource.rsmrc_sz) {
1183 /*
1184 * Allocate new array and copy current stuff into it
1185 */
1186 rsmresource_blk_t **p;
1187 uint_t newsz = (uint_t)rsm_resource.rsmrc_sz +
1188 RSMRC_BLKSZ;
1189 /*
1190 * Don't allocate more that max valid rnum
1191 */
1192 if (rsm_resource.rsmrc_len*RSMRC_BLKSZ >=
1193 max_segs + 1) {
1194 rw_exit(&rsm_resource.rsmrc_lock);
1195 return (RSMERR_INSUFFICIENT_RESOURCES);
1196 }
1197
1198 p = (rsmresource_blk_t **)kmem_zalloc(
1199 newsz * sizeof (*p),
1200 KM_SLEEP);
1201
1202 if (rsm_resource.rsmrc_root) {
1203 uint_t oldsz;
1204
1205 oldsz = (uint_t)(rsm_resource.rsmrc_sz *
1206 (int)sizeof (*p));
1207
1208 /*
1209 * Copy old data into new space and
1210 * free old stuff
1211 */
1212 bcopy(rsm_resource.rsmrc_root, p, oldsz);
1213 kmem_free(rsm_resource.rsmrc_root, oldsz);
1214 }
1215
1216 rsm_resource.rsmrc_root = p;
1217 rsm_resource.rsmrc_sz = (int)newsz;
1218 }
1219
1220 empty = rsm_resource.rsmrc_len;
1221 rsm_resource.rsmrc_len++;
1222 }
1223
1224 /*
1225 * Allocate a new blk
1226 */
1227 blk = (rsmresource_blk_t *)kmem_zalloc(sizeof (*blk), KM_SLEEP);
1228 ASSERT(rsm_resource.rsmrc_root[empty] == NULL);
1229 rsm_resource.rsmrc_root[empty] = blk;
1230 blk->rsmrcblk_avail = RSMRC_BLKSZ - 1;
1231
1232 /*
1233 * Allocate slot
1234 */
1235
1236 *rnum = (minor_t)(empty * RSMRC_BLKSZ);
1237
1238 /*
1239 * watch out not to exceed bounds of barrier page
1240 */
1241 if (*rnum >= max_segs + 1) {
1242 rw_exit(&rsm_resource.rsmrc_lock);
1243 DBG_PRINTF((RSM_KERNEL_ALL, RSM_ERR,
1244 "rsmresource_alloc failed %d\n", *rnum));
1245
1246 return (RSMERR_INSUFFICIENT_RESOURCES);
1247 }
1248 blk->rsmrcblk_blks[0] = RSMRC_RESERVED;
1249
1250
1251 rw_exit(&rsm_resource.rsmrc_lock);
1252
1253 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1254 "rsmresource_alloc done\n"));
1255
1256 return (RSM_SUCCESS);
1257 }
1258
1259 static rsmresource_t *
1260 rsmresource_free(minor_t rnum)
1261 {
1262
1263 /* search for available resource slot */
1264 int i, j;
1265 rsmresource_blk_t *blk;
1266 rsmresource_t *p;
1267
1268 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1269 "rsmresource_free enter\n"));
1270
1271 i = (int)(rnum / RSMRC_BLKSZ);
1272 j = (int)(rnum % RSMRC_BLKSZ);
1273
1274 if (i >= rsm_resource.rsmrc_len) {
1275 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1276 "rsmresource_free done\n"));
1277 return (NULL);
1278 }
1279
1280 rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER);
1281
1282 ASSERT(rsm_resource.rsmrc_root);
1283 ASSERT(i < rsm_resource.rsmrc_len);
1284 ASSERT(i < rsm_resource.rsmrc_sz);
1285 blk = rsm_resource.rsmrc_root[i];
1286 if (blk == NULL) {
1287 rw_exit(&rsm_resource.rsmrc_lock);
1288 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1289 "rsmresource_free done\n"));
1290 return (NULL);
1291 }
1292
1293 ASSERT(blk->rsmrcblk_blks[j]); /* reserved or full */
1294
1295 p = blk->rsmrcblk_blks[j];
1296 if (p == RSMRC_RESERVED) {
1297 p = NULL;
1298 }
1299
1300 blk->rsmrcblk_blks[j] = NULL;
1301 blk->rsmrcblk_avail++;
1302 if (blk->rsmrcblk_avail == RSMRC_BLKSZ) {
1303 /* free this blk */
1304 kmem_free(blk, sizeof (*blk));
1305 rsm_resource.rsmrc_root[i] = NULL;
1306 }
1307
1308 rw_exit(&rsm_resource.rsmrc_lock);
1309
1310 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1311 "rsmresource_free done\n"));
1312
1313 return (p);
1314 }
1315
1316 static rsmresource_t *
1317 rsmresource_lookup(minor_t rnum, int lock)
1318 {
1319 int i, j;
1320 rsmresource_blk_t *blk;
1321 rsmresource_t *p;
1322
1323 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1324 "rsmresource_lookup enter\n"));
1325
1326 /* Find resource and lock it in READER mode */
1327 /* search for available resource slot */
1328
1329 i = (int)(rnum / RSMRC_BLKSZ);
1330 j = (int)(rnum % RSMRC_BLKSZ);
1331
1332 if (i >= rsm_resource.rsmrc_len) {
1333 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1334 "rsmresource_lookup done\n"));
1335 return (NULL);
1336 }
1337
1338 rw_enter(&rsm_resource.rsmrc_lock, RW_READER);
1339
1340 blk = rsm_resource.rsmrc_root[i];
1341 if (blk != NULL) {
1342 ASSERT(i < rsm_resource.rsmrc_len);
1343 ASSERT(i < rsm_resource.rsmrc_sz);
1344
1345 p = blk->rsmrcblk_blks[j];
1346 if (lock == RSM_LOCK) {
1347 if (p != RSMRC_RESERVED) {
1348 mutex_enter(&p->rsmrc_lock);
1349 } else {
1350 p = NULL;
1351 }
1352 }
1353 } else {
1354 p = NULL;
1355 }
1356 rw_exit(&rsm_resource.rsmrc_lock);
1357
1358 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1359 "rsmresource_lookup done\n"));
1360
1361 return (p);
1362 }
1363
1364 static void
1365 rsmresource_insert(minor_t rnum, rsmresource_t *p, rsm_resource_type_t type)
1366 {
1367 /* Find resource and lock it in READER mode */
1368 /* Caller can upgrade if need be */
1369 /* search for available resource slot */
1370 int i, j;
1371 rsmresource_blk_t *blk;
1372
1373 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1374 "rsmresource_insert enter\n"));
1375
1376 i = (int)(rnum / RSMRC_BLKSZ);
1377 j = (int)(rnum % RSMRC_BLKSZ);
1378
1379 p->rsmrc_type = type;
1380 p->rsmrc_num = rnum;
1381
1382 rw_enter(&rsm_resource.rsmrc_lock, RW_READER);
1383
1384 ASSERT(rsm_resource.rsmrc_root);
1385 ASSERT(i < rsm_resource.rsmrc_len);
1386 ASSERT(i < rsm_resource.rsmrc_sz);
1387
1388 blk = rsm_resource.rsmrc_root[i];
1389 ASSERT(blk);
1390
1391 ASSERT(blk->rsmrcblk_blks[j] == RSMRC_RESERVED);
1392
1393 blk->rsmrcblk_blks[j] = p;
1394
1395 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1396 "rsmresource_insert done\n"));
1397
1398 rw_exit(&rsm_resource.rsmrc_lock);
1399 }
1400
1401 static void
1402 rsmresource_destroy()
1403 {
1404 int i, j;
1405
1406 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1407 "rsmresource_destroy enter\n"));
1408
1409 rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER);
1410
1411 for (i = 0; i < rsm_resource.rsmrc_len; i++) {
1412 rsmresource_blk_t *blk;
1413
1414 blk = rsm_resource.rsmrc_root[i];
1415 if (blk == NULL) {
1416 continue;
1417 }
1418 for (j = 0; j < RSMRC_BLKSZ; j++) {
1419 if (blk->rsmrcblk_blks[j] != NULL) {
1420 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1421 "Not null slot %d, %lx\n", j,
1422 (size_t)blk->rsmrcblk_blks[j]));
1423 }
1424 }
1425 kmem_free(blk, sizeof (*blk));
1426 rsm_resource.rsmrc_root[i] = NULL;
1427 }
1428 if (rsm_resource.rsmrc_root) {
1429 i = rsm_resource.rsmrc_sz * (int)sizeof (rsmresource_blk_t *);
1430 kmem_free(rsm_resource.rsmrc_root, (uint_t)i);
1431 rsm_resource.rsmrc_root = NULL;
1432 rsm_resource.rsmrc_len = 0;
1433 rsm_resource.rsmrc_sz = 0;
1434 }
1435
1436 DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
1437 "rsmresource_destroy done\n"));
1438
1439 rw_exit(&rsm_resource.rsmrc_lock);
1440 }
1441
1442
1443 /* ******************** Generic Key Hash Table Management ********* */
1444 static rsmresource_t *
1445 rsmhash_lookup(rsmhash_table_t *rhash, rsm_memseg_id_t key,
1446 rsm_resource_state_t state)
1447 {
1448 rsmresource_t *p;
1449 uint_t hashval;
1450 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1451
1452 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_lookup enter\n"));
1453
1454 hashval = rsmhash(key);
1455
1456 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_lookup %u=%d\n",
1457 key, hashval));
1458
1459 rw_enter(&rhash->rsmhash_rw, RW_READER);
1460
1461 p = (rsmresource_t *)rsmhash_getbkt(rhash, hashval);
1462
1463 for (; p; p = p->rsmrc_next) {
1464 if (p->rsmrc_key == key) {
1465 /* acquire resource lock */
1466 RSMRC_LOCK(p);
1467 break;
1468 }
1469 }
1470
1471 rw_exit(&rhash->rsmhash_rw);
1472
1473 if (p != NULL && p->rsmrc_state != state) {
1474 /* state changed, release lock and return null */
1475 RSMRC_UNLOCK(p);
1476 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
1477 "rsmhash_lookup done: state changed\n"));
1478 return (NULL);
1479 }
1480
1481 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_lookup done\n"));
1482
1483 return (p);
1484 }
1485
1486 static void
1487 rsmhash_rm(rsmhash_table_t *rhash, rsmresource_t *rcelm)
1488 {
1489 rsmresource_t *p, **back;
1490 uint_t hashval;
1491 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1492
1493 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_rm enter\n"));
1494
1495 hashval = rsmhash(rcelm->rsmrc_key);
1496
1497 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_rm %u=%d\n",
1498 rcelm->rsmrc_key, hashval));
1499
1500 /*
1501 * It's ok not to find the segment.
1502 */
1503 rw_enter(&rhash->rsmhash_rw, RW_WRITER);
1504
1505 back = (rsmresource_t **)rsmhash_bktaddr(rhash, hashval);
1506
1507 for (; (p = *back) != NULL; back = &p->rsmrc_next) {
1508 if (p == rcelm) {
1509 *back = rcelm->rsmrc_next;
1510 break;
1511 }
1512 }
1513
1514 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_rm done\n"));
1515
1516 rw_exit(&rhash->rsmhash_rw);
1517 }
1518
1519 static int
1520 rsmhash_add(rsmhash_table_t *rhash, rsmresource_t *new, rsm_memseg_id_t key,
1521 int dup_check, rsm_resource_state_t state)
1522 {
1523 rsmresource_t *p = NULL, **bktp;
1524 uint_t hashval;
1525 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1526
1527 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_add enter\n"));
1528
1529 /* lock table */
1530 rw_enter(&rhash->rsmhash_rw, RW_WRITER);
1531
1532 /*
1533 * If the current resource state is other than the state passed in
1534 * then the resource is (probably) already on the list. eg. for an
1535 * import segment if the state is not RSM_STATE_NEW then it's on the
1536 * list already.
1537 */
1538 RSMRC_LOCK(new);
1539 if (new->rsmrc_state != state) {
1540 RSMRC_UNLOCK(new);
1541 rw_exit(&rhash->rsmhash_rw);
1542 return (RSMERR_BAD_SEG_HNDL);
1543 }
1544
1545 hashval = rsmhash(key);
1546 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_add %d\n", hashval));
1547
1548 if (dup_check) {
1549 /*
1550 * Used for checking export segments; don't want to have
1551 * the same key used for multiple segments.
1552 */
1553
1554 p = (rsmresource_t *)rsmhash_getbkt(rhash, hashval);
1555
1556 for (; p; p = p->rsmrc_next) {
1557 if (p->rsmrc_key == key) {
1558 RSMRC_UNLOCK(new);
1559 break;
1560 }
1561 }
1562 }
1563
1564 if (p == NULL) {
1565 /* Key doesn't exist, add it */
1566
1567 bktp = (rsmresource_t **)rsmhash_bktaddr(rhash, hashval);
1568
1569 new->rsmrc_key = key;
1570 new->rsmrc_next = *bktp;
1571 *bktp = new;
1572 }
1573
1574 rw_exit(&rhash->rsmhash_rw);
1575
1576 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_add done\n"));
1577
1578 return (p == NULL ? RSM_SUCCESS : RSMERR_SEGID_IN_USE);
1579 }
1580
1581 /*
1582 * XOR each byte of the key.
1583 */
1584 static uint_t
1585 rsmhash(rsm_memseg_id_t key)
1586 {
1587 uint_t hash = key;
1588
1589 hash ^= (key >> 8);
1590 hash ^= (key >> 16);
1591 hash ^= (key >> 24);
1592
1593 return (hash % rsm_hash_size);
1594
1595 }
1596
1597 /*
1598 * generic function to get a specific bucket
1599 */
1600 static void *
1601 rsmhash_getbkt(rsmhash_table_t *rhash, uint_t hashval)
1602 {
1603
1604 if (rhash->bucket == NULL)
1605 return (NULL);
1606 else
1607 return ((void *)rhash->bucket[hashval]);
1608 }
1609
1610 /*
1611 * generic function to get a specific bucket's address
1612 */
1613 static void **
1614 rsmhash_bktaddr(rsmhash_table_t *rhash, uint_t hashval)
1615 {
1616 if (rhash->bucket == NULL)
1617 return (NULL);
1618 else
1619 return ((void **)&(rhash->bucket[hashval]));
1620 }
1621
1622 /*
1623 * generic function to alloc a hash table
1624 */
1625 static void
1626 rsmhash_alloc(rsmhash_table_t *rhash, int size)
1627 {
1628 rhash->bucket = (rsmresource_t **)
1629 kmem_zalloc(size * sizeof (rsmresource_t *), KM_SLEEP);
1630 }
1631
1632 /*
1633 * generic function to free a hash table
1634 */
1635 static void
1636 rsmhash_free(rsmhash_table_t *rhash, int size)
1637 {
1638
1639 kmem_free((void *)rhash->bucket, size * sizeof (caddr_t));
1640 rhash->bucket = NULL;
1641
1642 }
1643 /* *********************** Exported Segment Key Management ************ */
1644
1645 #define rsmexport_add(new, key) \
1646 rsmhash_add(&rsm_export_segs, (rsmresource_t *)new, key, 1, \
1647 RSM_STATE_BIND)
1648
1649 #define rsmexport_rm(arg) \
1650 rsmhash_rm(&rsm_export_segs, (rsmresource_t *)(arg))
1651
1652 #define rsmexport_lookup(key) \
1653 (rsmseg_t *)rsmhash_lookup(&rsm_export_segs, key, RSM_STATE_EXPORT)
1654
1655 /* ************************** Import Segment List Management ********** */
1656
1657 /*
1658 * Add segment to import list. This will be useful for paging and loopback
1659 * segment unloading.
1660 */
1661 #define rsmimport_add(arg, key) \
1662 rsmhash_add(&rsm_import_segs, (rsmresource_t *)(arg), (key), 0, \
1663 RSM_STATE_NEW)
1664
1665 #define rsmimport_rm(arg) \
1666 rsmhash_rm(&rsm_import_segs, (rsmresource_t *)(arg))
1667
1668 /*
1669 * #define rsmimport_lookup(key) \
1670 * (rsmseg_t *)rsmhash_lookup(&rsm_import_segs, (key), RSM_STATE_CONNECT)
1671 */
1672
1673 /*
1674 * increase the ref count and make the import segment point to the
1675 * shared data structure. Return a pointer to the share data struct
1676 * and the shared data struct is locked upon return
1677 */
1678 static rsm_import_share_t *
1679 rsmshare_get(rsm_memseg_id_t key, rsm_node_id_t node, adapter_t *adapter,
1680 rsmseg_t *segp)
1681 {
1682 uint_t hash;
1683 rsmresource_t *p;
1684 rsm_import_share_t *shdatap;
1685 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1686
1687 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmshare_get enter\n"));
1688
1689 hash = rsmhash(key);
1690 /* lock table */
1691 rw_enter(&rsm_import_segs.rsmhash_rw, RW_WRITER);
1692 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmshare_get:key=%u, hash=%d\n",
1693 key, hash));
1694
1695 p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hash);
1696
1697 for (; p; p = p->rsmrc_next) {
1698 /*
1699 * Look for an entry that is importing the same exporter
1700 * with the share data structure allocated.
1701 */
1702 if ((p->rsmrc_key == key) &&
1703 (p->rsmrc_node == node) &&
1704 (p->rsmrc_adapter == adapter) &&
1705 (((rsmseg_t *)p)->s_share != NULL)) {
1706 shdatap = ((rsmseg_t *)p)->s_share;
1707 break;
1708 }
1709 }
1710
1711 if (p == NULL) {
1712 /* we are the first importer, create the shared data struct */
1713 shdatap = kmem_zalloc(sizeof (rsm_import_share_t), KM_SLEEP);
1714 shdatap->rsmsi_state = RSMSI_STATE_NEW;
1715 shdatap->rsmsi_segid = key;
1716 shdatap->rsmsi_node = node;
1717 mutex_init(&shdatap->rsmsi_lock, NULL, MUTEX_DRIVER, NULL);
1718 cv_init(&shdatap->rsmsi_cv, NULL, CV_DRIVER, 0);
1719 }
1720
1721 rsmseglock_acquire(segp);
1722
1723 /* we grab the shared lock before returning from this function */
1724 mutex_enter(&shdatap->rsmsi_lock);
1725
1726 shdatap->rsmsi_refcnt++;
1727 segp->s_share = shdatap;
1728
1729 rsmseglock_release(segp);
1730
1731 rw_exit(&rsm_import_segs.rsmhash_rw);
1732
1733 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmshare_get done\n"));
1734
1735 return (shdatap);
1736 }
1737
1738 /*
1739 * the shared data structure should be locked before calling
1740 * rsmsharecv_signal().
1741 * Change the state and signal any waiting segments.
1742 */
1743 void
1744 rsmsharecv_signal(rsmseg_t *seg, int oldstate, int newstate)
1745 {
1746 ASSERT(rsmsharelock_held(seg));
1747
1748 if (seg->s_share->rsmsi_state == oldstate) {
1749 seg->s_share->rsmsi_state = newstate;
1750 cv_broadcast(&seg->s_share->rsmsi_cv);
1751 }
1752 }
1753
1754 /*
1755 * Add to the hash table
1756 */
1757 static void
1758 importer_list_add(rsm_node_id_t node, rsm_memseg_id_t key, rsm_addr_t hwaddr,
1759 void *cookie)
1760 {
1761
1762 importing_token_t *head;
1763 importing_token_t *new_token;
1764 int index;
1765
1766 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1767
1768 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_add enter\n"));
1769
1770 new_token = kmem_zalloc(sizeof (importing_token_t), KM_SLEEP);
1771 new_token->importing_node = node;
1772 new_token->key = key;
1773 new_token->import_segment_cookie = cookie;
1774 new_token->importing_adapter_hwaddr = hwaddr;
1775
1776 index = rsmhash(key);
1777
1778 mutex_enter(&importer_list.lock);
1779
1780 head = importer_list.bucket[index];
1781 importer_list.bucket[index] = new_token;
1782 new_token->next = head;
1783 mutex_exit(&importer_list.lock);
1784
1785 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_add done\n"));
1786 }
1787
1788 static void
1789 importer_list_rm(rsm_node_id_t node, rsm_memseg_id_t key, void *cookie)
1790 {
1791
1792 importing_token_t *prev, *token = NULL;
1793 int index;
1794 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1795
1796 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_rm enter\n"));
1797
1798 index = rsmhash(key);
1799
1800 mutex_enter(&importer_list.lock);
1801
1802 token = importer_list.bucket[index];
1803
1804 prev = token;
1805 while (token != NULL) {
1806 if (token->importing_node == node &&
1807 token->import_segment_cookie == cookie) {
1808 if (prev == token)
1809 importer_list.bucket[index] = token->next;
1810 else
1811 prev->next = token->next;
1812 kmem_free((void *)token, sizeof (*token));
1813 break;
1814 } else {
1815 prev = token;
1816 token = token->next;
1817 }
1818 }
1819
1820 mutex_exit(&importer_list.lock);
1821
1822 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_rm done\n"));
1823
1824
1825 }
1826
1827 /* **************************Segment Structure Management ************* */
1828
1829 /*
1830 * Free segment structure
1831 */
1832 static void
1833 rsmseg_free(rsmseg_t *seg)
1834 {
1835
1836 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1837
1838 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_free enter\n"));
1839
1840 /* need to take seglock here to avoid race with rsmmap_unmap() */
1841 rsmseglock_acquire(seg);
1842 if (seg->s_ckl != NULL) {
1843 /* Segment is still busy */
1844 seg->s_state = RSM_STATE_END;
1845 rsmseglock_release(seg);
1846 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
1847 "rsmseg_free done\n"));
1848 return;
1849 }
1850
1851 rsmseglock_release(seg);
1852
1853 ASSERT(seg->s_state == RSM_STATE_END || seg->s_state == RSM_STATE_NEW);
1854
1855 /*
1856 * If it's an importer decrement the refcount
1857 * and if its down to zero free the shared data structure.
1858 * This is where failures during rsm_connect() are unrefcounted
1859 */
1860 if (seg->s_share != NULL) {
1861
1862 ASSERT(seg->s_type == RSM_RESOURCE_IMPORT_SEGMENT);
1863
1864 rsmsharelock_acquire(seg);
1865
1866 ASSERT(seg->s_share->rsmsi_refcnt > 0);
1867
1868 seg->s_share->rsmsi_refcnt--;
1869
1870 if (seg->s_share->rsmsi_refcnt == 0) {
1871 rsmsharelock_release(seg);
1872 mutex_destroy(&seg->s_share->rsmsi_lock);
1873 cv_destroy(&seg->s_share->rsmsi_cv);
1874 kmem_free((void *)(seg->s_share),
1875 sizeof (rsm_import_share_t));
1876 } else {
1877 rsmsharelock_release(seg);
1878 }
1879 /*
1880 * The following needs to be done after any
1881 * rsmsharelock calls which use seg->s_share.
1882 */
1883 seg->s_share = NULL;
1884 }
1885
1886 cv_destroy(&seg->s_cv);
1887 mutex_destroy(&seg->s_lock);
1888 rsmacl_free(seg->s_acl, seg->s_acl_len);
1889 rsmpiacl_free(seg->s_acl_in, seg->s_acl_len);
1890 if (seg->s_adapter)
1891 rsmka_release_adapter(seg->s_adapter);
1892
1893 kmem_free((void *)seg, sizeof (*seg));
1894
1895 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_free done\n"));
1896
1897 }
1898
1899
1900 static rsmseg_t *
1901 rsmseg_alloc(minor_t num, struct cred *cred)
1902 {
1903 rsmseg_t *new;
1904 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
1905
1906 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_alloc enter\n"));
1907 /*
1908 * allocate memory for new segment. This should be a segkmem cache.
1909 */
1910 new = (rsmseg_t *)kmem_zalloc(sizeof (*new), KM_SLEEP);
1911
1912 new->s_state = RSM_STATE_NEW;
1913 new->s_minor = num;
1914 new->s_acl_len = 0;
1915 new->s_cookie = NULL;
1916 new->s_adapter = NULL;
1917
1918 new->s_mode = 0777 & ~PTOU((ttoproc(curthread)))->u_cmask;
1919 /* we don't have a key yet, will set at export/connect */
1920 new->s_uid = crgetuid(cred);
1921 new->s_gid = crgetgid(cred);
1922
1923 mutex_init(&new->s_lock, NULL, MUTEX_DRIVER, (void *)NULL);
1924 cv_init(&new->s_cv, NULL, CV_DRIVER, 0);
1925
1926 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_alloc done\n"));
1927
1928 return (new);
1929 }
1930
1931 /* ******************************** Driver Open/Close/Poll *************** */
1932
1933 /*ARGSUSED1*/
1934 static int
1935 rsm_open(dev_t *devp, int flag, int otyp, struct cred *cred)
1936 {
1937 minor_t rnum;
1938 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI);
1939
1940 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_open enter\n"));
1941 /*
1942 * Char only
1943 */
1944 if (otyp != OTYP_CHR) {
1945 DBG_PRINTF((category, RSM_ERR, "rsm_open: bad otyp\n"));
1946 return (EINVAL);
1947 }
1948
1949 /*
1950 * Only zero can be opened, clones are used for resources.
1951 */
1952 if (getminor(*devp) != RSM_DRIVER_MINOR) {
1953 DBG_PRINTF((category, RSM_ERR,
1954 "rsm_open: bad minor %d\n", getminor(*devp)));
1955 return (ENODEV);
1956 }
1957
1958 if ((flag & FEXCL) != 0 && secpolicy_excl_open(cred) != 0) {
1959 DBG_PRINTF((category, RSM_ERR, "rsm_open: bad perm\n"));
1960 return (EPERM);
1961 }
1962
1963 if (!(flag & FWRITE)) {
1964 /*
1965 * The library function _rsm_librsm_init calls open for
1966 * /dev/rsm with flag set to O_RDONLY. We want a valid
1967 * file descriptor to be returned for minor device zero.
1968 */
1969
1970 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
1971 "rsm_open RDONLY done\n"));
1972 return (DDI_SUCCESS);
1973 }
1974
1975 /*
1976 * - allocate new minor number and segment.
1977 * - add segment to list of all segments.
1978 * - set minordev data to segment
1979 * - update devp argument to new device
1980 * - update s_cred to cred; make sure you do crhold(cred);
1981 */
1982
1983 /* allocate a new resource number */
1984 if (rsmresource_alloc(&rnum) == RSM_SUCCESS) {
1985 /*
1986 * We will bind this minor to a specific resource in first
1987 * ioctl
1988 */
1989 *devp = makedevice(getmajor(*devp), rnum);
1990 } else {
1991 return (EAGAIN);
1992 }
1993
1994 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_open done\n"));
1995 return (DDI_SUCCESS);
1996 }
1997
1998 static void
1999 rsmseg_close(rsmseg_t *seg, int force_flag)
2000 {
2001 int e = RSM_SUCCESS;
2002
2003 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI);
2004
2005 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_close enter\n"));
2006
2007 rsmseglock_acquire(seg);
2008 if (!force_flag && (seg->s_hdr.rsmrc_type ==
2009 RSM_RESOURCE_EXPORT_SEGMENT)) {
2010 /*
2011 * If we are processing rsm_close wait for force_destroy
2012 * processing to complete since force_destroy processing
2013 * needs to finish first before we can free the segment.
2014 * force_destroy is only for export segments
2015 */
2016 while (seg->s_flags & RSM_FORCE_DESTROY_WAIT) {
2017 cv_wait(&seg->s_cv, &seg->s_lock);
2018 }
2019 }
2020 rsmseglock_release(seg);
2021
2022 /* It's ok to read the state without a lock */
2023 switch (seg->s_state) {
2024 case RSM_STATE_EXPORT:
2025 case RSM_STATE_EXPORT_QUIESCING:
2026 case RSM_STATE_EXPORT_QUIESCED:
2027 e = rsm_unpublish(seg, 1);
2028 /* FALLTHRU */
2029 case RSM_STATE_BIND_QUIESCED:
2030 /* FALLTHRU */
2031 case RSM_STATE_BIND:
2032 e = rsm_unbind(seg);
2033 if (e != RSM_SUCCESS && force_flag == 1)
2034 return;
2035 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT);
2036 /* FALLTHRU */
2037 case RSM_STATE_NEW_QUIESCED:
2038 rsmseglock_acquire(seg);
2039 seg->s_state = RSM_STATE_NEW;
2040 cv_broadcast(&seg->s_cv);
2041 rsmseglock_release(seg);
2042 break;
2043 case RSM_STATE_NEW:
2044 break;
2045 case RSM_STATE_ZOMBIE:
2046 /*
2047 * Segments in this state have been removed off the
2048 * exported segments list and have been unpublished
2049 * and unbind. These segments have been removed during
2050 * a callback to the rsm_export_force_destroy, which
2051 * is called for the purpose of unlocking these
2052 * exported memory segments when a process exits but
2053 * leaves the segments locked down since rsm_close is
2054 * is not called for the segments. This can happen
2055 * when a process calls fork or exec and then exits.
2056 * Once the segments are in the ZOMBIE state, all that
2057 * remains is to destroy them when rsm_close is called.
2058 * This is done here. Thus, for such segments the
2059 * the state is changed to new so that later in this
2060 * function rsmseg_free is called.
2061 */
2062 rsmseglock_acquire(seg);
2063 seg->s_state = RSM_STATE_NEW;
2064 rsmseglock_release(seg);
2065 break;
2066 case RSM_STATE_MAP_QUIESCE:
2067 case RSM_STATE_ACTIVE:
2068 /* Disconnect will handle the unmap */
2069 case RSM_STATE_CONN_QUIESCE:
2070 case RSM_STATE_CONNECT:
2071 case RSM_STATE_DISCONNECT:
2072 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
2073 (void) rsm_disconnect(seg);
2074 break;
2075 case RSM_STATE_MAPPING:
2076 /*FALLTHRU*/
2077 case RSM_STATE_END:
2078 DBG_PRINTF((category, RSM_ERR,
2079 "Invalid segment state %d in rsm_close\n", seg->s_state));
2080 break;
2081 default:
2082 DBG_PRINTF((category, RSM_ERR,
2083 "Invalid segment state %d in rsm_close\n", seg->s_state));
2084 break;
2085 }
2086
2087 /*
2088 * check state.
2089 * - make sure you do crfree(s_cred);
2090 * release segment and minor number
2091 */
2092 ASSERT(seg->s_state == RSM_STATE_NEW);
2093
2094 /*
2095 * The export_force_destroy callback is created to unlock
2096 * the exported segments of a process
2097 * when the process does a fork or exec and then exits calls this
2098 * function with the force flag set to 1 which indicates that the
2099 * segment state must be converted to ZOMBIE. This state means that the
2100 * segments still exist and have been unlocked and most importantly the
2101 * only operation allowed is to destroy them on an rsm_close.
2102 */
2103 if (force_flag) {
2104 rsmseglock_acquire(seg);
2105 seg->s_state = RSM_STATE_ZOMBIE;
2106 rsmseglock_release(seg);
2107 } else {
2108 rsmseg_free(seg);
2109 }
2110
2111 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_close done\n"));
2112 }
2113
2114 static int
2115 rsm_close(dev_t dev, int flag, int otyp, cred_t *cred)
2116 {
2117 minor_t rnum = getminor(dev);
2118 rsmresource_t *res;
2119 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI);
2120
2121 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close enter\n"));
2122
2123 flag = flag; cred = cred;
2124
2125 if (otyp != OTYP_CHR)
2126 return (EINVAL);
2127
2128 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rnum = %d\n", rnum));
2129
2130 /*
2131 * At this point we are the last reference to the resource.
2132 * Free resource number from resource table.
2133 * It's ok to remove number before we free the segment.
2134 * We need to lock the resource to protect against remote calls.
2135 */
2136 if (rnum == RSM_DRIVER_MINOR ||
2137 (res = rsmresource_free(rnum)) == NULL) {
2138 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close done\n"));
2139 return (DDI_SUCCESS);
2140 }
2141
2142 switch (res->rsmrc_type) {
2143 case RSM_RESOURCE_EXPORT_SEGMENT:
2144 case RSM_RESOURCE_IMPORT_SEGMENT:
2145 rsmseg_close((rsmseg_t *)res, 0);
2146 break;
2147 case RSM_RESOURCE_BAR:
2148 DBG_PRINTF((category, RSM_ERR, "bad resource in rsm_close\n"));
2149 break;
2150 default:
2151 break;
2152 }
2153
2154 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close done\n"));
2155
2156 return (DDI_SUCCESS);
2157 }
2158
2159 /*
2160 * rsm_inc_pgcnt
2161 *
2162 * Description: increment rsm page counter.
2163 *
2164 * Parameters: pgcnt_t pnum; number of pages to be used
2165 *
2166 * Returns: RSM_SUCCESS if memory limit not exceeded
2167 * ENOSPC if memory limit exceeded. In this case, the
2168 * page counter remains unchanged.
2169 *
2170 */
2171 static int
2172 rsm_inc_pgcnt(pgcnt_t pnum)
2173 {
2174 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2175 if (rsm_pgcnt_max == 0) { /* no upper limit has been set */
2176 return (RSM_SUCCESS);
2177 }
2178
2179 mutex_enter(&rsm_pgcnt_lock);
2180
2181 if (rsm_pgcnt + pnum > rsm_pgcnt_max) {
2182 /* ensure that limits have not been exceeded */
2183 mutex_exit(&rsm_pgcnt_lock);
2184 return (RSMERR_INSUFFICIENT_MEM);
2185 }
2186
2187 rsm_pgcnt += pnum;
2188 DBG_PRINTF((category, RSM_DEBUG, "rsm_pgcnt incr to %d.\n",
2189 rsm_pgcnt));
2190 mutex_exit(&rsm_pgcnt_lock);
2191
2192 return (RSM_SUCCESS);
2193 }
2194
2195 /*
2196 * rsm_dec_pgcnt
2197 *
2198 * Description: decrement rsm page counter.
2199 *
2200 * Parameters: pgcnt_t pnum; number of pages freed
2201 *
2202 */
2203 static void
2204 rsm_dec_pgcnt(pgcnt_t pnum)
2205 {
2206 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2207
2208 if (rsm_pgcnt_max == 0) { /* no upper limit has been set */
2209 return;
2210 }
2211
2212 mutex_enter(&rsm_pgcnt_lock);
2213 ASSERT(rsm_pgcnt >= pnum);
2214 rsm_pgcnt -= pnum;
2215 DBG_PRINTF((category, RSM_DEBUG, "rsm_pgcnt decr to %d.\n",
2216 rsm_pgcnt));
2217 mutex_exit(&rsm_pgcnt_lock);
2218 }
2219
2220 static struct umem_callback_ops rsm_as_ops = {
2221 UMEM_CALLBACK_VERSION, /* version number */
2222 rsm_export_force_destroy,
2223 };
2224
2225 static int
2226 rsm_bind_pages(ddi_umem_cookie_t *cookie, caddr_t vaddr, size_t len,
2227 proc_t *procp)
2228 {
2229 int error = RSM_SUCCESS;
2230 ulong_t pnum;
2231 struct umem_callback_ops *callbackops = &rsm_as_ops;
2232
2233 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2234
2235 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind_pages enter\n"));
2236
2237 /*
2238 * Make sure vaddr and len are aligned on a page boundary
2239 */
2240 if ((uintptr_t)vaddr & (PAGESIZE - 1)) {
2241 return (RSMERR_BAD_ADDR);
2242 }
2243
2244 if (len & (PAGESIZE - 1)) {
2245 return (RSMERR_BAD_LENGTH);
2246 }
2247
2248 /*
2249 * Find number of pages
2250 */
2251 pnum = btopr(len);
2252 error = rsm_inc_pgcnt(pnum);
2253 if (error != RSM_SUCCESS) {
2254 DBG_PRINTF((category, RSM_ERR,
2255 "rsm_bind_pages:mem limit exceeded\n"));
2256 return (RSMERR_INSUFFICIENT_MEM);
2257 }
2258
2259 error = umem_lockmemory(vaddr, len,
2260 DDI_UMEMLOCK_WRITE|DDI_UMEMLOCK_READ|DDI_UMEMLOCK_LONGTERM,
2261 cookie,
2262 callbackops, procp);
2263
2264 if (error) {
2265 rsm_dec_pgcnt(pnum);
2266 DBG_PRINTF((category, RSM_ERR,
2267 "rsm_bind_pages:ddi_umem_lock failed\n"));
2268 /*
2269 * ddi_umem_lock, in the case of failure, returns one of
2270 * the following three errors. These are translated into
2271 * the RSMERR namespace and returned.
2272 */
2273 if (error == EFAULT)
2274 return (RSMERR_BAD_ADDR);
2275 else if (error == EACCES)
2276 return (RSMERR_PERM_DENIED);
2277 else
2278 return (RSMERR_INSUFFICIENT_MEM);
2279 }
2280
2281 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind_pages done\n"));
2282
2283 return (error);
2284
2285 }
2286
2287 static int
2288 rsm_unbind_pages(rsmseg_t *seg)
2289 {
2290 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2291
2292 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind_pages enter\n"));
2293
2294 ASSERT(rsmseglock_held(seg));
2295
2296 if (seg->s_cookie != NULL) {
2297 /* unlock address range */
2298 ddi_umem_unlock(seg->s_cookie);
2299 rsm_dec_pgcnt(btopr(seg->s_len));
2300 seg->s_cookie = NULL;
2301 }
2302
2303 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind_pages done\n"));
2304
2305 return (RSM_SUCCESS);
2306 }
2307
2308
2309 static int
2310 rsm_bind(rsmseg_t *seg, rsm_ioctlmsg_t *msg, intptr_t dataptr, int mode)
2311 {
2312 int e;
2313 adapter_t *adapter;
2314 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2315
2316 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind enter\n"));
2317
2318 adapter = rsm_getadapter(msg, mode);
2319 if (adapter == NULL) {
2320 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2321 "rsm_bind done:no adapter\n"));
2322 return (RSMERR_CTLR_NOT_PRESENT);
2323 }
2324
2325 /* lock address range */
2326 if (msg->vaddr == NULL) {
2327 rsmka_release_adapter(adapter);
2328 DBG_PRINTF((category, RSM_ERR,
2329 "rsm: rsm_bind done: invalid vaddr\n"));
2330 return (RSMERR_BAD_ADDR);
2331 }
2332 if (msg->len <= 0) {
2333 rsmka_release_adapter(adapter);
2334 DBG_PRINTF((category, RSM_ERR,
2335 "rsm_bind: invalid length\n"));
2336 return (RSMERR_BAD_LENGTH);
2337 }
2338
2339 /* Lock segment */
2340 rsmseglock_acquire(seg);
2341
2342 while (seg->s_state == RSM_STATE_NEW_QUIESCED) {
2343 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
2344 DBG_PRINTF((category, RSM_DEBUG,
2345 "rsm_bind done: cv_wait INTERRUPTED"));
2346 rsmka_release_adapter(adapter);
2347 rsmseglock_release(seg);
2348 return (RSMERR_INTERRUPTED);
2349 }
2350 }
2351
2352 ASSERT(seg->s_state == RSM_STATE_NEW);
2353
2354 ASSERT(seg->s_cookie == NULL);
2355
2356 e = rsm_bind_pages(&seg->s_cookie, msg->vaddr, msg->len, curproc);
2357 if (e == RSM_SUCCESS) {
2358 seg->s_flags |= RSM_USER_MEMORY;
2359 if (msg->perm & RSM_ALLOW_REBIND) {
2360 seg->s_flags |= RSMKA_ALLOW_UNBIND_REBIND;
2361 }
2362 if (msg->perm & RSM_CREATE_SEG_DONTWAIT) {
2363 seg->s_flags |= RSMKA_SET_RESOURCE_DONTWAIT;
2364 }
2365 seg->s_region.r_vaddr = msg->vaddr;
2366 /*
2367 * Set the s_pid value in the segment structure. This is used
2368 * to identify exported segments belonging to a particular
2369 * process so that when the process exits, these segments can
2370 * be unlocked forcefully even if rsm_close is not called on
2371 * process exit since there maybe other processes referencing
2372 * them (for example on a fork or exec).
2373 * The s_pid value is also used to authenticate the process
2374 * doing a publish or unpublish on the export segment. Only
2375 * the creator of the export segment has a right to do a
2376 * publish or unpublish and unbind on the segment.
2377 */
2378 seg->s_pid = ddi_get_pid();
2379 seg->s_len = msg->len;
2380 seg->s_state = RSM_STATE_BIND;
2381 seg->s_adapter = adapter;
2382 seg->s_proc = curproc;
2383 } else {
2384 rsmka_release_adapter(adapter);
2385 DBG_PRINTF((category, RSM_WARNING,
2386 "unable to lock down pages\n"));
2387 }
2388
2389 msg->rnum = seg->s_minor;
2390 /* Unlock segment */
2391 rsmseglock_release(seg);
2392
2393 if (e == RSM_SUCCESS) {
2394 /* copyout the resource number */
2395 #ifdef _MULTI_DATAMODEL
2396 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
2397 rsm_ioctlmsg32_t msg32;
2398
2399 msg32.rnum = msg->rnum;
2400 if (ddi_copyout((caddr_t)&msg32.rnum,
2401 (caddr_t)&((rsm_ioctlmsg32_t *)dataptr)->rnum,
2402 sizeof (minor_t), mode)) {
2403 rsmka_release_adapter(adapter);
2404 e = RSMERR_BAD_ADDR;
2405 }
2406 }
2407 #endif
2408 if (ddi_copyout((caddr_t)&msg->rnum,
2409 (caddr_t)&((rsm_ioctlmsg_t *)dataptr)->rnum,
2410 sizeof (minor_t), mode)) {
2411 rsmka_release_adapter(adapter);
2412 e = RSMERR_BAD_ADDR;
2413 }
2414 }
2415
2416 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind done\n"));
2417
2418 return (e);
2419 }
2420
2421 static void
2422 rsm_remap_local_importers(rsm_node_id_t src_nodeid,
2423 rsm_memseg_id_t ex_segid, ddi_umem_cookie_t cookie)
2424 {
2425 rsmresource_t *p = NULL;
2426 rsmhash_table_t *rhash = &rsm_import_segs;
2427 uint_t index;
2428
2429 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_FUNC_ALL, RSM_DEBUG_VERBOSE,
2430 "rsm_remap_local_importers enter\n"));
2431
2432 index = rsmhash(ex_segid);
2433
2434 rw_enter(&rhash->rsmhash_rw, RW_READER);
2435
2436 p = rsmhash_getbkt(rhash, index);
2437
2438 for (; p; p = p->rsmrc_next) {
2439 rsmseg_t *seg = (rsmseg_t *)p;
2440 rsmseglock_acquire(seg);
2441 /*
2442 * Change the s_cookie value of only the local importers
2443 * which have been mapped (in state RSM_STATE_ACTIVE).
2444 * Note that there is no need to change the s_cookie value
2445 * if the imported segment is in RSM_STATE_MAPPING since
2446 * eventually the s_cookie will be updated via the mapping
2447 * functionality.
2448 */
2449 if ((seg->s_segid == ex_segid) && (seg->s_node == src_nodeid) &&
2450 (seg->s_state == RSM_STATE_ACTIVE)) {
2451 seg->s_cookie = cookie;
2452 }
2453 rsmseglock_release(seg);
2454 }
2455 rw_exit(&rhash->rsmhash_rw);
2456
2457 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_FUNC_ALL, RSM_DEBUG_VERBOSE,
2458 "rsm_remap_local_importers done\n"));
2459 }
2460
2461 static int
2462 rsm_rebind(rsmseg_t *seg, rsm_ioctlmsg_t *msg)
2463 {
2464 int e;
2465 adapter_t *adapter;
2466 ddi_umem_cookie_t cookie;
2467 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2468
2469 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind enter\n"));
2470
2471 /* Check for permissions to rebind */
2472 if (!(seg->s_flags & RSMKA_ALLOW_UNBIND_REBIND)) {
2473 return (RSMERR_REBIND_NOT_ALLOWED);
2474 }
2475
2476 if (seg->s_pid != ddi_get_pid() &&
2477 ddi_get_pid() != 0) {
2478 DBG_PRINTF((category, RSM_ERR, "rsm_rebind: Not owner\n"));
2479 return (RSMERR_NOT_CREATOR);
2480 }
2481
2482 /*
2483 * We will not be allowing partial rebind and hence length passed
2484 * in must be same as segment length
2485 */
2486 if (msg->vaddr == NULL) {
2487 DBG_PRINTF((category, RSM_ERR,
2488 "rsm_rebind done: null msg->vaddr\n"));
2489 return (RSMERR_BAD_ADDR);
2490 }
2491 if (msg->len != seg->s_len) {
2492 DBG_PRINTF((category, RSM_ERR,
2493 "rsm_rebind: invalid length\n"));
2494 return (RSMERR_BAD_LENGTH);
2495 }
2496
2497 /* Lock segment */
2498 rsmseglock_acquire(seg);
2499
2500 while ((seg->s_state == RSM_STATE_BIND_QUIESCED) ||
2501 (seg->s_state == RSM_STATE_EXPORT_QUIESCING) ||
2502 (seg->s_state == RSM_STATE_EXPORT_QUIESCED)) {
2503 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
2504 rsmseglock_release(seg);
2505 DBG_PRINTF((category, RSM_DEBUG,
2506 "rsm_rebind done: cv_wait INTERRUPTED"));
2507 return (RSMERR_INTERRUPTED);
2508 }
2509 }
2510
2511 /* verify segment state */
2512 if ((seg->s_state != RSM_STATE_BIND) &&
2513 (seg->s_state != RSM_STATE_EXPORT)) {
2514 /* Unlock segment */
2515 rsmseglock_release(seg);
2516 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2517 "rsm_rebind done: invalid state\n"));
2518 return (RSMERR_BAD_SEG_HNDL);
2519 }
2520
2521 ASSERT(seg->s_cookie != NULL);
2522
2523 if (msg->vaddr == seg->s_region.r_vaddr) {
2524 rsmseglock_release(seg);
2525 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind done\n"));
2526 return (RSM_SUCCESS);
2527 }
2528
2529 e = rsm_bind_pages(&cookie, msg->vaddr, msg->len, curproc);
2530 if (e == RSM_SUCCESS) {
2531 struct buf *xbuf;
2532 dev_t sdev = 0;
2533 rsm_memory_local_t mem;
2534
2535 xbuf = ddi_umem_iosetup(cookie, 0, msg->len, B_WRITE,
2536 sdev, 0, NULL, DDI_UMEM_SLEEP);
2537 ASSERT(xbuf != NULL);
2538
2539 mem.ms_type = RSM_MEM_BUF;
2540 mem.ms_bp = xbuf;
2541
2542 adapter = seg->s_adapter;
2543 e = adapter->rsmpi_ops->rsm_rebind(
2544 seg->s_handle.out, 0, &mem,
2545 RSM_RESOURCE_DONTWAIT, NULL);
2546
2547 if (e == RSM_SUCCESS) {
2548 /*
2549 * unbind the older pages, and unload local importers;
2550 * but don't disconnect importers
2551 */
2552 (void) rsm_unbind_pages(seg);
2553 seg->s_cookie = cookie;
2554 seg->s_region.r_vaddr = msg->vaddr;
2555 rsm_remap_local_importers(my_nodeid, seg->s_segid,
2556 cookie);
2557 } else {
2558 /*
2559 * Unbind the pages associated with "cookie" by the
2560 * rsm_bind_pages calls prior to this. This is
2561 * similar to what is done in the rsm_unbind_pages
2562 * routine for the seg->s_cookie.
2563 */
2564 ddi_umem_unlock(cookie);
2565 rsm_dec_pgcnt(btopr(msg->len));
2566 DBG_PRINTF((category, RSM_ERR,
2567 "rsm_rebind failed with %d\n", e));
2568 }
2569 /*
2570 * At present there is no dependency on the existence of xbuf.
2571 * So we can free it here. If in the future this changes, it can
2572 * be freed sometime during the segment destroy.
2573 */
2574 freerbuf(xbuf);
2575 }
2576
2577 /* Unlock segment */
2578 rsmseglock_release(seg);
2579
2580 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind done\n"));
2581
2582 return (e);
2583 }
2584
2585 static int
2586 rsm_unbind(rsmseg_t *seg)
2587 {
2588 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2589
2590 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind enter\n"));
2591
2592 rsmseglock_acquire(seg);
2593
2594 /* verify segment state */
2595 if ((seg->s_state != RSM_STATE_BIND) &&
2596 (seg->s_state != RSM_STATE_BIND_QUIESCED)) {
2597 rsmseglock_release(seg);
2598 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2599 "rsm_unbind: invalid state\n"));
2600 return (RSMERR_BAD_SEG_HNDL);
2601 }
2602
2603 /* unlock current range */
2604 (void) rsm_unbind_pages(seg);
2605
2606 if (seg->s_state == RSM_STATE_BIND) {
2607 seg->s_state = RSM_STATE_NEW;
2608 } else if (seg->s_state == RSM_STATE_BIND_QUIESCED) {
2609 seg->s_state = RSM_STATE_NEW_QUIESCED;
2610 }
2611
2612 rsmseglock_release(seg);
2613
2614 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind done\n"));
2615
2616 return (RSM_SUCCESS);
2617 }
2618
2619 /* **************************** Exporter Access List Management ******* */
2620 static void
2621 rsmacl_free(rsmapi_access_entry_t *acl, int acl_len)
2622 {
2623 int acl_sz;
2624 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2625
2626 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_free enter\n"));
2627
2628 /* acl could be NULL */
2629
2630 if (acl != NULL && acl_len > 0) {
2631 acl_sz = acl_len * sizeof (rsmapi_access_entry_t);
2632 kmem_free((void *)acl, acl_sz);
2633 }
2634
2635 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_free done\n"));
2636 }
2637
2638 static void
2639 rsmpiacl_free(rsm_access_entry_t *acl, int acl_len)
2640 {
2641 int acl_sz;
2642 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2643
2644 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_free enter\n"));
2645
2646 if (acl != NULL && acl_len > 0) {
2647 acl_sz = acl_len * sizeof (rsm_access_entry_t);
2648 kmem_free((void *)acl, acl_sz);
2649 }
2650
2651 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_free done\n"));
2652
2653 }
2654
2655 static int
2656 rsmacl_build(rsm_ioctlmsg_t *msg, int mode,
2657 rsmapi_access_entry_t **list, int *len, int loopback)
2658 {
2659 rsmapi_access_entry_t *acl;
2660 int acl_len;
2661 int i;
2662 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2663
2664 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_build enter\n"));
2665
2666 *len = 0;
2667 *list = NULL;
2668
2669 acl_len = msg->acl_len;
2670 if ((loopback && acl_len > 1) || (acl_len < 0) ||
2671 (acl_len > MAX_NODES)) {
2672 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2673 "rsmacl_build done: acl invalid\n"));
2674 return (RSMERR_BAD_ACL);
2675 }
2676
2677 if (acl_len > 0 && acl_len <= MAX_NODES) {
2678 size_t acl_size = acl_len * sizeof (rsmapi_access_entry_t);
2679
2680 acl = kmem_alloc(acl_size, KM_SLEEP);
2681
2682 if (ddi_copyin((caddr_t)msg->acl, (caddr_t)acl,
2683 acl_size, mode)) {
2684 kmem_free((void *) acl, acl_size);
2685 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2686 "rsmacl_build done: BAD_ADDR\n"));
2687 return (RSMERR_BAD_ADDR);
2688 }
2689
2690 /*
2691 * Verify access list
2692 */
2693 for (i = 0; i < acl_len; i++) {
2694 if (acl[i].ae_node > MAX_NODES ||
2695 (loopback && (acl[i].ae_node != my_nodeid)) ||
2696 acl[i].ae_permission > RSM_ACCESS_TRUSTED) {
2697 /* invalid entry */
2698 kmem_free((void *) acl, acl_size);
2699 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2700 "rsmacl_build done: EINVAL\n"));
2701 return (RSMERR_BAD_ACL);
2702 }
2703 }
2704
2705 *len = acl_len;
2706 *list = acl;
2707 }
2708
2709 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_build done\n"));
2710
2711 return (DDI_SUCCESS);
2712 }
2713
2714 static int
2715 rsmpiacl_create(rsmapi_access_entry_t *src, rsm_access_entry_t **dest,
2716 int acl_len, adapter_t *adapter)
2717 {
2718 rsm_access_entry_t *acl;
2719 rsm_addr_t hwaddr;
2720 int i;
2721 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2722
2723 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_create enter\n"));
2724
2725 if (src != NULL) {
2726 size_t acl_size = acl_len * sizeof (rsm_access_entry_t);
2727 acl = kmem_alloc(acl_size, KM_SLEEP);
2728
2729 /*
2730 * translate access list
2731 */
2732 for (i = 0; i < acl_len; i++) {
2733 if (src[i].ae_node == my_nodeid) {
2734 acl[i].ae_addr = adapter->hwaddr;
2735 } else {
2736 hwaddr = get_remote_hwaddr(adapter,
2737 src[i].ae_node);
2738 if ((int64_t)hwaddr < 0) {
2739 /* invalid hwaddr */
2740 kmem_free((void *) acl, acl_size);
2741 DBG_PRINTF((category,
2742 RSM_DEBUG_VERBOSE,
2743 "rsmpiacl_create done:"
2744 "EINVAL hwaddr\n"));
2745 return (RSMERR_INTERNAL_ERROR);
2746 }
2747 acl[i].ae_addr = hwaddr;
2748 }
2749 /* rsmpi understands only RSM_PERM_XXXX */
2750 acl[i].ae_permission =
2751 src[i].ae_permission & RSM_PERM_RDWR;
2752 }
2753 *dest = acl;
2754 } else {
2755 *dest = NULL;
2756 }
2757
2758 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_create done\n"));
2759
2760 return (RSM_SUCCESS);
2761 }
2762
2763 static int
2764 rsmsegacl_validate(rsmipc_request_t *req, rsm_node_id_t rnode,
2765 rsmipc_reply_t *reply)
2766 {
2767
2768 int i;
2769 rsmseg_t *seg;
2770 rsm_memseg_id_t key = req->rsmipc_key;
2771 rsm_permission_t perm = req->rsmipc_perm;
2772 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2773
2774 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2775 "rsmsegacl_validate enter\n"));
2776
2777 /*
2778 * Find segment and grab its lock. The reason why we grab the segment
2779 * lock in side the search is to avoid the race when the segment is
2780 * being deleted and we already have a pointer to it.
2781 */
2782 seg = rsmexport_lookup(key);
2783 if (!seg) {
2784 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2785 "rsmsegacl_validate done: %u ENXIO\n", key));
2786 return (RSMERR_SEG_NOT_PUBLISHED);
2787 }
2788
2789 ASSERT(rsmseglock_held(seg));
2790 ASSERT(seg->s_state == RSM_STATE_EXPORT);
2791
2792 /*
2793 * We implement a 2-level protection scheme.
2794 * First, we check if local/remote host has access rights.
2795 * Second, we check if the user has access rights.
2796 *
2797 * This routine only validates the rnode access_list
2798 */
2799 if (seg->s_acl_len > 0) {
2800 /*
2801 * Check host access list
2802 */
2803 ASSERT(seg->s_acl != NULL);
2804 for (i = 0; i < seg->s_acl_len; i++) {
2805 if (seg->s_acl[i].ae_node == rnode) {
2806 perm &= seg->s_acl[i].ae_permission;
2807 goto found;
2808 }
2809 }
2810 /* rnode is not found in the list */
2811 rsmseglock_release(seg);
2812 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
2813 "rsmsegacl_validate done: EPERM\n"));
2814 return (RSMERR_SEG_NOT_PUBLISHED_TO_NODE);
2815 } else {
2816 /* use default owner creation umask */
2817 perm &= seg->s_mode;
2818 }
2819
2820 found:
2821 /* update perm for this node */
2822 reply->rsmipc_mode = perm;
2823 reply->rsmipc_uid = seg->s_uid;
2824 reply->rsmipc_gid = seg->s_gid;
2825 reply->rsmipc_segid = seg->s_segid;
2826 reply->rsmipc_seglen = seg->s_len;
2827
2828 /*
2829 * Perm of requesting node is valid; source will validate user
2830 */
2831 rsmseglock_release(seg);
2832
2833 /*
2834 * Add the importer to the list right away, if connect fails
2835 * the importer will ask the exporter to remove it.
2836 */
2837 importer_list_add(rnode, key, req->rsmipc_adapter_hwaddr,
2838 req->rsmipc_segment_cookie);
2839
2840 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegacl_validate done\n"));
2841
2842 return (RSM_SUCCESS);
2843 }
2844
2845
2846 /* ************************** Exporter Calls ************************* */
2847
2848 static int
2849 rsm_publish(rsmseg_t *seg, rsm_ioctlmsg_t *msg, intptr_t dataptr, int mode)
2850 {
2851 int e;
2852 int acl_len;
2853 rsmapi_access_entry_t *acl;
2854 rsm_access_entry_t *rsmpi_acl;
2855 rsm_memory_local_t mem;
2856 struct buf *xbuf;
2857 dev_t sdev = 0;
2858 adapter_t *adapter;
2859 rsm_memseg_id_t segment_id = 0;
2860 int loopback_flag = 0;
2861 int create_flags = 0;
2862 rsm_resource_callback_t callback_flag;
2863 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
2864
2865 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_publish enter\n"));
2866
2867 if (seg->s_adapter == &loopback_adapter)
2868 loopback_flag = 1;
2869
2870 if (seg->s_pid != ddi_get_pid() &&
2871 ddi_get_pid() != 0) {
2872 DBG_PRINTF((category, RSM_ERR,
2873 "rsm_publish: Not creator\n"));
2874 return (RSMERR_NOT_CREATOR);
2875 }
2876
2877 /*
2878 * Get per node access list
2879 */
2880 e = rsmacl_build(msg, mode, &acl, &acl_len, loopback_flag);
2881 if (e != DDI_SUCCESS) {
2882 DBG_PRINTF((category, RSM_ERR,
2883 "rsm_publish done: rsmacl_build failed\n"));
2884 return (e);
2885 }
2886
2887 /*
2888 * The application provided msg->key is used for resolving a
2889 * segment id according to the following:
2890 * key = 0 Kernel Agent selects the segment id
2891 * key <= RSM_DLPI_ID_END Reserved for system usage except
2892 * RSMLIB range
2893 * key < RSM_USER_APP_ID_BASE segment id = key
2894 * key >= RSM_USER_APP_ID_BASE Reserved for KA selections
2895 *
2896 * rsm_nextavail_segmentid is initialized to 0x80000000 and
2897 * overflows to zero after 0x80000000 allocations.
2898 * An algorithm is needed which allows reinitialization and provides
2899 * for reallocation after overflow. For now, ENOMEM is returned
2900 * once the overflow condition has occurred.
2901 */
2902 if (msg->key == 0) {
2903 mutex_enter(&rsm_lock);
2904 segment_id = rsm_nextavail_segmentid;
2905 if (segment_id != 0) {
2906 rsm_nextavail_segmentid++;
2907 mutex_exit(&rsm_lock);
2908 } else {
2909 mutex_exit(&rsm_lock);
2910 DBG_PRINTF((category, RSM_ERR,
2911 "rsm_publish done: no more keys avlbl\n"));
2912 return (RSMERR_INSUFFICIENT_RESOURCES);
2913 }
2914 } else if BETWEEN(msg->key, RSM_RSMLIB_ID_BASE, RSM_RSMLIB_ID_END)
2915 /* range reserved for internal use by base/ndi libraries */
2916 segment_id = msg->key;
2917 else if (msg->key <= RSM_DLPI_ID_END)
2918 return (RSMERR_RESERVED_SEGID);
2919 else if (msg->key <= (uint_t)RSM_USER_APP_ID_BASE -1)
2920 segment_id = msg->key;
2921 else {
2922 DBG_PRINTF((category, RSM_ERR,
2923 "rsm_publish done: invalid key %u\n", msg->key));
2924 return (RSMERR_RESERVED_SEGID);
2925 }
2926
2927 /* Add key to exportlist; The segment lock is held on success */
2928 e = rsmexport_add(seg, segment_id);
2929 if (e) {
2930 rsmacl_free(acl, acl_len);
2931 DBG_PRINTF((category, RSM_ERR,
2932 "rsm_publish done: export_add failed: %d\n", e));
2933 return (e);
2934 }
2935
2936 seg->s_segid = segment_id;
2937
2938 if ((seg->s_state != RSM_STATE_BIND) &&
2939 (seg->s_state != RSM_STATE_BIND_QUIESCED)) {
2940 /* state changed since then, free acl and return */
2941 rsmseglock_release(seg);
2942 rsmexport_rm(seg);
2943 rsmacl_free(acl, acl_len);
2944 DBG_PRINTF((category, RSM_ERR,
2945 "rsm_publish done: segment in wrong state: %d\n",
2946 seg->s_state));
2947 return (RSMERR_BAD_SEG_HNDL);
2948 }
2949
2950 /*
2951 * If this is for a local memory handle and permissions are zero,
2952 * then the surrogate segment is very large and we want to skip
2953 * allocation of DVMA space.
2954 *
2955 * Careful! If the user didn't use an ACL list, acl will be a NULL
2956 * pointer. Check that before dereferencing it.
2957 */
2958 if (acl != (rsmapi_access_entry_t *)NULL) {
2959 if (acl[0].ae_node == my_nodeid && acl[0].ae_permission == 0)
2960 goto skipdriver;
2961 }
2962
2963 /* create segment */
2964 xbuf = ddi_umem_iosetup(seg->s_cookie, 0, seg->s_len, B_WRITE,
2965 sdev, 0, NULL, DDI_UMEM_SLEEP);
2966 ASSERT(xbuf != NULL);
2967
2968 mem.ms_type = RSM_MEM_BUF;
2969 mem.ms_bp = xbuf;
2970
2971 /* This call includes a bind operations */
2972
2973 adapter = seg->s_adapter;
2974 /*
2975 * create a acl list with hwaddr for RSMPI publish
2976 */
2977 e = rsmpiacl_create(acl, &rsmpi_acl, acl_len, adapter);
2978
2979 if (e != RSM_SUCCESS) {
2980 rsmseglock_release(seg);
2981 rsmexport_rm(seg);
2982 rsmacl_free(acl, acl_len);
2983 freerbuf(xbuf);
2984 DBG_PRINTF((category, RSM_ERR,
2985 "rsm_publish done: rsmpiacl_create failed: %d\n", e));
2986 return (e);
2987 }
2988
2989 if (seg->s_state == RSM_STATE_BIND) {
2990 /* create segment */
2991
2992 /* This call includes a bind operations */
2993
2994 if (seg->s_flags & RSMKA_ALLOW_UNBIND_REBIND) {
2995 create_flags = RSM_ALLOW_UNBIND_REBIND;
2996 }
2997
2998 if (seg->s_flags & RSMKA_SET_RESOURCE_DONTWAIT) {
2999 callback_flag = RSM_RESOURCE_DONTWAIT;
3000 } else {
3001 callback_flag = RSM_RESOURCE_SLEEP;
3002 }
3003
3004 e = adapter->rsmpi_ops->rsm_seg_create(
3005 adapter->rsmpi_handle,
3006 &seg->s_handle.out, seg->s_len,
3007 create_flags, &mem,
3008 callback_flag, NULL);
3009 /*
3010 * At present there is no dependency on the existence of xbuf.
3011 * So we can free it here. If in the future this changes, it can
3012 * be freed sometime during the segment destroy.
3013 */
3014 freerbuf(xbuf);
3015
3016 if (e != RSM_SUCCESS) {
3017 rsmseglock_release(seg);
3018 rsmexport_rm(seg);
3019 rsmacl_free(acl, acl_len);
3020 rsmpiacl_free(rsmpi_acl, acl_len);
3021 DBG_PRINTF((category, RSM_ERR,
3022 "rsm_publish done: export_create failed: %d\n", e));
3023 /*
3024 * The following assertion ensures that the two errors
3025 * related to the length and its alignment do not occur
3026 * since they have been checked during export_create
3027 */
3028 ASSERT(e != RSMERR_BAD_MEM_ALIGNMENT &&
3029 e != RSMERR_BAD_LENGTH);
3030 if (e == RSMERR_NOT_MEM)
3031 e = RSMERR_INSUFFICIENT_MEM;
3032
3033 return (e);
3034 }
3035 /* export segment, this should create an IMMU mapping */
3036 e = adapter->rsmpi_ops->rsm_publish(
3037 seg->s_handle.out,
3038 rsmpi_acl, acl_len,
3039 seg->s_segid,
3040 RSM_RESOURCE_DONTWAIT, NULL);
3041
3042 if (e != RSM_SUCCESS) {
3043 adapter->rsmpi_ops->rsm_seg_destroy(seg->s_handle.out);
3044 rsmseglock_release(seg);
3045 rsmexport_rm(seg);
3046 rsmacl_free(acl, acl_len);
3047 rsmpiacl_free(rsmpi_acl, acl_len);
3048 DBG_PRINTF((category, RSM_ERR,
3049 "rsm_publish done: export_publish failed: %d\n",
3050 e));
3051 return (e);
3052 }
3053 }
3054
3055 seg->s_acl_in = rsmpi_acl;
3056
3057 skipdriver:
3058 /* defer s_acl/s_acl_len -> avoid crash in rsmseg_free */
3059 seg->s_acl_len = acl_len;
3060 seg->s_acl = acl;
3061
3062 if (seg->s_state == RSM_STATE_BIND) {
3063 seg->s_state = RSM_STATE_EXPORT;
3064 } else if (seg->s_state == RSM_STATE_BIND_QUIESCED) {
3065 seg->s_state = RSM_STATE_EXPORT_QUIESCED;
3066 cv_broadcast(&seg->s_cv);
3067 }
3068
3069 rsmseglock_release(seg);
3070
3071 /*
3072 * If the segment id was solicited, then return it in
3073 * the original incoming message.
3074 */
3075 if (msg->key == 0) {
3076 msg->key = segment_id;
3077 #ifdef _MULTI_DATAMODEL
3078 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
3079 rsm_ioctlmsg32_t msg32;
3080
3081 msg32.key = msg->key;
3082 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3083 "rsm_publish done\n"));
3084 return (ddi_copyout((caddr_t)&msg32,
3085 (caddr_t)dataptr, sizeof (msg32), mode));
3086 }
3087 #endif
3088 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3089 "rsm_publish done\n"));
3090 return (ddi_copyout((caddr_t)msg,
3091 (caddr_t)dataptr, sizeof (*msg), mode));
3092 }
3093
3094 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_publish done\n"));
3095 return (DDI_SUCCESS);
3096 }
3097
3098 /*
3099 * This function modifies the access control list of an already published
3100 * segment. There is no effect on import segments which are already
3101 * connected.
3102 */
3103 static int
3104 rsm_republish(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int mode)
3105 {
3106 rsmapi_access_entry_t *new_acl, *old_acl, *tmp_acl;
3107 rsm_access_entry_t *rsmpi_new_acl, *rsmpi_old_acl;
3108 int new_acl_len, old_acl_len, tmp_acl_len;
3109 int e, i;
3110 adapter_t *adapter;
3111 int loopback_flag = 0;
3112 rsm_memseg_id_t key;
3113 rsm_permission_t permission;
3114 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
3115
3116 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_republish enter\n"));
3117
3118 if ((seg->s_state != RSM_STATE_EXPORT) &&
3119 (seg->s_state != RSM_STATE_EXPORT_QUIESCED) &&
3120 (seg->s_state != RSM_STATE_EXPORT_QUIESCING))
3121 return (RSMERR_SEG_NOT_PUBLISHED);
3122
3123 if (seg->s_pid != ddi_get_pid() &&
3124 ddi_get_pid() != 0) {
3125 DBG_PRINTF((category, RSM_ERR,
3126 "rsm_republish: Not owner\n"));
3127 return (RSMERR_NOT_CREATOR);
3128 }
3129
3130 if (seg->s_adapter == &loopback_adapter)
3131 loopback_flag = 1;
3132
3133 /*
3134 * Build new list first
3135 */
3136 e = rsmacl_build(msg, mode, &new_acl, &new_acl_len, loopback_flag);
3137 if (e) {
3138 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3139 "rsm_republish done: rsmacl_build failed %d", e));
3140 return (e);
3141 }
3142
3143 /* Lock segment */
3144 rsmseglock_acquire(seg);
3145 /*
3146 * a republish is in progress - REPUBLISH message is being
3147 * sent to the importers so wait for it to complete OR
3148 * wait till DR completes
3149 */
3150 while (((seg->s_state == RSM_STATE_EXPORT) &&
3151 (seg->s_flags & RSM_REPUBLISH_WAIT)) ||
3152 (seg->s_state == RSM_STATE_EXPORT_QUIESCED) ||
3153 (seg->s_state == RSM_STATE_EXPORT_QUIESCING)) {
3154 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
3155 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3156 "rsm_republish done: cv_wait INTERRUPTED"));
3157 rsmseglock_release(seg);
3158 rsmacl_free(new_acl, new_acl_len);
3159 return (RSMERR_INTERRUPTED);
3160 }
3161 }
3162
3163 /* recheck if state is valid */
3164 if (seg->s_state != RSM_STATE_EXPORT) {
3165 rsmseglock_release(seg);
3166 rsmacl_free(new_acl, new_acl_len);
3167 return (RSMERR_SEG_NOT_PUBLISHED);
3168 }
3169
3170 key = seg->s_key;
3171 old_acl = seg->s_acl;
3172 old_acl_len = seg->s_acl_len;
3173
3174 seg->s_acl = new_acl;
3175 seg->s_acl_len = new_acl_len;
3176
3177 /*
3178 * This call will only be meaningful if and when the interconnect
3179 * layer makes use of the access list
3180 */
3181 adapter = seg->s_adapter;
3182 /*
3183 * create a acl list with hwaddr for RSMPI publish
3184 */
3185 e = rsmpiacl_create(new_acl, &rsmpi_new_acl, new_acl_len, adapter);
3186
3187 if (e != RSM_SUCCESS) {
3188 seg->s_acl = old_acl;
3189 seg->s_acl_len = old_acl_len;
3190 rsmseglock_release(seg);
3191 rsmacl_free(new_acl, new_acl_len);
3192 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3193 "rsm_republish done: rsmpiacl_create failed %d", e));
3194 return (e);
3195 }
3196 rsmpi_old_acl = seg->s_acl_in;
3197 seg->s_acl_in = rsmpi_new_acl;
3198
3199 e = adapter->rsmpi_ops->rsm_republish(seg->s_handle.out,
3200 seg->s_acl_in, seg->s_acl_len,
3201 RSM_RESOURCE_DONTWAIT, NULL);
3202
3203 if (e != RSM_SUCCESS) {
3204 seg->s_acl = old_acl;
3205 seg->s_acl_in = rsmpi_old_acl;
3206 seg->s_acl_len = old_acl_len;
3207 rsmseglock_release(seg);
3208 rsmacl_free(new_acl, new_acl_len);
3209 rsmpiacl_free(rsmpi_new_acl, new_acl_len);
3210
3211 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3212 "rsm_republish done: rsmpi republish failed %d\n", e));
3213 return (e);
3214 }
3215
3216 /* create a tmp copy of the new acl */
3217 tmp_acl_len = new_acl_len;
3218 if (tmp_acl_len > 0) {
3219 tmp_acl = kmem_zalloc(new_acl_len*sizeof (*tmp_acl), KM_SLEEP);
3220 for (i = 0; i < tmp_acl_len; i++) {
3221 tmp_acl[i].ae_node = new_acl[i].ae_node;
3222 tmp_acl[i].ae_permission = new_acl[i].ae_permission;
3223 }
3224 /*
3225 * The default permission of a node which was in the old
3226 * ACL but not in the new ACL is 0 ie no access.
3227 */
3228 permission = 0;
3229 } else {
3230 /*
3231 * NULL acl means all importers can connect and
3232 * default permission will be owner creation umask
3233 */
3234 tmp_acl = NULL;
3235 permission = seg->s_mode;
3236 }
3237
3238 /* make other republishers to wait for republish to complete */
3239 seg->s_flags |= RSM_REPUBLISH_WAIT;
3240
3241 rsmseglock_release(seg);
3242
3243 /* send the new perms to the importing nodes */
3244 rsm_send_republish(key, tmp_acl, tmp_acl_len, permission);
3245
3246 rsmseglock_acquire(seg);
3247 seg->s_flags &= ~RSM_REPUBLISH_WAIT;
3248 /* wake up any one waiting for republish to complete */
3249 cv_broadcast(&seg->s_cv);
3250 rsmseglock_release(seg);
3251
3252 rsmacl_free(tmp_acl, tmp_acl_len);
3253 rsmacl_free(old_acl, old_acl_len);
3254 rsmpiacl_free(rsmpi_old_acl, old_acl_len);
3255
3256 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_republish done\n"));
3257 return (DDI_SUCCESS);
3258 }
3259
3260 static int
3261 rsm_unpublish(rsmseg_t *seg, int mode)
3262 {
3263 rsmapi_access_entry_t *acl;
3264 rsm_access_entry_t *rsmpi_acl;
3265 int acl_len;
3266 int e;
3267 adapter_t *adapter;
3268 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
3269
3270 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unpublish enter\n"));
3271
3272 if (seg->s_pid != ddi_get_pid() &&
3273 ddi_get_pid() != 0) {
3274 DBG_PRINTF((category, RSM_ERR,
3275 "rsm_unpublish: Not creator\n"));
3276 return (RSMERR_NOT_CREATOR);
3277 }
3278
3279 rsmseglock_acquire(seg);
3280 /*
3281 * wait for QUIESCING to complete here before rsmexport_rm
3282 * is called because the SUSPEND_COMPLETE mesg which changes
3283 * the seg state from EXPORT_QUIESCING to EXPORT_QUIESCED and
3284 * signals the cv_wait needs to find it in the hashtable.
3285 */
3286 while ((seg->s_state == RSM_STATE_EXPORT_QUIESCING) ||
3287 ((seg->s_state == RSM_STATE_EXPORT) && (seg->s_rdmacnt > 0))) {
3288 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
3289 rsmseglock_release(seg);
3290 DBG_PRINTF((category, RSM_ERR,
3291 "rsm_unpublish done: cv_wait INTR qscing"
3292 "getv/putv in progress"));
3293 return (RSMERR_INTERRUPTED);
3294 }
3295 }
3296
3297 /* verify segment state */
3298 if ((seg->s_state != RSM_STATE_EXPORT) &&
3299 (seg->s_state != RSM_STATE_EXPORT_QUIESCED)) {
3300 rsmseglock_release(seg);
3301 DBG_PRINTF((category, RSM_ERR,
3302 "rsm_unpublish done: bad state %x\n", seg->s_state));
3303 return (RSMERR_SEG_NOT_PUBLISHED);
3304 }
3305
3306 rsmseglock_release(seg);
3307
3308 rsmexport_rm(seg);
3309
3310 rsm_send_importer_disconnects(seg->s_segid, my_nodeid);
3311
3312 rsmseglock_acquire(seg);
3313 /*
3314 * wait for republish to complete
3315 */
3316 while ((seg->s_state == RSM_STATE_EXPORT) &&
3317 (seg->s_flags & RSM_REPUBLISH_WAIT)) {
3318 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
3319 DBG_PRINTF((category, RSM_ERR,
3320 "rsm_unpublish done: cv_wait INTR repubing"));
3321 rsmseglock_release(seg);
3322 return (RSMERR_INTERRUPTED);
3323 }
3324 }
3325
3326 if ((seg->s_state != RSM_STATE_EXPORT) &&
3327 (seg->s_state != RSM_STATE_EXPORT_QUIESCED)) {
3328 DBG_PRINTF((category, RSM_ERR,
3329 "rsm_unpublish done: invalid state"));
3330 rsmseglock_release(seg);
3331 return (RSMERR_SEG_NOT_PUBLISHED);
3332 }
3333
3334 /*
3335 * check for putv/get surrogate segment which was not published
3336 * to the driver.
3337 *
3338 * Be certain to see if there is an ACL first! If this segment was
3339 * not published with an ACL, acl will be a null pointer. Check
3340 * that before dereferencing it.
3341 */
3342 acl = seg->s_acl;
3343 if (acl != (rsmapi_access_entry_t *)NULL) {
3344 if (acl[0].ae_node == my_nodeid && acl[0].ae_permission == 0)
3345 goto bypass;
3346 }
3347
3348 /* The RSMPI unpublish/destroy has been done if seg is QUIESCED */
3349 if (seg->s_state == RSM_STATE_EXPORT_QUIESCED)
3350 goto bypass;
3351
3352 adapter = seg->s_adapter;
3353 for (;;) {
3354 if (seg->s_state != RSM_STATE_EXPORT) {
3355 rsmseglock_release(seg);
3356 DBG_PRINTF((category, RSM_ERR,
3357 "rsm_unpublish done: bad state %x\n",
3358 seg->s_state));
3359 return (RSMERR_SEG_NOT_PUBLISHED);
3360 }
3361
3362 /* unpublish from adapter */
3363 e = adapter->rsmpi_ops->rsm_unpublish(seg->s_handle.out);
3364
3365 if (e == RSM_SUCCESS) {
3366 break;
3367 }
3368
3369 if (e == RSMERR_SEG_IN_USE && mode == 1) {
3370 /*
3371 * wait for unpublish to succeed, it's busy.
3372 */
3373 seg->s_flags |= RSM_EXPORT_WAIT;
3374
3375 /* wait for a max of 1 ms - this is an empirical */
3376 /* value that was found by some minimal testing */
3377 /* can be fine tuned when we have better numbers */
3378 /* A long term fix would be to send cv_signal */
3379 /* from the intr callback routine */
3380 /* currently nobody signals this wait */
3381 (void) cv_reltimedwait(&seg->s_cv, &seg->s_lock,
3382 drv_usectohz(1000), TR_CLOCK_TICK);
3383
3384 DBG_PRINTF((category, RSM_ERR,
3385 "rsm_unpublish: SEG_IN_USE\n"));
3386
3387 seg->s_flags &= ~RSM_EXPORT_WAIT;
3388 } else {
3389 if (mode == 1) {
3390 DBG_PRINTF((category, RSM_ERR,
3391 "rsm:rsmpi unpublish err %x\n", e));
3392 seg->s_state = RSM_STATE_BIND;
3393 }
3394 rsmseglock_release(seg);
3395 return (e);
3396 }
3397 }
3398
3399 /* Free segment */
3400 e = adapter->rsmpi_ops->rsm_seg_destroy(seg->s_handle.out);
3401
3402 if (e != RSM_SUCCESS) {
3403 DBG_PRINTF((category, RSM_ERR,
3404 "rsm_unpublish: rsmpi destroy key=%x failed %x\n",
3405 seg->s_key, e));
3406 }
3407
3408 bypass:
3409 acl = seg->s_acl;
3410 rsmpi_acl = seg->s_acl_in;
3411 acl_len = seg->s_acl_len;
3412
3413 seg->s_acl = NULL;
3414 seg->s_acl_in = NULL;
3415 seg->s_acl_len = 0;
3416
3417 if (seg->s_state == RSM_STATE_EXPORT) {
3418 seg->s_state = RSM_STATE_BIND;
3419 } else if (seg->s_state == RSM_STATE_EXPORT_QUIESCED) {
3420 seg->s_state = RSM_STATE_BIND_QUIESCED;
3421 cv_broadcast(&seg->s_cv);
3422 }
3423
3424 rsmseglock_release(seg);
3425
3426 rsmacl_free(acl, acl_len);
3427 rsmpiacl_free(rsmpi_acl, acl_len);
3428
3429 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unpublish done\n"));
3430
3431 return (DDI_SUCCESS);
3432 }
3433
3434 /*
3435 * Called from rsm_unpublish to force an unload and disconnection of all
3436 * importers of the unpublished segment.
3437 *
3438 * First build the list of segments requiring a force disconnect, then
3439 * send a request for each.
3440 */
3441 static void
3442 rsm_send_importer_disconnects(rsm_memseg_id_t ex_segid,
3443 rsm_node_id_t ex_nodeid)
3444 {
3445 rsmipc_request_t request;
3446 importing_token_t *prev_token, *token, *tmp_token, *tokp;
3447 importing_token_t *force_disconnect_list = NULL;
3448 int index;
3449
3450 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
3451 "rsm_send_importer_disconnects enter\n"));
3452
3453 index = rsmhash(ex_segid);
3454
3455 mutex_enter(&importer_list.lock);
3456
3457 prev_token = NULL;
3458 token = importer_list.bucket[index];
3459
3460 while (token != NULL) {
3461 if (token->key == ex_segid) {
3462 /*
3463 * take it off the importer list and add it
3464 * to the force disconnect list.
3465 */
3466 if (prev_token == NULL)
3467 importer_list.bucket[index] = token->next;
3468 else
3469 prev_token->next = token->next;
3470 tmp_token = token;
3471 token = token->next;
3472 if (force_disconnect_list == NULL) {
3473 force_disconnect_list = tmp_token;
3474 tmp_token->next = NULL;
3475 } else {
3476 tokp = force_disconnect_list;
3477 /*
3478 * make sure that the tmp_token's node
3479 * is not already on the force disconnect
3480 * list.
3481 */
3482 while (tokp != NULL) {
3483 if (tokp->importing_node ==
3484 tmp_token->importing_node) {
3485 break;
3486 }
3487 tokp = tokp->next;
3488 }
3489 if (tokp == NULL) {
3490 tmp_token->next =
3491 force_disconnect_list;
3492 force_disconnect_list = tmp_token;
3493 } else {
3494 kmem_free((void *)tmp_token,
3495 sizeof (*token));
3496 }
3497 }
3498
3499 } else {
3500 prev_token = token;
3501 token = token->next;
3502 }
3503 }
3504 mutex_exit(&importer_list.lock);
3505
3506 token = force_disconnect_list;
3507 while (token != NULL) {
3508 if (token->importing_node == my_nodeid) {
3509 rsm_force_unload(ex_nodeid, ex_segid,
3510 DISCONNECT);
3511 } else {
3512 request.rsmipc_hdr.rsmipc_type =
3513 RSMIPC_MSG_DISCONNECT;
3514 request.rsmipc_key = token->key;
3515 for (;;) {
3516 if (rsmipc_send(token->importing_node,
3517 &request,
3518 RSM_NO_REPLY) == RSM_SUCCESS) {
3519 break;
3520 } else {
3521 delay(drv_usectohz(10000));
3522 }
3523 }
3524 }
3525 tmp_token = token;
3526 token = token->next;
3527 kmem_free((void *)tmp_token, sizeof (*token));
3528 }
3529
3530 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
3531 "rsm_send_importer_disconnects done\n"));
3532 }
3533
3534 /*
3535 * This function is used as a callback for unlocking the pages locked
3536 * down by a process which then does a fork or an exec.
3537 * It marks the export segments corresponding to umem cookie given by
3538 * the *arg to be in a ZOMBIE state(by calling rsmseg_close to be
3539 * destroyed later when an rsm_close occurs).
3540 */
3541 static void
3542 rsm_export_force_destroy(ddi_umem_cookie_t *ck)
3543 {
3544 rsmresource_blk_t *blk;
3545 rsmresource_t *p;
3546 rsmseg_t *eseg = NULL;
3547 int i, j;
3548 int found = 0;
3549
3550 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
3551 "rsm_export_force_destroy enter\n"));
3552
3553 /*
3554 * Walk the resource list and locate the export segment (either
3555 * in the BIND or the EXPORT state) which corresponds to the
3556 * ddi_umem_cookie_t being freed up, and call rsmseg_close.
3557 * Change the state to ZOMBIE by calling rsmseg_close with the
3558 * force_flag argument (the second argument) set to 1. Also,
3559 * unpublish and unbind the segment, but don't free it. Free it
3560 * only on a rsm_close call for the segment.
3561 */
3562 rw_enter(&rsm_resource.rsmrc_lock, RW_READER);
3563
3564 for (i = 0; i < rsm_resource.rsmrc_len; i++) {
3565 blk = rsm_resource.rsmrc_root[i];
3566 if (blk == NULL) {
3567 continue;
3568 }
3569
3570 for (j = 0; j < RSMRC_BLKSZ; j++) {
3571 p = blk->rsmrcblk_blks[j];
3572 if ((p != NULL) && (p != RSMRC_RESERVED) &&
3573 (p->rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT)) {
3574 eseg = (rsmseg_t *)p;
3575 if (eseg->s_cookie != ck)
3576 continue; /* continue searching */
3577 /*
3578 * Found the segment, set flag to indicate
3579 * force destroy processing is in progress
3580 */
3581 rsmseglock_acquire(eseg);
3582 eseg->s_flags |= RSM_FORCE_DESTROY_WAIT;
3583 rsmseglock_release(eseg);
3584 found = 1;
3585 break;
3586 }
3587 }
3588
3589 if (found)
3590 break;
3591 }
3592
3593 rw_exit(&rsm_resource.rsmrc_lock);
3594
3595 if (found) {
3596 ASSERT(eseg != NULL);
3597 /* call rsmseg_close with force flag set to 1 */
3598 rsmseg_close(eseg, 1);
3599 /*
3600 * force destroy processing done, clear flag and signal any
3601 * thread waiting in rsmseg_close.
3602 */
3603 rsmseglock_acquire(eseg);
3604 eseg->s_flags &= ~RSM_FORCE_DESTROY_WAIT;
3605 cv_broadcast(&eseg->s_cv);
3606 rsmseglock_release(eseg);
3607 }
3608
3609 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
3610 "rsm_export_force_destroy done\n"));
3611 }
3612
3613 /* ******************************* Remote Calls *********************** */
3614 static void
3615 rsm_intr_segconnect(rsm_node_id_t src, rsmipc_request_t *req)
3616 {
3617 rsmipc_reply_t reply;
3618 DBG_DEFINE(category,
3619 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
3620
3621 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3622 "rsm_intr_segconnect enter\n"));
3623
3624 reply.rsmipc_status = (short)rsmsegacl_validate(req, src, &reply);
3625
3626 reply.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPLY;
3627 reply.rsmipc_hdr.rsmipc_cookie = req->rsmipc_hdr.rsmipc_cookie;
3628
3629 (void) rsmipc_send(src, NULL, &reply);
3630
3631 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3632 "rsm_intr_segconnect done\n"));
3633 }
3634
3635
3636 /*
3637 * When an exported segment is unpublished the exporter sends an ipc
3638 * message (RSMIPC_MSG_DISCONNECT) to all importers. The recv ipc dispatcher
3639 * calls this function. The import list is scanned; segments which match the
3640 * exported segment id are unloaded and disconnected.
3641 *
3642 * Will also be called from rsm_rebind with disconnect_flag FALSE.
3643 *
3644 */
3645 static void
3646 rsm_force_unload(rsm_node_id_t src_nodeid, rsm_memseg_id_t ex_segid,
3647 boolean_t disconnect_flag)
3648 {
3649 rsmresource_t *p = NULL;
3650 rsmhash_table_t *rhash = &rsm_import_segs;
3651 uint_t index;
3652 DBG_DEFINE(category,
3653 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
3654
3655 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_force_unload enter\n"));
3656
3657 index = rsmhash(ex_segid);
3658
3659 rw_enter(&rhash->rsmhash_rw, RW_READER);
3660
3661 p = rsmhash_getbkt(rhash, index);
3662
3663 for (; p; p = p->rsmrc_next) {
3664 rsmseg_t *seg = (rsmseg_t *)p;
3665 if ((seg->s_segid == ex_segid) && (seg->s_node == src_nodeid)) {
3666 /*
3667 * In order to make rsmseg_unload and rsm_force_unload
3668 * thread safe, acquire the segment lock here.
3669 * rsmseg_unload is responsible for releasing the lock.
3670 * rsmseg_unload releases the lock just before a call
3671 * to rsmipc_send or in case of an early exit which
3672 * occurs if the segment was in the state
3673 * RSM_STATE_CONNECTING or RSM_STATE_NEW.
3674 */
3675 rsmseglock_acquire(seg);
3676 if (disconnect_flag)
3677 seg->s_flags |= RSM_FORCE_DISCONNECT;
3678 rsmseg_unload(seg);
3679 }
3680 }
3681 rw_exit(&rhash->rsmhash_rw);
3682
3683 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_force_unload done\n"));
3684 }
3685
3686 static void
3687 rsm_intr_reply(rsmipc_msghdr_t *msg)
3688 {
3689 /*
3690 * Find slot for cookie in reply.
3691 * Match sequence with sequence in cookie
3692 * If no match; return
3693 * Try to grap lock of slot, if locked return
3694 * copy data into reply slot area
3695 * signal waiter
3696 */
3697 rsmipc_slot_t *slot;
3698 rsmipc_cookie_t *cookie;
3699 void *data = (void *) msg;
3700 size_t size = sizeof (rsmipc_reply_t);
3701 DBG_DEFINE(category,
3702 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
3703
3704 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_reply enter\n"));
3705
3706 cookie = &msg->rsmipc_cookie;
3707 if (cookie->ic.index >= RSMIPC_SZ) {
3708 DBG_PRINTF((category, RSM_ERR,
3709 "rsm: rsm_intr_reply bad cookie %d\n", cookie->ic.index));
3710 return;
3711 }
3712
3713 ASSERT(cookie->ic.index < RSMIPC_SZ);
3714 slot = &rsm_ipc.slots[cookie->ic.index];
3715 mutex_enter(&slot->rsmipc_lock);
3716 if (slot->rsmipc_cookie.value == cookie->value) {
3717 /* found a match */
3718 if (RSMIPC_GET(slot, RSMIPC_PENDING)) {
3719 bcopy(data, slot->rsmipc_data, size);
3720 RSMIPC_CLEAR(slot, RSMIPC_PENDING);
3721 cv_signal(&slot->rsmipc_cv);
3722 }
3723 } else {
3724 DBG_PRINTF((category, RSM_DEBUG,
3725 "rsm: rsm_intr_reply mismatched reply %d\n",
3726 cookie->ic.index));
3727 }
3728 mutex_exit(&slot->rsmipc_lock);
3729 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_reply done\n"));
3730 }
3731
3732 /*
3733 * This function gets dispatched on the worker thread when we receive
3734 * the SQREADY message. This function sends the SQREADY_ACK message.
3735 */
3736 static void
3737 rsm_sqready_ack_deferred(void *arg)
3738 {
3739 path_t *path = (path_t *)arg;
3740 DBG_DEFINE(category,
3741 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
3742
3743 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3744 "rsm_sqready_ack_deferred enter\n"));
3745
3746 mutex_enter(&path->mutex);
3747
3748 /*
3749 * If path is not active no point in sending the ACK
3750 * because the whole SQREADY protocol will again start
3751 * when the path becomes active.
3752 */
3753 if (path->state != RSMKA_PATH_ACTIVE) {
3754 /*
3755 * decrement the path refcnt incremented in rsm_proc_sqready
3756 */
3757 PATH_RELE_NOLOCK(path);
3758 mutex_exit(&path->mutex);
3759 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3760 "rsm_sqready_ack_deferred done:!ACTIVE\n"));
3761 return;
3762 }
3763
3764 /* send an SQREADY_ACK message */
3765 (void) rsmipc_send_controlmsg(path, RSMIPC_MSG_SQREADY_ACK);
3766
3767 /* initialize credits to the max level */
3768 path->sendq_token.msgbuf_avail = RSMIPC_MAX_MESSAGES;
3769
3770 /* wake up any send that is waiting for credits */
3771 cv_broadcast(&path->sendq_token.sendq_cv);
3772
3773 /*
3774 * decrement the path refcnt since we incremented it in
3775 * rsm_proc_sqready
3776 */
3777 PATH_RELE_NOLOCK(path);
3778
3779 mutex_exit(&path->mutex);
3780
3781 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3782 "rsm_sqready_ack_deferred done\n"));
3783 }
3784
3785 /*
3786 * Process the SQREADY message
3787 */
3788 static void
3789 rsm_proc_sqready(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr,
3790 rsm_intr_hand_arg_t arg)
3791 {
3792 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)msg;
3793 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg;
3794 path_t *path;
3795 DBG_DEFINE(category,
3796 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
3797
3798 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_proc_sqready enter\n"));
3799
3800 /* look up the path - incr the path refcnt */
3801 path = rsm_find_path(hdlr_argp->adapter_name,
3802 hdlr_argp->adapter_instance, src_hwaddr);
3803
3804 /*
3805 * No path exists or path is not active - drop the message
3806 */
3807 if (path == NULL) {
3808 DBG_PRINTF((category, RSM_DEBUG,
3809 "rsm_proc_sqready done: msg dropped no path\n"));
3810 return;
3811 }
3812
3813 mutex_exit(&path->mutex);
3814
3815 /* drain any tasks from the previous incarnation */
3816 taskq_wait(path->recv_taskq);
3817
3818 mutex_enter(&path->mutex);
3819 /*
3820 * If we'd sent an SQREADY message and were waiting for SQREADY_ACK
3821 * in the meanwhile we received an SQREADY message, blindly reset
3822 * the WAIT_FOR_SQACK flag because we'll just send SQREADY_ACK
3823 * and forget about the SQREADY that we sent.
3824 */
3825 path->flags &= ~RSMKA_WAIT_FOR_SQACK;
3826
3827 if (path->state != RSMKA_PATH_ACTIVE) {
3828 /* decr refcnt and drop the mutex */
3829 PATH_RELE_NOLOCK(path);
3830 mutex_exit(&path->mutex);
3831 DBG_PRINTF((category, RSM_DEBUG,
3832 "rsm_proc_sqready done: msg dropped path !ACTIVE\n"));
3833 return;
3834 }
3835
3836 DBG_PRINTF((category, RSM_DEBUG, "rsm_proc_sqready:path=%lx "
3837 " src=%lx:%llx\n", path, msghdr->rsmipc_src, src_hwaddr));
3838
3839 /*
3840 * The sender's local incarnation number is our remote incarnation
3841 * number save it in the path data structure
3842 */
3843 path->remote_incn = msg->rsmipc_local_incn;
3844 path->sendq_token.msgbuf_avail = 0;
3845 path->procmsg_cnt = 0;
3846
3847 /*
3848 * path is active - dispatch task to send SQREADY_ACK - remember
3849 * RSMPI calls can't be done in interrupt context
3850 *
3851 * We can use the recv_taskq to send because the remote endpoint
3852 * cannot start sending messages till it receives SQREADY_ACK hence
3853 * at this point there are no tasks on recv_taskq.
3854 *
3855 * The path refcnt will be decremented in rsm_sqready_ack_deferred.
3856 */
3857 (void) taskq_dispatch(path->recv_taskq,
3858 rsm_sqready_ack_deferred, path, KM_NOSLEEP);
3859
3860 mutex_exit(&path->mutex);
3861
3862
3863 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_proc_sqready done\n"));
3864 }
3865
3866 /*
3867 * Process the SQREADY_ACK message
3868 */
3869 static void
3870 rsm_proc_sqready_ack(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr,
3871 rsm_intr_hand_arg_t arg)
3872 {
3873 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)msg;
3874 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg;
3875 path_t *path;
3876 DBG_DEFINE(category,
3877 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
3878
3879 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3880 "rsm_proc_sqready_ack enter\n"));
3881
3882 /* look up the path - incr the path refcnt */
3883 path = rsm_find_path(hdlr_argp->adapter_name,
3884 hdlr_argp->adapter_instance, src_hwaddr);
3885
3886 /*
3887 * drop the message if - no path exists or path is not active
3888 * or if its not waiting for SQREADY_ACK message
3889 */
3890 if (path == NULL) {
3891 DBG_PRINTF((category, RSM_DEBUG,
3892 "rsm_proc_sqready_ack done: msg dropped no path\n"));
3893 return;
3894 }
3895
3896 if ((path->state != RSMKA_PATH_ACTIVE) ||
3897 !(path->flags & RSMKA_WAIT_FOR_SQACK)) {
3898 /* decrement the refcnt */
3899 PATH_RELE_NOLOCK(path);
3900 mutex_exit(&path->mutex);
3901 DBG_PRINTF((category, RSM_DEBUG,
3902 "rsm_proc_sqready_ack done: msg dropped\n"));
3903 return;
3904 }
3905
3906 /*
3907 * Check if this message is in response to the last RSMIPC_MSG_SQREADY
3908 * sent, if not drop it.
3909 */
3910 if (path->local_incn != msghdr->rsmipc_incn) {
3911 /* decrement the refcnt */
3912 PATH_RELE_NOLOCK(path);
3913 mutex_exit(&path->mutex);
3914 DBG_PRINTF((category, RSM_DEBUG,
3915 "rsm_proc_sqready_ack done: msg old incn %lld\n",
3916 msghdr->rsmipc_incn));
3917 return;
3918 }
3919
3920 DBG_PRINTF((category, RSM_DEBUG, "rsm_proc_sqready_ack:path=%lx "
3921 " src=%lx:%llx\n", path, msghdr->rsmipc_src, src_hwaddr));
3922
3923 /*
3924 * clear the WAIT_FOR_SQACK flag since we have recvd the ack
3925 */
3926 path->flags &= ~RSMKA_WAIT_FOR_SQACK;
3927
3928 /* save the remote sendq incn number */
3929 path->remote_incn = msg->rsmipc_local_incn;
3930
3931 /* initialize credits to the max level */
3932 path->sendq_token.msgbuf_avail = RSMIPC_MAX_MESSAGES;
3933
3934 /* wake up any send that is waiting for credits */
3935 cv_broadcast(&path->sendq_token.sendq_cv);
3936
3937 /* decrement the refcnt */
3938 PATH_RELE_NOLOCK(path);
3939
3940 mutex_exit(&path->mutex);
3941
3942 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
3943 "rsm_proc_sqready_ack done\n"));
3944 }
3945
3946 /*
3947 * process the RSMIPC_MSG_CREDIT message
3948 */
3949 static void
3950 rsm_add_credits(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr,
3951 rsm_intr_hand_arg_t arg)
3952 {
3953 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)msg;
3954 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg;
3955 path_t *path;
3956 DBG_DEFINE(category,
3957 RSM_KERNEL_AGENT | RSM_FUNC_ALL |
3958 RSM_INTR_CALLBACK | RSM_FLOWCONTROL);
3959
3960 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_add_credits enter\n"));
3961
3962 /* look up the path - incr the path refcnt */
3963 path = rsm_find_path(hdlr_argp->adapter_name,
3964 hdlr_argp->adapter_instance, src_hwaddr);
3965
3966 if (path == NULL) {
3967 DBG_PRINTF((category, RSM_DEBUG,
3968 "rsm_add_credits enter: path not found\n"));
3969 return;
3970 }
3971
3972 /* the path is not active - discard credits */
3973 if (path->state != RSMKA_PATH_ACTIVE) {
3974 PATH_RELE_NOLOCK(path);
3975 mutex_exit(&path->mutex);
3976 DBG_PRINTF((category, RSM_DEBUG,
3977 "rsm_add_credits enter:path=%lx !ACTIVE\n", path));
3978 return;
3979 }
3980
3981 /*
3982 * Check if these credits are for current incarnation of the path.
3983 */
3984 if (path->local_incn != msghdr->rsmipc_incn) {
3985 /* decrement the refcnt */
3986 PATH_RELE_NOLOCK(path);
3987 mutex_exit(&path->mutex);
3988 DBG_PRINTF((category, RSM_DEBUG,
3989 "rsm_add_credits enter: old incn %lld\n",
3990 msghdr->rsmipc_incn));
3991 return;
3992 }
3993
3994 DBG_PRINTF((category, RSM_DEBUG,
3995 "rsm_add_credits:path=%lx new-creds=%d "
3996 "curr credits=%d src=%lx:%llx\n", path, msg->rsmipc_credits,
3997 path->sendq_token.msgbuf_avail, msghdr->rsmipc_src,
3998 src_hwaddr));
3999
4000
4001 /* add credits to the path's sendq */
4002 path->sendq_token.msgbuf_avail += msg->rsmipc_credits;
4003
4004 ASSERT(path->sendq_token.msgbuf_avail <= RSMIPC_MAX_MESSAGES);
4005
4006 /* wake up any send that is waiting for credits */
4007 cv_broadcast(&path->sendq_token.sendq_cv);
4008
4009 /* decrement the refcnt */
4010 PATH_RELE_NOLOCK(path);
4011
4012 mutex_exit(&path->mutex);
4013
4014 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_add_credits done\n"));
4015 }
4016
4017 static void
4018 rsm_intr_event(rsmipc_request_t *msg)
4019 {
4020 rsmseg_t *seg;
4021 rsmresource_t *p;
4022 rsm_node_id_t src_node;
4023 DBG_DEFINE(category,
4024 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4025
4026 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_event enter\n"));
4027
4028 src_node = msg->rsmipc_hdr.rsmipc_src;
4029
4030 if ((seg = msg->rsmipc_segment_cookie) != NULL) {
4031 /* This is for an import segment */
4032 uint_t hashval = rsmhash(msg->rsmipc_key);
4033
4034 rw_enter(&rsm_import_segs.rsmhash_rw, RW_READER);
4035
4036 p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hashval);
4037
4038 for (; p; p = p->rsmrc_next) {
4039 if ((p->rsmrc_key == msg->rsmipc_key) &&
4040 (p->rsmrc_node == src_node)) {
4041 seg = (rsmseg_t *)p;
4042 rsmseglock_acquire(seg);
4043
4044 atomic_inc_32(&seg->s_pollevent);
4045
4046 if (seg->s_pollflag & RSM_SEGMENT_POLL)
4047 pollwakeup(&seg->s_poll, POLLRDNORM);
4048
4049 rsmseglock_release(seg);
4050 }
4051 }
4052
4053 rw_exit(&rsm_import_segs.rsmhash_rw);
4054 } else {
4055 /* This is for an export segment */
4056 seg = rsmexport_lookup(msg->rsmipc_key);
4057 if (!seg) {
4058 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4059 "rsm_intr_event done: exp seg not found\n"));
4060 return;
4061 }
4062
4063 ASSERT(rsmseglock_held(seg));
4064
4065 atomic_inc_32(&seg->s_pollevent);
4066
4067 /*
4068 * We must hold the segment lock here, or else the segment
4069 * can be freed while pollwakeup is using it. This implies
4070 * that we MUST NOT grab the segment lock during rsm_chpoll,
4071 * as outlined in the chpoll(2) man page.
4072 */
4073 if (seg->s_pollflag & RSM_SEGMENT_POLL)
4074 pollwakeup(&seg->s_poll, POLLRDNORM);
4075
4076 rsmseglock_release(seg);
4077 }
4078
4079 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_event done\n"));
4080 }
4081
4082 /*
4083 * The exporter did a republish and changed the ACL - this change is only
4084 * visible to new importers.
4085 */
4086 static void
4087 importer_update(rsm_node_id_t src_node, rsm_memseg_id_t key,
4088 rsm_permission_t perm)
4089 {
4090
4091 rsmresource_t *p;
4092 rsmseg_t *seg;
4093 uint_t hashval = rsmhash(key);
4094 DBG_DEFINE(category,
4095 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4096
4097 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_update enter\n"));
4098
4099 rw_enter(&rsm_import_segs.rsmhash_rw, RW_READER);
4100
4101 p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hashval);
4102
4103 for (; p; p = p->rsmrc_next) {
4104 /*
4105 * find the importer and update the permission in the shared
4106 * data structure. Any new importers will use the new perms
4107 */
4108 if ((p->rsmrc_key == key) && (p->rsmrc_node == src_node)) {
4109 seg = (rsmseg_t *)p;
4110
4111 rsmseglock_acquire(seg);
4112 rsmsharelock_acquire(seg);
4113 seg->s_share->rsmsi_mode = perm;
4114 rsmsharelock_release(seg);
4115 rsmseglock_release(seg);
4116
4117 break;
4118 }
4119 }
4120
4121 rw_exit(&rsm_import_segs.rsmhash_rw);
4122
4123 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_update done\n"));
4124 }
4125
4126 void
4127 rsm_suspend_complete(rsm_node_id_t src_node, int flag)
4128 {
4129 int done = 1; /* indicate all SUSPENDS have been acked */
4130 list_element_t *elem;
4131 DBG_DEFINE(category,
4132 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4133
4134 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4135 "rsm_suspend_complete enter\n"));
4136
4137 mutex_enter(&rsm_suspend_list.list_lock);
4138
4139 if (rsm_suspend_list.list_head == NULL) {
4140 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4141 "rsm_suspend_complete done: suspend_list is empty\n"));
4142 mutex_exit(&rsm_suspend_list.list_lock);
4143 return;
4144 }
4145
4146 elem = rsm_suspend_list.list_head;
4147 while (elem != NULL) {
4148 if (elem->nodeid == src_node) {
4149 /* clear the pending flag for the node */
4150 elem->flags &= ~RSM_SUSPEND_ACKPENDING;
4151 elem->flags |= flag;
4152 }
4153
4154 if (done && (elem->flags & RSM_SUSPEND_ACKPENDING))
4155 done = 0; /* still some nodes have not yet ACKED */
4156
4157 elem = elem->next;
4158 }
4159
4160 mutex_exit(&rsm_suspend_list.list_lock);
4161
4162 if (!done) {
4163 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4164 "rsm_suspend_complete done: acks pending\n"));
4165 return;
4166 }
4167 /*
4168 * Now that we are done with suspending all the remote importers
4169 * time to quiesce the local exporters
4170 */
4171 exporter_quiesce();
4172
4173 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4174 "rsm_suspend_complete done\n"));
4175 }
4176
4177 static void
4178 exporter_quiesce()
4179 {
4180 int i, e;
4181 rsmresource_t *current;
4182 rsmseg_t *seg;
4183 adapter_t *adapter;
4184 DBG_DEFINE(category,
4185 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4186
4187 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exporter_quiesce enter\n"));
4188 /*
4189 * The importers send a SUSPEND_COMPLETE to the exporter node
4190 * Unpublish, unbind the export segment and
4191 * move the segments to the EXPORT_QUIESCED state
4192 */
4193
4194 rw_enter(&rsm_export_segs.rsmhash_rw, RW_READER);
4195
4196 for (i = 0; i < rsm_hash_size; i++) {
4197 current = rsm_export_segs.bucket[i];
4198 while (current != NULL) {
4199 seg = (rsmseg_t *)current;
4200 rsmseglock_acquire(seg);
4201 if (current->rsmrc_state ==
4202 RSM_STATE_EXPORT_QUIESCING) {
4203 adapter = seg->s_adapter;
4204 /*
4205 * some local memory handles are not published
4206 * check if it was published
4207 */
4208 if ((seg->s_acl == NULL) ||
4209 (seg->s_acl[0].ae_node != my_nodeid) ||
4210 (seg->s_acl[0].ae_permission != 0)) {
4211
4212 e = adapter->rsmpi_ops->rsm_unpublish(
4213 seg->s_handle.out);
4214 DBG_PRINTF((category, RSM_DEBUG,
4215 "exporter_quiesce:unpub %d\n", e));
4216
4217 e = adapter->rsmpi_ops->rsm_seg_destroy(
4218 seg->s_handle.out);
4219
4220 DBG_PRINTF((category, RSM_DEBUG,
4221 "exporter_quiesce:destroy %d\n",
4222 e));
4223 }
4224
4225 (void) rsm_unbind_pages(seg);
4226 seg->s_state = RSM_STATE_EXPORT_QUIESCED;
4227 cv_broadcast(&seg->s_cv);
4228 }
4229 rsmseglock_release(seg);
4230 current = current->rsmrc_next;
4231 }
4232 }
4233 rw_exit(&rsm_export_segs.rsmhash_rw);
4234
4235 /*
4236 * All the local segments we are done with the pre-del processing
4237 * - time to move to PREDEL_COMPLETED.
4238 */
4239
4240 mutex_enter(&rsm_drv_data.drv_lock);
4241
4242 ASSERT(rsm_drv_data.drv_state == RSM_DRV_PREDEL_STARTED);
4243
4244 rsm_drv_data.drv_state = RSM_DRV_PREDEL_COMPLETED;
4245
4246 cv_broadcast(&rsm_drv_data.drv_cv);
4247
4248 mutex_exit(&rsm_drv_data.drv_lock);
4249
4250 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exporter_quiesce done\n"));
4251 }
4252
4253 static void
4254 importer_suspend(rsm_node_id_t src_node)
4255 {
4256 int i;
4257 int susp_flg; /* true means already suspended */
4258 int num_importers;
4259 rsmresource_t *p = NULL, *curp;
4260 rsmhash_table_t *rhash = &rsm_import_segs;
4261 rsmseg_t *seg;
4262 rsmipc_request_t request;
4263 DBG_DEFINE(category,
4264 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4265
4266 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_suspend enter\n"));
4267
4268 rw_enter(&rhash->rsmhash_rw, RW_READER);
4269 for (i = 0; i < rsm_hash_size; i++) {
4270 p = rhash->bucket[i];
4271
4272 /*
4273 * Suspend all importers with same <node, key> pair.
4274 * After the last one of the shared importers has been
4275 * suspended - suspend the shared mappings/connection.
4276 */
4277 for (; p; p = p->rsmrc_next) {
4278 rsmseg_t *first = (rsmseg_t *)p;
4279 if ((first->s_node != src_node) ||
4280 (first->s_state == RSM_STATE_DISCONNECT))
4281 continue; /* go to next entry */
4282 /*
4283 * search the rest of the bucket for
4284 * other siblings (imprtrs with the same key)
4285 * of "first" and suspend them.
4286 * All importers with same key fall in
4287 * the same bucket.
4288 */
4289 num_importers = 0;
4290 for (curp = p; curp; curp = curp->rsmrc_next) {
4291 seg = (rsmseg_t *)curp;
4292
4293 rsmseglock_acquire(seg);
4294
4295 if ((seg->s_node != first->s_node) ||
4296 (seg->s_key != first->s_key) ||
4297 (seg->s_state == RSM_STATE_DISCONNECT)) {
4298 /*
4299 * either not a peer segment or its a
4300 * disconnected segment - skip it
4301 */
4302 rsmseglock_release(seg);
4303 continue;
4304 }
4305
4306 rsmseg_suspend(seg, &susp_flg);
4307
4308 if (susp_flg) { /* seg already suspended */
4309 rsmseglock_release(seg);
4310 break; /* the inner for loop */
4311 }
4312
4313 num_importers++;
4314 rsmsharelock_acquire(seg);
4315 /*
4316 * we've processed all importers that are
4317 * siblings of "first"
4318 */
4319 if (num_importers ==
4320 seg->s_share->rsmsi_refcnt) {
4321 rsmsharelock_release(seg);
4322 rsmseglock_release(seg);
4323 break;
4324 }
4325 rsmsharelock_release(seg);
4326 rsmseglock_release(seg);
4327 }
4328
4329 /*
4330 * All the importers with the same key and
4331 * nodeid as "first" have been suspended.
4332 * Now suspend the shared connect/mapping.
4333 * This is done only once.
4334 */
4335 if (!susp_flg) {
4336 rsmsegshare_suspend(seg);
4337 }
4338 }
4339 }
4340
4341 rw_exit(&rhash->rsmhash_rw);
4342
4343 /* send an ACK for SUSPEND message */
4344 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SUSPEND_DONE;
4345 (void) rsmipc_send(src_node, &request, RSM_NO_REPLY);
4346
4347
4348 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_suspend done\n"));
4349
4350 }
4351
4352 static void
4353 rsmseg_suspend(rsmseg_t *seg, int *susp_flg)
4354 {
4355 int recheck_state;
4356 rsmcookie_t *hdl;
4357 DBG_DEFINE(category,
4358 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4359
4360 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4361 "rsmseg_suspend enter: key=%u\n", seg->s_key));
4362
4363 *susp_flg = 0;
4364
4365 ASSERT(rsmseglock_held(seg));
4366 /* wait if putv/getv is in progress */
4367 while (seg->s_rdmacnt > 0)
4368 cv_wait(&seg->s_cv, &seg->s_lock);
4369
4370 do {
4371 recheck_state = 0;
4372
4373 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4374 "rsmseg_suspend:segment %x state=%d\n",
4375 seg->s_key, seg->s_state));
4376
4377 switch (seg->s_state) {
4378 case RSM_STATE_NEW:
4379 /* not a valid state */
4380 break;
4381 case RSM_STATE_CONNECTING:
4382 seg->s_state = RSM_STATE_ABORT_CONNECT;
4383 break;
4384 case RSM_STATE_ABORT_CONNECT:
4385 break;
4386 case RSM_STATE_CONNECT:
4387 seg->s_handle.in = NULL;
4388 seg->s_state = RSM_STATE_CONN_QUIESCE;
4389 break;
4390 case RSM_STATE_MAPPING:
4391 /* wait until segment leaves the mapping state */
4392 while (seg->s_state == RSM_STATE_MAPPING)
4393 cv_wait(&seg->s_cv, &seg->s_lock);
4394 recheck_state = 1;
4395 break;
4396 case RSM_STATE_ACTIVE:
4397 /* unload the mappings */
4398 if (seg->s_ckl != NULL) {
4399 hdl = seg->s_ckl;
4400 for (; hdl != NULL; hdl = hdl->c_next) {
4401 (void) devmap_unload(hdl->c_dhp,
4402 hdl->c_off, hdl->c_len);
4403 }
4404 }
4405 seg->s_mapinfo = NULL;
4406 seg->s_state = RSM_STATE_MAP_QUIESCE;
4407 break;
4408 case RSM_STATE_CONN_QUIESCE:
4409 /* FALLTHRU */
4410 case RSM_STATE_MAP_QUIESCE:
4411 /* rsmseg_suspend already done for seg */
4412 *susp_flg = 1;
4413 break;
4414 case RSM_STATE_DISCONNECT:
4415 break;
4416 default:
4417 ASSERT(0); /* invalid state */
4418 }
4419 } while (recheck_state);
4420
4421 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_suspend done\n"));
4422 }
4423
4424 static void
4425 rsmsegshare_suspend(rsmseg_t *seg)
4426 {
4427 int e;
4428 adapter_t *adapter;
4429 rsm_import_share_t *sharedp;
4430 DBG_DEFINE(category,
4431 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4432
4433 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4434 "rsmsegshare_suspend enter\n"));
4435
4436 rsmseglock_acquire(seg);
4437 rsmsharelock_acquire(seg);
4438
4439 sharedp = seg->s_share;
4440 adapter = seg->s_adapter;
4441 switch (sharedp->rsmsi_state) {
4442 case RSMSI_STATE_NEW:
4443 break;
4444 case RSMSI_STATE_CONNECTING:
4445 sharedp->rsmsi_state = RSMSI_STATE_ABORT_CONNECT;
4446 break;
4447 case RSMSI_STATE_ABORT_CONNECT:
4448 break;
4449 case RSMSI_STATE_CONNECTED:
4450 /* do the rsmpi disconnect */
4451 if (sharedp->rsmsi_node != my_nodeid) {
4452 e = adapter->rsmpi_ops->
4453 rsm_disconnect(sharedp->rsmsi_handle);
4454
4455 DBG_PRINTF((category, RSM_DEBUG,
4456 "rsm:rsmpi disconnect seg=%x:err=%d\n",
4457 sharedp->rsmsi_segid, e));
4458 }
4459
4460 sharedp->rsmsi_handle = NULL;
4461
4462 sharedp->rsmsi_state = RSMSI_STATE_CONN_QUIESCE;
4463 break;
4464 case RSMSI_STATE_CONN_QUIESCE:
4465 break;
4466 case RSMSI_STATE_MAPPED:
4467 /* do the rsmpi unmap and disconnect */
4468 if (sharedp->rsmsi_node != my_nodeid) {
4469 e = adapter->rsmpi_ops->rsm_unmap(seg->s_handle.in);
4470
4471 DBG_PRINTF((category, RSM_DEBUG,
4472 "rsmshare_suspend: rsmpi unmap %d\n", e));
4473
4474 e = adapter->rsmpi_ops->
4475 rsm_disconnect(sharedp->rsmsi_handle);
4476 DBG_PRINTF((category, RSM_DEBUG,
4477 "rsm:rsmpi disconnect seg=%x:err=%d\n",
4478 sharedp->rsmsi_segid, e));
4479 }
4480
4481 sharedp->rsmsi_handle = NULL;
4482
4483 sharedp->rsmsi_state = RSMSI_STATE_MAP_QUIESCE;
4484 break;
4485 case RSMSI_STATE_MAP_QUIESCE:
4486 break;
4487 case RSMSI_STATE_DISCONNECTED:
4488 break;
4489 default:
4490 ASSERT(0); /* invalid state */
4491 }
4492
4493 rsmsharelock_release(seg);
4494 rsmseglock_release(seg);
4495
4496 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4497 "rsmsegshare_suspend done\n"));
4498 }
4499
4500 /*
4501 * This should get called on receiving a RESUME message or from
4502 * the pathmanger if the node undergoing DR dies.
4503 */
4504 static void
4505 importer_resume(rsm_node_id_t src_node)
4506 {
4507 int i;
4508 rsmresource_t *p = NULL;
4509 rsmhash_table_t *rhash = &rsm_import_segs;
4510 void *cookie;
4511 DBG_DEFINE(category,
4512 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4513
4514 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_resume enter\n"));
4515
4516 rw_enter(&rhash->rsmhash_rw, RW_READER);
4517
4518 for (i = 0; i < rsm_hash_size; i++) {
4519 p = rhash->bucket[i];
4520
4521 for (; p; p = p->rsmrc_next) {
4522 rsmseg_t *seg = (rsmseg_t *)p;
4523
4524 rsmseglock_acquire(seg);
4525
4526 /* process only importers of node undergoing DR */
4527 if (seg->s_node != src_node) {
4528 rsmseglock_release(seg);
4529 continue;
4530 }
4531
4532 if (rsmseg_resume(seg, &cookie) != RSM_SUCCESS) {
4533 rsmipc_request_t request;
4534 /*
4535 * rsmpi map/connect failed
4536 * inform the exporter so that it can
4537 * remove the importer.
4538 */
4539 request.rsmipc_hdr.rsmipc_type =
4540 RSMIPC_MSG_NOTIMPORTING;
4541 request.rsmipc_key = seg->s_segid;
4542 request.rsmipc_segment_cookie = cookie;
4543 rsmseglock_release(seg);
4544 (void) rsmipc_send(seg->s_node, &request,
4545 RSM_NO_REPLY);
4546 } else {
4547 rsmseglock_release(seg);
4548 }
4549 }
4550 }
4551
4552 rw_exit(&rhash->rsmhash_rw);
4553
4554 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_resume done\n"));
4555 }
4556
4557 static int
4558 rsmseg_resume(rsmseg_t *seg, void **cookie)
4559 {
4560 int e;
4561 int retc;
4562 off_t dev_offset;
4563 size_t maplen;
4564 uint_t maxprot;
4565 rsm_mapinfo_t *p;
4566 rsmcookie_t *hdl;
4567 rsm_import_share_t *sharedp;
4568 DBG_DEFINE(category,
4569 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4570
4571 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4572 "rsmseg_resume enter: key=%u\n", seg->s_key));
4573
4574 *cookie = NULL;
4575
4576 ASSERT(rsmseglock_held(seg));
4577
4578 if ((seg->s_state != RSM_STATE_CONN_QUIESCE) &&
4579 (seg->s_state != RSM_STATE_MAP_QUIESCE)) {
4580 return (RSM_SUCCESS);
4581 }
4582
4583 sharedp = seg->s_share;
4584
4585 rsmsharelock_acquire(seg);
4586
4587 /* resume the shared connection and/or mapping */
4588 retc = rsmsegshare_resume(seg);
4589
4590 if (seg->s_state == RSM_STATE_CONN_QUIESCE) {
4591 /* shared state can either be connected or mapped */
4592 if ((sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) ||
4593 (sharedp->rsmsi_state == RSMSI_STATE_MAPPED)) {
4594 ASSERT(retc == RSM_SUCCESS);
4595 seg->s_handle.in = sharedp->rsmsi_handle;
4596 rsmsharelock_release(seg);
4597 seg->s_state = RSM_STATE_CONNECT;
4598
4599 } else { /* error in rsmpi connect during resume */
4600 seg->s_handle.in = NULL;
4601 seg->s_state = RSM_STATE_DISCONNECT;
4602
4603 sharedp->rsmsi_refcnt--;
4604 cookie = (void *)sharedp->rsmsi_cookie;
4605
4606 if (sharedp->rsmsi_refcnt == 0) {
4607 ASSERT(sharedp->rsmsi_mapcnt == 0);
4608 rsmsharelock_release(seg);
4609
4610 /* clean up the shared data structure */
4611 mutex_destroy(&sharedp->rsmsi_lock);
4612 cv_destroy(&sharedp->rsmsi_cv);
4613 kmem_free((void *)(sharedp),
4614 sizeof (rsm_import_share_t));
4615
4616 } else {
4617 rsmsharelock_release(seg);
4618 }
4619 /*
4620 * The following needs to be done after any
4621 * rsmsharelock calls which use seg->s_share.
4622 */
4623 seg->s_share = NULL;
4624 }
4625
4626 /* signal any waiting segment */
4627 cv_broadcast(&seg->s_cv);
4628
4629 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4630 "rsmseg_resume done:state=%d\n", seg->s_state));
4631 return (retc);
4632 }
4633
4634 ASSERT(seg->s_state == RSM_STATE_MAP_QUIESCE);
4635
4636 /* Setup protections for remap */
4637 maxprot = PROT_USER;
4638 if (seg->s_mode & RSM_PERM_READ) {
4639 maxprot |= PROT_READ;
4640 }
4641 if (seg->s_mode & RSM_PERM_WRITE) {
4642 maxprot |= PROT_WRITE;
4643 }
4644
4645 if (sharedp->rsmsi_state != RSMSI_STATE_MAPPED) {
4646 /* error in rsmpi connect or map during resume */
4647
4648 /* remap to trash page */
4649 ASSERT(seg->s_ckl != NULL);
4650
4651 for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) {
4652 e = devmap_umem_remap(hdl->c_dhp, rsm_dip,
4653 remap_cookie, hdl->c_off, hdl->c_len,
4654 maxprot, 0, NULL);
4655
4656 DBG_PRINTF((category, RSM_ERR,
4657 "rsmseg_resume:remap=%d\n", e));
4658 }
4659
4660 seg->s_handle.in = NULL;
4661 seg->s_state = RSM_STATE_DISCONNECT;
4662
4663 sharedp->rsmsi_refcnt--;
4664
4665 sharedp->rsmsi_mapcnt--;
4666 seg->s_mapinfo = NULL;
4667
4668 if (sharedp->rsmsi_refcnt == 0) {
4669 ASSERT(sharedp->rsmsi_mapcnt == 0);
4670 rsmsharelock_release(seg);
4671
4672 /* clean up the shared data structure */
4673 mutex_destroy(&sharedp->rsmsi_lock);
4674 cv_destroy(&sharedp->rsmsi_cv);
4675 kmem_free((void *)(sharedp),
4676 sizeof (rsm_import_share_t));
4677
4678 } else {
4679 rsmsharelock_release(seg);
4680 }
4681 /*
4682 * The following needs to be done after any
4683 * rsmsharelock calls which use seg->s_share.
4684 */
4685 seg->s_share = NULL;
4686
4687 /* signal any waiting segment */
4688 cv_broadcast(&seg->s_cv);
4689
4690 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4691 "rsmseg_resume done:seg=%x,err=%d\n",
4692 seg->s_key, retc));
4693 return (retc);
4694
4695 }
4696
4697 seg->s_handle.in = sharedp->rsmsi_handle;
4698
4699 if (seg->s_node == my_nodeid) { /* loopback */
4700 ASSERT(seg->s_mapinfo == NULL);
4701
4702 for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) {
4703 e = devmap_umem_remap(hdl->c_dhp,
4704 rsm_dip, seg->s_cookie,
4705 hdl->c_off, hdl->c_len,
4706 maxprot, 0, NULL);
4707
4708 DBG_PRINTF((category, RSM_ERR,
4709 "rsmseg_resume:remap=%d\n", e));
4710 }
4711 } else { /* remote exporter */
4712 /* remap to the new rsmpi maps */
4713 seg->s_mapinfo = sharedp->rsmsi_mapinfo;
4714
4715 for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) {
4716 p = rsm_get_mapinfo(seg, hdl->c_off, hdl->c_len,
4717 &dev_offset, &maplen);
4718 e = devmap_devmem_remap(hdl->c_dhp,
4719 p->dip, p->dev_register, dev_offset,
4720 maplen, maxprot, 0, NULL);
4721
4722 DBG_PRINTF((category, RSM_ERR,
4723 "rsmseg_resume:remap=%d\n", e));
4724 }
4725 }
4726
4727 rsmsharelock_release(seg);
4728
4729 seg->s_state = RSM_STATE_ACTIVE;
4730 cv_broadcast(&seg->s_cv);
4731
4732 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_resume done\n"));
4733
4734 return (retc);
4735 }
4736
4737 static int
4738 rsmsegshare_resume(rsmseg_t *seg)
4739 {
4740 int e = RSM_SUCCESS;
4741 adapter_t *adapter;
4742 rsm_import_share_t *sharedp;
4743 DBG_DEFINE(category,
4744 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4745
4746 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegshare_resume enter\n"));
4747
4748 ASSERT(rsmseglock_held(seg));
4749 ASSERT(rsmsharelock_held(seg));
4750
4751 sharedp = seg->s_share;
4752
4753 /*
4754 * If we are not in a xxxx_QUIESCE state that means shared
4755 * connect/mapping processing has been already been done
4756 * so return success.
4757 */
4758 if ((sharedp->rsmsi_state != RSMSI_STATE_CONN_QUIESCE) &&
4759 (sharedp->rsmsi_state != RSMSI_STATE_MAP_QUIESCE)) {
4760 return (RSM_SUCCESS);
4761 }
4762
4763 adapter = seg->s_adapter;
4764
4765 if (sharedp->rsmsi_node != my_nodeid) {
4766 rsm_addr_t hwaddr;
4767 hwaddr = get_remote_hwaddr(adapter, sharedp->rsmsi_node);
4768
4769 e = adapter->rsmpi_ops->rsm_connect(
4770 adapter->rsmpi_handle, hwaddr,
4771 sharedp->rsmsi_segid, &sharedp->rsmsi_handle);
4772
4773 DBG_PRINTF((category, RSM_DEBUG,
4774 "rsmsegshare_resume:rsmpi connect seg=%x:err=%d\n",
4775 sharedp->rsmsi_segid, e));
4776
4777 if (e != RSM_SUCCESS) {
4778 /* when do we send the NOT_IMPORTING message */
4779 sharedp->rsmsi_handle = NULL;
4780 sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED;
4781 /* signal any waiting segment */
4782 cv_broadcast(&sharedp->rsmsi_cv);
4783 return (e);
4784 }
4785 }
4786
4787 if (sharedp->rsmsi_state == RSMSI_STATE_CONN_QUIESCE) {
4788 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
4789 /* signal any waiting segment */
4790 cv_broadcast(&sharedp->rsmsi_cv);
4791 return (e);
4792 }
4793
4794 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAP_QUIESCE);
4795
4796 /* do the rsmpi map of the whole segment here */
4797 if (sharedp->rsmsi_node != my_nodeid) {
4798 size_t mapped_len;
4799 rsm_mapinfo_t *p;
4800
4801 /*
4802 * We need to do rsmpi maps with <off, lens> identical to
4803 * the old mapinfo list because the segment mapping handles
4804 * dhp and such need the fragmentation of rsmpi maps to be
4805 * identical to what it was during the mmap of the segment
4806 */
4807 p = sharedp->rsmsi_mapinfo;
4808
4809 while (p != NULL) {
4810 mapped_len = 0;
4811
4812 e = adapter->rsmpi_ops->rsm_map(
4813 sharedp->rsmsi_handle, p->start_offset,
4814 p->individual_len, &mapped_len,
4815 &p->dip, &p->dev_register, &p->dev_offset,
4816 NULL, NULL);
4817
4818 if (e != 0) {
4819 DBG_PRINTF((category, RSM_ERR,
4820 "rsmsegshare_resume: rsmpi map err=%d\n",
4821 e));
4822 break;
4823 }
4824
4825 if (mapped_len != p->individual_len) {
4826 DBG_PRINTF((category, RSM_ERR,
4827 "rsmsegshare_resume: rsmpi maplen"
4828 "< reqlen=%lx\n", mapped_len));
4829 e = RSMERR_BAD_LENGTH;
4830 break;
4831 }
4832
4833 p = p->next;
4834
4835 }
4836
4837
4838 if (e != RSM_SUCCESS) { /* rsmpi map failed */
4839 int err;
4840 /* Check if this is the first rsm_map */
4841 if (p != sharedp->rsmsi_mapinfo) {
4842 /*
4843 * A single rsm_unmap undoes multiple rsm_maps.
4844 */
4845 (void) seg->s_adapter->rsmpi_ops->
4846 rsm_unmap(sharedp->rsmsi_handle);
4847 }
4848
4849 rsm_free_mapinfo(sharedp->rsmsi_mapinfo);
4850 sharedp->rsmsi_mapinfo = NULL;
4851
4852 err = adapter->rsmpi_ops->
4853 rsm_disconnect(sharedp->rsmsi_handle);
4854
4855 DBG_PRINTF((category, RSM_DEBUG,
4856 "rsmsegshare_resume:disconn seg=%x:err=%d\n",
4857 sharedp->rsmsi_segid, err));
4858
4859 sharedp->rsmsi_handle = NULL;
4860 sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED;
4861
4862 /* signal the waiting segments */
4863 cv_broadcast(&sharedp->rsmsi_cv);
4864 DBG_PRINTF((category, RSM_DEBUG,
4865 "rsmsegshare_resume done: rsmpi map err\n"));
4866 return (e);
4867 }
4868 }
4869
4870 sharedp->rsmsi_state = RSMSI_STATE_MAPPED;
4871
4872 /* signal any waiting segment */
4873 cv_broadcast(&sharedp->rsmsi_cv);
4874
4875 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegshare_resume done\n"));
4876
4877 return (e);
4878 }
4879
4880 /*
4881 * this is the routine that gets called by recv_taskq which is the
4882 * thread that processes messages that are flow-controlled.
4883 */
4884 static void
4885 rsm_intr_proc_deferred(void *arg)
4886 {
4887 path_t *path = (path_t *)arg;
4888 rsmipc_request_t *msg;
4889 rsmipc_msghdr_t *msghdr;
4890 rsm_node_id_t src_node;
4891 msgbuf_elem_t *head;
4892 int e;
4893 DBG_DEFINE(category,
4894 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4895
4896 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4897 "rsm_intr_proc_deferred enter\n"));
4898
4899 mutex_enter(&path->mutex);
4900
4901 /* use the head of the msgbuf_queue */
4902 head = rsmka_gethead_msgbuf(path);
4903
4904 mutex_exit(&path->mutex);
4905
4906 msg = (rsmipc_request_t *)&(head->msg);
4907 msghdr = (rsmipc_msghdr_t *)msg;
4908
4909 src_node = msghdr->rsmipc_src;
4910
4911 /*
4912 * messages that need to send a reply should check the message version
4913 * before processing the message. And all messages that need to
4914 * send a reply should be processed here by the worker thread.
4915 */
4916 switch (msghdr->rsmipc_type) {
4917 case RSMIPC_MSG_SEGCONNECT:
4918 if (msghdr->rsmipc_version != RSM_VERSION) {
4919 rsmipc_reply_t reply;
4920 reply.rsmipc_status = RSMERR_BAD_DRIVER_VERSION;
4921 reply.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPLY;
4922 reply.rsmipc_hdr.rsmipc_cookie = msghdr->rsmipc_cookie;
4923 (void) rsmipc_send(msghdr->rsmipc_src, NULL, &reply);
4924 } else {
4925 rsm_intr_segconnect(src_node, msg);
4926 }
4927 break;
4928 case RSMIPC_MSG_DISCONNECT:
4929 rsm_force_unload(src_node, msg->rsmipc_key, DISCONNECT);
4930 break;
4931 case RSMIPC_MSG_SUSPEND:
4932 importer_suspend(src_node);
4933 break;
4934 case RSMIPC_MSG_SUSPEND_DONE:
4935 rsm_suspend_complete(src_node, 0);
4936 break;
4937 case RSMIPC_MSG_RESUME:
4938 importer_resume(src_node);
4939 break;
4940 default:
4941 ASSERT(0);
4942 }
4943
4944 mutex_enter(&path->mutex);
4945
4946 rsmka_dequeue_msgbuf(path);
4947
4948 /* incr procmsg_cnt can be at most RSMIPC_MAX_MESSAGES */
4949 if (path->procmsg_cnt < RSMIPC_MAX_MESSAGES)
4950 path->procmsg_cnt++;
4951
4952 ASSERT(path->procmsg_cnt <= RSMIPC_MAX_MESSAGES);
4953
4954 /* No need to send credits if path is going down */
4955 if ((path->state == RSMKA_PATH_ACTIVE) &&
4956 (path->procmsg_cnt >= RSMIPC_LOTSFREE_MSGBUFS)) {
4957 /*
4958 * send credits and reset procmsg_cnt if success otherwise
4959 * credits will be sent after processing the next message
4960 */
4961 e = rsmipc_send_controlmsg(path, RSMIPC_MSG_CREDIT);
4962 if (e == 0)
4963 path->procmsg_cnt = 0;
4964 else
4965 DBG_PRINTF((category, RSM_ERR,
4966 "rsm_intr_proc_deferred:send credits err=%d\n", e));
4967 }
4968
4969 /*
4970 * decrement the path refcnt since we incremented it in
4971 * rsm_intr_callback_dispatch
4972 */
4973 PATH_RELE_NOLOCK(path);
4974
4975 mutex_exit(&path->mutex);
4976
4977 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4978 "rsm_intr_proc_deferred done\n"));
4979 }
4980
4981 /*
4982 * Flow-controlled messages are enqueued and dispatched onto a taskq here
4983 */
4984 static void
4985 rsm_intr_callback_dispatch(void *data, rsm_addr_t src_hwaddr,
4986 rsm_intr_hand_arg_t arg)
4987 {
4988 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg;
4989 path_t *path;
4990 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)data;
4991 DBG_DEFINE(category,
4992 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
4993
4994 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
4995 "rsm_intr_callback_dispatch enter\n"));
4996 ASSERT(data && hdlr_argp);
4997
4998 /* look up the path - incr the path refcnt */
4999 path = rsm_find_path(hdlr_argp->adapter_name,
5000 hdlr_argp->adapter_instance, src_hwaddr);
5001
5002 /* the path has been removed - drop this message */
5003 if (path == NULL) {
5004 DBG_PRINTF((category, RSM_DEBUG,
5005 "rsm_intr_callback_dispatch done: msg dropped\n"));
5006 return;
5007 }
5008 /* the path is not active - don't accept new messages */
5009 if (path->state != RSMKA_PATH_ACTIVE) {
5010 PATH_RELE_NOLOCK(path);
5011 mutex_exit(&path->mutex);
5012 DBG_PRINTF((category, RSM_DEBUG,
5013 "rsm_intr_callback_dispatch done: msg dropped"
5014 " path=%lx !ACTIVE\n", path));
5015 return;
5016 }
5017
5018 /*
5019 * Check if this message was sent to an older incarnation
5020 * of the path/sendq.
5021 */
5022 if (path->local_incn != msghdr->rsmipc_incn) {
5023 /* decrement the refcnt */
5024 PATH_RELE_NOLOCK(path);
5025 mutex_exit(&path->mutex);
5026 DBG_PRINTF((category, RSM_DEBUG,
5027 "rsm_intr_callback_dispatch done: old incn %lld\n",
5028 msghdr->rsmipc_incn));
5029 return;
5030 }
5031
5032 /* copy and enqueue msg on the path's msgbuf queue */
5033 rsmka_enqueue_msgbuf(path, data);
5034
5035 /*
5036 * schedule task to process messages - ignore retval from
5037 * task_dispatch because we sender cannot send more than
5038 * what receiver can handle.
5039 */
5040 (void) taskq_dispatch(path->recv_taskq,
5041 rsm_intr_proc_deferred, path, KM_NOSLEEP);
5042
5043 mutex_exit(&path->mutex);
5044
5045 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5046 "rsm_intr_callback_dispatch done\n"));
5047 }
5048
5049 /*
5050 * This procedure is called from rsm_srv_func when a remote node creates a
5051 * a send queue. This event is used as a hint that an earlier failed
5052 * attempt to create a send queue to that remote node may now succeed and
5053 * should be retried. Indication of an earlier failed attempt is provided
5054 * by the RSMKA_SQCREATE_PENDING flag.
5055 */
5056 static void
5057 rsm_sqcreateop_callback(rsm_addr_t src_hwaddr, rsm_intr_hand_arg_t arg)
5058 {
5059 srv_handler_arg_t *hdlr_argp = (srv_handler_arg_t *)arg;
5060 path_t *path;
5061 DBG_DEFINE(category,
5062 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
5063
5064 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5065 "rsm_sqcreateop_callback enter\n"));
5066
5067 /* look up the path - incr the path refcnt */
5068 path = rsm_find_path(hdlr_argp->adapter_name,
5069 hdlr_argp->adapter_instance, src_hwaddr);
5070
5071 if (path == NULL) {
5072 DBG_PRINTF((category, RSM_DEBUG,
5073 "rsm_sqcreateop_callback done: no path\n"));
5074 return;
5075 }
5076
5077 if ((path->state == RSMKA_PATH_UP) &&
5078 (path->flags & RSMKA_SQCREATE_PENDING)) {
5079 /*
5080 * previous attempt to create sendq had failed, retry
5081 * it and move to RSMKA_PATH_ACTIVE state if successful.
5082 * the refcnt will be decremented in the do_deferred_work
5083 */
5084 (void) rsmka_do_path_active(path, RSMKA_NO_SLEEP);
5085 } else {
5086 /* decrement the refcnt */
5087 PATH_RELE_NOLOCK(path);
5088 }
5089 mutex_exit(&path->mutex);
5090
5091 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5092 "rsm_sqcreateop_callback done\n"));
5093 }
5094
5095 static void
5096 rsm_intr_callback(void *data, rsm_addr_t src_hwaddr, rsm_intr_hand_arg_t arg)
5097 {
5098 rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)data;
5099 rsmipc_request_t *msg = (rsmipc_request_t *)data;
5100 rsmipc_controlmsg_t *ctrlmsg = (rsmipc_controlmsg_t *)data;
5101 rsm_node_id_t src_node;
5102 DBG_DEFINE(category,
5103 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
5104
5105 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_callback enter:"
5106 "src=%d, type=%d\n", msghdr->rsmipc_src,
5107 msghdr->rsmipc_type));
5108
5109 /*
5110 * Check for the version number in the msg header. If it is not
5111 * RSM_VERSION, drop the message. In the future, we need to manage
5112 * incompatible version numbers in some way
5113 */
5114 if (msghdr->rsmipc_version != RSM_VERSION) {
5115 DBG_PRINTF((category, RSM_ERR, "wrong KA version\n"));
5116 /*
5117 * Drop requests that don't have a reply right here
5118 * Request with reply will send a BAD_VERSION reply
5119 * when they get processed by the worker thread.
5120 */
5121 if (msghdr->rsmipc_type != RSMIPC_MSG_SEGCONNECT) {
5122 return;
5123 }
5124
5125 }
5126
5127 src_node = msghdr->rsmipc_src;
5128
5129 switch (msghdr->rsmipc_type) {
5130 case RSMIPC_MSG_SEGCONNECT:
5131 case RSMIPC_MSG_DISCONNECT:
5132 case RSMIPC_MSG_SUSPEND:
5133 case RSMIPC_MSG_SUSPEND_DONE:
5134 case RSMIPC_MSG_RESUME:
5135 /*
5136 * These message types are handled by a worker thread using
5137 * the flow-control algorithm.
5138 * Any message processing that does one or more of the
5139 * following should be handled in a worker thread.
5140 * - allocates resources and might sleep
5141 * - makes RSMPI calls down to the interconnect driver
5142 * this by defn include requests with reply.
5143 * - takes a long duration of time
5144 */
5145 rsm_intr_callback_dispatch(data, src_hwaddr, arg);
5146 break;
5147 case RSMIPC_MSG_NOTIMPORTING:
5148 importer_list_rm(src_node, msg->rsmipc_key,
5149 msg->rsmipc_segment_cookie);
5150 break;
5151 case RSMIPC_MSG_SQREADY:
5152 rsm_proc_sqready(data, src_hwaddr, arg);
5153 break;
5154 case RSMIPC_MSG_SQREADY_ACK:
5155 rsm_proc_sqready_ack(data, src_hwaddr, arg);
5156 break;
5157 case RSMIPC_MSG_CREDIT:
5158 rsm_add_credits(ctrlmsg, src_hwaddr, arg);
5159 break;
5160 case RSMIPC_MSG_REPLY:
5161 rsm_intr_reply(msghdr);
5162 break;
5163 case RSMIPC_MSG_BELL:
5164 rsm_intr_event(msg);
5165 break;
5166 case RSMIPC_MSG_IMPORTING:
5167 importer_list_add(src_node, msg->rsmipc_key,
5168 msg->rsmipc_adapter_hwaddr,
5169 msg->rsmipc_segment_cookie);
5170 break;
5171 case RSMIPC_MSG_REPUBLISH:
5172 importer_update(src_node, msg->rsmipc_key, msg->rsmipc_perm);
5173 break;
5174 default:
5175 DBG_PRINTF((category, RSM_DEBUG,
5176 "rsm_intr_callback: bad msg %lx type %d data %lx\n",
5177 (size_t)msg, (int)(msghdr->rsmipc_type), (size_t)data));
5178 }
5179
5180 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_callback done\n"));
5181
5182 }
5183
5184 rsm_intr_hand_ret_t rsm_srv_func(rsm_controller_object_t *chd,
5185 rsm_intr_q_op_t opcode, rsm_addr_t src,
5186 void *data, size_t size, rsm_intr_hand_arg_t arg)
5187 {
5188 DBG_DEFINE(category,
5189 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
5190
5191 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_srv_func enter\n"));
5192
5193 switch (opcode) {
5194 case RSM_INTR_Q_OP_CREATE:
5195 DBG_PRINTF((category, RSM_DEBUG, "rsm_srv_func:OP_CREATE\n"));
5196 rsm_sqcreateop_callback(src, arg);
5197 break;
5198 case RSM_INTR_Q_OP_DESTROY:
5199 DBG_PRINTF((category, RSM_DEBUG, "rsm_srv_func:OP_DESTROY\n"));
5200 break;
5201 case RSM_INTR_Q_OP_RECEIVE:
5202 rsm_intr_callback(data, src, arg);
5203 break;
5204 default:
5205 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5206 "rsm_srv_func: unknown opcode = %x\n", opcode));
5207 }
5208
5209 chd = chd;
5210 size = size;
5211
5212 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_srv_func done\n"));
5213
5214 return (RSM_INTR_HAND_CLAIMED);
5215 }
5216
5217 /* *************************** IPC slots ************************* */
5218 static rsmipc_slot_t *
5219 rsmipc_alloc()
5220 {
5221 int i;
5222 rsmipc_slot_t *slot;
5223 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
5224
5225 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_alloc enter\n"));
5226
5227 /* try to find a free slot, if not wait */
5228 mutex_enter(&rsm_ipc.lock);
5229
5230 while (rsm_ipc.count == 0) {
5231 rsm_ipc.wanted = 1;
5232 cv_wait(&rsm_ipc.cv, &rsm_ipc.lock);
5233 }
5234
5235 /* An empty slot is available, find it */
5236 slot = &rsm_ipc.slots[0];
5237 for (i = 0; i < RSMIPC_SZ; i++, slot++) {
5238 if (RSMIPC_GET(slot, RSMIPC_FREE)) {
5239 RSMIPC_CLEAR(slot, RSMIPC_FREE);
5240 break;
5241 }
5242 }
5243
5244 ASSERT(i < RSMIPC_SZ);
5245 rsm_ipc.count--; /* one less is available */
5246 rsm_ipc.sequence++; /* new sequence */
5247
5248 slot->rsmipc_cookie.ic.sequence = (uint_t)rsm_ipc.sequence;
5249 slot->rsmipc_cookie.ic.index = (uint_t)i;
5250
5251 mutex_exit(&rsm_ipc.lock);
5252
5253 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_alloc done\n"));
5254
5255 return (slot);
5256 }
5257
5258 static void
5259 rsmipc_free(rsmipc_slot_t *slot)
5260 {
5261 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
5262
5263 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_free enter\n"));
5264
5265 ASSERT(MUTEX_HELD(&slot->rsmipc_lock));
5266 ASSERT(&rsm_ipc.slots[slot->rsmipc_cookie.ic.index] == slot);
5267
5268 mutex_enter(&rsm_ipc.lock);
5269
5270 RSMIPC_SET(slot, RSMIPC_FREE);
5271
5272 slot->rsmipc_cookie.ic.sequence = 0;
5273
5274 mutex_exit(&slot->rsmipc_lock);
5275 rsm_ipc.count++;
5276 ASSERT(rsm_ipc.count <= RSMIPC_SZ);
5277 if (rsm_ipc.wanted) {
5278 rsm_ipc.wanted = 0;
5279 cv_broadcast(&rsm_ipc.cv);
5280 }
5281
5282 mutex_exit(&rsm_ipc.lock);
5283
5284 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_free done\n"));
5285 }
5286
5287 static int
5288 rsmipc_send(rsm_node_id_t dest, rsmipc_request_t *req, rsmipc_reply_t *reply)
5289 {
5290 int e = 0;
5291 int credit_check = 0;
5292 int retry_cnt = 0;
5293 int min_retry_cnt = 10;
5294 rsm_send_t is;
5295 rsmipc_slot_t *rslot;
5296 adapter_t *adapter;
5297 path_t *path;
5298 sendq_token_t *sendq_token;
5299 sendq_token_t *used_sendq_token = NULL;
5300 rsm_send_q_handle_t ipc_handle;
5301 DBG_DEFINE(category,
5302 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
5303
5304 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_send enter:dest=%d",
5305 dest));
5306
5307 /*
5308 * Check if this is a local case
5309 */
5310 if (dest == my_nodeid) {
5311 switch (req->rsmipc_hdr.rsmipc_type) {
5312 case RSMIPC_MSG_SEGCONNECT:
5313 reply->rsmipc_status = (short)rsmsegacl_validate(
5314 req, dest, reply);
5315 break;
5316 case RSMIPC_MSG_BELL:
5317 req->rsmipc_hdr.rsmipc_src = dest;
5318 rsm_intr_event(req);
5319 break;
5320 case RSMIPC_MSG_IMPORTING:
5321 importer_list_add(dest, req->rsmipc_key,
5322 req->rsmipc_adapter_hwaddr,
5323 req->rsmipc_segment_cookie);
5324 break;
5325 case RSMIPC_MSG_NOTIMPORTING:
5326 importer_list_rm(dest, req->rsmipc_key,
5327 req->rsmipc_segment_cookie);
5328 break;
5329 case RSMIPC_MSG_REPUBLISH:
5330 importer_update(dest, req->rsmipc_key,
5331 req->rsmipc_perm);
5332 break;
5333 case RSMIPC_MSG_SUSPEND:
5334 importer_suspend(dest);
5335 break;
5336 case RSMIPC_MSG_SUSPEND_DONE:
5337 rsm_suspend_complete(dest, 0);
5338 break;
5339 case RSMIPC_MSG_RESUME:
5340 importer_resume(dest);
5341 break;
5342 default:
5343 ASSERT(0);
5344 }
5345 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5346 "rsmipc_send done\n"));
5347 return (0);
5348 }
5349
5350 if (dest >= MAX_NODES) {
5351 DBG_PRINTF((category, RSM_ERR,
5352 "rsm: rsmipc_send bad node number %x\n", dest));
5353 return (RSMERR_REMOTE_NODE_UNREACHABLE);
5354 }
5355
5356 /*
5357 * Oh boy! we are going remote.
5358 */
5359
5360 /*
5361 * identify if we need to have credits to send this message
5362 * - only selected requests are flow controlled
5363 */
5364 if (req != NULL) {
5365 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5366 "rsmipc_send:request type=%d\n",
5367 req->rsmipc_hdr.rsmipc_type));
5368
5369 switch (req->rsmipc_hdr.rsmipc_type) {
5370 case RSMIPC_MSG_SEGCONNECT:
5371 case RSMIPC_MSG_DISCONNECT:
5372 case RSMIPC_MSG_IMPORTING:
5373 case RSMIPC_MSG_SUSPEND:
5374 case RSMIPC_MSG_SUSPEND_DONE:
5375 case RSMIPC_MSG_RESUME:
5376 credit_check = 1;
5377 break;
5378 default:
5379 credit_check = 0;
5380 }
5381 }
5382
5383 again:
5384 if (retry_cnt++ == min_retry_cnt) {
5385 /* backoff before further retries for 10ms */
5386 delay(drv_usectohz(10000));
5387 retry_cnt = 0; /* reset retry_cnt */
5388 }
5389 sendq_token = rsmka_get_sendq_token(dest, used_sendq_token);
5390 if (sendq_token == NULL) {
5391 DBG_PRINTF((category, RSM_ERR,
5392 "rsm: rsmipc_send no device to reach node %d\n", dest));
5393 return (RSMERR_REMOTE_NODE_UNREACHABLE);
5394 }
5395
5396 if ((sendq_token == used_sendq_token) &&
5397 ((e == RSMERR_CONN_ABORTED) || (e == RSMERR_TIMEOUT) ||
5398 (e == RSMERR_COMM_ERR_MAYBE_DELIVERED))) {
5399 rele_sendq_token(sendq_token);
5400 DBG_PRINTF((category, RSM_DEBUG, "rsmipc_send done=%d\n", e));
5401 return (RSMERR_CONN_ABORTED);
5402 } else
5403 used_sendq_token = sendq_token;
5404
5405 /* lint -save -e413 */
5406 path = SQ_TOKEN_TO_PATH(sendq_token);
5407 adapter = path->local_adapter;
5408 /* lint -restore */
5409 ipc_handle = sendq_token->rsmpi_sendq_handle;
5410
5411 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5412 "rsmipc_send: path=%lx sendq_hdl=%lx\n", path, ipc_handle));
5413
5414 if (reply == NULL) {
5415 /* Send request without ack */
5416 /*
5417 * Set the rsmipc_version number in the msghdr for KA
5418 * communication versioning
5419 */
5420 req->rsmipc_hdr.rsmipc_version = RSM_VERSION;
5421 req->rsmipc_hdr.rsmipc_src = my_nodeid;
5422 /*
5423 * remote endpoints incn should match the value in our
5424 * path's remote_incn field. No need to grab any lock
5425 * since we have refcnted the path in rsmka_get_sendq_token
5426 */
5427 req->rsmipc_hdr.rsmipc_incn = path->remote_incn;
5428
5429 is.is_data = (void *)req;
5430 is.is_size = sizeof (*req);
5431 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
5432 is.is_wait = 0;
5433
5434 if (credit_check) {
5435 mutex_enter(&path->mutex);
5436 /*
5437 * wait till we recv credits or path goes down. If path
5438 * goes down rsm_send will fail and we handle the error
5439 * then
5440 */
5441 while ((sendq_token->msgbuf_avail == 0) &&
5442 (path->state == RSMKA_PATH_ACTIVE)) {
5443 e = cv_wait_sig(&sendq_token->sendq_cv,
5444 &path->mutex);
5445 if (e == 0) {
5446 mutex_exit(&path->mutex);
5447 no_reply_cnt++;
5448 rele_sendq_token(sendq_token);
5449 DBG_PRINTF((category, RSM_DEBUG,
5450 "rsmipc_send done: "
5451 "cv_wait INTERRUPTED"));
5452 return (RSMERR_INTERRUPTED);
5453 }
5454 }
5455
5456 /*
5457 * path is not active retry on another path.
5458 */
5459 if (path->state != RSMKA_PATH_ACTIVE) {
5460 mutex_exit(&path->mutex);
5461 rele_sendq_token(sendq_token);
5462 e = RSMERR_CONN_ABORTED;
5463 DBG_PRINTF((category, RSM_ERR,
5464 "rsm: rsmipc_send: path !ACTIVE"));
5465 goto again;
5466 }
5467
5468 ASSERT(sendq_token->msgbuf_avail > 0);
5469
5470 /*
5471 * reserve a msgbuf
5472 */
5473 sendq_token->msgbuf_avail--;
5474
5475 mutex_exit(&path->mutex);
5476
5477 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
5478 NULL);
5479
5480 if (e != RSM_SUCCESS) {
5481 mutex_enter(&path->mutex);
5482 /*
5483 * release the reserved msgbuf since
5484 * the send failed
5485 */
5486 sendq_token->msgbuf_avail++;
5487 cv_broadcast(&sendq_token->sendq_cv);
5488 mutex_exit(&path->mutex);
5489 }
5490 } else
5491 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
5492 NULL);
5493
5494 no_reply_cnt++;
5495 rele_sendq_token(sendq_token);
5496 if (e != RSM_SUCCESS) {
5497 DBG_PRINTF((category, RSM_ERR,
5498 "rsm: rsmipc_send no reply send"
5499 " err = %d no reply count = %d\n",
5500 e, no_reply_cnt));
5501 ASSERT(e != RSMERR_QUEUE_FENCE_UP &&
5502 e != RSMERR_BAD_BARRIER_HNDL);
5503 atomic_inc_64(&rsm_ipcsend_errcnt);
5504 goto again;
5505 } else {
5506 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5507 "rsmipc_send done\n"));
5508 return (e);
5509 }
5510
5511 }
5512
5513 if (req == NULL) {
5514 /* Send reply - No flow control is done for reply */
5515 /*
5516 * Set the version in the msg header for KA communication
5517 * versioning
5518 */
5519 reply->rsmipc_hdr.rsmipc_version = RSM_VERSION;
5520 reply->rsmipc_hdr.rsmipc_src = my_nodeid;
5521 /* incn number is not used for reply msgs currently */
5522 reply->rsmipc_hdr.rsmipc_incn = path->remote_incn;
5523
5524 is.is_data = (void *)reply;
5525 is.is_size = sizeof (*reply);
5526 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
5527 is.is_wait = 0;
5528 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, NULL);
5529 rele_sendq_token(sendq_token);
5530 if (e != RSM_SUCCESS) {
5531 DBG_PRINTF((category, RSM_ERR,
5532 "rsm: rsmipc_send reply send"
5533 " err = %d\n", e));
5534 atomic_inc_64(&rsm_ipcsend_errcnt);
5535 goto again;
5536 } else {
5537 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5538 "rsmipc_send done\n"));
5539 return (e);
5540 }
5541 }
5542
5543 /* Reply needed */
5544 rslot = rsmipc_alloc(); /* allocate a new ipc slot */
5545
5546 mutex_enter(&rslot->rsmipc_lock);
5547
5548 rslot->rsmipc_data = (void *)reply;
5549 RSMIPC_SET(rslot, RSMIPC_PENDING);
5550
5551 while (RSMIPC_GET(rslot, RSMIPC_PENDING)) {
5552 /*
5553 * Set the rsmipc_version number in the msghdr for KA
5554 * communication versioning
5555 */
5556 req->rsmipc_hdr.rsmipc_version = RSM_VERSION;
5557 req->rsmipc_hdr.rsmipc_src = my_nodeid;
5558 req->rsmipc_hdr.rsmipc_cookie = rslot->rsmipc_cookie;
5559 /*
5560 * remote endpoints incn should match the value in our
5561 * path's remote_incn field. No need to grab any lock
5562 * since we have refcnted the path in rsmka_get_sendq_token
5563 */
5564 req->rsmipc_hdr.rsmipc_incn = path->remote_incn;
5565
5566 is.is_data = (void *)req;
5567 is.is_size = sizeof (*req);
5568 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
5569 is.is_wait = 0;
5570 if (credit_check) {
5571
5572 mutex_enter(&path->mutex);
5573 /*
5574 * wait till we recv credits or path goes down. If path
5575 * goes down rsm_send will fail and we handle the error
5576 * then.
5577 */
5578 while ((sendq_token->msgbuf_avail == 0) &&
5579 (path->state == RSMKA_PATH_ACTIVE)) {
5580 e = cv_wait_sig(&sendq_token->sendq_cv,
5581 &path->mutex);
5582 if (e == 0) {
5583 mutex_exit(&path->mutex);
5584 RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
5585 rsmipc_free(rslot);
5586 rele_sendq_token(sendq_token);
5587 DBG_PRINTF((category, RSM_DEBUG,
5588 "rsmipc_send done: "
5589 "cv_wait INTERRUPTED"));
5590 return (RSMERR_INTERRUPTED);
5591 }
5592 }
5593
5594 /*
5595 * path is not active retry on another path.
5596 */
5597 if (path->state != RSMKA_PATH_ACTIVE) {
5598 mutex_exit(&path->mutex);
5599 RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
5600 rsmipc_free(rslot);
5601 rele_sendq_token(sendq_token);
5602 e = RSMERR_CONN_ABORTED;
5603 DBG_PRINTF((category, RSM_ERR,
5604 "rsm: rsmipc_send: path !ACTIVE"));
5605 goto again;
5606 }
5607
5608 ASSERT(sendq_token->msgbuf_avail > 0);
5609
5610 /*
5611 * reserve a msgbuf
5612 */
5613 sendq_token->msgbuf_avail--;
5614
5615 mutex_exit(&path->mutex);
5616
5617 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
5618 NULL);
5619
5620 if (e != RSM_SUCCESS) {
5621 mutex_enter(&path->mutex);
5622 /*
5623 * release the reserved msgbuf since
5624 * the send failed
5625 */
5626 sendq_token->msgbuf_avail++;
5627 cv_broadcast(&sendq_token->sendq_cv);
5628 mutex_exit(&path->mutex);
5629 }
5630 } else
5631 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
5632 NULL);
5633
5634 if (e != RSM_SUCCESS) {
5635 DBG_PRINTF((category, RSM_ERR,
5636 "rsm: rsmipc_send rsmpi send err = %d\n", e));
5637 RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
5638 rsmipc_free(rslot);
5639 rele_sendq_token(sendq_token);
5640 atomic_inc_64(&rsm_ipcsend_errcnt);
5641 goto again;
5642 }
5643
5644 /* wait for a reply signal, a SIGINT, or 5 sec. timeout */
5645 e = cv_reltimedwait_sig(&rslot->rsmipc_cv, &rslot->rsmipc_lock,
5646 drv_usectohz(5000000), TR_CLOCK_TICK);
5647 if (e < 0) {
5648 /* timed out - retry */
5649 e = RSMERR_TIMEOUT;
5650 } else if (e == 0) {
5651 /* signalled - return error */
5652 e = RSMERR_INTERRUPTED;
5653 break;
5654 } else {
5655 e = RSM_SUCCESS;
5656 }
5657 }
5658
5659 RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
5660 rsmipc_free(rslot);
5661 rele_sendq_token(sendq_token);
5662
5663 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_send done=%d\n", e));
5664 return (e);
5665 }
5666
5667 static int
5668 rsm_send_notimporting(rsm_node_id_t dest, rsm_memseg_id_t segid, void *cookie)
5669 {
5670 rsmipc_request_t request;
5671
5672 /*
5673 * inform the exporter to delete this importer
5674 */
5675 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_NOTIMPORTING;
5676 request.rsmipc_key = segid;
5677 request.rsmipc_segment_cookie = cookie;
5678 return (rsmipc_send(dest, &request, RSM_NO_REPLY));
5679 }
5680
5681 static void
5682 rsm_send_republish(rsm_memseg_id_t segid, rsmapi_access_entry_t *acl,
5683 int acl_len, rsm_permission_t default_permission)
5684 {
5685 int i;
5686 importing_token_t *token;
5687 rsmipc_request_t request;
5688 republish_token_t *republish_list = NULL;
5689 republish_token_t *rp;
5690 rsm_permission_t permission;
5691 int index;
5692
5693 /*
5694 * send the new access mode to all the nodes that have imported
5695 * this segment.
5696 * If the new acl does not have a node that was present in
5697 * the old acl a access permission of 0 is sent.
5698 */
5699
5700 index = rsmhash(segid);
5701
5702 /*
5703 * create a list of node/permissions to send the republish message
5704 */
5705 mutex_enter(&importer_list.lock);
5706
5707 token = importer_list.bucket[index];
5708 while (token != NULL) {
5709 if (segid == token->key) {
5710 permission = default_permission;
5711
5712 for (i = 0; i < acl_len; i++) {
5713 if (token->importing_node == acl[i].ae_node) {
5714 permission = acl[i].ae_permission;
5715 break;
5716 }
5717 }
5718 rp = kmem_zalloc(sizeof (republish_token_t), KM_SLEEP);
5719
5720 rp->key = segid;
5721 rp->importing_node = token->importing_node;
5722 rp->permission = permission;
5723 rp->next = republish_list;
5724 republish_list = rp;
5725 }
5726 token = token->next;
5727 }
5728
5729 mutex_exit(&importer_list.lock);
5730
5731 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPUBLISH;
5732 request.rsmipc_key = segid;
5733
5734 while (republish_list != NULL) {
5735 request.rsmipc_perm = republish_list->permission;
5736 (void) rsmipc_send(republish_list->importing_node,
5737 &request, RSM_NO_REPLY);
5738 rp = republish_list;
5739 republish_list = republish_list->next;
5740 kmem_free(rp, sizeof (republish_token_t));
5741 }
5742 }
5743
5744 static void
5745 rsm_send_suspend()
5746 {
5747 int i, e;
5748 rsmipc_request_t request;
5749 list_element_t *tokp;
5750 list_element_t *head = NULL;
5751 importing_token_t *token;
5752 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
5753 "rsm_send_suspend enter\n"));
5754
5755 /*
5756 * create a list of node to send the suspend message
5757 *
5758 * Currently the whole importer list is scanned and we obtain
5759 * all the nodes - this basically gets all nodes that at least
5760 * import one segment from the local node.
5761 *
5762 * no need to grab the rsm_suspend_list lock here since we are
5763 * single threaded when suspend is called.
5764 */
5765
5766 mutex_enter(&importer_list.lock);
5767 for (i = 0; i < rsm_hash_size; i++) {
5768
5769 token = importer_list.bucket[i];
5770
5771 while (token != NULL) {
5772
5773 tokp = head;
5774
5775 /*
5776 * make sure that the token's node
5777 * is not already on the suspend list
5778 */
5779 while (tokp != NULL) {
5780 if (tokp->nodeid == token->importing_node) {
5781 break;
5782 }
5783 tokp = tokp->next;
5784 }
5785
5786 if (tokp == NULL) { /* not in suspend list */
5787 tokp = kmem_zalloc(sizeof (list_element_t),
5788 KM_SLEEP);
5789 tokp->nodeid = token->importing_node;
5790 tokp->next = head;
5791 head = tokp;
5792 }
5793
5794 token = token->next;
5795 }
5796 }
5797 mutex_exit(&importer_list.lock);
5798
5799 if (head == NULL) { /* no importers so go ahead and quiesce segments */
5800 exporter_quiesce();
5801 return;
5802 }
5803
5804 mutex_enter(&rsm_suspend_list.list_lock);
5805 ASSERT(rsm_suspend_list.list_head == NULL);
5806 /*
5807 * update the suspend list righaway so that if a node dies the
5808 * pathmanager can set the NODE dead flag
5809 */
5810 rsm_suspend_list.list_head = head;
5811 mutex_exit(&rsm_suspend_list.list_lock);
5812
5813 tokp = head;
5814
5815 while (tokp != NULL) {
5816 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SUSPEND;
5817 e = rsmipc_send(tokp->nodeid, &request, RSM_NO_REPLY);
5818 /*
5819 * Error in rsmipc_send currently happens due to inaccessibility
5820 * of the remote node.
5821 */
5822 if (e == RSM_SUCCESS) { /* send failed - don't wait for ack */
5823 tokp->flags |= RSM_SUSPEND_ACKPENDING;
5824 }
5825
5826 tokp = tokp->next;
5827 }
5828
5829 DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
5830 "rsm_send_suspend done\n"));
5831
5832 }
5833
5834 static void
5835 rsm_send_resume()
5836 {
5837 rsmipc_request_t request;
5838 list_element_t *elem, *head;
5839
5840 /*
5841 * save the suspend list so that we know where to send
5842 * the resume messages and make the suspend list head
5843 * NULL.
5844 */
5845 mutex_enter(&rsm_suspend_list.list_lock);
5846 head = rsm_suspend_list.list_head;
5847 rsm_suspend_list.list_head = NULL;
5848 mutex_exit(&rsm_suspend_list.list_lock);
5849
5850 while (head != NULL) {
5851 elem = head;
5852 head = head->next;
5853
5854 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_RESUME;
5855
5856 (void) rsmipc_send(elem->nodeid, &request, RSM_NO_REPLY);
5857
5858 kmem_free((void *)elem, sizeof (list_element_t));
5859
5860 }
5861
5862 }
5863
5864 /*
5865 * This function takes path and sends a message using the sendq
5866 * corresponding to it. The RSMIPC_MSG_SQREADY, RSMIPC_MSG_SQREADY_ACK
5867 * and RSMIPC_MSG_CREDIT are sent using this function.
5868 */
5869 int
5870 rsmipc_send_controlmsg(path_t *path, int msgtype)
5871 {
5872 int e;
5873 int retry_cnt = 0;
5874 int min_retry_cnt = 10;
5875 adapter_t *adapter;
5876 rsm_send_t is;
5877 rsm_send_q_handle_t ipc_handle;
5878 rsmipc_controlmsg_t msg;
5879 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_FLOWCONTROL);
5880
5881 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5882 "rsmipc_send_controlmsg enter\n"));
5883
5884 ASSERT(MUTEX_HELD(&path->mutex));
5885
5886 adapter = path->local_adapter;
5887
5888 DBG_PRINTF((category, RSM_DEBUG, "rsmipc_send_controlmsg:path=%lx "
5889 "msgtype=%d %lx:%llx->%lx:%llx procmsg=%d\n", path, msgtype,
5890 my_nodeid, adapter->hwaddr, path->remote_node,
5891 path->remote_hwaddr, path->procmsg_cnt));
5892
5893 if (path->state != RSMKA_PATH_ACTIVE) {
5894 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5895 "rsmipc_send_controlmsg done: ! RSMKA_PATH_ACTIVE"));
5896 return (1);
5897 }
5898
5899 ipc_handle = path->sendq_token.rsmpi_sendq_handle;
5900
5901 msg.rsmipc_hdr.rsmipc_version = RSM_VERSION;
5902 msg.rsmipc_hdr.rsmipc_src = my_nodeid;
5903 msg.rsmipc_hdr.rsmipc_type = msgtype;
5904 msg.rsmipc_hdr.rsmipc_incn = path->remote_incn;
5905
5906 if (msgtype == RSMIPC_MSG_CREDIT)
5907 msg.rsmipc_credits = path->procmsg_cnt;
5908
5909 msg.rsmipc_local_incn = path->local_incn;
5910
5911 msg.rsmipc_adapter_hwaddr = adapter->hwaddr;
5912 /* incr the sendq, path refcnt */
5913 PATH_HOLD_NOLOCK(path);
5914 SENDQ_TOKEN_HOLD(path);
5915
5916 do {
5917 /* drop the path lock before doing the rsm_send */
5918 mutex_exit(&path->mutex);
5919
5920 is.is_data = (void *)&msg;
5921 is.is_size = sizeof (msg);
5922 is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
5923 is.is_wait = 0;
5924
5925 e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, NULL);
5926
5927 ASSERT(e != RSMERR_QUEUE_FENCE_UP &&
5928 e != RSMERR_BAD_BARRIER_HNDL);
5929
5930 mutex_enter(&path->mutex);
5931
5932 if (e == RSM_SUCCESS) {
5933 break;
5934 }
5935 /* error counter for statistics */
5936 atomic_inc_64(&rsm_ctrlmsg_errcnt);
5937
5938 DBG_PRINTF((category, RSM_ERR,
5939 "rsmipc_send_controlmsg:rsm_send error=%d", e));
5940
5941 if (++retry_cnt == min_retry_cnt) { /* backoff before retry */
5942 (void) cv_reltimedwait(&path->sendq_token.sendq_cv,
5943 &path->mutex, drv_usectohz(10000), TR_CLOCK_TICK);
5944 retry_cnt = 0;
5945 }
5946 } while (path->state == RSMKA_PATH_ACTIVE);
5947
5948 /* decrement the sendq,path refcnt that we incr before rsm_send */
5949 SENDQ_TOKEN_RELE(path);
5950 PATH_RELE_NOLOCK(path);
5951
5952 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
5953 "rsmipc_send_controlmsg done=%d", e));
5954 return (e);
5955 }
5956
5957 /*
5958 * Called from rsm_force_unload and path_importer_disconnect. The memory
5959 * mapping for the imported segment is removed and the segment is
5960 * disconnected at the interconnect layer if disconnect_flag is TRUE.
5961 * rsm_force_unload will get disconnect_flag TRUE from rsm_intr_callback
5962 * and FALSE from rsm_rebind.
5963 *
5964 * When subsequent accesses cause page faulting, the dummy page is mapped
5965 * to resolve the fault, and the mapping generation number is incremented
5966 * so that the application can be notified on a close barrier operation.
5967 *
5968 * It is important to note that the caller of rsmseg_unload is responsible for
5969 * acquiring the segment lock before making a call to rsmseg_unload. This is
5970 * required to make the caller and rsmseg_unload thread safe. The segment lock
5971 * will be released by the rsmseg_unload function.
5972 */
5973 void
5974 rsmseg_unload(rsmseg_t *im_seg)
5975 {
5976 rsmcookie_t *hdl;
5977 void *shared_cookie;
5978 rsmipc_request_t request;
5979 uint_t maxprot;
5980
5981 DBG_DEFINE(category,
5982 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);
5983
5984 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_unload enter\n"));
5985
5986 ASSERT(im_seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
5987
5988 /* wait until segment leaves the mapping state */
5989 while (im_seg->s_state == RSM_STATE_MAPPING)
5990 cv_wait(&im_seg->s_cv, &im_seg->s_lock);
5991 /*
5992 * An unload is only necessary if the segment is connected. However,
5993 * if the segment was on the import list in state RSM_STATE_CONNECTING
5994 * then a connection was in progress. Change to RSM_STATE_NEW
5995 * here to cause an early exit from the connection process.
5996 */
5997 if (im_seg->s_state == RSM_STATE_NEW) {
5998 rsmseglock_release(im_seg);
5999 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6000 "rsmseg_unload done: RSM_STATE_NEW\n"));
6001 return;
6002 } else if (im_seg->s_state == RSM_STATE_CONNECTING) {
6003 im_seg->s_state = RSM_STATE_ABORT_CONNECT;
6004 rsmsharelock_acquire(im_seg);
6005 im_seg->s_share->rsmsi_state = RSMSI_STATE_ABORT_CONNECT;
6006 rsmsharelock_release(im_seg);
6007 rsmseglock_release(im_seg);
6008 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6009 "rsmseg_unload done: RSM_STATE_CONNECTING\n"));
6010 return;
6011 }
6012
6013 if (im_seg->s_flags & RSM_FORCE_DISCONNECT) {
6014 if (im_seg->s_ckl != NULL) {
6015 int e;
6016 /* Setup protections for remap */
6017 maxprot = PROT_USER;
6018 if (im_seg->s_mode & RSM_PERM_READ) {
6019 maxprot |= PROT_READ;
6020 }
6021 if (im_seg->s_mode & RSM_PERM_WRITE) {
6022 maxprot |= PROT_WRITE;
6023 }
6024 hdl = im_seg->s_ckl;
6025 for (; hdl != NULL; hdl = hdl->c_next) {
6026 e = devmap_umem_remap(hdl->c_dhp, rsm_dip,
6027 remap_cookie,
6028 hdl->c_off, hdl->c_len,
6029 maxprot, 0, NULL);
6030
6031 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6032 "remap returns %d\n", e));
6033 }
6034 }
6035
6036 (void) rsm_closeconnection(im_seg, &shared_cookie);
6037
6038 if (shared_cookie != NULL) {
6039 /*
6040 * inform the exporting node so this import
6041 * can be deleted from the list of importers.
6042 */
6043 request.rsmipc_hdr.rsmipc_type =
6044 RSMIPC_MSG_NOTIMPORTING;
6045 request.rsmipc_key = im_seg->s_segid;
6046 request.rsmipc_segment_cookie = shared_cookie;
6047 rsmseglock_release(im_seg);
6048 (void) rsmipc_send(im_seg->s_node, &request,
6049 RSM_NO_REPLY);
6050 } else {
6051 rsmseglock_release(im_seg);
6052 }
6053 }
6054 else
6055 rsmseglock_release(im_seg);
6056
6057 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_unload done\n"));
6058
6059 }
6060
6061 /* ****************************** Importer Calls ************************ */
6062
6063 static int
6064 rsm_access(uid_t owner, gid_t group, int perm, int mode, const struct cred *cr)
6065 {
6066 int shifts = 0;
6067
6068 if (crgetuid(cr) != owner) {
6069 shifts += 3;
6070 if (!groupmember(group, cr))
6071 shifts += 3;
6072 }
6073
6074 mode &= ~(perm << shifts);
6075
6076 if (mode == 0)
6077 return (0);
6078
6079 return (secpolicy_rsm_access(cr, owner, mode));
6080 }
6081
6082
6083 static int
6084 rsm_connect(rsmseg_t *seg, rsm_ioctlmsg_t *msg, cred_t *cred,
6085 intptr_t dataptr, int mode)
6086 {
6087 int e;
6088 int recheck_state = 0;
6089 void *shared_cookie;
6090 rsmipc_request_t request;
6091 rsmipc_reply_t reply;
6092 rsm_permission_t access;
6093 adapter_t *adapter;
6094 rsm_addr_t addr = 0;
6095 rsm_import_share_t *sharedp;
6096 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);
6097
6098 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_connect enter\n"));
6099
6100 adapter = rsm_getadapter(msg, mode);
6101 if (adapter == NULL) {
6102 DBG_PRINTF((category, RSM_ERR,
6103 "rsm_connect done:ENODEV adapter=NULL\n"));
6104 return (RSMERR_CTLR_NOT_PRESENT);
6105 }
6106
6107 if ((adapter == &loopback_adapter) && (msg->nodeid != my_nodeid)) {
6108 rsmka_release_adapter(adapter);
6109 DBG_PRINTF((category, RSM_ERR,
6110 "rsm_connect done:ENODEV loopback\n"));
6111 return (RSMERR_CTLR_NOT_PRESENT);
6112 }
6113
6114
6115 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
6116 ASSERT(seg->s_state == RSM_STATE_NEW);
6117
6118 /*
6119 * Translate perm to access
6120 */
6121 if (msg->perm & ~RSM_PERM_RDWR) {
6122 rsmka_release_adapter(adapter);
6123 DBG_PRINTF((category, RSM_ERR,
6124 "rsm_connect done:EINVAL invalid perms\n"));
6125 return (RSMERR_BAD_PERMS);
6126 }
6127 access = 0;
6128 if (msg->perm & RSM_PERM_READ)
6129 access |= RSM_ACCESS_READ;
6130 if (msg->perm & RSM_PERM_WRITE)
6131 access |= RSM_ACCESS_WRITE;
6132
6133 seg->s_node = msg->nodeid;
6134
6135 /*
6136 * Adding to the import list locks the segment; release the segment
6137 * lock so we can get the reply for the send.
6138 */
6139 e = rsmimport_add(seg, msg->key);
6140 if (e) {
6141 rsmka_release_adapter(adapter);
6142 DBG_PRINTF((category, RSM_ERR,
6143 "rsm_connect done:rsmimport_add failed %d\n", e));
6144 return (e);
6145 }
6146 seg->s_state = RSM_STATE_CONNECTING;
6147
6148 /*
6149 * Set the s_adapter field here so as to have a valid comparison of
6150 * the adapter and the s_adapter value during rsmshare_get. For
6151 * any error, set s_adapter to NULL before doing a release_adapter
6152 */
6153 seg->s_adapter = adapter;
6154
6155 rsmseglock_release(seg);
6156
6157 /*
6158 * get the pointer to the shared data structure; the
6159 * shared data is locked and refcount has been incremented
6160 */
6161 sharedp = rsmshare_get(msg->key, msg->nodeid, adapter, seg);
6162
6163 ASSERT(rsmsharelock_held(seg));
6164
6165 do {
6166 /* flag indicates whether we need to recheck the state */
6167 recheck_state = 0;
6168 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6169 "rsm_connect:RSMSI_STATE=%d\n", sharedp->rsmsi_state));
6170 switch (sharedp->rsmsi_state) {
6171 case RSMSI_STATE_NEW:
6172 sharedp->rsmsi_state = RSMSI_STATE_CONNECTING;
6173 break;
6174 case RSMSI_STATE_CONNECTING:
6175 /* FALLTHRU */
6176 case RSMSI_STATE_CONN_QUIESCE:
6177 /* FALLTHRU */
6178 case RSMSI_STATE_MAP_QUIESCE:
6179 /* wait for the state to change */
6180 while ((sharedp->rsmsi_state ==
6181 RSMSI_STATE_CONNECTING) ||
6182 (sharedp->rsmsi_state ==
6183 RSMSI_STATE_CONN_QUIESCE) ||
6184 (sharedp->rsmsi_state ==
6185 RSMSI_STATE_MAP_QUIESCE)) {
6186 if (cv_wait_sig(&sharedp->rsmsi_cv,
6187 &sharedp->rsmsi_lock) == 0) {
6188 /* signalled - clean up and return */
6189 rsmsharelock_release(seg);
6190 rsmimport_rm(seg);
6191 seg->s_adapter = NULL;
6192 rsmka_release_adapter(adapter);
6193 seg->s_state = RSM_STATE_NEW;
6194 DBG_PRINTF((category, RSM_ERR,
6195 "rsm_connect done: INTERRUPTED\n"));
6196 return (RSMERR_INTERRUPTED);
6197 }
6198 }
6199 /*
6200 * the state changed, loop back and check what it is
6201 */
6202 recheck_state = 1;
6203 break;
6204 case RSMSI_STATE_ABORT_CONNECT:
6205 /* exit the loop and clean up further down */
6206 break;
6207 case RSMSI_STATE_CONNECTED:
6208 /* already connected, good - fall through */
6209 case RSMSI_STATE_MAPPED:
6210 /* already mapped, wow - fall through */
6211 /* access validation etc is done further down */
6212 break;
6213 case RSMSI_STATE_DISCONNECTED:
6214 /* disconnected - so reconnect now */
6215 sharedp->rsmsi_state = RSMSI_STATE_CONNECTING;
6216 break;
6217 default:
6218 ASSERT(0); /* Invalid State */
6219 }
6220 } while (recheck_state);
6221
6222 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) {
6223 /* we are the first to connect */
6224 rsmsharelock_release(seg);
6225
6226 if (msg->nodeid != my_nodeid) {
6227 addr = get_remote_hwaddr(adapter, msg->nodeid);
6228
6229 if ((int64_t)addr < 0) {
6230 rsmsharelock_acquire(seg);
6231 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING,
6232 RSMSI_STATE_NEW);
6233 rsmsharelock_release(seg);
6234 rsmimport_rm(seg);
6235 seg->s_adapter = NULL;
6236 rsmka_release_adapter(adapter);
6237 seg->s_state = RSM_STATE_NEW;
6238 DBG_PRINTF((category, RSM_ERR,
6239 "rsm_connect done: hwaddr<0\n"));
6240 return (RSMERR_INTERNAL_ERROR);
6241 }
6242 } else {
6243 addr = adapter->hwaddr;
6244 }
6245
6246 /*
6247 * send request to node [src, dest, key, msgid] and get back
6248 * [status, msgid, cookie]
6249 */
6250 request.rsmipc_key = msg->key;
6251 /*
6252 * we need the s_mode of the exporter so pass
6253 * RSM_ACCESS_TRUSTED
6254 */
6255 request.rsmipc_perm = RSM_ACCESS_TRUSTED;
6256 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SEGCONNECT;
6257 request.rsmipc_adapter_hwaddr = addr;
6258 request.rsmipc_segment_cookie = sharedp;
6259
6260 e = (int)rsmipc_send(msg->nodeid, &request, &reply);
6261 if (e) {
6262 rsmsharelock_acquire(seg);
6263 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING,
6264 RSMSI_STATE_NEW);
6265 rsmsharelock_release(seg);
6266 rsmimport_rm(seg);
6267 seg->s_adapter = NULL;
6268 rsmka_release_adapter(adapter);
6269 seg->s_state = RSM_STATE_NEW;
6270 DBG_PRINTF((category, RSM_ERR,
6271 "rsm_connect done:rsmipc_send failed %d\n", e));
6272 return (e);
6273 }
6274
6275 if (reply.rsmipc_status != RSM_SUCCESS) {
6276 rsmsharelock_acquire(seg);
6277 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING,
6278 RSMSI_STATE_NEW);
6279 rsmsharelock_release(seg);
6280 rsmimport_rm(seg);
6281 seg->s_adapter = NULL;
6282 rsmka_release_adapter(adapter);
6283 seg->s_state = RSM_STATE_NEW;
6284 DBG_PRINTF((category, RSM_ERR,
6285 "rsm_connect done:rsmipc_send reply err %d\n",
6286 reply.rsmipc_status));
6287 return (reply.rsmipc_status);
6288 }
6289
6290 rsmsharelock_acquire(seg);
6291 /* store the information recvd into the shared data struct */
6292 sharedp->rsmsi_mode = reply.rsmipc_mode;
6293 sharedp->rsmsi_uid = reply.rsmipc_uid;
6294 sharedp->rsmsi_gid = reply.rsmipc_gid;
6295 sharedp->rsmsi_seglen = reply.rsmipc_seglen;
6296 sharedp->rsmsi_cookie = sharedp;
6297 }
6298
6299 rsmsharelock_release(seg);
6300
6301 /*
6302 * Get the segment lock and check for a force disconnect
6303 * from the export side which would have changed the state
6304 * back to RSM_STATE_NEW. Once the segment lock is acquired a
6305 * force disconnect will be held off until the connection
6306 * has completed.
6307 */
6308 rsmseglock_acquire(seg);
6309 rsmsharelock_acquire(seg);
6310 ASSERT(seg->s_state == RSM_STATE_CONNECTING ||
6311 seg->s_state == RSM_STATE_ABORT_CONNECT);
6312
6313 shared_cookie = sharedp->rsmsi_cookie;
6314
6315 if ((seg->s_state == RSM_STATE_ABORT_CONNECT) ||
6316 (sharedp->rsmsi_state == RSMSI_STATE_ABORT_CONNECT)) {
6317 seg->s_state = RSM_STATE_NEW;
6318 seg->s_adapter = NULL;
6319 rsmsharelock_release(seg);
6320 rsmseglock_release(seg);
6321 rsmimport_rm(seg);
6322 rsmka_release_adapter(adapter);
6323
6324 rsmsharelock_acquire(seg);
6325 if (!(sharedp->rsmsi_flags & RSMSI_FLAGS_ABORTDONE)) {
6326 /*
6327 * set a flag indicating abort handling has been
6328 * done
6329 */
6330 sharedp->rsmsi_flags |= RSMSI_FLAGS_ABORTDONE;
6331 rsmsharelock_release(seg);
6332 /* send a message to exporter - only once */
6333 (void) rsm_send_notimporting(msg->nodeid,
6334 msg->key, shared_cookie);
6335 rsmsharelock_acquire(seg);
6336 /*
6337 * wake up any waiting importers and inform that
6338 * connection has been aborted
6339 */
6340 cv_broadcast(&sharedp->rsmsi_cv);
6341 }
6342 rsmsharelock_release(seg);
6343
6344 DBG_PRINTF((category, RSM_ERR,
6345 "rsm_connect done: RSM_STATE_ABORT_CONNECT\n"));
6346 return (RSMERR_INTERRUPTED);
6347 }
6348
6349
6350 /*
6351 * We need to verify that this process has access
6352 */
6353 e = rsm_access(sharedp->rsmsi_uid, sharedp->rsmsi_gid,
6354 access & sharedp->rsmsi_mode,
6355 (int)(msg->perm & RSM_PERM_RDWR), cred);
6356 if (e) {
6357 rsmsharelock_release(seg);
6358 seg->s_state = RSM_STATE_NEW;
6359 seg->s_adapter = NULL;
6360 rsmseglock_release(seg);
6361 rsmimport_rm(seg);
6362 rsmka_release_adapter(adapter);
6363 /*
6364 * No need to lock segment it has been removed
6365 * from the hash table
6366 */
6367 rsmsharelock_acquire(seg);
6368 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) {
6369 rsmsharelock_release(seg);
6370 /* this is the first importer */
6371
6372 (void) rsm_send_notimporting(msg->nodeid, msg->key,
6373 shared_cookie);
6374 rsmsharelock_acquire(seg);
6375 sharedp->rsmsi_state = RSMSI_STATE_NEW;
6376 cv_broadcast(&sharedp->rsmsi_cv);
6377 }
6378 rsmsharelock_release(seg);
6379
6380 DBG_PRINTF((category, RSM_ERR,
6381 "rsm_connect done: ipcaccess failed\n"));
6382 return (RSMERR_PERM_DENIED);
6383 }
6384
6385 /* update state and cookie */
6386 seg->s_segid = sharedp->rsmsi_segid;
6387 seg->s_len = sharedp->rsmsi_seglen;
6388 seg->s_mode = access & sharedp->rsmsi_mode;
6389 seg->s_pid = ddi_get_pid();
6390 seg->s_mapinfo = NULL;
6391
6392 if (seg->s_node != my_nodeid) {
6393 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) {
6394 e = adapter->rsmpi_ops->rsm_connect(
6395 adapter->rsmpi_handle,
6396 addr, seg->s_segid, &sharedp->rsmsi_handle);
6397
6398 if (e != RSM_SUCCESS) {
6399 seg->s_state = RSM_STATE_NEW;
6400 seg->s_adapter = NULL;
6401 rsmsharelock_release(seg);
6402 rsmseglock_release(seg);
6403 rsmimport_rm(seg);
6404 rsmka_release_adapter(adapter);
6405 /*
6406 * inform the exporter to delete this importer
6407 */
6408 (void) rsm_send_notimporting(msg->nodeid,
6409 msg->key, shared_cookie);
6410
6411 /*
6412 * Now inform any waiting importers to
6413 * retry connect. This needs to be done
6414 * after sending notimporting so that
6415 * the notimporting is sent before a waiting
6416 * importer sends a segconnect while retrying
6417 *
6418 * No need to lock segment it has been removed
6419 * from the hash table
6420 */
6421
6422 rsmsharelock_acquire(seg);
6423 sharedp->rsmsi_state = RSMSI_STATE_NEW;
6424 cv_broadcast(&sharedp->rsmsi_cv);
6425 rsmsharelock_release(seg);
6426
6427 DBG_PRINTF((category, RSM_ERR,
6428 "rsm_connect error %d\n", e));
6429 if (e == RSMERR_SEG_NOT_PUBLISHED_TO_RSM_ADDR)
6430 return (
6431 RSMERR_SEG_NOT_PUBLISHED_TO_NODE);
6432 else if ((e == RSMERR_RSM_ADDR_UNREACHABLE) ||
6433 (e == RSMERR_UNKNOWN_RSM_ADDR))
6434 return (RSMERR_REMOTE_NODE_UNREACHABLE);
6435 else
6436 return (e);
6437 }
6438
6439 }
6440 seg->s_handle.in = sharedp->rsmsi_handle;
6441
6442 }
6443
6444 seg->s_state = RSM_STATE_CONNECT;
6445
6446
6447 seg->s_flags &= ~RSM_IMPORT_DUMMY; /* clear dummy flag */
6448 if (bar_va) {
6449 /* increment generation number on barrier page */
6450 atomic_inc_16(bar_va + seg->s_hdr.rsmrc_num);
6451 /* return user off into barrier page where status will be */
6452 msg->off = (int)seg->s_hdr.rsmrc_num;
6453 msg->gnum = bar_va[msg->off]; /* gnum race */
6454 } else {
6455 msg->off = 0;
6456 msg->gnum = 0; /* gnum race */
6457 }
6458
6459 msg->len = (int)sharedp->rsmsi_seglen;
6460 msg->rnum = seg->s_minor;
6461 rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING, RSMSI_STATE_CONNECTED);
6462 rsmsharelock_release(seg);
6463 rsmseglock_release(seg);
6464
6465 /* Return back to user the segment size & perm in case it's needed */
6466
6467 #ifdef _MULTI_DATAMODEL
6468 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
6469 rsm_ioctlmsg32_t msg32;
6470
6471 if (msg->len > UINT_MAX)
6472 msg32.len = RSM_MAXSZ_PAGE_ALIGNED;
6473 else
6474 msg32.len = msg->len;
6475 msg32.off = msg->off;
6476 msg32.perm = msg->perm;
6477 msg32.gnum = msg->gnum;
6478 msg32.rnum = msg->rnum;
6479
6480 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6481 "rsm_connect done\n"));
6482
6483 if (ddi_copyout((caddr_t)&msg32, (caddr_t)dataptr,
6484 sizeof (msg32), mode))
6485 return (RSMERR_BAD_ADDR);
6486 else
6487 return (RSM_SUCCESS);
6488 }
6489 #endif
6490 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_connect done\n"));
6491
6492 if (ddi_copyout((caddr_t)msg, (caddr_t)dataptr, sizeof (*msg),
6493 mode))
6494 return (RSMERR_BAD_ADDR);
6495 else
6496 return (RSM_SUCCESS);
6497 }
6498
6499 static int
6500 rsm_unmap(rsmseg_t *seg)
6501 {
6502 int err;
6503 adapter_t *adapter;
6504 rsm_import_share_t *sharedp;
6505 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);
6506
6507 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6508 "rsm_unmap enter %u\n", seg->s_segid));
6509
6510 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
6511
6512 /* assert seg is locked */
6513 ASSERT(rsmseglock_held(seg));
6514 ASSERT(seg->s_state != RSM_STATE_MAPPING);
6515
6516 if ((seg->s_state != RSM_STATE_ACTIVE) &&
6517 (seg->s_state != RSM_STATE_MAP_QUIESCE)) {
6518 /* segment unmap has already been done */
6519 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unmap done\n"));
6520 return (RSM_SUCCESS);
6521 }
6522
6523 sharedp = seg->s_share;
6524
6525 rsmsharelock_acquire(seg);
6526
6527 /*
6528 * - shared data struct is in MAPPED or MAP_QUIESCE state
6529 */
6530
6531 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED ||
6532 sharedp->rsmsi_state == RSMSI_STATE_MAP_QUIESCE);
6533
6534 /*
6535 * Unmap pages - previously rsm_memseg_import_unmap was called only if
6536 * the segment cookie list was NULL; but it is always NULL when
6537 * called from rsmmap_unmap and won't be NULL when called for
6538 * a force disconnect - so the check for NULL cookie list was removed
6539 */
6540
6541 ASSERT(sharedp->rsmsi_mapcnt > 0);
6542
6543 sharedp->rsmsi_mapcnt--;
6544
6545 if (sharedp->rsmsi_mapcnt == 0) {
6546 if (sharedp->rsmsi_state == RSMSI_STATE_MAPPED) {
6547 /* unmap the shared RSMPI mapping */
6548 adapter = seg->s_adapter;
6549 if (seg->s_node != my_nodeid) {
6550 ASSERT(sharedp->rsmsi_handle != NULL);
6551 err = adapter->rsmpi_ops->
6552 rsm_unmap(sharedp->rsmsi_handle);
6553 DBG_PRINTF((category, RSM_DEBUG,
6554 "rsm_unmap: rsmpi unmap %d\n", err));
6555 rsm_free_mapinfo(sharedp->rsmsi_mapinfo);
6556 sharedp->rsmsi_mapinfo = NULL;
6557 }
6558 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
6559 } else { /* MAP_QUIESCE --munmap()--> CONN_QUIESCE */
6560 sharedp->rsmsi_state = RSMSI_STATE_CONN_QUIESCE;
6561 }
6562 }
6563
6564 rsmsharelock_release(seg);
6565
6566 /*
6567 * The s_cookie field is used to store the cookie returned from the
6568 * ddi_umem_lock when binding the pages for an export segment. This
6569 * is the primary use of the s_cookie field and does not normally
6570 * pertain to any importing segment except in the loopback case.
6571 * For the loopback case, the import segment and export segment are
6572 * on the same node, the s_cookie field of the segment structure for
6573 * the importer is initialized to the s_cookie field in the exported
6574 * segment during the map operation and is used during the call to
6575 * devmap_umem_setup for the import mapping.
6576 * Thus, during unmap, we simply need to set s_cookie to NULL to
6577 * indicate that the mapping no longer exists.
6578 */
6579 seg->s_cookie = NULL;
6580
6581 seg->s_mapinfo = NULL;
6582
6583 if (seg->s_state == RSM_STATE_ACTIVE)
6584 seg->s_state = RSM_STATE_CONNECT;
6585 else
6586 seg->s_state = RSM_STATE_CONN_QUIESCE;
6587
6588 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unmap done\n"));
6589
6590 return (RSM_SUCCESS);
6591 }
6592
6593 /*
6594 * cookie returned here if not null indicates that it is
6595 * the last importer and it can be used in the RSMIPC_NOT_IMPORTING
6596 * message.
6597 */
6598 static int
6599 rsm_closeconnection(rsmseg_t *seg, void **cookie)
6600 {
6601 int e;
6602 adapter_t *adapter;
6603 rsm_import_share_t *sharedp;
6604 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);
6605
6606 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6607 "rsm_closeconnection enter\n"));
6608
6609 *cookie = (void *)NULL;
6610
6611 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
6612
6613 /* assert seg is locked */
6614 ASSERT(rsmseglock_held(seg));
6615
6616 if (seg->s_state == RSM_STATE_DISCONNECT) {
6617 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6618 "rsm_closeconnection done: already disconnected\n"));
6619 return (RSM_SUCCESS);
6620 }
6621
6622 /* wait for all putv/getv ops to get done */
6623 while (seg->s_rdmacnt > 0) {
6624 cv_wait(&seg->s_cv, &seg->s_lock);
6625 }
6626
6627 (void) rsm_unmap(seg);
6628
6629 ASSERT(seg->s_state == RSM_STATE_CONNECT ||
6630 seg->s_state == RSM_STATE_CONN_QUIESCE);
6631
6632 adapter = seg->s_adapter;
6633 sharedp = seg->s_share;
6634
6635 ASSERT(sharedp != NULL);
6636
6637 rsmsharelock_acquire(seg);
6638
6639 /*
6640 * Disconnect on adapter
6641 *
6642 * The current algorithm is stateless, I don't have to contact
6643 * server when I go away. It only gives me permissions. Of course,
6644 * the adapters will talk to terminate the connect.
6645 *
6646 * disconnect is needed only if we are CONNECTED not in CONN_QUIESCE
6647 */
6648 if ((sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) &&
6649 (sharedp->rsmsi_node != my_nodeid)) {
6650
6651 if (sharedp->rsmsi_refcnt == 1) {
6652 /* this is the last importer */
6653 ASSERT(sharedp->rsmsi_mapcnt == 0);
6654
6655 e = adapter->rsmpi_ops->
6656 rsm_disconnect(sharedp->rsmsi_handle);
6657 if (e != RSM_SUCCESS) {
6658 DBG_PRINTF((category, RSM_DEBUG,
6659 "rsm:disconnect failed seg=%x:err=%d\n",
6660 seg->s_key, e));
6661 }
6662 }
6663 }
6664
6665 seg->s_handle.in = NULL;
6666
6667 sharedp->rsmsi_refcnt--;
6668
6669 if (sharedp->rsmsi_refcnt == 0) {
6670 *cookie = (void *)sharedp->rsmsi_cookie;
6671 sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED;
6672 sharedp->rsmsi_handle = NULL;
6673 rsmsharelock_release(seg);
6674
6675 /* clean up the shared data structure */
6676 mutex_destroy(&sharedp->rsmsi_lock);
6677 cv_destroy(&sharedp->rsmsi_cv);
6678 kmem_free((void *)(sharedp), sizeof (rsm_import_share_t));
6679
6680 } else {
6681 rsmsharelock_release(seg);
6682 }
6683
6684 /* increment generation number on barrier page */
6685 if (bar_va) {
6686 atomic_inc_16(bar_va + seg->s_hdr.rsmrc_num);
6687 }
6688
6689 /*
6690 * The following needs to be done after any
6691 * rsmsharelock calls which use seg->s_share.
6692 */
6693 seg->s_share = NULL;
6694
6695 seg->s_state = RSM_STATE_DISCONNECT;
6696 /* signal anyone waiting in the CONN_QUIESCE state */
6697 cv_broadcast(&seg->s_cv);
6698
6699 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6700 "rsm_closeconnection done\n"));
6701
6702 return (RSM_SUCCESS);
6703 }
6704
6705 int
6706 rsm_disconnect(rsmseg_t *seg)
6707 {
6708 rsmipc_request_t request;
6709 void *shared_cookie;
6710 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);
6711
6712 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_disconnect enter\n"));
6713
6714 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
6715
6716 /* assert seg isn't locked */
6717 ASSERT(!rsmseglock_held(seg));
6718
6719
6720 /* Remove segment from imported list */
6721 rsmimport_rm(seg);
6722
6723 /* acquire the segment */
6724 rsmseglock_acquire(seg);
6725
6726 /* wait until segment leaves the mapping state */
6727 while (seg->s_state == RSM_STATE_MAPPING)
6728 cv_wait(&seg->s_cv, &seg->s_lock);
6729
6730 if (seg->s_state == RSM_STATE_DISCONNECT) {
6731 seg->s_state = RSM_STATE_NEW;
6732 rsmseglock_release(seg);
6733 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6734 "rsm_disconnect done: already disconnected\n"));
6735 return (RSM_SUCCESS);
6736 }
6737
6738 (void) rsm_closeconnection(seg, &shared_cookie);
6739
6740 /* update state */
6741 seg->s_state = RSM_STATE_NEW;
6742
6743 if (shared_cookie != NULL) {
6744 /*
6745 * This is the last importer so inform the exporting node
6746 * so this import can be deleted from the list of importers.
6747 */
6748 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_NOTIMPORTING;
6749 request.rsmipc_key = seg->s_segid;
6750 request.rsmipc_segment_cookie = shared_cookie;
6751 rsmseglock_release(seg);
6752 (void) rsmipc_send(seg->s_node, &request, RSM_NO_REPLY);
6753 } else {
6754 rsmseglock_release(seg);
6755 }
6756
6757 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_disconnect done\n"));
6758
6759 return (DDI_SUCCESS);
6760 }
6761
6762 static int
6763 rsm_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
6764 struct pollhead **phpp)
6765 {
6766 minor_t rnum;
6767 rsmresource_t *res;
6768 rsmseg_t *seg;
6769 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
6770
6771 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_chpoll enter\n"));
6772
6773 /* find minor, no lock */
6774 rnum = getminor(dev);
6775 res = rsmresource_lookup(rnum, RSM_NOLOCK);
6776
6777 /* poll is supported only for export/import segments */
6778 if ((res == NULL) || (res == RSMRC_RESERVED) ||
6779 (res->rsmrc_type == RSM_RESOURCE_BAR)) {
6780 return (ENXIO);
6781 }
6782
6783 /*
6784 * An exported segment must be in state RSM_STATE_EXPORT; an
6785 * imported segment must be in state RSM_STATE_ACTIVE.
6786 */
6787 seg = (rsmseg_t *)res;
6788
6789 if (seg->s_pollevent) {
6790 *reventsp = POLLRDNORM;
6791 } else {
6792 *reventsp = 0;
6793 }
6794
6795 if ((*reventsp == 0 && !anyyet) || (events & POLLET)) {
6796 /* cannot take segment lock here */
6797 *phpp = &seg->s_poll;
6798 seg->s_pollflag |= RSM_SEGMENT_POLL;
6799 }
6800 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_chpoll done\n"));
6801 return (0);
6802 }
6803
6804
6805
6806 /* ************************* IOCTL Commands ********************* */
6807
6808 static rsmseg_t *
6809 rsmresource_seg(rsmresource_t *res, minor_t rnum, cred_t *credp,
6810 rsm_resource_type_t type)
6811 {
6812 /* get segment from resource handle */
6813 rsmseg_t *seg;
6814 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL);
6815
6816 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmresource_seg enter\n"));
6817
6818
6819 if (res != RSMRC_RESERVED) {
6820 seg = (rsmseg_t *)res;
6821 } else {
6822 /* Allocate segment now and bind it */
6823 seg = rsmseg_alloc(rnum, credp);
6824
6825 /*
6826 * if DR pre-processing is going on or DR is in progress
6827 * then the new export segments should be in the NEW_QSCD state
6828 */
6829 if (type == RSM_RESOURCE_EXPORT_SEGMENT) {
6830 mutex_enter(&rsm_drv_data.drv_lock);
6831 if ((rsm_drv_data.drv_state ==
6832 RSM_DRV_PREDEL_STARTED) ||
6833 (rsm_drv_data.drv_state ==
6834 RSM_DRV_PREDEL_COMPLETED) ||
6835 (rsm_drv_data.drv_state ==
6836 RSM_DRV_DR_IN_PROGRESS)) {
6837 seg->s_state = RSM_STATE_NEW_QUIESCED;
6838 }
6839 mutex_exit(&rsm_drv_data.drv_lock);
6840 }
6841
6842 rsmresource_insert(rnum, (rsmresource_t *)seg, type);
6843 }
6844
6845 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmresource_seg done\n"));
6846
6847 return (seg);
6848 }
6849
6850 static int
6851 rsmexport_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg,
6852 int mode, cred_t *credp)
6853 {
6854 int error;
6855 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT | RSM_IOCTL);
6856
6857 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmexport_ioctl enter\n"));
6858
6859 arg = arg;
6860 credp = credp;
6861
6862 ASSERT(seg != NULL);
6863
6864 switch (cmd) {
6865 case RSM_IOCTL_BIND:
6866 error = rsm_bind(seg, msg, arg, mode);
6867 break;
6868 case RSM_IOCTL_REBIND:
6869 error = rsm_rebind(seg, msg);
6870 break;
6871 case RSM_IOCTL_UNBIND:
6872 error = ENOTSUP;
6873 break;
6874 case RSM_IOCTL_PUBLISH:
6875 error = rsm_publish(seg, msg, arg, mode);
6876 break;
6877 case RSM_IOCTL_REPUBLISH:
6878 error = rsm_republish(seg, msg, mode);
6879 break;
6880 case RSM_IOCTL_UNPUBLISH:
6881 error = rsm_unpublish(seg, 1);
6882 break;
6883 default:
6884 error = EINVAL;
6885 break;
6886 }
6887
6888 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmexport_ioctl done: %d\n",
6889 error));
6890
6891 return (error);
6892 }
6893 static int
6894 rsmimport_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg,
6895 int mode, cred_t *credp)
6896 {
6897 int error;
6898 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
6899
6900 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmimport_ioctl enter\n"));
6901
6902 ASSERT(seg);
6903
6904 switch (cmd) {
6905 case RSM_IOCTL_CONNECT:
6906 error = rsm_connect(seg, msg, credp, arg, mode);
6907 break;
6908 default:
6909 error = EINVAL;
6910 break;
6911 }
6912
6913 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmimport_ioctl done: %d\n",
6914 error));
6915 return (error);
6916 }
6917
6918 static int
6919 rsmbar_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg,
6920 int mode)
6921 {
6922 int e;
6923 adapter_t *adapter;
6924 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
6925
6926 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmbar_ioctl enter\n"));
6927
6928
6929 if ((seg->s_flags & RSM_IMPORT_DUMMY) != 0) {
6930 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6931 "rsmbar_ioctl done: RSM_IMPORT_DUMMY\n"));
6932 return (RSMERR_CONN_ABORTED);
6933 } else if (seg->s_node == my_nodeid) {
6934 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6935 "rsmbar_ioctl done: loopback\n"));
6936 return (RSM_SUCCESS);
6937 }
6938
6939 adapter = seg->s_adapter;
6940
6941 switch (cmd) {
6942 case RSM_IOCTL_BAR_CHECK:
6943 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6944 "rsmbar_ioctl done: RSM_BAR_CHECK %d\n", bar_va));
6945 return (bar_va ? RSM_SUCCESS : EINVAL);
6946 case RSM_IOCTL_BAR_OPEN:
6947 e = adapter->rsmpi_ops->
6948 rsm_open_barrier_ctrl(adapter->rsmpi_handle, &msg->bar);
6949 break;
6950 case RSM_IOCTL_BAR_ORDER:
6951 e = adapter->rsmpi_ops->rsm_order_barrier(&msg->bar);
6952 break;
6953 case RSM_IOCTL_BAR_CLOSE:
6954 e = adapter->rsmpi_ops->rsm_close_barrier(&msg->bar);
6955 break;
6956 default:
6957 e = EINVAL;
6958 break;
6959 }
6960
6961 if (e == RSM_SUCCESS) {
6962 #ifdef _MULTI_DATAMODEL
6963 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
6964 rsm_ioctlmsg32_t msg32;
6965 int i;
6966
6967 for (i = 0; i < 4; i++) {
6968 msg32.bar.comp[i].u64 = msg->bar.comp[i].u64;
6969 }
6970
6971 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6972 "rsmbar_ioctl done\n"));
6973 if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg,
6974 sizeof (msg32), mode))
6975 return (RSMERR_BAD_ADDR);
6976 else
6977 return (RSM_SUCCESS);
6978 }
6979 #endif
6980 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6981 "rsmbar_ioctl done\n"));
6982 if (ddi_copyout((caddr_t)&msg->bar, (caddr_t)arg,
6983 sizeof (*msg), mode))
6984 return (RSMERR_BAD_ADDR);
6985 else
6986 return (RSM_SUCCESS);
6987 }
6988
6989 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
6990 "rsmbar_ioctl done: error=%d\n", e));
6991
6992 return (e);
6993 }
6994
6995 /*
6996 * Ring the doorbell of the export segment to which this segment is
6997 * connected.
6998 */
6999 static int
7000 exportbell_ioctl(rsmseg_t *seg, int cmd /*ARGSUSED*/)
7001 {
7002 int e = 0;
7003 rsmipc_request_t request;
7004
7005 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
7006
7007 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exportbell_ioctl enter\n"));
7008
7009 request.rsmipc_key = seg->s_segid;
7010 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL;
7011 request.rsmipc_segment_cookie = NULL;
7012 e = rsmipc_send(seg->s_node, &request, RSM_NO_REPLY);
7013
7014 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7015 "exportbell_ioctl done: %d\n", e));
7016
7017 return (e);
7018 }
7019
7020 /*
7021 * Ring the doorbells of all segments importing this segment
7022 */
7023 static int
7024 importbell_ioctl(rsmseg_t *seg, int cmd /*ARGSUSED*/)
7025 {
7026 importing_token_t *token = NULL;
7027 rsmipc_request_t request;
7028 int index;
7029
7030 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT | RSM_IOCTL);
7031
7032 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importbell_ioctl enter\n"));
7033
7034 ASSERT(seg->s_state != RSM_STATE_NEW &&
7035 seg->s_state != RSM_STATE_NEW_QUIESCED);
7036
7037 request.rsmipc_key = seg->s_segid;
7038 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL;
7039
7040 index = rsmhash(seg->s_segid);
7041
7042 token = importer_list.bucket[index];
7043
7044 while (token != NULL) {
7045 if (seg->s_key == token->key) {
7046 request.rsmipc_segment_cookie =
7047 token->import_segment_cookie;
7048 (void) rsmipc_send(token->importing_node,
7049 &request, RSM_NO_REPLY);
7050 }
7051 token = token->next;
7052 }
7053
7054 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7055 "importbell_ioctl done\n"));
7056 return (RSM_SUCCESS);
7057 }
7058
7059 static int
7060 rsm_consumeevent_copyin(caddr_t arg, rsm_consume_event_msg_t *msgp,
7061 rsm_poll_event_t **eventspp, int mode)
7062 {
7063 rsm_poll_event_t *evlist = NULL;
7064 size_t evlistsz;
7065 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL);
7066
7067 #ifdef _MULTI_DATAMODEL
7068 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7069 int i;
7070 rsm_consume_event_msg32_t cemsg32 = {0};
7071 rsm_poll_event32_t event32[RSM_MAX_POLLFDS];
7072 rsm_poll_event32_t *evlist32;
7073 size_t evlistsz32;
7074
7075 /* copyin the ioctl message */
7076 if (ddi_copyin(arg, (caddr_t)&cemsg32,
7077 sizeof (rsm_consume_event_msg32_t), mode)) {
7078 DBG_PRINTF((category, RSM_ERR,
7079 "consumeevent_copyin msgp: RSMERR_BAD_ADDR\n"));
7080 return (RSMERR_BAD_ADDR);
7081 }
7082 msgp->seglist = (caddr_t)(uintptr_t)cemsg32.seglist;
7083 msgp->numents = (int)cemsg32.numents;
7084
7085 evlistsz32 = sizeof (rsm_poll_event32_t) * msgp->numents;
7086 /*
7087 * If numents is large alloc events list on heap otherwise
7088 * use the address of array that was passed in.
7089 */
7090 if (msgp->numents > RSM_MAX_POLLFDS) {
7091 if (msgp->numents > max_segs) { /* validate numents */
7092 DBG_PRINTF((category, RSM_ERR,
7093 "consumeevent_copyin: "
7094 "RSMERR_BAD_ARGS_ERRORS\n"));
7095 return (RSMERR_BAD_ARGS_ERRORS);
7096 }
7097 evlist32 = kmem_zalloc(evlistsz32, KM_SLEEP);
7098 } else {
7099 evlist32 = event32;
7100 }
7101
7102 /* copyin the seglist into the rsm_poll_event32_t array */
7103 if (ddi_copyin((caddr_t)msgp->seglist, (caddr_t)evlist32,
7104 evlistsz32, mode)) {
7105 if ((msgp->numents > RSM_MAX_POLLFDS) && evlist32) {
7106 kmem_free(evlist32, evlistsz32);
7107 }
7108 DBG_PRINTF((category, RSM_ERR,
7109 "consumeevent_copyin evlist: RSMERR_BAD_ADDR\n"));
7110 return (RSMERR_BAD_ADDR);
7111 }
7112
7113 /* evlist and evlistsz are based on rsm_poll_event_t type */
7114 evlistsz = sizeof (rsm_poll_event_t)* msgp->numents;
7115
7116 if (msgp->numents > RSM_MAX_POLLFDS) {
7117 evlist = kmem_zalloc(evlistsz, KM_SLEEP);
7118 *eventspp = evlist;
7119 } else {
7120 evlist = *eventspp;
7121 }
7122 /*
7123 * copy the rsm_poll_event32_t array to the rsm_poll_event_t
7124 * array
7125 */
7126 for (i = 0; i < msgp->numents; i++) {
7127 evlist[i].rnum = evlist32[i].rnum;
7128 evlist[i].fdsidx = evlist32[i].fdsidx;
7129 evlist[i].revent = evlist32[i].revent;
7130 }
7131 /* free the temp 32-bit event list */
7132 if ((msgp->numents > RSM_MAX_POLLFDS) && evlist32) {
7133 kmem_free(evlist32, evlistsz32);
7134 }
7135
7136 return (RSM_SUCCESS);
7137 }
7138 #endif
7139 /* copyin the ioctl message */
7140 if (ddi_copyin(arg, (caddr_t)msgp, sizeof (rsm_consume_event_msg_t),
7141 mode)) {
7142 DBG_PRINTF((category, RSM_ERR,
7143 "consumeevent_copyin msgp: RSMERR_BAD_ADDR\n"));
7144 return (RSMERR_BAD_ADDR);
7145 }
7146 /*
7147 * If numents is large alloc events list on heap otherwise
7148 * use the address of array that was passed in.
7149 */
7150 if (msgp->numents > RSM_MAX_POLLFDS) {
7151 if (msgp->numents > max_segs) { /* validate numents */
7152 DBG_PRINTF((category, RSM_ERR,
7153 "consumeevent_copyin: RSMERR_BAD_ARGS_ERRORS\n"));
7154 return (RSMERR_BAD_ARGS_ERRORS);
7155 }
7156 evlistsz = sizeof (rsm_poll_event_t)*msgp->numents;
7157 evlist = kmem_zalloc(evlistsz, KM_SLEEP);
7158 *eventspp = evlist;
7159 }
7160
7161 /* copyin the seglist */
7162 if (ddi_copyin((caddr_t)msgp->seglist, (caddr_t)(*eventspp),
7163 sizeof (rsm_poll_event_t)*msgp->numents, mode)) {
7164 if (evlist) {
7165 kmem_free(evlist, evlistsz);
7166 *eventspp = NULL;
7167 }
7168 DBG_PRINTF((category, RSM_ERR,
7169 "consumeevent_copyin evlist: RSMERR_BAD_ADDR\n"));
7170 return (RSMERR_BAD_ADDR);
7171 }
7172
7173 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7174 "consumeevent_copyin done\n"));
7175 return (RSM_SUCCESS);
7176 }
7177
7178 static int
7179 rsm_consumeevent_copyout(rsm_consume_event_msg_t *msgp,
7180 rsm_poll_event_t *eventsp, int mode)
7181 {
7182 size_t evlistsz;
7183 int err = RSM_SUCCESS;
7184 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL);
7185
7186 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7187 "consumeevent_copyout enter: numents(%d) eventsp(%p)\n",
7188 msgp->numents, eventsp));
7189
7190 #ifdef _MULTI_DATAMODEL
7191 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7192 int i;
7193 rsm_poll_event32_t event32[RSM_MAX_POLLFDS];
7194 rsm_poll_event32_t *evlist32;
7195 size_t evlistsz32;
7196
7197 evlistsz32 = sizeof (rsm_poll_event32_t)*msgp->numents;
7198 if (msgp->numents > RSM_MAX_POLLFDS) {
7199 evlist32 = kmem_zalloc(evlistsz32, KM_SLEEP);
7200 } else {
7201 evlist32 = event32;
7202 }
7203
7204 /*
7205 * copy the rsm_poll_event_t array to the rsm_poll_event32_t
7206 * array
7207 */
7208 for (i = 0; i < msgp->numents; i++) {
7209 evlist32[i].rnum = eventsp[i].rnum;
7210 evlist32[i].fdsidx = eventsp[i].fdsidx;
7211 evlist32[i].revent = eventsp[i].revent;
7212 }
7213
7214 if (ddi_copyout((caddr_t)evlist32, (caddr_t)msgp->seglist,
7215 evlistsz32, mode)) {
7216 err = RSMERR_BAD_ADDR;
7217 }
7218
7219 if (msgp->numents > RSM_MAX_POLLFDS) {
7220 if (evlist32) { /* free the temp 32-bit event list */
7221 kmem_free(evlist32, evlistsz32);
7222 }
7223 /*
7224 * eventsp and evlistsz are based on rsm_poll_event_t
7225 * type
7226 */
7227 evlistsz = sizeof (rsm_poll_event_t)*msgp->numents;
7228 /* event list on the heap and needs to be freed here */
7229 if (eventsp) {
7230 kmem_free(eventsp, evlistsz);
7231 }
7232 }
7233
7234 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7235 "consumeevent_copyout done: err=%d\n", err));
7236 return (err);
7237 }
7238 #endif
7239 evlistsz = sizeof (rsm_poll_event_t)*msgp->numents;
7240
7241 if (ddi_copyout((caddr_t)eventsp, (caddr_t)msgp->seglist, evlistsz,
7242 mode)) {
7243 err = RSMERR_BAD_ADDR;
7244 }
7245
7246 if ((msgp->numents > RSM_MAX_POLLFDS) && eventsp) {
7247 /* event list on the heap and needs to be freed here */
7248 kmem_free(eventsp, evlistsz);
7249 }
7250
7251 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7252 "consumeevent_copyout done: err=%d\n", err));
7253 return (err);
7254 }
7255
7256 static int
7257 rsm_consumeevent_ioctl(caddr_t arg, int mode)
7258 {
7259 int rc;
7260 int i;
7261 minor_t rnum;
7262 rsm_consume_event_msg_t msg = {0};
7263 rsmseg_t *seg;
7264 rsm_poll_event_t *event_list;
7265 rsm_poll_event_t events[RSM_MAX_POLLFDS];
7266 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL);
7267
7268 event_list = events;
7269
7270 if ((rc = rsm_consumeevent_copyin(arg, &msg, &event_list, mode)) !=
7271 RSM_SUCCESS) {
7272 return (rc);
7273 }
7274
7275 for (i = 0; i < msg.numents; i++) {
7276 rnum = event_list[i].rnum;
7277 event_list[i].revent = 0;
7278 /* get the segment structure */
7279 seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_LOCK);
7280 if (seg) {
7281 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7282 "consumeevent_ioctl: rnum(%d) seg(%p)\n", rnum,
7283 seg));
7284 if (seg->s_pollevent) {
7285 /* consume the event */
7286 atomic_dec_32(&seg->s_pollevent);
7287 event_list[i].revent = POLLRDNORM;
7288 }
7289 rsmseglock_release(seg);
7290 }
7291 }
7292
7293 if ((rc = rsm_consumeevent_copyout(&msg, event_list, mode)) !=
7294 RSM_SUCCESS) {
7295 return (rc);
7296 }
7297
7298 return (RSM_SUCCESS);
7299 }
7300
7301 static int
7302 iovec_copyin(caddr_t user_vec, rsmka_iovec_t *iovec, int count, int mode)
7303 {
7304 int size;
7305 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
7306
7307 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "iovec_copyin enter\n"));
7308
7309 #ifdef _MULTI_DATAMODEL
7310 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7311 rsmka_iovec32_t *iovec32, *iovec32_base;
7312 int i;
7313
7314 size = count * sizeof (rsmka_iovec32_t);
7315 iovec32_base = iovec32 = kmem_zalloc(size, KM_SLEEP);
7316 if (ddi_copyin((caddr_t)user_vec,
7317 (caddr_t)iovec32, size, mode)) {
7318 kmem_free(iovec32, size);
7319 DBG_PRINTF((category, RSM_DEBUG,
7320 "iovec_copyin: returning RSMERR_BAD_ADDR\n"));
7321 return (RSMERR_BAD_ADDR);
7322 }
7323
7324 for (i = 0; i < count; i++, iovec++, iovec32++) {
7325 iovec->io_type = (int)iovec32->io_type;
7326 if (iovec->io_type == RSM_HANDLE_TYPE)
7327 iovec->local.segid = (rsm_memseg_id_t)
7328 iovec32->local;
7329 else
7330 iovec->local.vaddr =
7331 (caddr_t)(uintptr_t)iovec32->local;
7332 iovec->local_offset = (size_t)iovec32->local_offset;
7333 iovec->remote_offset = (size_t)iovec32->remote_offset;
7334 iovec->transfer_len = (size_t)iovec32->transfer_len;
7335
7336 }
7337 kmem_free(iovec32_base, size);
7338 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7339 "iovec_copyin done\n"));
7340 return (DDI_SUCCESS);
7341 }
7342 #endif
7343
7344 size = count * sizeof (rsmka_iovec_t);
7345 if (ddi_copyin((caddr_t)user_vec, (caddr_t)iovec, size, mode)) {
7346 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7347 "iovec_copyin done: RSMERR_BAD_ADDR\n"));
7348 return (RSMERR_BAD_ADDR);
7349 }
7350
7351 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "iovec_copyin done\n"));
7352
7353 return (DDI_SUCCESS);
7354 }
7355
7356
7357 static int
7358 sgio_copyin(caddr_t arg, rsmka_scat_gath_t *sg_io, int mode)
7359 {
7360 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
7361
7362 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_copyin enter\n"));
7363
7364 #ifdef _MULTI_DATAMODEL
7365 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7366 rsmka_scat_gath32_t sg_io32;
7367
7368 if (ddi_copyin(arg, (caddr_t)&sg_io32, sizeof (sg_io32),
7369 mode)) {
7370 DBG_PRINTF((category, RSM_DEBUG,
7371 "sgio_copyin done: returning EFAULT\n"));
7372 return (RSMERR_BAD_ADDR);
7373 }
7374 sg_io->local_nodeid = (rsm_node_id_t)sg_io32.local_nodeid;
7375 sg_io->io_request_count = (size_t)sg_io32.io_request_count;
7376 sg_io->io_residual_count = (size_t)sg_io32.io_residual_count;
7377 sg_io->flags = (size_t)sg_io32.flags;
7378 sg_io->remote_handle = (rsm_memseg_import_handle_t)
7379 (uintptr_t)sg_io32.remote_handle;
7380 sg_io->iovec = (rsmka_iovec_t *)(uintptr_t)sg_io32.iovec;
7381 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7382 "sgio_copyin done\n"));
7383 return (DDI_SUCCESS);
7384 }
7385 #endif
7386 if (ddi_copyin(arg, (caddr_t)sg_io, sizeof (rsmka_scat_gath_t),
7387 mode)) {
7388 DBG_PRINTF((category, RSM_DEBUG,
7389 "sgio_copyin done: returning EFAULT\n"));
7390 return (RSMERR_BAD_ADDR);
7391 }
7392 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_copyin done\n"));
7393 return (DDI_SUCCESS);
7394 }
7395
7396 static int
7397 sgio_resid_copyout(caddr_t arg, rsmka_scat_gath_t *sg_io, int mode)
7398 {
7399 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
7400
7401 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7402 "sgio_resid_copyout enter\n"));
7403
7404 #ifdef _MULTI_DATAMODEL
7405 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7406 rsmka_scat_gath32_t sg_io32;
7407
7408 sg_io32.io_residual_count = sg_io->io_residual_count;
7409 sg_io32.flags = sg_io->flags;
7410
7411 if (ddi_copyout((caddr_t)&sg_io32.io_residual_count,
7412 (caddr_t)&((rsmka_scat_gath32_t *)arg)->io_residual_count,
7413 sizeof (uint32_t), mode)) {
7414
7415 DBG_PRINTF((category, RSM_ERR,
7416 "sgio_resid_copyout error: rescnt\n"));
7417 return (RSMERR_BAD_ADDR);
7418 }
7419
7420 if (ddi_copyout((caddr_t)&sg_io32.flags,
7421 (caddr_t)&((rsmka_scat_gath32_t *)arg)->flags,
7422 sizeof (uint32_t), mode)) {
7423
7424 DBG_PRINTF((category, RSM_ERR,
7425 "sgio_resid_copyout error: flags\n"));
7426 return (RSMERR_BAD_ADDR);
7427 }
7428 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7429 "sgio_resid_copyout done\n"));
7430 return (DDI_SUCCESS);
7431 }
7432 #endif
7433 if (ddi_copyout((caddr_t)&sg_io->io_residual_count,
7434 (caddr_t)&((rsmka_scat_gath_t *)arg)->io_residual_count,
7435 sizeof (ulong_t), mode)) {
7436
7437 DBG_PRINTF((category, RSM_ERR,
7438 "sgio_resid_copyout error:rescnt\n"));
7439 return (RSMERR_BAD_ADDR);
7440 }
7441
7442 if (ddi_copyout((caddr_t)&sg_io->flags,
7443 (caddr_t)&((rsmka_scat_gath_t *)arg)->flags,
7444 sizeof (uint_t), mode)) {
7445
7446 DBG_PRINTF((category, RSM_ERR,
7447 "sgio_resid_copyout error:flags\n"));
7448 return (RSMERR_BAD_ADDR);
7449 }
7450
7451 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_resid_copyout done\n"));
7452 return (DDI_SUCCESS);
7453 }
7454
7455
7456 static int
7457 rsm_iovec_ioctl(dev_t dev, caddr_t arg, int cmd, int mode, cred_t *credp)
7458 {
7459 rsmka_scat_gath_t sg_io;
7460 rsmka_iovec_t ka_iovec_arr[RSM_MAX_IOVLEN];
7461 rsmka_iovec_t *ka_iovec;
7462 rsmka_iovec_t *ka_iovec_start;
7463 rsmpi_scat_gath_t rsmpi_sg_io;
7464 rsmpi_iovec_t iovec_arr[RSM_MAX_IOVLEN];
7465 rsmpi_iovec_t *iovec;
7466 rsmpi_iovec_t *iovec_start = NULL;
7467 rsmapi_access_entry_t *acl;
7468 rsmresource_t *res;
7469 minor_t rnum;
7470 rsmseg_t *im_seg, *ex_seg;
7471 int e;
7472 int error = 0;
7473 uint_t i;
7474 uint_t iov_proc = 0; /* num of iovecs processed */
7475 size_t size = 0;
7476 size_t ka_size;
7477
7478 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);
7479
7480 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_iovec_ioctl enter\n"));
7481
7482 credp = credp;
7483
7484 /*
7485 * Copyin the scatter/gather structure and build new structure
7486 * for rsmpi.
7487 */
7488 e = sgio_copyin(arg, &sg_io, mode);
7489 if (e != DDI_SUCCESS) {
7490 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7491 "rsm_iovec_ioctl done: sgio_copyin %d\n", e));
7492 return (e);
7493 }
7494
7495 if (sg_io.io_request_count > RSM_MAX_SGIOREQS) {
7496 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7497 "rsm_iovec_ioctl done: request_count(%d) too large\n",
7498 sg_io.io_request_count));
7499 return (RSMERR_BAD_SGIO);
7500 }
7501
7502 rsmpi_sg_io.io_request_count = sg_io.io_request_count;
7503 rsmpi_sg_io.io_residual_count = sg_io.io_request_count;
7504 rsmpi_sg_io.io_segflg = 0;
7505
7506 /* Allocate memory and copyin io vector array */
7507 if (sg_io.io_request_count > RSM_MAX_IOVLEN) {
7508 ka_size = sg_io.io_request_count * sizeof (rsmka_iovec_t);
7509 ka_iovec_start = ka_iovec = kmem_zalloc(ka_size, KM_SLEEP);
7510 } else {
7511 ka_iovec_start = ka_iovec = ka_iovec_arr;
7512 }
7513 e = iovec_copyin((caddr_t)sg_io.iovec, ka_iovec,
7514 sg_io.io_request_count, mode);
7515 if (e != DDI_SUCCESS) {
7516 if (sg_io.io_request_count > RSM_MAX_IOVLEN)
7517 kmem_free(ka_iovec, ka_size);
7518 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7519 "rsm_iovec_ioctl done: iovec_copyin %d\n", e));
7520 return (e);
7521 }
7522
7523 /* get the import segment descriptor */
7524 rnum = getminor(dev);
7525 res = rsmresource_lookup(rnum, RSM_LOCK);
7526
7527 /*
7528 * The following sequence of locking may (or MAY NOT) cause a
7529 * deadlock but this is currently not addressed here since the
7530 * implementation will be changed to incorporate the use of
7531 * reference counting for both the import and the export segments.
7532 */
7533
7534 /* rsmseglock_acquire(im_seg) done in rsmresource_lookup */
7535
7536 im_seg = (rsmseg_t *)res;
7537
7538 if (im_seg == NULL) {
7539 if (sg_io.io_request_count > RSM_MAX_IOVLEN)
7540 kmem_free(ka_iovec, ka_size);
7541 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7542 "rsm_iovec_ioctl done: rsmresource_lookup failed\n"));
7543 return (EINVAL);
7544 }
7545 /* putv/getv supported is supported only on import segments */
7546 if (im_seg->s_type != RSM_RESOURCE_IMPORT_SEGMENT) {
7547 rsmseglock_release(im_seg);
7548 if (sg_io.io_request_count > RSM_MAX_IOVLEN)
7549 kmem_free(ka_iovec, ka_size);
7550 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7551 "rsm_iovec_ioctl done: not an import segment\n"));
7552 return (EINVAL);
7553 }
7554
7555 /*
7556 * wait for a remote DR to complete ie. for segments to get UNQUIESCED
7557 * as well as wait for a local DR to complete.
7558 */
7559 while ((im_seg->s_state == RSM_STATE_CONN_QUIESCE) ||
7560 (im_seg->s_state == RSM_STATE_MAP_QUIESCE) ||
7561 (im_seg->s_flags & RSM_DR_INPROGRESS)) {
7562 if (cv_wait_sig(&im_seg->s_cv, &im_seg->s_lock) == 0) {
7563 DBG_PRINTF((category, RSM_DEBUG,
7564 "rsm_iovec_ioctl done: cv_wait INTR"));
7565 rsmseglock_release(im_seg);
7566 return (RSMERR_INTERRUPTED);
7567 }
7568 }
7569
7570 if ((im_seg->s_state != RSM_STATE_CONNECT) &&
7571 (im_seg->s_state != RSM_STATE_ACTIVE)) {
7572
7573 ASSERT(im_seg->s_state == RSM_STATE_DISCONNECT ||
7574 im_seg->s_state == RSM_STATE_NEW);
7575
7576 DBG_PRINTF((category, RSM_DEBUG,
7577 "rsm_iovec_ioctl done: im_seg not conn/map"));
7578 rsmseglock_release(im_seg);
7579 e = RSMERR_BAD_SGIO;
7580 goto out;
7581 }
7582
7583 im_seg->s_rdmacnt++;
7584 rsmseglock_release(im_seg);
7585
7586 /*
7587 * Allocate and set up the io vector for rsmpi
7588 */
7589 if (sg_io.io_request_count > RSM_MAX_IOVLEN) {
7590 size = sg_io.io_request_count * sizeof (rsmpi_iovec_t);
7591 iovec_start = iovec = kmem_zalloc(size, KM_SLEEP);
7592 } else {
7593 iovec_start = iovec = iovec_arr;
7594 }
7595
7596 rsmpi_sg_io.iovec = iovec;
7597 for (iov_proc = 0; iov_proc < sg_io.io_request_count; iov_proc++) {
7598 if (ka_iovec->io_type == RSM_HANDLE_TYPE) {
7599 ex_seg = rsmexport_lookup(ka_iovec->local.segid);
7600
7601 if (ex_seg == NULL) {
7602 e = RSMERR_BAD_SGIO;
7603 break;
7604 }
7605 ASSERT(ex_seg->s_state == RSM_STATE_EXPORT);
7606
7607 acl = ex_seg->s_acl;
7608 if (acl[0].ae_permission == 0) {
7609 struct buf *xbuf;
7610 dev_t sdev = 0;
7611
7612 xbuf = ddi_umem_iosetup(ex_seg->s_cookie,
7613 0, ex_seg->s_len, B_WRITE,
7614 sdev, 0, NULL, DDI_UMEM_SLEEP);
7615
7616 ASSERT(xbuf != NULL);
7617
7618 iovec->local_mem.ms_type = RSM_MEM_BUF;
7619 iovec->local_mem.ms_memory.bp = xbuf;
7620 } else {
7621 iovec->local_mem.ms_type = RSM_MEM_HANDLE;
7622 iovec->local_mem.ms_memory.handle =
7623 ex_seg->s_handle.out;
7624 }
7625 ex_seg->s_rdmacnt++; /* refcnt the handle */
7626 rsmseglock_release(ex_seg);
7627 } else {
7628 iovec->local_mem.ms_type = RSM_MEM_VADDR;
7629 iovec->local_mem.ms_memory.vr.vaddr =
7630 ka_iovec->local.vaddr;
7631 }
7632
7633 iovec->local_offset = ka_iovec->local_offset;
7634 iovec->remote_handle = im_seg->s_handle.in;
7635 iovec->remote_offset = ka_iovec->remote_offset;
7636 iovec->transfer_length = ka_iovec->transfer_len;
7637 iovec++;
7638 ka_iovec++;
7639 }
7640
7641 if (iov_proc < sg_io.io_request_count) {
7642 /* error while processing handle */
7643 rsmseglock_acquire(im_seg);
7644 im_seg->s_rdmacnt--; /* decrement the refcnt for importseg */
7645 if (im_seg->s_rdmacnt == 0) {
7646 cv_broadcast(&im_seg->s_cv);
7647 }
7648 rsmseglock_release(im_seg);
7649 goto out;
7650 }
7651
7652 /* call rsmpi */
7653 if (cmd == RSM_IOCTL_PUTV)
7654 e = im_seg->s_adapter->rsmpi_ops->rsm_memseg_import_putv(
7655 im_seg->s_adapter->rsmpi_handle,
7656 &rsmpi_sg_io);
7657 else if (cmd == RSM_IOCTL_GETV)
7658 e = im_seg->s_adapter->rsmpi_ops->rsm_memseg_import_getv(
7659 im_seg->s_adapter->rsmpi_handle,
7660 &rsmpi_sg_io);
7661 else {
7662 e = EINVAL;
7663 DBG_PRINTF((category, RSM_DEBUG,
7664 "iovec_ioctl: bad command = %x\n", cmd));
7665 }
7666
7667
7668 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7669 "rsm_iovec_ioctl RSMPI oper done %d\n", e));
7670
7671 sg_io.io_residual_count = rsmpi_sg_io.io_residual_count;
7672
7673 /*
7674 * Check for implicit signal post flag and do the signal
7675 * post if needed
7676 */
7677 if (sg_io.flags & RSM_IMPLICIT_SIGPOST &&
7678 e == RSM_SUCCESS) {
7679 rsmipc_request_t request;
7680
7681 request.rsmipc_key = im_seg->s_segid;
7682 request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL;
7683 request.rsmipc_segment_cookie = NULL;
7684 e = rsmipc_send(im_seg->s_node, &request, RSM_NO_REPLY);
7685 /*
7686 * Reset the implicit signal post flag to 0 to indicate
7687 * that the signal post has been done and need not be
7688 * done in the RSMAPI library
7689 */
7690 sg_io.flags &= ~RSM_IMPLICIT_SIGPOST;
7691 }
7692
7693 rsmseglock_acquire(im_seg);
7694 im_seg->s_rdmacnt--;
7695 if (im_seg->s_rdmacnt == 0) {
7696 cv_broadcast(&im_seg->s_cv);
7697 }
7698 rsmseglock_release(im_seg);
7699 error = sgio_resid_copyout(arg, &sg_io, mode);
7700 out:
7701 iovec = iovec_start;
7702 ka_iovec = ka_iovec_start;
7703 for (i = 0; i < iov_proc; i++) {
7704 if (ka_iovec->io_type == RSM_HANDLE_TYPE) {
7705 ex_seg = rsmexport_lookup(ka_iovec->local.segid);
7706
7707 ASSERT(ex_seg != NULL);
7708 ASSERT(ex_seg->s_state == RSM_STATE_EXPORT);
7709
7710 ex_seg->s_rdmacnt--; /* unrefcnt the handle */
7711 if (ex_seg->s_rdmacnt == 0) {
7712 cv_broadcast(&ex_seg->s_cv);
7713 }
7714 rsmseglock_release(ex_seg);
7715 }
7716
7717 ASSERT(iovec != NULL); /* true if iov_proc > 0 */
7718
7719 /*
7720 * At present there is no dependency on the existence of xbufs
7721 * created by ddi_umem_iosetup for each of the iovecs. So we
7722 * can these xbufs here.
7723 */
7724 if (iovec->local_mem.ms_type == RSM_MEM_BUF) {
7725 freerbuf(iovec->local_mem.ms_memory.bp);
7726 }
7727
7728 iovec++;
7729 ka_iovec++;
7730 }
7731
7732 if (sg_io.io_request_count > RSM_MAX_IOVLEN) {
7733 if (iovec_start)
7734 kmem_free(iovec_start, size);
7735 kmem_free(ka_iovec_start, ka_size);
7736 }
7737
7738 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7739 "rsm_iovec_ioctl done %d\n", e));
7740 /* if RSMPI call fails return that else return copyout's retval */
7741 return ((e != RSM_SUCCESS) ? e : error);
7742
7743 }
7744
7745
7746 static int
7747 rsmaddr_ioctl(int cmd, rsm_ioctlmsg_t *msg, int mode)
7748 {
7749 adapter_t *adapter;
7750 rsm_addr_t addr;
7751 rsm_node_id_t node;
7752 int rval = DDI_SUCCESS;
7753 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL);
7754
7755 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmaddr_ioctl enter\n"));
7756
7757 adapter = rsm_getadapter(msg, mode);
7758 if (adapter == NULL) {
7759 DBG_PRINTF((category, RSM_DEBUG,
7760 "rsmaddr_ioctl done: adapter not found\n"));
7761 return (RSMERR_CTLR_NOT_PRESENT);
7762 }
7763
7764 switch (cmd) {
7765 case RSM_IOCTL_MAP_TO_ADDR: /* nodeid to hwaddr mapping */
7766 /* returns the hwaddr in msg->hwaddr */
7767 if (msg->nodeid == my_nodeid) {
7768 msg->hwaddr = adapter->hwaddr;
7769 } else {
7770 addr = get_remote_hwaddr(adapter, msg->nodeid);
7771 if ((int64_t)addr < 0) {
7772 rval = RSMERR_INTERNAL_ERROR;
7773 } else {
7774 msg->hwaddr = addr;
7775 }
7776 }
7777 break;
7778 case RSM_IOCTL_MAP_TO_NODEID: /* hwaddr to nodeid mapping */
7779 /* returns the nodeid in msg->nodeid */
7780 if (msg->hwaddr == adapter->hwaddr) {
7781 msg->nodeid = my_nodeid;
7782 } else {
7783 node = get_remote_nodeid(adapter, msg->hwaddr);
7784 if ((int)node < 0) {
7785 rval = RSMERR_INTERNAL_ERROR;
7786 } else {
7787 msg->nodeid = (rsm_node_id_t)node;
7788 }
7789 }
7790 break;
7791 default:
7792 rval = EINVAL;
7793 break;
7794 }
7795
7796 rsmka_release_adapter(adapter);
7797 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7798 "rsmaddr_ioctl done: %d\n", rval));
7799 return (rval);
7800 }
7801
7802 static int
7803 rsm_ddi_copyin(caddr_t arg, rsm_ioctlmsg_t *msg, int mode)
7804 {
7805 DBG_DEFINE(category,
7806 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL | RSM_DDI);
7807
7808 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ddi_copyin enter\n"));
7809
7810 #ifdef _MULTI_DATAMODEL
7811
7812 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7813 rsm_ioctlmsg32_t msg32;
7814 int i;
7815
7816 if (ddi_copyin(arg, (caddr_t)&msg32, sizeof (msg32), mode)) {
7817 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7818 "rsm_ddi_copyin done: EFAULT\n"));
7819 return (RSMERR_BAD_ADDR);
7820 }
7821 msg->len = msg32.len;
7822 msg->vaddr = (caddr_t)(uintptr_t)msg32.vaddr;
7823 msg->arg = (caddr_t)(uintptr_t)msg32.arg;
7824 msg->key = msg32.key;
7825 msg->acl_len = msg32.acl_len;
7826 msg->acl = (rsmapi_access_entry_t *)(uintptr_t)msg32.acl;
7827 msg->cnum = msg32.cnum;
7828 msg->cname = (caddr_t)(uintptr_t)msg32.cname;
7829 msg->cname_len = msg32.cname_len;
7830 msg->nodeid = msg32.nodeid;
7831 msg->hwaddr = msg32.hwaddr;
7832 msg->perm = msg32.perm;
7833 for (i = 0; i < 4; i++) {
7834 msg->bar.comp[i].u64 = msg32.bar.comp[i].u64;
7835 }
7836 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7837 "rsm_ddi_copyin done\n"));
7838 return (RSM_SUCCESS);
7839 }
7840 #endif
7841 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ddi_copyin done\n"));
7842 if (ddi_copyin(arg, (caddr_t)msg, sizeof (*msg), mode))
7843 return (RSMERR_BAD_ADDR);
7844 else
7845 return (RSM_SUCCESS);
7846 }
7847
7848 static int
7849 rsmattr_ddi_copyout(adapter_t *adapter, caddr_t arg, int mode)
7850 {
7851 rsmka_int_controller_attr_t rsm_cattr;
7852 DBG_DEFINE(category,
7853 RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL | RSM_DDI);
7854
7855 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7856 "rsmattr_ddi_copyout enter\n"));
7857 /*
7858 * need to copy appropriate data from rsm_controller_attr_t
7859 * to rsmka_int_controller_attr_t
7860 */
7861 #ifdef _MULTI_DATAMODEL
7862 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
7863 rsmka_int_controller_attr32_t rsm_cattr32;
7864
7865 rsm_cattr32.attr_direct_access_sizes =
7866 adapter->rsm_attr.attr_direct_access_sizes;
7867 rsm_cattr32.attr_atomic_sizes =
7868 adapter->rsm_attr.attr_atomic_sizes;
7869 rsm_cattr32.attr_page_size =
7870 adapter->rsm_attr.attr_page_size;
7871 if (adapter->rsm_attr.attr_max_export_segment_size >
7872 UINT_MAX)
7873 rsm_cattr32.attr_max_export_segment_size =
7874 RSM_MAXSZ_PAGE_ALIGNED;
7875 else
7876 rsm_cattr32.attr_max_export_segment_size =
7877 adapter->rsm_attr.attr_max_export_segment_size;
7878 if (adapter->rsm_attr.attr_tot_export_segment_size >
7879 UINT_MAX)
7880 rsm_cattr32.attr_tot_export_segment_size =
7881 RSM_MAXSZ_PAGE_ALIGNED;
7882 else
7883 rsm_cattr32.attr_tot_export_segment_size =
7884 adapter->rsm_attr.attr_tot_export_segment_size;
7885 if (adapter->rsm_attr.attr_max_export_segments >
7886 UINT_MAX)
7887 rsm_cattr32.attr_max_export_segments =
7888 UINT_MAX;
7889 else
7890 rsm_cattr32.attr_max_export_segments =
7891 adapter->rsm_attr.attr_max_export_segments;
7892 if (adapter->rsm_attr.attr_max_import_map_size >
7893 UINT_MAX)
7894 rsm_cattr32.attr_max_import_map_size =
7895 RSM_MAXSZ_PAGE_ALIGNED;
7896 else
7897 rsm_cattr32.attr_max_import_map_size =
7898 adapter->rsm_attr.attr_max_import_map_size;
7899 if (adapter->rsm_attr.attr_tot_import_map_size >
7900 UINT_MAX)
7901 rsm_cattr32.attr_tot_import_map_size =
7902 RSM_MAXSZ_PAGE_ALIGNED;
7903 else
7904 rsm_cattr32.attr_tot_import_map_size =
7905 adapter->rsm_attr.attr_tot_import_map_size;
7906 if (adapter->rsm_attr.attr_max_import_segments >
7907 UINT_MAX)
7908 rsm_cattr32.attr_max_import_segments =
7909 UINT_MAX;
7910 else
7911 rsm_cattr32.attr_max_import_segments =
7912 adapter->rsm_attr.attr_max_import_segments;
7913 rsm_cattr32.attr_controller_addr =
7914 adapter->rsm_attr.attr_controller_addr;
7915
7916 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7917 "rsmattr_ddi_copyout done\n"));
7918 if (ddi_copyout((caddr_t)&rsm_cattr32, arg,
7919 sizeof (rsmka_int_controller_attr32_t), mode)) {
7920 return (RSMERR_BAD_ADDR);
7921 }
7922 else
7923 return (RSM_SUCCESS);
7924 }
7925 #endif
7926 rsm_cattr.attr_direct_access_sizes =
7927 adapter->rsm_attr.attr_direct_access_sizes;
7928 rsm_cattr.attr_atomic_sizes =
7929 adapter->rsm_attr.attr_atomic_sizes;
7930 rsm_cattr.attr_page_size =
7931 adapter->rsm_attr.attr_page_size;
7932 rsm_cattr.attr_max_export_segment_size =
7933 adapter->rsm_attr.attr_max_export_segment_size;
7934 rsm_cattr.attr_tot_export_segment_size =
7935 adapter->rsm_attr.attr_tot_export_segment_size;
7936 rsm_cattr.attr_max_export_segments =
7937 adapter->rsm_attr.attr_max_export_segments;
7938 rsm_cattr.attr_max_import_map_size =
7939 adapter->rsm_attr.attr_max_import_map_size;
7940 rsm_cattr.attr_tot_import_map_size =
7941 adapter->rsm_attr.attr_tot_import_map_size;
7942 rsm_cattr.attr_max_import_segments =
7943 adapter->rsm_attr.attr_max_import_segments;
7944 rsm_cattr.attr_controller_addr =
7945 adapter->rsm_attr.attr_controller_addr;
7946 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7947 "rsmattr_ddi_copyout done\n"));
7948 if (ddi_copyout((caddr_t)&rsm_cattr, arg,
7949 sizeof (rsmka_int_controller_attr_t), mode)) {
7950 return (RSMERR_BAD_ADDR);
7951 }
7952 else
7953 return (RSM_SUCCESS);
7954 }
7955
7956 /*ARGSUSED*/
7957 static int
7958 rsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
7959 int *rvalp)
7960 {
7961 rsmseg_t *seg;
7962 rsmresource_t *res;
7963 minor_t rnum;
7964 rsm_ioctlmsg_t msg = {0};
7965 int error;
7966 adapter_t *adapter;
7967 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL);
7968
7969 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ioctl enter\n"));
7970
7971 if (cmd == RSM_IOCTL_CONSUMEEVENT) {
7972 error = rsm_consumeevent_ioctl((caddr_t)arg, mode);
7973 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7974 "rsm_ioctl RSM_IOCTL_CONSUMEEVENT done: %d\n", error));
7975 return (error);
7976 }
7977
7978 /* topology cmd does not use the arg common to other cmds */
7979 if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_TOPOLOGY) {
7980 error = rsmka_topology_ioctl((caddr_t)arg, cmd, mode);
7981 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7982 "rsm_ioctl done: %d\n", error));
7983 return (error);
7984 }
7985
7986 if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_IOVEC) {
7987 error = rsm_iovec_ioctl(dev, (caddr_t)arg, cmd, mode, credp);
7988 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7989 "rsm_ioctl done: %d\n", error));
7990 return (error);
7991 }
7992
7993 /*
7994 * try to load arguments
7995 */
7996 if (cmd != RSM_IOCTL_RING_BELL &&
7997 rsm_ddi_copyin((caddr_t)arg, &msg, mode)) {
7998 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
7999 "rsm_ioctl done: EFAULT\n"));
8000 return (RSMERR_BAD_ADDR);
8001 }
8002
8003 if (cmd == RSM_IOCTL_ATTR) {
8004 adapter = rsm_getadapter(&msg, mode);
8005 if (adapter == NULL) {
8006 DBG_PRINTF((category, RSM_DEBUG,
8007 "rsm_ioctl done: ENODEV\n"));
8008 return (RSMERR_CTLR_NOT_PRESENT);
8009 }
8010 error = rsmattr_ddi_copyout(adapter, msg.arg, mode);
8011 rsmka_release_adapter(adapter);
8012 DBG_PRINTF((category, RSM_DEBUG,
8013 "rsm_ioctl:after copyout %d\n", error));
8014 return (error);
8015 }
8016
8017 if (cmd == RSM_IOCTL_BAR_INFO) {
8018 /* Return library off,len of barrier page */
8019 msg.off = barrier_offset;
8020 msg.len = (int)barrier_size;
8021 #ifdef _MULTI_DATAMODEL
8022 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
8023 rsm_ioctlmsg32_t msg32;
8024
8025 if (msg.len > UINT_MAX)
8026 msg.len = RSM_MAXSZ_PAGE_ALIGNED;
8027 else
8028 msg32.len = (int32_t)msg.len;
8029 msg32.off = (int32_t)msg.off;
8030 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8031 "rsm_ioctl done\n"));
8032 if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg,
8033 sizeof (msg32), mode))
8034 return (RSMERR_BAD_ADDR);
8035 else
8036 return (RSM_SUCCESS);
8037 }
8038 #endif
8039 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8040 "rsm_ioctl done\n"));
8041 if (ddi_copyout((caddr_t)&msg, (caddr_t)arg,
8042 sizeof (msg), mode))
8043 return (RSMERR_BAD_ADDR);
8044 else
8045 return (RSM_SUCCESS);
8046 }
8047
8048 if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_MAP_ADDR) {
8049 /* map the nodeid or hwaddr */
8050 error = rsmaddr_ioctl(cmd, &msg, mode);
8051 if (error == RSM_SUCCESS) {
8052 #ifdef _MULTI_DATAMODEL
8053 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
8054 rsm_ioctlmsg32_t msg32;
8055
8056 msg32.hwaddr = (uint64_t)msg.hwaddr;
8057 msg32.nodeid = (uint32_t)msg.nodeid;
8058
8059 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8060 "rsm_ioctl done\n"));
8061 if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg,
8062 sizeof (msg32), mode))
8063 return (RSMERR_BAD_ADDR);
8064 else
8065 return (RSM_SUCCESS);
8066 }
8067 #endif
8068 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8069 "rsm_ioctl done\n"));
8070 if (ddi_copyout((caddr_t)&msg, (caddr_t)arg,
8071 sizeof (msg), mode))
8072 return (RSMERR_BAD_ADDR);
8073 else
8074 return (RSM_SUCCESS);
8075 }
8076 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8077 "rsm_ioctl done: %d\n", error));
8078 return (error);
8079 }
8080
8081 /* Find resource and look it in read mode */
8082 rnum = getminor(dev);
8083 res = rsmresource_lookup(rnum, RSM_NOLOCK);
8084 ASSERT(res != NULL);
8085
8086 /*
8087 * Find command group
8088 */
8089 switch (RSM_IOCTL_CMDGRP(cmd)) {
8090 case RSM_IOCTL_EXPORT_SEG:
8091 /*
8092 * Export list is searched during publish, loopback and
8093 * remote lookup call.
8094 */
8095 seg = rsmresource_seg(res, rnum, credp,
8096 RSM_RESOURCE_EXPORT_SEGMENT);
8097 if (seg->s_type == RSM_RESOURCE_EXPORT_SEGMENT) {
8098 error = rsmexport_ioctl(seg, &msg, cmd, arg, mode,
8099 credp);
8100 } else { /* export ioctl on an import/barrier resource */
8101 error = RSMERR_BAD_SEG_HNDL;
8102 }
8103 break;
8104 case RSM_IOCTL_IMPORT_SEG:
8105 /* Import list is searched during remote unmap call. */
8106 seg = rsmresource_seg(res, rnum, credp,
8107 RSM_RESOURCE_IMPORT_SEGMENT);
8108 if (seg->s_type == RSM_RESOURCE_IMPORT_SEGMENT) {
8109 error = rsmimport_ioctl(seg, &msg, cmd, arg, mode,
8110 credp);
8111 } else { /* import ioctl on an export/barrier resource */
8112 error = RSMERR_BAD_SEG_HNDL;
8113 }
8114 break;
8115 case RSM_IOCTL_BAR:
8116 if (res != RSMRC_RESERVED &&
8117 res->rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT) {
8118 error = rsmbar_ioctl((rsmseg_t *)res, &msg, cmd, arg,
8119 mode);
8120 } else { /* invalid res value */
8121 error = RSMERR_BAD_SEG_HNDL;
8122 }
8123 break;
8124 case RSM_IOCTL_BELL:
8125 if (res != RSMRC_RESERVED) {
8126 if (res->rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT)
8127 error = exportbell_ioctl((rsmseg_t *)res, cmd);
8128 else if (res->rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT)
8129 error = importbell_ioctl((rsmseg_t *)res, cmd);
8130 else /* RSM_RESOURCE_BAR */
8131 error = RSMERR_BAD_SEG_HNDL;
8132 } else { /* invalid res value */
8133 error = RSMERR_BAD_SEG_HNDL;
8134 }
8135 break;
8136 default:
8137 error = EINVAL;
8138 }
8139
8140 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ioctl done: %d\n",
8141 error));
8142 return (error);
8143 }
8144
8145
8146 /* **************************** Segment Mapping Operations ********* */
8147 static rsm_mapinfo_t *
8148 rsm_get_mapinfo(rsmseg_t *seg, off_t off, size_t len, off_t *dev_offset,
8149 size_t *map_len)
8150 {
8151 rsm_mapinfo_t *p;
8152 /*
8153 * Find the correct mapinfo structure to use during the mapping
8154 * from the seg->s_mapinfo list.
8155 * The seg->s_mapinfo list contains in reverse order the mappings
8156 * as returned by the RSMPI rsm_map. In rsm_devmap, we need to
8157 * access the correct entry within this list for the mapping
8158 * requested.
8159 *
8160 * The algorithm for selecting a list entry is as follows:
8161 *
8162 * When start_offset of an entry <= off we have found the entry
8163 * we were looking for. Adjust the dev_offset and map_len (needs
8164 * to be PAGESIZE aligned).
8165 */
8166 p = seg->s_mapinfo;
8167 for (; p; p = p->next) {
8168 if (p->start_offset <= off) {
8169 *dev_offset = p->dev_offset + off - p->start_offset;
8170 *map_len = (len > p->individual_len) ?
8171 p->individual_len : ptob(btopr(len));
8172 return (p);
8173 }
8174 p = p->next;
8175 }
8176
8177 return (NULL);
8178 }
8179
8180 static void
8181 rsm_free_mapinfo(rsm_mapinfo_t *mapinfo)
8182 {
8183 rsm_mapinfo_t *p;
8184
8185 while (mapinfo != NULL) {
8186 p = mapinfo;
8187 mapinfo = mapinfo->next;
8188 kmem_free(p, sizeof (*p));
8189 }
8190 }
8191
8192 static int
8193 rsmmap_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, offset_t off,
8194 size_t len, void **pvtp)
8195 {
8196 rsmcookie_t *p;
8197 rsmresource_t *res;
8198 rsmseg_t *seg;
8199 minor_t rnum;
8200 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
8201
8202 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_map enter\n"));
8203
8204 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8205 "rsmmap_map: dhp = %x\n", dhp));
8206
8207 flags = flags;
8208
8209 rnum = getminor(dev);
8210 res = (rsmresource_t *)rsmresource_lookup(rnum, RSM_NOLOCK);
8211 ASSERT(res != NULL);
8212
8213 seg = (rsmseg_t *)res;
8214
8215 rsmseglock_acquire(seg);
8216
8217 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
8218
8219 /*
8220 * Allocate structure and add cookie to segment list
8221 */
8222 p = kmem_alloc(sizeof (*p), KM_SLEEP);
8223
8224 p->c_dhp = dhp;
8225 p->c_off = off;
8226 p->c_len = len;
8227 p->c_next = seg->s_ckl;
8228 seg->s_ckl = p;
8229
8230 *pvtp = (void *)seg;
8231
8232 rsmseglock_release(seg);
8233
8234 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_map done\n"));
8235 return (DDI_SUCCESS);
8236 }
8237
8238 /*
8239 * Page fault handling is done here. The prerequisite mapping setup
8240 * has been done in rsm_devmap with calls to ddi_devmem_setup or
8241 * ddi_umem_setup
8242 */
8243 static int
8244 rsmmap_access(devmap_cookie_t dhp, void *pvt, offset_t offset, size_t len,
8245 uint_t type, uint_t rw)
8246 {
8247 int e;
8248 rsmseg_t *seg = (rsmseg_t *)pvt;
8249 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
8250
8251 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_access enter\n"));
8252
8253 rsmseglock_acquire(seg);
8254
8255 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
8256
8257 while (seg->s_state == RSM_STATE_MAP_QUIESCE) {
8258 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
8259 DBG_PRINTF((category, RSM_DEBUG,
8260 "rsmmap_access done: cv_wait INTR"));
8261 rsmseglock_release(seg);
8262 return (RSMERR_INTERRUPTED);
8263 }
8264 }
8265
8266 ASSERT(seg->s_state == RSM_STATE_DISCONNECT ||
8267 seg->s_state == RSM_STATE_ACTIVE);
8268
8269 if (seg->s_state == RSM_STATE_DISCONNECT)
8270 seg->s_flags |= RSM_IMPORT_DUMMY;
8271
8272 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8273 "rsmmap_access: dhp = %x\n", dhp));
8274
8275 rsmseglock_release(seg);
8276
8277 if (e = devmap_load(dhp, offset, len, type, rw)) {
8278 DBG_PRINTF((category, RSM_ERR, "devmap_load failed\n"));
8279 }
8280
8281
8282 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_access done\n"));
8283
8284 return (e);
8285 }
8286
8287 static int
8288 rsmmap_dup(devmap_cookie_t dhp, void *oldpvt, devmap_cookie_t new_dhp,
8289 void **newpvt)
8290 {
8291 rsmseg_t *seg = (rsmseg_t *)oldpvt;
8292 rsmcookie_t *p, *old;
8293 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
8294
8295 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_dup enter\n"));
8296
8297 /*
8298 * Same as map, create an entry to hold cookie and add it to
8299 * connect segment list. The oldpvt is a pointer to segment.
8300 * Return segment pointer in newpvt.
8301 */
8302 rsmseglock_acquire(seg);
8303
8304 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
8305
8306 /*
8307 * Find old cookie
8308 */
8309 for (old = seg->s_ckl; old != NULL; old = old->c_next) {
8310 if (old->c_dhp == dhp) {
8311 break;
8312 }
8313 }
8314 if (old == NULL) {
8315 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8316 "rsmmap_dup done: EINVAL\n"));
8317 rsmseglock_release(seg);
8318 return (EINVAL);
8319 }
8320
8321 p = kmem_alloc(sizeof (*p), KM_SLEEP);
8322
8323 p->c_dhp = new_dhp;
8324 p->c_off = old->c_off;
8325 p->c_len = old->c_len;
8326 p->c_next = seg->s_ckl;
8327 seg->s_ckl = p;
8328
8329 *newpvt = (void *)seg;
8330
8331 rsmseglock_release(seg);
8332
8333 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_dup done\n"));
8334
8335 return (DDI_SUCCESS);
8336 }
8337
8338 static void
8339 rsmmap_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off, size_t len,
8340 devmap_cookie_t new_dhp1, void **pvtp1,
8341 devmap_cookie_t new_dhp2, void **pvtp2)
8342 {
8343 /*
8344 * Remove pvtp structure from segment list.
8345 */
8346 rsmseg_t *seg = (rsmseg_t *)pvtp;
8347 int freeflag;
8348
8349 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
8350
8351 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_unmap enter\n"));
8352
8353 off = off; len = len;
8354 pvtp1 = pvtp1; pvtp2 = pvtp2;
8355
8356 rsmseglock_acquire(seg);
8357
8358 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
8359
8360 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8361 "rsmmap_unmap: dhp = %x\n", dhp));
8362 /*
8363 * We can go ahead and remove the dhps even if we are in
8364 * the MAPPING state because the dhps being removed here
8365 * belong to a different mmap and we are holding the segment
8366 * lock.
8367 */
8368 if (new_dhp1 == NULL && new_dhp2 == NULL) {
8369 /* find and remove dhp handle */
8370 rsmcookie_t *tmp, **back = &seg->s_ckl;
8371
8372 while (*back != NULL) {
8373 tmp = *back;
8374 if (tmp->c_dhp == dhp) {
8375 *back = tmp->c_next;
8376 kmem_free(tmp, sizeof (*tmp));
8377 break;
8378 }
8379 back = &tmp->c_next;
8380 }
8381 } else {
8382 DBG_PRINTF((category, RSM_DEBUG_LVL2,
8383 "rsmmap_unmap:parital unmap"
8384 "new_dhp1 %lx, new_dhp2 %lx\n",
8385 (size_t)new_dhp1, (size_t)new_dhp2));
8386 }
8387
8388 /*
8389 * rsmmap_unmap is called for each mapping cookie on the list.
8390 * When the list becomes empty and we are not in the MAPPING
8391 * state then unmap in the rsmpi driver.
8392 */
8393 if ((seg->s_ckl == NULL) && (seg->s_state != RSM_STATE_MAPPING))
8394 (void) rsm_unmap(seg);
8395
8396 if (seg->s_state == RSM_STATE_END && seg->s_ckl == NULL) {
8397 freeflag = 1;
8398 } else {
8399 freeflag = 0;
8400 }
8401
8402 rsmseglock_release(seg);
8403
8404 if (freeflag) {
8405 /* Free the segment structure */
8406 rsmseg_free(seg);
8407 }
8408 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_unmap done\n"));
8409
8410 }
8411
8412 static struct devmap_callback_ctl rsmmap_ops = {
8413 DEVMAP_OPS_REV, /* devmap_ops version number */
8414 rsmmap_map, /* devmap_ops map routine */
8415 rsmmap_access, /* devmap_ops access routine */
8416 rsmmap_dup, /* devmap_ops dup routine */
8417 rsmmap_unmap, /* devmap_ops unmap routine */
8418 };
8419
8420 static int
8421 rsm_devmap(dev_t dev, devmap_cookie_t dhc, offset_t off, size_t len,
8422 size_t *maplen, uint_t model /*ARGSUSED*/)
8423 {
8424 struct devmap_callback_ctl *callbackops = &rsmmap_ops;
8425 int err;
8426 uint_t maxprot;
8427 minor_t rnum;
8428 rsmseg_t *seg;
8429 off_t dev_offset;
8430 size_t cur_len;
8431 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
8432
8433 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_devmap enter\n"));
8434
8435 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8436 "rsm_devmap: off = %lx, len = %lx\n", off, len));
8437 rnum = getminor(dev);
8438 seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_NOLOCK);
8439 ASSERT(seg != NULL);
8440
8441 if (seg->s_hdr.rsmrc_type == RSM_RESOURCE_BAR) {
8442 if ((off == barrier_offset) &&
8443 (len == barrier_size)) {
8444
8445 ASSERT(bar_va != NULL && bar_cookie != NULL);
8446
8447 /*
8448 * The offset argument in devmap_umem_setup represents
8449 * the offset within the kernel memory defined by the
8450 * cookie. We use this offset as barrier_offset.
8451 */
8452 err = devmap_umem_setup(dhc, rsm_dip, NULL, bar_cookie,
8453 barrier_offset, len, PROT_USER|PROT_READ,
8454 DEVMAP_DEFAULTS, 0);
8455
8456 if (err != 0) {
8457 DBG_PRINTF((category, RSM_ERR,
8458 "rsm_devmap done: %d\n", err));
8459 return (RSMERR_MAP_FAILED);
8460 }
8461 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8462 "rsm_devmap done: %d\n", err));
8463
8464 *maplen = barrier_size;
8465
8466 return (err);
8467 } else {
8468 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8469 "rsm_devmap done: %d\n", err));
8470 return (RSMERR_MAP_FAILED);
8471 }
8472 }
8473
8474 ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
8475 ASSERT(seg->s_state == RSM_STATE_MAPPING);
8476
8477 /*
8478 * Make sure we still have permission for the map operation.
8479 */
8480 maxprot = PROT_USER;
8481 if (seg->s_mode & RSM_PERM_READ) {
8482 maxprot |= PROT_READ;
8483 }
8484
8485 if (seg->s_mode & RSM_PERM_WRITE) {
8486 maxprot |= PROT_WRITE;
8487 }
8488
8489 /*
8490 * For each devmap call, rsmmap_map is called. This maintains driver
8491 * private information for the mapping. Thus, if there are multiple
8492 * devmap calls there will be multiple rsmmap_map calls and for each
8493 * call, the mapping information will be stored.
8494 * In case of an error during the processing of the devmap call, error
8495 * will be returned. This error return causes the caller of rsm_devmap
8496 * to undo all the mappings by calling rsmmap_unmap for each one.
8497 * rsmmap_unmap will free up the private information for the requested
8498 * mapping.
8499 */
8500 if (seg->s_node != my_nodeid) {
8501 rsm_mapinfo_t *p;
8502
8503 p = rsm_get_mapinfo(seg, off, len, &dev_offset, &cur_len);
8504 if (p == NULL) {
8505 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8506 "rsm_devmap: incorrect mapping info\n"));
8507 return (RSMERR_MAP_FAILED);
8508 }
8509 err = devmap_devmem_setup(dhc, p->dip,
8510 callbackops, p->dev_register,
8511 dev_offset, cur_len, maxprot,
8512 DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS, 0);
8513
8514 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8515 "rsm_devmap: dip=%lx,dreg=%lu,doff=%lx,"
8516 "off=%lx,len=%lx\n",
8517 p->dip, p->dev_register, dev_offset, off, cur_len));
8518
8519 if (err != 0) {
8520 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8521 "rsm_devmap: devmap_devmem_setup failed %d\n",
8522 err));
8523 return (RSMERR_MAP_FAILED);
8524 }
8525 /* cur_len is always an integral multiple pagesize */
8526 ASSERT((cur_len & (PAGESIZE-1)) == 0);
8527 *maplen = cur_len;
8528 return (err);
8529
8530 } else {
8531 err = devmap_umem_setup(dhc, rsm_dip, callbackops,
8532 seg->s_cookie, off, len, maxprot,
8533 DEVMAP_ALLOW_REMAP|DEVMAP_DEFAULTS, 0);
8534 if (err != 0) {
8535 DBG_PRINTF((category, RSM_DEBUG,
8536 "rsm_devmap: devmap_umem_setup failed %d\n",
8537 err));
8538 return (RSMERR_MAP_FAILED);
8539 }
8540 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8541 "rsm_devmap: loopback done\n"));
8542
8543 *maplen = ptob(btopr(len));
8544
8545 return (err);
8546 }
8547 }
8548
8549 /*
8550 * We can use the devmap framework for mapping device memory to user space by
8551 * specifying this routine in the rsm_cb_ops structure. The kernel mmap
8552 * processing calls this entry point and devmap_setup is called within this
8553 * function, which eventually calls rsm_devmap
8554 */
8555 static int
8556 rsm_segmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len,
8557 uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
8558 {
8559 int error = 0;
8560 int old_state;
8561 minor_t rnum;
8562 rsmseg_t *seg, *eseg;
8563 adapter_t *adapter;
8564 rsm_import_share_t *sharedp;
8565 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);
8566
8567 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_segmap enter\n"));
8568
8569 /*
8570 * find segment
8571 */
8572 rnum = getminor(dev);
8573 seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_LOCK);
8574
8575 if (seg == NULL) {
8576 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8577 "rsm_segmap done: invalid segment\n"));
8578 return (EINVAL);
8579 }
8580
8581 /*
8582 * the user is trying to map a resource that has not been
8583 * defined yet. The library uses this to map in the
8584 * barrier page.
8585 */
8586 if (seg->s_hdr.rsmrc_type == RSM_RESOURCE_BAR) {
8587 rsmseglock_release(seg);
8588
8589 /*
8590 * The mapping for the barrier page is identified
8591 * by the special offset barrier_offset
8592 */
8593
8594 if (off == (off_t)barrier_offset ||
8595 len == (off_t)barrier_size) {
8596 if (bar_cookie == NULL || bar_va == NULL) {
8597 DBG_PRINTF((category, RSM_DEBUG,
8598 "rsm_segmap: bar cookie/va is NULL\n"));
8599 return (EINVAL);
8600 }
8601
8602 error = devmap_setup(dev, (offset_t)off, as, addrp,
8603 (size_t)len, prot, maxprot, flags, cred);
8604
8605 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8606 "rsm_segmap done: %d\n", error));
8607 return (error);
8608 } else {
8609 DBG_PRINTF((category, RSM_DEBUG,
8610 "rsm_segmap: bad offset/length\n"));
8611 return (EINVAL);
8612 }
8613 }
8614
8615 /* Make sure you can only map imported segments */
8616 if (seg->s_hdr.rsmrc_type != RSM_RESOURCE_IMPORT_SEGMENT) {
8617 rsmseglock_release(seg);
8618 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8619 "rsm_segmap done: not an import segment\n"));
8620 return (EINVAL);
8621 }
8622 /* check means library is broken */
8623 ASSERT(seg->s_hdr.rsmrc_num == rnum);
8624
8625 /* wait for the segment to become unquiesced */
8626 while (seg->s_state == RSM_STATE_CONN_QUIESCE) {
8627 if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
8628 rsmseglock_release(seg);
8629 DBG_PRINTF((category, RSM_DEBUG,
8630 "rsm_segmap done: cv_wait INTR"));
8631 return (ENODEV);
8632 }
8633 }
8634
8635 /* wait until segment leaves the mapping state */
8636 while (seg->s_state == RSM_STATE_MAPPING)
8637 cv_wait(&seg->s_cv, &seg->s_lock);
8638
8639 /*
8640 * we allow multiple maps of the same segment in the KA
8641 * and it works because we do an rsmpi map of the whole
8642 * segment during the first map and all the device mapping
8643 * information needed in rsm_devmap is in the mapinfo list.
8644 */
8645 if ((seg->s_state != RSM_STATE_CONNECT) &&
8646 (seg->s_state != RSM_STATE_ACTIVE)) {
8647 rsmseglock_release(seg);
8648 DBG_PRINTF((category, RSM_DEBUG,
8649 "rsm_segmap done: segment not connected\n"));
8650 return (ENODEV);
8651 }
8652
8653 /*
8654 * Make sure we are not mapping a larger segment than what's
8655 * exported
8656 */
8657 if ((size_t)off + ptob(btopr(len)) > seg->s_len) {
8658 rsmseglock_release(seg);
8659 DBG_PRINTF((category, RSM_DEBUG,
8660 "rsm_segmap done: off+len>seg size\n"));
8661 return (ENXIO);
8662 }
8663
8664 /*
8665 * Make sure we still have permission for the map operation.
8666 */
8667 maxprot = PROT_USER;
8668 if (seg->s_mode & RSM_PERM_READ) {
8669 maxprot |= PROT_READ;
8670 }
8671
8672 if (seg->s_mode & RSM_PERM_WRITE) {
8673 maxprot |= PROT_WRITE;
8674 }
8675
8676 if ((prot & maxprot) != prot) {
8677 /* No permission */
8678 rsmseglock_release(seg);
8679 DBG_PRINTF((category, RSM_DEBUG,
8680 "rsm_segmap done: no permission\n"));
8681 return (EACCES);
8682 }
8683
8684 old_state = seg->s_state;
8685
8686 ASSERT(seg->s_share != NULL);
8687
8688 rsmsharelock_acquire(seg);
8689
8690 sharedp = seg->s_share;
8691
8692 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
8693 "rsm_segmap:RSMSI_STATE=%d\n", sharedp->rsmsi_state));
8694
8695 if ((sharedp->rsmsi_state != RSMSI_STATE_CONNECTED) &&
8696 (sharedp->rsmsi_state != RSMSI_STATE_MAPPED)) {
8697 rsmsharelock_release(seg);
8698 rsmseglock_release(seg);
8699 DBG_PRINTF((category, RSM_DEBUG,
8700 "rsm_segmap done:RSMSI_STATE %d invalid\n",
8701 sharedp->rsmsi_state));
8702 return (ENODEV);
8703 }
8704
8705 /*
8706 * Do the map - since we want importers to share mappings
8707 * we do the rsmpi map for the whole segment
8708 */
8709 if (seg->s_node != my_nodeid) {
8710 uint_t dev_register;
8711 off_t dev_offset;
8712 dev_info_t *dip;
8713 size_t tmp_len;
8714 size_t total_length_mapped = 0;
8715 size_t length_to_map = seg->s_len;
8716 off_t tmp_off = 0;
8717 rsm_mapinfo_t *p;
8718
8719 /*
8720 * length_to_map = seg->s_len is always an integral
8721 * multiple of PAGESIZE. Length mapped in each entry in mapinfo
8722 * list is a multiple of PAGESIZE - RSMPI map ensures this
8723 */
8724
8725 adapter = seg->s_adapter;
8726 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_CONNECTED ||
8727 sharedp->rsmsi_state == RSMSI_STATE_MAPPED);
8728
8729 if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) {
8730 error = 0;
8731 /* map the whole segment */
8732 while (total_length_mapped < seg->s_len) {
8733 tmp_len = 0;
8734
8735 error = adapter->rsmpi_ops->rsm_map(
8736 seg->s_handle.in, tmp_off,
8737 length_to_map, &tmp_len,
8738 &dip, &dev_register, &dev_offset,
8739 NULL, NULL);
8740
8741 if (error != 0)
8742 break;
8743
8744 /*
8745 * Store the mapping info obtained from rsm_map
8746 */
8747 p = kmem_alloc(sizeof (*p), KM_SLEEP);
8748 p->dev_register = dev_register;
8749 p->dev_offset = dev_offset;
8750 p->dip = dip;
8751 p->individual_len = tmp_len;
8752 p->start_offset = tmp_off;
8753 p->next = sharedp->rsmsi_mapinfo;
8754 sharedp->rsmsi_mapinfo = p;
8755
8756 total_length_mapped += tmp_len;
8757 length_to_map -= tmp_len;
8758 tmp_off += tmp_len;
8759 }
8760 seg->s_mapinfo = sharedp->rsmsi_mapinfo;
8761
8762 if (error != RSM_SUCCESS) {
8763 /* Check if this is the the first rsm_map */
8764 if (sharedp->rsmsi_mapinfo != NULL) {
8765 /*
8766 * A single rsm_unmap undoes
8767 * multiple rsm_maps.
8768 */
8769 (void) seg->s_adapter->rsmpi_ops->
8770 rsm_unmap(sharedp->rsmsi_handle);
8771 rsm_free_mapinfo(sharedp->
8772 rsmsi_mapinfo);
8773 }
8774 sharedp->rsmsi_mapinfo = NULL;
8775 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
8776 rsmsharelock_release(seg);
8777 rsmseglock_release(seg);
8778 DBG_PRINTF((category, RSM_DEBUG,
8779 "rsm_segmap done: rsmpi map err %d\n",
8780 error));
8781 ASSERT(error != RSMERR_BAD_LENGTH &&
8782 error != RSMERR_BAD_MEM_ALIGNMENT &&
8783 error != RSMERR_BAD_SEG_HNDL);
8784 if (error == RSMERR_UNSUPPORTED_OPERATION)
8785 return (ENOTSUP);
8786 else if (error == RSMERR_INSUFFICIENT_RESOURCES)
8787 return (EAGAIN);
8788 else if (error == RSMERR_CONN_ABORTED)
8789 return (ENODEV);
8790 else
8791 return (error);
8792 } else {
8793 sharedp->rsmsi_state = RSMSI_STATE_MAPPED;
8794 }
8795 } else {
8796 seg->s_mapinfo = sharedp->rsmsi_mapinfo;
8797 }
8798
8799 sharedp->rsmsi_mapcnt++;
8800
8801 rsmsharelock_release(seg);
8802
8803 /* move to an intermediate mapping state */
8804 seg->s_state = RSM_STATE_MAPPING;
8805 rsmseglock_release(seg);
8806
8807 error = devmap_setup(dev, (offset_t)off, as, addrp,
8808 len, prot, maxprot, flags, cred);
8809
8810 rsmseglock_acquire(seg);
8811 ASSERT(seg->s_state == RSM_STATE_MAPPING);
8812
8813 if (error == DDI_SUCCESS) {
8814 seg->s_state = RSM_STATE_ACTIVE;
8815 } else {
8816 rsmsharelock_acquire(seg);
8817
8818 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED);
8819
8820 sharedp->rsmsi_mapcnt--;
8821 if (sharedp->rsmsi_mapcnt == 0) {
8822 /* unmap the shared RSMPI mapping */
8823 ASSERT(sharedp->rsmsi_handle != NULL);
8824 (void) adapter->rsmpi_ops->
8825 rsm_unmap(sharedp->rsmsi_handle);
8826 rsm_free_mapinfo(sharedp->rsmsi_mapinfo);
8827 sharedp->rsmsi_mapinfo = NULL;
8828 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
8829 }
8830
8831 rsmsharelock_release(seg);
8832 seg->s_state = old_state;
8833 DBG_PRINTF((category, RSM_ERR,
8834 "rsm: devmap_setup failed %d\n", error));
8835 }
8836 cv_broadcast(&seg->s_cv);
8837 rsmseglock_release(seg);
8838 DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsm_segmap done: %d\n",
8839 error));
8840 return (error);
8841 } else {
8842 /*
8843 * For loopback, the export segment mapping cookie (s_cookie)
8844 * is also used as the s_cookie value for its import segments
8845 * during mapping.
8846 * Note that reference counting for s_cookie of the export
8847 * segment is not required due to the following:
8848 * We never have a case of the export segment being destroyed,
8849 * leaving the import segments with a stale value for the
8850 * s_cookie field, since a force disconnect is done prior to a
8851 * destroy of an export segment. The force disconnect causes
8852 * the s_cookie value to be reset to NULL. Also for the
8853 * rsm_rebind operation, we change the s_cookie value of the
8854 * export segment as well as of all its local (loopback)
8855 * importers.
8856 */
8857 DBG_ADDCATEGORY(category, RSM_LOOPBACK);
8858
8859 rsmsharelock_release(seg);
8860 /*
8861 * In order to maintain the lock ordering between the export
8862 * and import segment locks, we need to acquire the export
8863 * segment lock first and only then acquire the import
8864 * segment lock.
8865 * The above is necessary to avoid any deadlock scenarios
8866 * with rsm_rebind which also acquires both the export
8867 * and import segment locks in the above mentioned order.
8868 * Based on code inspection, there seem to be no other
8869 * situations in which both the export and import segment
8870 * locks are acquired either in the same or opposite order
8871 * as mentioned above.
8872 * Thus in order to conform to the above lock order, we
8873 * need to change the state of the import segment to
8874 * RSM_STATE_MAPPING, release the lock. Once this is done we
8875 * can now safely acquire the export segment lock first
8876 * followed by the import segment lock which is as per
8877 * the lock order mentioned above.
8878 */
8879 /* move to an intermediate mapping state */
8880 seg->s_state = RSM_STATE_MAPPING;
8881 rsmseglock_release(seg);
8882
8883 eseg = rsmexport_lookup(seg->s_key);
8884
8885 if (eseg == NULL) {
8886 rsmseglock_acquire(seg);
8887 /*
8888 * Revert to old_state and signal any waiters
8889 * The shared state is not changed
8890 */
8891
8892 seg->s_state = old_state;
8893 cv_broadcast(&seg->s_cv);
8894 rsmseglock_release(seg);
8895 DBG_PRINTF((category, RSM_DEBUG,
8896 "rsm_segmap done: key %d not found\n", seg->s_key));
8897 return (ENODEV);
8898 }
8899
8900 rsmsharelock_acquire(seg);
8901 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_CONNECTED ||
8902 sharedp->rsmsi_state == RSMSI_STATE_MAPPED);
8903
8904 sharedp->rsmsi_mapcnt++;
8905 sharedp->rsmsi_state = RSMSI_STATE_MAPPED;
8906 rsmsharelock_release(seg);
8907
8908 ASSERT(eseg->s_cookie != NULL);
8909
8910 /*
8911 * It is not required or necessary to acquire the import
8912 * segment lock here to change the value of s_cookie since
8913 * no one will touch the import segment as long as it is
8914 * in the RSM_STATE_MAPPING state.
8915 */
8916 seg->s_cookie = eseg->s_cookie;
8917
8918 rsmseglock_release(eseg);
8919
8920 error = devmap_setup(dev, (offset_t)off, as, addrp, (size_t)len,
8921 prot, maxprot, flags, cred);
8922
8923 rsmseglock_acquire(seg);
8924 ASSERT(seg->s_state == RSM_STATE_MAPPING);
8925 if (error == 0) {
8926 seg->s_state = RSM_STATE_ACTIVE;
8927 } else {
8928 rsmsharelock_acquire(seg);
8929
8930 ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED);
8931
8932 sharedp->rsmsi_mapcnt--;
8933 if (sharedp->rsmsi_mapcnt == 0) {
8934 sharedp->rsmsi_mapinfo = NULL;
8935 sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
8936 }
8937 rsmsharelock_release(seg);
8938 seg->s_state = old_state;
8939 seg->s_cookie = NULL;
8940 }
8941 cv_broadcast(&seg->s_cv);
8942 rsmseglock_release(seg);
8943 DBG_PRINTF((category, RSM_DEBUG_LVL2,
8944 "rsm_segmap done: %d\n", error));
8945 return (error);
8946 }
8947 }
8948
8949 int
8950 rsmka_null_seg_create(
8951 rsm_controller_handle_t argcp,
8952 rsm_memseg_export_handle_t *handle,
8953 size_t size,
8954 uint_t flags,
8955 rsm_memory_local_t *memory,
8956 rsm_resource_callback_t callback,
8957 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/)
8958 {
8959 return (RSM_SUCCESS);
8960 }
8961
8962
8963 int
8964 rsmka_null_seg_destroy(
8965 rsm_memseg_export_handle_t argmemseg /*ARGSUSED*/)
8966 {
8967 return (RSM_SUCCESS);
8968 }
8969
8970
8971 int
8972 rsmka_null_bind(
8973 rsm_memseg_export_handle_t argmemseg,
8974 off_t offset,
8975 rsm_memory_local_t *argmemory,
8976 rsm_resource_callback_t callback,
8977 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/)
8978 {
8979 return (RSM_SUCCESS);
8980 }
8981
8982
8983 int
8984 rsmka_null_unbind(
8985 rsm_memseg_export_handle_t argmemseg,
8986 off_t offset,
8987 size_t length /*ARGSUSED*/)
8988 {
8989 return (DDI_SUCCESS);
8990 }
8991
8992 int
8993 rsmka_null_rebind(
8994 rsm_memseg_export_handle_t argmemseg,
8995 off_t offset,
8996 rsm_memory_local_t *memory,
8997 rsm_resource_callback_t callback,
8998 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/)
8999 {
9000 return (RSM_SUCCESS);
9001 }
9002
9003 int
9004 rsmka_null_publish(
9005 rsm_memseg_export_handle_t argmemseg,
9006 rsm_access_entry_t access_list[],
9007 uint_t access_list_length,
9008 rsm_memseg_id_t segment_id,
9009 rsm_resource_callback_t callback,
9010 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/)
9011 {
9012 return (RSM_SUCCESS);
9013 }
9014
9015
9016 int
9017 rsmka_null_republish(
9018 rsm_memseg_export_handle_t memseg,
9019 rsm_access_entry_t access_list[],
9020 uint_t access_list_length,
9021 rsm_resource_callback_t callback,
9022 rsm_resource_callback_arg_t callback_arg /*ARGSUSED*/)
9023 {
9024 return (RSM_SUCCESS);
9025 }
9026
9027 int
9028 rsmka_null_unpublish(
9029 rsm_memseg_export_handle_t argmemseg /*ARGSUSED*/)
9030 {
9031 return (RSM_SUCCESS);
9032 }
9033
9034
9035 void
9036 rsmka_init_loopback()
9037 {
9038 rsm_ops_t *ops = &null_rsmpi_ops;
9039 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_LOOPBACK);
9040
9041 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9042 "rsmka_init_loopback enter\n"));
9043
9044 /* initialize null ops vector */
9045 ops->rsm_seg_create = rsmka_null_seg_create;
9046 ops->rsm_seg_destroy = rsmka_null_seg_destroy;
9047 ops->rsm_bind = rsmka_null_bind;
9048 ops->rsm_unbind = rsmka_null_unbind;
9049 ops->rsm_rebind = rsmka_null_rebind;
9050 ops->rsm_publish = rsmka_null_publish;
9051 ops->rsm_unpublish = rsmka_null_unpublish;
9052 ops->rsm_republish = rsmka_null_republish;
9053
9054 /* initialize attributes for loopback adapter */
9055 loopback_attr.attr_name = loopback_str;
9056 loopback_attr.attr_page_size = 0x8; /* 8K */
9057
9058 /* initialize loopback adapter */
9059 loopback_adapter.rsm_attr = loopback_attr;
9060 loopback_adapter.rsmpi_ops = &null_rsmpi_ops;
9061 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9062 "rsmka_init_loopback done\n"));
9063 }
9064
9065 /* ************** DR functions ********************************** */
9066 static void
9067 rsm_quiesce_exp_seg(rsmresource_t *resp)
9068 {
9069 int recheck_state;
9070 rsmseg_t *segp = (rsmseg_t *)resp;
9071 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9072 DBG_DEFINE_STR(function, "rsm_unquiesce_exp_seg");
9073
9074 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9075 "%s enter: key=%u\n", function, segp->s_key));
9076
9077 rsmseglock_acquire(segp);
9078 do {
9079 recheck_state = 0;
9080 if ((segp->s_state == RSM_STATE_NEW_QUIESCED) ||
9081 (segp->s_state == RSM_STATE_BIND_QUIESCED) ||
9082 (segp->s_state == RSM_STATE_EXPORT_QUIESCING) ||
9083 (segp->s_state == RSM_STATE_EXPORT_QUIESCED)) {
9084 rsmseglock_release(segp);
9085 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9086 "%s done:state =%d\n", function,
9087 segp->s_state));
9088 return;
9089 }
9090
9091 if (segp->s_state == RSM_STATE_NEW) {
9092 segp->s_state = RSM_STATE_NEW_QUIESCED;
9093 rsmseglock_release(segp);
9094 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9095 "%s done:state =%d\n", function,
9096 segp->s_state));
9097 return;
9098 }
9099
9100 if (segp->s_state == RSM_STATE_BIND) {
9101 /* unbind */
9102 (void) rsm_unbind_pages(segp);
9103 segp->s_state = RSM_STATE_BIND_QUIESCED;
9104 rsmseglock_release(segp);
9105 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9106 "%s done:state =%d\n", function,
9107 segp->s_state));
9108 return;
9109 }
9110
9111 if (segp->s_state == RSM_STATE_EXPORT) {
9112 /*
9113 * wait for putv/getv to complete if the segp is
9114 * a local memory handle
9115 */
9116 while ((segp->s_state == RSM_STATE_EXPORT) &&
9117 (segp->s_rdmacnt != 0)) {
9118 cv_wait(&segp->s_cv, &segp->s_lock);
9119 }
9120
9121 if (segp->s_state != RSM_STATE_EXPORT) {
9122 /*
9123 * state changed need to see what it
9124 * should be changed to.
9125 */
9126 recheck_state = 1;
9127 continue;
9128 }
9129
9130 segp->s_state = RSM_STATE_EXPORT_QUIESCING;
9131 rsmseglock_release(segp);
9132 /*
9133 * send SUSPEND messages - currently it will be
9134 * done at the end
9135 */
9136 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9137 "%s done:state =%d\n", function,
9138 segp->s_state));
9139 return;
9140 }
9141 } while (recheck_state);
9142
9143 rsmseglock_release(segp);
9144 }
9145
9146 static void
9147 rsm_unquiesce_exp_seg(rsmresource_t *resp)
9148 {
9149 int ret;
9150 rsmseg_t *segp = (rsmseg_t *)resp;
9151 rsmapi_access_entry_t *acl;
9152 rsm_access_entry_t *rsmpi_acl;
9153 int acl_len;
9154 int create_flags = 0;
9155 struct buf *xbuf;
9156 rsm_memory_local_t mem;
9157 adapter_t *adapter;
9158 dev_t sdev = 0;
9159 rsm_resource_callback_t callback_flag;
9160 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9161 DBG_DEFINE_STR(function, "rsm_unquiesce_exp_seg");
9162
9163 rsmseglock_acquire(segp);
9164
9165 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9166 "%s enter: key=%u, state=%d\n", function, segp->s_key,
9167 segp->s_state));
9168
9169 if ((segp->s_state == RSM_STATE_NEW) ||
9170 (segp->s_state == RSM_STATE_BIND) ||
9171 (segp->s_state == RSM_STATE_EXPORT)) {
9172 rsmseglock_release(segp);
9173 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done:state=%d\n",
9174 function, segp->s_state));
9175 return;
9176 }
9177
9178 if (segp->s_state == RSM_STATE_NEW_QUIESCED) {
9179 segp->s_state = RSM_STATE_NEW;
9180 cv_broadcast(&segp->s_cv);
9181 rsmseglock_release(segp);
9182 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done:state=%d\n",
9183 function, segp->s_state));
9184 return;
9185 }
9186
9187 if (segp->s_state == RSM_STATE_BIND_QUIESCED) {
9188 /* bind the segment */
9189 ret = rsm_bind_pages(&segp->s_cookie, segp->s_region.r_vaddr,
9190 segp->s_len, segp->s_proc);
9191 if (ret == RSM_SUCCESS) { /* bind successful */
9192 segp->s_state = RSM_STATE_BIND;
9193 } else { /* bind failed - resource unavailable */
9194 segp->s_state = RSM_STATE_NEW;
9195 }
9196 cv_broadcast(&segp->s_cv);
9197 rsmseglock_release(segp);
9198 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9199 "%s done: bind_qscd bind = %d\n", function, ret));
9200 return;
9201 }
9202
9203 while (segp->s_state == RSM_STATE_EXPORT_QUIESCING) {
9204 /* wait for the segment to move to EXPORT_QUIESCED state */
9205 cv_wait(&segp->s_cv, &segp->s_lock);
9206 }
9207
9208 if (segp->s_state == RSM_STATE_EXPORT_QUIESCED) {
9209 /* bind the segment */
9210 ret = rsm_bind_pages(&segp->s_cookie, segp->s_region.r_vaddr,
9211 segp->s_len, segp->s_proc);
9212
9213 if (ret != RSM_SUCCESS) {
9214 /* bind failed - resource unavailable */
9215 acl_len = segp->s_acl_len;
9216 acl = segp->s_acl;
9217 rsmpi_acl = segp->s_acl_in;
9218 segp->s_acl_len = 0;
9219 segp->s_acl = NULL;
9220 segp->s_acl_in = NULL;
9221 rsmseglock_release(segp);
9222
9223 rsmexport_rm(segp);
9224 rsmacl_free(acl, acl_len);
9225 rsmpiacl_free(rsmpi_acl, acl_len);
9226
9227 rsmseglock_acquire(segp);
9228 segp->s_state = RSM_STATE_NEW;
9229 cv_broadcast(&segp->s_cv);
9230 rsmseglock_release(segp);
9231 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9232 "%s done: exp_qscd bind failed = %d\n",
9233 function, ret));
9234 return;
9235 }
9236 /*
9237 * publish the segment
9238 * if successful
9239 * segp->s_state = RSM_STATE_EXPORT;
9240 * else failed
9241 * segp->s_state = RSM_STATE_BIND;
9242 */
9243
9244 /* check whether it is a local_memory_handle */
9245 if (segp->s_acl != (rsmapi_access_entry_t *)NULL) {
9246 if ((segp->s_acl[0].ae_node == my_nodeid) &&
9247 (segp->s_acl[0].ae_permission == 0)) {
9248 segp->s_state = RSM_STATE_EXPORT;
9249 cv_broadcast(&segp->s_cv);
9250 rsmseglock_release(segp);
9251 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9252 "%s done:exp_qscd\n", function));
9253 return;
9254 }
9255 }
9256 xbuf = ddi_umem_iosetup(segp->s_cookie, 0, segp->s_len, B_WRITE,
9257 sdev, 0, NULL, DDI_UMEM_SLEEP);
9258 ASSERT(xbuf != NULL);
9259
9260 mem.ms_type = RSM_MEM_BUF;
9261 mem.ms_bp = xbuf;
9262
9263 adapter = segp->s_adapter;
9264
9265 if (segp->s_flags & RSMKA_ALLOW_UNBIND_REBIND) {
9266 create_flags = RSM_ALLOW_UNBIND_REBIND;
9267 }
9268
9269 if (segp->s_flags & RSMKA_SET_RESOURCE_DONTWAIT) {
9270 callback_flag = RSM_RESOURCE_DONTWAIT;
9271 } else {
9272 callback_flag = RSM_RESOURCE_SLEEP;
9273 }
9274
9275 ret = adapter->rsmpi_ops->rsm_seg_create(
9276 adapter->rsmpi_handle, &segp->s_handle.out,
9277 segp->s_len, create_flags, &mem,
9278 callback_flag, NULL);
9279
9280 if (ret != RSM_SUCCESS) {
9281 acl_len = segp->s_acl_len;
9282 acl = segp->s_acl;
9283 rsmpi_acl = segp->s_acl_in;
9284 segp->s_acl_len = 0;
9285 segp->s_acl = NULL;
9286 segp->s_acl_in = NULL;
9287 rsmseglock_release(segp);
9288
9289 rsmexport_rm(segp);
9290 rsmacl_free(acl, acl_len);
9291 rsmpiacl_free(rsmpi_acl, acl_len);
9292
9293 rsmseglock_acquire(segp);
9294 segp->s_state = RSM_STATE_BIND;
9295 cv_broadcast(&segp->s_cv);
9296 rsmseglock_release(segp);
9297 DBG_PRINTF((category, RSM_ERR,
9298 "%s done: exp_qscd create failed = %d\n",
9299 function, ret));
9300 return;
9301 }
9302
9303 ret = adapter->rsmpi_ops->rsm_publish(
9304 segp->s_handle.out, segp->s_acl_in, segp->s_acl_len,
9305 segp->s_segid, RSM_RESOURCE_DONTWAIT, NULL);
9306
9307 if (ret != RSM_SUCCESS) {
9308 acl_len = segp->s_acl_len;
9309 acl = segp->s_acl;
9310 rsmpi_acl = segp->s_acl_in;
9311 segp->s_acl_len = 0;
9312 segp->s_acl = NULL;
9313 segp->s_acl_in = NULL;
9314 adapter->rsmpi_ops->rsm_seg_destroy(segp->s_handle.out);
9315 rsmseglock_release(segp);
9316
9317 rsmexport_rm(segp);
9318 rsmacl_free(acl, acl_len);
9319 rsmpiacl_free(rsmpi_acl, acl_len);
9320
9321 rsmseglock_acquire(segp);
9322 segp->s_state = RSM_STATE_BIND;
9323 cv_broadcast(&segp->s_cv);
9324 rsmseglock_release(segp);
9325 DBG_PRINTF((category, RSM_ERR,
9326 "%s done: exp_qscd publish failed = %d\n",
9327 function, ret));
9328 return;
9329 }
9330
9331 segp->s_state = RSM_STATE_EXPORT;
9332 cv_broadcast(&segp->s_cv);
9333 rsmseglock_release(segp);
9334 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done: exp_qscd\n",
9335 function));
9336 return;
9337 }
9338
9339 rsmseglock_release(segp);
9340
9341 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function));
9342 }
9343
9344 static void
9345 rsm_quiesce_imp_seg(rsmresource_t *resp)
9346 {
9347 rsmseg_t *segp = (rsmseg_t *)resp;
9348 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9349 DBG_DEFINE_STR(function, "rsm_quiesce_imp_seg");
9350
9351 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9352 "%s enter: key=%u\n", function, segp->s_key));
9353
9354 rsmseglock_acquire(segp);
9355 segp->s_flags |= RSM_DR_INPROGRESS;
9356
9357 while (segp->s_rdmacnt != 0) {
9358 /* wait for the RDMA to complete */
9359 cv_wait(&segp->s_cv, &segp->s_lock);
9360 }
9361
9362 rsmseglock_release(segp);
9363
9364 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function));
9365
9366 }
9367
9368 static void
9369 rsm_unquiesce_imp_seg(rsmresource_t *resp)
9370 {
9371 rsmseg_t *segp = (rsmseg_t *)resp;
9372 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9373 DBG_DEFINE_STR(function, "rsm_unquiesce_imp_seg");
9374
9375 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9376 "%s enter: key=%u\n", function, segp->s_key));
9377
9378 rsmseglock_acquire(segp);
9379
9380 segp->s_flags &= ~RSM_DR_INPROGRESS;
9381 /* wake up any waiting putv/getv ops */
9382 cv_broadcast(&segp->s_cv);
9383
9384 rsmseglock_release(segp);
9385
9386 DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function));
9387
9388
9389 }
9390
9391 static void
9392 rsm_process_exp_seg(rsmresource_t *resp, int event)
9393 {
9394 if (event == RSM_DR_QUIESCE)
9395 rsm_quiesce_exp_seg(resp);
9396 else /* UNQUIESCE */
9397 rsm_unquiesce_exp_seg(resp);
9398 }
9399
9400 static void
9401 rsm_process_imp_seg(rsmresource_t *resp, int event)
9402 {
9403 if (event == RSM_DR_QUIESCE)
9404 rsm_quiesce_imp_seg(resp);
9405 else /* UNQUIESCE */
9406 rsm_unquiesce_imp_seg(resp);
9407 }
9408
9409 static void
9410 rsm_dr_process_local_segments(int event)
9411 {
9412
9413 int i, j;
9414 rsmresource_blk_t *blk;
9415 rsmresource_t *p;
9416 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9417
9418 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9419 "rsm_dr_process_local_segments enter\n"));
9420
9421 /* iterate through the resource structure */
9422
9423 rw_enter(&rsm_resource.rsmrc_lock, RW_READER);
9424
9425 for (i = 0; i < rsm_resource.rsmrc_len; i++) {
9426 blk = rsm_resource.rsmrc_root[i];
9427 if (blk != NULL) {
9428 for (j = 0; j < RSMRC_BLKSZ; j++) {
9429 p = blk->rsmrcblk_blks[j];
9430 if ((p != NULL) && (p != RSMRC_RESERVED)) {
9431 /* valid resource */
9432 if (p->rsmrc_type ==
9433 RSM_RESOURCE_EXPORT_SEGMENT)
9434 rsm_process_exp_seg(p, event);
9435 else if (p->rsmrc_type ==
9436 RSM_RESOURCE_IMPORT_SEGMENT)
9437 rsm_process_imp_seg(p, event);
9438 }
9439 }
9440 }
9441 }
9442
9443 rw_exit(&rsm_resource.rsmrc_lock);
9444
9445 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9446 "rsm_dr_process_local_segments done\n"));
9447 }
9448
9449 /* *************** DR callback functions ************ */
9450 static void
9451 rsm_dr_callback_post_add(void *arg, pgcnt_t delta /* ARGSUSED */)
9452 {
9453 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9454 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9455 "rsm_dr_callback_post_add is a no-op\n"));
9456 /* Noop */
9457 }
9458
9459 static int
9460 rsm_dr_callback_pre_del(void *arg, pgcnt_t delta /* ARGSUSED */)
9461 {
9462 int recheck_state = 0;
9463 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9464
9465 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9466 "rsm_dr_callback_pre_del enter\n"));
9467
9468 mutex_enter(&rsm_drv_data.drv_lock);
9469
9470 do {
9471 recheck_state = 0;
9472 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9473 "rsm_dr_callback_pre_del:state=%d\n",
9474 rsm_drv_data.drv_state));
9475
9476 switch (rsm_drv_data.drv_state) {
9477 case RSM_DRV_NEW:
9478 /*
9479 * The state should usually never be RSM_DRV_NEW
9480 * since in this state the callbacks have not yet
9481 * been registered. So, ASSERT.
9482 */
9483 ASSERT(0);
9484 return (0);
9485 case RSM_DRV_REG_PROCESSING:
9486 /*
9487 * The driver is in the process of registering
9488 * with the DR framework. So, wait till the
9489 * registration process is complete.
9490 */
9491 recheck_state = 1;
9492 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
9493 break;
9494 case RSM_DRV_UNREG_PROCESSING:
9495 /*
9496 * If the state is RSM_DRV_UNREG_PROCESSING, the
9497 * module is in the process of detaching and
9498 * unregistering the callbacks from the DR
9499 * framework. So, simply return.
9500 */
9501 mutex_exit(&rsm_drv_data.drv_lock);
9502 DBG_PRINTF((category, RSM_DEBUG,
9503 "rsm_dr_callback_pre_del:"
9504 "pre-del on NEW/UNREG\n"));
9505 return (0);
9506 case RSM_DRV_OK:
9507 rsm_drv_data.drv_state = RSM_DRV_PREDEL_STARTED;
9508 break;
9509 case RSM_DRV_PREDEL_STARTED:
9510 /* FALLTHRU */
9511 case RSM_DRV_PREDEL_COMPLETED:
9512 /* FALLTHRU */
9513 case RSM_DRV_POSTDEL_IN_PROGRESS:
9514 recheck_state = 1;
9515 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
9516 break;
9517 case RSM_DRV_DR_IN_PROGRESS:
9518 rsm_drv_data.drv_memdel_cnt++;
9519 mutex_exit(&rsm_drv_data.drv_lock);
9520 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9521 "rsm_dr_callback_pre_del done\n"));
9522 return (0);
9523 /* break; */
9524 default:
9525 ASSERT(0);
9526 break;
9527 }
9528
9529 } while (recheck_state);
9530
9531 rsm_drv_data.drv_memdel_cnt++;
9532
9533 mutex_exit(&rsm_drv_data.drv_lock);
9534
9535 /* Do all the quiescing stuff here */
9536 DBG_PRINTF((category, RSM_DEBUG,
9537 "rsm_dr_callback_pre_del: quiesce things now\n"));
9538
9539 rsm_dr_process_local_segments(RSM_DR_QUIESCE);
9540
9541 /*
9542 * now that all local segments have been quiesced lets inform
9543 * the importers
9544 */
9545 rsm_send_suspend();
9546
9547 /*
9548 * In response to the suspend message the remote node(s) will process
9549 * the segments and send a suspend_complete message. Till all
9550 * the nodes send the suspend_complete message we wait in the
9551 * RSM_DRV_PREDEL_STARTED state. In the exporter_quiesce
9552 * function we transition to the RSM_DRV_PREDEL_COMPLETED state.
9553 */
9554 mutex_enter(&rsm_drv_data.drv_lock);
9555
9556 while (rsm_drv_data.drv_state == RSM_DRV_PREDEL_STARTED) {
9557 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
9558 }
9559
9560 ASSERT(rsm_drv_data.drv_state == RSM_DRV_PREDEL_COMPLETED);
9561
9562 rsm_drv_data.drv_state = RSM_DRV_DR_IN_PROGRESS;
9563 cv_broadcast(&rsm_drv_data.drv_cv);
9564
9565 mutex_exit(&rsm_drv_data.drv_lock);
9566
9567 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9568 "rsm_dr_callback_pre_del done\n"));
9569
9570 return (0);
9571 }
9572
9573 static void
9574 rsm_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled /* ARGSUSED */)
9575 {
9576 int recheck_state = 0;
9577 DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
9578
9579 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9580 "rsm_dr_callback_post_del enter\n"));
9581
9582 mutex_enter(&rsm_drv_data.drv_lock);
9583
9584 do {
9585 recheck_state = 0;
9586 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9587 "rsm_dr_callback_post_del:state=%d\n",
9588 rsm_drv_data.drv_state));
9589
9590 switch (rsm_drv_data.drv_state) {
9591 case RSM_DRV_NEW:
9592 /*
9593 * The driver state cannot not be RSM_DRV_NEW
9594 * since in this state the callbacks have not
9595 * yet been registered.
9596 */
9597 ASSERT(0);
9598 return;
9599 case RSM_DRV_REG_PROCESSING:
9600 /*
9601 * The driver is in the process of registering with
9602 * the DR framework. Wait till the registration is
9603 * complete.
9604 */
9605 recheck_state = 1;
9606 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
9607 break;
9608 case RSM_DRV_UNREG_PROCESSING:
9609 /*
9610 * RSM_DRV_UNREG_PROCESSING state means the module
9611 * is detaching and unregistering the callbacks
9612 * from the DR framework. So simply return.
9613 */
9614 /* FALLTHRU */
9615 case RSM_DRV_OK:
9616 /*
9617 * RSM_DRV_OK means we missed the pre-del
9618 * corresponding to this post-del coz we had not
9619 * registered yet, so simply return.
9620 */
9621 mutex_exit(&rsm_drv_data.drv_lock);
9622 DBG_PRINTF((category, RSM_DEBUG,
9623 "rsm_dr_callback_post_del:"
9624 "post-del on OK/UNREG\n"));
9625 return;
9626 /* break; */
9627 case RSM_DRV_PREDEL_STARTED:
9628 /* FALLTHRU */
9629 case RSM_DRV_PREDEL_COMPLETED:
9630 /* FALLTHRU */
9631 case RSM_DRV_POSTDEL_IN_PROGRESS:
9632 recheck_state = 1;
9633 cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
9634 break;
9635 case RSM_DRV_DR_IN_PROGRESS:
9636 rsm_drv_data.drv_memdel_cnt--;
9637 if (rsm_drv_data.drv_memdel_cnt > 0) {
9638 mutex_exit(&rsm_drv_data.drv_lock);
9639 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9640 "rsm_dr_callback_post_del done:\n"));
9641 return;
9642 }
9643 rsm_drv_data.drv_state = RSM_DRV_POSTDEL_IN_PROGRESS;
9644 break;
9645 default:
9646 ASSERT(0);
9647 return;
9648 /* break; */
9649 }
9650 } while (recheck_state);
9651
9652 mutex_exit(&rsm_drv_data.drv_lock);
9653
9654 /* Do all the unquiescing stuff here */
9655 DBG_PRINTF((category, RSM_DEBUG,
9656 "rsm_dr_callback_post_del: unquiesce things now\n"));
9657
9658 rsm_dr_process_local_segments(RSM_DR_UNQUIESCE);
9659
9660 /*
9661 * now that all local segments have been unquiesced lets inform
9662 * the importers
9663 */
9664 rsm_send_resume();
9665
9666 mutex_enter(&rsm_drv_data.drv_lock);
9667
9668 rsm_drv_data.drv_state = RSM_DRV_OK;
9669
9670 cv_broadcast(&rsm_drv_data.drv_cv);
9671
9672 mutex_exit(&rsm_drv_data.drv_lock);
9673
9674 DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
9675 "rsm_dr_callback_post_del done\n"));
9676
9677 return;
9678
9679 }