1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
14 */
15
16 #include <sys/types.h>
17 #include <sys/conf.h>
18 #include <sys/sysmacros.h>
19 #include <sys/cmn_err.h>
20 #include <sys/stat.h>
21 #include <sys/ddi.h>
22 #include <sys/sunddi.h>
23 #include <sys/sdt.h>
24 #include <sys/avl.h>
25 #include <sys/debug.h>
26 #include <sys/sysevent.h>
27 #include <sys/sysevent_impl.h>
28
29 #include "krrp_protocol.h"
30 #include "krrp_connection.h"
31 #include "krrp_params.h"
32 #include "krrp_svc.h"
33
34 static krrp_sess_t *krrp_svc_lookup_session_no_lock(const char *sess_id);
35 static void krrp_svc_unregister_all_sessions(void);
36 static void krrp_svc_unregister_session_common(krrp_sess_t *s);
37 static void krrp_svc_on_new_ks_cb(ksocket_t new_ks);
38 static void krrp_svc_on_server_error_cb(krrp_error_t *error);
39 static void krrp_svc_new_conn_handler(void *void_conn);
40
41 static void krrp_svc_ref_cnt_wait(void);
42 static void krrp_svc_send_error(krrp_conn_t *conn, krrp_error_t *error);
43
44 int krrp_ping_period = 5000;
45
46 krrp_svc_t krrp_svc;
47 int krrp_svc_created = 0;
48
49 krrp_svc_t *
50 krrp_svc_get_instance(void)
51 {
52 if (!krrp_svc_created) {
53 (void) memset(&krrp_svc, 0, sizeof (krrp_svc_t));
54 krrp_svc.state = KRRP_SVCS_CREATED;
55 krrp_svc_created = 1;
56 }
57
58 return (&krrp_svc);
59 }
60
61 void
62 krrp_svc_init(void)
63 {
64 VERIFY3U(krrp_svc.state, ==, KRRP_SVCS_CREATED);
65
66 mutex_init(&krrp_svc.mtx, NULL, MUTEX_DEFAULT, NULL);
67 cv_init(&krrp_svc.cv, NULL, CV_DEFAULT, NULL);
68
69 avl_create(&krrp_svc.sessions, &krrp_sess_compare_id,
70 sizeof (krrp_sess_t), offsetof(krrp_sess_t, node));
71
72 krrp_svc.ev_chan = NULL;
73 krrp_svc.state = KRRP_SVCS_DETACHED;
74 }
75
76 void
77 krrp_svc_fini(void)
78 {
79 mutex_enter(&krrp_svc.mtx);
80
81 avl_destroy(&krrp_svc.sessions);
82 cv_destroy(&krrp_svc.cv);
83
84 if (krrp_svc.ev_chan)
85 (void) sysevent_evc_unbind(krrp_svc.ev_chan);
86
87 mutex_exit(&krrp_svc.mtx);
88 mutex_destroy(&krrp_svc.mtx);
89
90 (void) memset(&krrp_svc, 0, sizeof (krrp_svc_t));
91
92 krrp_svc.state = KRRP_SVCS_CREATED;
93 }
94
95 boolean_t
96 krrp_svc_is_enabled(void)
97 {
98 boolean_t res;
99
100 krrp_svc_lock(&krrp_svc);
101 res = krrp_svc.state == KRRP_SVCS_ENABLED ? B_TRUE : B_FALSE;
102 krrp_svc_unlock(&krrp_svc);
103
104 return (res);
105 }
106
107 void
108 krrp_svc_attach(dev_info_t *dip)
109 {
110 krrp_svc_lock(&krrp_svc);
111 VERIFY3U(krrp_svc.state, ==, KRRP_SVCS_DETACHED);
112 krrp_svc.state = KRRP_SVCS_DISABLED;
113 krrp_svc.dip = dip;
114 krrp_svc_unlock(&krrp_svc);
115 }
116
117 int
118 krrp_svc_detach(void)
119 {
120 krrp_svc_lock(&krrp_svc);
121 if (krrp_svc.state != KRRP_SVCS_DISABLED) {
122 krrp_svc_unlock(&krrp_svc);
123 return (-1);
124 }
125
126 krrp_svc.state = KRRP_SVCS_DETACHED;
127 krrp_svc.dip = NULL;
128 krrp_svc_unlock(&krrp_svc);
129
130 return (0);
131 }
132
133 int
134 krrp_svc_enable(krrp_error_t *error)
135 {
136 int rc = 0;
137
138 krrp_svc_lock(&krrp_svc);
139
140 switch (krrp_svc.state) {
141 case KRRP_SVCS_DISABLED:
142 krrp_svc.state = KRRP_SVCS_ENABLING;
143 break;
144 case KRRP_SVCS_ENABLING:
145 case KRRP_SVCS_DISABLING:
146 krrp_error_set(error, KRRP_ERRNO_BUSY, 0);
147 rc = -1;
148 break;
149 default:
150 VERIFY(0);
151 }
152
153 krrp_svc_unlock(&krrp_svc);
154
155 if (rc != 0)
156 return (rc);
157
158 /*
159 * This taskq will process the following tasks:
160 * - PDU rele
161 * - DBLK engine destroy
162 * - processing of new incoming connections
163 */
164 krrp_svc.aux_taskq = taskq_create("krrp_aux_taskq", 10,
165 minclsyspri, 128, 16384, TASKQ_DYNAMIC);
166
167 if (krrp_pdu_engine_global_init() != 0)
168 goto err;
169
170 krrp_server_create(&krrp_svc.server, &krrp_svc_on_new_ks_cb,
171 &krrp_svc_on_server_error_cb);
172
173 krrp_svc_lock(&krrp_svc);
174 krrp_svc.state = KRRP_SVCS_ENABLED;
175 krrp_svc_unlock(&krrp_svc);
176
177 return (0);
178
179 err:
180 if (krrp_svc.aux_taskq != NULL) {
181 taskq_destroy(krrp_svc.aux_taskq);
182 krrp_svc.aux_taskq = NULL;
183 }
184
185 krrp_svc_lock(&krrp_svc);
186 krrp_svc.state = KRRP_SVCS_DISABLED;
187 krrp_svc_unlock(&krrp_svc);
188
189 return (-1);
190 }
191
192 int
193 krrp_svc_disable(krrp_error_t *error)
194 {
195 int rc = 0;
196
197 krrp_svc_lock(&krrp_svc);
198
199 switch (krrp_svc.state) {
200 case KRRP_SVCS_ENABLED:
201 krrp_svc.state = KRRP_SVCS_DISABLING;
202 break;
203 case KRRP_SVCS_ENABLING:
204 case KRRP_SVCS_DISABLING:
205 krrp_error_set(error, KRRP_ERRNO_BUSY, 0);
206 rc = -1;
207 break;
208 default:
209 VERIFY(0);
210 }
211
212 krrp_svc_unlock(&krrp_svc);
213
214 if (rc != 0)
215 return (rc);
216
217 krrp_server_destroy(krrp_svc.server);
218 krrp_svc.server = NULL;
219
220 /* Wait until all our clients finished */
221 krrp_svc_ref_cnt_wait();
222
223 krrp_svc_unregister_all_sessions();
224
225 krrp_pdu_engine_global_fini();
226
227 taskq_destroy(krrp_svc.aux_taskq);
228
229 krrp_svc_lock(&krrp_svc);
230 krrp_svc.state = KRRP_SVCS_DISABLED;
231 krrp_svc_unlock(&krrp_svc);
232
233 return (0);
234 }
235
236 int
237 krrp_svc_config(nvlist_t *params, nvlist_t *result,
238 krrp_error_t *error)
239 {
240 int rc;
241 krrp_cfg_type_t cfg_type;
242
243 rc = krrp_param_get(KRRP_PARAM_CFG_TYPE, params,
244 (void *) &cfg_type);
245 if (rc != 0) {
246 krrp_error_set(error, KRRP_ERRNO_CFGTYPE, ENOENT);
247 goto out;
248 }
249
250 switch (cfg_type) {
251 case KRRP_SVC_CFG_TYPE_SERVER:
252 if (result == NULL) {
253 rc = krrp_server_set_config(krrp_svc.server,
254 params, error);
255 } else {
256 rc = krrp_server_get_config(krrp_svc.server,
257 result, error);
258 }
259
260 break;
261 default:
262 krrp_error_set(error, KRRP_ERRNO_CFGTYPE, EINVAL);
263 rc = -1;
264 }
265
266 out:
267 return (rc);
268 }
269
270 void
271 krrp_svc_state(nvlist_t *out_nvl)
272 {
273 boolean_t svc_enabled, srv_running = B_FALSE;
274
275 svc_enabled = krrp_svc_is_enabled();
276 if (svc_enabled)
277 srv_running = krrp_server_is_running(krrp_svc.server);
278
279 VERIFY3U(krrp_param_put(KRRP_PARAM_SVC_ENABLED,
280 out_nvl, &svc_enabled), ==, 0);
281
282 VERIFY3U(krrp_param_put(KRRP_PARAM_SRV_RUNNING,
283 out_nvl, &srv_running), ==, 0);
284 }
285
286 int
287 krrp_svc_register_session(krrp_sess_t *sess, krrp_error_t *error)
288 {
289 krrp_svc_lock(&krrp_svc);
290
291 if (krrp_svc_lookup_session_no_lock(sess->id) != NULL) {
292 krrp_svc_unlock(&krrp_svc);
293 krrp_error_set(error, KRRP_ERRNO_SESS, EALREADY);
294 return (-1);
295 }
296
297 avl_add(&krrp_svc.sessions, sess);
298
299 cmn_err(CE_NOTE, "A new session has been registered (id:[%s])",
300 sess->id);
301
302 krrp_svc_unlock(&krrp_svc);
303
304 return (0);
305 }
306
307 int
308 krrp_svc_unregister_session(krrp_sess_t *sess, krrp_error_t *error)
309 {
310 krrp_svc_lock(&krrp_svc);
311
312 if (krrp_svc_lookup_session_no_lock(sess->id) == NULL) {
313 krrp_svc_unlock(&krrp_svc);
314 krrp_error_set(error, KRRP_ERRNO_SESS, ENOENT);
315 return (-1);
316 }
317
318 krrp_svc_unregister_session_common(sess);
319
320 krrp_svc_unlock(&krrp_svc);
321
322 return (0);
323 }
324
325 void
326 krrp_svc_post_uevent(const char *subclass, nvlist_t *attr_list)
327 {
328 int rc;
329
330 if (krrp_svc.ev_chan == NULL) {
331 if (sysevent_evc_bind(KRRP_EVENT_CHANNEL, &krrp_svc.ev_chan,
332 EVCH_HOLD_PEND | EVCH_CREAT) != 0) {
333 cmn_err(CE_WARN, "Failed to bind to krrp event "
334 "channel");
335 return;
336 }
337 }
338
339 cmn_err(CE_NOTE, "Publishing KRRP event %s %s", EC_KRRP, subclass);
340
341 rc = sysevent_evc_publish(krrp_svc.ev_chan, EC_KRRP, subclass,
342 KRRP_EVENT_VENDOR, KRRP_EVENT_PUBLISHER, attr_list, EVCH_NOSLEEP);
343
344 if (rc != 0)
345 cmn_err(CE_WARN, "Failed to publish KRRP event (%d)", rc);
346 }
347
348 void
349 krrp_svc_dispatch_task(task_func_t func, void *arg)
350 {
351 VERIFY(taskq_dispatch(krrp_svc.aux_taskq,
352 func, arg, TQ_SLEEP) != 0);
353 }
354
355 static void
356 krrp_svc_unregister_all_sessions()
357 {
358 krrp_sess_t *s, *s_next;
359
360 krrp_svc_lock(&krrp_svc);
361
362 s = avl_first(&krrp_svc.sessions);
363 while (s != NULL) {
364 s_next = AVL_NEXT(&krrp_svc.sessions, s);
365 krrp_svc_unregister_session_common(s);
366
367 krrp_sess_destroy(s);
368
369 s = s_next;
370 }
371
372 krrp_svc_unlock(&krrp_svc);
373 }
374
375 static void
376 krrp_svc_unregister_session_common(krrp_sess_t *s)
377 {
378 avl_remove(&krrp_svc.sessions, s);
379 cmn_err(CE_NOTE, "A session has been unregistered (id:[%s])", s->id);
380 }
381
382 void
383 krrp_svc_list_sessions(nvlist_t *out_nvl)
384 {
385 struct krrp_nvlist_list {
386 nvlist_t *sess_nvl;
387 struct krrp_nvlist_list *next;
388 } *nvls_head = NULL, *nvls_cur, **s_nvl_cur;
389 s_nvl_cur = &nvls_head;
390 krrp_param_array_t param_array;
391 size_t nvl_cnt = 0, i;
392 krrp_sess_t *sess;
393
394
395 krrp_svc_lock(&krrp_svc);
396
397 sess = avl_first(&krrp_svc.sessions);
398 while (sess != NULL) {
399 boolean_t sess_started;
400 boolean_t sess_running;
401
402 sess_started = krrp_sess_is_started(sess);
403 sess_running = krrp_sess_is_running(sess);
404 nvls_cur = kmem_zalloc(sizeof (struct krrp_nvlist_list),
405 KM_SLEEP);
406
407 nvls_cur->sess_nvl = fnvlist_alloc();
408
409 VERIFY3U(krrp_param_put(KRRP_PARAM_SESS_ID,
410 nvls_cur->sess_nvl, (void *)sess->id), ==, 0);
411 VERIFY3U(krrp_param_put(KRRP_PARAM_SESS_KSTAT_ID,
412 nvls_cur->sess_nvl, (void *)sess->kstat.id), ==, 0);
413 VERIFY3U(krrp_param_put(KRRP_PARAM_SESS_STARTED,
414 nvls_cur->sess_nvl, (void *)&sess_started), ==, 0);
415 VERIFY3U(krrp_param_put(KRRP_PARAM_SESS_RUNNING,
416 nvls_cur->sess_nvl, (void *)&sess_running), ==, 0);
417
418 sess = AVL_NEXT(&krrp_svc.sessions, sess);
419 nvl_cnt++;
420
421 *s_nvl_cur = nvls_cur;
422 s_nvl_cur = &(nvls_cur->next);
423 }
424
425 krrp_svc_unlock(&krrp_svc);
426
427 if (nvl_cnt != 0) {
428 param_array.array =
429 kmem_zalloc(sizeof (nvlist_t *) * nvl_cnt, KM_SLEEP);
430 param_array.nelem = (uint_t)nvl_cnt;
431
432 for (i = 0; i < nvl_cnt; i++) {
433 nvls_cur = nvls_head;
434 nvls_head = nvls_head->next;
435
436 param_array.array[i] = nvls_cur->sess_nvl;
437
438 kmem_free(nvls_cur, sizeof (struct krrp_nvlist_list));
439 }
440
441 VERIFY3U(krrp_param_put(KRRP_PARAM_SESSIONS,
442 out_nvl, (void *)¶m_array), ==, 0);
443
444 /*
445 * Need to free the nvls that have
446 * been added to the output nvl
447 */
448 for (i = 0; i < nvl_cnt; i++)
449 fnvlist_free(param_array.array[i]);
450
451 kmem_free(param_array.array, sizeof (nvlist_t *) * nvl_cnt);
452 }
453 }
454
455 krrp_sess_t *
456 krrp_svc_lookup_session(const char *sess_id)
457 {
458 krrp_sess_t *res;
459
460 krrp_svc_lock(&krrp_svc);
461 res = krrp_svc_lookup_session_no_lock(sess_id);
462 krrp_svc_unlock(&krrp_svc);
463
464 return (res);
465 }
466
467 static krrp_sess_t *
468 krrp_svc_lookup_session_no_lock(const char *sess_id)
469 {
470 krrp_error_t error;
471 krrp_sess_t srch_sess, *s;
472
473 if (krrp_sess_set_id(&srch_sess, sess_id, &error) != 0)
474 return (NULL);
475
476 s = avl_find(&krrp_svc.sessions,
477 (const void *)&srch_sess, NULL);
478
479 return (s);
480 }
481
482 static void
483 krrp_svc_on_new_ks_cb(ksocket_t new_ks)
484 {
485 krrp_error_t error;
486 krrp_conn_t *conn = NULL;
487
488 if (krrp_svc_ref_cnt_try_hold() != 0) {
489 (void) ksocket_close(new_ks, CRED());
490 return;
491 }
492
493 if (krrp_conn_create_from_ksocket(&conn, new_ks, &error) != 0) {
494 (void) ksocket_close(new_ks, CRED());
495 return;
496 }
497
498 krrp_svc_dispatch_task(krrp_svc_new_conn_handler, conn);
499 }
500
501 static void
502 krrp_svc_on_server_error_cb(krrp_error_t *error)
503 {
504 nvlist_t *attrs = fnvlist_alloc();
505
506 krrp_error_to_nvl(error, &attrs);
507
508 krrp_svc_post_uevent(ESC_KRRP_SERVER_ERROR, attrs);
509
510 fnvlist_free(attrs);
511 }
512
513 /*
514 * In this stage we are doing Phase I of Session establishment:
515 *
516 * The remote hosts must send to us ctrl-pdu with KRRP_OPCODE_ATTACH_SESS
517 * opcode and the payload must contain packed NVL that contains
518 * KRRP_PARAM_SESS_ID param. After extracting of KRRP_PARAM_SESS_ID
519 * we are trying to lookup a session with the extracted sess_id.
520 * If the session exists we attach the connection to the session,
521 * otherwise send the error to the remote side
522 */
523 static void
524 krrp_svc_new_conn_handler(void *void_conn)
525 {
526 int rc = -1;
527 nvlist_t *params = NULL;
528 krrp_conn_t *conn = void_conn;
529 krrp_sess_t *sess;
530 const char *sess_id = NULL;
531 krrp_pdu_ctrl_t *pdu = NULL;
532 krrp_error_t error;
533
534 krrp_error_init(&error);
535 if (krrp_conn_rx_ctrl_pdu(conn, &pdu, &error) != 0)
536 goto out;
537
538 if (krrp_pdu_opcode(pdu) != KRRP_OPCODE_ATTACH_SESS) {
539 cmn_err(CE_WARN, "Received an unexpected opcode [%d]",
540 krrp_pdu_opcode(pdu));
541 krrp_error_set(&error, KRRP_ERRNO_PROTO, EINVAL);
542 goto send_error;
543 }
544
545 if (pdu->cur_data_sz == 0) {
546 krrp_error_set(&error, KRRP_ERRNO_PROTO, ENODATA);
547 goto send_error;
548 }
549
550 rc = nvlist_unpack((char *)pdu->dblk->data,
551 pdu->cur_data_sz, ¶ms, KM_SLEEP);
552 if (rc != 0) {
553 krrp_error_set(&error, KRRP_ERRNO_PROTO, EBADMSG);
554 goto send_error;
555 }
556
557 rc = krrp_param_get(KRRP_PARAM_SESS_ID, params,
558 (void *) &sess_id);
559 if (rc != 0) {
560 krrp_error_set(&error, KRRP_ERRNO_SESSID, ENOENT);
561 goto send_error;
562 }
563
564 rc = -1;
565 sess = krrp_svc_lookup_session(sess_id);
566 if (sess == NULL) {
567 krrp_error_set(&error, KRRP_ERRNO_SESS, ENOENT);
568 goto send_error;
569 }
570
571 rc = krrp_sess_target_attach_conn(sess, conn, params, &error);
572 if (rc != 0 && error.krrp_errno == KRRP_ERRNO_SENDFAIL)
573 goto out;
574
575 send_error:
576 if (rc != 0)
577 krrp_svc_send_error(conn, &error);
578
579 out:
580 if (pdu != NULL)
581 krrp_pdu_rele((krrp_pdu_t *)pdu);
582
583 if (rc != 0) {
584 if (error.krrp_errno != 0) {
585 cmn_err(CE_WARN, "Session estabishment error: %s [%d]",
586 krrp_error_errno_to_str(error.krrp_errno),
587 error.unix_errno);
588 }
589
590 krrp_conn_destroy(conn);
591 }
592
593 if (params != NULL)
594 fnvlist_free(params);
595
596 krrp_svc_ref_cnt_rele();
597 }
598
599 /*
600 * At now this function is used only by ksvc_disable logic
601 * Before process an ioctl we incement ref_cnt,
602 * ksvc_disable logic also works under ref_cnt,
603 * so here we wait for all exclude the the ksvc_disable itself
604 */
605 static void
606 krrp_svc_ref_cnt_wait()
607 {
608 krrp_svc_lock(&krrp_svc);
609 while (krrp_svc.ref_cnt > 1)
610 cv_wait(&krrp_svc.cv, &krrp_svc.mtx);
611 krrp_svc_unlock(&krrp_svc);
612 }
613
614 int
615 krrp_svc_ref_cnt_try_hold()
616 {
617 krrp_svc_lock(&krrp_svc);
618 if (krrp_svc.state != KRRP_SVCS_ENABLED) {
619 krrp_svc_unlock(&krrp_svc);
620 return (-1);
621 }
622
623 krrp_svc.ref_cnt++;
624 krrp_svc_unlock(&krrp_svc);
625 return (0);
626 }
627
628 void
629 krrp_svc_ref_cnt_rele()
630 {
631 krrp_svc_lock(&krrp_svc);
632 VERIFY(krrp_svc.ref_cnt > 0);
633 krrp_svc.ref_cnt--;
634 cv_signal(&krrp_svc.cv);
635 krrp_svc_unlock(&krrp_svc);
636 }
637
638 static void
639 krrp_svc_send_error(krrp_conn_t *conn, krrp_error_t *error)
640 {
641 nvlist_t *err_nvl = NULL;
642 krrp_error_t err;
643 int rc;
644
645 krrp_error_to_nvl(error, &err_nvl);
646
647 rc = krrp_conn_send_ctrl_data(conn,
648 KRRP_OPCODE_ERROR, err_nvl, &err);
649 if (rc != 0)
650 cmn_err(CE_WARN, "Failed to send Error to "
651 "the remote host: %s [%d]",
652 krrp_error_errno_to_str(err.krrp_errno),
653 err.unix_errno);
654
655 fnvlist_free(err_nvl);
656 }