Print this page
6452 ilbd leaks lmalloc() memory due to posix_spawn() sloppiness
Reviewed by: Albert Lee <trisk@omniti.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilbd_hc.c
+++ new/usr/src/cmd/cmd-inet/usr.lib/ilbd/ilbd_hc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 * Copyright 2012 Milan Jurik. All rights reserved.
26 + * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
26 27 */
27 28
28 29 #include <sys/types.h>
29 30 #include <sys/socket.h>
30 31 #include <sys/list.h>
31 32 #include <sys/stropts.h>
32 33 #include <sys/siginfo.h>
33 34 #include <sys/wait.h>
34 35 #include <arpa/inet.h>
35 36 #include <netinet/in.h>
36 37 #include <stdlib.h>
37 38 #include <stdio.h>
38 39 #include <strings.h>
39 40 #include <stddef.h>
40 41 #include <unistd.h>
41 42 #include <libilb.h>
42 43 #include <port.h>
43 44 #include <time.h>
44 45 #include <signal.h>
45 46 #include <assert.h>
46 47 #include <errno.h>
47 48 #include <spawn.h>
48 49 #include <fcntl.h>
49 50 #include <limits.h>
50 51 #include "libilb_impl.h"
51 52 #include "ilbd.h"
52 53
53 54 /* Global list of HC objects */
54 55 list_t ilbd_hc_list;
55 56
56 57 /* Timer queue for all hc related timers. */
57 58 static iu_tq_t *ilbd_hc_timer_q;
58 59
59 60 /* Indicate whether the timer needs to be updated */
60 61 static boolean_t hc_timer_restarted;
61 62
62 63 static void ilbd_hc_probe_timer(iu_tq_t *, void *);
63 64 static ilb_status_t ilbd_hc_restart_timer(ilbd_hc_t *, ilbd_hc_srv_t *);
64 65 static boolean_t ilbd_run_probe(ilbd_hc_srv_t *);
65 66
66 67 #define MAX(a, b) ((a) > (b) ? (a) : (b))
67 68
68 69 /*
69 70 * Number of arguments passed to a probe. argc[0] is the path name of
70 71 * the probe.
71 72 */
72 73 #define HC_PROBE_ARGC 8
73 74
74 75 /*
75 76 * Max number of characters to be read from the output of a probe. It
76 77 * is long enough to read in a 64 bit integer.
77 78 */
78 79 #define HC_MAX_PROBE_OUTPUT 24
79 80
80 81 void
81 82 i_ilbd_setup_hc_list(void)
82 83 {
83 84 list_create(&ilbd_hc_list, sizeof (ilbd_hc_t),
84 85 offsetof(ilbd_hc_t, ihc_link));
85 86 }
86 87
87 88 /*
88 89 * Given a hc object name, return a pointer to hc object if found.
89 90 */
90 91 ilbd_hc_t *
91 92 ilbd_get_hc(const char *name)
92 93 {
93 94 ilbd_hc_t *hc;
94 95
95 96 for (hc = list_head(&ilbd_hc_list); hc != NULL;
96 97 hc = list_next(&ilbd_hc_list, hc)) {
97 98 if (strcasecmp(hc->ihc_name, name) == 0)
98 99 return (hc);
99 100 }
100 101 return (NULL);
101 102 }
102 103
103 104 /*
104 105 * Generates an audit record for create-healthcheck,
105 106 * delete-healtcheck subcommands.
106 107 */
107 108 static void
108 109 ilbd_audit_hc_event(const char *audit_hcname,
109 110 const ilb_hc_info_t *audit_hcinfo, ilbd_cmd_t cmd,
110 111 ilb_status_t rc, ucred_t *ucredp)
111 112 {
112 113 adt_session_data_t *ah;
113 114 adt_event_data_t *event;
114 115 au_event_t flag;
115 116 int audit_error;
116 117
117 118 if ((ucredp == NULL) && (cmd == ILBD_CREATE_HC)) {
118 119 /*
119 120 * we came here from the path where ilbd incorporates
120 121 * the configuration that is listed in SCF:
121 122 * i_ilbd_read_config->ilbd_walk_hc_pgs->
122 123 * ->ilbd_scf_instance_walk_pg->ilbd_create_hc
123 124 * We skip auditing in that case
124 125 */
125 126 logdebug("ilbd_audit_hc_event: skipping auditing");
126 127 return;
127 128 }
128 129
129 130 if (adt_start_session(&ah, NULL, 0) != 0) {
130 131 logerr("ilbd_audit_hc_event: adt_start_session failed");
131 132 exit(EXIT_FAILURE);
132 133 }
133 134 if (adt_set_from_ucred(ah, ucredp, ADT_NEW) != 0) {
134 135 (void) adt_end_session(ah);
135 136 logerr("ilbd_audit_rule_event: adt_set_from_ucred failed");
136 137 exit(EXIT_FAILURE);
137 138 }
138 139 if (cmd == ILBD_CREATE_HC)
139 140 flag = ADT_ilb_create_healthcheck;
140 141 else if (cmd == ILBD_DESTROY_HC)
141 142 flag = ADT_ilb_delete_healthcheck;
142 143
143 144 if ((event = adt_alloc_event(ah, flag)) == NULL) {
144 145 logerr("ilbd_audit_hc_event: adt_alloc_event failed");
145 146 exit(EXIT_FAILURE);
146 147 }
147 148 (void) memset((char *)event, 0, sizeof (adt_event_data_t));
148 149
149 150 switch (cmd) {
150 151 case ILBD_CREATE_HC:
151 152 event->adt_ilb_create_healthcheck.auth_used =
152 153 NET_ILB_CONFIG_AUTH;
153 154 event->adt_ilb_create_healthcheck.hc_test =
154 155 (char *)audit_hcinfo->hci_test;
155 156 event->adt_ilb_create_healthcheck.hc_name =
156 157 (char *)audit_hcinfo->hci_name;
157 158
158 159 /*
159 160 * If the value 0 is stored, the default values are
160 161 * set in the kernel. User land does not know about them
161 162 * So if the user does not specify them, audit record
162 163 * will show them as 0
163 164 */
164 165 event->adt_ilb_create_healthcheck.hc_timeout =
165 166 audit_hcinfo->hci_timeout;
166 167 event->adt_ilb_create_healthcheck.hc_count =
167 168 audit_hcinfo->hci_count;
168 169 event->adt_ilb_create_healthcheck.hc_interval =
169 170 audit_hcinfo->hci_interval;
170 171 break;
171 172 case ILBD_DESTROY_HC:
172 173 event->adt_ilb_delete_healthcheck.auth_used =
173 174 NET_ILB_CONFIG_AUTH;
174 175 event->adt_ilb_delete_healthcheck.hc_name =
175 176 (char *)audit_hcname;
176 177 break;
177 178 }
178 179
179 180 /* Fill in success/failure */
180 181 if (rc == ILB_STATUS_OK) {
181 182 if (adt_put_event(event, ADT_SUCCESS, ADT_SUCCESS) != 0) {
182 183 logerr("ilbd_audit_hc_event: adt_put_event failed");
183 184 exit(EXIT_FAILURE);
184 185 }
185 186 } else {
186 187 audit_error = ilberror2auditerror(rc);
187 188 if (adt_put_event(event, ADT_FAILURE, audit_error) != 0) {
188 189 logerr("ilbd_audit_hc_event: adt_put_event failed");
189 190 exit(EXIT_FAILURE);
190 191 }
191 192 }
192 193 adt_free_event(event);
193 194 (void) adt_end_session(ah);
194 195 }
195 196
196 197 /*
197 198 * Given the ilb_hc_info_t passed in (from the libilb), create a hc object
198 199 * in ilbd. The parameter ev_port is not used, refer to comments of
199 200 * ilbd_create_sg() in ilbd_sg.c
200 201 */
201 202 /* ARGSUSED */
202 203 ilb_status_t
203 204 ilbd_create_hc(const ilb_hc_info_t *hc_info, int ev_port,
204 205 const struct passwd *ps, ucred_t *ucredp)
205 206 {
206 207 ilbd_hc_t *hc;
207 208 ilb_status_t ret = ILB_STATUS_OK;
208 209
209 210 /*
210 211 * ps == NULL is from the daemon when it starts and load configuration
211 212 * ps != NULL is from client.
212 213 */
213 214 if (ps != NULL) {
214 215 ret = ilbd_check_client_config_auth(ps);
215 216 if (ret != ILB_STATUS_OK) {
216 217 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
217 218 ret, ucredp);
218 219 return (ret);
219 220 }
220 221 }
221 222
222 223 if (hc_info->hci_name[0] == '\0') {
223 224 logdebug("ilbd_create_hc: missing healthcheck info");
224 225 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
225 226 ILB_STATUS_ENOHCINFO, ucredp);
226 227 return (ILB_STATUS_ENOHCINFO);
227 228 }
228 229
229 230 hc = ilbd_get_hc(hc_info->hci_name);
230 231 if (hc != NULL) {
231 232 logdebug("ilbd_create_hc: healthcheck name %s already"
232 233 " exists", hc_info->hci_name);
233 234 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
234 235 ILB_STATUS_EEXIST, ucredp);
235 236 return (ILB_STATUS_EEXIST);
236 237 }
237 238
238 239 /*
239 240 * Sanity check on user supplied probe. The given path name
240 241 * must be a full path name (starts with '/') and is
241 242 * executable.
242 243 */
243 244 if (strcasecmp(hc_info->hci_test, ILB_HC_STR_TCP) != 0 &&
244 245 strcasecmp(hc_info->hci_test, ILB_HC_STR_UDP) != 0 &&
245 246 strcasecmp(hc_info->hci_test, ILB_HC_STR_PING) != 0 &&
246 247 (hc_info->hci_test[0] != '/' ||
247 248 access(hc_info->hci_test, X_OK) == -1)) {
248 249 if (errno == ENOENT) {
249 250 logdebug("ilbd_create_hc: user script %s doesn't "
250 251 "exist", hc_info->hci_test);
251 252 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
252 253 ILB_STATUS_ENOENT, ucredp);
253 254 return (ILB_STATUS_ENOENT);
254 255 } else {
255 256 logdebug("ilbd_create_hc: user script %s is "
256 257 "invalid", hc_info->hci_test);
257 258 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
258 259 ILB_STATUS_EINVAL, ucredp);
259 260 return (ILB_STATUS_EINVAL);
260 261 }
261 262 }
262 263
263 264 /* Create and add the hc object */
264 265 hc = calloc(1, sizeof (ilbd_hc_t));
265 266 if (hc == NULL) {
266 267 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
267 268 ILB_STATUS_ENOMEM, ucredp);
268 269 return (ILB_STATUS_ENOMEM);
269 270 }
270 271 (void) memcpy(&hc->ihc_info, hc_info, sizeof (ilb_hc_info_t));
271 272 if (strcasecmp(hc->ihc_test, ILB_HC_STR_TCP) == 0)
272 273 hc->ihc_test_type = ILBD_HC_TCP;
273 274 else if (strcasecmp(hc->ihc_test, ILB_HC_STR_UDP) == 0)
274 275 hc->ihc_test_type = ILBD_HC_UDP;
275 276 else if (strcasecmp(hc->ihc_test, ILB_HC_STR_PING) == 0)
276 277 hc->ihc_test_type = ILBD_HC_PING;
277 278 else
278 279 hc->ihc_test_type = ILBD_HC_USER;
279 280 list_create(&hc->ihc_rules, sizeof (ilbd_hc_rule_t),
280 281 offsetof(ilbd_hc_rule_t, hcr_link));
281 282
282 283 /* Update SCF */
283 284 if (ps != NULL) {
284 285 if ((ret = ilbd_create_pg(ILBD_SCF_HC, (void *)hc)) !=
285 286 ILB_STATUS_OK) {
286 287 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
287 288 ret, ucredp);
288 289 list_destroy(&hc->ihc_rules);
289 290 free(hc);
290 291 return (ret);
291 292 }
292 293 }
293 294
294 295 /* Everything is fine, now add it to the global list. */
295 296 list_insert_tail(&ilbd_hc_list, hc);
296 297 ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC, ret, ucredp);
297 298 return (ret);
298 299 }
299 300
300 301 /*
301 302 * Given a name of a hc object, destroy it.
302 303 */
303 304 ilb_status_t
304 305 ilbd_destroy_hc(const char *hc_name, const struct passwd *ps,
305 306 ucred_t *ucredp)
306 307 {
307 308 ilb_status_t ret;
308 309 ilbd_hc_t *hc;
309 310
310 311 /*
311 312 * No need to check ps == NULL, daemon won't call any destroy func
312 313 * at start up.
313 314 */
314 315 ret = ilbd_check_client_config_auth(ps);
315 316 if (ret != ILB_STATUS_OK) {
316 317 ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
317 318 ret, ucredp);
318 319 return (ret);
319 320 }
320 321
321 322 hc = ilbd_get_hc(hc_name);
322 323 if (hc == NULL) {
323 324 logdebug("ilbd_destroy_hc: healthcheck %s does not exist",
324 325 hc_name);
325 326 ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
326 327 ILB_STATUS_ENOENT, ucredp);
327 328 return (ILB_STATUS_ENOENT);
328 329 }
329 330
330 331 /* If hc is in use, cannot delete it */
331 332 if (hc->ihc_rule_cnt > 0) {
332 333 logdebug("ilbd_destroy_hc: healthcheck %s is associated"
333 334 " with a rule - cannot remove", hc_name);
334 335 ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
335 336 ILB_STATUS_INUSE, ucredp);
336 337 return (ILB_STATUS_INUSE);
337 338 }
338 339
339 340 if ((ret = ilbd_destroy_pg(ILBD_SCF_HC, hc_name)) !=
340 341 ILB_STATUS_OK) {
341 342 logdebug("ilbd_destroy_hc: cannot destroy healthcheck %s "
342 343 "property group", hc_name);
343 344 ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
344 345 ret, ucredp);
345 346 return (ret);
346 347 }
347 348
348 349 list_remove(&ilbd_hc_list, hc);
349 350 list_destroy(&hc->ihc_rules);
350 351 free(hc);
351 352 ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC, ret, ucredp);
352 353 return (ret);
353 354 }
354 355
355 356 /*
356 357 * Given a hc object name, return its information. Used by libilb to
357 358 * get hc info.
358 359 */
359 360 ilb_status_t
360 361 ilbd_get_hc_info(const char *hc_name, uint32_t *rbuf, size_t *rbufsz)
361 362 {
362 363 ilbd_hc_t *hc;
363 364 ilb_hc_info_t *hc_info;
364 365 ilb_comm_t *ic = (ilb_comm_t *)rbuf;
365 366
366 367 hc = ilbd_get_hc(hc_name);
367 368 if (hc == NULL) {
368 369 logdebug("%s: healthcheck %s does not exist", __func__,
369 370 hc_name);
370 371 return (ILB_STATUS_ENOENT);
371 372 }
372 373 ilbd_reply_ok(rbuf, rbufsz);
373 374 hc_info = (ilb_hc_info_t *)&ic->ic_data;
374 375
375 376 (void) strlcpy(hc_info->hci_name, hc->ihc_name, sizeof (hc->ihc_name));
376 377 (void) strlcpy(hc_info->hci_test, hc->ihc_test, sizeof (hc->ihc_test));
377 378 hc_info->hci_timeout = hc->ihc_timeout;
378 379 hc_info->hci_count = hc->ihc_count;
379 380 hc_info->hci_interval = hc->ihc_interval;
380 381 hc_info->hci_def_ping = hc->ihc_def_ping;
381 382
382 383 *rbufsz += sizeof (ilb_hc_info_t);
383 384
384 385 return (ILB_STATUS_OK);
385 386 }
386 387
387 388 static void
388 389 ilbd_hc_copy_srvs(uint32_t *rbuf, size_t *rbufsz, ilbd_hc_rule_t *hc_rule,
389 390 const char *rulename)
390 391 {
391 392 ilbd_hc_srv_t *tmp_srv;
392 393 ilb_hc_srv_t *dst_srv;
393 394 ilb_hc_rule_srv_t *srvs;
394 395 size_t tmp_rbufsz;
395 396 int i;
396 397
397 398 tmp_rbufsz = *rbufsz;
398 399 /* Set up the reply buffer. rbufsz will be set to the new size. */
399 400 ilbd_reply_ok(rbuf, rbufsz);
400 401
401 402 /* Calculate how much space is left for holding server info. */
402 403 *rbufsz += sizeof (ilb_hc_rule_srv_t);
403 404 tmp_rbufsz -= *rbufsz;
404 405
405 406 srvs = (ilb_hc_rule_srv_t *)&((ilb_comm_t *)rbuf)->ic_data;
406 407
407 408 tmp_srv = list_head(&hc_rule->hcr_servers);
408 409 for (i = 0; tmp_srv != NULL && tmp_rbufsz >= sizeof (*dst_srv); i++) {
409 410 dst_srv = &srvs->rs_srvs[i];
410 411
411 412 (void) strlcpy(dst_srv->hcs_rule_name, rulename, ILB_NAMESZ);
412 413 (void) strlcpy(dst_srv->hcs_ID, tmp_srv->shc_sg_srv->sgs_srvID,
413 414 ILB_NAMESZ);
414 415 (void) strlcpy(dst_srv->hcs_hc_name,
415 416 tmp_srv->shc_hc->ihc_name, ILB_NAMESZ);
416 417 dst_srv->hcs_IP = tmp_srv->shc_sg_srv->sgs_addr;
417 418 dst_srv->hcs_fail_cnt = tmp_srv->shc_fail_cnt;
418 419 dst_srv->hcs_status = tmp_srv->shc_status;
419 420 dst_srv->hcs_rtt = tmp_srv->shc_rtt;
420 421 dst_srv->hcs_lasttime = tmp_srv->shc_lasttime;
421 422 dst_srv->hcs_nexttime = tmp_srv->shc_nexttime;
422 423
423 424 tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv);
424 425 tmp_rbufsz -= sizeof (*dst_srv);
425 426 }
426 427 srvs->rs_num_srvs = i;
427 428 *rbufsz += i * sizeof (*dst_srv);
428 429 }
429 430
430 431 /*
431 432 * Given a rule name, return the hc status of its servers.
432 433 */
433 434 ilb_status_t
434 435 ilbd_get_hc_srvs(const char *rulename, uint32_t *rbuf, size_t *rbufsz)
435 436 {
436 437 ilbd_hc_t *hc;
437 438 ilbd_hc_rule_t *hc_rule;
438 439
439 440 for (hc = list_head(&ilbd_hc_list); hc != NULL;
440 441 hc = list_next(&ilbd_hc_list, hc)) {
441 442 for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
442 443 hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
443 444 if (strcasecmp(hc_rule->hcr_rule->irl_name,
444 445 rulename) != 0) {
445 446 continue;
446 447 }
447 448 ilbd_hc_copy_srvs(rbuf, rbufsz, hc_rule, rulename);
448 449 return (ILB_STATUS_OK);
449 450 }
450 451 }
451 452 return (ILB_STATUS_RULE_NO_HC);
452 453 }
453 454
454 455 /*
455 456 * Initialize the hc timer and associate the notification of timeout to
456 457 * the given event port.
457 458 */
458 459 void
459 460 ilbd_hc_timer_init(int ev_port, ilbd_timer_event_obj_t *ev_obj)
460 461 {
461 462 struct sigevent sigev;
462 463 port_notify_t notify;
463 464
464 465 if ((ilbd_hc_timer_q = iu_tq_create()) == NULL) {
465 466 logerr("%s: cannot create hc timer queue", __func__);
466 467 exit(EXIT_FAILURE);
467 468 }
468 469 hc_timer_restarted = B_FALSE;
469 470
470 471 ev_obj->ev = ILBD_EVENT_TIMER;
471 472 ev_obj->timerid = -1;
472 473
473 474 notify.portnfy_port = ev_port;
474 475 notify.portnfy_user = ev_obj;
475 476 sigev.sigev_notify = SIGEV_PORT;
476 477 sigev.sigev_value.sival_ptr = ¬ify;
477 478 if (timer_create(CLOCK_REALTIME, &sigev, &ev_obj->timerid) == -1) {
478 479 logerr("%s: cannot create timer", __func__);
479 480 exit(EXIT_FAILURE);
480 481 }
481 482 }
482 483
483 484 /*
484 485 * HC timeout handler.
485 486 */
486 487 void
487 488 ilbd_hc_timeout(void)
488 489 {
489 490 (void) iu_expire_timers(ilbd_hc_timer_q);
490 491 hc_timer_restarted = B_TRUE;
491 492 }
492 493
493 494 /*
494 495 * Set up the timer to fire at the earliest timeout.
495 496 */
496 497 void
497 498 ilbd_hc_timer_update(ilbd_timer_event_obj_t *ev_obj)
498 499 {
499 500 itimerspec_t itimeout;
500 501 int timeout;
501 502
502 503 /*
503 504 * There is no change on the timer list, so no need to set up the
504 505 * timer again.
505 506 */
506 507 if (!hc_timer_restarted)
507 508 return;
508 509
509 510 restart:
510 511 if ((timeout = iu_earliest_timer(ilbd_hc_timer_q)) == INFTIM) {
511 512 hc_timer_restarted = B_FALSE;
512 513 return;
513 514 } else if (timeout == 0) {
514 515 /*
515 516 * Handle the timeout immediately. After that (clearing all
516 517 * the expired timers), check to see if there are still
517 518 * timers running. If yes, start them.
518 519 */
519 520 (void) iu_expire_timers(ilbd_hc_timer_q);
520 521 goto restart;
521 522 }
522 523
523 524 itimeout.it_value.tv_sec = timeout / MILLISEC + 1;
524 525 itimeout.it_value.tv_nsec = 0;
525 526 itimeout.it_interval.tv_sec = 0;
526 527 itimeout.it_interval.tv_nsec = 0;
527 528
528 529 /*
529 530 * Failure to set a timeout is "OK" since hopefully there will be
530 531 * other events and timer_settime() will be called again. So
531 532 * we will only miss some timeouts. But in the worst case, no event
532 533 * will happen and ilbd will get stuck...
533 534 */
534 535 if (timer_settime(ev_obj->timerid, 0, &itimeout, NULL) == -1)
535 536 logerr("%s: cannot set timer", __func__);
536 537 hc_timer_restarted = B_FALSE;
537 538 }
538 539
539 540 /*
540 541 * Kill the probe process of a server.
541 542 */
542 543 static void
543 544 ilbd_hc_kill_probe(ilbd_hc_srv_t *srv)
544 545 {
545 546 /*
546 547 * First dissociate the fd from the event port. It should not
547 548 * fail.
548 549 */
549 550 if (port_dissociate(srv->shc_ev_port, PORT_SOURCE_FD,
550 551 srv->shc_child_fd) != 0) {
551 552 logdebug("%s: port_dissociate: %s", __func__, strerror(errno));
552 553 }
553 554 (void) close(srv->shc_child_fd);
554 555 free(srv->shc_ev);
555 556 srv->shc_ev = NULL;
556 557
557 558 /* Then kill the probe process. */
558 559 if (kill(srv->shc_child_pid, SIGKILL) != 0) {
559 560 logerr("%s: rule %s server %s: %s", __func__,
560 561 srv->shc_hc_rule->hcr_rule->irl_name,
561 562 srv->shc_sg_srv->sgs_srvID, strerror(errno));
562 563 }
563 564 /* Should not fail... */
564 565 if (waitpid(srv->shc_child_pid, NULL, 0) != srv->shc_child_pid) {
565 566 logdebug("%s: waitpid: rule %s server %s", __func__,
566 567 srv->shc_hc_rule->hcr_rule->irl_name,
567 568 srv->shc_sg_srv->sgs_srvID);
568 569 }
569 570 srv->shc_child_pid = 0;
570 571 }
571 572
572 573 /*
573 574 * Disable the server, either because the server is dead or because a timer
574 575 * cannot be started for this server. Note that this only affects the
575 576 * transient configuration, meaning only in memory. The persistent
576 577 * configuration is not affected.
577 578 */
578 579 static void
579 580 ilbd_mark_server_disabled(ilbd_hc_srv_t *srv)
580 581 {
581 582 srv->shc_status = ILB_HCS_DISABLED;
582 583
583 584 /* Disable the server in kernel. */
584 585 if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
585 586 srv->shc_hc_rule->hcr_rule->irl_name,
586 587 stat_declare_srv_dead) != ILB_STATUS_OK) {
587 588 logerr("%s: cannot disable server in kernel: rule %s "
588 589 "server %s", __func__,
589 590 srv->shc_hc_rule->hcr_rule->irl_name,
590 591 srv->shc_sg_srv->sgs_srvID);
591 592 }
592 593 }
593 594
594 595 /*
595 596 * A probe fails, set the state of the server.
596 597 */
597 598 static void
598 599 ilbd_set_fail_state(ilbd_hc_srv_t *srv)
599 600 {
600 601 if (++srv->shc_fail_cnt < srv->shc_hc->ihc_count) {
601 602 /* Probe again */
602 603 ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
603 604 return;
604 605 }
605 606
606 607 logdebug("%s: rule %s server %s fails %u", __func__,
607 608 srv->shc_hc_rule->hcr_rule->irl_name, srv->shc_sg_srv->sgs_srvID,
608 609 srv->shc_fail_cnt);
609 610
610 611 /*
611 612 * If this is a ping test, mark the server as
612 613 * unreachable instead of dead.
613 614 */
614 615 if (srv->shc_hc->ihc_test_type == ILBD_HC_PING ||
615 616 srv->shc_state == ilbd_hc_def_pinging) {
616 617 srv->shc_status = ILB_HCS_UNREACH;
617 618 } else {
618 619 srv->shc_status = ILB_HCS_DEAD;
619 620 }
620 621
621 622 /* Disable the server in kernel. */
622 623 if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
623 624 srv->shc_hc_rule->hcr_rule->irl_name, stat_declare_srv_dead) !=
624 625 ILB_STATUS_OK) {
625 626 logerr("%s: cannot disable server in kernel: rule %s "
626 627 "server %s", __func__,
627 628 srv->shc_hc_rule->hcr_rule->irl_name,
628 629 srv->shc_sg_srv->sgs_srvID);
629 630 }
630 631
631 632 /* Still keep probing in case the server is alive again. */
632 633 if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
633 634 /* Only thing to do is to disable the server... */
634 635 logerr("%s: cannot restart timer: rule %s server %s", __func__,
635 636 srv->shc_hc_rule->hcr_rule->irl_name,
636 637 srv->shc_sg_srv->sgs_srvID);
637 638 srv->shc_status = ILB_HCS_DISABLED;
638 639 }
639 640 }
640 641
641 642 /*
642 643 * A probe process has not returned for the ihc_timeout period, we should
643 644 * kill it. This function is the handler of this.
644 645 */
645 646 /* ARGSUSED */
646 647 static void
647 648 ilbd_hc_kill_timer(iu_tq_t *tq, void *arg)
648 649 {
649 650 ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;
650 651
651 652 ilbd_hc_kill_probe(srv);
652 653 ilbd_set_fail_state(srv);
653 654 }
654 655
655 656 /*
656 657 * Probe timeout handler. Send out the appropriate probe.
657 658 */
658 659 /* ARGSUSED */
659 660 static void
660 661 ilbd_hc_probe_timer(iu_tq_t *tq, void *arg)
661 662 {
662 663 ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;
663 664
664 665 /*
665 666 * If starting the probe fails, just pretend that the timeout has
666 667 * extended.
667 668 */
668 669 if (!ilbd_run_probe(srv)) {
669 670 /*
670 671 * If we cannot restart the timer, the only thing we can do
671 672 * is to disable this server. Hopefully the sys admin will
672 673 * notice this and enable this server again later.
673 674 */
674 675 if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
675 676 logerr("%s: cannot restart timer: rule %s server %s, "
676 677 "disabling it", __func__,
677 678 srv->shc_hc_rule->hcr_rule->irl_name,
678 679 srv->shc_sg_srv->sgs_srvID);
679 680 ilbd_mark_server_disabled(srv);
680 681 }
681 682 return;
682 683 }
683 684
684 685 /*
685 686 * Similar to above, if kill timer cannot be started, disable the
686 687 * server.
687 688 */
688 689 if ((srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q,
689 690 srv->shc_hc->ihc_timeout, ilbd_hc_kill_timer, srv)) == -1) {
690 691 logerr("%s: cannot start kill timer: rule %s server %s, "
691 692 "disabling it", __func__,
692 693 srv->shc_hc_rule->hcr_rule->irl_name,
693 694 srv->shc_sg_srv->sgs_srvID);
694 695 ilbd_mark_server_disabled(srv);
695 696 }
696 697 hc_timer_restarted = B_TRUE;
697 698 }
698 699
699 700 /* Restart the periodic timer for a given server. */
700 701 static ilb_status_t
701 702 ilbd_hc_restart_timer(ilbd_hc_t *hc, ilbd_hc_srv_t *srv)
702 703 {
703 704 int timeout;
704 705
705 706 /* Don't allow the timeout interval to be less than 1s */
706 707 timeout = MAX((hc->ihc_interval >> 1) + (gethrtime() %
707 708 (hc->ihc_interval + 1)), 1);
708 709
709 710 /*
710 711 * If the probe is actually a ping probe, there is no need to
711 712 * do default pinging. Just skip the step.
712 713 */
713 714 if (hc->ihc_def_ping && hc->ihc_test_type != ILBD_HC_PING)
714 715 srv->shc_state = ilbd_hc_def_pinging;
715 716 else
716 717 srv->shc_state = ilbd_hc_probing;
717 718 srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q, timeout,
718 719 ilbd_hc_probe_timer, srv);
719 720
720 721 if (srv->shc_tid == -1)
721 722 return (ILB_STATUS_TIMER);
722 723 srv->shc_lasttime = time(NULL);
723 724 srv->shc_nexttime = time(NULL) + timeout;
724 725
725 726 hc_timer_restarted = B_TRUE;
726 727 return (ILB_STATUS_OK);
727 728 }
728 729
729 730 /* Helper routine to associate a server with its hc object. */
730 731 static ilb_status_t
731 732 ilbd_hc_srv_add(ilbd_hc_t *hc, ilbd_hc_rule_t *hc_rule,
732 733 const ilb_sg_srv_t *srv, int ev_port)
733 734 {
734 735 ilbd_hc_srv_t *new_srv;
735 736 ilb_status_t ret;
736 737
737 738 if ((new_srv = calloc(1, sizeof (ilbd_hc_srv_t))) == NULL)
738 739 return (ILB_STATUS_ENOMEM);
739 740 new_srv->shc_hc = hc;
740 741 new_srv->shc_hc_rule = hc_rule;
741 742 new_srv->shc_sg_srv = srv;
742 743 new_srv->shc_ev_port = ev_port;
743 744 new_srv->shc_tid = -1;
744 745 new_srv->shc_nexttime = time(NULL);
745 746 new_srv->shc_lasttime = new_srv->shc_nexttime;
746 747
747 748 if ((hc_rule->hcr_rule->irl_flags & ILB_FLAGS_RULE_ENABLED) &&
748 749 ILB_IS_SRV_ENABLED(srv->sgs_flags)) {
749 750 new_srv->shc_status = ILB_HCS_UNINIT;
750 751 ret = ilbd_hc_restart_timer(hc, new_srv);
751 752 if (ret != ILB_STATUS_OK) {
752 753 free(new_srv);
753 754 return (ret);
754 755 }
755 756 } else {
756 757 new_srv->shc_status = ILB_HCS_DISABLED;
757 758 }
758 759
759 760 list_insert_tail(&hc_rule->hcr_servers, new_srv);
760 761 return (ILB_STATUS_OK);
761 762 }
762 763
763 764 /* Handy macro to cancel a server's timer. */
764 765 #define HC_CANCEL_TIMER(srv) \
765 766 { \
766 767 void *arg; \
767 768 int ret; \
768 769 if ((srv)->shc_tid != -1) { \
769 770 ret = iu_cancel_timer(ilbd_hc_timer_q, (srv)->shc_tid, &arg); \
770 771 (srv)->shc_tid = -1; \
771 772 assert(ret == 1); \
772 773 assert(arg == (srv)); \
773 774 } \
774 775 hc_timer_restarted = B_TRUE; \
775 776 }
776 777
777 778 /* Helper routine to dissociate a server from its hc object. */
778 779 static ilb_status_t
779 780 ilbd_hc_srv_rem(ilbd_hc_rule_t *hc_rule, const ilb_sg_srv_t *srv)
780 781 {
781 782 ilbd_hc_srv_t *tmp_srv;
782 783
783 784 for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
784 785 tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
785 786 if (tmp_srv->shc_sg_srv == srv) {
786 787 list_remove(&hc_rule->hcr_servers, tmp_srv);
787 788 HC_CANCEL_TIMER(tmp_srv);
788 789 if (tmp_srv->shc_child_pid != 0)
789 790 ilbd_hc_kill_probe(tmp_srv);
790 791 free(tmp_srv);
791 792 return (ILB_STATUS_OK);
792 793 }
793 794 }
794 795 return (ILB_STATUS_ENOENT);
795 796 }
796 797
797 798 /* Helper routine to dissociate all servers of a rule from its hc object. */
798 799 static void
799 800 ilbd_hc_srv_rem_all(ilbd_hc_rule_t *hc_rule)
800 801 {
801 802 ilbd_hc_srv_t *srv;
802 803
803 804 while ((srv = list_remove_head(&hc_rule->hcr_servers)) != NULL) {
804 805 HC_CANCEL_TIMER(srv);
805 806 if (srv->shc_child_pid != 0)
806 807 ilbd_hc_kill_probe(srv);
807 808 free(srv);
808 809 }
809 810 }
810 811
811 812 /* Associate a rule with its hc object. */
812 813 ilb_status_t
813 814 ilbd_hc_associate_rule(const ilbd_rule_t *rule, int ev_port)
814 815 {
815 816 ilbd_hc_t *hc;
816 817 ilbd_hc_rule_t *hc_rule;
817 818 ilb_status_t ret;
818 819 ilbd_sg_t *sg;
819 820 ilbd_srv_t *ilbd_srv;
820 821
821 822 /* The rule is assumed to be initialized appropriately. */
822 823 if ((hc = ilbd_get_hc(rule->irl_hcname)) == NULL) {
823 824 logdebug("ilbd_hc_associate_rule: healthcheck %s does not "
824 825 "exist", rule->irl_hcname);
825 826 return (ILB_STATUS_ENOHCINFO);
826 827 }
827 828 if ((hc->ihc_test_type == ILBD_HC_TCP &&
828 829 rule->irl_proto != IPPROTO_TCP) ||
829 830 (hc->ihc_test_type == ILBD_HC_UDP &&
830 831 rule->irl_proto != IPPROTO_UDP)) {
831 832 return (ILB_STATUS_RULE_HC_MISMATCH);
832 833 }
833 834 if ((hc_rule = calloc(1, sizeof (ilbd_hc_rule_t))) == NULL) {
834 835 logdebug("ilbd_hc_associate_rule: out of memory");
835 836 return (ILB_STATUS_ENOMEM);
836 837 }
837 838
838 839 hc_rule->hcr_rule = rule;
839 840 list_create(&hc_rule->hcr_servers, sizeof (ilbd_hc_srv_t),
840 841 offsetof(ilbd_hc_srv_t, shc_srv_link));
841 842
842 843 /* Add all the servers. */
843 844 sg = rule->irl_sg;
844 845 for (ilbd_srv = list_head(&sg->isg_srvlist); ilbd_srv != NULL;
845 846 ilbd_srv = list_next(&sg->isg_srvlist, ilbd_srv)) {
846 847 if ((ret = ilbd_hc_srv_add(hc, hc_rule, &ilbd_srv->isv_srv,
847 848 ev_port)) != ILB_STATUS_OK) {
848 849 /* Remove all previously added servers */
849 850 ilbd_hc_srv_rem_all(hc_rule);
850 851 list_destroy(&hc_rule->hcr_servers);
851 852 free(hc_rule);
852 853 return (ret);
853 854 }
854 855 }
855 856 list_insert_tail(&hc->ihc_rules, hc_rule);
856 857 hc->ihc_rule_cnt++;
857 858
858 859 return (ILB_STATUS_OK);
859 860 }
860 861
861 862 /* Dissociate a rule from its hc object. */
862 863 ilb_status_t
863 864 ilbd_hc_dissociate_rule(const ilbd_rule_t *rule)
864 865 {
865 866 ilbd_hc_t *hc;
866 867 ilbd_hc_rule_t *hc_rule;
867 868
868 869 /* The rule is assumed to be initialized appropriately. */
869 870 if ((hc = ilbd_get_hc(rule->irl_hcname)) == NULL) {
870 871 logdebug("ilbd_hc_dissociate_rule: healthcheck %s does not "
871 872 "exist", rule->irl_hcname);
872 873 return (ILB_STATUS_ENOENT);
873 874 }
874 875 for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
875 876 hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
876 877 if (hc_rule->hcr_rule == rule)
877 878 break;
878 879 }
879 880 if (hc_rule == NULL) {
880 881 logdebug("ilbd_hc_dissociate_rule: rule %s is not associated "
881 882 "with healtcheck %s", rule->irl_hcname, hc->ihc_name);
882 883 return (ILB_STATUS_ENOENT);
883 884 }
884 885 ilbd_hc_srv_rem_all(hc_rule);
885 886 list_remove(&hc->ihc_rules, hc_rule);
886 887 hc->ihc_rule_cnt--;
887 888 list_destroy(&hc_rule->hcr_servers);
888 889 free(hc_rule);
889 890 return (ILB_STATUS_OK);
890 891 }
891 892
892 893 /*
893 894 * Given a hc object name and a rule, check to see if the rule is associated
894 895 * with the hc object. If it is, the hc object is returned in **hc and the
895 896 * ilbd_hc_rule_t is returned in **hc_rule.
896 897 */
897 898 static boolean_t
898 899 ilbd_hc_check_rule(const char *hc_name, const ilbd_rule_t *rule,
899 900 ilbd_hc_t **hc, ilbd_hc_rule_t **hc_rule)
900 901 {
901 902 ilbd_hc_t *tmp_hc;
902 903 ilbd_hc_rule_t *tmp_hc_rule;
903 904
904 905 if ((tmp_hc = ilbd_get_hc(hc_name)) == NULL)
905 906 return (B_FALSE);
906 907 for (tmp_hc_rule = list_head(&tmp_hc->ihc_rules); tmp_hc_rule != NULL;
907 908 tmp_hc_rule = list_next(&tmp_hc->ihc_rules, tmp_hc_rule)) {
908 909 if (tmp_hc_rule->hcr_rule == rule) {
909 910 *hc = tmp_hc;
910 911 *hc_rule = tmp_hc_rule;
911 912 return (B_TRUE);
912 913 }
913 914 }
914 915 return (B_FALSE);
915 916 }
916 917
917 918 /* Associate a server with its hc object. */
918 919 ilb_status_t
919 920 ilbd_hc_add_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv,
920 921 int ev_port)
921 922 {
922 923 ilbd_hc_t *hc;
923 924 ilbd_hc_rule_t *hc_rule;
924 925
925 926 if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
926 927 return (ILB_STATUS_ENOENT);
927 928 return (ilbd_hc_srv_add(hc, hc_rule, srv, ev_port));
928 929 }
929 930
930 931 /* Dissociate a server from its hc object. */
931 932 ilb_status_t
932 933 ilbd_hc_del_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
933 934 {
934 935 ilbd_hc_t *hc;
935 936 ilbd_hc_rule_t *hc_rule;
936 937
937 938 if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
938 939 return (ILB_STATUS_ENOENT);
939 940 return (ilbd_hc_srv_rem(hc_rule, srv));
940 941 }
941 942
942 943 /* Helper routine to enable/disable a server's hc probe. */
943 944 static ilb_status_t
944 945 ilbd_hc_toggle_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv,
945 946 boolean_t enable)
946 947 {
947 948 ilbd_hc_t *hc;
948 949 ilbd_hc_rule_t *hc_rule;
949 950 ilbd_hc_srv_t *tmp_srv;
950 951 ilb_status_t ret;
951 952
952 953 if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
953 954 return (ILB_STATUS_ENOENT);
954 955 for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
955 956 tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
956 957 if (tmp_srv->shc_sg_srv != srv) {
957 958 continue;
958 959 }
959 960 if (enable) {
960 961 if (tmp_srv->shc_status == ILB_HCS_DISABLED) {
961 962 ret = ilbd_hc_restart_timer(hc, tmp_srv);
962 963 if (ret != ILB_STATUS_OK) {
963 964 logerr("%s: cannot start timers for "
964 965 "rule %s server %s", __func__,
965 966 rule->irl_name,
966 967 tmp_srv->shc_sg_srv->sgs_srvID);
967 968 return (ret);
968 969 }
969 970 /* Start from fresh... */
970 971 tmp_srv->shc_status = ILB_HCS_UNINIT;
971 972 tmp_srv->shc_rtt = 0;
972 973 tmp_srv->shc_fail_cnt = 0;
973 974 }
974 975 } else {
975 976 if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
976 977 tmp_srv->shc_status = ILB_HCS_DISABLED;
977 978 HC_CANCEL_TIMER(tmp_srv);
978 979 if (tmp_srv->shc_child_pid != 0)
979 980 ilbd_hc_kill_probe(tmp_srv);
980 981 }
981 982 }
982 983 return (ILB_STATUS_OK);
983 984 }
984 985 return (ILB_STATUS_ENOENT);
985 986 }
986 987
987 988 ilb_status_t
988 989 ilbd_hc_enable_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
989 990 {
990 991 return (ilbd_hc_toggle_server(rule, srv, B_TRUE));
991 992 }
992 993
993 994 ilb_status_t
994 995 ilbd_hc_disable_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
995 996 {
996 997 return (ilbd_hc_toggle_server(rule, srv, B_FALSE));
997 998 }
998 999
999 1000 /*
1000 1001 * Helper routine to enable/disable a rule's hc probe (including all its
1001 1002 * servers).
1002 1003 */
1003 1004 static ilb_status_t
1004 1005 ilbd_hc_toggle_rule(const ilbd_rule_t *rule, boolean_t enable)
1005 1006 {
1006 1007 ilbd_hc_t *hc;
1007 1008 ilbd_hc_rule_t *hc_rule;
1008 1009 ilbd_hc_srv_t *tmp_srv;
1009 1010 int ret;
1010 1011
1011 1012 if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
1012 1013 return (ILB_STATUS_ENOENT);
1013 1014
1014 1015 for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
1015 1016 tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
1016 1017 if (enable) {
1017 1018 /*
1018 1019 * If the server is disabled in the rule, do not
1019 1020 * restart its timer.
1020 1021 */
1021 1022 if (tmp_srv->shc_status == ILB_HCS_DISABLED &&
1022 1023 ILB_IS_SRV_ENABLED(
1023 1024 tmp_srv->shc_sg_srv->sgs_flags)) {
1024 1025 ret = ilbd_hc_restart_timer(hc, tmp_srv);
1025 1026 if (ret != ILB_STATUS_OK) {
1026 1027 logerr("%s: cannot start timers for "
1027 1028 "rule %s server %s", __func__,
1028 1029 rule->irl_name,
1029 1030 tmp_srv->shc_sg_srv->sgs_srvID);
1030 1031 goto rollback;
1031 1032 } else {
1032 1033 /* Start from fresh... */
1033 1034 tmp_srv->shc_status = ILB_HCS_UNINIT;
1034 1035 tmp_srv->shc_rtt = 0;
1035 1036 tmp_srv->shc_fail_cnt = 0;
1036 1037 }
1037 1038 }
1038 1039 } else {
1039 1040 if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
1040 1041 HC_CANCEL_TIMER(tmp_srv);
1041 1042 tmp_srv->shc_status = ILB_HCS_DISABLED;
1042 1043 if (tmp_srv->shc_child_pid != 0)
1043 1044 ilbd_hc_kill_probe(tmp_srv);
1044 1045 }
1045 1046 }
1046 1047 }
1047 1048 return (ILB_STATUS_OK);
1048 1049 rollback:
1049 1050 enable = !enable;
1050 1051 for (tmp_srv = list_prev(&hc_rule->hcr_servers, tmp_srv);
1051 1052 tmp_srv != NULL;
1052 1053 tmp_srv = list_prev(&hc_rule->hcr_servers, tmp_srv)) {
1053 1054 if (enable) {
1054 1055 if (tmp_srv->shc_status == ILB_HCS_DISABLED &&
1055 1056 ILB_IS_SRV_ENABLED(
1056 1057 tmp_srv->shc_sg_srv->sgs_flags)) {
1057 1058 (void) ilbd_hc_restart_timer(hc, tmp_srv);
1058 1059 tmp_srv->shc_status = ILB_HCS_UNINIT;
1059 1060 tmp_srv->shc_rtt = 0;
1060 1061 tmp_srv->shc_fail_cnt = 0;
1061 1062 }
1062 1063 } else {
1063 1064 if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
1064 1065 HC_CANCEL_TIMER(tmp_srv);
1065 1066 tmp_srv->shc_status = ILB_HCS_DISABLED;
1066 1067 if (tmp_srv->shc_child_pid != 0)
1067 1068 ilbd_hc_kill_probe(tmp_srv);
1068 1069 }
1069 1070 }
1070 1071 }
1071 1072 return (ret);
1072 1073 }
1073 1074
1074 1075 ilb_status_t
1075 1076 ilbd_hc_enable_rule(const ilbd_rule_t *rule)
1076 1077 {
1077 1078 return (ilbd_hc_toggle_rule(rule, B_TRUE));
1078 1079 }
1079 1080
1080 1081 ilb_status_t
1081 1082 ilbd_hc_disable_rule(const ilbd_rule_t *rule)
1082 1083 {
1083 1084 return (ilbd_hc_toggle_rule(rule, B_FALSE));
1084 1085 }
1085 1086
1086 1087 static const char *
1087 1088 topo_2_str(ilb_topo_t topo)
1088 1089 {
1089 1090 switch (topo) {
1090 1091 case ILB_TOPO_DSR:
1091 1092 return ("DSR");
1092 1093 case ILB_TOPO_NAT:
1093 1094 return ("NAT");
1094 1095 case ILB_TOPO_HALF_NAT:
1095 1096 return ("HALF_NAT");
1096 1097 default:
1097 1098 /* Should not happen. */
1098 1099 logerr("%s: unknown topology", __func__);
1099 1100 break;
1100 1101 }
1101 1102 return ("");
1102 1103 }
1103 1104
1104 1105 /*
1105 1106 * Create the argument list to be passed to a hc probe command.
1106 1107 * The passed in argv is assumed to have HC_PROBE_ARGC elements.
1107 1108 */
1108 1109 static boolean_t
1109 1110 create_argv(ilbd_hc_srv_t *srv, char *argv[])
1110 1111 {
1111 1112 char buf[INET6_ADDRSTRLEN];
1112 1113 ilbd_rule_t const *rule;
1113 1114 ilb_sg_srv_t const *sg_srv;
1114 1115 struct in_addr v4_addr;
1115 1116 in_port_t port;
1116 1117 int i;
1117 1118
1118 1119 rule = srv->shc_hc_rule->hcr_rule;
1119 1120 sg_srv = srv->shc_sg_srv;
1120 1121
1121 1122 if (srv->shc_state == ilbd_hc_def_pinging) {
1122 1123 if ((argv[0] = strdup(ILB_PROBE_PING)) == NULL)
1123 1124 return (B_FALSE);
1124 1125 } else {
1125 1126 switch (srv->shc_hc->ihc_test_type) {
1126 1127 case ILBD_HC_USER:
1127 1128 if ((argv[0] = strdup(srv->shc_hc->ihc_test)) == NULL)
1128 1129 return (B_FALSE);
1129 1130 break;
1130 1131 case ILBD_HC_TCP:
1131 1132 case ILBD_HC_UDP:
1132 1133 if ((argv[0] = strdup(ILB_PROBE_PROTO)) ==
1133 1134 NULL) {
1134 1135 return (B_FALSE);
1135 1136 }
1136 1137 break;
1137 1138 case ILBD_HC_PING:
1138 1139 if ((argv[0] = strdup(ILB_PROBE_PING)) == NULL) {
1139 1140 return (B_FALSE);
1140 1141 }
1141 1142 break;
1142 1143 }
1143 1144 }
1144 1145
1145 1146 /*
1146 1147 * argv[1] is the VIP.
1147 1148 *
1148 1149 * Right now, the VIP and the backend server addresses should be
1149 1150 * in the same IP address family. Here we don't do that in case
1150 1151 * this assumption is changed in future.
1151 1152 */
1152 1153 if (IN6_IS_ADDR_V4MAPPED(&rule->irl_vip)) {
1153 1154 IN6_V4MAPPED_TO_INADDR(&rule->irl_vip, &v4_addr);
1154 1155 if (inet_ntop(AF_INET, &v4_addr, buf, sizeof (buf)) == NULL)
1155 1156 goto cleanup;
1156 1157 } else {
1157 1158 if (inet_ntop(AF_INET6, &rule->irl_vip, buf,
1158 1159 sizeof (buf)) == NULL) {
1159 1160 goto cleanup;
1160 1161 }
1161 1162 }
1162 1163 if ((argv[1] = strdup(buf)) == NULL)
1163 1164 goto cleanup;
1164 1165
1165 1166 /*
1166 1167 * argv[2] is the backend server address.
1167 1168 */
1168 1169 if (IN6_IS_ADDR_V4MAPPED(&sg_srv->sgs_addr)) {
1169 1170 IN6_V4MAPPED_TO_INADDR(&sg_srv->sgs_addr, &v4_addr);
1170 1171 if (inet_ntop(AF_INET, &v4_addr, buf, sizeof (buf)) == NULL)
1171 1172 goto cleanup;
1172 1173 } else {
1173 1174 if (inet_ntop(AF_INET6, &sg_srv->sgs_addr, buf,
1174 1175 sizeof (buf)) == NULL) {
1175 1176 goto cleanup;
1176 1177 }
1177 1178 }
1178 1179 if ((argv[2] = strdup(buf)) == NULL)
1179 1180 goto cleanup;
1180 1181
1181 1182 /*
1182 1183 * argv[3] is the transport protocol used in the rule.
1183 1184 */
1184 1185 switch (rule->irl_proto) {
1185 1186 case IPPROTO_TCP:
1186 1187 argv[3] = strdup("TCP");
1187 1188 break;
1188 1189 case IPPROTO_UDP:
1189 1190 argv[3] = strdup("UDP");
1190 1191 break;
1191 1192 default:
1192 1193 logerr("%s: unknown protocol", __func__);
1193 1194 goto cleanup;
1194 1195 }
1195 1196 if (argv[3] == NULL)
1196 1197 goto cleanup;
1197 1198
1198 1199 /*
1199 1200 * argv[4] is the load balance mode, DSR, NAT, HALF-NAT.
1200 1201 */
1201 1202 if ((argv[4] = strdup(topo_2_str(rule->irl_topo))) == NULL)
1202 1203 goto cleanup;
1203 1204
1204 1205 /*
1205 1206 * argv[5] is the port range. Right now, there should only be 1 port.
1206 1207 */
1207 1208 switch (rule->irl_hcpflag) {
1208 1209 case ILB_HCI_PROBE_FIX:
1209 1210 port = ntohs(rule->irl_hcport);
1210 1211 break;
1211 1212 case ILB_HCI_PROBE_ANY: {
1212 1213 in_port_t min, max;
1213 1214
1214 1215 if (ntohs(sg_srv->sgs_minport) == 0) {
1215 1216 min = ntohs(rule->irl_minport);
1216 1217 max = ntohs(rule->irl_maxport);
1217 1218 } else {
1218 1219 min = ntohs(sg_srv->sgs_minport);
1219 1220 max = ntohs(sg_srv->sgs_maxport);
1220 1221 }
1221 1222 if (max > min)
1222 1223 port = min + gethrtime() % (max - min + 1);
1223 1224 else
1224 1225 port = min;
1225 1226 break;
1226 1227 }
1227 1228 default:
1228 1229 logerr("%s: unknown HC flag", __func__);
1229 1230 goto cleanup;
1230 1231 }
1231 1232 (void) sprintf(buf, "%d", port);
1232 1233 if ((argv[5] = strdup(buf)) == NULL)
1233 1234 goto cleanup;
1234 1235
1235 1236 /*
1236 1237 * argv[6] is the probe timeout.
1237 1238 */
1238 1239 (void) sprintf(buf, "%d", srv->shc_hc->ihc_timeout);
1239 1240 if ((argv[6] = strdup(buf)) == NULL)
1240 1241 goto cleanup;
1241 1242
1242 1243 argv[7] = NULL;
1243 1244 return (B_TRUE);
1244 1245
1245 1246 cleanup:
1246 1247 for (i = 0; i < HC_PROBE_ARGC; i++) {
1247 1248 if (argv[i] != NULL)
1248 1249 free(argv[i]);
1249 1250 }
1250 1251 return (B_FALSE);
1251 1252 }
1252 1253
1253 1254 static void
1254 1255 destroy_argv(char *argv[])
1255 1256 {
1256 1257 int i;
1257 1258
1258 1259 for (i = 0; argv[i] != NULL; i++)
1259 1260 free(argv[i]);
1260 1261 }
1261 1262
1262 1263 /* Spawn a process to run the hc probe on the given server. */
1263 1264 static boolean_t
1264 1265 ilbd_run_probe(ilbd_hc_srv_t *srv)
1265 1266 {
1266 1267 posix_spawn_file_actions_t fd_actions;
1267 1268 posix_spawnattr_t attr;
1268 1269 sigset_t child_sigset;
1269 1270 int fds[2];
1270 1271 int fdflags;
1271 1272 pid_t pid;
1272 1273 char *child_argv[HC_PROBE_ARGC];
1273 1274 ilbd_hc_probe_event_t *probe_ev;
1274 1275 char *probe_name;
1275 1276
1276 1277 bzero(child_argv, HC_PROBE_ARGC * sizeof (char *));
1277 1278 if ((probe_ev = calloc(1, sizeof (*probe_ev))) == NULL) {
1278 1279 logdebug("ilbd_run_probe: calloc");
1279 1280 return (B_FALSE);
1280 1281 }
|
↓ open down ↓ |
1245 lines elided |
↑ open up ↑ |
1281 1282
1282 1283 /* Set up a pipe to get output from probe command. */
1283 1284 if (pipe(fds) < 0) {
1284 1285 logdebug("ilbd_run_probe: cannot create pipe");
1285 1286 free(probe_ev);
1286 1287 return (B_FALSE);
1287 1288 }
1288 1289 /* Set our side of the pipe to be non-blocking */
1289 1290 if ((fdflags = fcntl(fds[0], F_GETFL, 0)) == -1) {
1290 1291 logdebug("ilbd_run_probe: fcntl(F_GETFL)");
1291 - goto cleanup;
1292 + goto cleanup_noactions;
1292 1293 }
1293 1294 if (fcntl(fds[0], F_SETFL, fdflags | O_NONBLOCK) == -1) {
1294 1295 logdebug("ilbd_run_probe: fcntl(F_SETFL)");
1295 - goto cleanup;
1296 + goto cleanup_noactions;
1296 1297 }
1297 1298
1298 1299 if (posix_spawn_file_actions_init(&fd_actions) != 0) {
1299 1300 logdebug("ilbd_run_probe: posix_spawn_file_actions_init");
1300 - goto cleanup;
1301 + goto cleanup_noactions;
1301 1302 }
1302 1303 if (posix_spawnattr_init(&attr) != 0) {
1303 1304 logdebug("ilbd_run_probe: posix_spawnattr_init");
1304 - goto cleanup;
1305 + goto cleanup_noattr;
1305 1306 }
1306 1307 if (posix_spawn_file_actions_addclose(&fd_actions, fds[0]) != 0) {
1307 1308 logdebug("ilbd_run_probe: posix_spawn_file_actions_addclose");
1308 1309 goto cleanup;
1309 1310 }
1310 1311 if (posix_spawn_file_actions_adddup2(&fd_actions, fds[1],
1311 1312 STDOUT_FILENO) != 0) {
1312 1313 logdebug("ilbd_run_probe: posix_spawn_file_actions_dup2");
1313 1314 goto cleanup;
1314 1315 }
1315 1316 if (posix_spawn_file_actions_addclose(&fd_actions, fds[1]) != 0) {
1316 1317 logdebug("ilbd_run_probe: posix_spawn_file_actions_addclose");
1317 1318 goto cleanup;
1318 1319 }
1319 1320
1320 1321 /* Reset all signal handling of the child to default. */
1321 1322 (void) sigfillset(&child_sigset);
1322 1323 if (posix_spawnattr_setsigdefault(&attr, &child_sigset) != 0) {
1323 1324 logdebug("ilbd_run_probe: posix_spawnattr_setsigdefault");
1324 1325 goto cleanup;
1325 1326 }
1326 1327 /* Don't want SIGCHLD. */
1327 1328 if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_NOSIGCHLD_NP|
1328 1329 POSIX_SPAWN_SETSIGDEF) != 0) {
1329 1330 logdebug("ilbd_run_probe: posix_spawnattr_setflags");
1330 1331 goto cleanup;
1331 1332 }
1332 1333
1333 1334 if (!create_argv(srv, child_argv)) {
1334 1335 logdebug("ilbd_run_probe: create_argv");
1335 1336 goto cleanup;
1336 1337 }
1337 1338
1338 1339 /*
1339 1340 * If we are doing default pinging or not using a user supplied
1340 1341 * probe, we should execute our standard supplied probe. The
1341 1342 * supplied probe command handles all types of probes. And the
1342 1343 * type used depends on argv[0], as filled in by create_argv().
1343 1344 */
1344 1345 if (srv->shc_state == ilbd_hc_def_pinging ||
1345 1346 srv->shc_hc->ihc_test_type != ILBD_HC_USER) {
1346 1347 probe_name = ILB_PROBE_PROTO;
1347 1348 } else {
|
↓ open down ↓ |
33 lines elided |
↑ open up ↑ |
1348 1349 probe_name = srv->shc_hc->ihc_test;
1349 1350 }
1350 1351 if (posix_spawn(&pid, probe_name, &fd_actions, &attr, child_argv,
1351 1352 NULL) != 0) {
1352 1353 logerr("%s: posix_spawn: %s for server %s: %s", __func__,
1353 1354 srv->shc_hc->ihc_test, srv->shc_sg_srv->sgs_srvID,
1354 1355 strerror(errno));
1355 1356 goto cleanup;
1356 1357 }
1357 1358
1359 + (void) posix_spawnattr_destroy(&attr);
1360 + (void) posix_spawn_file_actions_destroy(&fd_actions);
1358 1361 (void) close(fds[1]);
1359 - destroy_argv(child_argv);
1360 1362 srv->shc_child_pid = pid;
1361 1363 srv->shc_child_fd = fds[0];
1362 1364 srv->shc_ev = probe_ev;
1363 1365
1364 1366 probe_ev->ihp_ev = ILBD_EVENT_PROBE;
1365 1367 probe_ev->ihp_srv = srv;
1366 1368 probe_ev->ihp_pid = pid;
1367 1369 if (port_associate(srv->shc_ev_port, PORT_SOURCE_FD, fds[0],
1368 1370 POLLRDNORM, probe_ev) != 0) {
1369 1371 /*
1370 1372 * Need to kill the child. It will free the srv->shc_ev,
1371 1373 * which is probe_ev. So set probe_ev to NULL.
1372 1374 */
1373 1375 ilbd_hc_kill_probe(srv);
1374 1376 probe_ev = NULL;
1375 - goto cleanup;
1377 + /* posix_spawn attrs & actions already destroyed. */
1378 + goto cleanup_noactions;
1376 1379 }
1380 + destroy_argv(child_argv);
1377 1381
1378 1382 return (B_TRUE);
1379 1383
1380 1384 cleanup:
1385 + (void) posix_spawnattr_destroy(&attr);
1386 +cleanup_noattr:
1387 + (void) posix_spawn_file_actions_destroy(&fd_actions);
1388 +cleanup_noactions:
1381 1389 (void) close(fds[0]);
1382 1390 (void) close(fds[1]);
1383 1391 destroy_argv(child_argv);
1384 1392 if (probe_ev != NULL)
1385 1393 free(probe_ev);
1386 1394 return (B_FALSE);
1387 1395 }
1388 1396
1389 1397 /*
1390 1398 * Called by ild_hc_probe_return() to re-associate the fd to a child to
1391 1399 * the event port.
1392 1400 */
1393 1401 static void
1394 1402 reassociate_port(int ev_port, int fd, ilbd_hc_probe_event_t *ev)
1395 1403 {
1396 1404 if (port_associate(ev_port, PORT_SOURCE_FD, fd,
1397 1405 POLLRDNORM, ev) != 0) {
1398 1406 /*
1399 1407 * If we cannot reassociate with the port, the only
1400 1408 * thing we can do now is to kill the child and
1401 1409 * do a blocking wait here...
1402 1410 */
1403 1411 logdebug("%s: port_associate: %s", __func__, strerror(errno));
1404 1412 if (kill(ev->ihp_pid, SIGKILL) != 0)
1405 1413 logerr("%s: kill: %s", __func__, strerror(errno));
1406 1414 if (waitpid(ev->ihp_pid, NULL, 0) != ev->ihp_pid)
1407 1415 logdebug("%s: waitpid: %s", __func__, strerror(errno));
1408 1416 free(ev);
1409 1417 }
1410 1418 }
1411 1419
1412 1420 /*
1413 1421 * To handle a child probe process hanging up.
1414 1422 */
1415 1423 static void
1416 1424 ilbd_hc_child_hup(int ev_port, int fd, ilbd_hc_probe_event_t *ev)
1417 1425 {
1418 1426 ilbd_hc_srv_t *srv;
1419 1427 pid_t ret_pid;
1420 1428 int ret;
1421 1429
1422 1430 srv = ev->ihp_srv;
1423 1431
1424 1432 if (!ev->ihp_done) {
1425 1433 /* ilbd does not care about this process anymore ... */
1426 1434 ev->ihp_done = B_TRUE;
1427 1435 srv->shc_ev = NULL;
1428 1436 srv->shc_child_pid = 0;
1429 1437 HC_CANCEL_TIMER(srv);
1430 1438 ilbd_set_fail_state(srv);
1431 1439 }
1432 1440 ret_pid = waitpid(ev->ihp_pid, &ret, WNOHANG);
1433 1441 switch (ret_pid) {
1434 1442 case -1:
1435 1443 logperror("ilbd_hc_child_hup: waitpid");
1436 1444 /* FALLTHROUGH */
1437 1445 case 0:
1438 1446 /* The child has not completed the exit. Wait again. */
1439 1447 reassociate_port(ev_port, fd, ev);
1440 1448 break;
1441 1449 default:
1442 1450 /* Right now, we just ignore the exit status. */
1443 1451 if (WIFEXITED(ret))
1444 1452 ret = WEXITSTATUS(ret);
1445 1453 (void) close(fd);
1446 1454 free(ev);
1447 1455 }
1448 1456 }
1449 1457
1450 1458 /*
1451 1459 * To read the output of a child probe process.
1452 1460 */
1453 1461 static void
1454 1462 ilbd_hc_child_data(int fd, ilbd_hc_probe_event_t *ev)
1455 1463 {
1456 1464 ilbd_hc_srv_t *srv;
1457 1465 char buf[HC_MAX_PROBE_OUTPUT];
1458 1466 int ret;
1459 1467 int64_t rtt;
1460 1468
1461 1469 srv = ev->ihp_srv;
1462 1470
1463 1471 bzero(buf, HC_MAX_PROBE_OUTPUT);
1464 1472 ret = read(fd, buf, HC_MAX_PROBE_OUTPUT - 1);
1465 1473 /* Should not happen since event port should have caught this. */
1466 1474 assert(ret > 0);
1467 1475
1468 1476 /*
1469 1477 * We expect the probe command to print out the RTT only. But
1470 1478 * the command may misbehave and print out more than what we intend to
1471 1479 * read in. So need to do this check below to "flush" out all the
1472 1480 * output from the command.
1473 1481 */
1474 1482 if (!ev->ihp_done) {
1475 1483 ev->ihp_done = B_TRUE;
1476 1484 /* We don't need to know about this event anymore. */
1477 1485 srv->shc_ev = NULL;
1478 1486 srv->shc_child_pid = 0;
1479 1487 HC_CANCEL_TIMER(srv);
1480 1488 } else {
1481 1489 return;
1482 1490 }
1483 1491
1484 1492 rtt = strtoll(buf, NULL, 10);
1485 1493
1486 1494 /*
1487 1495 * -1 means the server is dead or the probe somehow fails. Treat
1488 1496 * them both as server is dead.
1489 1497 */
1490 1498 if (rtt == -1) {
1491 1499 ilbd_set_fail_state(srv);
1492 1500 return;
1493 1501 } else if (rtt > 0) {
1494 1502 /* If the returned RTT value is not valid, just ignore it. */
1495 1503 if (rtt > 0 && rtt <= UINT_MAX) {
1496 1504 /* Set rtt to be the simple smoothed average. */
1497 1505 if (srv->shc_rtt == 0) {
1498 1506 srv->shc_rtt = rtt;
1499 1507 } else {
1500 1508 srv->shc_rtt = 3 * ((srv)->shc_rtt >> 2) +
1501 1509 (rtt >> 2);
1502 1510 }
1503 1511 }
1504 1512
1505 1513 }
1506 1514
1507 1515 switch (srv->shc_state) {
1508 1516 case ilbd_hc_def_pinging:
1509 1517 srv->shc_state = ilbd_hc_probing;
1510 1518
1511 1519 /* Ping is OK, now start the probe. */
1512 1520 ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
1513 1521 break;
1514 1522 case ilbd_hc_probing:
1515 1523 srv->shc_fail_cnt = 0;
1516 1524
1517 1525 /* Server is dead before, re-enable it. */
1518 1526 if (srv->shc_status == ILB_HCS_UNREACH ||
1519 1527 srv->shc_status == ILB_HCS_DEAD) {
1520 1528 /*
1521 1529 * If enabling the server in kernel fails now,
1522 1530 * hopefully when the timer fires again later, the
1523 1531 * enabling can be done.
1524 1532 */
1525 1533 if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
1526 1534 srv->shc_hc_rule->hcr_rule->irl_name,
1527 1535 stat_declare_srv_alive) != ILB_STATUS_OK) {
1528 1536 logerr("%s: cannot enable server in kernel: "
1529 1537 " rule %s server %s", __func__,
1530 1538 srv->shc_hc_rule->hcr_rule->irl_name,
1531 1539 srv->shc_sg_srv->sgs_srvID);
1532 1540 } else {
1533 1541 srv->shc_status = ILB_HCS_ALIVE;
1534 1542 }
1535 1543 } else {
1536 1544 srv->shc_status = ILB_HCS_ALIVE;
1537 1545 }
1538 1546 if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
1539 1547 logerr("%s: cannot restart timer: rule %s server %s",
1540 1548 __func__, srv->shc_hc_rule->hcr_rule->irl_name,
1541 1549 srv->shc_sg_srv->sgs_srvID);
1542 1550 ilbd_mark_server_disabled(srv);
1543 1551 }
1544 1552 break;
1545 1553 default:
1546 1554 logdebug("%s: unknown state", __func__);
1547 1555 break;
1548 1556 }
1549 1557 }
1550 1558
1551 1559 /*
1552 1560 * Handle the return event of a child probe fd.
1553 1561 */
1554 1562 void
1555 1563 ilbd_hc_probe_return(int ev_port, int fd, int port_events,
1556 1564 ilbd_hc_probe_event_t *ev)
1557 1565 {
1558 1566 /*
1559 1567 * Note that there can be more than one events delivered to us at
1560 1568 * the same time. So we need to check them individually.
1561 1569 */
1562 1570 if (port_events & POLLRDNORM)
1563 1571 ilbd_hc_child_data(fd, ev);
1564 1572
1565 1573 if (port_events & (POLLHUP|POLLERR)) {
1566 1574 ilbd_hc_child_hup(ev_port, fd, ev);
1567 1575 return;
1568 1576 }
1569 1577
1570 1578 /*
1571 1579 * Re-associate the fd with the port so that when the child
1572 1580 * exits, we can reap the status.
1573 1581 */
1574 1582 reassociate_port(ev_port, fd, ev);
1575 1583 }
|
↓ open down ↓ |
185 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX