Print this page
8634 epoll fails to wake on certain edge-triggered conditions
8635 epoll should not emit POLLNVAL
8636 recursive epoll should emit EPOLLRDNORM
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Igor Kozhukhov <igor@dilos.org>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/crypto/api/kcf_random.c
+++ new/usr/src/uts/common/crypto/api/kcf_random.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
24 - * Copyright (c) 2015, Joyent, Inc.
24 + * Copyright 2017 Joyent, Inc.
25 25 */
26 26
27 27 /*
28 28 * This file implements the interfaces that the /dev/random
29 29 * driver uses for read(2), write(2) and poll(2) on /dev/random or
30 30 * /dev/urandom. It also implements the kernel API - random_add_entropy(),
31 31 * random_add_pseudo_entropy(), random_get_pseudo_bytes()
32 32 * and random_get_bytes().
33 33 *
34 34 * We periodically collect random bits from providers which are registered
35 35 * with the Kernel Cryptographic Framework (kCF) as capable of random
36 36 * number generation. The random bits are maintained in a cache and
37 37 * it is used for high quality random numbers (/dev/random) requests.
38 38 * We pick a provider and call its SPI routine, if the cache does not have
39 39 * enough bytes to satisfy a request.
40 40 *
41 41 * /dev/urandom requests use a software-based generator algorithm that uses the
42 42 * random bits in the cache as a seed. We create one pseudo-random generator
43 43 * (for /dev/urandom) per possible CPU on the system, and use it,
44 44 * kmem-magazine-style, to avoid cache line contention.
45 45 *
46 46 * LOCKING HIERARCHY:
47 47 * 1) rmp->rm_mag.rm_lock protects the per-cpu pseudo-random generators.
48 48 * 2) rndpool_lock protects the high-quality randomness pool.
49 49 * It may be locked while a rmp->rm_mag.rm_lock is held.
50 50 *
51 51 * A history note: The kernel API and the software-based algorithms in this
52 52 * file used to be part of the /dev/random driver.
53 53 */
54 54
55 55 #include <sys/types.h>
56 56 #include <sys/conf.h>
57 57 #include <sys/sunddi.h>
58 58 #include <sys/disp.h>
59 59 #include <sys/modctl.h>
60 60 #include <sys/ddi.h>
61 61 #include <sys/crypto/common.h>
62 62 #include <sys/crypto/api.h>
63 63 #include <sys/crypto/impl.h>
64 64 #include <sys/crypto/sched_impl.h>
65 65 #include <sys/crypto/ioctladmin.h>
66 66 #include <sys/random.h>
67 67 #include <sys/sha1.h>
68 68 #include <sys/time.h>
69 69 #include <sys/sysmacros.h>
70 70 #include <sys/cpuvar.h>
71 71 #include <sys/taskq.h>
72 72 #include <rng/fips_random.h>
73 73
74 74 #define RNDPOOLSIZE 1024 /* Pool size in bytes */
75 75 #define MINEXTRACTBYTES 20
76 76 #define MAXEXTRACTBYTES 1024
77 77 #define PRNG_MAXOBLOCKS 1310720 /* Max output block per prng key */
78 78 #define TIMEOUT_INTERVAL 5 /* Periodic mixing interval in secs */
79 79
80 80 typedef enum extract_type {
81 81 NONBLOCK_EXTRACT,
82 82 BLOCKING_EXTRACT,
83 83 ALWAYS_EXTRACT
84 84 } extract_type_t;
85 85
86 86 /*
87 87 * Hash-algo generic definitions. For now, they are SHA1's. We use SHA1
88 88 * routines directly instead of using k-API because we can't return any
89 89 * error code in /dev/urandom case and we can get an error using k-API
90 90 * if a mechanism is disabled.
91 91 */
92 92 #define HASHSIZE 20
93 93 #define HASH_CTX SHA1_CTX
94 94 #define HashInit(ctx) SHA1Init((ctx))
95 95 #define HashUpdate(ctx, p, s) SHA1Update((ctx), (p), (s))
96 96 #define HashFinal(d, ctx) SHA1Final((d), (ctx))
97 97
98 98 /* HMAC-SHA1 */
99 99 #define HMAC_KEYSIZE 20
100 100
101 101 /*
102 102 * Cache of random bytes implemented as a circular buffer. findex and rindex
103 103 * track the front and back of the circular buffer.
104 104 */
105 105 uint8_t rndpool[RNDPOOLSIZE];
106 106 static int findex, rindex;
107 107 static int rnbyte_cnt; /* Number of bytes in the cache */
108 108
109 109 static kmutex_t rndpool_lock; /* protects r/w accesses to the cache, */
110 110 /* and the global variables */
111 111 static kcondvar_t rndpool_read_cv; /* serializes poll/read syscalls */
112 112 static int num_waiters; /* #threads waiting to read from /dev/random */
113 113
114 114 static struct pollhead rnd_pollhead;
115 115 /* LINTED E_STATIC_UNUSED */
116 116 static timeout_id_t kcf_rndtimeout_id;
117 117 static crypto_mech_type_t rngmech_type = CRYPTO_MECH_INVALID;
118 118 rnd_stats_t rnd_stats;
119 119 static boolean_t rng_prov_found = B_TRUE;
120 120 static boolean_t rng_ok_to_log = B_TRUE;
121 121 static boolean_t rngprov_task_idle = B_TRUE;
122 122
123 123 static void rndc_addbytes(uint8_t *, size_t);
124 124 static void rndc_getbytes(uint8_t *ptr, size_t len);
125 125 static void rnd_handler(void *);
126 126 static void rnd_alloc_magazines(void);
127 127 static void rnd_fips_discard_initial(void);
128 128 static void rnd_init2(void *);
129 129 static void rnd_schedule_timeout(void);
130 130
131 131 /*
132 132 * Called from kcf:_init()
133 133 */
134 134 void
135 135 kcf_rnd_init()
136 136 {
137 137 hrtime_t ts;
138 138 time_t now;
139 139
140 140 mutex_init(&rndpool_lock, NULL, MUTEX_DEFAULT, NULL);
141 141 cv_init(&rndpool_read_cv, NULL, CV_DEFAULT, NULL);
142 142
143 143 /*
144 144 * Add bytes to the cache using
145 145 * . 2 unpredictable times: high resolution time since the boot-time,
146 146 * and the current time-of-the day.
147 147 * This is used only to make the timeout value in the timer
148 148 * unpredictable.
149 149 */
150 150 ts = gethrtime();
151 151 rndc_addbytes((uint8_t *)&ts, sizeof (ts));
152 152
153 153 (void) drv_getparm(TIME, &now);
154 154 rndc_addbytes((uint8_t *)&now, sizeof (now));
155 155
156 156 rnbyte_cnt = 0;
157 157 findex = rindex = 0;
158 158 num_waiters = 0;
159 159
160 160 rnd_alloc_magazines();
161 161
162 162 (void) taskq_dispatch(system_taskq, rnd_init2, NULL, TQ_SLEEP);
163 163 }
164 164
165 165 /*
166 166 * This is called via the system taskq, so that we can do further
167 167 * initializations that have to wait until the kcf module itself is
168 168 * done loading. (After kcf:_init returns.)
169 169 */
170 170 static void
171 171 rnd_init2(void *unused)
172 172 {
173 173
174 174 _NOTE(ARGUNUSED(unused));
175 175
176 176 /*
177 177 * This will load a randomness provider; typically "swrand",
178 178 * but could be another provider if so configured.
179 179 */
180 180 rngmech_type = crypto_mech2id(SUN_RANDOM);
181 181
182 182 /* Update rng_prov_found etc. */
183 183 (void) kcf_rngprov_check();
184 184
185 185 /* FIPS 140-2 init. */
186 186 rnd_fips_discard_initial();
187 187
188 188 /* Start rnd_handler calls. */
189 189 rnd_schedule_timeout();
190 190 }
191 191
192 192 /*
193 193 * Return TRUE if at least one provider exists that can
194 194 * supply random numbers.
195 195 */
196 196 boolean_t
197 197 kcf_rngprov_check(void)
198 198 {
199 199 int rv;
200 200 kcf_provider_desc_t *pd;
201 201
202 202 if ((pd = kcf_get_mech_provider(rngmech_type, NULL, NULL, &rv,
203 203 NULL, CRYPTO_FG_RANDOM, 0)) != NULL) {
204 204 KCF_PROV_REFRELE(pd);
205 205 /*
206 206 * We logged a warning once about no provider being available
207 207 * and now a provider became available. So, set the flag so
208 208 * that we can log again if the problem recurs.
209 209 */
210 210 rng_ok_to_log = B_TRUE;
211 211 rng_prov_found = B_TRUE;
212 212 return (B_TRUE);
213 213 } else {
214 214 rng_prov_found = B_FALSE;
215 215 return (B_FALSE);
216 216 }
217 217 }
218 218
219 219 /*
220 220 * Pick a software-based provider and submit a request to seed
221 221 * its random number generator.
222 222 */
223 223 static void
224 224 rngprov_seed(uint8_t *buf, int len, uint_t entropy_est, uint32_t flags)
225 225 {
226 226 kcf_provider_desc_t *pd = NULL;
227 227
228 228 if (kcf_get_sw_prov(rngmech_type, &pd, NULL, B_FALSE) ==
229 229 CRYPTO_SUCCESS) {
230 230 (void) KCF_PROV_SEED_RANDOM(pd, pd->pd_sid, buf, len,
231 231 entropy_est, flags, NULL);
232 232 KCF_PROV_REFRELE(pd);
233 233 }
234 234 }
235 235
236 236 /*
237 237 * This routine is called for blocking reads.
238 238 *
239 239 * The argument is_taskq_thr indicates whether the caller is
240 240 * the taskq thread dispatched by the timeout handler routine.
241 241 * In this case, we cycle through all the providers
242 242 * submitting a request to each provider to generate random numbers.
243 243 *
244 244 * For other cases, we pick a provider and submit a request to generate
245 245 * random numbers. We retry using another provider if we get an error.
246 246 *
247 247 * Returns the number of bytes that are written to 'ptr'. Returns -1
248 248 * if no provider is found. ptr and need are unchanged.
249 249 */
250 250 static int
251 251 rngprov_getbytes(uint8_t *ptr, size_t need, boolean_t is_taskq_thr)
252 252 {
253 253 int rv;
254 254 int prov_cnt = 0;
255 255 int total_bytes = 0;
256 256 kcf_provider_desc_t *pd;
257 257 kcf_req_params_t params;
258 258 kcf_prov_tried_t *list = NULL;
259 259
260 260 while ((pd = kcf_get_mech_provider(rngmech_type, NULL, NULL, &rv,
261 261 list, CRYPTO_FG_RANDOM, 0)) != NULL) {
262 262
263 263 prov_cnt++;
264 264
265 265 KCF_WRAP_RANDOM_OPS_PARAMS(¶ms, KCF_OP_RANDOM_GENERATE,
266 266 pd->pd_sid, ptr, need, 0, 0);
267 267 rv = kcf_submit_request(pd, NULL, NULL, ¶ms, B_FALSE);
268 268 ASSERT(rv != CRYPTO_QUEUED);
269 269
270 270 if (rv == CRYPTO_SUCCESS) {
271 271 total_bytes += need;
272 272 if (is_taskq_thr)
273 273 rndc_addbytes(ptr, need);
274 274 else {
275 275 KCF_PROV_REFRELE(pd);
276 276 break;
277 277 }
278 278 }
279 279
280 280 if (is_taskq_thr || rv != CRYPTO_SUCCESS) {
281 281 /* Add pd to the linked list of providers tried. */
282 282 if (kcf_insert_triedlist(&list, pd, KM_SLEEP) == NULL) {
283 283 KCF_PROV_REFRELE(pd);
284 284 break;
285 285 }
286 286 }
287 287
288 288 }
289 289
290 290 if (list != NULL)
291 291 kcf_free_triedlist(list);
292 292
293 293 if (prov_cnt == 0) { /* no provider could be found. */
294 294 rng_prov_found = B_FALSE;
295 295 return (-1);
296 296 } else {
297 297 rng_prov_found = B_TRUE;
298 298 /* See comments in kcf_rngprov_check() */
299 299 rng_ok_to_log = B_TRUE;
300 300 }
301 301
302 302 return (total_bytes);
303 303 }
304 304
305 305 static void
306 306 notify_done(void *arg, int rv)
307 307 {
308 308 uchar_t *rndbuf = arg;
309 309
310 310 if (rv == CRYPTO_SUCCESS)
311 311 rndc_addbytes(rndbuf, MINEXTRACTBYTES);
312 312
313 313 bzero(rndbuf, MINEXTRACTBYTES);
314 314 kmem_free(rndbuf, MINEXTRACTBYTES);
315 315 }
316 316
317 317 /*
318 318 * Cycle through all the providers submitting a request to each provider
319 319 * to generate random numbers. This is called for the modes - NONBLOCK_EXTRACT
320 320 * and ALWAYS_EXTRACT.
321 321 *
322 322 * Returns the number of bytes that are written to 'ptr'. Returns -1
323 323 * if no provider is found. ptr and len are unchanged.
324 324 */
325 325 static int
326 326 rngprov_getbytes_nblk(uint8_t *ptr, size_t len)
327 327 {
328 328 int rv, total_bytes;
329 329 size_t blen;
330 330 uchar_t *rndbuf;
331 331 kcf_provider_desc_t *pd;
332 332 kcf_req_params_t params;
333 333 crypto_call_req_t req;
334 334 kcf_prov_tried_t *list = NULL;
335 335 int prov_cnt = 0;
336 336
337 337 blen = 0;
338 338 total_bytes = 0;
339 339 req.cr_flag = CRYPTO_SKIP_REQID;
340 340 req.cr_callback_func = notify_done;
341 341
342 342 while ((pd = kcf_get_mech_provider(rngmech_type, NULL, NULL, &rv,
343 343 list, CRYPTO_FG_RANDOM, 0)) != NULL) {
344 344
345 345 prov_cnt ++;
346 346 switch (pd->pd_prov_type) {
347 347 case CRYPTO_HW_PROVIDER:
348 348 /*
349 349 * We have to allocate a buffer here as we can not
350 350 * assume that the input buffer will remain valid
351 351 * when the callback comes. We use a fixed size buffer
352 352 * to simplify the book keeping.
353 353 */
354 354 rndbuf = kmem_alloc(MINEXTRACTBYTES, KM_NOSLEEP);
355 355 if (rndbuf == NULL) {
356 356 KCF_PROV_REFRELE(pd);
357 357 if (list != NULL)
358 358 kcf_free_triedlist(list);
359 359 return (total_bytes);
360 360 }
361 361 req.cr_callback_arg = rndbuf;
362 362 KCF_WRAP_RANDOM_OPS_PARAMS(¶ms,
363 363 KCF_OP_RANDOM_GENERATE,
364 364 pd->pd_sid, rndbuf, MINEXTRACTBYTES, 0, 0);
365 365 break;
366 366
367 367 case CRYPTO_SW_PROVIDER:
368 368 /*
369 369 * We do not need to allocate a buffer in the software
370 370 * provider case as there is no callback involved. We
371 371 * avoid any extra data copy by directly passing 'ptr'.
372 372 */
373 373 KCF_WRAP_RANDOM_OPS_PARAMS(¶ms,
374 374 KCF_OP_RANDOM_GENERATE,
375 375 pd->pd_sid, ptr, len, 0, 0);
376 376 break;
377 377 }
378 378
379 379 rv = kcf_submit_request(pd, NULL, &req, ¶ms, B_FALSE);
380 380 if (rv == CRYPTO_SUCCESS) {
381 381 switch (pd->pd_prov_type) {
382 382 case CRYPTO_HW_PROVIDER:
383 383 /*
384 384 * Since we have the input buffer handy,
385 385 * we directly copy to it rather than
386 386 * adding to the pool.
387 387 */
388 388 blen = min(MINEXTRACTBYTES, len);
389 389 bcopy(rndbuf, ptr, blen);
390 390 if (len < MINEXTRACTBYTES)
391 391 rndc_addbytes(rndbuf + len,
392 392 MINEXTRACTBYTES - len);
393 393 ptr += blen;
394 394 len -= blen;
395 395 total_bytes += blen;
396 396 break;
397 397
398 398 case CRYPTO_SW_PROVIDER:
399 399 total_bytes += len;
400 400 len = 0;
401 401 break;
402 402 }
403 403 }
404 404
405 405 /*
406 406 * We free the buffer in the callback routine
407 407 * for the CRYPTO_QUEUED case.
408 408 */
409 409 if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
410 410 rv != CRYPTO_QUEUED) {
411 411 bzero(rndbuf, MINEXTRACTBYTES);
412 412 kmem_free(rndbuf, MINEXTRACTBYTES);
413 413 }
414 414
415 415 if (len == 0) {
416 416 KCF_PROV_REFRELE(pd);
417 417 break;
418 418 }
419 419
420 420 if (rv != CRYPTO_SUCCESS) {
421 421 /* Add pd to the linked list of providers tried. */
422 422 if (kcf_insert_triedlist(&list, pd, KM_NOSLEEP) ==
423 423 NULL) {
424 424 KCF_PROV_REFRELE(pd);
425 425 break;
426 426 }
427 427 }
428 428 }
429 429
430 430 if (list != NULL) {
431 431 kcf_free_triedlist(list);
432 432 }
433 433
434 434 if (prov_cnt == 0) { /* no provider could be found. */
435 435 rng_prov_found = B_FALSE;
436 436 return (-1);
437 437 } else {
438 438 rng_prov_found = B_TRUE;
439 439 /* See comments in kcf_rngprov_check() */
440 440 rng_ok_to_log = B_TRUE;
441 441 }
442 442
443 443 return (total_bytes);
444 444 }
445 445
446 446 static void
447 447 rngprov_task(void *arg)
448 448 {
449 449 int len = (int)(uintptr_t)arg;
450 450 uchar_t tbuf[MAXEXTRACTBYTES];
451 451
452 452 ASSERT(len <= MAXEXTRACTBYTES);
453 453 (void) rngprov_getbytes(tbuf, len, B_TRUE);
454 454 rngprov_task_idle = B_TRUE;
455 455 }
456 456
457 457 /*
458 458 * Returns "len" random or pseudo-random bytes in *ptr.
459 459 * Will block if not enough random bytes are available and the
460 460 * call is blocking.
461 461 *
462 462 * Called with rndpool_lock held (allowing caller to do optimistic locking;
463 463 * releases the lock before return).
464 464 */
465 465 static int
466 466 rnd_get_bytes(uint8_t *ptr, size_t len, extract_type_t how)
467 467 {
468 468 size_t bytes;
469 469 int got;
470 470
471 471 ASSERT(mutex_owned(&rndpool_lock));
472 472 /*
473 473 * Check if the request can be satisfied from the cache
474 474 * of random bytes.
475 475 */
476 476 if (len <= rnbyte_cnt) {
477 477 rndc_getbytes(ptr, len);
478 478 mutex_exit(&rndpool_lock);
479 479 return (0);
480 480 }
481 481 mutex_exit(&rndpool_lock);
482 482
483 483 switch (how) {
484 484 case BLOCKING_EXTRACT:
485 485 if ((got = rngprov_getbytes(ptr, len, B_FALSE)) == -1)
486 486 break; /* No provider found */
487 487
488 488 if (got == len)
489 489 return (0);
490 490 len -= got;
491 491 ptr += got;
492 492 break;
493 493
494 494 case NONBLOCK_EXTRACT:
495 495 case ALWAYS_EXTRACT:
496 496 if ((got = rngprov_getbytes_nblk(ptr, len)) == -1) {
497 497 /* No provider found */
498 498 if (how == NONBLOCK_EXTRACT) {
499 499 return (EAGAIN);
500 500 }
501 501 } else {
502 502 if (got == len)
503 503 return (0);
504 504 len -= got;
505 505 ptr += got;
506 506 }
507 507 if (how == NONBLOCK_EXTRACT && (rnbyte_cnt < len))
508 508 return (EAGAIN);
509 509 break;
510 510 }
511 511
512 512 mutex_enter(&rndpool_lock);
513 513 while (len > 0) {
514 514 if (how == BLOCKING_EXTRACT) {
515 515 /* Check if there is enough */
516 516 while (rnbyte_cnt < MINEXTRACTBYTES) {
517 517 num_waiters++;
518 518 if (cv_wait_sig(&rndpool_read_cv,
519 519 &rndpool_lock) == 0) {
520 520 num_waiters--;
521 521 mutex_exit(&rndpool_lock);
522 522 return (EINTR);
523 523 }
524 524 num_waiters--;
525 525 }
526 526 }
527 527
528 528 /* Figure out how many bytes to extract */
529 529 bytes = min(len, rnbyte_cnt);
530 530 rndc_getbytes(ptr, bytes);
531 531
532 532 len -= bytes;
533 533 ptr += bytes;
534 534
535 535 if (len > 0 && how == ALWAYS_EXTRACT) {
536 536 /*
537 537 * There are not enough bytes, but we can not block.
538 538 * This only happens in the case of /dev/urandom which
539 539 * runs an additional generation algorithm. So, there
540 540 * is no problem.
541 541 */
542 542 while (len > 0) {
543 543 *ptr = rndpool[findex];
544 544 ptr++; len--;
545 545 rindex = findex = (findex + 1) &
546 546 (RNDPOOLSIZE - 1);
547 547 }
548 548 break;
549 549 }
550 550 }
551 551
552 552 mutex_exit(&rndpool_lock);
553 553 return (0);
554 554 }
555 555
556 556 int
557 557 kcf_rnd_get_bytes(uint8_t *ptr, size_t len, boolean_t noblock)
558 558 {
559 559 extract_type_t how;
560 560 int error;
561 561
562 562 how = noblock ? NONBLOCK_EXTRACT : BLOCKING_EXTRACT;
563 563 mutex_enter(&rndpool_lock);
564 564 if ((error = rnd_get_bytes(ptr, len, how)) != 0)
565 565 return (error);
566 566
567 567 BUMP_RND_STATS(rs_rndOut, len);
568 568 return (0);
569 569 }
570 570
571 571 /*
572 572 * Revisit this if the structs grow or we come up with a better way
573 573 * of cache-line-padding structures.
574 574 */
575 575 #define RND_CPU_CACHE_SIZE 64
576 576 #define RND_CPU_PAD_SIZE RND_CPU_CACHE_SIZE*6
577 577 #define RND_CPU_PAD (RND_CPU_PAD_SIZE - \
578 578 sizeof (rndmag_t))
579 579 /*
580 580 * Per-CPU random state. Somewhat like like kmem's magazines, this provides
581 581 * a per-CPU instance of the pseudo-random generator. We have it much easier
582 582 * than kmem, as we can afford to "leak" random bits if a CPU is DR'ed out.
583 583 *
584 584 * Note that this usage is preemption-safe; a thread
585 585 * entering a critical section remembers which generator it locked
586 586 * and unlocks the same one; should it be preempted and wind up running on
587 587 * a different CPU, there will be a brief period of increased contention
588 588 * before it exits the critical section but nothing will melt.
589 589 */
590 590 typedef struct rndmag_s
591 591 {
592 592 kmutex_t rm_lock;
593 593 uint8_t *rm_buffer; /* Start of buffer */
594 594 uint8_t *rm_eptr; /* End of buffer */
595 595 uint8_t *rm_rptr; /* Current read pointer */
596 596 uint32_t rm_oblocks; /* time to rekey? */
597 597 uint32_t rm_ofuzz; /* Rekey backoff state */
598 598 uint32_t rm_olimit; /* Hard rekey limit */
599 599 rnd_stats_t rm_stats; /* Per-CPU Statistics */
600 600 uint32_t rm_key[HASHSIZE/BYTES_IN_WORD]; /* FIPS XKEY */
601 601 uint32_t rm_seed[HASHSIZE/BYTES_IN_WORD]; /* seed for rekey */
602 602 uint32_t rm_previous[HASHSIZE/BYTES_IN_WORD]; /* prev random */
603 603 } rndmag_t;
604 604
605 605 typedef struct rndmag_pad_s
606 606 {
607 607 rndmag_t rm_mag;
608 608 uint8_t rm_pad[RND_CPU_PAD];
609 609 } rndmag_pad_t;
610 610
611 611 /*
612 612 * Generate random bytes for /dev/urandom by applying the
613 613 * FIPS 186-2 algorithm with a key created from bytes extracted
614 614 * from the pool. A maximum of PRNG_MAXOBLOCKS output blocks
615 615 * is generated before a new key is obtained.
616 616 *
617 617 * Note that callers to this routine are likely to assume it can't fail.
618 618 *
619 619 * Called with rmp locked; releases lock.
620 620 */
621 621 static int
622 622 rnd_generate_pseudo_bytes(rndmag_pad_t *rmp, uint8_t *ptr, size_t len)
623 623 {
624 624 size_t bytes = len, size;
625 625 int nblock;
626 626 uint32_t oblocks;
627 627 uint32_t tempout[HASHSIZE/BYTES_IN_WORD];
628 628 uint32_t seed[HASHSIZE/BYTES_IN_WORD];
629 629 int i;
630 630 hrtime_t timestamp;
631 631 uint8_t *src, *dst;
632 632
633 633 ASSERT(mutex_owned(&rmp->rm_mag.rm_lock));
634 634
635 635 /* Nothing is being asked */
636 636 if (len == 0) {
637 637 mutex_exit(&rmp->rm_mag.rm_lock);
638 638 return (0);
639 639 }
640 640
641 641 nblock = howmany(len, HASHSIZE);
642 642
643 643 rmp->rm_mag.rm_oblocks += nblock;
644 644 oblocks = rmp->rm_mag.rm_oblocks;
645 645
646 646 do {
647 647 if (oblocks >= rmp->rm_mag.rm_olimit) {
648 648
649 649 /*
650 650 * Contention-avoiding rekey: see if
651 651 * the pool is locked, and if so, wait a bit.
652 652 * Do an 'exponential back-in' to ensure we don't
653 653 * run too long without rekey.
654 654 */
655 655 if (rmp->rm_mag.rm_ofuzz) {
656 656 /*
657 657 * Decaying exponential back-in for rekey.
658 658 */
659 659 if ((rnbyte_cnt < MINEXTRACTBYTES) ||
660 660 (!mutex_tryenter(&rndpool_lock))) {
661 661 rmp->rm_mag.rm_olimit +=
662 662 rmp->rm_mag.rm_ofuzz;
663 663 rmp->rm_mag.rm_ofuzz >>= 1;
664 664 goto punt;
665 665 }
666 666 } else {
667 667 mutex_enter(&rndpool_lock);
668 668 }
669 669
670 670 /* Get a new chunk of entropy */
671 671 (void) rnd_get_bytes((uint8_t *)rmp->rm_mag.rm_key,
672 672 HMAC_KEYSIZE, ALWAYS_EXTRACT);
673 673
674 674 rmp->rm_mag.rm_olimit = PRNG_MAXOBLOCKS/2;
675 675 rmp->rm_mag.rm_ofuzz = PRNG_MAXOBLOCKS/4;
676 676 oblocks = 0;
677 677 rmp->rm_mag.rm_oblocks = nblock;
678 678 }
679 679 punt:
680 680 timestamp = gethrtime();
681 681
682 682 src = (uint8_t *)×tamp;
683 683 dst = (uint8_t *)rmp->rm_mag.rm_seed;
684 684
685 685 for (i = 0; i < HASHSIZE; i++) {
686 686 dst[i] ^= src[i % sizeof (timestamp)];
687 687 }
688 688
689 689 bcopy(rmp->rm_mag.rm_seed, seed, HASHSIZE);
690 690
691 691 fips_random_inner(rmp->rm_mag.rm_key, tempout,
692 692 seed);
693 693
694 694 if (bytes >= HASHSIZE) {
695 695 size = HASHSIZE;
696 696 } else {
697 697 size = min(bytes, HASHSIZE);
698 698 }
699 699
700 700 /*
701 701 * FIPS 140-2: Continuous RNG test - each generation
702 702 * of an n-bit block shall be compared with the previously
703 703 * generated block. Test shall fail if any two compared
704 704 * n-bit blocks are equal.
705 705 */
706 706 for (i = 0; i < HASHSIZE/BYTES_IN_WORD; i++) {
707 707 if (tempout[i] != rmp->rm_mag.rm_previous[i])
708 708 break;
709 709 }
710 710 if (i == HASHSIZE/BYTES_IN_WORD) {
711 711 cmn_err(CE_WARN, "kcf_random: The value of 160-bit "
712 712 "block random bytes are same as the previous "
713 713 "one.\n");
714 714 /* discard random bytes and return error */
715 715 mutex_exit(&rmp->rm_mag.rm_lock);
716 716 return (EIO);
717 717 }
718 718
719 719 bcopy(tempout, rmp->rm_mag.rm_previous,
720 720 HASHSIZE);
721 721
722 722 bcopy(tempout, ptr, size);
723 723 ptr += size;
724 724 bytes -= size;
725 725 oblocks++;
726 726 nblock--;
727 727 } while (bytes > 0);
728 728
729 729 /* Zero out sensitive information */
730 730 bzero(seed, HASHSIZE);
731 731 bzero(tempout, HASHSIZE);
732 732 mutex_exit(&rmp->rm_mag.rm_lock);
733 733 return (0);
734 734 }
735 735
736 736 /*
737 737 * Per-CPU Random magazines.
738 738 */
739 739 static rndmag_pad_t *rndmag;
740 740 static uint8_t *rndbuf;
741 741 static size_t rndmag_total;
742 742 /*
743 743 * common/os/cpu.c says that platform support code can shrinkwrap
744 744 * max_ncpus. On the off chance that we get loaded very early, we
745 745 * read it exactly once, to copy it here.
746 746 */
747 747 static uint32_t random_max_ncpus = 0;
748 748
749 749 /*
750 750 * Boot-time tunables, for experimentation.
751 751 */
752 752 size_t rndmag_threshold = 2560;
753 753 size_t rndbuf_len = 5120;
754 754 size_t rndmag_size = 1280;
755 755
756 756
757 757 int
758 758 kcf_rnd_get_pseudo_bytes(uint8_t *ptr, size_t len)
759 759 {
760 760 rndmag_pad_t *rmp;
761 761 uint8_t *cptr, *eptr;
762 762
763 763 /*
764 764 * Anyone who asks for zero bytes of randomness should get slapped.
765 765 */
766 766 ASSERT(len > 0);
767 767
768 768 /*
769 769 * Fast path.
770 770 */
771 771 for (;;) {
772 772 rmp = &rndmag[CPU->cpu_seqid];
773 773 mutex_enter(&rmp->rm_mag.rm_lock);
774 774
775 775 /*
776 776 * Big requests bypass buffer and tail-call the
777 777 * generate routine directly.
778 778 */
779 779 if (len > rndmag_threshold) {
780 780 BUMP_CPU_RND_STATS(rmp, rs_urndOut, len);
781 781 return (rnd_generate_pseudo_bytes(rmp, ptr, len));
782 782 }
783 783
784 784 cptr = rmp->rm_mag.rm_rptr;
785 785 eptr = cptr + len;
786 786
787 787 if (eptr <= rmp->rm_mag.rm_eptr) {
788 788 rmp->rm_mag.rm_rptr = eptr;
789 789 bcopy(cptr, ptr, len);
790 790 BUMP_CPU_RND_STATS(rmp, rs_urndOut, len);
791 791 mutex_exit(&rmp->rm_mag.rm_lock);
792 792
793 793 return (0);
794 794 }
795 795 /*
796 796 * End fast path.
797 797 */
798 798 rmp->rm_mag.rm_rptr = rmp->rm_mag.rm_buffer;
799 799 /*
800 800 * Note: We assume the generate routine always succeeds
801 801 * in this case (because it does at present..)
802 802 * It also always releases rm_lock.
803 803 */
804 804 (void) rnd_generate_pseudo_bytes(rmp, rmp->rm_mag.rm_buffer,
805 805 rndbuf_len);
806 806 }
807 807 }
808 808
809 809 /*
810 810 * We set up (empty) magazines for all of max_ncpus, possibly wasting a
811 811 * little memory on big systems that don't have the full set installed.
812 812 * See above; "empty" means "rptr equal to eptr"; this will trigger the
813 813 * refill path in rnd_get_pseudo_bytes above on the first call for each CPU.
814 814 *
815 815 * TODO: make rndmag_size tunable at run time!
816 816 */
817 817 static void
818 818 rnd_alloc_magazines()
819 819 {
820 820 rndmag_pad_t *rmp;
821 821 int i;
822 822
823 823 rndbuf_len = roundup(rndbuf_len, HASHSIZE);
824 824 if (rndmag_size < rndbuf_len)
825 825 rndmag_size = rndbuf_len;
826 826 rndmag_size = roundup(rndmag_size, RND_CPU_CACHE_SIZE);
827 827
828 828 random_max_ncpus = max_ncpus;
829 829 rndmag_total = rndmag_size * random_max_ncpus;
830 830
831 831 rndbuf = kmem_alloc(rndmag_total, KM_SLEEP);
832 832 rndmag = kmem_zalloc(sizeof (rndmag_pad_t) * random_max_ncpus,
833 833 KM_SLEEP);
834 834
835 835 for (i = 0; i < random_max_ncpus; i++) {
836 836 uint8_t *buf;
837 837
838 838 rmp = &rndmag[i];
839 839 mutex_init(&rmp->rm_mag.rm_lock, NULL, MUTEX_DRIVER, NULL);
840 840
841 841 buf = rndbuf + i * rndmag_size;
842 842
843 843 rmp->rm_mag.rm_buffer = buf;
844 844 rmp->rm_mag.rm_eptr = buf + rndbuf_len;
845 845 rmp->rm_mag.rm_rptr = buf + rndbuf_len;
846 846 rmp->rm_mag.rm_oblocks = 1;
847 847 }
848 848 }
849 849
850 850 /*
851 851 * FIPS 140-2: the first n-bit (n > 15) block generated
852 852 * after power-up, initialization, or reset shall not
853 853 * be used, but shall be saved for comparison.
854 854 */
855 855 static void
856 856 rnd_fips_discard_initial(void)
857 857 {
858 858 uint8_t discard_buf[HASHSIZE];
859 859 rndmag_pad_t *rmp;
860 860 int i;
861 861
862 862 for (i = 0; i < random_max_ncpus; i++) {
863 863 rmp = &rndmag[i];
864 864
865 865 /* rnd_get_bytes() will call mutex_exit(&rndpool_lock) */
866 866 mutex_enter(&rndpool_lock);
867 867 (void) rnd_get_bytes(discard_buf,
868 868 HMAC_KEYSIZE, ALWAYS_EXTRACT);
869 869 bcopy(discard_buf, rmp->rm_mag.rm_previous,
870 870 HMAC_KEYSIZE);
871 871 /* rnd_get_bytes() will call mutex_exit(&rndpool_lock) */
872 872 mutex_enter(&rndpool_lock);
873 873 (void) rnd_get_bytes((uint8_t *)rmp->rm_mag.rm_key,
874 874 HMAC_KEYSIZE, ALWAYS_EXTRACT);
875 875 /* rnd_get_bytes() will call mutex_exit(&rndpool_lock) */
876 876 mutex_enter(&rndpool_lock);
877 877 (void) rnd_get_bytes((uint8_t *)rmp->rm_mag.rm_seed,
878 878 HMAC_KEYSIZE, ALWAYS_EXTRACT);
879 879 }
880 880 }
881 881
882 882 static void
883 883 rnd_schedule_timeout(void)
884 884 {
885 885 clock_t ut; /* time in microseconds */
886 886
887 887 /*
888 888 * The new timeout value is taken from the buffer of random bytes.
889 889 * We're merely reading the first 32 bits from the buffer here, not
890 890 * consuming any random bytes.
891 891 * The timeout multiplier value is a random value between 0.5 sec and
892 892 * 1.544480 sec (0.5 sec + 0xFF000 microseconds).
893 893 * The new timeout is TIMEOUT_INTERVAL times that multiplier.
894 894 */
895 895 ut = 500000 + (clock_t)((((uint32_t)rndpool[findex]) << 12) & 0xFF000);
896 896 kcf_rndtimeout_id = timeout(rnd_handler, NULL,
897 897 TIMEOUT_INTERVAL * drv_usectohz(ut));
898 898 }
899 899
900 900 /*
901 901 * Called from the driver for a poll on /dev/random
902 902 * . POLLOUT always succeeds.
903 903 * . POLLIN and POLLRDNORM will block until a
904 904 * minimum amount of entropy is available.
905 905 *
906 906 * &rnd_pollhead is passed in *phpp in order to indicate the calling thread
907 907 * will block. When enough random bytes are available, later, the timeout
908 908 * handler routine will issue the pollwakeup() calls.
909 909 */
910 910 void
911 911 kcf_rnd_chpoll(short events, int anyyet, short *reventsp,
912 912 struct pollhead **phpp)
913 913 {
914 914 *reventsp = events & POLLOUT;
|
↓ open down ↓ |
880 lines elided |
↑ open up ↑ |
915 915
916 916 if (events & (POLLIN | POLLRDNORM)) {
917 917 /*
918 918 * Sampling of rnbyte_cnt is an atomic
919 919 * operation. Hence we do not need any locking.
920 920 */
921 921 if (rnbyte_cnt >= MINEXTRACTBYTES)
922 922 *reventsp |= (events & (POLLIN | POLLRDNORM));
923 923 }
924 924
925 - if (*reventsp == 0 && !anyyet)
925 + if ((*reventsp == 0 && !anyyet) || (events & POLLET))
926 926 *phpp = &rnd_pollhead;
927 927 }
928 928
929 929 /*ARGSUSED*/
930 930 static void
931 931 rnd_handler(void *arg)
932 932 {
933 933 int len = 0;
934 934
935 935 if (!rng_prov_found && rng_ok_to_log) {
936 936 cmn_err(CE_WARN, "No randomness provider enabled for "
937 937 "/dev/random. Use cryptoadm(1M) to enable a provider.");
938 938 rng_ok_to_log = B_FALSE;
939 939 }
940 940
941 941 if (num_waiters > 0)
942 942 /*
943 943 * Note: len has no relationship with how many bytes
944 944 * a poll thread needs.
945 945 */
946 946 len = MAXEXTRACTBYTES;
947 947 else if (rnbyte_cnt < RNDPOOLSIZE)
948 948 len = MINEXTRACTBYTES;
949 949
950 950 /*
951 951 * Only one thread gets to set rngprov_task_idle at a given point
952 952 * of time and the order of the writes is defined. Also, it is OK
953 953 * if we read an older value of it and skip the dispatch once
954 954 * since we will get the correct value during the next time here.
955 955 * So, no locking is needed here.
956 956 */
957 957 if (len > 0 && rngprov_task_idle) {
958 958 rngprov_task_idle = B_FALSE;
959 959
960 960 /*
961 961 * It is OK if taskq_dispatch fails here. We will retry
962 962 * the next time around. Meanwhile, a thread doing a
963 963 * read() will go to the provider directly, if the
964 964 * cache becomes empty.
965 965 */
966 966 if (taskq_dispatch(system_taskq, rngprov_task,
967 967 (void *)(uintptr_t)len, TQ_NOSLEEP | TQ_NOQUEUE) == 0) {
968 968 rngprov_task_idle = B_TRUE;
969 969 }
970 970 }
971 971
972 972 mutex_enter(&rndpool_lock);
973 973 /*
974 974 * Wake up threads waiting in poll() or for enough accumulated
975 975 * random bytes to read from /dev/random. In case a poll() is
976 976 * concurrent with a read(), the polling process may be woken up
977 977 * indicating that enough randomness is now available for reading,
978 978 * and another process *steals* the bits from the pool, causing the
979 979 * subsequent read() from the first process to block. It is acceptable
980 980 * since the blocking will eventually end, after the timeout
981 981 * has expired enough times to honor the read.
982 982 *
983 983 * Note - Since we hold the rndpool_lock across the pollwakeup() call
984 984 * we MUST NOT grab the rndpool_lock in kcf_rndchpoll().
985 985 */
986 986 if (rnbyte_cnt >= MINEXTRACTBYTES)
987 987 pollwakeup(&rnd_pollhead, POLLIN | POLLRDNORM);
988 988
989 989 if (num_waiters > 0)
990 990 cv_broadcast(&rndpool_read_cv);
991 991 mutex_exit(&rndpool_lock);
992 992
993 993 rnd_schedule_timeout();
994 994 }
995 995
996 996 static void
997 997 rndc_addbytes(uint8_t *ptr, size_t len)
998 998 {
999 999 ASSERT(ptr != NULL && len > 0);
1000 1000 ASSERT(rnbyte_cnt <= RNDPOOLSIZE);
1001 1001
1002 1002 mutex_enter(&rndpool_lock);
1003 1003 while ((len > 0) && (rnbyte_cnt < RNDPOOLSIZE)) {
1004 1004 rndpool[rindex] ^= *ptr;
1005 1005 ptr++; len--;
1006 1006 rindex = (rindex + 1) & (RNDPOOLSIZE - 1);
1007 1007 rnbyte_cnt++;
1008 1008 }
1009 1009
1010 1010 /* Handle buffer full case */
1011 1011 while (len > 0) {
1012 1012 rndpool[rindex] ^= *ptr;
1013 1013 ptr++; len--;
1014 1014 findex = rindex = (rindex + 1) & (RNDPOOLSIZE - 1);
1015 1015 }
1016 1016 mutex_exit(&rndpool_lock);
1017 1017 }
1018 1018
1019 1019 /*
1020 1020 * Caller should check len <= rnbyte_cnt under the
1021 1021 * rndpool_lock before calling.
1022 1022 */
1023 1023 static void
1024 1024 rndc_getbytes(uint8_t *ptr, size_t len)
1025 1025 {
1026 1026 ASSERT(MUTEX_HELD(&rndpool_lock));
1027 1027 ASSERT(len <= rnbyte_cnt && rnbyte_cnt <= RNDPOOLSIZE);
1028 1028
1029 1029 BUMP_RND_STATS(rs_rndcOut, len);
1030 1030
1031 1031 while (len > 0) {
1032 1032 *ptr = rndpool[findex];
1033 1033 ptr++; len--;
1034 1034 findex = (findex + 1) & (RNDPOOLSIZE - 1);
1035 1035 rnbyte_cnt--;
1036 1036 }
1037 1037 }
1038 1038
1039 1039 /* Random number exported entry points */
1040 1040
1041 1041 /*
1042 1042 * Mix the supplied bytes into the entropy pool of a kCF
1043 1043 * RNG provider.
1044 1044 */
1045 1045 int
1046 1046 random_add_pseudo_entropy(uint8_t *ptr, size_t len, uint_t entropy_est)
1047 1047 {
1048 1048 if (len < 1)
1049 1049 return (-1);
1050 1050
1051 1051 rngprov_seed(ptr, len, entropy_est, 0);
1052 1052
1053 1053 return (0);
1054 1054 }
1055 1055
1056 1056 /*
1057 1057 * Mix the supplied bytes into the entropy pool of a kCF
1058 1058 * RNG provider. Mix immediately.
1059 1059 */
1060 1060 int
1061 1061 random_add_entropy(uint8_t *ptr, size_t len, uint_t entropy_est)
1062 1062 {
1063 1063 if (len < 1)
1064 1064 return (-1);
1065 1065
1066 1066 rngprov_seed(ptr, len, entropy_est, CRYPTO_SEED_NOW);
1067 1067
1068 1068 return (0);
1069 1069 }
1070 1070
1071 1071 /*
1072 1072 * Get bytes from the /dev/urandom generator. This function
1073 1073 * always succeeds. Returns 0.
1074 1074 */
1075 1075 int
1076 1076 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1077 1077 {
1078 1078 ASSERT(!mutex_owned(&rndpool_lock));
1079 1079
1080 1080 if (len < 1)
1081 1081 return (0);
1082 1082 return (kcf_rnd_get_pseudo_bytes(ptr, len));
1083 1083 }
1084 1084
1085 1085 /*
1086 1086 * Get bytes from the /dev/random generator. Returns 0
1087 1087 * on success. Returns EAGAIN if there is insufficient entropy.
1088 1088 */
1089 1089 int
1090 1090 random_get_bytes(uint8_t *ptr, size_t len)
1091 1091 {
1092 1092 ASSERT(!mutex_owned(&rndpool_lock));
1093 1093
1094 1094 if (len < 1)
1095 1095 return (0);
1096 1096 return (kcf_rnd_get_bytes(ptr, len, B_TRUE));
1097 1097 }
1098 1098
1099 1099 int
1100 1100 random_get_blocking_bytes(uint8_t *ptr, size_t len)
1101 1101 {
1102 1102 ASSERT(!mutex_owned(&rndpool_lock));
1103 1103
1104 1104 if (len < 1)
1105 1105 return (0);
1106 1106 return (kcf_rnd_get_bytes(ptr, len, B_FALSE));
1107 1107 }
|
↓ open down ↓ |
172 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX