Print this page
13902 Fix for 13717 may break 8-disk raidz2
13915 installctx() blocking allocate causes problems
Portions contributed by: Jerry Jelinek <gjelinek@gmail.com>
Change-Id: I934d69946cec42630fc541fa8c7385b862b69ca2
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/kcpc.c
+++ new/usr/src/uts/common/os/kcpc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 + * Copyright 2021 Joyent, Inc.
24 25 */
25 26
26 27 #include <sys/param.h>
27 28 #include <sys/thread.h>
28 29 #include <sys/cpuvar.h>
29 30 #include <sys/inttypes.h>
30 31 #include <sys/cmn_err.h>
31 32 #include <sys/time.h>
32 33 #include <sys/ksynch.h>
33 34 #include <sys/systm.h>
34 35 #include <sys/kcpc.h>
35 36 #include <sys/cpc_impl.h>
36 37 #include <sys/cpc_pcbe.h>
37 38 #include <sys/atomic.h>
38 39 #include <sys/sunddi.h>
39 40 #include <sys/modctl.h>
40 41 #include <sys/sdt.h>
41 42 #include <sys/archsystm.h>
42 43 #include <sys/promif.h>
43 44 #include <sys/x_call.h>
44 45 #include <sys/cap_util.h>
45 46 #if defined(__x86)
46 47 #include <asm/clock.h>
47 48 #include <sys/xc_levels.h>
48 49 #endif
49 50
50 51 static kmutex_t kcpc_ctx_llock[CPC_HASH_BUCKETS]; /* protects ctx_list */
51 52 static kcpc_ctx_t *kcpc_ctx_list[CPC_HASH_BUCKETS]; /* head of list */
52 53
53 54
54 55 krwlock_t kcpc_cpuctx_lock; /* lock for 'kcpc_cpuctx' below */
55 56 int kcpc_cpuctx; /* number of cpu-specific contexts */
56 57
57 58 int kcpc_counts_include_idle = 1; /* Project Private /etc/system variable */
58 59
59 60 /*
60 61 * These are set when a PCBE module is loaded.
61 62 */
62 63 uint_t cpc_ncounters = 0;
63 64 pcbe_ops_t *pcbe_ops = NULL;
64 65
65 66 /*
66 67 * Statistics on (mis)behavior
67 68 */
68 69 static uint32_t kcpc_intrctx_count; /* # overflows in an interrupt handler */
69 70 static uint32_t kcpc_nullctx_count; /* # overflows in a thread with no ctx */
70 71
71 72 /*
72 73 * By setting 'kcpc_nullctx_panic' to 1, any overflow interrupts in a thread
73 74 * with no valid context will result in a panic.
74 75 */
75 76 static int kcpc_nullctx_panic = 0;
76 77
77 78 static void kcpc_lwp_create(kthread_t *t, kthread_t *ct);
78 79 static void kcpc_restore(kcpc_ctx_t *ctx);
79 80 static void kcpc_save(kcpc_ctx_t *ctx);
80 81 static void kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx);
81 82 static int kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch);
82 83 static kcpc_set_t *kcpc_dup_set(kcpc_set_t *set);
83 84 static kcpc_set_t *kcpc_set_create(kcpc_request_t *reqs, int nreqs,
84 85 int set_flags, int kmem_flags);
85 86
86 87 /*
87 88 * Macros to manipulate context flags. All flag updates should use one of these
88 89 * two macros
89 90 *
90 91 * Flags should be always be updated atomically since some of the updates are
91 92 * not protected by locks.
92 93 */
93 94 #define KCPC_CTX_FLAG_SET(ctx, flag) atomic_or_uint(&(ctx)->kc_flags, (flag))
94 95 #define KCPC_CTX_FLAG_CLR(ctx, flag) atomic_and_uint(&(ctx)->kc_flags, ~(flag))
95 96
96 97 /*
97 98 * The IS_HIPIL() macro verifies that the code is executed either from a
98 99 * cross-call or from high-PIL interrupt
99 100 */
100 101 #ifdef DEBUG
101 102 #define IS_HIPIL() (getpil() >= XCALL_PIL)
102 103 #else
103 104 #define IS_HIPIL()
104 105 #endif /* DEBUG */
105 106
106 107
107 108 extern int kcpc_hw_load_pcbe(void);
108 109
109 110 /*
110 111 * Return value from kcpc_hw_load_pcbe()
111 112 */
112 113 static int kcpc_pcbe_error = 0;
113 114
114 115 /*
115 116 * Perform one-time initialization of kcpc framework.
116 117 * This function performs the initialization only the first time it is called.
117 118 * It is safe to call it multiple times.
118 119 */
119 120 int
120 121 kcpc_init(void)
121 122 {
122 123 long hash;
123 124 static uint32_t kcpc_initialized = 0;
124 125
125 126 /*
126 127 * We already tried loading platform pcbe module and failed
127 128 */
128 129 if (kcpc_pcbe_error != 0)
129 130 return (-1);
130 131
131 132 /*
132 133 * The kcpc framework should be initialized at most once
133 134 */
134 135 if (atomic_cas_32(&kcpc_initialized, 0, 1) != 0)
135 136 return (0);
136 137
137 138 rw_init(&kcpc_cpuctx_lock, NULL, RW_DEFAULT, NULL);
138 139 for (hash = 0; hash < CPC_HASH_BUCKETS; hash++)
139 140 mutex_init(&kcpc_ctx_llock[hash],
140 141 NULL, MUTEX_DRIVER, (void *)(uintptr_t)15);
141 142
142 143 /*
143 144 * Load platform-specific pcbe module
144 145 */
145 146 kcpc_pcbe_error = kcpc_hw_load_pcbe();
146 147
147 148 return (kcpc_pcbe_error == 0 ? 0 : -1);
148 149 }
149 150
150 151 void
151 152 kcpc_register_pcbe(pcbe_ops_t *ops)
152 153 {
153 154 pcbe_ops = ops;
154 155 cpc_ncounters = pcbe_ops->pcbe_ncounters();
155 156 }
156 157
157 158 void
158 159 kcpc_register_dcpc(void (*func)(uint64_t))
159 160 {
160 161 dtrace_cpc_fire = func;
161 162 }
162 163
163 164 void
164 165 kcpc_unregister_dcpc(void)
165 166 {
166 167 dtrace_cpc_fire = NULL;
167 168 }
168 169
169 170 int
170 171 kcpc_bind_cpu(kcpc_set_t *set, processorid_t cpuid, int *subcode)
171 172 {
172 173 cpu_t *cp;
173 174 kcpc_ctx_t *ctx;
174 175 int error;
175 176 int save_spl;
176 177
177 178 ctx = kcpc_ctx_alloc(KM_SLEEP);
178 179
179 180 if (kcpc_assign_reqs(set, ctx) != 0) {
180 181 kcpc_ctx_free(ctx);
181 182 *subcode = CPC_RESOURCE_UNAVAIL;
182 183 return (EINVAL);
183 184 }
184 185
185 186 ctx->kc_cpuid = cpuid;
186 187 ctx->kc_thread = curthread;
187 188
188 189 set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP);
189 190
190 191 if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) {
191 192 kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
192 193 kcpc_ctx_free(ctx);
193 194 return (error);
194 195 }
195 196
196 197 set->ks_ctx = ctx;
197 198 ctx->kc_set = set;
198 199
199 200 /*
200 201 * We must hold cpu_lock to prevent DR, offlining, or unbinding while
201 202 * we are manipulating the cpu_t and programming the hardware, else the
202 203 * the cpu_t could go away while we're looking at it.
203 204 */
204 205 mutex_enter(&cpu_lock);
205 206 cp = cpu_get(cpuid);
206 207
207 208 if (cp == NULL)
208 209 /*
209 210 * The CPU could have been DRd out while we were getting set up.
210 211 */
211 212 goto unbound;
212 213
213 214 mutex_enter(&cp->cpu_cpc_ctxlock);
214 215 kpreempt_disable();
215 216 save_spl = spl_xcall();
216 217
217 218 /*
218 219 * Check to see whether counters for CPU already being used by someone
219 220 * other than kernel for capacity and utilization (since kernel will
220 221 * let go of counters for user in kcpc_program() below)
221 222 */
222 223 if (cp->cpu_cpc_ctx != NULL && !CU_CPC_ON(cp)) {
223 224 /*
224 225 * If this CPU already has a bound set, return an error.
225 226 */
226 227 splx(save_spl);
227 228 kpreempt_enable();
228 229 mutex_exit(&cp->cpu_cpc_ctxlock);
229 230 goto unbound;
230 231 }
231 232
232 233 if (curthread->t_bind_cpu != cpuid) {
233 234 splx(save_spl);
234 235 kpreempt_enable();
235 236 mutex_exit(&cp->cpu_cpc_ctxlock);
236 237 goto unbound;
237 238 }
238 239
239 240 kcpc_program(ctx, B_FALSE, B_TRUE);
240 241
241 242 splx(save_spl);
242 243 kpreempt_enable();
243 244
244 245 mutex_exit(&cp->cpu_cpc_ctxlock);
245 246 mutex_exit(&cpu_lock);
246 247
247 248 mutex_enter(&set->ks_lock);
248 249 set->ks_state |= KCPC_SET_BOUND;
249 250 cv_signal(&set->ks_condv);
250 251 mutex_exit(&set->ks_lock);
251 252
252 253 return (0);
253 254
254 255 unbound:
255 256 mutex_exit(&cpu_lock);
256 257 set->ks_ctx = NULL;
257 258 kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
258 259 kcpc_ctx_free(ctx);
259 260 return (EAGAIN);
260 261 }
261 262
262 263 int
263 264 kcpc_bind_thread(kcpc_set_t *set, kthread_t *t, int *subcode)
264 265 {
265 266 kcpc_ctx_t *ctx;
266 267 int error;
267 268
268 269 /*
269 270 * Only one set is allowed per context, so ensure there is no
270 271 * existing context.
271 272 */
272 273
273 274 if (t->t_cpc_ctx != NULL)
274 275 return (EEXIST);
275 276
276 277 ctx = kcpc_ctx_alloc(KM_SLEEP);
277 278
278 279 /*
279 280 * The context must begin life frozen until it has been properly
280 281 * programmed onto the hardware. This prevents the context ops from
281 282 * worrying about it until we're ready.
282 283 */
283 284 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE);
284 285 ctx->kc_hrtime = gethrtime();
285 286
286 287 if (kcpc_assign_reqs(set, ctx) != 0) {
287 288 kcpc_ctx_free(ctx);
288 289 *subcode = CPC_RESOURCE_UNAVAIL;
289 290 return (EINVAL);
290 291 }
291 292
292 293 ctx->kc_cpuid = -1;
293 294 if (set->ks_flags & CPC_BIND_LWP_INHERIT)
294 295 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_LWPINHERIT);
295 296 ctx->kc_thread = t;
296 297 t->t_cpc_ctx = ctx;
297 298 /*
298 299 * Permit threads to look at their own hardware counters from userland.
299 300 */
300 301 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_NONPRIV);
301 302
302 303 /*
303 304 * Create the data store for this set.
304 305 */
305 306 set->ks_data = kmem_alloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP);
306 307
307 308 if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) {
308 309 kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
309 310 kcpc_ctx_free(ctx);
310 311 t->t_cpc_ctx = NULL;
|
↓ open down ↓ |
277 lines elided |
↑ open up ↑ |
311 312 return (error);
312 313 }
313 314
314 315 set->ks_ctx = ctx;
315 316 ctx->kc_set = set;
316 317
317 318 /*
318 319 * Add a device context to the subject thread.
319 320 */
320 321 installctx(t, ctx, kcpc_save, kcpc_restore, NULL,
321 - kcpc_lwp_create, NULL, kcpc_free);
322 + kcpc_lwp_create, NULL, kcpc_free, NULL);
322 323
323 324 /*
324 325 * Ask the backend to program the hardware.
325 326 */
326 327 if (t == curthread) {
327 328 int save_spl;
328 329
329 330 kpreempt_disable();
330 331 save_spl = spl_xcall();
331 332 kcpc_program(ctx, B_TRUE, B_TRUE);
332 333 splx(save_spl);
333 334 kpreempt_enable();
334 335 } else {
335 336 /*
336 337 * Since we are the agent LWP, we know the victim LWP is stopped
337 338 * until we're done here; no need to worry about preemption or
338 339 * migration here. We still use an atomic op to clear the flag
339 340 * to ensure the flags are always self-consistent; they can
340 341 * still be accessed from, for instance, another CPU doing a
341 342 * kcpc_invalidate_all().
342 343 */
343 344 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE);
344 345 }
345 346
346 347 mutex_enter(&set->ks_lock);
347 348 set->ks_state |= KCPC_SET_BOUND;
348 349 cv_signal(&set->ks_condv);
349 350 mutex_exit(&set->ks_lock);
350 351
351 352 return (0);
352 353 }
353 354
354 355 /*
355 356 * Walk through each request in the set and ask the PCBE to configure a
356 357 * corresponding counter.
357 358 */
358 359 int
359 360 kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode)
360 361 {
361 362 int i;
362 363 int ret;
363 364 kcpc_request_t *rp;
364 365
365 366 for (i = 0; i < set->ks_nreqs; i++) {
366 367 int n;
367 368 rp = &set->ks_req[i];
368 369
369 370 n = rp->kr_picnum;
370 371
371 372 ASSERT(n >= 0 && n < cpc_ncounters);
372 373
373 374 ASSERT(ctx->kc_pics[n].kp_req == NULL);
374 375
375 376 if (rp->kr_flags & CPC_OVF_NOTIFY_EMT) {
376 377 if ((pcbe_ops->pcbe_caps & CPC_CAP_OVERFLOW_INTERRUPT)
377 378 == 0) {
378 379 *subcode = -1;
379 380 return (ENOTSUP);
380 381 }
381 382 /*
382 383 * If any of the counters have requested overflow
383 384 * notification, we flag the context as being one that
384 385 * cares about overflow.
385 386 */
386 387 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_SIGOVF);
387 388 }
388 389
389 390 rp->kr_config = NULL;
390 391 if ((ret = pcbe_ops->pcbe_configure(n, rp->kr_event,
391 392 rp->kr_preset, rp->kr_flags, rp->kr_nattrs, rp->kr_attr,
392 393 &(rp->kr_config), (void *)ctx)) != 0) {
393 394 kcpc_free_configs(set);
394 395 *subcode = ret;
395 396 switch (ret) {
396 397 case CPC_ATTR_REQUIRES_PRIVILEGE:
397 398 case CPC_HV_NO_ACCESS:
398 399 return (EACCES);
399 400 default:
400 401 return (EINVAL);
401 402 }
402 403 }
403 404
404 405 ctx->kc_pics[n].kp_req = rp;
405 406 rp->kr_picp = &ctx->kc_pics[n];
406 407 rp->kr_data = set->ks_data + rp->kr_index;
407 408 *rp->kr_data = rp->kr_preset;
408 409 }
409 410
410 411 return (0);
411 412 }
412 413
413 414 void
414 415 kcpc_free_configs(kcpc_set_t *set)
415 416 {
416 417 int i;
417 418
418 419 for (i = 0; i < set->ks_nreqs; i++)
419 420 if (set->ks_req[i].kr_config != NULL)
420 421 pcbe_ops->pcbe_free(set->ks_req[i].kr_config);
421 422 }
422 423
423 424 /*
424 425 * buf points to a user address and the data should be copied out to that
425 426 * address in the current process.
426 427 */
427 428 int
428 429 kcpc_sample(kcpc_set_t *set, uint64_t *buf, hrtime_t *hrtime, uint64_t *tick)
429 430 {
430 431 kcpc_ctx_t *ctx = set->ks_ctx;
431 432 int save_spl;
432 433
433 434 mutex_enter(&set->ks_lock);
434 435 if ((set->ks_state & KCPC_SET_BOUND) == 0) {
435 436 mutex_exit(&set->ks_lock);
436 437 return (EINVAL);
437 438 }
438 439 mutex_exit(&set->ks_lock);
439 440
440 441 /*
441 442 * Kernel preemption must be disabled while reading the hardware regs,
442 443 * and if this is a CPU-bound context, while checking the CPU binding of
443 444 * the current thread.
444 445 */
445 446 kpreempt_disable();
446 447 save_spl = spl_xcall();
447 448
448 449 if (ctx->kc_flags & KCPC_CTX_INVALID) {
449 450 splx(save_spl);
450 451 kpreempt_enable();
451 452 return (EAGAIN);
452 453 }
453 454
454 455 if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) {
455 456 if (ctx->kc_cpuid != -1) {
456 457 if (curthread->t_bind_cpu != ctx->kc_cpuid) {
457 458 splx(save_spl);
458 459 kpreempt_enable();
459 460 return (EAGAIN);
460 461 }
461 462 }
462 463
463 464 if (ctx->kc_thread == curthread) {
464 465 uint64_t curtick = KCPC_GET_TICK();
465 466
466 467 ctx->kc_hrtime = gethrtime_waitfree();
467 468 pcbe_ops->pcbe_sample(ctx);
468 469 ctx->kc_vtick += curtick - ctx->kc_rawtick;
469 470 ctx->kc_rawtick = curtick;
470 471 }
471 472
472 473 /*
473 474 * The config may have been invalidated by
474 475 * the pcbe_sample op.
475 476 */
476 477 if (ctx->kc_flags & KCPC_CTX_INVALID) {
477 478 splx(save_spl);
478 479 kpreempt_enable();
479 480 return (EAGAIN);
480 481 }
481 482
482 483 }
483 484
484 485 splx(save_spl);
485 486 kpreempt_enable();
486 487
487 488 if (copyout(set->ks_data, buf,
488 489 set->ks_nreqs * sizeof (uint64_t)) == -1)
489 490 return (EFAULT);
490 491 if (copyout(&ctx->kc_hrtime, hrtime, sizeof (uint64_t)) == -1)
491 492 return (EFAULT);
492 493 if (copyout(&ctx->kc_vtick, tick, sizeof (uint64_t)) == -1)
493 494 return (EFAULT);
494 495
495 496 return (0);
496 497 }
497 498
498 499 /*
499 500 * Stop the counters on the CPU this context is bound to.
500 501 */
501 502 static void
502 503 kcpc_stop_hw(kcpc_ctx_t *ctx)
503 504 {
504 505 cpu_t *cp;
505 506
506 507 kpreempt_disable();
507 508
508 509 if (ctx->kc_cpuid == CPU->cpu_id) {
509 510 cp = CPU;
510 511 } else {
511 512 cp = cpu_get(ctx->kc_cpuid);
512 513 }
513 514
514 515 ASSERT(cp != NULL && cp->cpu_cpc_ctx == ctx);
515 516 kcpc_cpu_stop(cp, B_FALSE);
516 517
517 518 kpreempt_enable();
518 519 }
519 520
520 521 int
521 522 kcpc_unbind(kcpc_set_t *set)
522 523 {
523 524 kcpc_ctx_t *ctx;
524 525 kthread_t *t;
525 526
526 527 /*
527 528 * We could be racing with the process's agent thread as it
528 529 * binds the set; we must wait for the set to finish binding
529 530 * before attempting to tear it down.
530 531 */
531 532 mutex_enter(&set->ks_lock);
532 533 while ((set->ks_state & KCPC_SET_BOUND) == 0)
533 534 cv_wait(&set->ks_condv, &set->ks_lock);
534 535 mutex_exit(&set->ks_lock);
535 536
536 537 ctx = set->ks_ctx;
537 538
538 539 /*
539 540 * Use kc_lock to synchronize with kcpc_restore().
540 541 */
541 542 mutex_enter(&ctx->kc_lock);
542 543 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID);
543 544 mutex_exit(&ctx->kc_lock);
544 545
545 546 if (ctx->kc_cpuid == -1) {
546 547 t = ctx->kc_thread;
547 548 /*
548 549 * The context is thread-bound and therefore has a device
549 550 * context. It will be freed via removectx() calling
550 551 * freectx() calling kcpc_free().
551 552 */
552 553 if (t == curthread) {
553 554 int save_spl;
554 555
555 556 kpreempt_disable();
556 557 save_spl = spl_xcall();
557 558 if (!(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED))
558 559 kcpc_unprogram(ctx, B_TRUE);
559 560 splx(save_spl);
560 561 kpreempt_enable();
561 562 }
562 563 #ifdef DEBUG
563 564 if (removectx(t, ctx, kcpc_save, kcpc_restore, NULL,
564 565 kcpc_lwp_create, NULL, kcpc_free) == 0)
565 566 panic("kcpc_unbind: context %p not preset on thread %p",
566 567 (void *)ctx, (void *)t);
567 568 #else
568 569 (void) removectx(t, ctx, kcpc_save, kcpc_restore, NULL,
569 570 kcpc_lwp_create, NULL, kcpc_free);
570 571 #endif /* DEBUG */
571 572 t->t_cpc_set = NULL;
572 573 t->t_cpc_ctx = NULL;
573 574 } else {
574 575 /*
575 576 * If we are unbinding a CPU-bound set from a remote CPU, the
576 577 * native CPU's idle thread could be in the midst of programming
577 578 * this context onto the CPU. We grab the context's lock here to
578 579 * ensure that the idle thread is done with it. When we release
579 580 * the lock, the CPU no longer has a context and the idle thread
580 581 * will move on.
581 582 *
582 583 * cpu_lock must be held to prevent the CPU from being DR'd out
583 584 * while we disassociate the context from the cpu_t.
584 585 */
585 586 cpu_t *cp;
586 587 mutex_enter(&cpu_lock);
587 588 cp = cpu_get(ctx->kc_cpuid);
588 589 if (cp != NULL) {
589 590 /*
590 591 * The CPU may have been DR'd out of the system.
591 592 */
592 593 mutex_enter(&cp->cpu_cpc_ctxlock);
593 594 if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0)
594 595 kcpc_stop_hw(ctx);
595 596 ASSERT(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED);
596 597 mutex_exit(&cp->cpu_cpc_ctxlock);
597 598 }
598 599 mutex_exit(&cpu_lock);
599 600 if (ctx->kc_thread == curthread) {
600 601 kcpc_free(ctx, 0);
601 602 curthread->t_cpc_set = NULL;
602 603 }
603 604 }
604 605
605 606 return (0);
606 607 }
607 608
608 609 int
609 610 kcpc_preset(kcpc_set_t *set, int index, uint64_t preset)
610 611 {
611 612 int i;
612 613
613 614 ASSERT(set != NULL);
614 615 ASSERT(set->ks_state & KCPC_SET_BOUND);
615 616 ASSERT(set->ks_ctx->kc_thread == curthread);
616 617 ASSERT(set->ks_ctx->kc_cpuid == -1);
617 618
618 619 if (index < 0 || index >= set->ks_nreqs)
619 620 return (EINVAL);
620 621
621 622 for (i = 0; i < set->ks_nreqs; i++)
622 623 if (set->ks_req[i].kr_index == index)
623 624 break;
624 625 ASSERT(i != set->ks_nreqs);
625 626
626 627 set->ks_req[i].kr_preset = preset;
627 628 return (0);
628 629 }
629 630
630 631 int
631 632 kcpc_restart(kcpc_set_t *set)
632 633 {
633 634 kcpc_ctx_t *ctx = set->ks_ctx;
634 635 int i;
635 636 int save_spl;
636 637
637 638 ASSERT(set->ks_state & KCPC_SET_BOUND);
638 639 ASSERT(ctx->kc_thread == curthread);
639 640 ASSERT(ctx->kc_cpuid == -1);
640 641
641 642 for (i = 0; i < set->ks_nreqs; i++) {
642 643 *(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset;
643 644 pcbe_ops->pcbe_configure(0, NULL, set->ks_req[i].kr_preset,
644 645 0, 0, NULL, &set->ks_req[i].kr_config, NULL);
645 646 }
646 647
647 648 kpreempt_disable();
648 649 save_spl = spl_xcall();
649 650
650 651 /*
651 652 * If the user is doing this on a running set, make sure the counters
652 653 * are stopped first.
653 654 */
654 655 if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0)
655 656 pcbe_ops->pcbe_allstop();
656 657
657 658 /*
658 659 * Ask the backend to program the hardware.
659 660 */
660 661 ctx->kc_rawtick = KCPC_GET_TICK();
661 662 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE);
662 663 pcbe_ops->pcbe_program(ctx);
663 664 splx(save_spl);
664 665 kpreempt_enable();
665 666
666 667 return (0);
667 668 }
668 669
669 670 /*
670 671 * Caller must hold kcpc_cpuctx_lock.
671 672 */
672 673 int
673 674 kcpc_enable(kthread_t *t, int cmd, int enable)
674 675 {
675 676 kcpc_ctx_t *ctx = t->t_cpc_ctx;
676 677 kcpc_set_t *set = t->t_cpc_set;
677 678 kcpc_set_t *newset;
678 679 int i;
679 680 int flag;
680 681 int err;
681 682
682 683 ASSERT(RW_READ_HELD(&kcpc_cpuctx_lock));
683 684
684 685 if (ctx == NULL) {
685 686 /*
686 687 * This thread has a set but no context; it must be a
687 688 * CPU-bound set.
688 689 */
689 690 ASSERT(t->t_cpc_set != NULL);
690 691 ASSERT(t->t_cpc_set->ks_ctx->kc_cpuid != -1);
691 692 return (EINVAL);
692 693 } else if (ctx->kc_flags & KCPC_CTX_INVALID)
693 694 return (EAGAIN);
694 695
695 696 if (cmd == CPC_ENABLE) {
696 697 if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0)
697 698 return (EINVAL);
698 699 kpreempt_disable();
699 700 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE);
700 701 kcpc_restore(ctx);
701 702 kpreempt_enable();
702 703 } else if (cmd == CPC_DISABLE) {
703 704 if (ctx->kc_flags & KCPC_CTX_FREEZE)
704 705 return (EINVAL);
705 706 kpreempt_disable();
706 707 kcpc_save(ctx);
707 708 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE);
708 709 kpreempt_enable();
709 710 } else if (cmd == CPC_USR_EVENTS || cmd == CPC_SYS_EVENTS) {
710 711 /*
711 712 * Strategy for usr/sys: stop counters and update set's presets
712 713 * with current counter values, unbind, update requests with
713 714 * new config, then re-bind.
714 715 */
715 716 flag = (cmd == CPC_USR_EVENTS) ?
716 717 CPC_COUNT_USER: CPC_COUNT_SYSTEM;
717 718
718 719 kpreempt_disable();
719 720 KCPC_CTX_FLAG_SET(ctx,
720 721 KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);
721 722 pcbe_ops->pcbe_allstop();
722 723 kpreempt_enable();
723 724
724 725 for (i = 0; i < set->ks_nreqs; i++) {
725 726 set->ks_req[i].kr_preset = *(set->ks_req[i].kr_data);
726 727 if (enable)
727 728 set->ks_req[i].kr_flags |= flag;
728 729 else
729 730 set->ks_req[i].kr_flags &= ~flag;
730 731 }
731 732 newset = kcpc_dup_set(set);
732 733 if (kcpc_unbind(set) != 0)
733 734 return (EINVAL);
734 735 t->t_cpc_set = newset;
735 736 if (kcpc_bind_thread(newset, t, &err) != 0) {
736 737 t->t_cpc_set = NULL;
737 738 kcpc_free_set(newset);
738 739 return (EINVAL);
739 740 }
740 741 } else
741 742 return (EINVAL);
742 743
743 744 return (0);
744 745 }
745 746
746 747 /*
747 748 * Provide PCBEs with a way of obtaining the configs of every counter which will
748 749 * be programmed together.
749 750 *
750 751 * If current is NULL, provide the first config.
751 752 *
752 753 * If data != NULL, caller wants to know where the data store associated with
753 754 * the config we return is located.
754 755 */
755 756 void *
756 757 kcpc_next_config(void *token, void *current, uint64_t **data)
757 758 {
758 759 int i;
759 760 kcpc_pic_t *pic;
760 761 kcpc_ctx_t *ctx = (kcpc_ctx_t *)token;
761 762
762 763 if (current == NULL) {
763 764 /*
764 765 * Client would like the first config, which may not be in
765 766 * counter 0; we need to search through the counters for the
766 767 * first config.
767 768 */
768 769 for (i = 0; i < cpc_ncounters; i++)
769 770 if (ctx->kc_pics[i].kp_req != NULL)
770 771 break;
771 772 /*
772 773 * There are no counters configured for the given context.
773 774 */
774 775 if (i == cpc_ncounters)
775 776 return (NULL);
776 777 } else {
777 778 /*
778 779 * There surely is a faster way to do this.
779 780 */
780 781 for (i = 0; i < cpc_ncounters; i++) {
781 782 pic = &ctx->kc_pics[i];
782 783
783 784 if (pic->kp_req != NULL &&
784 785 current == pic->kp_req->kr_config)
785 786 break;
786 787 }
787 788
788 789 /*
789 790 * We found the current config at picnum i. Now search for the
790 791 * next configured PIC.
791 792 */
792 793 for (i++; i < cpc_ncounters; i++) {
793 794 pic = &ctx->kc_pics[i];
794 795 if (pic->kp_req != NULL)
795 796 break;
796 797 }
797 798
798 799 if (i == cpc_ncounters)
799 800 return (NULL);
800 801 }
801 802
802 803 if (data != NULL) {
803 804 *data = ctx->kc_pics[i].kp_req->kr_data;
804 805 }
805 806
806 807 return (ctx->kc_pics[i].kp_req->kr_config);
807 808 }
808 809
809 810
810 811 kcpc_ctx_t *
811 812 kcpc_ctx_alloc(int kmem_flags)
812 813 {
813 814 kcpc_ctx_t *ctx;
814 815 long hash;
815 816
816 817 ctx = (kcpc_ctx_t *)kmem_zalloc(sizeof (kcpc_ctx_t), kmem_flags);
817 818 if (ctx == NULL)
818 819 return (NULL);
819 820
820 821 hash = CPC_HASH_CTX(ctx);
821 822 mutex_enter(&kcpc_ctx_llock[hash]);
822 823 ctx->kc_next = kcpc_ctx_list[hash];
823 824 kcpc_ctx_list[hash] = ctx;
824 825 mutex_exit(&kcpc_ctx_llock[hash]);
825 826
826 827 ctx->kc_pics = (kcpc_pic_t *)kmem_zalloc(sizeof (kcpc_pic_t) *
827 828 cpc_ncounters, KM_SLEEP);
828 829
829 830 ctx->kc_cpuid = -1;
830 831
831 832 return (ctx);
832 833 }
833 834
834 835 /*
835 836 * Copy set from ctx to the child context, cctx, if it has CPC_BIND_LWP_INHERIT
836 837 * in the flags.
837 838 */
838 839 static void
839 840 kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx)
840 841 {
841 842 kcpc_set_t *ks = ctx->kc_set, *cks;
842 843 int i, j;
843 844 int code;
844 845
845 846 ASSERT(ks != NULL);
846 847
847 848 if ((ks->ks_flags & CPC_BIND_LWP_INHERIT) == 0)
848 849 return;
849 850
850 851 cks = kmem_zalloc(sizeof (*cks), KM_SLEEP);
851 852 cks->ks_state &= ~KCPC_SET_BOUND;
852 853 cctx->kc_set = cks;
853 854 cks->ks_flags = ks->ks_flags;
854 855 cks->ks_nreqs = ks->ks_nreqs;
855 856 cks->ks_req = kmem_alloc(cks->ks_nreqs *
856 857 sizeof (kcpc_request_t), KM_SLEEP);
857 858 cks->ks_data = kmem_alloc(cks->ks_nreqs * sizeof (uint64_t),
858 859 KM_SLEEP);
859 860 cks->ks_ctx = cctx;
860 861
861 862 for (i = 0; i < cks->ks_nreqs; i++) {
862 863 cks->ks_req[i].kr_index = ks->ks_req[i].kr_index;
863 864 cks->ks_req[i].kr_picnum = ks->ks_req[i].kr_picnum;
864 865 (void) strncpy(cks->ks_req[i].kr_event,
865 866 ks->ks_req[i].kr_event, CPC_MAX_EVENT_LEN);
866 867 cks->ks_req[i].kr_preset = ks->ks_req[i].kr_preset;
867 868 cks->ks_req[i].kr_flags = ks->ks_req[i].kr_flags;
868 869 cks->ks_req[i].kr_nattrs = ks->ks_req[i].kr_nattrs;
869 870 if (ks->ks_req[i].kr_nattrs > 0) {
870 871 cks->ks_req[i].kr_attr =
871 872 kmem_alloc(ks->ks_req[i].kr_nattrs *
872 873 sizeof (kcpc_attr_t), KM_SLEEP);
873 874 }
874 875 for (j = 0; j < ks->ks_req[i].kr_nattrs; j++) {
875 876 (void) strncpy(cks->ks_req[i].kr_attr[j].ka_name,
876 877 ks->ks_req[i].kr_attr[j].ka_name,
877 878 CPC_MAX_ATTR_LEN);
878 879 cks->ks_req[i].kr_attr[j].ka_val =
879 880 ks->ks_req[i].kr_attr[j].ka_val;
880 881 }
881 882 }
882 883 if (kcpc_configure_reqs(cctx, cks, &code) != 0)
883 884 kcpc_invalidate_config(cctx);
884 885
885 886 mutex_enter(&cks->ks_lock);
886 887 cks->ks_state |= KCPC_SET_BOUND;
887 888 cv_signal(&cks->ks_condv);
888 889 mutex_exit(&cks->ks_lock);
889 890 }
890 891
891 892
892 893 void
893 894 kcpc_ctx_free(kcpc_ctx_t *ctx)
894 895 {
895 896 kcpc_ctx_t **loc;
896 897 long hash = CPC_HASH_CTX(ctx);
897 898
898 899 mutex_enter(&kcpc_ctx_llock[hash]);
899 900 loc = &kcpc_ctx_list[hash];
900 901 ASSERT(*loc != NULL);
901 902 while (*loc != ctx)
902 903 loc = &(*loc)->kc_next;
903 904 *loc = ctx->kc_next;
904 905 mutex_exit(&kcpc_ctx_llock[hash]);
905 906
906 907 kmem_free(ctx->kc_pics, cpc_ncounters * sizeof (kcpc_pic_t));
907 908 cv_destroy(&ctx->kc_condv);
908 909 mutex_destroy(&ctx->kc_lock);
909 910 kmem_free(ctx, sizeof (*ctx));
910 911 }
911 912
912 913 /*
913 914 * Generic interrupt handler used on hardware that generates
914 915 * overflow interrupts.
915 916 *
916 917 * Note: executed at high-level interrupt context!
917 918 */
918 919 /*ARGSUSED*/
919 920 kcpc_ctx_t *
920 921 kcpc_overflow_intr(caddr_t arg, uint64_t bitmap)
921 922 {
922 923 kcpc_ctx_t *ctx;
923 924 kthread_t *t = curthread;
924 925 int i;
925 926
926 927 /*
927 928 * On both x86 and UltraSPARC, we may deliver the high-level
928 929 * interrupt in kernel mode, just after we've started to run an
929 930 * interrupt thread. (That's because the hardware helpfully
930 931 * delivers the overflow interrupt some random number of cycles
931 932 * after the instruction that caused the overflow by which time
932 933 * we're in some part of the kernel, not necessarily running on
933 934 * the right thread).
934 935 *
935 936 * Check for this case here -- find the pinned thread
936 937 * that was running when the interrupt went off.
937 938 */
938 939 if (t->t_flag & T_INTR_THREAD) {
939 940 klwp_t *lwp;
940 941
941 942 atomic_inc_32(&kcpc_intrctx_count);
942 943
943 944 /*
944 945 * Note that t_lwp is always set to point at the underlying
945 946 * thread, thus this will work in the presence of nested
946 947 * interrupts.
947 948 */
948 949 ctx = NULL;
949 950 if ((lwp = t->t_lwp) != NULL) {
950 951 t = lwptot(lwp);
951 952 ctx = t->t_cpc_ctx;
952 953 }
953 954 } else
954 955 ctx = t->t_cpc_ctx;
955 956
956 957 if (ctx == NULL) {
957 958 /*
958 959 * This can easily happen if we're using the counters in
959 960 * "shared" mode, for example, and an overflow interrupt
960 961 * occurs while we are running cpustat. In that case, the
961 962 * bound thread that has the context that belongs to this
962 963 * CPU is almost certainly sleeping (if it was running on
963 964 * the CPU we'd have found it above), and the actual
964 965 * interrupted thread has no knowledge of performance counters!
965 966 */
966 967 ctx = curthread->t_cpu->cpu_cpc_ctx;
967 968 if (ctx != NULL) {
968 969 /*
969 970 * Return the bound context for this CPU to
970 971 * the interrupt handler so that it can synchronously
971 972 * sample the hardware counters and restart them.
972 973 */
973 974 return (ctx);
974 975 }
975 976
976 977 /*
977 978 * As long as the overflow interrupt really is delivered early
978 979 * enough after trapping into the kernel to avoid switching
979 980 * threads, we must always be able to find the cpc context,
980 981 * or something went terribly wrong i.e. we ended up
981 982 * running a passivated interrupt thread, a kernel
982 983 * thread or we interrupted idle, all of which are Very Bad.
983 984 *
984 985 * We also could end up here owing to an incredibly unlikely
985 986 * race condition that exists on x86 based architectures when
986 987 * the cpc provider is in use; overflow interrupts are directed
987 988 * to the cpc provider if the 'dtrace_cpc_in_use' variable is
988 989 * set when we enter the handler. This variable is unset after
989 990 * overflow interrupts have been disabled on all CPUs and all
990 991 * contexts have been torn down. To stop interrupts, the cpc
991 992 * provider issues a xcall to the remote CPU before it tears
992 993 * down that CPUs context. As high priority xcalls, on an x86
993 994 * architecture, execute at a higher PIL than this handler, it
994 995 * is possible (though extremely unlikely) that the xcall could
995 996 * interrupt the overflow handler before the handler has
996 997 * checked the 'dtrace_cpc_in_use' variable, stop the counters,
997 998 * return to the cpc provider which could then rip down
998 999 * contexts and unset 'dtrace_cpc_in_use' *before* the CPUs
999 1000 * overflow handler has had a chance to check the variable. In
1000 1001 * that case, the handler would direct the overflow into this
1001 1002 * code and no valid context will be found. The default behavior
1002 1003 * when no valid context is found is now to shout a warning to
1003 1004 * the console and bump the 'kcpc_nullctx_count' variable.
1004 1005 */
1005 1006 if (kcpc_nullctx_panic)
1006 1007 panic("null cpc context, thread %p", (void *)t);
1007 1008 #ifdef DEBUG
1008 1009 cmn_err(CE_NOTE,
1009 1010 "null cpc context found in overflow handler!\n");
1010 1011 #endif
1011 1012 atomic_inc_32(&kcpc_nullctx_count);
1012 1013 } else if ((ctx->kc_flags & KCPC_CTX_INVALID) == 0) {
1013 1014 /*
1014 1015 * Schedule an ast to sample the counters, which will
1015 1016 * propagate any overflow into the virtualized performance
1016 1017 * counter(s), and may deliver a signal.
1017 1018 */
1018 1019 ttolwp(t)->lwp_pcb.pcb_flags |= CPC_OVERFLOW;
1019 1020 /*
1020 1021 * If a counter has overflowed which was counting on behalf of
1021 1022 * a request which specified CPC_OVF_NOTIFY_EMT, send the
1022 1023 * process a signal.
1023 1024 */
1024 1025 for (i = 0; i < cpc_ncounters; i++) {
1025 1026 if (ctx->kc_pics[i].kp_req != NULL &&
1026 1027 bitmap & (1 << i) &&
1027 1028 ctx->kc_pics[i].kp_req->kr_flags &
1028 1029 CPC_OVF_NOTIFY_EMT) {
1029 1030 /*
1030 1031 * A signal has been requested for this PIC, so
1031 1032 * so freeze the context. The interrupt handler
1032 1033 * has already stopped the counter hardware.
1033 1034 */
1034 1035 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE);
1035 1036 atomic_or_uint(&ctx->kc_pics[i].kp_flags,
1036 1037 KCPC_PIC_OVERFLOWED);
1037 1038 }
1038 1039 }
1039 1040 aston(t);
1040 1041 } else if (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) {
1041 1042 /*
1042 1043 * Thread context is no longer valid, but here may be a valid
1043 1044 * CPU context.
1044 1045 */
1045 1046 return (curthread->t_cpu->cpu_cpc_ctx);
1046 1047 }
1047 1048
1048 1049 return (NULL);
1049 1050 }
1050 1051
1051 1052 /*
1052 1053 * The current thread context had an overflow interrupt; we're
1053 1054 * executing here in high-level interrupt context.
1054 1055 */
1055 1056 /*ARGSUSED*/
1056 1057 uint_t
1057 1058 kcpc_hw_overflow_intr(caddr_t arg1, caddr_t arg2)
1058 1059 {
1059 1060 kcpc_ctx_t *ctx;
1060 1061 uint64_t bitmap;
1061 1062 uint8_t *state;
1062 1063 int save_spl;
1063 1064
1064 1065 if (pcbe_ops == NULL ||
1065 1066 (bitmap = pcbe_ops->pcbe_overflow_bitmap()) == 0)
1066 1067 return (DDI_INTR_UNCLAIMED);
1067 1068
1068 1069 /*
1069 1070 * Prevent any further interrupts.
1070 1071 */
1071 1072 pcbe_ops->pcbe_allstop();
1072 1073
1073 1074 if (dtrace_cpc_in_use) {
1074 1075 state = &cpu_core[CPU->cpu_id].cpuc_dcpc_intr_state;
1075 1076
1076 1077 /*
1077 1078 * Set the per-CPU state bit to indicate that we are currently
1078 1079 * processing an interrupt if it is currently free. Drop the
1079 1080 * interrupt if the state isn't free (i.e. a configuration
1080 1081 * event is taking place).
1081 1082 */
1082 1083 if (atomic_cas_8(state, DCPC_INTR_FREE,
1083 1084 DCPC_INTR_PROCESSING) == DCPC_INTR_FREE) {
1084 1085 int i;
1085 1086 kcpc_request_t req;
1086 1087
1087 1088 ASSERT(dtrace_cpc_fire != NULL);
1088 1089
1089 1090 (*dtrace_cpc_fire)(bitmap);
1090 1091
1091 1092 ctx = curthread->t_cpu->cpu_cpc_ctx;
1092 1093 if (ctx == NULL) {
1093 1094 #ifdef DEBUG
1094 1095 cmn_err(CE_NOTE, "null cpc context in"
1095 1096 "hardware overflow handler!\n");
1096 1097 #endif
1097 1098 return (DDI_INTR_CLAIMED);
1098 1099 }
1099 1100
1100 1101 /* Reset any counters that have overflowed */
1101 1102 for (i = 0; i < ctx->kc_set->ks_nreqs; i++) {
1102 1103 req = ctx->kc_set->ks_req[i];
1103 1104
1104 1105 if (bitmap & (1 << req.kr_picnum)) {
1105 1106 pcbe_ops->pcbe_configure(req.kr_picnum,
1106 1107 req.kr_event, req.kr_preset,
1107 1108 req.kr_flags, req.kr_nattrs,
1108 1109 req.kr_attr, &(req.kr_config),
1109 1110 (void *)ctx);
1110 1111 }
1111 1112 }
1112 1113 pcbe_ops->pcbe_program(ctx);
1113 1114
1114 1115 /*
1115 1116 * We've finished processing the interrupt so set
1116 1117 * the state back to free.
1117 1118 */
1118 1119 cpu_core[CPU->cpu_id].cpuc_dcpc_intr_state =
1119 1120 DCPC_INTR_FREE;
1120 1121 membar_producer();
1121 1122 }
1122 1123 return (DDI_INTR_CLAIMED);
1123 1124 }
1124 1125
1125 1126 /*
1126 1127 * DTrace isn't involved so pass on accordingly.
1127 1128 *
1128 1129 * If the interrupt has occurred in the context of an lwp owning
1129 1130 * the counters, then the handler posts an AST to the lwp to
1130 1131 * trigger the actual sampling, and optionally deliver a signal or
1131 1132 * restart the counters, on the way out of the kernel using
1132 1133 * kcpc_hw_overflow_ast() (see below).
1133 1134 *
1134 1135 * On the other hand, if the handler returns the context to us
1135 1136 * directly, then it means that there are no other threads in
1136 1137 * the middle of updating it, no AST has been posted, and so we
1137 1138 * should sample the counters here, and restart them with no
1138 1139 * further fuss.
1139 1140 *
1140 1141 * The CPU's CPC context may disappear as a result of cross-call which
1141 1142 * has higher PIL on x86, so protect the context by raising PIL to the
1142 1143 * cross-call level.
1143 1144 */
1144 1145 save_spl = spl_xcall();
1145 1146 if ((ctx = kcpc_overflow_intr(arg1, bitmap)) != NULL) {
1146 1147 uint64_t curtick = KCPC_GET_TICK();
1147 1148
1148 1149 ctx->kc_hrtime = gethrtime_waitfree();
1149 1150 ctx->kc_vtick += curtick - ctx->kc_rawtick;
1150 1151 ctx->kc_rawtick = curtick;
1151 1152 pcbe_ops->pcbe_sample(ctx);
1152 1153 pcbe_ops->pcbe_program(ctx);
1153 1154 }
1154 1155 splx(save_spl);
1155 1156
1156 1157 return (DDI_INTR_CLAIMED);
1157 1158 }
1158 1159
1159 1160 /*
1160 1161 * Called from trap() when processing the ast posted by the high-level
1161 1162 * interrupt handler.
1162 1163 */
1163 1164 int
1164 1165 kcpc_overflow_ast()
1165 1166 {
1166 1167 kcpc_ctx_t *ctx = curthread->t_cpc_ctx;
1167 1168 int i;
1168 1169 int found = 0;
1169 1170 uint64_t curtick = KCPC_GET_TICK();
1170 1171
1171 1172 ASSERT(ctx != NULL); /* Beware of interrupt skid. */
1172 1173
1173 1174 /*
1174 1175 * An overflow happened: sample the context to ensure that
1175 1176 * the overflow is propagated into the upper bits of the
1176 1177 * virtualized 64-bit counter(s).
1177 1178 */
1178 1179 kpreempt_disable();
1179 1180 ctx->kc_hrtime = gethrtime_waitfree();
1180 1181 pcbe_ops->pcbe_sample(ctx);
1181 1182 kpreempt_enable();
1182 1183
1183 1184 ctx->kc_vtick += curtick - ctx->kc_rawtick;
1184 1185
1185 1186 /*
1186 1187 * The interrupt handler has marked any pics with KCPC_PIC_OVERFLOWED
1187 1188 * if that pic generated an overflow and if the request it was counting
1188 1189 * on behalf of had CPC_OVERFLOW_REQUEST specified. We go through all
1189 1190 * pics in the context and clear the KCPC_PIC_OVERFLOWED flags. If we
1190 1191 * found any overflowed pics, keep the context frozen and return true
1191 1192 * (thus causing a signal to be sent).
1192 1193 */
1193 1194 for (i = 0; i < cpc_ncounters; i++) {
1194 1195 if (ctx->kc_pics[i].kp_flags & KCPC_PIC_OVERFLOWED) {
1195 1196 atomic_and_uint(&ctx->kc_pics[i].kp_flags,
1196 1197 ~KCPC_PIC_OVERFLOWED);
1197 1198 found = 1;
1198 1199 }
1199 1200 }
1200 1201 if (found)
1201 1202 return (1);
1202 1203
1203 1204 /*
1204 1205 * Otherwise, re-enable the counters and continue life as before.
1205 1206 */
1206 1207 kpreempt_disable();
1207 1208 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE);
1208 1209 pcbe_ops->pcbe_program(ctx);
1209 1210 kpreempt_enable();
1210 1211 return (0);
1211 1212 }
1212 1213
1213 1214 /*
1214 1215 * Called when switching away from current thread.
1215 1216 */
1216 1217 static void
1217 1218 kcpc_save(kcpc_ctx_t *ctx)
1218 1219 {
1219 1220 int err;
1220 1221 int save_spl;
1221 1222
1222 1223 kpreempt_disable();
1223 1224 save_spl = spl_xcall();
1224 1225
1225 1226 if (ctx->kc_flags & KCPC_CTX_INVALID) {
1226 1227 if (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) {
1227 1228 splx(save_spl);
1228 1229 kpreempt_enable();
1229 1230 return;
1230 1231 }
1231 1232 /*
1232 1233 * This context has been invalidated but the counters have not
1233 1234 * been stopped. Stop them here and mark the context stopped.
1234 1235 */
1235 1236 kcpc_unprogram(ctx, B_TRUE);
1236 1237 splx(save_spl);
1237 1238 kpreempt_enable();
1238 1239 return;
1239 1240 }
1240 1241
1241 1242 pcbe_ops->pcbe_allstop();
1242 1243 if (ctx->kc_flags & KCPC_CTX_FREEZE) {
1243 1244 splx(save_spl);
1244 1245 kpreempt_enable();
1245 1246 return;
1246 1247 }
1247 1248
1248 1249 /*
1249 1250 * Need to sample for all reqs into each req's current mpic.
1250 1251 */
1251 1252 ctx->kc_hrtime = gethrtime_waitfree();
1252 1253 ctx->kc_vtick += KCPC_GET_TICK() - ctx->kc_rawtick;
1253 1254 pcbe_ops->pcbe_sample(ctx);
1254 1255
1255 1256 /*
1256 1257 * Program counter for measuring capacity and utilization since user
1257 1258 * thread isn't using counter anymore
1258 1259 */
1259 1260 ASSERT(ctx->kc_cpuid == -1);
1260 1261 cu_cpc_program(CPU, &err);
1261 1262 splx(save_spl);
1262 1263 kpreempt_enable();
1263 1264 }
1264 1265
1265 1266 static void
1266 1267 kcpc_restore(kcpc_ctx_t *ctx)
1267 1268 {
1268 1269 int save_spl;
1269 1270
1270 1271 mutex_enter(&ctx->kc_lock);
1271 1272
1272 1273 if ((ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED)) ==
1273 1274 KCPC_CTX_INVALID) {
1274 1275 /*
1275 1276 * The context is invalidated but has not been marked stopped.
1276 1277 * We mark it as such here because we will not start the
1277 1278 * counters during this context switch.
1278 1279 */
1279 1280 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID_STOPPED);
1280 1281 }
1281 1282
1282 1283 if (ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_FREEZE)) {
1283 1284 mutex_exit(&ctx->kc_lock);
1284 1285 return;
1285 1286 }
1286 1287
1287 1288 /*
1288 1289 * Set kc_flags to show that a kcpc_restore() is in progress to avoid
1289 1290 * ctx & set related memory objects being freed without us knowing.
1290 1291 * This can happen if an agent thread is executing a kcpc_unbind(),
1291 1292 * with this thread as the target, whilst we're concurrently doing a
1292 1293 * restorectx() during, for example, a proc_exit(). Effectively, by
1293 1294 * doing this, we're asking kcpc_free() to cv_wait() until
1294 1295 * kcpc_restore() has completed.
1295 1296 */
1296 1297 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_RESTORE);
1297 1298 mutex_exit(&ctx->kc_lock);
1298 1299
1299 1300 /*
1300 1301 * While programming the hardware, the counters should be stopped. We
1301 1302 * don't do an explicit pcbe_allstop() here because they should have
1302 1303 * been stopped already by the last consumer.
1303 1304 */
1304 1305 kpreempt_disable();
1305 1306 save_spl = spl_xcall();
1306 1307 kcpc_program(ctx, B_TRUE, B_TRUE);
1307 1308 splx(save_spl);
1308 1309 kpreempt_enable();
1309 1310
1310 1311 /*
1311 1312 * Wake the agent thread if it's waiting in kcpc_free().
1312 1313 */
1313 1314 mutex_enter(&ctx->kc_lock);
1314 1315 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_RESTORE);
1315 1316 cv_signal(&ctx->kc_condv);
1316 1317 mutex_exit(&ctx->kc_lock);
1317 1318 }
1318 1319
1319 1320 /*
1320 1321 * If kcpc_counts_include_idle is set to 0 by the sys admin, we add the the
1321 1322 * following context operators to the idle thread on each CPU. They stop the
1322 1323 * counters when the idle thread is switched on, and they start them again when
1323 1324 * it is switched off.
1324 1325 */
1325 1326 /*ARGSUSED*/
1326 1327 void
1327 1328 kcpc_idle_save(struct cpu *cp)
1328 1329 {
1329 1330 /*
1330 1331 * The idle thread shouldn't be run anywhere else.
1331 1332 */
1332 1333 ASSERT(CPU == cp);
1333 1334
1334 1335 /*
1335 1336 * We must hold the CPU's context lock to ensure the context isn't freed
1336 1337 * while we're looking at it.
1337 1338 */
1338 1339 mutex_enter(&cp->cpu_cpc_ctxlock);
1339 1340
1340 1341 if ((cp->cpu_cpc_ctx == NULL) ||
1341 1342 (cp->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID)) {
1342 1343 mutex_exit(&cp->cpu_cpc_ctxlock);
1343 1344 return;
1344 1345 }
1345 1346
1346 1347 pcbe_ops->pcbe_program(cp->cpu_cpc_ctx);
1347 1348 mutex_exit(&cp->cpu_cpc_ctxlock);
1348 1349 }
1349 1350
1350 1351 void
1351 1352 kcpc_idle_restore(struct cpu *cp)
1352 1353 {
1353 1354 /*
1354 1355 * The idle thread shouldn't be run anywhere else.
1355 1356 */
1356 1357 ASSERT(CPU == cp);
1357 1358
1358 1359 /*
1359 1360 * We must hold the CPU's context lock to ensure the context isn't freed
1360 1361 * while we're looking at it.
1361 1362 */
1362 1363 mutex_enter(&cp->cpu_cpc_ctxlock);
1363 1364
1364 1365 if ((cp->cpu_cpc_ctx == NULL) ||
1365 1366 (cp->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID)) {
1366 1367 mutex_exit(&cp->cpu_cpc_ctxlock);
1367 1368 return;
1368 1369 }
1369 1370
1370 1371 pcbe_ops->pcbe_allstop();
1371 1372 mutex_exit(&cp->cpu_cpc_ctxlock);
1372 1373 }
1373 1374
1374 1375 /*ARGSUSED*/
1375 1376 static void
1376 1377 kcpc_lwp_create(kthread_t *t, kthread_t *ct)
1377 1378 {
1378 1379 kcpc_ctx_t *ctx = t->t_cpc_ctx, *cctx;
1379 1380 int i;
1380 1381
1381 1382 if (ctx == NULL || (ctx->kc_flags & KCPC_CTX_LWPINHERIT) == 0)
1382 1383 return;
1383 1384
1384 1385 rw_enter(&kcpc_cpuctx_lock, RW_READER);
1385 1386 if (ctx->kc_flags & KCPC_CTX_INVALID) {
1386 1387 rw_exit(&kcpc_cpuctx_lock);
1387 1388 return;
1388 1389 }
1389 1390 cctx = kcpc_ctx_alloc(KM_SLEEP);
1390 1391 kcpc_ctx_clone(ctx, cctx);
1391 1392 rw_exit(&kcpc_cpuctx_lock);
1392 1393
1393 1394 /*
1394 1395 * Copy the parent context's kc_flags field, but don't overwrite
1395 1396 * the child's in case it was modified during kcpc_ctx_clone.
1396 1397 */
1397 1398 KCPC_CTX_FLAG_SET(cctx, ctx->kc_flags);
1398 1399 cctx->kc_thread = ct;
1399 1400 cctx->kc_cpuid = -1;
1400 1401 ct->t_cpc_set = cctx->kc_set;
1401 1402 ct->t_cpc_ctx = cctx;
1402 1403
1403 1404 if (cctx->kc_flags & KCPC_CTX_SIGOVF) {
1404 1405 kcpc_set_t *ks = cctx->kc_set;
1405 1406 /*
1406 1407 * Our contract with the user requires us to immediately send an
1407 1408 * overflow signal to all children if we have the LWPINHERIT
1408 1409 * and SIGOVF flags set. In addition, all counters should be
1409 1410 * set to UINT64_MAX, and their pic's overflow flag turned on
1410 1411 * so that our trap() processing knows to send a signal.
1411 1412 */
1412 1413 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_FREEZE);
1413 1414 for (i = 0; i < ks->ks_nreqs; i++) {
1414 1415 kcpc_request_t *kr = &ks->ks_req[i];
1415 1416
1416 1417 if (kr->kr_flags & CPC_OVF_NOTIFY_EMT) {
|
↓ open down ↓ |
1085 lines elided |
↑ open up ↑ |
1417 1418 *(kr->kr_data) = UINT64_MAX;
1418 1419 atomic_or_uint(&kr->kr_picp->kp_flags,
1419 1420 KCPC_PIC_OVERFLOWED);
1420 1421 }
1421 1422 }
1422 1423 ttolwp(ct)->lwp_pcb.pcb_flags |= CPC_OVERFLOW;
1423 1424 aston(ct);
1424 1425 }
1425 1426
1426 1427 installctx(ct, cctx, kcpc_save, kcpc_restore,
1427 - NULL, kcpc_lwp_create, NULL, kcpc_free);
1428 + NULL, kcpc_lwp_create, NULL, kcpc_free, NULL);
1428 1429 }
1429 1430
1430 1431 /*
1431 1432 * Counter Stoppage Theory
1432 1433 *
1433 1434 * The counters may need to be stopped properly at the following occasions:
1434 1435 *
1435 1436 * 1) An LWP exits.
1436 1437 * 2) A thread exits.
1437 1438 * 3) An LWP performs an exec().
1438 1439 * 4) A bound set is unbound.
1439 1440 *
1440 1441 * In addition to stopping the counters, the CPC context (a kcpc_ctx_t) may need
1441 1442 * to be freed as well.
1442 1443 *
1443 1444 * Case 1: kcpc_passivate(), called via lwp_exit(), stops the counters. Later on
1444 1445 * when the thread is freed, kcpc_free(), called by freectx(), frees the
1445 1446 * context.
1446 1447 *
1447 1448 * Case 2: same as case 1 except kcpc_passivate is called from thread_exit().
1448 1449 *
1449 1450 * Case 3: kcpc_free(), called via freectx() via exec(), recognizes that it has
1450 1451 * been called from exec. It stops the counters _and_ frees the context.
1451 1452 *
1452 1453 * Case 4: kcpc_unbind() stops the hardware _and_ frees the context.
1453 1454 *
1454 1455 * CPU-bound counters are always stopped via kcpc_unbind().
1455 1456 */
1456 1457
1457 1458 /*
1458 1459 * We're being called to delete the context; we ensure that all associated data
1459 1460 * structures are freed, and that the hardware is passivated if this is an exec.
1460 1461 */
1461 1462
1462 1463 /*ARGSUSED*/
1463 1464 void
1464 1465 kcpc_free(kcpc_ctx_t *ctx, int isexec)
1465 1466 {
1466 1467 int i;
1467 1468 kcpc_set_t *set = ctx->kc_set;
1468 1469
1469 1470 ASSERT(set != NULL);
1470 1471
1471 1472 /*
1472 1473 * Wait for kcpc_restore() to finish before we tear things down.
1473 1474 */
1474 1475 mutex_enter(&ctx->kc_lock);
1475 1476 while (ctx->kc_flags & KCPC_CTX_RESTORE)
1476 1477 cv_wait(&ctx->kc_condv, &ctx->kc_lock);
1477 1478 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID);
1478 1479 mutex_exit(&ctx->kc_lock);
1479 1480
1480 1481 if (isexec) {
1481 1482 /*
1482 1483 * This thread is execing, and after the exec it should not have
1483 1484 * any performance counter context. Stop the counters properly
1484 1485 * here so the system isn't surprised by an overflow interrupt
1485 1486 * later.
1486 1487 */
1487 1488 if (ctx->kc_cpuid != -1) {
1488 1489 cpu_t *cp;
1489 1490 /*
1490 1491 * CPU-bound context; stop the appropriate CPU's ctrs.
1491 1492 * Hold cpu_lock while examining the CPU to ensure it
1492 1493 * doesn't go away.
1493 1494 */
1494 1495 mutex_enter(&cpu_lock);
1495 1496 cp = cpu_get(ctx->kc_cpuid);
1496 1497 /*
1497 1498 * The CPU could have been DR'd out, so only stop the
1498 1499 * CPU and clear its context pointer if the CPU still
1499 1500 * exists.
1500 1501 */
1501 1502 if (cp != NULL) {
1502 1503 mutex_enter(&cp->cpu_cpc_ctxlock);
1503 1504 kcpc_stop_hw(ctx);
1504 1505 mutex_exit(&cp->cpu_cpc_ctxlock);
1505 1506 }
1506 1507 mutex_exit(&cpu_lock);
1507 1508 ASSERT(curthread->t_cpc_ctx == NULL);
1508 1509 } else {
1509 1510 int save_spl;
1510 1511
1511 1512 /*
1512 1513 * Thread-bound context; stop _this_ CPU's counters.
1513 1514 */
1514 1515 kpreempt_disable();
1515 1516 save_spl = spl_xcall();
1516 1517 kcpc_unprogram(ctx, B_TRUE);
1517 1518 curthread->t_cpc_ctx = NULL;
1518 1519 splx(save_spl);
1519 1520 kpreempt_enable();
1520 1521 }
1521 1522
1522 1523 /*
1523 1524 * Since we are being called from an exec and we know that
1524 1525 * exec is not permitted via the agent thread, we should clean
1525 1526 * up this thread's CPC state completely, and not leave dangling
1526 1527 * CPC pointers behind.
1527 1528 */
1528 1529 ASSERT(ctx->kc_thread == curthread);
1529 1530 curthread->t_cpc_set = NULL;
1530 1531 }
1531 1532
1532 1533 /*
1533 1534 * Walk through each request in this context's set and free the PCBE's
1534 1535 * configuration if it exists.
1535 1536 */
1536 1537 for (i = 0; i < set->ks_nreqs; i++) {
1537 1538 if (set->ks_req[i].kr_config != NULL)
1538 1539 pcbe_ops->pcbe_free(set->ks_req[i].kr_config);
1539 1540 }
1540 1541
1541 1542 kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
1542 1543 kcpc_ctx_free(ctx);
1543 1544 kcpc_free_set(set);
1544 1545 }
1545 1546
1546 1547 /*
1547 1548 * Free the memory associated with a request set.
1548 1549 */
1549 1550 void
1550 1551 kcpc_free_set(kcpc_set_t *set)
1551 1552 {
1552 1553 int i;
1553 1554 kcpc_request_t *req;
1554 1555
1555 1556 ASSERT(set->ks_req != NULL);
1556 1557
1557 1558 for (i = 0; i < set->ks_nreqs; i++) {
1558 1559 req = &set->ks_req[i];
1559 1560
1560 1561 if (req->kr_nattrs != 0) {
1561 1562 kmem_free(req->kr_attr,
1562 1563 req->kr_nattrs * sizeof (kcpc_attr_t));
1563 1564 }
1564 1565 }
1565 1566
1566 1567 kmem_free(set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs);
1567 1568 cv_destroy(&set->ks_condv);
1568 1569 mutex_destroy(&set->ks_lock);
1569 1570 kmem_free(set, sizeof (kcpc_set_t));
1570 1571 }
1571 1572
1572 1573 /*
1573 1574 * Grab every existing context and mark it as invalid.
1574 1575 */
1575 1576 void
1576 1577 kcpc_invalidate_all(void)
1577 1578 {
1578 1579 kcpc_ctx_t *ctx;
1579 1580 long hash;
1580 1581
1581 1582 for (hash = 0; hash < CPC_HASH_BUCKETS; hash++) {
1582 1583 mutex_enter(&kcpc_ctx_llock[hash]);
1583 1584 for (ctx = kcpc_ctx_list[hash]; ctx; ctx = ctx->kc_next)
1584 1585 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID);
1585 1586 mutex_exit(&kcpc_ctx_llock[hash]);
1586 1587 }
1587 1588 }
1588 1589
1589 1590 /*
1590 1591 * Interface for PCBEs to signal that an existing configuration has suddenly
1591 1592 * become invalid.
1592 1593 */
1593 1594 void
1594 1595 kcpc_invalidate_config(void *token)
1595 1596 {
1596 1597 kcpc_ctx_t *ctx = token;
1597 1598
1598 1599 ASSERT(ctx != NULL);
1599 1600
1600 1601 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID);
1601 1602 }
1602 1603
1603 1604 /*
1604 1605 * Called from lwp_exit() and thread_exit()
1605 1606 */
1606 1607 void
1607 1608 kcpc_passivate(void)
1608 1609 {
1609 1610 kcpc_ctx_t *ctx = curthread->t_cpc_ctx;
1610 1611 kcpc_set_t *set = curthread->t_cpc_set;
1611 1612 int save_spl;
1612 1613
1613 1614 if (set == NULL)
1614 1615 return;
1615 1616
1616 1617 if (ctx == NULL) {
1617 1618 /*
1618 1619 * This thread has a set but no context; it must be a CPU-bound
1619 1620 * set. The hardware will be stopped via kcpc_unbind() when the
1620 1621 * process exits and closes its file descriptors with
1621 1622 * kcpc_close(). Our only job here is to clean up this thread's
1622 1623 * state; the set will be freed with the unbind().
1623 1624 */
1624 1625 (void) kcpc_unbind(set);
1625 1626 /*
1626 1627 * Unbinding a set belonging to the current thread should clear
1627 1628 * its set pointer.
1628 1629 */
1629 1630 ASSERT(curthread->t_cpc_set == NULL);
1630 1631 return;
1631 1632 }
1632 1633
1633 1634 kpreempt_disable();
1634 1635 save_spl = spl_xcall();
1635 1636 curthread->t_cpc_set = NULL;
1636 1637
1637 1638 /*
1638 1639 * This thread/LWP is exiting but context switches will continue to
1639 1640 * happen for a bit as the exit proceeds. Kernel preemption must be
1640 1641 * disabled here to prevent a race between checking or setting the
1641 1642 * INVALID_STOPPED flag here and kcpc_restore() setting the flag during
1642 1643 * a context switch.
1643 1644 */
1644 1645 if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) {
1645 1646 kcpc_unprogram(ctx, B_TRUE);
1646 1647 KCPC_CTX_FLAG_SET(ctx,
1647 1648 KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);
1648 1649 }
1649 1650
1650 1651 /*
1651 1652 * We're cleaning up after this thread; ensure there are no dangling
1652 1653 * CPC pointers left behind. The context and set will be freed by
1653 1654 * freectx().
1654 1655 */
1655 1656 curthread->t_cpc_ctx = NULL;
1656 1657
1657 1658 splx(save_spl);
1658 1659 kpreempt_enable();
1659 1660 }
1660 1661
1661 1662 /*
1662 1663 * Assign the requests in the given set to the PICs in the context.
1663 1664 * Returns 0 if successful, -1 on failure.
1664 1665 */
1665 1666 /*ARGSUSED*/
1666 1667 int
1667 1668 kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx)
1668 1669 {
1669 1670 int i;
1670 1671 int *picnum_save;
1671 1672
1672 1673 ASSERT(set->ks_nreqs <= cpc_ncounters);
1673 1674
1674 1675 /*
1675 1676 * Provide kcpc_tryassign() with scratch space to avoid doing an
1676 1677 * alloc/free with every invocation.
1677 1678 */
1678 1679 picnum_save = kmem_alloc(set->ks_nreqs * sizeof (int), KM_SLEEP);
1679 1680 /*
1680 1681 * kcpc_tryassign() blindly walks through each request in the set,
1681 1682 * seeing if a counter can count its event. If yes, it assigns that
1682 1683 * counter. However, that counter may have been the only capable counter
1683 1684 * for _another_ request's event. The solution is to try every possible
1684 1685 * request first. Note that this does not cover all solutions, as
1685 1686 * that would require all unique orderings of requests, an n^n operation
1686 1687 * which would be unacceptable for architectures with many counters.
1687 1688 */
1688 1689 for (i = 0; i < set->ks_nreqs; i++)
1689 1690 if (kcpc_tryassign(set, i, picnum_save) == 0)
1690 1691 break;
1691 1692
1692 1693 kmem_free(picnum_save, set->ks_nreqs * sizeof (int));
1693 1694 if (i == set->ks_nreqs)
1694 1695 return (-1);
1695 1696 return (0);
1696 1697 }
1697 1698
1698 1699 static int
1699 1700 kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch)
1700 1701 {
1701 1702 int i;
1702 1703 int j;
1703 1704 uint64_t bitmap = 0, resmap = 0;
1704 1705 uint64_t ctrmap;
1705 1706
1706 1707 /*
1707 1708 * We are attempting to assign the reqs to pics, but we may fail. If we
1708 1709 * fail, we need to restore the state of the requests to what it was
1709 1710 * when we found it, as some reqs may have been explicitly assigned to
1710 1711 * a specific PIC beforehand. We do this by snapshotting the assignments
1711 1712 * now and restoring from it later if we fail.
1712 1713 *
1713 1714 * Also we note here which counters have already been claimed by
1714 1715 * requests with explicit counter assignments.
1715 1716 */
1716 1717 for (i = 0; i < set->ks_nreqs; i++) {
1717 1718 scratch[i] = set->ks_req[i].kr_picnum;
1718 1719 if (set->ks_req[i].kr_picnum != -1)
1719 1720 resmap |= (1 << set->ks_req[i].kr_picnum);
1720 1721 }
1721 1722
1722 1723 /*
1723 1724 * Walk through requests assigning them to the first PIC that is
1724 1725 * capable.
1725 1726 */
1726 1727 i = starting_req;
1727 1728 do {
1728 1729 if (set->ks_req[i].kr_picnum != -1) {
1729 1730 ASSERT((bitmap & (1 << set->ks_req[i].kr_picnum)) == 0);
1730 1731 bitmap |= (1 << set->ks_req[i].kr_picnum);
1731 1732 if (++i == set->ks_nreqs)
1732 1733 i = 0;
1733 1734 continue;
1734 1735 }
1735 1736
1736 1737 ctrmap = pcbe_ops->pcbe_event_coverage(set->ks_req[i].kr_event);
1737 1738 for (j = 0; j < cpc_ncounters; j++) {
1738 1739 if (ctrmap & (1 << j) && (bitmap & (1 << j)) == 0 &&
1739 1740 (resmap & (1 << j)) == 0) {
1740 1741 /*
1741 1742 * We can assign this counter because:
1742 1743 *
1743 1744 * 1. It can count the event (ctrmap)
1744 1745 * 2. It hasn't been assigned yet (bitmap)
1745 1746 * 3. It wasn't reserved by a request (resmap)
1746 1747 */
1747 1748 bitmap |= (1 << j);
1748 1749 break;
1749 1750 }
1750 1751 }
1751 1752 if (j == cpc_ncounters) {
1752 1753 for (i = 0; i < set->ks_nreqs; i++)
1753 1754 set->ks_req[i].kr_picnum = scratch[i];
1754 1755 return (-1);
1755 1756 }
1756 1757 set->ks_req[i].kr_picnum = j;
1757 1758
1758 1759 if (++i == set->ks_nreqs)
1759 1760 i = 0;
1760 1761 } while (i != starting_req);
1761 1762
1762 1763 return (0);
1763 1764 }
1764 1765
1765 1766 kcpc_set_t *
1766 1767 kcpc_dup_set(kcpc_set_t *set)
1767 1768 {
1768 1769 kcpc_set_t *new;
1769 1770 int i;
1770 1771 int j;
1771 1772
1772 1773 new = kmem_zalloc(sizeof (*new), KM_SLEEP);
1773 1774 new->ks_state &= ~KCPC_SET_BOUND;
1774 1775 new->ks_flags = set->ks_flags;
1775 1776 new->ks_nreqs = set->ks_nreqs;
1776 1777 new->ks_req = kmem_alloc(set->ks_nreqs * sizeof (kcpc_request_t),
1777 1778 KM_SLEEP);
1778 1779 new->ks_data = NULL;
1779 1780 new->ks_ctx = NULL;
1780 1781
1781 1782 for (i = 0; i < new->ks_nreqs; i++) {
1782 1783 new->ks_req[i].kr_config = NULL;
1783 1784 new->ks_req[i].kr_index = set->ks_req[i].kr_index;
1784 1785 new->ks_req[i].kr_picnum = set->ks_req[i].kr_picnum;
1785 1786 new->ks_req[i].kr_picp = NULL;
1786 1787 new->ks_req[i].kr_data = NULL;
1787 1788 (void) strncpy(new->ks_req[i].kr_event, set->ks_req[i].kr_event,
1788 1789 CPC_MAX_EVENT_LEN);
1789 1790 new->ks_req[i].kr_preset = set->ks_req[i].kr_preset;
1790 1791 new->ks_req[i].kr_flags = set->ks_req[i].kr_flags;
1791 1792 new->ks_req[i].kr_nattrs = set->ks_req[i].kr_nattrs;
1792 1793 new->ks_req[i].kr_attr = kmem_alloc(new->ks_req[i].kr_nattrs *
1793 1794 sizeof (kcpc_attr_t), KM_SLEEP);
1794 1795 for (j = 0; j < new->ks_req[i].kr_nattrs; j++) {
1795 1796 new->ks_req[i].kr_attr[j].ka_val =
1796 1797 set->ks_req[i].kr_attr[j].ka_val;
1797 1798 (void) strncpy(new->ks_req[i].kr_attr[j].ka_name,
1798 1799 set->ks_req[i].kr_attr[j].ka_name,
1799 1800 CPC_MAX_ATTR_LEN);
1800 1801 }
1801 1802 }
1802 1803
1803 1804 return (new);
1804 1805 }
1805 1806
1806 1807 int
1807 1808 kcpc_allow_nonpriv(void *token)
1808 1809 {
1809 1810 return (((kcpc_ctx_t *)token)->kc_flags & KCPC_CTX_NONPRIV);
1810 1811 }
1811 1812
1812 1813 void
1813 1814 kcpc_invalidate(kthread_t *t)
1814 1815 {
1815 1816 kcpc_ctx_t *ctx = t->t_cpc_ctx;
1816 1817
1817 1818 if (ctx != NULL)
1818 1819 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID);
1819 1820 }
1820 1821
1821 1822 /*
1822 1823 * Given a PCBE ID, attempt to load a matching PCBE module. The strings given
1823 1824 * are used to construct PCBE names, starting with the most specific,
1824 1825 * "pcbe.first.second.third.fourth" and ending with the least specific,
1825 1826 * "pcbe.first".
1826 1827 *
1827 1828 * Returns 0 if a PCBE was successfully loaded and -1 upon error.
1828 1829 */
1829 1830 int
1830 1831 kcpc_pcbe_tryload(const char *prefix, uint_t first, uint_t second, uint_t third)
1831 1832 {
1832 1833 uint_t s[3];
1833 1834
1834 1835 s[0] = first;
1835 1836 s[1] = second;
1836 1837 s[2] = third;
1837 1838
1838 1839 return (modload_qualified("pcbe",
1839 1840 "pcbe", prefix, ".", s, 3, NULL) < 0 ? -1 : 0);
1840 1841 }
1841 1842
1842 1843 /*
1843 1844 * Create one or more CPC context for given CPU with specified counter event
1844 1845 * requests
1845 1846 *
1846 1847 * If number of requested counter events is less than or equal number of
1847 1848 * hardware counters on a CPU and can all be assigned to the counters on a CPU
1848 1849 * at the same time, then make one CPC context.
1849 1850 *
1850 1851 * Otherwise, multiple CPC contexts are created to allow multiplexing more
1851 1852 * counter events than existing counters onto the counters by iterating through
1852 1853 * all of the CPC contexts, programming the counters with each CPC context one
1853 1854 * at a time and measuring the resulting counter values. Each of the resulting
1854 1855 * CPC contexts contains some number of requested counter events less than or
1855 1856 * equal the number of counters on a CPU depending on whether all the counter
1856 1857 * events can be programmed on all the counters at the same time or not.
1857 1858 *
1858 1859 * Flags to kmem_{,z}alloc() are passed in as an argument to allow specifying
1859 1860 * whether memory allocation should be non-blocking or not. The code will try
1860 1861 * to allocate *whole* CPC contexts if possible. If there is any memory
1861 1862 * allocation failure during the allocations needed for a given CPC context, it
1862 1863 * will skip allocating that CPC context because it cannot allocate the whole
1863 1864 * thing. Thus, the only time that it will end up allocating none (ie. no CPC
1864 1865 * contexts whatsoever) is when it cannot even allocate *one* whole CPC context
1865 1866 * without a memory allocation failure occurring.
1866 1867 */
1867 1868 int
1868 1869 kcpc_cpu_ctx_create(cpu_t *cp, kcpc_request_list_t *req_list, int kmem_flags,
1869 1870 kcpc_ctx_t ***ctx_ptr_array, size_t *ctx_ptr_array_sz)
1870 1871 {
1871 1872 kcpc_ctx_t **ctx_ptrs;
1872 1873 int nctx;
1873 1874 int nctx_ptrs;
1874 1875 int nreqs;
1875 1876 kcpc_request_t *reqs;
1876 1877
1877 1878 if (cp == NULL || ctx_ptr_array == NULL || ctx_ptr_array_sz == NULL ||
1878 1879 req_list == NULL || req_list->krl_cnt < 1)
1879 1880 return (-1);
1880 1881
1881 1882 /*
1882 1883 * Allocate number of sets assuming that each set contains one and only
1883 1884 * one counter event request for each counter on a CPU
1884 1885 */
1885 1886 nreqs = req_list->krl_cnt;
1886 1887 nctx_ptrs = (nreqs + cpc_ncounters - 1) / cpc_ncounters;
1887 1888 ctx_ptrs = kmem_zalloc(nctx_ptrs * sizeof (kcpc_ctx_t *), kmem_flags);
1888 1889 if (ctx_ptrs == NULL)
1889 1890 return (-2);
1890 1891
1891 1892 /*
1892 1893 * Fill in sets of requests
1893 1894 */
1894 1895 nctx = 0;
1895 1896 reqs = req_list->krl_list;
1896 1897 while (nreqs > 0) {
1897 1898 kcpc_ctx_t *ctx;
1898 1899 kcpc_set_t *set;
1899 1900 int subcode;
1900 1901
1901 1902 /*
1902 1903 * Allocate CPC context and set for requested counter events
1903 1904 */
1904 1905 ctx = kcpc_ctx_alloc(kmem_flags);
1905 1906 set = kcpc_set_create(reqs, nreqs, 0, kmem_flags);
1906 1907 if (set == NULL) {
1907 1908 kcpc_ctx_free(ctx);
1908 1909 break;
1909 1910 }
1910 1911
1911 1912 /*
1912 1913 * Determine assignment of requested counter events to specific
1913 1914 * counters
1914 1915 */
1915 1916 if (kcpc_assign_reqs(set, ctx) != 0) {
1916 1917 /*
1917 1918 * May not be able to assign requested counter events
1918 1919 * to all counters since all counters may not be able
1919 1920 * to do all events, so only do one counter event in
1920 1921 * set of counter requests when this happens since at
1921 1922 * least one of the counters must be able to do the
1922 1923 * event.
1923 1924 */
1924 1925 kcpc_free_set(set);
1925 1926 set = kcpc_set_create(reqs, 1, 0, kmem_flags);
1926 1927 if (set == NULL) {
1927 1928 kcpc_ctx_free(ctx);
1928 1929 break;
1929 1930 }
1930 1931 if (kcpc_assign_reqs(set, ctx) != 0) {
1931 1932 #ifdef DEBUG
1932 1933 cmn_err(CE_NOTE, "!kcpc_cpu_ctx_create: can't "
1933 1934 "assign counter event %s!\n",
1934 1935 set->ks_req->kr_event);
1935 1936 #endif
1936 1937 kcpc_free_set(set);
1937 1938 kcpc_ctx_free(ctx);
1938 1939 reqs++;
1939 1940 nreqs--;
1940 1941 continue;
1941 1942 }
1942 1943 }
1943 1944
1944 1945 /*
1945 1946 * Allocate memory needed to hold requested counter event data
1946 1947 */
1947 1948 set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t),
1948 1949 kmem_flags);
1949 1950 if (set->ks_data == NULL) {
1950 1951 kcpc_free_set(set);
1951 1952 kcpc_ctx_free(ctx);
1952 1953 break;
1953 1954 }
1954 1955
1955 1956 /*
1956 1957 * Configure requested counter events
1957 1958 */
1958 1959 if (kcpc_configure_reqs(ctx, set, &subcode) != 0) {
1959 1960 #ifdef DEBUG
1960 1961 cmn_err(CE_NOTE,
1961 1962 "!kcpc_cpu_ctx_create: can't configure "
1962 1963 "set of counter event requests!\n");
1963 1964 #endif
1964 1965 reqs += set->ks_nreqs;
1965 1966 nreqs -= set->ks_nreqs;
1966 1967 kmem_free(set->ks_data,
1967 1968 set->ks_nreqs * sizeof (uint64_t));
1968 1969 kcpc_free_set(set);
1969 1970 kcpc_ctx_free(ctx);
1970 1971 continue;
1971 1972 }
1972 1973
1973 1974 /*
1974 1975 * Point set of counter event requests at this context and fill
1975 1976 * in CPC context
1976 1977 */
1977 1978 set->ks_ctx = ctx;
1978 1979 ctx->kc_set = set;
1979 1980 ctx->kc_cpuid = cp->cpu_id;
1980 1981 ctx->kc_thread = curthread;
1981 1982
1982 1983 ctx_ptrs[nctx] = ctx;
1983 1984
1984 1985 /*
1985 1986 * Update requests and how many are left to be assigned to sets
1986 1987 */
1987 1988 reqs += set->ks_nreqs;
1988 1989 nreqs -= set->ks_nreqs;
1989 1990
1990 1991 /*
1991 1992 * Increment number of CPC contexts and allocate bigger array
1992 1993 * for context pointers as needed
1993 1994 */
1994 1995 nctx++;
1995 1996 if (nctx >= nctx_ptrs) {
1996 1997 kcpc_ctx_t **new;
1997 1998 int new_cnt;
1998 1999
1999 2000 /*
2000 2001 * Allocate more CPC contexts based on how many
2001 2002 * contexts allocated so far and how many counter
2002 2003 * requests left to assign
2003 2004 */
2004 2005 new_cnt = nctx_ptrs +
2005 2006 ((nreqs + cpc_ncounters - 1) / cpc_ncounters);
2006 2007 new = kmem_zalloc(new_cnt * sizeof (kcpc_ctx_t *),
2007 2008 kmem_flags);
2008 2009 if (new == NULL)
2009 2010 break;
2010 2011
2011 2012 /*
2012 2013 * Copy contents of old sets into new ones
2013 2014 */
2014 2015 bcopy(ctx_ptrs, new,
2015 2016 nctx_ptrs * sizeof (kcpc_ctx_t *));
2016 2017
2017 2018 /*
2018 2019 * Free old array of context pointers and use newly
2019 2020 * allocated one instead now
2020 2021 */
2021 2022 kmem_free(ctx_ptrs, nctx_ptrs * sizeof (kcpc_ctx_t *));
2022 2023 ctx_ptrs = new;
2023 2024 nctx_ptrs = new_cnt;
2024 2025 }
2025 2026 }
2026 2027
2027 2028 /*
2028 2029 * Return NULL if no CPC contexts filled in
2029 2030 */
2030 2031 if (nctx == 0) {
2031 2032 kmem_free(ctx_ptrs, nctx_ptrs * sizeof (kcpc_ctx_t *));
2032 2033 *ctx_ptr_array = NULL;
2033 2034 *ctx_ptr_array_sz = 0;
2034 2035 return (-2);
2035 2036 }
2036 2037
2037 2038 *ctx_ptr_array = ctx_ptrs;
2038 2039 *ctx_ptr_array_sz = nctx_ptrs * sizeof (kcpc_ctx_t *);
2039 2040 return (nctx);
2040 2041 }
2041 2042
2042 2043 /*
2043 2044 * Return whether PCBE supports given counter event
2044 2045 */
2045 2046 boolean_t
2046 2047 kcpc_event_supported(char *event)
2047 2048 {
2048 2049 if (pcbe_ops == NULL || pcbe_ops->pcbe_event_coverage(event) == 0)
2049 2050 return (B_FALSE);
2050 2051
2051 2052 return (B_TRUE);
2052 2053 }
2053 2054
2054 2055 /*
2055 2056 * Program counters on current CPU with given CPC context
2056 2057 *
2057 2058 * If kernel is interposing on counters to measure hardware capacity and
2058 2059 * utilization, then unprogram counters for kernel *before* programming them
2059 2060 * with specified CPC context.
2060 2061 *
2061 2062 * kcpc_{program,unprogram}() may be called either directly by a thread running
2062 2063 * on the target CPU or from a cross-call from another CPU. To protect
2063 2064 * programming and unprogramming from being interrupted by cross-calls, callers
2064 2065 * who execute kcpc_{program,unprogram} should raise PIL to the level used by
2065 2066 * cross-calls.
2066 2067 */
2067 2068 void
2068 2069 kcpc_program(kcpc_ctx_t *ctx, boolean_t for_thread, boolean_t cu_interpose)
2069 2070 {
2070 2071 int error;
2071 2072
2072 2073 ASSERT(IS_HIPIL());
2073 2074
2074 2075 /*
2075 2076 * CPC context shouldn't be NULL, its CPU field should specify current
2076 2077 * CPU or be -1 to specify any CPU when the context is bound to a
2077 2078 * thread, and preemption should be disabled
2078 2079 */
2079 2080 ASSERT(ctx != NULL && (ctx->kc_cpuid == CPU->cpu_id ||
2080 2081 ctx->kc_cpuid == -1) && curthread->t_preempt > 0);
2081 2082 if (ctx == NULL || (ctx->kc_cpuid != CPU->cpu_id &&
2082 2083 ctx->kc_cpuid != -1) || curthread->t_preempt < 1)
2083 2084 return;
2084 2085
2085 2086 /*
2086 2087 * Unprogram counters for kernel measuring hardware capacity and
2087 2088 * utilization
2088 2089 */
2089 2090 if (cu_interpose == B_TRUE) {
2090 2091 cu_cpc_unprogram(CPU, &error);
2091 2092 } else {
2092 2093 kcpc_set_t *set = ctx->kc_set;
2093 2094 int i;
2094 2095
2095 2096 ASSERT(set != NULL);
2096 2097
2097 2098 /*
2098 2099 * Since cu_interpose is false, we are programming CU context.
2099 2100 * In general, PCBE can continue from the state saved in the
2100 2101 * set, but it is not very reliable, so we start again from the
2101 2102 * preset value.
2102 2103 */
2103 2104 for (i = 0; i < set->ks_nreqs; i++) {
2104 2105 /*
2105 2106 * Reset the virtual counter value to the preset value.
2106 2107 */
2107 2108 *(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset;
2108 2109
2109 2110 /*
2110 2111 * Reset PCBE to the preset value.
2111 2112 */
2112 2113 pcbe_ops->pcbe_configure(0, NULL,
2113 2114 set->ks_req[i].kr_preset,
2114 2115 0, 0, NULL, &set->ks_req[i].kr_config, NULL);
2115 2116 }
2116 2117 }
2117 2118
2118 2119 /*
2119 2120 * Program counters with specified CPC context
2120 2121 */
2121 2122 ctx->kc_rawtick = KCPC_GET_TICK();
2122 2123 pcbe_ops->pcbe_program(ctx);
2123 2124
2124 2125 /*
2125 2126 * Denote that counters programmed for thread or CPU CPC context
2126 2127 * differently
2127 2128 */
2128 2129 if (for_thread == B_TRUE)
2129 2130 KCPC_CTX_FLAG_CLR(ctx, KCPC_CTX_FREEZE);
2130 2131 else
2131 2132 CPU->cpu_cpc_ctx = ctx;
2132 2133 }
2133 2134
2134 2135 /*
2135 2136 * Unprogram counters with given CPC context on current CPU
2136 2137 *
2137 2138 * If kernel is interposing on counters to measure hardware capacity and
2138 2139 * utilization, then program counters for the kernel capacity and utilization
2139 2140 * *after* unprogramming them for given CPC context.
2140 2141 *
2141 2142 * See the comment for kcpc_program regarding the synchronization with
2142 2143 * cross-calls.
2143 2144 */
2144 2145 void
2145 2146 kcpc_unprogram(kcpc_ctx_t *ctx, boolean_t cu_interpose)
2146 2147 {
2147 2148 int error;
2148 2149
2149 2150 ASSERT(IS_HIPIL());
2150 2151
2151 2152 /*
2152 2153 * CPC context shouldn't be NULL, its CPU field should specify current
2153 2154 * CPU or be -1 to specify any CPU when the context is bound to a
2154 2155 * thread, and preemption should be disabled
2155 2156 */
2156 2157 ASSERT(ctx != NULL && (ctx->kc_cpuid == CPU->cpu_id ||
2157 2158 ctx->kc_cpuid == -1) && curthread->t_preempt > 0);
2158 2159
2159 2160 if (ctx == NULL || (ctx->kc_cpuid != CPU->cpu_id &&
2160 2161 ctx->kc_cpuid != -1) || curthread->t_preempt < 1 ||
2161 2162 (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) != 0) {
2162 2163 return;
2163 2164 }
2164 2165
2165 2166 /*
2166 2167 * Specified CPC context to be unprogrammed should be bound to current
2167 2168 * CPU or thread
2168 2169 */
2169 2170 ASSERT(CPU->cpu_cpc_ctx == ctx || curthread->t_cpc_ctx == ctx);
2170 2171
2171 2172 /*
2172 2173 * Stop counters
2173 2174 */
2174 2175 pcbe_ops->pcbe_allstop();
2175 2176 KCPC_CTX_FLAG_SET(ctx, KCPC_CTX_INVALID_STOPPED);
2176 2177
2177 2178 /*
2178 2179 * Allow kernel to interpose on counters and program them for its own
2179 2180 * use to measure hardware capacity and utilization if cu_interpose
2180 2181 * argument is true
2181 2182 */
2182 2183 if (cu_interpose == B_TRUE)
2183 2184 cu_cpc_program(CPU, &error);
2184 2185 }
2185 2186
2186 2187 /*
2187 2188 * Read CPU Performance Counter (CPC) on current CPU and call specified update
2188 2189 * routine with data for each counter event currently programmed on CPU
2189 2190 */
2190 2191 int
2191 2192 kcpc_read(kcpc_update_func_t update_func)
2192 2193 {
2193 2194 kcpc_ctx_t *ctx;
2194 2195 int i;
2195 2196 kcpc_request_t *req;
2196 2197 int retval;
2197 2198 kcpc_set_t *set;
2198 2199
2199 2200 ASSERT(IS_HIPIL());
2200 2201
2201 2202 /*
2202 2203 * Can't grab locks or block because may be called inside dispatcher
2203 2204 */
2204 2205 kpreempt_disable();
2205 2206
2206 2207 ctx = CPU->cpu_cpc_ctx;
2207 2208 if (ctx == NULL) {
2208 2209 kpreempt_enable();
2209 2210 return (0);
2210 2211 }
2211 2212
2212 2213 /*
2213 2214 * Read counter data from current CPU
2214 2215 */
2215 2216 pcbe_ops->pcbe_sample(ctx);
2216 2217
2217 2218 set = ctx->kc_set;
2218 2219 if (set == NULL || set->ks_req == NULL) {
2219 2220 kpreempt_enable();
2220 2221 return (0);
2221 2222 }
2222 2223
2223 2224 /*
2224 2225 * Call update function with preset pointer and data for each CPC event
2225 2226 * request currently programmed on current CPU
2226 2227 */
2227 2228 req = set->ks_req;
2228 2229 retval = 0;
2229 2230 for (i = 0; i < set->ks_nreqs; i++) {
2230 2231 int ret;
2231 2232
2232 2233 if (req[i].kr_data == NULL)
2233 2234 break;
2234 2235
2235 2236 ret = update_func(req[i].kr_ptr, *req[i].kr_data);
2236 2237 if (ret < 0)
2237 2238 retval = ret;
2238 2239 }
2239 2240
2240 2241 kpreempt_enable();
2241 2242
2242 2243 return (retval);
2243 2244 }
2244 2245
2245 2246 /*
2246 2247 * Initialize list of counter event requests
2247 2248 */
2248 2249 kcpc_request_list_t *
2249 2250 kcpc_reqs_init(int nreqs, int kmem_flags)
2250 2251 {
2251 2252 kcpc_request_list_t *req_list;
2252 2253 kcpc_request_t *reqs;
2253 2254
2254 2255 if (nreqs < 1)
2255 2256 return (NULL);
2256 2257
2257 2258 req_list = kmem_zalloc(sizeof (kcpc_request_list_t), kmem_flags);
2258 2259 if (req_list == NULL)
2259 2260 return (NULL);
2260 2261
2261 2262 reqs = kmem_zalloc(nreqs * sizeof (kcpc_request_t), kmem_flags);
2262 2263 if (reqs == NULL) {
2263 2264 kmem_free(req_list, sizeof (kcpc_request_list_t));
2264 2265 return (NULL);
2265 2266 }
2266 2267
2267 2268 req_list->krl_list = reqs;
2268 2269 req_list->krl_cnt = 0;
2269 2270 req_list->krl_max = nreqs;
2270 2271 return (req_list);
2271 2272 }
2272 2273
2273 2274
2274 2275 /*
2275 2276 * Add counter event request to given list of counter event requests
2276 2277 */
2277 2278 int
2278 2279 kcpc_reqs_add(kcpc_request_list_t *req_list, char *event, uint64_t preset,
2279 2280 uint_t flags, uint_t nattrs, kcpc_attr_t *attr, void *ptr, int kmem_flags)
2280 2281 {
2281 2282 kcpc_request_t *req;
2282 2283
2283 2284 if (req_list == NULL || req_list->krl_list == NULL)
2284 2285 return (-1);
2285 2286
2286 2287 ASSERT(req_list->krl_max != 0);
2287 2288
2288 2289 /*
2289 2290 * Allocate more space (if needed)
2290 2291 */
2291 2292 if (req_list->krl_cnt > req_list->krl_max) {
2292 2293 kcpc_request_t *new;
2293 2294 kcpc_request_t *old;
2294 2295
2295 2296 old = req_list->krl_list;
2296 2297 new = kmem_zalloc((req_list->krl_max +
2297 2298 cpc_ncounters) * sizeof (kcpc_request_t), kmem_flags);
2298 2299 if (new == NULL)
2299 2300 return (-2);
2300 2301
2301 2302 req_list->krl_list = new;
2302 2303 bcopy(old, req_list->krl_list,
2303 2304 req_list->krl_cnt * sizeof (kcpc_request_t));
2304 2305 kmem_free(old, req_list->krl_max * sizeof (kcpc_request_t));
2305 2306 req_list->krl_cnt = 0;
2306 2307 req_list->krl_max += cpc_ncounters;
2307 2308 }
2308 2309
2309 2310 /*
2310 2311 * Fill in request as much as possible now, but some fields will need
2311 2312 * to be set when request is assigned to a set.
2312 2313 */
2313 2314 req = &req_list->krl_list[req_list->krl_cnt];
2314 2315 req->kr_config = NULL;
2315 2316 req->kr_picnum = -1; /* have CPC pick this */
2316 2317 req->kr_index = -1; /* set when assigning request to set */
2317 2318 req->kr_data = NULL; /* set when configuring request */
2318 2319 (void) strcpy(req->kr_event, event);
2319 2320 req->kr_preset = preset;
2320 2321 req->kr_flags = flags;
2321 2322 req->kr_nattrs = nattrs;
2322 2323 req->kr_attr = attr;
2323 2324 /*
2324 2325 * Keep pointer given by caller to give to update function when this
2325 2326 * counter event is sampled/read
2326 2327 */
2327 2328 req->kr_ptr = ptr;
2328 2329
2329 2330 req_list->krl_cnt++;
2330 2331
2331 2332 return (0);
2332 2333 }
2333 2334
2334 2335 /*
2335 2336 * Reset list of CPC event requests so its space can be used for another set
2336 2337 * of requests
2337 2338 */
2338 2339 int
2339 2340 kcpc_reqs_reset(kcpc_request_list_t *req_list)
2340 2341 {
2341 2342 /*
2342 2343 * Return when pointer to request list structure or request is NULL or
2343 2344 * when max requests is less than or equal to 0
2344 2345 */
2345 2346 if (req_list == NULL || req_list->krl_list == NULL ||
2346 2347 req_list->krl_max <= 0)
2347 2348 return (-1);
2348 2349
2349 2350 /*
2350 2351 * Zero out requests and number of requests used
2351 2352 */
2352 2353 bzero(req_list->krl_list, req_list->krl_max * sizeof (kcpc_request_t));
2353 2354 req_list->krl_cnt = 0;
2354 2355 return (0);
2355 2356 }
2356 2357
2357 2358 /*
2358 2359 * Free given list of counter event requests
2359 2360 */
2360 2361 int
2361 2362 kcpc_reqs_fini(kcpc_request_list_t *req_list)
2362 2363 {
2363 2364 kmem_free(req_list->krl_list,
2364 2365 req_list->krl_max * sizeof (kcpc_request_t));
2365 2366 kmem_free(req_list, sizeof (kcpc_request_list_t));
2366 2367 return (0);
2367 2368 }
2368 2369
2369 2370 /*
2370 2371 * Create set of given counter event requests
2371 2372 */
2372 2373 static kcpc_set_t *
2373 2374 kcpc_set_create(kcpc_request_t *reqs, int nreqs, int set_flags, int kmem_flags)
2374 2375 {
2375 2376 int i;
2376 2377 kcpc_set_t *set;
2377 2378
2378 2379 /*
2379 2380 * Allocate set and assign number of requests in set and flags
2380 2381 */
2381 2382 set = kmem_zalloc(sizeof (kcpc_set_t), kmem_flags);
2382 2383 if (set == NULL)
2383 2384 return (NULL);
2384 2385
2385 2386 if (nreqs < cpc_ncounters)
2386 2387 set->ks_nreqs = nreqs;
2387 2388 else
2388 2389 set->ks_nreqs = cpc_ncounters;
2389 2390
2390 2391 set->ks_flags = set_flags;
2391 2392
2392 2393 /*
2393 2394 * Allocate requests needed, copy requests into set, and set index into
2394 2395 * data for each request (which may change when we assign requested
2395 2396 * counter events to counters)
2396 2397 */
2397 2398 set->ks_req = (kcpc_request_t *)kmem_zalloc(sizeof (kcpc_request_t) *
2398 2399 set->ks_nreqs, kmem_flags);
2399 2400 if (set->ks_req == NULL) {
2400 2401 kmem_free(set, sizeof (kcpc_set_t));
2401 2402 return (NULL);
2402 2403 }
2403 2404
2404 2405 bcopy(reqs, set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs);
2405 2406
2406 2407 for (i = 0; i < set->ks_nreqs; i++)
2407 2408 set->ks_req[i].kr_index = i;
2408 2409
2409 2410 return (set);
2410 2411 }
2411 2412
2412 2413
2413 2414 /*
2414 2415 * Stop counters on current CPU.
2415 2416 *
2416 2417 * If preserve_context is true, the caller is interested in the CPU's CPC
2417 2418 * context and wants it to be preserved.
2418 2419 *
2419 2420 * If preserve_context is false, the caller does not need the CPU's CPC context
2420 2421 * to be preserved, so it is set to NULL.
2421 2422 */
2422 2423 static void
2423 2424 kcpc_cpustop_func(uintptr_t arg1, uintptr_t arg2 __unused)
2424 2425 {
2425 2426 boolean_t preserve_context;
2426 2427 kpreempt_disable();
2427 2428
2428 2429 preserve_context = (boolean_t)arg1;
2429 2430 /*
2430 2431 * Someone already stopped this context before us, so there is nothing
2431 2432 * to do.
2432 2433 */
2433 2434 if (CPU->cpu_cpc_ctx == NULL) {
2434 2435 kpreempt_enable();
2435 2436 return;
2436 2437 }
2437 2438
2438 2439 kcpc_unprogram(CPU->cpu_cpc_ctx, B_TRUE);
2439 2440 /*
2440 2441 * If CU does not use counters, then clear the CPU's CPC context
2441 2442 * If the caller requested to preserve context it should disable CU
2442 2443 * first, so there should be no CU context now.
2443 2444 */
2444 2445 ASSERT(!preserve_context || !CU_CPC_ON(CPU));
2445 2446 if (!preserve_context && CPU->cpu_cpc_ctx != NULL && !CU_CPC_ON(CPU))
2446 2447 CPU->cpu_cpc_ctx = NULL;
2447 2448
2448 2449 kpreempt_enable();
2449 2450 }
2450 2451
2451 2452 /*
2452 2453 * Stop counters on given CPU and set its CPC context to NULL unless
2453 2454 * preserve_context is true.
2454 2455 */
2455 2456 void
2456 2457 kcpc_cpu_stop(cpu_t *cp, boolean_t preserve_context)
2457 2458 {
2458 2459 cpu_call(cp, kcpc_cpustop_func, preserve_context, 0);
2459 2460 }
2460 2461
2461 2462 /*
2462 2463 * Program the context on the current CPU
2463 2464 */
2464 2465 static void
2465 2466 kcpc_remoteprogram_func(uintptr_t arg1, uintptr_t arg2)
2466 2467 {
2467 2468 kcpc_ctx_t *ctx = (kcpc_ctx_t *)arg1;
2468 2469 boolean_t for_thread = (boolean_t)arg2;
2469 2470
2470 2471 ASSERT(ctx != NULL);
2471 2472
2472 2473 kpreempt_disable();
2473 2474 kcpc_program(ctx, for_thread, B_TRUE);
2474 2475 kpreempt_enable();
2475 2476 }
2476 2477
2477 2478 /*
2478 2479 * Program counters on given CPU
2479 2480 */
2480 2481 void
2481 2482 kcpc_cpu_program(cpu_t *cp, kcpc_ctx_t *ctx)
2482 2483 {
2483 2484 cpu_call(cp, kcpc_remoteprogram_func, (uintptr_t)ctx,
2484 2485 (uintptr_t)B_FALSE);
2485 2486 }
2486 2487
2487 2488 char *
2488 2489 kcpc_list_attrs(void)
2489 2490 {
2490 2491 ASSERT(pcbe_ops != NULL);
2491 2492
2492 2493 return (pcbe_ops->pcbe_list_attrs());
2493 2494 }
2494 2495
2495 2496 char *
2496 2497 kcpc_list_events(uint_t pic)
2497 2498 {
2498 2499 ASSERT(pcbe_ops != NULL);
2499 2500
2500 2501 return (pcbe_ops->pcbe_list_events(pic));
2501 2502 }
2502 2503
2503 2504 uint_t
2504 2505 kcpc_pcbe_capabilities(void)
2505 2506 {
2506 2507 ASSERT(pcbe_ops != NULL);
2507 2508
2508 2509 return (pcbe_ops->pcbe_caps);
2509 2510 }
2510 2511
2511 2512 int
2512 2513 kcpc_pcbe_loaded(void)
2513 2514 {
2514 2515 return (pcbe_ops == NULL ? -1 : 0);
2515 2516 }
|
↓ open down ↓ |
1078 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX