1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
27 * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
28 * Copyright 2016 Joyent, Inc.
29 */
30
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/disp.h>
35 #include <sys/var.h>
36 #include <sys/cmn_err.h>
37 #include <sys/debug.h>
38 #include <sys/x86_archext.h>
39 #include <sys/archsystm.h>
40 #include <sys/cpuvar.h>
41 #include <sys/psm_defs.h>
42 #include <sys/clock.h>
43 #include <sys/atomic.h>
44 #include <sys/lockstat.h>
45 #include <sys/smp_impldefs.h>
46 #include <sys/dtrace.h>
47 #include <sys/time.h>
48 #include <sys/panic.h>
49 #include <sys/cpu.h>
50 #include <sys/sdt.h>
51 #include <sys/comm_page.h>
52
53 /*
54 * Using the Pentium's TSC register for gethrtime()
55 * ------------------------------------------------
56 *
57 * The Pentium family, like many chip architectures, has a high-resolution
58 * timestamp counter ("TSC") which increments once per CPU cycle. The contents
59 * of the timestamp counter are read with the RDTSC instruction.
60 *
61 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
62 * must be translated into nanoseconds in order to implement gethrtime().
63 * We avoid inducing floating point operations in this conversion by
64 * implementing the same nsec_scale algorithm as that found in the sun4u
65 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
66 * a detailed description of the algorithm; the comment is not reproduced
67 * here. This implementation differs only in its value for NSEC_SHIFT:
68 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
69 * 60 MHz Pentiums.
70 *
71 * While TSC and %tick are both cycle counting registers, TSC's functionality
72 * falls short in several critical ways:
73 *
74 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in
75 * practice they often _are_ in sync, this isn't guaranteed by the
76 * architecture.
77 *
78 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture
79 * only supports writing the low 32-bits of TSC, making it impractical
80 * to rewrite.
81 *
82 * (c) The architecture doesn't have the capacity to interrupt based on
83 * arbitrary values of TSC; there is no TICK_CMPR equivalent.
84 *
85 * Together, (a) and (b) imply that software must track the skew between
86 * TSCs and account for it (it is assumed that while there may exist skew,
87 * there does not exist drift). To determine the skew between CPUs, we
88 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
89 * the online operation calls tsc_sync_master().
90 *
91 * In the absence of time-of-day clock adjustments, gethrtime() must stay in
92 * sync with gettimeofday(). This is problematic; given (c), the software
93 * cannot drive its time-of-day source from TSC, and yet they must somehow be
94 * kept in sync. We implement this by having a routine, tsc_tick(), which
95 * is called once per second from the interrupt which drives time-of-day.
96 *
97 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
98 * atomically with nsec_scale under CLOCK_LOCK. This assures that time
99 * monotonically increases.
100 */
101
102 #define NSEC_SHIFT 5
103
104 static uint_t nsec_unscale;
105
106 /*
107 * These two variables used to be grouped together inside of a structure that
108 * lived on a single cache line. A regression (bug ID 4623398) caused the
109 * compiler to emit code that "optimized" away the while-loops below. The
110 * result was that no synchronization between the onlining and onlined CPUs
111 * took place.
112 */
113 static volatile int tsc_ready;
114 static volatile int tsc_sync_go;
115
116 /*
117 * Used as indices into the tsc_sync_snaps[] array.
118 */
119 #define TSC_MASTER 0
120 #define TSC_SLAVE 1
121
122 /*
123 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
124 */
125 #define TSC_SYNC_STOP 1
126 #define TSC_SYNC_GO 2
127 #define TSC_SYNC_DONE 3
128 #define SYNC_ITERATIONS 10
129
130 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \
131 unsigned int *_l = (unsigned int *)&(tsc); \
132 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \
133 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
134 }
135
136 #define TSC_CONVERT(tsc, hrt, scale) { \
137 unsigned int *_l = (unsigned int *)&(tsc); \
138 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \
139 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
140 }
141
142 int tsc_master_slave_sync_needed = 1;
143
144 typedef struct tsc_sync {
145 volatile hrtime_t master_tsc, slave_tsc;
146 } tsc_sync_t;
147 static tsc_sync_t *tscp;
148
149 static hrtime_t tsc_last_jumped = 0;
150 static int tsc_jumped = 0;
151 static uint32_t tsc_wayback = 0;
152 /*
153 * The cap of 1 second was chosen since it is the frequency at which the
154 * tsc_tick() function runs which means that when gethrtime() is called it
155 * should never be more than 1 second since tsc_last was updated.
156 */
157 static hrtime_t tsc_resume_cap_ns = NANOSEC; /* 1s */
158
159 static hrtime_t shadow_tsc_hrtime_base;
160 static hrtime_t shadow_tsc_last;
161 static uint_t shadow_nsec_scale;
162 static uint32_t shadow_hres_lock;
163 int get_tsc_ready();
164
165 static inline
166 hrtime_t tsc_protect(hrtime_t a) {
167 if (a > tsc_resume_cap) {
168 atomic_inc_32(&tsc_wayback);
169 DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last,
170 uint32_t, tsc_wayback);
171 return (tsc_resume_cap);
172 }
173 return (a);
174 }
175
176 hrtime_t
177 tsc_gethrtime(void)
178 {
179 uint32_t old_hres_lock;
180 hrtime_t tsc, hrt;
181
182 do {
183 old_hres_lock = hres_lock;
184
185 if ((tsc = tsc_read()) >= tsc_last) {
186 /*
187 * It would seem to be obvious that this is true
188 * (that is, the past is less than the present),
189 * but it isn't true in the presence of suspend/resume
190 * cycles. If we manage to call gethrtime()
191 * after a resume, but before the first call to
192 * tsc_tick(), we will see the jump. In this case,
193 * we will simply use the value in TSC as the delta.
194 */
195 tsc -= tsc_last;
196 } else if (tsc >= tsc_last - 2*tsc_max_delta) {
197 /*
198 * There is a chance that tsc_tick() has just run on
199 * another CPU, and we have drifted just enough so that
200 * we appear behind tsc_last. In this case, force the
201 * delta to be zero.
202 */
203 tsc = 0;
204 } else {
205 /*
206 * If we reach this else clause we assume that we have
207 * gone through a suspend/resume cycle and use the
208 * current tsc value as the delta.
209 *
210 * In rare cases we can reach this else clause due to
211 * a lack of monotonicity in the TSC value. In such
212 * cases using the current TSC value as the delta would
213 * cause us to return a value ~2x of what it should
214 * be. To protect against these cases we cap the
215 * suspend/resume delta at tsc_resume_cap.
216 */
217 tsc = tsc_protect(tsc);
218 }
219
220 hrt = tsc_hrtime_base;
221
222 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
223 } while ((old_hres_lock & ~1) != hres_lock);
224
225 return (hrt);
226 }
227
228 hrtime_t
229 tsc_gethrtime_delta(void)
230 {
231 uint32_t old_hres_lock;
232 hrtime_t tsc, hrt;
233 ulong_t flags;
234
235 do {
236 old_hres_lock = hres_lock;
237
238 /*
239 * We need to disable interrupts here to assure that we
240 * don't migrate between the call to tsc_read() and
241 * adding the CPU's TSC tick delta. Note that disabling
242 * and reenabling preemption is forbidden here because
243 * we may be in the middle of a fast trap. In the amd64
244 * kernel we cannot tolerate preemption during a fast
245 * trap. See _update_sregs().
246 */
247
248 flags = clear_int_flag();
249 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
250 restore_int_flag(flags);
251
252 /* See comments in tsc_gethrtime() above */
253
254 if (tsc >= tsc_last) {
255 tsc -= tsc_last;
256 } else if (tsc >= tsc_last - 2 * tsc_max_delta) {
257 tsc = 0;
258 } else {
259 tsc = tsc_protect(tsc);
260 }
261
262 hrt = tsc_hrtime_base;
263
264 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
265 } while ((old_hres_lock & ~1) != hres_lock);
266
267 return (hrt);
268 }
269
270 hrtime_t
271 tsc_gethrtime_tick_delta(void)
272 {
273 hrtime_t hrt;
274 ulong_t flags;
275
276 flags = clear_int_flag();
277 hrt = tsc_sync_tick_delta[CPU->cpu_id];
278 restore_int_flag(flags);
279
280 return (hrt);
281 }
282
283 /*
284 * This is similar to the above, but it cannot actually spin on hres_lock.
285 * As a result, it caches all of the variables it needs; if the variables
286 * don't change, it's done.
287 */
288 hrtime_t
289 dtrace_gethrtime(void)
290 {
291 uint32_t old_hres_lock;
292 hrtime_t tsc, hrt;
293 ulong_t flags;
294
295 do {
296 old_hres_lock = hres_lock;
297
298 /*
299 * Interrupts are disabled to ensure that the thread isn't
300 * migrated between the tsc_read() and adding the CPU's
301 * TSC tick delta.
302 */
303 flags = clear_int_flag();
304
305 tsc = tsc_read();
306
307 if (gethrtimef == tsc_gethrtime_delta)
308 tsc += tsc_sync_tick_delta[CPU->cpu_id];
309
310 restore_int_flag(flags);
311
312 /*
313 * See the comments in tsc_gethrtime(), above.
314 */
315 if (tsc >= tsc_last)
316 tsc -= tsc_last;
317 else if (tsc >= tsc_last - 2*tsc_max_delta)
318 tsc = 0;
319 else
320 tsc = tsc_protect(tsc);
321
322 hrt = tsc_hrtime_base;
323
324 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
325
326 if ((old_hres_lock & ~1) == hres_lock)
327 break;
328
329 /*
330 * If we're here, the clock lock is locked -- or it has been
331 * unlocked and locked since we looked. This may be due to
332 * tsc_tick() running on another CPU -- or it may be because
333 * some code path has ended up in dtrace_probe() with
334 * CLOCK_LOCK held. We'll try to determine that we're in
335 * the former case by taking another lap if the lock has
336 * changed since when we first looked at it.
337 */
338 if (old_hres_lock != hres_lock)
339 continue;
340
341 /*
342 * So the lock was and is locked. We'll use the old data
343 * instead.
344 */
345 old_hres_lock = shadow_hres_lock;
346
347 /*
348 * Again, disable interrupts to ensure that the thread
349 * isn't migrated between the tsc_read() and adding
350 * the CPU's TSC tick delta.
351 */
352 flags = clear_int_flag();
353
354 tsc = tsc_read();
355
356 if (gethrtimef == tsc_gethrtime_delta)
357 tsc += tsc_sync_tick_delta[CPU->cpu_id];
358
359 restore_int_flag(flags);
360
361 /*
362 * See the comments in tsc_gethrtime(), above.
363 */
364 if (tsc >= shadow_tsc_last)
365 tsc -= shadow_tsc_last;
366 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
367 tsc = 0;
368 else
369 tsc = tsc_protect(tsc);
370
371 hrt = shadow_tsc_hrtime_base;
372
373 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
374 } while ((old_hres_lock & ~1) != shadow_hres_lock);
375
376 return (hrt);
377 }
378
379 hrtime_t
380 tsc_gethrtimeunscaled(void)
381 {
382 uint32_t old_hres_lock;
383 hrtime_t tsc;
384
385 do {
386 old_hres_lock = hres_lock;
387
388 /* See tsc_tick(). */
389 tsc = tsc_read() + tsc_last_jumped;
390 } while ((old_hres_lock & ~1) != hres_lock);
391
392 return (tsc);
393 }
394
395 /*
396 * Convert a nanosecond based timestamp to tsc
397 */
398 uint64_t
399 tsc_unscalehrtime(hrtime_t nsec)
400 {
401 hrtime_t tsc;
402
403 if (tsc_gethrtime_enable) {
404 TSC_CONVERT(nsec, tsc, nsec_unscale);
405 return (tsc);
406 }
407 return ((uint64_t)nsec);
408 }
409
410 /* Convert a tsc timestamp to nanoseconds */
411 void
412 tsc_scalehrtime(hrtime_t *tsc)
413 {
414 hrtime_t hrt;
415 hrtime_t mytsc;
416
417 if (tsc == NULL)
418 return;
419 mytsc = *tsc;
420
421 TSC_CONVERT(mytsc, hrt, nsec_scale);
422 *tsc = hrt;
423 }
424
425 hrtime_t
426 tsc_gethrtimeunscaled_delta(void)
427 {
428 hrtime_t hrt;
429 ulong_t flags;
430
431 /*
432 * Similarly to tsc_gethrtime_delta, we need to disable preemption
433 * to prevent migration between the call to tsc_gethrtimeunscaled
434 * and adding the CPU's hrtime delta. Note that disabling and
435 * reenabling preemption is forbidden here because we may be in the
436 * middle of a fast trap. In the amd64 kernel we cannot tolerate
437 * preemption during a fast trap. See _update_sregs().
438 */
439
440 flags = clear_int_flag();
441 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
442 restore_int_flag(flags);
443
444 return (hrt);
445 }
446
447 /*
448 * TSC Sync Master
449 *
450 * Typically called on the boot CPU, this attempts to quantify TSC skew between
451 * different CPUs. If an appreciable difference is found, gethrtimef will be
452 * changed to point to tsc_gethrtime_delta().
453 *
454 * Calculating skews is precise only when the master and slave TSCs are read
455 * simultaneously; however, there is no algorithm that can read both CPUs in
456 * perfect simultaneity. The proposed algorithm is an approximate method based
457 * on the behaviour of cache management. The slave CPU continuously polls the
458 * TSC while reading a global variable updated by the master CPU. The latest
459 * TSC reading is saved when the master's update (forced via mfence) reaches
460 * visibility on the slave. The master will also take a TSC reading
461 * immediately following the mfence.
462 *
463 * While the delay between cache line invalidation on the slave and mfence
464 * completion on the master is not repeatable, the error is heuristically
465 * assumed to be 1/4th of the write time recorded by the master. Multiple
466 * samples are taken to control for the variance caused by external factors
467 * such as bus contention. Each sample set is independent per-CPU to control
468 * for differing memory latency on NUMA systems.
469 *
470 * TSC sync is disabled in the context of virtualization because the CPUs
471 * assigned to the guest are virtual CPUs which means the real CPUs on which
472 * guest runs keep changing during life time of guest OS. So we would end up
473 * calculating TSC skews for a set of CPUs during boot whereas the guest
474 * might migrate to a different set of physical CPUs at a later point of
475 * time.
476 */
477 void
478 tsc_sync_master(processorid_t slave)
479 {
480 ulong_t flags, source, min_write_time = ~0UL;
481 hrtime_t write_time, mtsc_after, last_delta = 0;
482 tsc_sync_t *tsc = tscp;
483 int cnt;
484 int hwtype;
485
486 hwtype = get_hwenv();
487 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
488 return;
489
490 flags = clear_int_flag();
491 source = CPU->cpu_id;
492
493 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
494 while (tsc_sync_go != TSC_SYNC_GO)
495 SMT_PAUSE();
496
497 tsc->master_tsc = tsc_read();
498 membar_enter();
499 mtsc_after = tsc_read();
500 while (tsc_sync_go != TSC_SYNC_DONE)
501 SMT_PAUSE();
502 write_time = mtsc_after - tsc->master_tsc;
503 if (write_time <= min_write_time) {
504 hrtime_t tdelta;
505
506 tdelta = tsc->slave_tsc - mtsc_after;
507 if (tdelta < 0)
508 tdelta = -tdelta;
509 /*
510 * If the margin exists, subtract 1/4th of the measured
511 * write time from the master's TSC value. This is an
512 * estimate of how late the mfence completion came
513 * after the slave noticed the cache line change.
514 */
515 if (tdelta > (write_time/4)) {
516 tdelta = tsc->slave_tsc -
517 (mtsc_after - (write_time/4));
518 } else {
519 tdelta = tsc->slave_tsc - mtsc_after;
520 }
521 last_delta = tsc_sync_tick_delta[source] - tdelta;
522 tsc_sync_tick_delta[slave] = last_delta;
523 min_write_time = write_time;
524 }
525
526 tsc->master_tsc = tsc->slave_tsc = write_time = 0;
527 membar_enter();
528 tsc_sync_go = TSC_SYNC_STOP;
529 }
530
531 /*
532 * Only enable the delta variants of the TSC functions if the measured
533 * skew is greater than the fastest write time.
534 */
535 last_delta = (last_delta < 0) ? -last_delta : last_delta;
536 if (last_delta > min_write_time) {
537 gethrtimef = tsc_gethrtime_delta;
538 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
539 tsc_ncpu = NCPU;
540 }
541 restore_int_flag(flags);
542 }
543
544 /*
545 * TSC Sync Slave
546 *
547 * Called by a CPU which has just been onlined. It is expected that the CPU
548 * performing the online operation will call tsc_sync_master().
549 *
550 * Like tsc_sync_master, this logic is skipped on virtualized platforms.
551 */
552 void
553 tsc_sync_slave(void)
554 {
555 ulong_t flags;
556 hrtime_t s1;
557 tsc_sync_t *tsc = tscp;
558 int cnt;
559 int hwtype;
560
561 hwtype = get_hwenv();
562 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
563 return;
564
565 flags = clear_int_flag();
566
567 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
568 /* Re-fill the cache line */
569 s1 = tsc->master_tsc;
570 membar_enter();
571 tsc_sync_go = TSC_SYNC_GO;
572 do {
573 /*
574 * Do not put an SMT_PAUSE here. If the master and
575 * slave are the same hyper-threaded CPU, we want the
576 * master to yield as quickly as possible to the slave.
577 */
578 s1 = tsc_read();
579 } while (tsc->master_tsc == 0);
580 tsc->slave_tsc = s1;
581 membar_enter();
582 tsc_sync_go = TSC_SYNC_DONE;
583
584 while (tsc_sync_go != TSC_SYNC_STOP)
585 SMT_PAUSE();
586 }
587
588 restore_int_flag(flags);
589 }
590
591 /*
592 * Called once per second on a CPU from the cyclic subsystem's
593 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only)
594 */
595 void
596 tsc_tick(void)
597 {
598 hrtime_t now, delta;
599 ushort_t spl;
600
601 /*
602 * Before we set the new variables, we set the shadow values. This
603 * allows for lock free operation in dtrace_gethrtime().
604 */
605 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
606 ipltospl(CBE_HIGH_PIL), &spl);
607
608 shadow_tsc_hrtime_base = tsc_hrtime_base;
609 shadow_tsc_last = tsc_last;
610 shadow_nsec_scale = nsec_scale;
611
612 shadow_hres_lock++;
613 splx(spl);
614
615 CLOCK_LOCK(&spl);
616
617 now = tsc_read();
618
619 if (gethrtimef == tsc_gethrtime_delta)
620 now += tsc_sync_tick_delta[CPU->cpu_id];
621
622 if (now < tsc_last) {
623 /*
624 * The TSC has just jumped into the past. We assume that
625 * this is due to a suspend/resume cycle, and we're going
626 * to use the _current_ value of TSC as the delta. This
627 * will keep tsc_hrtime_base correct. We're also going to
628 * assume that rate of tsc does not change after a suspend
629 * resume (i.e nsec_scale remains the same).
630 */
631 delta = now;
632 delta = tsc_protect(delta);
633 tsc_last_jumped += tsc_last;
634 tsc_jumped = 1;
635 } else {
636 /*
637 * Determine the number of TSC ticks since the last clock
638 * tick, and add that to the hrtime base.
639 */
640 delta = now - tsc_last;
641 }
642
643 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
644 tsc_last = now;
645
646 CLOCK_UNLOCK(spl);
647 }
648
649 void
650 tsc_hrtimeinit(uint64_t cpu_freq_hz)
651 {
652 extern int gethrtime_hires;
653 longlong_t tsc;
654 ulong_t flags;
655
656 /*
657 * cpu_freq_hz is the measured cpu frequency in hertz
658 */
659
660 /*
661 * We can't accommodate CPUs slower than 31.25 MHz.
662 */
663 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
664 nsec_scale =
665 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
666 nsec_unscale =
667 (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
668
669 flags = clear_int_flag();
670 tsc = tsc_read();
671 (void) tsc_gethrtime();
672 tsc_max_delta = tsc_read() - tsc;
673 restore_int_flag(flags);
674 gethrtimef = tsc_gethrtime;
675 gethrtimeunscaledf = tsc_gethrtimeunscaled;
676 scalehrtimef = tsc_scalehrtime;
677 unscalehrtimef = tsc_unscalehrtime;
678 hrtime_tick = tsc_tick;
679 gethrtime_hires = 1;
680 /*
681 * Being part of the comm page, tsc_ncpu communicates the published
682 * length of the tsc_sync_tick_delta array. This is kept zeroed to
683 * ignore the absent delta data while the TSCs are synced.
684 */
685 tsc_ncpu = 0;
686 /*
687 * Allocate memory for the structure used in the tsc sync logic.
688 * This structure should be aligned on a multiple of cache line size.
689 */
690 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
691
692 /*
693 * Convert the TSC resume cap ns value into its unscaled TSC value.
694 * See tsc_gethrtime().
695 */
696 if (tsc_resume_cap == 0)
697 TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale);
698 }
699
700 int
701 get_tsc_ready()
702 {
703 return (tsc_ready);
704 }
705
706 /*
707 * Adjust all the deltas by adding the passed value to the array and activate
708 * the "delta" versions of the gethrtime functions. It is possible that the
709 * adjustment could be negative. Such may occur if the SunOS instance was
710 * moved by a virtual manager to a machine with a higher value of TSC.
711 */
712 void
713 tsc_adjust_delta(hrtime_t tdelta)
714 {
715 int i;
716
717 for (i = 0; i < NCPU; i++) {
718 tsc_sync_tick_delta[i] += tdelta;
719 }
720
721 gethrtimef = tsc_gethrtime_delta;
722 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
723 tsc_ncpu = NCPU;
724 }
725
726 /*
727 * Functions to manage TSC and high-res time on suspend and resume.
728 */
729
730 /* tod_ops from "uts/i86pc/io/todpc_subr.c" */
731 extern tod_ops_t *tod_ops;
732
733 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
734 static timestruc_t tsc_saved_ts;
735 static int tsc_needs_resume = 0; /* We only want to do this once. */
736 int tsc_delta_onsuspend = 0;
737 int tsc_adjust_seconds = 1;
738 int tsc_suspend_count = 0;
739 int tsc_resume_in_cyclic = 0;
740
741 /*
742 * Take snapshots of the current time and do any other pre-suspend work.
743 */
744 void
745 tsc_suspend(void)
746 {
747 /*
748 * We need to collect the time at which we suspended here so we know
749 * now much should be added during the resume. This is called by each
750 * CPU, so reentry must be properly handled.
751 */
752 if (tsc_gethrtime_enable) {
753 /*
754 * Perform the tsc_read after acquiring the lock to make it as
755 * accurate as possible in the face of contention.
756 */
757 mutex_enter(&tod_lock);
758 tsc_saved_tsc = tsc_read();
759 tsc_saved_ts = TODOP_GET(tod_ops);
760 mutex_exit(&tod_lock);
761 /* We only want to do this once. */
762 if (tsc_needs_resume == 0) {
763 if (tsc_delta_onsuspend) {
764 tsc_adjust_delta(tsc_saved_tsc);
765 } else {
766 tsc_adjust_delta(nsec_scale);
767 }
768 tsc_suspend_count++;
769 }
770 }
771
772 invalidate_cache();
773 tsc_needs_resume = 1;
774 }
775
776 /*
777 * Restore all timestamp state based on the snapshots taken at suspend time.
778 */
779 void
780 tsc_resume(void)
781 {
782 /*
783 * We only need to (and want to) do this once. So let the first
784 * caller handle this (we are locked by the cpu lock), as it
785 * is preferential that we get the earliest sync.
786 */
787 if (tsc_needs_resume) {
788 /*
789 * If using the TSC, adjust the delta based on how long
790 * we were sleeping (or away). We also adjust for
791 * migration and a grown TSC.
792 */
793 if (tsc_saved_tsc != 0) {
794 timestruc_t ts;
795 hrtime_t now, sleep_tsc = 0;
796 int sleep_sec;
797 extern void tsc_tick(void);
798 extern uint64_t cpu_freq_hz;
799
800 /* tsc_read() MUST be before TODOP_GET() */
801 mutex_enter(&tod_lock);
802 now = tsc_read();
803 ts = TODOP_GET(tod_ops);
804 mutex_exit(&tod_lock);
805
806 /* Compute seconds of sleep time */
807 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
808
809 /*
810 * If the saved sec is less that or equal to
811 * the current ts, then there is likely a
812 * problem with the clock. Assume at least
813 * one second has passed, so that time goes forward.
814 */
815 if (sleep_sec <= 0) {
816 sleep_sec = 1;
817 }
818
819 /* How many TSC's should have occured while sleeping */
820 if (tsc_adjust_seconds)
821 sleep_tsc = sleep_sec * cpu_freq_hz;
822
823 /*
824 * We also want to subtract from the "sleep_tsc"
825 * the current value of tsc_read(), so that our
826 * adjustment accounts for the amount of time we
827 * have been resumed _or_ an adjustment based on
828 * the fact that we didn't actually power off the
829 * CPU (migration is another issue, but _should_
830 * also comply with this calculation). If the CPU
831 * never powered off, then:
832 * 'now == sleep_tsc + saved_tsc'
833 * and the delta will effectively be "0".
834 */
835 sleep_tsc -= now;
836 if (tsc_delta_onsuspend) {
837 tsc_adjust_delta(sleep_tsc);
838 } else {
839 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
840 }
841 tsc_saved_tsc = 0;
842
843 tsc_tick();
844 }
845 tsc_needs_resume = 0;
846 }
847
848 }