1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 #include <sys/types.h>
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/disp.h>
33 #include <sys/var.h>
34 #include <sys/cmn_err.h>
35 #include <sys/debug.h>
36 #include <sys/x86_archext.h>
37 #include <sys/archsystm.h>
38 #include <sys/cpuvar.h>
39 #include <sys/psm_defs.h>
40 #include <sys/clock.h>
41 #include <sys/atomic.h>
42 #include <sys/lockstat.h>
43 #include <sys/smp_impldefs.h>
44 #include <sys/dtrace.h>
45 #include <sys/time.h>
46 #include <sys/panic.h>
47 #include <sys/cpu.h>
48
49 /*
50 * Using the Pentium's TSC register for gethrtime()
51 * ------------------------------------------------
52 *
53 * The Pentium family, like many chip architectures, has a high-resolution
54 * timestamp counter ("TSC") which increments once per CPU cycle. The contents
55 * of the timestamp counter are read with the RDTSC instruction.
56 *
57 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
58 * must be translated into nanoseconds in order to implement gethrtime().
59 * We avoid inducing floating point operations in this conversion by
60 * implementing the same nsec_scale algorithm as that found in the sun4u
61 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
62 * a detailed description of the algorithm; the comment is not reproduced
63 * here. This implementation differs only in its value for NSEC_SHIFT:
64 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
65 * 60 MHz Pentiums.
66 *
67 * While TSC and %tick are both cycle counting registers, TSC's functionality
68 * falls short in several critical ways:
69 *
70 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in
71 * practice they often _are_ in sync, this isn't guaranteed by the
72 * architecture.
73 *
74 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture
75 * only supports writing the low 32-bits of TSC, making it impractical
76 * to rewrite.
77 *
78 * (c) The architecture doesn't have the capacity to interrupt based on
79 * arbitrary values of TSC; there is no TICK_CMPR equivalent.
80 *
81 * Together, (a) and (b) imply that software must track the skew between
82 * TSCs and account for it (it is assumed that while there may exist skew,
83 * there does not exist drift). To determine the skew between CPUs, we
84 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
85 * the online operation calls tsc_sync_master().
86 *
87 * In the absence of time-of-day clock adjustments, gethrtime() must stay in
88 * sync with gettimeofday(). This is problematic; given (c), the software
89 * cannot drive its time-of-day source from TSC, and yet they must somehow be
90 * kept in sync. We implement this by having a routine, tsc_tick(), which
91 * is called once per second from the interrupt which drives time-of-day.
92 *
93 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
94 * atomically with nsec_scale under CLOCK_LOCK. This assures that time
95 * monotonically increases.
96 */
97
98 #define NSEC_SHIFT 5
99
100 static uint_t nsec_scale;
101 static uint_t nsec_unscale;
102
103 /*
104 * These two variables used to be grouped together inside of a structure that
105 * lived on a single cache line. A regression (bug ID 4623398) caused the
106 * compiler to emit code that "optimized" away the while-loops below. The
107 * result was that no synchronization between the onlining and onlined CPUs
108 * took place.
109 */
110 static volatile int tsc_ready;
111 static volatile int tsc_sync_go;
112
113 /*
114 * Used as indices into the tsc_sync_snaps[] array.
115 */
116 #define TSC_MASTER 0
117 #define TSC_SLAVE 1
118
119 /*
120 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
121 */
122 #define TSC_SYNC_STOP 1
123 #define TSC_SYNC_GO 2
124 #define TSC_SYNC_DONE 3
125 #define SYNC_ITERATIONS 10
126
127 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \
128 unsigned int *_l = (unsigned int *)&(tsc); \
129 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \
130 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
131 }
132
133 #define TSC_CONVERT(tsc, hrt, scale) { \
134 unsigned int *_l = (unsigned int *)&(tsc); \
135 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \
136 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
137 }
138
139 int tsc_master_slave_sync_needed = 1;
140
141 static int tsc_max_delta;
142 static hrtime_t tsc_sync_tick_delta[NCPU];
143 typedef struct tsc_sync {
144 volatile hrtime_t master_tsc, slave_tsc;
145 } tsc_sync_t;
146 static tsc_sync_t *tscp;
147 static hrtime_t largest_tsc_delta = 0;
148 static ulong_t shortest_write_time = ~0UL;
149
150 static hrtime_t tsc_last = 0;
151 static hrtime_t tsc_last_jumped = 0;
152 static hrtime_t tsc_hrtime_base = 0;
153 static int tsc_jumped = 0;
154
155 static hrtime_t shadow_tsc_hrtime_base;
156 static hrtime_t shadow_tsc_last;
157 static uint_t shadow_nsec_scale;
158 static uint32_t shadow_hres_lock;
159 int get_tsc_ready();
160
161 hrtime_t
162 tsc_gethrtime(void)
163 {
164 uint32_t old_hres_lock;
165 hrtime_t tsc, hrt;
166
167 do {
168 old_hres_lock = hres_lock;
169
170 if ((tsc = tsc_read()) >= tsc_last) {
171 /*
172 * It would seem to be obvious that this is true
173 * (that is, the past is less than the present),
174 * but it isn't true in the presence of suspend/resume
175 * cycles. If we manage to call gethrtime()
176 * after a resume, but before the first call to
177 * tsc_tick(), we will see the jump. In this case,
178 * we will simply use the value in TSC as the delta.
179 */
180 tsc -= tsc_last;
181 } else if (tsc >= tsc_last - 2*tsc_max_delta) {
182 /*
183 * There is a chance that tsc_tick() has just run on
184 * another CPU, and we have drifted just enough so that
185 * we appear behind tsc_last. In this case, force the
186 * delta to be zero.
187 */
188 tsc = 0;
189 }
190
191 hrt = tsc_hrtime_base;
192
193 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
194 } while ((old_hres_lock & ~1) != hres_lock);
195
196 return (hrt);
197 }
198
199 hrtime_t
200 tsc_gethrtime_delta(void)
201 {
202 uint32_t old_hres_lock;
203 hrtime_t tsc, hrt;
204 ulong_t flags;
205
206 do {
207 old_hres_lock = hres_lock;
208
209 /*
210 * We need to disable interrupts here to assure that we
211 * don't migrate between the call to tsc_read() and
212 * adding the CPU's TSC tick delta. Note that disabling
213 * and reenabling preemption is forbidden here because
214 * we may be in the middle of a fast trap. In the amd64
215 * kernel we cannot tolerate preemption during a fast
216 * trap. See _update_sregs().
217 */
218
219 flags = clear_int_flag();
220 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
221 restore_int_flag(flags);
222
223 /* See comments in tsc_gethrtime() above */
224
225 if (tsc >= tsc_last) {
226 tsc -= tsc_last;
227 } else if (tsc >= tsc_last - 2 * tsc_max_delta) {
228 tsc = 0;
229 }
230
231 hrt = tsc_hrtime_base;
232
233 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
234 } while ((old_hres_lock & ~1) != hres_lock);
235
236 return (hrt);
237 }
238
239 hrtime_t
240 tsc_gethrtime_tick_delta(void)
241 {
242 hrtime_t hrt;
243 ulong_t flags;
244
245 flags = clear_int_flag();
246 hrt = tsc_sync_tick_delta[CPU->cpu_id];
247 restore_int_flag(flags);
248
249 return (hrt);
250 }
251
252 /*
253 * This is similar to the above, but it cannot actually spin on hres_lock.
254 * As a result, it caches all of the variables it needs; if the variables
255 * don't change, it's done.
256 */
257 hrtime_t
258 dtrace_gethrtime(void)
259 {
260 uint32_t old_hres_lock;
261 hrtime_t tsc, hrt;
262 ulong_t flags;
263
264 do {
265 old_hres_lock = hres_lock;
266
267 /*
268 * Interrupts are disabled to ensure that the thread isn't
269 * migrated between the tsc_read() and adding the CPU's
270 * TSC tick delta.
271 */
272 flags = clear_int_flag();
273
274 tsc = tsc_read();
275
276 if (gethrtimef == tsc_gethrtime_delta)
277 tsc += tsc_sync_tick_delta[CPU->cpu_id];
278
279 restore_int_flag(flags);
280
281 /*
282 * See the comments in tsc_gethrtime(), above.
283 */
284 if (tsc >= tsc_last)
285 tsc -= tsc_last;
286 else if (tsc >= tsc_last - 2*tsc_max_delta)
287 tsc = 0;
288
289 hrt = tsc_hrtime_base;
290
291 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
292
293 if ((old_hres_lock & ~1) == hres_lock)
294 break;
295
296 /*
297 * If we're here, the clock lock is locked -- or it has been
298 * unlocked and locked since we looked. This may be due to
299 * tsc_tick() running on another CPU -- or it may be because
300 * some code path has ended up in dtrace_probe() with
301 * CLOCK_LOCK held. We'll try to determine that we're in
302 * the former case by taking another lap if the lock has
303 * changed since when we first looked at it.
304 */
305 if (old_hres_lock != hres_lock)
306 continue;
307
308 /*
309 * So the lock was and is locked. We'll use the old data
310 * instead.
311 */
312 old_hres_lock = shadow_hres_lock;
313
314 /*
315 * Again, disable interrupts to ensure that the thread
316 * isn't migrated between the tsc_read() and adding
317 * the CPU's TSC tick delta.
318 */
319 flags = clear_int_flag();
320
321 tsc = tsc_read();
322
323 if (gethrtimef == tsc_gethrtime_delta)
324 tsc += tsc_sync_tick_delta[CPU->cpu_id];
325
326 restore_int_flag(flags);
327
328 /*
329 * See the comments in tsc_gethrtime(), above.
330 */
331 if (tsc >= shadow_tsc_last)
332 tsc -= shadow_tsc_last;
333 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
334 tsc = 0;
335
336 hrt = shadow_tsc_hrtime_base;
337
338 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
339 } while ((old_hres_lock & ~1) != shadow_hres_lock);
340
341 return (hrt);
342 }
343
344 hrtime_t
345 tsc_gethrtimeunscaled(void)
346 {
347 uint32_t old_hres_lock;
348 hrtime_t tsc;
349
350 do {
351 old_hres_lock = hres_lock;
352
353 /* See tsc_tick(). */
354 tsc = tsc_read() + tsc_last_jumped;
355 } while ((old_hres_lock & ~1) != hres_lock);
356
357 return (tsc);
358 }
359
360 /*
361 * Convert a nanosecond based timestamp to tsc
362 */
363 uint64_t
364 tsc_unscalehrtime(hrtime_t nsec)
365 {
366 hrtime_t tsc;
367
368 if (tsc_gethrtime_enable) {
369 TSC_CONVERT(nsec, tsc, nsec_unscale);
370 return (tsc);
371 }
372 return ((uint64_t)nsec);
373 }
374
375 /* Convert a tsc timestamp to nanoseconds */
376 void
377 tsc_scalehrtime(hrtime_t *tsc)
378 {
379 hrtime_t hrt;
380 hrtime_t mytsc;
381
382 if (tsc == NULL)
383 return;
384 mytsc = *tsc;
385
386 TSC_CONVERT(mytsc, hrt, nsec_scale);
387 *tsc = hrt;
388 }
389
390 hrtime_t
391 tsc_gethrtimeunscaled_delta(void)
392 {
393 hrtime_t hrt;
394 ulong_t flags;
395
396 /*
397 * Similarly to tsc_gethrtime_delta, we need to disable preemption
398 * to prevent migration between the call to tsc_gethrtimeunscaled
399 * and adding the CPU's hrtime delta. Note that disabling and
400 * reenabling preemption is forbidden here because we may be in the
401 * middle of a fast trap. In the amd64 kernel we cannot tolerate
402 * preemption during a fast trap. See _update_sregs().
403 */
404
405 flags = clear_int_flag();
406 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
407 restore_int_flag(flags);
408
409 return (hrt);
410 }
411
412 /*
413 * Called by the master in the TSC sync operation (usually the boot CPU).
414 * If the slave is discovered to have a skew, gethrtimef will be changed to
415 * point to tsc_gethrtime_delta(). Calculating skews is precise only when
416 * the master and slave TSCs are read simultaneously; however, there is no
417 * algorithm that can read both CPUs in perfect simultaneity. The proposed
418 * algorithm is an approximate method based on the behaviour of cache
419 * management. The slave CPU continuously reads TSC and then reads a global
420 * variable which the master CPU updates. The moment the master's update reaches
421 * the slave's visibility (being forced by an mfence operation) we use the TSC
422 * reading taken on the slave. A corresponding TSC read will be taken on the
423 * master as soon as possible after finishing the mfence operation. But the
424 * delay between causing the slave to notice the invalid cache line and the
425 * competion of mfence is not repeatable. This error is heuristically assumed
426 * to be 1/4th of the total write time as being measured by the two TSC reads
427 * on the master sandwiching the mfence. Furthermore, due to the nature of
428 * bus arbitration, contention on memory bus, etc., the time taken for the write
429 * to reflect globally can vary a lot. So instead of taking a single reading,
430 * a set of readings are taken and the one with least write time is chosen
431 * to calculate the final skew.
432 *
433 * TSC sync is disabled in the context of virtualization because the CPUs
434 * assigned to the guest are virtual CPUs which means the real CPUs on which
435 * guest runs keep changing during life time of guest OS. So we would end up
436 * calculating TSC skews for a set of CPUs during boot whereas the guest
437 * might migrate to a different set of physical CPUs at a later point of
438 * time.
439 */
440 void
441 tsc_sync_master(processorid_t slave)
442 {
443 ulong_t flags, source, min_write_time = ~0UL;
444 hrtime_t write_time, x, mtsc_after, tdelta;
445 tsc_sync_t *tsc = tscp;
446 int cnt;
447 int hwtype;
448
449 hwtype = get_hwenv();
450 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
451 return;
452
453 flags = clear_int_flag();
454 source = CPU->cpu_id;
455
456 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
457 while (tsc_sync_go != TSC_SYNC_GO)
458 SMT_PAUSE();
459
460 tsc->master_tsc = tsc_read();
461 membar_enter();
462 mtsc_after = tsc_read();
463 while (tsc_sync_go != TSC_SYNC_DONE)
464 SMT_PAUSE();
465 write_time = mtsc_after - tsc->master_tsc;
466 if (write_time <= min_write_time) {
467 min_write_time = write_time;
468 /*
469 * Apply heuristic adjustment only if the calculated
470 * delta is > 1/4th of the write time.
471 */
472 x = tsc->slave_tsc - mtsc_after;
473 if (x < 0)
474 x = -x;
475 if (x > (min_write_time/4))
476 /*
477 * Subtract 1/4th of the measured write time
478 * from the master's TSC value, as an estimate
479 * of how late the mfence completion came
480 * after the slave noticed the cache line
481 * change.
482 */
483 tdelta = tsc->slave_tsc -
484 (mtsc_after - (min_write_time/4));
485 else
486 tdelta = tsc->slave_tsc - mtsc_after;
487 tsc_sync_tick_delta[slave] =
488 tsc_sync_tick_delta[source] - tdelta;
489 }
490
491 tsc->master_tsc = tsc->slave_tsc = write_time = 0;
492 membar_enter();
493 tsc_sync_go = TSC_SYNC_STOP;
494 }
495 if (tdelta < 0)
496 tdelta = -tdelta;
497 if (tdelta > largest_tsc_delta)
498 largest_tsc_delta = tdelta;
499 if (min_write_time < shortest_write_time)
500 shortest_write_time = min_write_time;
501 /*
502 * Enable delta variants of tsc functions if the largest of all chosen
503 * deltas is > smallest of the write time.
504 */
505 if (largest_tsc_delta > shortest_write_time) {
506 gethrtimef = tsc_gethrtime_delta;
507 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
508 }
509 restore_int_flag(flags);
510 }
511
512 /*
513 * Called by a CPU which has just been onlined. It is expected that the CPU
514 * performing the online operation will call tsc_sync_master().
515 *
516 * TSC sync is disabled in the context of virtualization. See comments
517 * above tsc_sync_master.
518 */
519 void
520 tsc_sync_slave(void)
521 {
522 ulong_t flags;
523 hrtime_t s1;
524 tsc_sync_t *tsc = tscp;
525 int cnt;
526 int hwtype;
527
528 hwtype = get_hwenv();
529 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
530 return;
531
532 flags = clear_int_flag();
533
534 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
535 /* Re-fill the cache line */
536 s1 = tsc->master_tsc;
537 membar_enter();
538 tsc_sync_go = TSC_SYNC_GO;
539 do {
540 /*
541 * Do not put an SMT_PAUSE here. For instance,
542 * if the master and slave are really the same
543 * hyper-threaded CPU, then you want the master
544 * to yield to the slave as quickly as possible here,
545 * but not the other way.
546 */
547 s1 = tsc_read();
548 } while (tsc->master_tsc == 0);
549 tsc->slave_tsc = s1;
550 membar_enter();
551 tsc_sync_go = TSC_SYNC_DONE;
552
553 while (tsc_sync_go != TSC_SYNC_STOP)
554 SMT_PAUSE();
555 }
556
557 restore_int_flag(flags);
558 }
559
560 /*
561 * Called once per second on a CPU from the cyclic subsystem's
562 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only)
563 */
564 void
565 tsc_tick(void)
566 {
567 hrtime_t now, delta;
568 ushort_t spl;
569
570 /*
571 * Before we set the new variables, we set the shadow values. This
572 * allows for lock free operation in dtrace_gethrtime().
573 */
574 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
575 ipltospl(CBE_HIGH_PIL), &spl);
576
577 shadow_tsc_hrtime_base = tsc_hrtime_base;
578 shadow_tsc_last = tsc_last;
579 shadow_nsec_scale = nsec_scale;
580
581 shadow_hres_lock++;
582 splx(spl);
583
584 CLOCK_LOCK(&spl);
585
586 now = tsc_read();
587
588 if (gethrtimef == tsc_gethrtime_delta)
589 now += tsc_sync_tick_delta[CPU->cpu_id];
590
591 if (now < tsc_last) {
592 /*
593 * The TSC has just jumped into the past. We assume that
594 * this is due to a suspend/resume cycle, and we're going
595 * to use the _current_ value of TSC as the delta. This
596 * will keep tsc_hrtime_base correct. We're also going to
597 * assume that rate of tsc does not change after a suspend
598 * resume (i.e nsec_scale remains the same).
599 */
600 delta = now;
601 tsc_last_jumped += tsc_last;
602 tsc_jumped = 1;
603 } else {
604 /*
605 * Determine the number of TSC ticks since the last clock
606 * tick, and add that to the hrtime base.
607 */
608 delta = now - tsc_last;
609 }
610
611 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
612 tsc_last = now;
613
614 CLOCK_UNLOCK(spl);
615 }
616
617 void
618 tsc_hrtimeinit(uint64_t cpu_freq_hz)
619 {
620 extern int gethrtime_hires;
621 longlong_t tsc;
622 ulong_t flags;
623
624 /*
625 * cpu_freq_hz is the measured cpu frequency in hertz
626 */
627
628 /*
629 * We can't accommodate CPUs slower than 31.25 MHz.
630 */
631 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
632 nsec_scale =
633 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
634 nsec_unscale =
635 (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
636
637 flags = clear_int_flag();
638 tsc = tsc_read();
639 (void) tsc_gethrtime();
640 tsc_max_delta = tsc_read() - tsc;
641 restore_int_flag(flags);
642 gethrtimef = tsc_gethrtime;
643 gethrtimeunscaledf = tsc_gethrtimeunscaled;
644 scalehrtimef = tsc_scalehrtime;
645 unscalehrtimef = tsc_unscalehrtime;
646 hrtime_tick = tsc_tick;
647 gethrtime_hires = 1;
648 /*
649 * Allocate memory for the structure used in the tsc sync logic.
650 * This structure should be aligned on a multiple of cache line size.
651 */
652 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
653 }
654
655 int
656 get_tsc_ready()
657 {
658 return (tsc_ready);
659 }
660
661 /*
662 * Adjust all the deltas by adding the passed value to the array.
663 * Then use the "delt" versions of the the gethrtime functions.
664 * Note that 'tdelta' _could_ be a negative number, which should
665 * reduce the values in the array (used, for example, if the Solaris
666 * instance was moved by a virtual manager to a machine with a higher
667 * value of tsc).
668 */
669 void
670 tsc_adjust_delta(hrtime_t tdelta)
671 {
672 int i;
673
674 for (i = 0; i < NCPU; i++) {
675 tsc_sync_tick_delta[i] += tdelta;
676 }
677
678 gethrtimef = tsc_gethrtime_delta;
679 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
680 }
681
682 /*
683 * Functions to manage TSC and high-res time on suspend and resume.
684 */
685
686 /*
687 * declarations needed for time adjustment
688 */
689 extern void rtcsync(void);
690 extern tod_ops_t *tod_ops;
691 /* There must be a better way than exposing nsec_scale! */
692 extern uint_t nsec_scale;
693 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
694 static timestruc_t tsc_saved_ts;
695 static int tsc_needs_resume = 0; /* We only want to do this once. */
696 int tsc_delta_onsuspend = 0;
697 int tsc_adjust_seconds = 1;
698 int tsc_suspend_count = 0;
699 int tsc_resume_in_cyclic = 0;
700
701 /*
702 * Let timestamp.c know that we are suspending. It needs to take
703 * snapshots of the current time, and do any pre-suspend work.
704 */
705 void
706 tsc_suspend(void)
707 {
708 /*
709 * What we need to do here, is to get the time we suspended, so that we
710 * know how much we should add to the resume.
711 * This routine is called by each CPU, so we need to handle reentry.
712 */
713 if (tsc_gethrtime_enable) {
714 /*
715 * We put the tsc_read() inside the lock as it
716 * as no locking constraints, and it puts the
717 * aquired value closer to the time stamp (in
718 * case we delay getting the lock).
719 */
720 mutex_enter(&tod_lock);
721 tsc_saved_tsc = tsc_read();
722 tsc_saved_ts = TODOP_GET(tod_ops);
723 mutex_exit(&tod_lock);
724 /* We only want to do this once. */
725 if (tsc_needs_resume == 0) {
726 if (tsc_delta_onsuspend) {
727 tsc_adjust_delta(tsc_saved_tsc);
728 } else {
729 tsc_adjust_delta(nsec_scale);
730 }
731 tsc_suspend_count++;
732 }
733 }
734
735 invalidate_cache();
736 tsc_needs_resume = 1;
737 }
738
739 /*
740 * Restore all timestamp state based on the snapshots taken at
741 * suspend time.
742 */
743 void
744 tsc_resume(void)
745 {
746 /*
747 * We only need to (and want to) do this once. So let the first
748 * caller handle this (we are locked by the cpu lock), as it
749 * is preferential that we get the earliest sync.
750 */
751 if (tsc_needs_resume) {
752 /*
753 * If using the TSC, adjust the delta based on how long
754 * we were sleeping (or away). We also adjust for
755 * migration and a grown TSC.
756 */
757 if (tsc_saved_tsc != 0) {
758 timestruc_t ts;
759 hrtime_t now, sleep_tsc = 0;
760 int sleep_sec;
761 extern void tsc_tick(void);
762 extern uint64_t cpu_freq_hz;
763
764 /* tsc_read() MUST be before TODOP_GET() */
765 mutex_enter(&tod_lock);
766 now = tsc_read();
767 ts = TODOP_GET(tod_ops);
768 mutex_exit(&tod_lock);
769
770 /* Compute seconds of sleep time */
771 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
772
773 /*
774 * If the saved sec is less that or equal to
775 * the current ts, then there is likely a
776 * problem with the clock. Assume at least
777 * one second has passed, so that time goes forward.
778 */
779 if (sleep_sec <= 0) {
780 sleep_sec = 1;
781 }
782
783 /* How many TSC's should have occured while sleeping */
784 if (tsc_adjust_seconds)
785 sleep_tsc = sleep_sec * cpu_freq_hz;
786
787 /*
788 * We also want to subtract from the "sleep_tsc"
789 * the current value of tsc_read(), so that our
790 * adjustment accounts for the amount of time we
791 * have been resumed _or_ an adjustment based on
792 * the fact that we didn't actually power off the
793 * CPU (migration is another issue, but _should_
794 * also comply with this calculation). If the CPU
795 * never powered off, then:
796 * 'now == sleep_tsc + saved_tsc'
797 * and the delta will effectively be "0".
798 */
799 sleep_tsc -= now;
800 if (tsc_delta_onsuspend) {
801 tsc_adjust_delta(sleep_tsc);
802 } else {
803 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
804 }
805 tsc_saved_tsc = 0;
806
807 tsc_tick();
808 }
809 tsc_needs_resume = 0;
810 }
811
812 }