5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 * Copyright 2016 Gary Mills
27 */
28
29 /*
30 * VM - Hardware Address Translation management for Spitfire MMU.
31 *
32 * This file implements the machine specific hardware translation
33 * needed by the VM system. The machine independent interface is
34 * described in <vm/hat.h> while the machine dependent interface
35 * and data structures are described in <vm/hat_sfmmu.h>.
36 *
37 * The hat layer manages the address translation hardware as a cache
38 * driven by calls from the higher levels in the VM system.
39 */
40
41 #include <sys/types.h>
42 #include <sys/kstat.h>
43 #include <vm/hat.h>
44 #include <vm/hat_sfmmu.h>
45 #include <vm/page.h>
643
644 /*
645 * Used for growing the TSB based on the process RSS.
646 * tsb_rss_factor is based on the smallest TSB, and is
647 * shifted by the TSB size to determine if we need to grow.
648 * The default will grow the TSB if the number of TTEs for
649 * this page size exceeds 75% of the number of TSB entries,
650 * which should _almost_ eliminate all conflict misses
651 * (at the expense of using up lots and lots of memory).
652 */
653 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75)
654 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc)
655 #define SELECT_TSB_SIZECODE(pgcnt) ( \
656 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \
657 default_tsb_size)
658 #define TSB_OK_SHRINK() \
659 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree)
660 #define TSB_OK_GROW() \
661 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree)
662
663 int enable_tsb_rss_sizing = 1;
664 int tsb_rss_factor = (int)TSB_RSS_FACTOR;
665
666 /* which TSB size code to use for new address spaces or if rss sizing off */
667 int default_tsb_size = TSB_8K_SZCODE;
668
669 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */
670 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */
671 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32
672
673 #ifdef DEBUG
674 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */
675 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */
676 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */
677 static int tsb_alloc_fail_mtbf = 0;
678 static int tsb_alloc_count = 0;
679 #endif /* DEBUG */
680
681 /* if set to 1, will remap valid TTEs when growing TSB. */
682 int tsb_remap_ttes = 1;
683
684 /*
685 * If we have more than this many mappings, allocate a second TSB.
686 * This default is chosen because the I/D fully associative TLBs are
687 * assumed to have at least 8 available entries. Platforms with a
688 * larger fully-associative TLB could probably override the default.
689 */
690
6170 if (pml) {
6171 sfmmu_mlist_exit(pml);
6172 }
6173
6174 addr += TTEBYTES(ttesz);
6175 sfhmep++;
6176 DEMAP_RANGE_NEXTPG(dmrp);
6177 }
6178 /*
6179 * For shared hmeblks this routine is only called when region is freed
6180 * and no longer referenced. So no need to decrement ttecnt
6181 * in the region structure here.
6182 */
6183 if (ttecnt > 0 && sfmmup != NULL) {
6184 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt);
6185 }
6186 return (addr);
6187 }
6188
6189 /*
6190 * Invalidate a virtual address range for the local CPU.
6191 * For best performance ensure that the va range is completely
6192 * mapped, otherwise the entire TLB will be flushed.
6193 */
6194 void
6195 hat_flush_range(struct hat *sfmmup, caddr_t va, size_t size)
6196 {
6197 ssize_t sz;
6198 caddr_t endva = va + size;
6199
6200 while (va < endva) {
6201 sz = hat_getpagesize(sfmmup, va);
6202 if (sz < 0) {
6203 vtag_flushall();
6204 break;
6205 }
6206 vtag_flushpage(va, (uint64_t)sfmmup);
6207 va += sz;
6208 }
6209 }
6210
6211 /*
6212 * Synchronize all the mappings in the range [addr..addr+len).
6213 * Can be called with clearflag having two states:
6214 * HAT_SYNC_DONTZERO means just return the rm stats
6215 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats
6216 */
6217 void
6218 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)
6219 {
6220 struct hmehash_bucket *hmebp;
6221 hmeblk_tag hblktag;
6222 int hmeshift, hashno = 1;
6223 struct hme_blk *hmeblkp, *list = NULL;
6224 caddr_t endaddr;
6225 cpuset_t cpuset;
6226
6227 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
6228 ASSERT((len & MMU_PAGEOFFSET) == 0);
|
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
26 * Copyright 2016 Gary Mills
27 */
28
29 /*
30 * VM - Hardware Address Translation management for Spitfire MMU.
31 *
32 * This file implements the machine specific hardware translation
33 * needed by the VM system. The machine independent interface is
34 * described in <vm/hat.h> while the machine dependent interface
35 * and data structures are described in <vm/hat_sfmmu.h>.
36 *
37 * The hat layer manages the address translation hardware as a cache
38 * driven by calls from the higher levels in the VM system.
39 */
40
41 #include <sys/types.h>
42 #include <sys/kstat.h>
43 #include <vm/hat.h>
44 #include <vm/hat_sfmmu.h>
45 #include <vm/page.h>
643
644 /*
645 * Used for growing the TSB based on the process RSS.
646 * tsb_rss_factor is based on the smallest TSB, and is
647 * shifted by the TSB size to determine if we need to grow.
648 * The default will grow the TSB if the number of TTEs for
649 * this page size exceeds 75% of the number of TSB entries,
650 * which should _almost_ eliminate all conflict misses
651 * (at the expense of using up lots and lots of memory).
652 */
653 #define TSB_RSS_FACTOR (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75)
654 #define SFMMU_RSS_TSBSIZE(tsbszc) (tsb_rss_factor << tsbszc)
655 #define SELECT_TSB_SIZECODE(pgcnt) ( \
656 (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \
657 default_tsb_size)
658 #define TSB_OK_SHRINK() \
659 (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree)
660 #define TSB_OK_GROW() \
661 (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree)
662
663 volatile int enable_tsb_rss_sizing = 1;
664 volatile int tsb_rss_factor = (int)TSB_RSS_FACTOR;
665
666 /* which TSB size code to use for new address spaces or if rss sizing off */
667 volatile int default_tsb_size = TSB_8K_SZCODE;
668
669 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */
670 volatile uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = */
671 /* physmem / this */
672 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT 32
673
674 #ifdef DEBUG
675 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */
676 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */
677 static int tsb_alloc_mtbf = 0; /* fail allocation every n attempts */
678 static int tsb_alloc_fail_mtbf = 0;
679 static int tsb_alloc_count = 0;
680 #endif /* DEBUG */
681
682 /* if set to 1, will remap valid TTEs when growing TSB. */
683 int tsb_remap_ttes = 1;
684
685 /*
686 * If we have more than this many mappings, allocate a second TSB.
687 * This default is chosen because the I/D fully associative TLBs are
688 * assumed to have at least 8 available entries. Platforms with a
689 * larger fully-associative TLB could probably override the default.
690 */
691
6171 if (pml) {
6172 sfmmu_mlist_exit(pml);
6173 }
6174
6175 addr += TTEBYTES(ttesz);
6176 sfhmep++;
6177 DEMAP_RANGE_NEXTPG(dmrp);
6178 }
6179 /*
6180 * For shared hmeblks this routine is only called when region is freed
6181 * and no longer referenced. So no need to decrement ttecnt
6182 * in the region structure here.
6183 */
6184 if (ttecnt > 0 && sfmmup != NULL) {
6185 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt);
6186 }
6187 return (addr);
6188 }
6189
6190 /*
6191 * Flush the TLB for the local CPU
6192 * Invoked from a slave CPU during panic() dumps.
6193 */
6194 void
6195 hat_flush(void)
6196 {
6197 vtag_flushall();
6198 }
6199
6200 /*
6201 * Synchronize all the mappings in the range [addr..addr+len).
6202 * Can be called with clearflag having two states:
6203 * HAT_SYNC_DONTZERO means just return the rm stats
6204 * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats
6205 */
6206 void
6207 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)
6208 {
6209 struct hmehash_bucket *hmebp;
6210 hmeblk_tag hblktag;
6211 int hmeshift, hashno = 1;
6212 struct hme_blk *hmeblkp, *list = NULL;
6213 caddr_t endaddr;
6214 cpuset_t cpuset;
6215
6216 ASSERT((sfmmup == ksfmmup) || AS_LOCK_HELD(sfmmup->sfmmu_as));
6217 ASSERT((len & MMU_PAGEOFFSET) == 0);
|