9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
  25  * Copyright 2017 Nexenta Systems, Inc.
  26  * Copyright 2015 Joyent, Inc.
  27  * Copyright (c) 2015 by Delphix. All rights reserved.
  28  */

  29 /*
  30  * Copyright (c) 2010, Intel Corporation.
  31  * All rights reserved.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/t_lock.h>
  36 #include <sys/param.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/signal.h>
  39 #include <sys/systm.h>
  40 #include <sys/user.h>
  41 #include <sys/mman.h>
  42 #include <sys/vm.h>
  43 #include <sys/conf.h>
  44 #include <sys/avintr.h>
  45 #include <sys/autoconf.h>
  46 #include <sys/disp.h>
  47 #include <sys/class.h>
  48 #include <sys/bitmap.h>

 108 #include <sys/machsystm.h>
 109 #include <vm/hat.h>
 110 #include <vm/hat_i86.h>
 111 #include <sys/pmem.h>
 112 #include <sys/smp_impldefs.h>
 113 #include <sys/x86_archext.h>
 114 #include <sys/cpuvar.h>
 115 #include <sys/segments.h>
 116 #include <sys/clconf.h>
 117 #include <sys/kobj.h>
 118 #include <sys/kobj_lex.h>
 119 #include <sys/cpc_impl.h>
 120 #include <sys/cpu_module.h>
 121 #include <sys/smbios.h>
 122 #include <sys/debug_info.h>
 123 #include <sys/bootinfo.h>
 124 #include <sys/ddi_periodic.h>
 125 #include <sys/systeminfo.h>
 126 #include <sys/multiboot.h>
 127 #include <sys/ramdisk.h>

 128 
 129 #ifdef  __xpv
 130 
 131 #include <sys/hypervisor.h>
 132 #include <sys/xen_mmu.h>
 133 #include <sys/evtchn_impl.h>
 134 #include <sys/gnttab.h>
 135 #include <sys/xpv_panic.h>
 136 #include <xen/sys/xenbus_comms.h>
 137 #include <xen/public/physdev.h>
 138 
 139 extern void xen_late_startup(void);
 140 
 141 struct xen_evt_data cpu0_evt_data;
 142 
 143 #else   /* __xpv */
 144 #include <sys/memlist_impl.h>
 145 
 146 extern void mem_config_init(void);
 147 #endif /* __xpv */

 213  * kernelbase being set too high.
 214  */
 215 #define PHYSMEM 0x400000
 216 
 217 #else /* __amd64 */
 218 
 219 /*
 220  * For now we can handle memory with physical addresses up to about
 221  * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly
 222  * half the VA space for seg_kpm. When systems get bigger than 64TB this
 223  * code will need revisiting. There is an implicit assumption that there
 224  * are no *huge* holes in the physical address space too.
 225  */
 226 #define TERABYTE                (1ul << 40)
 227 #define PHYSMEM_MAX64           mmu_btop(64 * TERABYTE)
 228 #define PHYSMEM                 PHYSMEM_MAX64
 229 #define AMD64_VA_HOLE_END       0xFFFF800000000000ul
 230 
 231 #endif /* __amd64 */
 232 
 233 pgcnt_t physmem = PHYSMEM;
 234 pgcnt_t obp_pages;      /* Memory used by PROM for its text and data */
 235 
 236 char *kobj_file_buf;
 237 int kobj_file_bufsize;  /* set in /etc/system */
 238 
 239 /* Global variables for MP support. Used in mp_startup */
 240 caddr_t rm_platter_va = 0;
 241 uint32_t rm_platter_pa;
 242 
 243 int     auto_lpg_disable = 1;
 244 
 245 /*
 246  * Some CPUs have holes in the middle of the 64-bit virtual address range.
 247  */
 248 uintptr_t hole_start, hole_end;
 249 
 250 /*
 251  * kpm mapping window
 252  */
 253 caddr_t kpm_vbase;

 313 struct seg kpseg;               /* Segment used for pageable kernel virt mem */
 314 struct seg kmapseg;             /* Segment used for generic kernel mappings */
 315 struct seg kdebugseg;           /* Segment used for the kernel debugger */
 316 
 317 struct seg *segkmap = &kmapseg;     /* Kernel generic mapping segment */
 318 static struct seg *segmap = &kmapseg;       /* easier to use name for in here */
 319 
 320 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
 321 
 322 #if defined(__amd64)
 323 struct seg kvseg_core;          /* Segment used for the core heap */
 324 struct seg kpmseg;              /* Segment used for physical mapping */
 325 struct seg *segkpm = &kpmseg;       /* 64bit kernel physical mapping segment */
 326 #else
 327 struct seg *segkpm = NULL;      /* Unused on IA32 */
 328 #endif
 329 
 330 caddr_t segkp_base;             /* Base address of segkp */
 331 caddr_t segzio_base;            /* Base address of segzio */
 332 #if defined(__amd64)
 333 pgcnt_t segkpsize = btop(SEGKPDEFSIZE); /* size of segkp segment in pages */

 334 #else
 335 pgcnt_t segkpsize = 0;
 336 #endif
 337 pgcnt_t segziosize = 0;         /* size of zio segment in pages */
 338 
 339 /*
 340  * A static DR page_t VA map is reserved that can map the page structures
 341  * for a domain's entire RA space. The pages that back this space are
 342  * dynamically allocated and need not be physically contiguous.  The DR
 343  * map size is derived from KPM size.
 344  * This mechanism isn't used by x86 yet, so just stubs here.
 345  */
 346 int ppvm_enable = 0;            /* Static virtual map for page structs */
 347 page_t *ppvm_base = NULL;       /* Base of page struct map */
 348 pgcnt_t ppvm_size = 0;          /* Size of page struct map */
 349 
 350 /*
 351  * VA range available to the debugger
 352  */
 353 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
 354 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
 355 

2027          * KERNEL_TEXT and below kernelbase) is dealt with correctly.
2028          * Note this may never happen, but it might someday.
2029          */
2030         bootpages = NULL;
2031         PRM_POINT("Protecting boot pages");
2032 
2033         /*
2034          * Protect any pages mapped above KERNEL_TEXT that somehow have
2035          * page_t's. This can only happen if something weird allocated
2036          * in this range (like kadb/kmdb).
2037          */
2038         protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
2039 
2040         /*
2041          * Before we can take over memory allocation/mapping from the boot
2042          * loader we must remove from our free page lists any boot allocated
2043          * pages that stay mapped until release_bootstrap().
2044          */
2045         protect_boot_range(0, kernelbase, 1);
2046 
2047 
2048         /*
2049          * Switch to running on regular HAT (not boot_mmu)
2050          */
2051         PRM_POINT("Calling hat_kern_setup()...");
2052         hat_kern_setup();
2053 
2054         /*
2055          * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
2056          */
2057         bop_no_more_mem();
2058 
2059         PRM_POINT("hat_kern_setup() done");
2060 
2061         hat_cpu_online(CPU);
2062 
2063         /*
2064          * Initialize VM system
2065          */
2066         PRM_POINT("Calling kvm_init()...");
2067         kvm_init();

2504                 (void) snprintf(propname, sizeof (propname),
2505                     "module-size-%u", i);
2506                 if (do_bsys_getproplen(NULL, propname) <= 0)
2507                         break;
2508                 (void) do_bsys_getprop(NULL, propname, &size);
2509 
2510                 modranges[i].phys = start;
2511                 modranges[i].size = size;
2512         }
2513 
2514         /* unmount boot ramdisk and release kmem usage */
2515         kobj_boot_unmountroot();
2516 
2517         /*
2518          * We're finished using the boot loader so free its pages.
2519          */
2520         PRM_POINT("Unmapping lower boot pages");
2521 
2522         clear_boot_mappings(0, _userlimit);
2523 










2524         postbootkernelbase = kernelbase;
2525 
2526         /*
2527          * If root isn't on ramdisk, destroy the hardcoded
2528          * ramdisk node now and release the memory. Else,
2529          * ramdisk memory is kept in rd_pages.
2530          */
2531         root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
2532         if (!root_is_ramdisk) {
2533                 dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
2534                 ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
2535                 ndi_rele_devi(dip);     /* held from ddi_find_devinfo */
2536                 (void) ddi_remove_child(dip, 0);
2537         }
2538 
2539         PRM_POINT("Releasing boot pages");
2540         while (bootpages) {
2541                 extern uint64_t ramdisk_start, ramdisk_end;
2542                 pp = bootpages;
2543                 bootpages = pp->p_next;

   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
  25  * Copyright 2017 Nexenta Systems, Inc.
  26  * Copyright 2015 Joyent, Inc.
  27  * Copyright (c) 2015 by Delphix. All rights reserved.
  28  */
  29 
  30 /*
  31  * Copyright (c) 2010, Intel Corporation.
  32  * All rights reserved.
  33  */
  34 
  35 #include <sys/types.h>
  36 #include <sys/t_lock.h>
  37 #include <sys/param.h>
  38 #include <sys/sysmacros.h>
  39 #include <sys/signal.h>
  40 #include <sys/systm.h>
  41 #include <sys/user.h>
  42 #include <sys/mman.h>
  43 #include <sys/vm.h>
  44 #include <sys/conf.h>
  45 #include <sys/avintr.h>
  46 #include <sys/autoconf.h>
  47 #include <sys/disp.h>
  48 #include <sys/class.h>
  49 #include <sys/bitmap.h>

 109 #include <sys/machsystm.h>
 110 #include <vm/hat.h>
 111 #include <vm/hat_i86.h>
 112 #include <sys/pmem.h>
 113 #include <sys/smp_impldefs.h>
 114 #include <sys/x86_archext.h>
 115 #include <sys/cpuvar.h>
 116 #include <sys/segments.h>
 117 #include <sys/clconf.h>
 118 #include <sys/kobj.h>
 119 #include <sys/kobj_lex.h>
 120 #include <sys/cpc_impl.h>
 121 #include <sys/cpu_module.h>
 122 #include <sys/smbios.h>
 123 #include <sys/debug_info.h>
 124 #include <sys/bootinfo.h>
 125 #include <sys/ddi_periodic.h>
 126 #include <sys/systeminfo.h>
 127 #include <sys/multiboot.h>
 128 #include <sys/ramdisk.h>
 129 #include <sys/framebuffer.h>
 130 
 131 #ifdef  __xpv
 132 
 133 #include <sys/hypervisor.h>
 134 #include <sys/xen_mmu.h>
 135 #include <sys/evtchn_impl.h>
 136 #include <sys/gnttab.h>
 137 #include <sys/xpv_panic.h>
 138 #include <xen/sys/xenbus_comms.h>
 139 #include <xen/public/physdev.h>
 140 
 141 extern void xen_late_startup(void);
 142 
 143 struct xen_evt_data cpu0_evt_data;
 144 
 145 #else   /* __xpv */
 146 #include <sys/memlist_impl.h>
 147 
 148 extern void mem_config_init(void);
 149 #endif /* __xpv */

 215  * kernelbase being set too high.
 216  */
 217 #define PHYSMEM 0x400000
 218 
 219 #else /* __amd64 */
 220 
 221 /*
 222  * For now we can handle memory with physical addresses up to about
 223  * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly
 224  * half the VA space for seg_kpm. When systems get bigger than 64TB this
 225  * code will need revisiting. There is an implicit assumption that there
 226  * are no *huge* holes in the physical address space too.
 227  */
 228 #define TERABYTE                (1ul << 40)
 229 #define PHYSMEM_MAX64           mmu_btop(64 * TERABYTE)
 230 #define PHYSMEM                 PHYSMEM_MAX64
 231 #define AMD64_VA_HOLE_END       0xFFFF800000000000ul
 232 
 233 #endif /* __amd64 */
 234 
 235 volatile pgcnt_t physmem = PHYSMEM;
 236 pgcnt_t obp_pages;      /* Memory used by PROM for its text and data */
 237 
 238 char *kobj_file_buf;
 239 int kobj_file_bufsize;  /* set in /etc/system */
 240 
 241 /* Global variables for MP support. Used in mp_startup */
 242 caddr_t rm_platter_va = 0;
 243 uint32_t rm_platter_pa;
 244 
 245 int     auto_lpg_disable = 1;
 246 
 247 /*
 248  * Some CPUs have holes in the middle of the 64-bit virtual address range.
 249  */
 250 uintptr_t hole_start, hole_end;
 251 
 252 /*
 253  * kpm mapping window
 254  */
 255 caddr_t kpm_vbase;

 315 struct seg kpseg;               /* Segment used for pageable kernel virt mem */
 316 struct seg kmapseg;             /* Segment used for generic kernel mappings */
 317 struct seg kdebugseg;           /* Segment used for the kernel debugger */
 318 
 319 struct seg *segkmap = &kmapseg;     /* Kernel generic mapping segment */
 320 static struct seg *segmap = &kmapseg;       /* easier to use name for in here */
 321 
 322 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
 323 
 324 #if defined(__amd64)
 325 struct seg kvseg_core;          /* Segment used for the core heap */
 326 struct seg kpmseg;              /* Segment used for physical mapping */
 327 struct seg *segkpm = &kpmseg;       /* 64bit kernel physical mapping segment */
 328 #else
 329 struct seg *segkpm = NULL;      /* Unused on IA32 */
 330 #endif
 331 
 332 caddr_t segkp_base;             /* Base address of segkp */
 333 caddr_t segzio_base;            /* Base address of segzio */
 334 #if defined(__amd64)
 335 volatile pgcnt_t segkpsize = btop(SEGKPDEFSIZE); /* size of segkp segment in */
 336                                                 /* pages */
 337 #else
 338 volatile pgcnt_t segkpsize = 0;
 339 #endif
 340 pgcnt_t segziosize = 0;         /* size of zio segment in pages */
 341 
 342 /*
 343  * A static DR page_t VA map is reserved that can map the page structures
 344  * for a domain's entire RA space. The pages that back this space are
 345  * dynamically allocated and need not be physically contiguous.  The DR
 346  * map size is derived from KPM size.
 347  * This mechanism isn't used by x86 yet, so just stubs here.
 348  */
 349 int ppvm_enable = 0;            /* Static virtual map for page structs */
 350 page_t *ppvm_base = NULL;       /* Base of page struct map */
 351 pgcnt_t ppvm_size = 0;          /* Size of page struct map */
 352 
 353 /*
 354  * VA range available to the debugger
 355  */
 356 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
 357 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
 358 

2030          * KERNEL_TEXT and below kernelbase) is dealt with correctly.
2031          * Note this may never happen, but it might someday.
2032          */
2033         bootpages = NULL;
2034         PRM_POINT("Protecting boot pages");
2035 
2036         /*
2037          * Protect any pages mapped above KERNEL_TEXT that somehow have
2038          * page_t's. This can only happen if something weird allocated
2039          * in this range (like kadb/kmdb).
2040          */
2041         protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
2042 
2043         /*
2044          * Before we can take over memory allocation/mapping from the boot
2045          * loader we must remove from our free page lists any boot allocated
2046          * pages that stay mapped until release_bootstrap().
2047          */
2048         protect_boot_range(0, kernelbase, 1);
2049 

2050         /*
2051          * Switch to running on regular HAT (not boot_mmu)
2052          */
2053         PRM_POINT("Calling hat_kern_setup()...");
2054         hat_kern_setup();
2055 
2056         /*
2057          * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
2058          */
2059         bop_no_more_mem();
2060 
2061         PRM_POINT("hat_kern_setup() done");
2062 
2063         hat_cpu_online(CPU);
2064 
2065         /*
2066          * Initialize VM system
2067          */
2068         PRM_POINT("Calling kvm_init()...");
2069         kvm_init();

2506                 (void) snprintf(propname, sizeof (propname),
2507                     "module-size-%u", i);
2508                 if (do_bsys_getproplen(NULL, propname) <= 0)
2509                         break;
2510                 (void) do_bsys_getprop(NULL, propname, &size);
2511 
2512                 modranges[i].phys = start;
2513                 modranges[i].size = size;
2514         }
2515 
2516         /* unmount boot ramdisk and release kmem usage */
2517         kobj_boot_unmountroot();
2518 
2519         /*
2520          * We're finished using the boot loader so free its pages.
2521          */
2522         PRM_POINT("Unmapping lower boot pages");
2523 
2524         clear_boot_mappings(0, _userlimit);
2525 
2526 #if 0
2527         if (fb_info.paddr != 0 && fb_info.fb_type != FB_TYPE_EGA_TEXT) {
2528                 clear_boot_mappings(fb_info.paddr,
2529                     P2ROUNDUP(fb_info.paddr + fb_info.fb_size, MMU_PAGESIZE));
2530                 clear_boot_mappings((uintptr_t)fb_info.fb,
2531                     P2ROUNDUP((uintptr_t)fb_info.fb + fb_info.fb_size,
2532                     MMU_PAGESIZE));
2533         }
2534 #endif
2535 
2536         postbootkernelbase = kernelbase;
2537 
2538         /*
2539          * If root isn't on ramdisk, destroy the hardcoded
2540          * ramdisk node now and release the memory. Else,
2541          * ramdisk memory is kept in rd_pages.
2542          */
2543         root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
2544         if (!root_is_ramdisk) {
2545                 dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
2546                 ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
2547                 ndi_rele_devi(dip);     /* held from ddi_find_devinfo */
2548                 (void) ddi_remove_child(dip, 0);
2549         }
2550 
2551         PRM_POINT("Releasing boot pages");
2552         while (bootpages) {
2553                 extern uint64_t ramdisk_start, ramdisk_end;
2554                 pp = bootpages;
2555                 bootpages = pp->p_next;