Print this page
NEX-17501 Enable parallel crash dump
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
re #13613 rb4516 Tunables needs volatile keyword
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/sun4/os/startup.c
+++ new/usr/src/uts/sun4/os/startup.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 + * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
24 25 * Copyright (c) 2016 by Delphix. All rights reserved.
25 26 */
26 27
27 28 #include <sys/machsystm.h>
28 29 #include <sys/archsystm.h>
29 30 #include <sys/vm.h>
30 31 #include <sys/cpu.h>
31 32 #include <sys/atomic.h>
32 33 #include <sys/reboot.h>
33 34 #include <sys/kdi.h>
34 35 #include <sys/bootconf.h>
35 36 #include <sys/memlist_plat.h>
36 37 #include <sys/memlist_impl.h>
37 38 #include <sys/prom_plat.h>
38 39 #include <sys/prom_isa.h>
39 40 #include <sys/autoconf.h>
40 41 #include <sys/ivintr.h>
41 42 #include <sys/fpu/fpusystm.h>
42 43 #include <sys/iommutsb.h>
43 44 #include <vm/vm_dep.h>
44 45 #include <vm/seg_dev.h>
45 46 #include <vm/seg_kmem.h>
46 47 #include <vm/seg_kpm.h>
47 48 #include <vm/seg_map.h>
48 49 #include <vm/seg_kp.h>
49 50 #include <sys/sysconf.h>
50 51 #include <vm/hat_sfmmu.h>
51 52 #include <sys/kobj.h>
52 53 #include <sys/sun4asi.h>
53 54 #include <sys/clconf.h>
54 55 #include <sys/platform_module.h>
55 56 #include <sys/panic.h>
56 57 #include <sys/cpu_sgnblk_defs.h>
57 58 #include <sys/clock.h>
58 59 #include <sys/cmn_err.h>
59 60 #include <sys/dumphdr.h>
60 61 #include <sys/promif.h>
61 62 #include <sys/prom_debug.h>
62 63 #include <sys/traptrace.h>
63 64 #include <sys/memnode.h>
64 65 #include <sys/mem_cage.h>
65 66 #include <sys/mmu.h>
66 67 #include <sys/swap.h>
67 68
68 69 extern void setup_trap_table(void);
69 70 extern int cpu_intrq_setup(struct cpu *);
70 71 extern void cpu_intrq_register(struct cpu *);
71 72 extern void contig_mem_init(void);
72 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
73 74 extern void mach_dump_buffer_init(void);
74 75 extern void mach_descrip_init(void);
75 76 extern void mach_descrip_startup_fini(void);
76 77 extern void mach_memscrub(void);
77 78 extern void mach_fpras(void);
78 79 extern void mach_cpu_halt_idle(void);
79 80 extern void mach_hw_copy_limit(void);
80 81 extern void load_mach_drivers(void);
81 82 extern void load_tod_module(void);
82 83 #pragma weak load_tod_module
83 84
84 85 extern int ndata_alloc_mmfsa(struct memlist *ndata);
85 86 #pragma weak ndata_alloc_mmfsa
86 87
87 88 extern void cif_init(void);
88 89 #pragma weak cif_init
89 90
90 91 extern void parse_idprom(void);
91 92 extern void add_vx_handler(char *, int, void (*)(cell_t *));
92 93 extern void mem_config_init(void);
93 94 extern void memseg_remap_init(void);
94 95
95 96 extern void mach_kpm_init(void);
96 97 extern void pcf_init();
97 98 extern int size_pse_array(pgcnt_t, int);
98 99 extern void pg_init();
99 100
100 101 /*
101 102 * External Data:
102 103 */
103 104 extern int vac_size; /* cache size in bytes */
104 105 extern uint_t vac_mask; /* VAC alignment consistency mask */
105 106 extern uint_t vac_colors;
106 107
107 108 /*
108 109 * Global Data Definitions:
109 110 */
110 111
|
↓ open down ↓ |
77 lines elided |
↑ open up ↑ |
111 112 /*
112 113 * XXX - Don't port this to new architectures
113 114 * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
114 115 * 'romp' has no use with a prom with an IEEE 1275 client interface.
115 116 * The driver doesn't use the value, but it depends on the symbol.
116 117 */
117 118 void *romp; /* veritas driver won't load without romp 4154976 */
118 119 /*
119 120 * Declare these as initialized data so we can patch them.
120 121 */
121 -pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */
122 -pgcnt_t segkpsize =
122 +volatile pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */
123 +volatile pgcnt_t segkpsize =
123 124 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */
124 -uint_t segmap_percent = 6; /* Size of segmap segment */
125 +volatile uint_t segmap_percent = 6; /* Size of segmap segment */
125 126
126 127 int use_cache = 1; /* cache not reliable (605 bugs) with MP */
127 128 int vac_copyback = 1;
128 129 char *cache_mode = NULL;
129 130 int use_mix = 1;
130 131 int prom_debug = 0;
131 132
132 133 caddr_t boot_tba; /* %tba at boot - used by kmdb */
133 134 uint_t tba_taken_over = 0;
134 135
135 136 caddr_t s_text; /* start of kernel text segment */
136 137 caddr_t e_text; /* end of kernel text segment */
137 138 caddr_t s_data; /* start of kernel data segment */
138 139 caddr_t e_data; /* end of kernel data segment */
139 140
140 141 caddr_t modtext; /* beginning of module text */
141 142 size_t modtext_sz; /* size of module text */
142 143 caddr_t moddata; /* beginning of module data reserve */
143 144 caddr_t e_moddata; /* end of module data reserve */
144 145
145 146 /*
146 147 * End of first block of contiguous kernel in 32-bit virtual address space
147 148 */
148 149 caddr_t econtig32; /* end of first blk of contiguous kernel */
149 150
150 151 caddr_t ncbase; /* beginning of non-cached segment */
151 152 caddr_t ncend; /* end of non-cached segment */
152 153
153 154 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */
154 155 caddr_t nalloc_base; /* beginning of nucleus allocation */
155 156 caddr_t nalloc_end; /* end of nucleus allocatable memory */
156 157 caddr_t valloc_base; /* beginning of kvalloc segment */
157 158
158 159 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */
159 160 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */
160 161 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */
161 162 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */
162 163 int kmem64_szc; /* page size code */
163 164 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */
164 165
165 166 uintptr_t shm_alignment; /* VAC address consistency modulus */
166 167 struct memlist *phys_install; /* Total installed physical memory */
167 168 struct memlist *phys_avail; /* Available (unreserved) physical memory */
168 169 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */
169 170 struct memlist *nopp_list; /* pages with no backing page structs */
170 171 struct memlist ndata; /* memlist of nucleus allocatable memory */
171 172 int memexp_flag; /* memory expansion card flag */
172 173 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */
173 174 pgcnt_t obp_pages; /* Physical pages used by OBP */
174 175
175 176 /*
176 177 * VM data structures
177 178 */
178 179 long page_hashsz; /* Size of page hash table (power of two) */
179 180 unsigned int page_hashsz_shift; /* log2(page_hashsz) */
180 181 struct page *pp_base; /* Base of system page struct array */
181 182 size_t pp_sz; /* Size in bytes of page struct array */
182 183 struct page **page_hash; /* Page hash table */
183 184 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */
184 185 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */
185 186 int pse_shift; /* log2(pse_table_size) */
186 187 struct seg ktextseg; /* Segment used for kernel executable image */
187 188 struct seg kvalloc; /* Segment used for "valloc" mapping */
188 189 struct seg kpseg; /* Segment used for pageable kernel virt mem */
189 190 struct seg ktexthole; /* Segment used for nucleus text hole */
190 191 struct seg kmapseg; /* Segment used for generic kernel mappings */
191 192 struct seg kpmseg; /* Segment used for physical mapping */
192 193 struct seg kdebugseg; /* Segment used for the kernel debugger */
193 194
194 195 void *kpm_pp_base; /* Base of system kpm_page array */
195 196 size_t kpm_pp_sz; /* Size of system kpm_page array */
196 197 pgcnt_t kpm_npages; /* How many kpm pages are managed */
197 198
198 199 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
199 200 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */
200 201 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */
201 202
202 203 int segzio_fromheap = 0; /* zio allocations occur from heap */
203 204 caddr_t segzio_base; /* Base address of segzio */
204 205 pgcnt_t segziosize = 0; /* size of zio segment in pages */
205 206
206 207 /*
207 208 * A static DR page_t VA map is reserved that can map the page structures
208 209 * for a domain's entire RA space. The pages that backs this space are
209 210 * dynamically allocated and need not be physically contiguous. The DR
210 211 * map size is derived from KPM size.
211 212 */
212 213 int ppvm_enable = 0; /* Static virtual map for page structs */
213 214 page_t *ppvm_base; /* Base of page struct map */
214 215 pgcnt_t ppvm_size = 0; /* Size of page struct map */
215 216
216 217 /*
217 218 * debugger pages (if allocated)
218 219 */
219 220 struct vnode kdebugvp;
220 221
221 222 /*
222 223 * VA range available to the debugger
223 224 */
224 225 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
225 226 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
226 227
227 228 /*
228 229 * Segment for relocated kernel structures in 64-bit large RAM kernels
229 230 */
230 231 struct seg kmem64;
231 232
232 233 struct memseg *memseg_free;
233 234
234 235 struct vnode unused_pages_vp;
235 236
236 237 /*
237 238 * VM data structures allocated early during boot.
238 239 */
239 240 size_t pagehash_sz;
240 241 uint64_t memlist_sz;
241 242
242 243 char tbr_wr_addr_inited = 0;
243 244
244 245 caddr_t mpo_heap32_buf = NULL;
245 246 size_t mpo_heap32_bufsz = 0;
246 247
247 248 /*
248 249 * Static Routines:
249 250 */
250 251 static int ndata_alloc_memseg(struct memlist *, size_t);
251 252 static void memlist_new(uint64_t, uint64_t, struct memlist **);
252 253 static void memlist_add(uint64_t, uint64_t,
253 254 struct memlist **, struct memlist **);
254 255 static void kphysm_init(void);
255 256 static void kvm_init(void);
256 257 static void install_kmem64_tte(void);
257 258
258 259 static void startup_init(void);
259 260 static void startup_memlist(void);
260 261 static void startup_modules(void);
261 262 static void startup_bop_gone(void);
262 263 static void startup_vm(void);
263 264 static void startup_end(void);
264 265 static void setup_cage_params(void);
265 266 static void startup_create_io_node(void);
266 267
267 268 static pgcnt_t npages;
268 269 static struct memlist *memlist;
269 270 void *memlist_end;
270 271
271 272 static pgcnt_t bop_alloc_pages;
272 273 static caddr_t hblk_base;
273 274 uint_t hblk_alloc_dynamic = 0;
274 275 uint_t hblk1_min = H1MIN;
275 276
276 277
277 278 /*
278 279 * Hooks for unsupported platforms and down-rev firmware
279 280 */
280 281 int iam_positron(void);
281 282 #pragma weak iam_positron
282 283 static void do_prom_version_check(void);
283 284
284 285 /*
285 286 * After receiving a thermal interrupt, this is the number of seconds
286 287 * to delay before shutting off the system, assuming
287 288 * shutdown fails. Use /etc/system to change the delay if this isn't
288 289 * large enough.
289 290 */
290 291 int thermal_powerdown_delay = 1200;
291 292
292 293 /*
293 294 * Used to hold off page relocations into the cage until OBP has completed
294 295 * its boot-time handoff of its resources to the kernel.
295 296 */
296 297 int page_relocate_ready = 0;
297 298
298 299 /*
299 300 * Indicate if kmem64 allocation was done in small chunks
300 301 */
301 302 int kmem64_smchunks = 0;
302 303
303 304 /*
304 305 * Enable some debugging messages concerning memory usage...
305 306 */
306 307 #ifdef DEBUGGING_MEM
307 308 static int debugging_mem;
308 309 static void
309 310 printmemlist(char *title, struct memlist *list)
310 311 {
311 312 if (!debugging_mem)
312 313 return;
313 314
314 315 printf("%s\n", title);
315 316
316 317 while (list) {
317 318 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
318 319 (uint32_t)(list->ml_address >> 32),
319 320 (uint32_t)list->ml_address,
320 321 (uint32_t)(list->ml_size >> 32),
321 322 (uint32_t)(list->ml_size));
322 323 list = list->ml_next;
323 324 }
324 325 }
325 326
326 327 void
327 328 printmemseg(struct memseg *memseg)
328 329 {
329 330 if (!debugging_mem)
330 331 return;
331 332
332 333 printf("memseg\n");
333 334
334 335 while (memseg) {
335 336 prom_printf("\tpage = 0x%p, epage = 0x%p, "
336 337 "pfn = 0x%x, epfn = 0x%x\n",
337 338 memseg->pages, memseg->epages,
338 339 memseg->pages_base, memseg->pages_end);
339 340 memseg = memseg->next;
340 341 }
341 342 }
342 343
343 344 #define debug_pause(str) halt((str))
344 345 #define MPRINTF(str) if (debugging_mem) prom_printf((str))
345 346 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a))
346 347 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b))
347 348 #define MPRINTF3(str, a, b, c) \
348 349 if (debugging_mem) prom_printf((str), (a), (b), (c))
349 350 #else /* DEBUGGING_MEM */
350 351 #define MPRINTF(str)
351 352 #define MPRINTF1(str, a)
352 353 #define MPRINTF2(str, a, b)
353 354 #define MPRINTF3(str, a, b, c)
354 355 #endif /* DEBUGGING_MEM */
355 356
356 357
357 358 /*
358 359 *
359 360 * Kernel's Virtual Memory Layout.
360 361 * /-----------------------\
361 362 * 0xFFFFFFFF.FFFFFFFF -| |-
362 363 * | OBP's virtual page |
363 364 * | tables |
364 365 * 0xFFFFFFFC.00000000 -|-----------------------|-
365 366 * : :
366 367 * : :
367 368 * -|-----------------------|-
368 369 * | segzio | (base and size vary)
369 370 * 0xFFFFFE00.00000000 -|-----------------------|-
370 371 * | | Ultrasparc I/II support
371 372 * | segkpm segment | up to 2TB of physical
372 373 * | (64-bit kernel ONLY) | memory, VAC has 2 colors
373 374 * | |
374 375 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment
375 376 * : :
376 377 * : :
377 378 * 0xFFFFF810.00000000 -|-----------------------|- hole_end
378 379 * | | ^
379 380 * | UltraSPARC I/II call | |
380 381 * | bug requires an extra | |
381 382 * | 4 GB of space between | |
382 383 * | hole and used RAM | |
383 384 * | | |
384 385 * 0xFFFFF800.00000000 -|-----------------------|- |
385 386 * | | |
386 387 * | Virtual Address Hole | UltraSPARC
387 388 * | on UltraSPARC I/II | I/II * ONLY *
388 389 * | | |
389 390 * 0x00000800.00000000 -|-----------------------|- |
390 391 * | | |
391 392 * | UltraSPARC I/II call | |
392 393 * | bug requires an extra | |
393 394 * | 4 GB of space between | |
394 395 * | hole and used RAM | |
395 396 * | | v
396 397 * 0x000007FF.00000000 -|-----------------------|- hole_start -----
397 398 * : : ^
398 399 * : : |
399 400 * |-----------------------| |
400 401 * | | |
401 402 * | ecache flush area | |
402 403 * | (twice largest e$) | |
403 404 * | | |
404 405 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ |
405 406 * | overmapped area | alignend_end |
406 407 * | (kmem64_alignsize | |
407 408 * | boundary) | |
408 409 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end |
409 410 * | | |
410 411 * | 64-bit kernel ONLY | |
411 412 * | | |
412 413 * | kmem64 segment | |
413 414 * | | |
414 415 * | (Relocated extra HME | Approximately
415 416 * | block allocations, | 1 TB of virtual
416 417 * | memnode freelists, | address space
417 418 * | HME hash buckets, | |
418 419 * | mml_table, kpmp_table,| |
419 420 * | page_t array and | |
420 421 * | hashblock pool to | |
421 422 * | avoid hard-coded | |
422 423 * | 32-bit vaddr | |
423 424 * | limitations) | |
424 425 * | | v
425 426 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base)
426 427 * | |
427 428 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB)
428 429 * | |
429 430 * 0x00000300.00000000 -|-----------------------|- SYSBASE
430 431 * : :
431 432 * : :
432 433 * -|-----------------------|-
433 434 * | |
434 435 * | segmap segment | SEGMAPSIZE (1/8th physmem,
435 436 * | | 256G MAX)
436 437 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE
437 438 * : :
438 439 * : :
439 440 * -|-----------------------|-
440 441 * | |
441 442 * | segkp | SEGKPSIZE (2GB)
442 443 * | |
443 444 * | |
444 445 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE
445 446 * | |
446 447 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE
447 448 * | | (SEGKPBASE - 0x400000)
448 449 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE
449 450 * | | (MEMSCRUBBASE - NCARGS)
450 451 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE
451 452 * | | (ARGSBASE - PPMAPSIZE)
452 453 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE
453 454 * | |
454 455 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE
455 456 * | |
456 457 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE
457 458 * : :
458 459 * : :
459 460 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR
460 461 * | |
461 462 * | OBP |
462 463 * | |
463 464 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR
464 465 * | kmdb |
465 466 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE
466 467 * : :
467 468 * : :
468 469 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32
469 470 * | |
470 471 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 =
471 472 * | | ~64MB)
472 473 * -|-----------------------|
473 474 * | IVSIZE |
474 475 * 0x00000000.70004000 -|-----------------------|
475 476 * | panicbuf |
476 477 * 0x00000000.70002000 -|-----------------------|
477 478 * | PAGESIZE |
478 479 * 0x00000000.70000000 -|-----------------------|- SYSBASE32
479 480 * | boot-time |
480 481 * | temporary space |
481 482 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE
482 483 * : :
483 484 * : :
484 485 * | |
485 486 * |-----------------------|- econtig32
486 487 * | vm structures |
487 488 * 0x00000000.01C00000 |-----------------------|- nalloc_end
488 489 * | TSBs |
489 490 * |-----------------------|- end/nalloc_base
490 491 * | kernel data & bss |
491 492 * 0x00000000.01800000 -|-----------------------|
492 493 * : nucleus text hole :
493 494 * 0x00000000.01400000 -|-----------------------|
494 495 * : :
495 496 * |-----------------------|
496 497 * | module text |
497 498 * |-----------------------|- e_text/modtext
498 499 * | kernel text |
499 500 * |-----------------------|
500 501 * | trap table (48k) |
501 502 * 0x00000000.01000000 -|-----------------------|- KERNELBASE
502 503 * | reserved for trapstat |} TSTAT_TOTAL_SIZE
503 504 * |-----------------------|
504 505 * | |
505 506 * | invalid |
506 507 * | |
507 508 * 0x00000000.00000000 _|_______________________|
508 509 *
509 510 *
510 511 *
511 512 * 32-bit User Virtual Memory Layout.
512 513 * /-----------------------\
513 514 * | |
514 515 * | invalid |
515 516 * | |
516 517 * 0xFFC00000 -|-----------------------|- USERLIMIT
517 518 * | user stack |
518 519 * : :
519 520 * : :
520 521 * : :
521 522 * | user data |
522 523 * -|-----------------------|-
523 524 * | user text |
524 525 * 0x00002000 -|-----------------------|-
525 526 * | invalid |
526 527 * 0x00000000 _|_______________________|
527 528 *
528 529 *
529 530 *
530 531 * 64-bit User Virtual Memory Layout.
531 532 * /-----------------------\
532 533 * | |
533 534 * | invalid |
534 535 * | |
535 536 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
536 537 * | user stack |
537 538 * : :
538 539 * : :
539 540 * : :
540 541 * | user data |
541 542 * -|-----------------------|-
542 543 * | user text |
543 544 * 0x00000000.01000000 -|-----------------------|-
544 545 * | invalid |
545 546 * 0x00000000.00000000 _|_______________________|
546 547 */
547 548
548 549 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
549 550 extern uint64_t ecache_flush_address(void);
550 551
551 552 #pragma weak load_platform_modules
552 553 #pragma weak plat_startup_memlist
553 554 #pragma weak ecache_init_scrub_flush_area
554 555 #pragma weak ecache_flush_address
555 556
556 557
557 558 /*
558 559 * By default the DR Cage is enabled for maximum OS
559 560 * MPSS performance. Users needing to disable the cage mechanism
560 561 * can set this variable to zero via /etc/system.
561 562 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
562 563 * will result in loss of DR functionality.
563 564 * Platforms wishing to disable kernel Cage by default
564 565 * should do so in their set_platform_defaults() routine.
565 566 */
566 567 int kernel_cage_enable = 1;
567 568
568 569 static void
569 570 setup_cage_params(void)
570 571 {
571 572 void (*func)(void);
572 573
573 574 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
574 575 if (func != NULL) {
575 576 (*func)();
576 577 return;
577 578 }
578 579
579 580 if (kernel_cage_enable == 0) {
580 581 return;
581 582 }
582 583 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
583 584
584 585 if (kcage_on) {
585 586 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
586 587 } else {
587 588 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
588 589 }
589 590
590 591 }
591 592
592 593 /*
593 594 * Machine-dependent startup code
594 595 */
595 596 void
596 597 startup(void)
597 598 {
598 599 startup_init();
599 600 if (&startup_platform)
600 601 startup_platform();
601 602 startup_memlist();
602 603 startup_modules();
603 604 setup_cage_params();
604 605 startup_bop_gone();
605 606 startup_vm();
606 607 startup_end();
607 608 }
608 609
609 610 struct regs sync_reg_buf;
610 611 uint64_t sync_tt;
611 612
612 613 void
613 614 sync_handler(void)
614 615 {
615 616 struct panic_trap_info ti;
616 617 int i;
617 618
618 619 /*
619 620 * Prevent trying to talk to the other CPUs since they are
620 621 * sitting in the prom and won't reply.
621 622 */
622 623 for (i = 0; i < NCPU; i++) {
|
↓ open down ↓ |
488 lines elided |
↑ open up ↑ |
623 624 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
624 625 cpu[i]->cpu_flags &= ~CPU_READY;
625 626 cpu[i]->cpu_flags |= CPU_QUIESCED;
626 627 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
627 628 }
628 629 }
629 630
630 631 /*
631 632 * Force a serial dump, since there are no CPUs to help.
632 633 */
633 - dump_plat_mincpu = 0;
634 + dump_ncpu_low = 0;
634 635
635 636 /*
636 637 * We've managed to get here without going through the
637 638 * normal panic code path. Try and save some useful
638 639 * information.
639 640 */
640 641 if (!panicstr && (curthread->t_panic_trap == NULL)) {
641 642 ti.trap_type = sync_tt;
642 643 ti.trap_regs = &sync_reg_buf;
643 644 ti.trap_addr = NULL;
644 645 ti.trap_mmu_fsr = 0x0;
645 646
646 647 curthread->t_panic_trap = &ti;
647 648 }
648 649
649 650 /*
650 651 * If we're re-entering the panic path, update the signature
651 652 * block so that the SC knows we're in the second part of panic.
652 653 */
653 654 if (panicstr)
654 655 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
655 656
656 657 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
657 658 panic("sync initiated");
658 659 }
659 660
660 661
661 662 static void
662 663 startup_init(void)
663 664 {
664 665 /*
665 666 * We want to save the registers while we're still in OBP
666 667 * so that we know they haven't been fiddled with since.
667 668 * (In principle, OBP can't change them just because it
668 669 * makes a callback, but we'd rather not depend on that
669 670 * behavior.)
670 671 */
671 672 char sync_str[] =
672 673 "warning @ warning off : sync "
673 674 "%%tl-c %%tstate h# %p x! "
674 675 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
675 676 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
676 677 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
677 678 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
678 679 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
679 680 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
680 681 "%%y h# %p l! %%tl-c %%tt h# %p x! "
681 682 "sync ; warning !";
682 683
683 684 /*
684 685 * 20 == num of %p substrings
685 686 * 16 == max num of chars %p will expand to.
686 687 */
687 688 char bp[sizeof (sync_str) + 16 * 20];
688 689
689 690 /*
690 691 * Initialize ptl1 stack for the 1st CPU.
691 692 */
692 693 ptl1_init_cpu(&cpu0);
693 694
694 695 /*
695 696 * Initialize the address map for cache consistent mappings
696 697 * to random pages; must be done after vac_size is set.
697 698 */
698 699 ppmapinit();
699 700
700 701 /*
701 702 * Initialize the PROM callback handler.
702 703 */
703 704 init_vx_handler();
704 705
705 706 /*
706 707 * have prom call sync_callback() to handle the sync and
707 708 * save some useful information which will be stored in the
708 709 * core file later.
709 710 */
710 711 (void) sprintf((char *)bp, sync_str,
711 712 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
712 713 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
713 714 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
714 715 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
715 716 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
716 717 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
717 718 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
718 719 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
719 720 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
720 721 (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
721 722 prom_interpret(bp, 0, 0, 0, 0, 0);
722 723 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
723 724 }
724 725
725 726
726 727 size_t
727 728 calc_pp_sz(pgcnt_t npages)
728 729 {
729 730
730 731 return (npages * sizeof (struct page));
731 732 }
732 733
733 734 size_t
734 735 calc_kpmpp_sz(pgcnt_t npages)
735 736 {
736 737
737 738 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
738 739 kpm_pgsz = 1ull << kpm_pgshft;
739 740 kpm_pgoff = kpm_pgsz - 1;
740 741 kpmp2pshft = kpm_pgshft - PAGESHIFT;
741 742 kpmpnpgs = 1 << kpmp2pshft;
742 743
743 744 if (kpm_smallpages == 0) {
744 745 /*
745 746 * Avoid fragmentation problems in kphysm_init()
746 747 * by allocating for all of physical memory
747 748 */
748 749 kpm_npages = ptokpmpr(physinstalled);
749 750 return (kpm_npages * sizeof (kpm_page_t));
750 751 } else {
751 752 kpm_npages = npages;
752 753 return (kpm_npages * sizeof (kpm_spage_t));
753 754 }
754 755 }
755 756
756 757 size_t
757 758 calc_pagehash_sz(pgcnt_t npages)
758 759 {
759 760 /* LINTED */
760 761 ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
761 762 /*
762 763 * The page structure hash table size is a power of 2
763 764 * such that the average hash chain length is PAGE_HASHAVELEN.
764 765 */
765 766 page_hashsz = npages / PAGE_HASHAVELEN;
766 767 page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
767 768 highbit(page_hashsz));
768 769 page_hashsz = 1 << page_hashsz_shift;
769 770 return (page_hashsz * sizeof (struct page *));
770 771 }
771 772
772 773 int testkmem64_smchunks = 0;
773 774
774 775 int
775 776 alloc_kmem64(caddr_t base, caddr_t end)
776 777 {
777 778 int i;
778 779 caddr_t aligned_end = NULL;
779 780
780 781 if (testkmem64_smchunks)
781 782 return (1);
782 783
783 784 /*
784 785 * Make one large memory alloc after figuring out the 64-bit size. This
785 786 * will enable use of the largest page size appropriate for the system
786 787 * architecture.
787 788 */
788 789 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
789 790 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
790 791 for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
791 792 size_t alloc_size, alignsize;
792 793 #if !defined(C_OBP)
793 794 unsigned long long pa;
794 795 #endif /* !C_OBP */
795 796
796 797 if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
797 798 continue;
798 799 alignsize = TTEBYTES(i);
799 800 kmem64_szc = i;
800 801
801 802 /* limit page size for small memory */
802 803 if (mmu_btop(alignsize) > (npages >> 2))
803 804 continue;
804 805
805 806 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
806 807 alloc_size = aligned_end - base;
807 808 #if !defined(C_OBP)
808 809 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
809 810 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
810 811 kmem64_pabase = pa;
811 812 kmem64_aligned_end = aligned_end;
812 813 install_kmem64_tte();
813 814 break;
814 815 } else {
815 816 prom_free_phys(alloc_size, pa);
816 817 }
817 818 }
818 819 #else /* !C_OBP */
819 820 if (prom_alloc(base, alloc_size, alignsize) == base) {
820 821 kmem64_pabase = va_to_pa(kmem64_base);
821 822 kmem64_aligned_end = aligned_end;
822 823 break;
823 824 }
824 825 #endif /* !C_OBP */
825 826 if (i == TTE8K) {
826 827 #ifdef sun4v
827 828 /* return failure to try small allocations */
828 829 return (1);
829 830 #else
830 831 prom_panic("kmem64 allocation failure");
831 832 #endif
832 833 }
833 834 }
834 835 ASSERT(aligned_end != NULL);
835 836 return (0);
836 837 }
837 838
838 839 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
839 840 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
840 841
841 842 #if !defined(C_OBP)
842 843 /*
843 844 * Install a temporary tte handler in OBP for kmem64 area.
844 845 *
845 846 * We map kmem64 area with large pages before the trap table is taken
846 847 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
847 848 * the same area. Duplicate tlb entries with different page sizes
848 849 * cause unpredicatble behavior. To avoid this, we don't create
849 850 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
850 851 * OBP). Instead, we manage translations with a temporary va>tte-data
851 852 * handler (kmem64-tte). This handler is replaced by unix-tte when
852 853 * the trap table is taken over.
853 854 *
854 855 * The temporary handler knows the physical address of the kmem64
855 856 * area. It uses the prom's pgmap@ Forth word for other addresses.
856 857 *
857 858 * We have to use BOP_ALLOC() method for C-OBP platforms because
858 859 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
859 860 * sun4u platforms. On sun4u we flush tlb after trap table is taken
860 861 * over if we use large pages for kernel heap and kmem64. Since sun4u
861 862 * prom (unlike sun4v) calls va>tte-data first for client address
862 863 * translation prom's ttes for kmem64 can't get into TLB even if we
863 864 * later switch to prom's trap table again. C-OBP uses 4M pages for
864 865 * client mappings when possible so on all platforms we get the
865 866 * benefit from large mappings for kmem64 area immediately during
866 867 * boot.
867 868 *
868 869 * pseudo code:
869 870 * if (context != 0) {
870 871 * return false
871 872 * } else if (miss_va in range[kmem64_base, kmem64_end)) {
872 873 * tte = tte_template +
873 874 * (((miss_va & pagemask) - kmem64_base));
874 875 * return tte, true
875 876 * } else {
876 877 * return pgmap@ result
877 878 * }
878 879 */
879 880 char kmem64_obp_str[] =
880 881 "h# %lx constant kmem64-base "
881 882 "h# %lx constant kmem64-end "
882 883 "h# %lx constant kmem64-pagemask "
883 884 "h# %lx constant kmem64-template "
884 885
885 886 ": kmem64-tte ( addr cnum -- false | tte-data true ) "
886 887 " if ( addr ) "
887 888 " drop false exit then ( false ) "
888 889 " dup kmem64-base kmem64-end within if ( addr ) "
889 890 " kmem64-pagemask and ( addr' ) "
890 891 " kmem64-base - ( addr' ) "
891 892 " kmem64-template + ( tte ) "
892 893 " true ( tte true ) "
893 894 " else ( addr ) "
894 895 " pgmap@ ( tte ) "
895 896 " dup 0< if true else drop false then ( tte true | false ) "
896 897 " then ( tte true | false ) "
897 898 "; "
898 899
899 900 "' kmem64-tte is va>tte-data "
900 901 ;
901 902
902 903 static void
903 904 install_kmem64_tte()
904 905 {
905 906 char b[sizeof (kmem64_obp_str) + (4 * 16)];
906 907 tte_t tte;
907 908
908 909 PRM_DEBUG(kmem64_pabase);
909 910 PRM_DEBUG(kmem64_szc);
910 911 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
911 912 PROC_DATA | HAT_NOSYNC, kmem64_szc);
912 913 PRM_DEBUG(tte.ll);
913 914 (void) sprintf(b, kmem64_obp_str,
914 915 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
915 916 ASSERT(strlen(b) < sizeof (b));
916 917 prom_interpret(b, 0, 0, 0, 0, 0);
917 918 }
918 919 #endif /* !C_OBP */
919 920
920 921 /*
921 922 * As OBP takes up some RAM when the system boots, pages will already be "lost"
922 923 * to the system and reflected in npages by the time we see it.
923 924 *
924 925 * We only want to allocate kernel structures in the 64-bit virtual address
925 926 * space on systems with enough RAM to make the overhead of keeping track of
926 927 * an extra kernel memory segment worthwhile.
927 928 *
928 929 * Since OBP has already performed its memory allocations by this point, if we
929 930 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
930 931 * memory in the 64-bit virtual address space; otherwise keep allocations
931 932 * contiguous with we've mapped so far in the 32-bit virtual address space.
932 933 */
933 934 #define MINMOVE_RAM_MB ((size_t)1900)
934 935 #define MB_TO_BYTES(mb) ((mb) * 1048576ul)
935 936 #define BYTES_TO_MB(b) ((b) / 1048576ul)
936 937
937 938 pgcnt_t tune_npages = (pgcnt_t)
938 939 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
939 940
940 941 #pragma weak page_set_colorequiv_arr_cpu
941 942 extern void page_set_colorequiv_arr_cpu(void);
942 943 extern void page_set_colorequiv_arr(void);
943 944
944 945 static pgcnt_t ramdisk_npages;
945 946 static struct memlist *old_phys_avail;
946 947
947 948 kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
948 949
949 950 static void
950 951 startup_memlist(void)
951 952 {
952 953 size_t hmehash_sz, pagelist_sz, tt_sz;
953 954 size_t psetable_sz;
954 955 caddr_t alloc_base;
955 956 caddr_t memspace;
956 957 struct memlist *cur;
957 958 size_t syslimit = (size_t)SYSLIMIT;
958 959 size_t sysbase = (size_t)SYSBASE;
959 960
960 961 /*
961 962 * Initialize enough of the system to allow kmem_alloc to work by
962 963 * calling boot to allocate its memory until the time that
963 964 * kvm_init is completed. The page structs are allocated after
964 965 * rounding up end to the nearest page boundary; the memsegs are
965 966 * initialized and the space they use comes from the kernel heap.
966 967 * With appropriate initialization, they can be reallocated later
967 968 * to a size appropriate for the machine's configuration.
968 969 *
969 970 * At this point, memory is allocated for things that will never
970 971 * need to be freed, this used to be "valloced". This allows a
971 972 * savings as the pages don't need page structures to describe
972 973 * them because them will not be managed by the vm system.
973 974 */
974 975
975 976 /*
976 977 * We're loaded by boot with the following configuration (as
977 978 * specified in the sun4u/conf/Mapfile):
978 979 *
979 980 * text: 4 MB chunk aligned on a 4MB boundary
980 981 * data & bss: 4 MB chunk aligned on a 4MB boundary
981 982 *
982 983 * These two chunks will eventually be mapped by 2 locked 4MB
983 984 * ttes and will represent the nucleus of the kernel. This gives
984 985 * us some free space that is already allocated, some or all of
985 986 * which is made available to kernel module text.
986 987 *
987 988 * The free space in the data-bss chunk is used for nucleus
988 989 * allocatable data structures and we reserve it using the
989 990 * nalloc_base and nalloc_end variables. This space is currently
990 991 * being used for hat data structures required for tlb miss
991 992 * handling operations. We align nalloc_base to a l2 cache
992 993 * linesize because this is the line size the hardware uses to
993 994 * maintain cache coherency.
994 995 * 512K is carved out for module data.
995 996 */
996 997
997 998 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
998 999 e_moddata = moddata + MODDATA;
999 1000 nalloc_base = e_moddata;
1000 1001
1001 1002 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
1002 1003 valloc_base = nalloc_base;
1003 1004
1004 1005 /*
1005 1006 * Calculate the start of the data segment.
1006 1007 */
1007 1008 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1008 1009 prom_panic("nucleus data overflow");
1009 1010
1010 1011 PRM_DEBUG(moddata);
1011 1012 PRM_DEBUG(nalloc_base);
1012 1013 PRM_DEBUG(nalloc_end);
1013 1014
1014 1015 /*
1015 1016 * Remember any slop after e_text so we can give it to the modules.
1016 1017 */
1017 1018 PRM_DEBUG(e_text);
1018 1019 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1019 1020 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1020 1021 prom_panic("nucleus text overflow");
1021 1022 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1022 1023 modtext;
1023 1024 PRM_DEBUG(modtext);
1024 1025 PRM_DEBUG(modtext_sz);
1025 1026
1026 1027 init_boot_memlists();
1027 1028 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1028 1029 &boot_physavail, &boot_physavail_len,
1029 1030 &boot_virtavail, &boot_virtavail_len);
1030 1031
1031 1032 /*
1032 1033 * Remember what the physically available highest page is
1033 1034 * so that dumpsys works properly, and find out how much
1034 1035 * memory is installed.
1035 1036 */
1036 1037 installed_top_size_memlist_array(boot_physinstalled,
1037 1038 boot_physinstalled_len, &physmax, &physinstalled);
1038 1039 PRM_DEBUG(physinstalled);
1039 1040 PRM_DEBUG(physmax);
1040 1041
1041 1042 /* Fill out memory nodes config structure */
1042 1043 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1043 1044
1044 1045 /*
1045 1046 * npages is the maximum of available physical memory possible.
1046 1047 * (ie. it will never be more than this)
1047 1048 *
1048 1049 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1049 1050 * using phys_avail will underestimate what will end up being freed.
1050 1051 * A better initial guess is just total memory minus the kernel text
1051 1052 */
1052 1053 npages = physinstalled - btop(MMU_PAGESIZE4M);
1053 1054
1054 1055 /*
1055 1056 * First allocate things that can go in the nucleus data page
1056 1057 * (fault status, TSBs, dmv, CPUs)
1057 1058 */
1058 1059 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1059 1060
1060 1061 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1061 1062 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1062 1063
1063 1064 if (ndata_alloc_tsbs(&ndata, npages) != 0)
1064 1065 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1065 1066
1066 1067 if (ndata_alloc_dmv(&ndata) != 0)
1067 1068 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1068 1069
1069 1070 if (ndata_alloc_page_mutexs(&ndata) != 0)
1070 1071 cmn_err(CE_PANIC,
1071 1072 "no more nucleus memory after page free lists alloc");
1072 1073
1073 1074 if (ndata_alloc_hat(&ndata) != 0)
1074 1075 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1075 1076
1076 1077 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1077 1078 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1078 1079
1079 1080 /*
1080 1081 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1081 1082 *
1082 1083 * There are comments all over the SFMMU code warning of dire
1083 1084 * consequences if the TSBs are moved out of 32-bit space. This
1084 1085 * is largely because the asm code uses "sethi %hi(addr)"-type
1085 1086 * instructions which will not provide the expected result if the
1086 1087 * address is a 64-bit one.
1087 1088 *
1088 1089 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1089 1090 */
1090 1091 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1091 1092 PRM_DEBUG(alloc_base);
1092 1093
1093 1094 alloc_base = sfmmu_ktsb_alloc(alloc_base);
1094 1095 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1095 1096 PRM_DEBUG(alloc_base);
1096 1097
1097 1098 /*
1098 1099 * Allocate IOMMU TSB array. We do this here so that the physical
1099 1100 * memory gets deducted from the PROM's physical memory list.
1100 1101 */
1101 1102 alloc_base = iommu_tsb_init(alloc_base);
1102 1103 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1103 1104 PRM_DEBUG(alloc_base);
1104 1105
1105 1106 /*
1106 1107 * Allow for an early allocation of physically contiguous memory.
1107 1108 */
1108 1109 alloc_base = contig_mem_prealloc(alloc_base, npages);
1109 1110
1110 1111 /*
1111 1112 * Platforms like Starcat and OPL need special structures assigned in
1112 1113 * 32-bit virtual address space because their probing routines execute
1113 1114 * FCode, and FCode can't handle 64-bit virtual addresses...
1114 1115 */
1115 1116 if (&plat_startup_memlist) {
1116 1117 alloc_base = plat_startup_memlist(alloc_base);
1117 1118 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1118 1119 ecache_alignsize);
1119 1120 PRM_DEBUG(alloc_base);
1120 1121 }
1121 1122
1122 1123 /*
1123 1124 * Save off where the contiguous allocations to date have ended
1124 1125 * in econtig32.
1125 1126 */
1126 1127 econtig32 = alloc_base;
1127 1128 PRM_DEBUG(econtig32);
1128 1129 if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1129 1130 cmn_err(CE_PANIC, "econtig32 too big");
1130 1131
1131 1132 pp_sz = calc_pp_sz(npages);
1132 1133 PRM_DEBUG(pp_sz);
1133 1134 if (kpm_enable) {
1134 1135 kpm_pp_sz = calc_kpmpp_sz(npages);
1135 1136 PRM_DEBUG(kpm_pp_sz);
1136 1137 }
1137 1138
1138 1139 hmehash_sz = calc_hmehash_sz(npages);
1139 1140 PRM_DEBUG(hmehash_sz);
1140 1141
1141 1142 pagehash_sz = calc_pagehash_sz(npages);
1142 1143 PRM_DEBUG(pagehash_sz);
1143 1144
1144 1145 pagelist_sz = calc_free_pagelist_sz();
1145 1146 PRM_DEBUG(pagelist_sz);
1146 1147
1147 1148 #ifdef TRAPTRACE
1148 1149 tt_sz = calc_traptrace_sz();
1149 1150 PRM_DEBUG(tt_sz);
1150 1151 #else
1151 1152 tt_sz = 0;
1152 1153 #endif /* TRAPTRACE */
1153 1154
1154 1155 /*
1155 1156 * Place the array that protects pp->p_selock in the kmem64 wad.
1156 1157 */
1157 1158 pse_shift = size_pse_array(npages, max_ncpus);
1158 1159 PRM_DEBUG(pse_shift);
1159 1160 pse_table_size = 1 << pse_shift;
1160 1161 PRM_DEBUG(pse_table_size);
1161 1162 psetable_sz = roundup(
1162 1163 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1163 1164 PRM_DEBUG(psetable_sz);
1164 1165
1165 1166 /*
1166 1167 * Now allocate the whole wad
1167 1168 */
1168 1169 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1169 1170 pagelist_sz + tt_sz + psetable_sz;
1170 1171 kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1171 1172 kmem64_base = (caddr_t)syslimit;
1172 1173 kmem64_end = kmem64_base + kmem64_sz;
1173 1174 if (alloc_kmem64(kmem64_base, kmem64_end)) {
1174 1175 /*
1175 1176 * Attempt for kmem64 to allocate one big
1176 1177 * contiguous chunk of memory failed.
1177 1178 * We get here because we are sun4v.
1178 1179 * We will proceed by breaking up
1179 1180 * the allocation into two attempts.
1180 1181 * First, we allocate kpm_pp_sz, hmehash_sz,
1181 1182 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1182 1183 * one contiguous chunk. This is a much smaller
1183 1184 * chunk and we should get it, if not we panic.
1184 1185 * Note that hmehash and tt need to be physically
1185 1186 * (in the real address sense) contiguous.
1186 1187 * Next, we use bop_alloc_chunk() to
1187 1188 * to allocate the page_t structures.
1188 1189 * This will allow the page_t to be allocated
1189 1190 * in multiple smaller chunks.
1190 1191 * In doing so, the assumption that page_t is
1191 1192 * physically contiguous no longer hold, this is ok
1192 1193 * for sun4v but not for sun4u.
1193 1194 */
1194 1195 size_t tmp_size;
1195 1196 caddr_t tmp_base;
1196 1197
1197 1198 pp_sz = roundup(pp_sz, PAGESIZE);
1198 1199
1199 1200 /*
1200 1201 * Allocate kpm_pp_sz, hmehash_sz,
1201 1202 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1202 1203 */
1203 1204 tmp_base = kmem64_base + pp_sz;
1204 1205 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1205 1206 pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1206 1207 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1207 1208 prom_panic("kmem64 prom_alloc contig failed");
1208 1209 PRM_DEBUG(tmp_base);
1209 1210 PRM_DEBUG(tmp_size);
1210 1211
1211 1212 /*
1212 1213 * Allocate the page_ts
1213 1214 */
1214 1215 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1215 1216 prom_panic("kmem64 bop_alloc_chunk page_t failed");
1216 1217 PRM_DEBUG(kmem64_base);
1217 1218 PRM_DEBUG(pp_sz);
1218 1219
1219 1220 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1220 1221 ASSERT(kmem64_aligned_end >= kmem64_end);
1221 1222
1222 1223 kmem64_smchunks = 1;
1223 1224 } else {
1224 1225
1225 1226 /*
1226 1227 * We need to adjust pp_sz for the normal
1227 1228 * case where kmem64 can allocate one large chunk
1228 1229 */
1229 1230 if (kpm_smallpages == 0) {
1230 1231 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1231 1232 } else {
1232 1233 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1233 1234 sizeof (kpm_spage_t));
1234 1235 }
1235 1236 pp_sz = npages * sizeof (struct page);
1236 1237 }
1237 1238
1238 1239 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1239 1240 cmn_err(CE_PANIC, "not enough kmem64 space");
1240 1241 PRM_DEBUG(kmem64_base);
1241 1242 PRM_DEBUG(kmem64_end);
1242 1243 PRM_DEBUG(kmem64_aligned_end);
1243 1244
1244 1245 /*
1245 1246 * ... and divy it up
1246 1247 */
1247 1248 alloc_base = kmem64_base;
1248 1249
1249 1250 pp_base = (page_t *)alloc_base;
1250 1251 alloc_base += pp_sz;
1251 1252 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1252 1253 PRM_DEBUG(pp_base);
1253 1254 PRM_DEBUG(npages);
1254 1255
1255 1256 if (kpm_enable) {
1256 1257 kpm_pp_base = alloc_base;
1257 1258 if (kpm_smallpages == 0) {
1258 1259 /* kpm_npages based on physinstalled, don't reset */
1259 1260 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1260 1261 } else {
1261 1262 kpm_npages = ptokpmpr(npages);
1262 1263 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1263 1264 }
1264 1265 alloc_base += kpm_pp_sz;
1265 1266 alloc_base =
1266 1267 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1267 1268 PRM_DEBUG(kpm_pp_base);
1268 1269 }
1269 1270
1270 1271 alloc_base = alloc_hmehash(alloc_base);
1271 1272 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1272 1273 PRM_DEBUG(alloc_base);
1273 1274
1274 1275 page_hash = (page_t **)alloc_base;
1275 1276 alloc_base += pagehash_sz;
1276 1277 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1277 1278 PRM_DEBUG(page_hash);
1278 1279
1279 1280 alloc_base = alloc_page_freelists(alloc_base);
1280 1281 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1281 1282 PRM_DEBUG(alloc_base);
1282 1283
1283 1284 #ifdef TRAPTRACE
1284 1285 ttrace_buf = alloc_base;
1285 1286 alloc_base += tt_sz;
1286 1287 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1287 1288 PRM_DEBUG(alloc_base);
1288 1289 #endif /* TRAPTRACE */
1289 1290
1290 1291 pse_mutex = (pad_mutex_t *)alloc_base;
1291 1292 alloc_base += psetable_sz;
1292 1293 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1293 1294 PRM_DEBUG(alloc_base);
1294 1295
1295 1296 /*
1296 1297 * Note that if we use small chunk allocations for
1297 1298 * kmem64, we need to ensure kmem64_end is the same as
1298 1299 * kmem64_aligned_end to prevent subsequent logic from
1299 1300 * trying to reuse the overmapping.
1300 1301 * Otherwise we adjust kmem64_end to what we really allocated.
1301 1302 */
1302 1303 if (kmem64_smchunks) {
1303 1304 kmem64_end = kmem64_aligned_end;
1304 1305 } else {
1305 1306 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1306 1307 }
1307 1308 kmem64_sz = kmem64_end - kmem64_base;
1308 1309
1309 1310 if (&ecache_init_scrub_flush_area) {
1310 1311 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1311 1312 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1312 1313 }
1313 1314
1314 1315 /*
1315 1316 * If physmem is patched to be non-zero, use it instead of
1316 1317 * the monitor value unless physmem is larger than the total
1317 1318 * amount of memory on hand.
1318 1319 */
1319 1320 if (physmem == 0 || physmem > npages)
1320 1321 physmem = npages;
1321 1322
1322 1323 /*
1323 1324 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1324 1325 * is mounted as root. This memory is held down by OBP and unlike
1325 1326 * the stub boot_archive is never released.
1326 1327 *
1327 1328 * In order to get things sized correctly on lower memory
1328 1329 * machines (where the memory used by the ramdisk represents
1329 1330 * a significant portion of memory), physmem is adjusted.
1330 1331 *
1331 1332 * This is done by subtracting the ramdisk_size which is set
1332 1333 * to the size of the ramdisk (in Kb) in /etc/system at the
1333 1334 * time the miniroot archive is constructed.
1334 1335 */
1335 1336 if (root_is_ramdisk == B_TRUE) {
1336 1337 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1337 1338 physmem -= ramdisk_npages;
1338 1339 }
1339 1340
1340 1341 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1341 1342 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1342 1343
1343 1344 /*
1344 1345 * Allocate space for the interrupt vector table.
1345 1346 */
1346 1347 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1347 1348 if (memspace != (caddr_t)intr_vec_table)
1348 1349 prom_panic("interrupt vector table allocation failure");
1349 1350
1350 1351 /*
1351 1352 * Between now and when we finish copying in the memory lists,
1352 1353 * allocations happen so the space gets fragmented and the
1353 1354 * lists longer. Leave enough space for lists twice as
1354 1355 * long as we have now; then roundup to a pagesize.
1355 1356 */
1356 1357 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1357 1358 prom_phys_avail_len() + prom_virt_avail_len());
1358 1359 memlist_sz *= 2;
1359 1360 memlist_sz = roundup(memlist_sz, PAGESIZE);
1360 1361 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1361 1362 if (memspace == NULL)
1362 1363 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1363 1364
1364 1365 memlist = (struct memlist *)memspace;
1365 1366 memlist_end = (char *)memspace + memlist_sz;
1366 1367 PRM_DEBUG(memlist);
1367 1368 PRM_DEBUG(memlist_end);
1368 1369
1369 1370 PRM_DEBUG(sysbase);
1370 1371 PRM_DEBUG(syslimit);
1371 1372 kernelheap_init((void *)sysbase, (void *)syslimit,
1372 1373 (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1373 1374
1374 1375 /*
1375 1376 * Take the most current snapshot we can by calling mem-update.
1376 1377 */
1377 1378 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1378 1379 &boot_physavail, &boot_physavail_len,
1379 1380 &boot_virtavail, &boot_virtavail_len);
1380 1381
1381 1382 /*
1382 1383 * Remove the space used by prom_alloc from the kernel heap
1383 1384 * plus the area actually used by the OBP (if any)
1384 1385 * ignoring virtual addresses in virt_avail, above syslimit.
1385 1386 */
1386 1387 virt_avail = memlist;
1387 1388 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1388 1389
1389 1390 for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1390 1391 uint64_t range_base, range_size;
1391 1392
1392 1393 if ((range_base = cur->ml_address + cur->ml_size) <
1393 1394 (uint64_t)sysbase)
1394 1395 continue;
1395 1396 if (range_base >= (uint64_t)syslimit)
1396 1397 break;
1397 1398 /*
1398 1399 * Limit the range to end at syslimit.
1399 1400 */
1400 1401 range_size = MIN(cur->ml_next->ml_address,
1401 1402 (uint64_t)syslimit) - range_base;
1402 1403 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1403 1404 0, 0, (void *)range_base, (void *)(range_base + range_size),
1404 1405 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1405 1406 }
1406 1407
1407 1408 phys_avail = memlist;
1408 1409 copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1409 1410
1410 1411 /*
1411 1412 * Add any extra memory at the end of the ndata region if there's at
1412 1413 * least a page to add. There might be a few more pages available in
1413 1414 * the middle of the ndata region, but for now they are ignored.
1414 1415 */
1415 1416 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1416 1417 if (nalloc_base == NULL)
1417 1418 nalloc_base = nalloc_end;
1418 1419 ndata_remain_sz = nalloc_end - nalloc_base;
1419 1420
1420 1421 /*
1421 1422 * Copy physinstalled list into kernel space.
1422 1423 */
1423 1424 phys_install = memlist;
1424 1425 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1425 1426
1426 1427 /*
1427 1428 * Create list of physical addrs we don't need pp's for:
1428 1429 * kernel text 4M page
1429 1430 * kernel data 4M page - ndata_remain_sz
1430 1431 * kmem64 pages
1431 1432 *
1432 1433 * NB if adding any pages here, make sure no kpm page
1433 1434 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1434 1435 */
1435 1436 nopp_list = memlist;
1436 1437 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1437 1438 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1438 1439 &memlist, &nopp_list);
1439 1440
1440 1441 /* Don't add to nopp_list if kmem64 was allocated in smchunks */
1441 1442 if (!kmem64_smchunks)
1442 1443 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1443 1444
1444 1445 if ((caddr_t)memlist > (memspace + memlist_sz))
1445 1446 prom_panic("memlist overflow");
1446 1447
1447 1448 /*
1448 1449 * Size the pcf array based on the number of cpus in the box at
1449 1450 * boot time.
1450 1451 */
1451 1452 pcf_init();
1452 1453
1453 1454 /*
1454 1455 * Initialize the page structures from the memory lists.
1455 1456 */
1456 1457 kphysm_init();
1457 1458
1458 1459 availrmem_initial = availrmem = freemem;
1459 1460 PRM_DEBUG(availrmem);
1460 1461
1461 1462 /*
1462 1463 * Some of the locks depend on page_hashsz being set!
1463 1464 * kmem_init() depends on this; so, keep it here.
1464 1465 */
1465 1466 page_lock_init();
1466 1467
1467 1468 /*
1468 1469 * Initialize kernel memory allocator.
1469 1470 */
1470 1471 kmem_init();
1471 1472
1472 1473 /*
1473 1474 * Factor in colorequiv to check additional 'equivalent' bins
1474 1475 */
1475 1476 if (&page_set_colorequiv_arr_cpu != NULL)
1476 1477 page_set_colorequiv_arr_cpu();
1477 1478 else
1478 1479 page_set_colorequiv_arr();
1479 1480
1480 1481 /*
1481 1482 * Initialize bp_mapin().
1482 1483 */
1483 1484 bp_init(shm_alignment, HAT_STRICTORDER);
1484 1485
1485 1486 /*
1486 1487 * Reserve space for MPO mblock structs from the 32-bit heap.
1487 1488 */
1488 1489
1489 1490 if (mpo_heap32_bufsz > (size_t)0) {
1490 1491 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1491 1492 PAGESIZE, 0, 0, mpo_heap32_buf,
1492 1493 mpo_heap32_buf + mpo_heap32_bufsz,
1493 1494 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1494 1495 }
1495 1496 mem_config_init();
1496 1497 }
1497 1498
1498 1499 static void
1499 1500 startup_modules(void)
1500 1501 {
1501 1502 int nhblk1, nhblk8;
1502 1503 size_t nhblksz;
1503 1504 pgcnt_t pages_per_hblk;
1504 1505 size_t hme8blk_sz, hme1blk_sz;
1505 1506
1506 1507 /*
1507 1508 * The system file /etc/system was read already under startup_memlist.
1508 1509 */
1509 1510 if (&set_platform_defaults)
1510 1511 set_platform_defaults();
1511 1512
1512 1513 /*
1513 1514 * Calculate default settings of system parameters based upon
1514 1515 * maxusers, yet allow to be overridden via the /etc/system file.
1515 1516 */
1516 1517 param_calc(0);
1517 1518
1518 1519 mod_setup();
1519 1520
1520 1521 /*
1521 1522 * If this is a positron, complain and halt.
1522 1523 */
1523 1524 if (&iam_positron && iam_positron()) {
1524 1525 cmn_err(CE_WARN, "This hardware platform is not supported"
1525 1526 " by this release of Solaris.\n");
1526 1527 #ifdef DEBUG
1527 1528 prom_enter_mon(); /* Type 'go' to resume */
1528 1529 cmn_err(CE_WARN, "Booting an unsupported platform.\n");
1529 1530 cmn_err(CE_WARN, "Booting with down-rev firmware.\n");
1530 1531
1531 1532 #else /* DEBUG */
1532 1533 halt(0);
1533 1534 #endif /* DEBUG */
1534 1535 }
1535 1536
1536 1537 /*
1537 1538 * If we are running firmware that isn't 64-bit ready
1538 1539 * then complain and halt.
1539 1540 */
1540 1541 do_prom_version_check();
1541 1542
1542 1543 /*
1543 1544 * Initialize system parameters
1544 1545 */
1545 1546 param_init();
1546 1547
1547 1548 /*
1548 1549 * maxmem is the amount of physical memory we're playing with.
1549 1550 */
1550 1551 maxmem = physmem;
1551 1552
1552 1553 /* Set segkp limits. */
1553 1554 ncbase = kdi_segdebugbase;
1554 1555 ncend = kdi_segdebugbase;
1555 1556
1556 1557 /*
1557 1558 * Initialize the hat layer.
1558 1559 */
1559 1560 hat_init();
1560 1561
1561 1562 /*
1562 1563 * Initialize segment management stuff.
1563 1564 */
1564 1565 seg_init();
1565 1566
1566 1567 /*
1567 1568 * Create the va>tte handler, so the prom can understand
1568 1569 * kernel translations. The handler is installed later, just
1569 1570 * as we are about to take over the trap table from the prom.
1570 1571 */
1571 1572 create_va_to_tte();
1572 1573
1573 1574 /*
1574 1575 * Load the forthdebugger (optional)
1575 1576 */
1576 1577 forthdebug_init();
1577 1578
1578 1579 /*
1579 1580 * Create OBP node for console input callbacks
1580 1581 * if it is needed.
1581 1582 */
1582 1583 startup_create_io_node();
1583 1584
1584 1585 if (modloadonly("fs", "specfs") == -1)
1585 1586 halt("Can't load specfs");
1586 1587
1587 1588 if (modloadonly("fs", "devfs") == -1)
1588 1589 halt("Can't load devfs");
1589 1590
1590 1591 if (modloadonly("fs", "procfs") == -1)
1591 1592 halt("Can't load procfs");
1592 1593
1593 1594 if (modloadonly("misc", "swapgeneric") == -1)
1594 1595 halt("Can't load swapgeneric");
1595 1596
1596 1597 (void) modloadonly("sys", "lbl_edition");
1597 1598
1598 1599 dispinit();
1599 1600
1600 1601 /*
1601 1602 * Infer meanings to the members of the idprom buffer.
1602 1603 */
1603 1604 parse_idprom();
1604 1605
1605 1606 /* Read cluster configuration data. */
1606 1607 clconf_init();
1607 1608
1608 1609 setup_ddi();
1609 1610
1610 1611 /*
1611 1612 * Lets take this opportunity to load the root device.
1612 1613 */
1613 1614 if (loadrootmodules() != 0)
1614 1615 debug_enter("Can't load the root filesystem");
1615 1616
1616 1617 /*
1617 1618 * Load tod driver module for the tod part found on this system.
1618 1619 * Recompute the cpu frequency/delays based on tod as tod part
1619 1620 * tends to keep time more accurately.
1620 1621 */
1621 1622 if (&load_tod_module)
1622 1623 load_tod_module();
1623 1624
1624 1625 /*
1625 1626 * Allow platforms to load modules which might
1626 1627 * be needed after bootops are gone.
1627 1628 */
1628 1629 if (&load_platform_modules)
1629 1630 load_platform_modules();
1630 1631
1631 1632 setcpudelay();
1632 1633
1633 1634 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1634 1635 &boot_physavail, &boot_physavail_len,
1635 1636 &boot_virtavail, &boot_virtavail_len);
1636 1637
1637 1638 /*
1638 1639 * Calculation and allocation of hmeblks needed to remap
1639 1640 * the memory allocated by PROM till now.
1640 1641 * Overestimate the number of hblk1 elements by assuming
1641 1642 * worst case of TTE64K mappings.
1642 1643 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1643 1644 */
1644 1645 bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1645 1646 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1646 1647 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1647 1648 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1648 1649
1649 1650 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1650 1651
1651 1652 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1652 1653 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1653 1654 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1654 1655
1655 1656 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1656 1657
1657 1658 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1658 1659 nhblk8 = 0;
1659 1660 while (bop_alloc_pages > 1) {
1660 1661 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1661 1662 nhblk8 += bop_alloc_pages /= pages_per_hblk;
1662 1663 bop_alloc_pages *= hme8blk_sz;
1663 1664 bop_alloc_pages = btopr(bop_alloc_pages);
1664 1665 }
1665 1666 nhblk8 += 2;
1666 1667
1667 1668 /*
1668 1669 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1669 1670 * boundary, the number of hblk8's needed to map the entries in the
1670 1671 * boot_virtavail list needs to be adjusted to take this into
1671 1672 * consideration. Thus, we need to add additional hblk8's since it
1672 1673 * is possible that an hblk8 will not have all 8 slots used due to
1673 1674 * alignment constraints. Since there were boot_virtavail_len entries
1674 1675 * in that list, we need to add that many hblk8's to the number
1675 1676 * already calculated to make sure we don't underestimate.
1676 1677 */
1677 1678 nhblk8 += boot_virtavail_len;
1678 1679 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1679 1680
1680 1681 /* Allocate in pagesize chunks */
1681 1682 nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1682 1683 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1683 1684 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1684 1685 }
1685 1686
1686 1687 static void
1687 1688 startup_bop_gone(void)
1688 1689 {
1689 1690
1690 1691 /*
1691 1692 * Destroy the MD initialized at startup
1692 1693 * The startup initializes the MD framework
1693 1694 * using prom and BOP alloc free it now.
1694 1695 */
1695 1696 mach_descrip_startup_fini();
1696 1697
1697 1698 /*
1698 1699 * We're done with prom allocations.
1699 1700 */
1700 1701 bop_fini();
1701 1702
1702 1703 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1703 1704 &boot_physavail, &boot_physavail_len,
1704 1705 &boot_virtavail, &boot_virtavail_len);
1705 1706
1706 1707 /*
1707 1708 * setup physically contiguous area twice as large as the ecache.
1708 1709 * this is used while doing displacement flush of ecaches
1709 1710 */
1710 1711 if (&ecache_flush_address) {
1711 1712 ecache_flushaddr = ecache_flush_address();
1712 1713 if (ecache_flushaddr == (uint64_t)-1) {
1713 1714 cmn_err(CE_PANIC,
1714 1715 "startup: no memory to set ecache_flushaddr");
1715 1716 }
1716 1717 }
1717 1718
1718 1719 /*
1719 1720 * Virtual available next.
1720 1721 */
1721 1722 ASSERT(virt_avail != NULL);
1722 1723 memlist_free_list(virt_avail);
1723 1724 virt_avail = memlist;
1724 1725 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1725 1726
1726 1727 }
1727 1728
1728 1729
1729 1730 /*
1730 1731 * startup_fixup_physavail - called from mach_sfmmu.c after the final
1731 1732 * allocations have been performed. We can't call it in startup_bop_gone
1732 1733 * since later operations can cause obp to allocate more memory.
1733 1734 */
1734 1735 void
1735 1736 startup_fixup_physavail(void)
1736 1737 {
1737 1738 struct memlist *cur;
1738 1739 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1739 1740
1740 1741 PRM_DEBUG(kmem64_overmap_size);
1741 1742
1742 1743 /*
1743 1744 * take the most current snapshot we can by calling mem-update
1744 1745 */
1745 1746 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1746 1747 &boot_physavail, &boot_physavail_len,
1747 1748 &boot_virtavail, &boot_virtavail_len);
1748 1749
1749 1750 /*
1750 1751 * Copy phys_avail list, again.
1751 1752 * Both the kernel/boot and the prom have been allocating
1752 1753 * from the original list we copied earlier.
1753 1754 */
1754 1755 cur = memlist;
1755 1756 copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1756 1757
1757 1758 /*
1758 1759 * Add any unused kmem64 memory from overmapped page
1759 1760 * (Note: va_to_pa does not work for kmem64_end)
1760 1761 */
1761 1762 if (kmem64_overmap_size) {
1762 1763 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1763 1764 kmem64_overmap_size, &memlist, &cur);
1764 1765 }
1765 1766
1766 1767 /*
1767 1768 * Add any extra memory after e_data we added to the phys_avail list
1768 1769 * back to the old list.
1769 1770 */
1770 1771 if (ndata_remain_sz >= MMU_PAGESIZE)
1771 1772 memlist_add(va_to_pa(nalloc_base),
1772 1773 (uint64_t)ndata_remain_sz, &memlist, &cur);
1773 1774
1774 1775 /*
1775 1776 * There isn't any bounds checking on the memlist area
1776 1777 * so ensure it hasn't overgrown.
1777 1778 */
1778 1779 if ((caddr_t)memlist > (caddr_t)memlist_end)
1779 1780 cmn_err(CE_PANIC, "startup: memlist size exceeded");
1780 1781
1781 1782 /*
1782 1783 * The kernel removes the pages that were allocated for it from
1783 1784 * the freelist, but we now have to find any -extra- pages that
1784 1785 * the prom has allocated for it's own book-keeping, and remove
1785 1786 * them from the freelist too. sigh.
1786 1787 */
1787 1788 sync_memlists(phys_avail, cur);
1788 1789
1789 1790 ASSERT(phys_avail != NULL);
1790 1791
1791 1792 old_phys_avail = phys_avail;
1792 1793 phys_avail = cur;
1793 1794 }
1794 1795
1795 1796 void
1796 1797 update_kcage_ranges(uint64_t addr, uint64_t len)
1797 1798 {
1798 1799 pfn_t base = btop(addr);
1799 1800 pgcnt_t num = btop(len);
1800 1801 int rv;
1801 1802
1802 1803 rv = kcage_range_add(base, num, kcage_startup_dir);
1803 1804
1804 1805 if (rv == ENOMEM) {
1805 1806 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1806 1807 (len == 0 ? 0 : BYTES_TO_MB(len)));
1807 1808 } else if (rv != 0) {
1808 1809 /* catch this in debug kernels */
1809 1810 ASSERT(0);
1810 1811
1811 1812 cmn_err(CE_WARN, "unexpected kcage_range_add"
1812 1813 " return value %d", rv);
1813 1814 }
1814 1815 }
1815 1816
1816 1817 static void
1817 1818 startup_vm(void)
1818 1819 {
1819 1820 size_t i;
1820 1821 struct segmap_crargs a;
1821 1822 struct segkpm_crargs b;
1822 1823
1823 1824 uint64_t avmem;
1824 1825 caddr_t va;
1825 1826 pgcnt_t max_phys_segkp;
1826 1827 int mnode;
1827 1828
1828 1829 extern int use_brk_lpg, use_stk_lpg;
1829 1830
1830 1831 /*
1831 1832 * get prom's mappings, create hments for them and switch
1832 1833 * to the kernel context.
1833 1834 */
1834 1835 hat_kern_setup();
1835 1836
1836 1837 /*
1837 1838 * Take over trap table
1838 1839 */
1839 1840 setup_trap_table();
1840 1841
1841 1842 /*
1842 1843 * Install the va>tte handler, so that the prom can handle
1843 1844 * misses and understand the kernel table layout in case
1844 1845 * we need call into the prom.
1845 1846 */
1846 1847 install_va_to_tte();
1847 1848
1848 1849 /*
1849 1850 * Set a flag to indicate that the tba has been taken over.
1850 1851 */
1851 1852 tba_taken_over = 1;
1852 1853
1853 1854 /* initialize MMU primary context register */
1854 1855 mmu_init_kcontext();
1855 1856
1856 1857 /*
1857 1858 * The boot cpu can now take interrupts, x-calls, x-traps
1858 1859 */
1859 1860 CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1860 1861 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1861 1862
1862 1863 /*
1863 1864 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1864 1865 */
1865 1866 tbr_wr_addr_inited = 1;
1866 1867
1867 1868 /*
1868 1869 * Initialize VM system, and map kernel address space.
1869 1870 */
1870 1871 kvm_init();
1871 1872
1872 1873 ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1873 1874 if (kernel_cage_enable) {
1874 1875 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1875 1876 }
1876 1877 memlist_free_list(old_phys_avail);
1877 1878
1878 1879 /*
1879 1880 * If the following is true, someone has patched
1880 1881 * phsymem to be less than the number of pages that
1881 1882 * the system actually has. Remove pages until system
1882 1883 * memory is limited to the requested amount. Since we
1883 1884 * have allocated page structures for all pages, we
1884 1885 * correct the amount of memory we want to remove
1885 1886 * by the size of the memory used to hold page structures
1886 1887 * for the non-used pages.
1887 1888 */
1888 1889 if (physmem + ramdisk_npages < npages) {
1889 1890 pgcnt_t diff, off;
1890 1891 struct page *pp;
1891 1892 struct seg kseg;
1892 1893
1893 1894 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1894 1895
1895 1896 off = 0;
1896 1897 diff = npages - (physmem + ramdisk_npages);
1897 1898 diff -= mmu_btopr(diff * sizeof (struct page));
1898 1899 kseg.s_as = &kas;
1899 1900 while (diff--) {
1900 1901 pp = page_create_va(&unused_pages_vp, (offset_t)off,
1901 1902 MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1902 1903 &kseg, (caddr_t)off);
1903 1904 if (pp == NULL)
1904 1905 cmn_err(CE_PANIC, "limited physmem too much!");
1905 1906 page_io_unlock(pp);
1906 1907 page_downgrade(pp);
1907 1908 availrmem--;
1908 1909 off += MMU_PAGESIZE;
1909 1910 }
1910 1911 }
1911 1912
1912 1913 /*
1913 1914 * When printing memory, show the total as physmem less
1914 1915 * that stolen by a debugger.
1915 1916 */
1916 1917 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1917 1918 (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1918 1919 (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1919 1920
1920 1921 avmem = (uint64_t)freemem << PAGESHIFT;
1921 1922 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1922 1923
1923 1924 /*
1924 1925 * For small memory systems disable automatic large pages.
1925 1926 */
1926 1927 if (physmem < privm_lpg_min_physmem) {
1927 1928 use_brk_lpg = 0;
1928 1929 use_stk_lpg = 0;
1929 1930 }
1930 1931
1931 1932 /*
1932 1933 * Perform platform specific freelist processing
1933 1934 */
1934 1935 if (&plat_freelist_process) {
1935 1936 for (mnode = 0; mnode < max_mem_nodes; mnode++)
1936 1937 if (mem_node_config[mnode].exists)
1937 1938 plat_freelist_process(mnode);
1938 1939 }
1939 1940
1940 1941 /*
1941 1942 * Initialize the segkp segment type. We position it
1942 1943 * after the configured tables and buffers (whose end
1943 1944 * is given by econtig) and before V_WKBASE_ADDR.
1944 1945 * Also in this area is segkmap (size SEGMAPSIZE).
1945 1946 */
1946 1947
1947 1948 /* XXX - cache alignment? */
1948 1949 va = (caddr_t)SEGKPBASE;
1949 1950 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1950 1951
1951 1952 max_phys_segkp = (physmem * 2);
1952 1953
1953 1954 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1954 1955 segkpsize = btop(SEGKPDEFSIZE);
1955 1956 cmn_err(CE_WARN, "Illegal value for segkpsize. "
1956 1957 "segkpsize has been reset to %ld pages", segkpsize);
1957 1958 }
1958 1959
1959 1960 i = ptob(MIN(segkpsize, max_phys_segkp));
1960 1961
1961 1962 rw_enter(&kas.a_lock, RW_WRITER);
1962 1963 if (seg_attach(&kas, va, i, segkp) < 0)
1963 1964 cmn_err(CE_PANIC, "startup: cannot attach segkp");
1964 1965 if (segkp_create(segkp) != 0)
1965 1966 cmn_err(CE_PANIC, "startup: segkp_create failed");
1966 1967 rw_exit(&kas.a_lock);
1967 1968
1968 1969 /*
1969 1970 * kpm segment
1970 1971 */
1971 1972 segmap_kpm = kpm_enable &&
1972 1973 segmap_kpm && PAGESIZE == MAXBSIZE;
1973 1974
1974 1975 if (kpm_enable) {
1975 1976 rw_enter(&kas.a_lock, RW_WRITER);
1976 1977
1977 1978 /*
1978 1979 * The segkpm virtual range range is larger than the
1979 1980 * actual physical memory size and also covers gaps in
1980 1981 * the physical address range for the following reasons:
1981 1982 * . keep conversion between segkpm and physical addresses
1982 1983 * simple, cheap and unambiguous.
1983 1984 * . avoid extension/shrink of the the segkpm in case of DR.
1984 1985 * . avoid complexity for handling of virtual addressed
1985 1986 * caches, segkpm and the regular mapping scheme must be
1986 1987 * kept in sync wrt. the virtual color of mapped pages.
1987 1988 * Any accesses to virtual segkpm ranges not backed by
1988 1989 * physical memory will fall through the memseg pfn hash
1989 1990 * and will be handled in segkpm_fault.
1990 1991 * Additional kpm_size spaces needed for vac alias prevention.
1991 1992 */
1992 1993 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1993 1994 segkpm) < 0)
1994 1995 cmn_err(CE_PANIC, "cannot attach segkpm");
1995 1996
1996 1997 b.prot = PROT_READ | PROT_WRITE;
1997 1998 b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1998 1999
1999 2000 if (segkpm_create(segkpm, (caddr_t)&b) != 0)
2000 2001 panic("segkpm_create segkpm");
2001 2002
2002 2003 rw_exit(&kas.a_lock);
2003 2004
2004 2005 mach_kpm_init();
2005 2006 }
2006 2007
2007 2008 va = kpm_vbase + (kpm_size * vac_colors);
2008 2009
2009 2010 if (!segzio_fromheap) {
2010 2011 size_t size;
2011 2012 size_t physmem_b = mmu_ptob(physmem);
2012 2013
2013 2014 /* size is in bytes, segziosize is in pages */
2014 2015 if (segziosize == 0) {
2015 2016 size = physmem_b;
2016 2017 } else {
2017 2018 size = mmu_ptob(segziosize);
2018 2019 }
2019 2020
2020 2021 if (size < SEGZIOMINSIZE) {
2021 2022 size = SEGZIOMINSIZE;
2022 2023 } else if (size > SEGZIOMAXSIZE) {
2023 2024 size = SEGZIOMAXSIZE;
2024 2025 /*
2025 2026 * On 64-bit x86, we only have 2TB of KVA. This exists
2026 2027 * for parity with x86.
2027 2028 *
2028 2029 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2029 2030 * doesn't consume all of KVA. However, if we have a
2030 2031 * system that has more thant 512gb of physical memory,
2031 2032 * we can actually consume about half of the difference
2032 2033 * between 512gb and the rest of the available physical
2033 2034 * memory.
2034 2035 */
2035 2036 if (physmem_b > SEGZIOMAXSIZE) {
2036 2037 size += (physmem_b - SEGZIOMAXSIZE) / 2;
2037 2038 }
2038 2039 }
2039 2040 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2040 2041 /* put the base of the ZIO segment after the kpm segment */
2041 2042 segzio_base = va;
2042 2043 va += mmu_ptob(segziosize);
2043 2044 PRM_DEBUG(segziosize);
2044 2045 PRM_DEBUG(segzio_base);
2045 2046
2046 2047 /*
2047 2048 * On some platforms, kvm_init is called after the kpm
2048 2049 * sizes have been determined. On SPARC, kvm_init is called
2049 2050 * before, so we have to attach the kzioseg after kvm is
2050 2051 * initialized, otherwise we'll try to allocate from the boot
2051 2052 * area since the kernel heap hasn't yet been configured.
2052 2053 */
2053 2054 rw_enter(&kas.a_lock, RW_WRITER);
2054 2055
2055 2056 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2056 2057 &kzioseg);
2057 2058 (void) segkmem_zio_create(&kzioseg);
2058 2059
2059 2060 /* create zio area covering new segment */
2060 2061 segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2061 2062
2062 2063 rw_exit(&kas.a_lock);
2063 2064 }
2064 2065
2065 2066 if (ppvm_enable) {
2066 2067 caddr_t ppvm_max;
2067 2068
2068 2069 /*
2069 2070 * ppvm refers to the static VA space used to map
2070 2071 * the page_t's for dynamically added memory.
2071 2072 *
2072 2073 * ppvm_base should not cross a potential VA hole.
2073 2074 *
2074 2075 * ppvm_size should be large enough to map the
2075 2076 * page_t's needed to manage all of KPM range.
2076 2077 */
2077 2078 ppvm_size =
2078 2079 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2079 2080 MMU_PAGESIZE);
2080 2081 ppvm_max = (caddr_t)(0ull - ppvm_size);
2081 2082 ppvm_base = (page_t *)va;
2082 2083
2083 2084 if ((caddr_t)ppvm_base <= hole_end) {
2084 2085 cmn_err(CE_WARN,
2085 2086 "Memory DR disabled: invalid DR map base: 0x%p\n",
2086 2087 (void *)ppvm_base);
2087 2088 ppvm_enable = 0;
2088 2089 } else if ((caddr_t)ppvm_base > ppvm_max) {
2089 2090 uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2090 2091
2091 2092 cmn_err(CE_WARN,
2092 2093 "Memory DR disabled: insufficient DR map size:"
2093 2094 " 0x%lx (needed 0x%lx)\n",
2094 2095 ppvm_size - diff, ppvm_size);
2095 2096 ppvm_enable = 0;
2096 2097 }
2097 2098 PRM_DEBUG(ppvm_size);
2098 2099 PRM_DEBUG(ppvm_base);
2099 2100 }
2100 2101
2101 2102 /*
2102 2103 * Now create generic mapping segment. This mapping
2103 2104 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total
2104 2105 * virtual address is greater than the amount of free
2105 2106 * memory that is available, then we trim back the
2106 2107 * segment size to that amount
2107 2108 */
2108 2109 va = (caddr_t)SEGMAPBASE;
2109 2110
2110 2111 /*
2111 2112 * 1201049: segkmap base address must be MAXBSIZE aligned
2112 2113 */
2113 2114 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2114 2115
2115 2116 /*
2116 2117 * Set size of segmap to percentage of freemem at boot,
2117 2118 * but stay within the allowable range
2118 2119 * Note we take percentage before converting from pages
2119 2120 * to bytes to avoid an overflow on 32-bit kernels.
2120 2121 */
2121 2122 i = mmu_ptob((freemem * segmap_percent) / 100);
2122 2123
2123 2124 if (i < MINMAPSIZE)
2124 2125 i = MINMAPSIZE;
2125 2126
2126 2127 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2127 2128 i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2128 2129
2129 2130 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */
2130 2131
2131 2132 rw_enter(&kas.a_lock, RW_WRITER);
2132 2133 if (seg_attach(&kas, va, i, segkmap) < 0)
2133 2134 cmn_err(CE_PANIC, "cannot attach segkmap");
2134 2135
2135 2136 a.prot = PROT_READ | PROT_WRITE;
2136 2137 a.shmsize = shm_alignment;
2137 2138 a.nfreelist = 0; /* use segmap driver defaults */
2138 2139
2139 2140 if (segmap_create(segkmap, (caddr_t)&a) != 0)
2140 2141 panic("segmap_create segkmap");
2141 2142 rw_exit(&kas.a_lock);
2142 2143
2143 2144 segdev_init();
2144 2145 }
2145 2146
2146 2147 static void
2147 2148 startup_end(void)
2148 2149 {
2149 2150 if ((caddr_t)memlist > (caddr_t)memlist_end)
2150 2151 panic("memlist overflow 2");
2151 2152 memlist_free_block((caddr_t)memlist,
2152 2153 ((caddr_t)memlist_end - (caddr_t)memlist));
2153 2154 memlist = NULL;
2154 2155
2155 2156 /* enable page_relocation since OBP is now done */
2156 2157 page_relocate_ready = 1;
2157 2158
2158 2159 /*
2159 2160 * Perform tasks that get done after most of the VM
2160 2161 * initialization has been done but before the clock
2161 2162 * and other devices get started.
2162 2163 */
2163 2164 kern_setup1();
2164 2165
2165 2166 /*
2166 2167 * Perform CPC initialization for this CPU.
2167 2168 */
2168 2169 kcpc_hw_init();
2169 2170
2170 2171 /*
2171 2172 * Intialize the VM arenas for allocating physically
2172 2173 * contiguus memory chunk for interrupt queues snd
2173 2174 * allocate/register boot cpu's queues, if any and
2174 2175 * allocate dump buffer for sun4v systems to store
2175 2176 * extra crash information during crash dump
2176 2177 */
2177 2178 contig_mem_init();
2178 2179 mach_descrip_init();
2179 2180
2180 2181 if (cpu_intrq_setup(CPU)) {
2181 2182 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2182 2183 }
2183 2184 cpu_intrq_register(CPU);
2184 2185 mach_htraptrace_setup(CPU->cpu_id);
2185 2186 mach_htraptrace_configure(CPU->cpu_id);
2186 2187 mach_dump_buffer_init();
2187 2188
2188 2189 /*
2189 2190 * Initialize interrupt related stuff
2190 2191 */
2191 2192 cpu_intr_alloc(CPU, NINTR_THREADS);
2192 2193
2193 2194 (void) splzs(); /* allow hi clock ints but not zs */
2194 2195
2195 2196 /*
2196 2197 * Initialize errors.
2197 2198 */
2198 2199 error_init();
2199 2200
2200 2201 /*
2201 2202 * Note that we may have already used kernel bcopy before this
2202 2203 * point - but if you really care about this, adb the use_hw_*
2203 2204 * variables to 0 before rebooting.
2204 2205 */
2205 2206 mach_hw_copy_limit();
2206 2207
2207 2208 /*
2208 2209 * Install the "real" preemption guards before DDI services
2209 2210 * are available.
2210 2211 */
2211 2212 (void) prom_set_preprom(kern_preprom);
2212 2213 (void) prom_set_postprom(kern_postprom);
2213 2214 CPU->cpu_m.mutex_ready = 1;
2214 2215
2215 2216 /*
2216 2217 * Initialize segnf (kernel support for non-faulting loads).
2217 2218 */
2218 2219 segnf_init();
2219 2220
2220 2221 /*
2221 2222 * Configure the root devinfo node.
2222 2223 */
2223 2224 configure(); /* set up devices */
2224 2225 mach_cpu_halt_idle();
2225 2226 }
2226 2227
2227 2228
2228 2229 void
2229 2230 post_startup(void)
2230 2231 {
2231 2232 #ifdef PTL1_PANIC_DEBUG
2232 2233 extern void init_ptl1_thread(void);
2233 2234 #endif /* PTL1_PANIC_DEBUG */
2234 2235 extern void abort_sequence_init(void);
2235 2236
2236 2237 /*
2237 2238 * Set the system wide, processor-specific flags to be passed
2238 2239 * to userland via the aux vector for performance hints and
2239 2240 * instruction set extensions.
2240 2241 */
2241 2242 bind_hwcap();
2242 2243
2243 2244 /*
2244 2245 * Startup memory scrubber (if any)
2245 2246 */
2246 2247 mach_memscrub();
2247 2248
2248 2249 /*
2249 2250 * Allocate soft interrupt to handle abort sequence.
2250 2251 */
2251 2252 abort_sequence_init();
2252 2253
2253 2254 /*
2254 2255 * Configure the rest of the system.
2255 2256 * Perform forceloading tasks for /etc/system.
2256 2257 */
2257 2258 (void) mod_sysctl(SYS_FORCELOAD, NULL);
2258 2259 /*
2259 2260 * ON4.0: Force /proc module in until clock interrupt handle fixed
2260 2261 * ON4.0: This must be fixed or restated in /etc/systems.
2261 2262 */
2262 2263 (void) modload("fs", "procfs");
2263 2264
2264 2265 /* load machine class specific drivers */
2265 2266 load_mach_drivers();
2266 2267
2267 2268 /* load platform specific drivers */
2268 2269 if (&load_platform_drivers)
2269 2270 load_platform_drivers();
2270 2271
2271 2272 /* load vis simulation module, if we are running w/fpu off */
2272 2273 if (!fpu_exists) {
2273 2274 if (modload("misc", "vis") == -1)
2274 2275 halt("Can't load vis");
2275 2276 }
2276 2277
2277 2278 mach_fpras();
2278 2279
2279 2280 maxmem = freemem;
2280 2281
2281 2282 pg_init();
2282 2283
2283 2284 #ifdef PTL1_PANIC_DEBUG
2284 2285 init_ptl1_thread();
2285 2286 #endif /* PTL1_PANIC_DEBUG */
2286 2287 }
2287 2288
2288 2289 #ifdef PTL1_PANIC_DEBUG
2289 2290 int ptl1_panic_test = 0;
2290 2291 int ptl1_panic_xc_one_test = 0;
2291 2292 int ptl1_panic_xc_all_test = 0;
2292 2293 int ptl1_panic_xt_one_test = 0;
2293 2294 int ptl1_panic_xt_all_test = 0;
2294 2295 kthread_id_t ptl1_thread_p = NULL;
2295 2296 kcondvar_t ptl1_cv;
2296 2297 kmutex_t ptl1_mutex;
2297 2298 int ptl1_recurse_count_threshold = 0x40;
2298 2299 int ptl1_recurse_trap_threshold = 0x3d;
2299 2300 extern void ptl1_recurse(int, int);
2300 2301 extern void ptl1_panic_xt(int, int);
2301 2302
2302 2303 /*
2303 2304 * Called once per second by timeout() to wake up
2304 2305 * the ptl1_panic thread to see if it should cause
2305 2306 * a trap to the ptl1_panic() code.
2306 2307 */
2307 2308 /* ARGSUSED */
2308 2309 static void
2309 2310 ptl1_wakeup(void *arg)
2310 2311 {
2311 2312 mutex_enter(&ptl1_mutex);
2312 2313 cv_signal(&ptl1_cv);
2313 2314 mutex_exit(&ptl1_mutex);
2314 2315 }
2315 2316
2316 2317 /*
2317 2318 * ptl1_panic cross call function:
2318 2319 * Needed because xc_one() and xc_some() can pass
2319 2320 * 64 bit args but ptl1_recurse() expects ints.
2320 2321 */
2321 2322 static void
2322 2323 ptl1_panic_xc(void)
2323 2324 {
2324 2325 ptl1_recurse(ptl1_recurse_count_threshold,
2325 2326 ptl1_recurse_trap_threshold);
2326 2327 }
2327 2328
2328 2329 /*
2329 2330 * The ptl1 thread waits for a global flag to be set
2330 2331 * and uses the recurse thresholds to set the stack depth
2331 2332 * to cause a ptl1_panic() directly via a call to ptl1_recurse
2332 2333 * or indirectly via the cross call and cross trap functions.
2333 2334 *
2334 2335 * This is useful testing stack overflows and normal
2335 2336 * ptl1_panic() states with a know stack frame.
2336 2337 *
2337 2338 * ptl1_recurse() is an asm function in ptl1_panic.s that
2338 2339 * sets the {In, Local, Out, and Global} registers to a
2339 2340 * know state on the stack and just prior to causing a
2340 2341 * test ptl1_panic trap.
2341 2342 */
2342 2343 static void
2343 2344 ptl1_thread(void)
2344 2345 {
2345 2346 mutex_enter(&ptl1_mutex);
2346 2347 while (ptl1_thread_p) {
2347 2348 cpuset_t other_cpus;
2348 2349 int cpu_id;
2349 2350 int my_cpu_id;
2350 2351 int target_cpu_id;
2351 2352 int target_found;
2352 2353
2353 2354 if (ptl1_panic_test) {
2354 2355 ptl1_recurse(ptl1_recurse_count_threshold,
2355 2356 ptl1_recurse_trap_threshold);
2356 2357 }
2357 2358
2358 2359 /*
2359 2360 * Find potential targets for x-call and x-trap,
2360 2361 * if any exist while preempt is disabled we
2361 2362 * start a ptl1_panic if requested via a
2362 2363 * globals.
2363 2364 */
2364 2365 kpreempt_disable();
2365 2366 my_cpu_id = CPU->cpu_id;
2366 2367 other_cpus = cpu_ready_set;
2367 2368 CPUSET_DEL(other_cpus, CPU->cpu_id);
2368 2369 target_found = 0;
2369 2370 if (!CPUSET_ISNULL(other_cpus)) {
2370 2371 /*
2371 2372 * Pick the first one
2372 2373 */
2373 2374 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2374 2375 if (cpu_id == my_cpu_id)
2375 2376 continue;
2376 2377
2377 2378 if (CPU_XCALL_READY(cpu_id)) {
2378 2379 target_cpu_id = cpu_id;
2379 2380 target_found = 1;
2380 2381 break;
2381 2382 }
2382 2383 }
2383 2384 ASSERT(target_found);
2384 2385
2385 2386 if (ptl1_panic_xc_one_test) {
2386 2387 xc_one(target_cpu_id,
2387 2388 (xcfunc_t *)ptl1_panic_xc, 0, 0);
2388 2389 }
2389 2390 if (ptl1_panic_xc_all_test) {
2390 2391 xc_some(other_cpus,
2391 2392 (xcfunc_t *)ptl1_panic_xc, 0, 0);
2392 2393 }
2393 2394 if (ptl1_panic_xt_one_test) {
2394 2395 xt_one(target_cpu_id,
2395 2396 (xcfunc_t *)ptl1_panic_xt, 0, 0);
2396 2397 }
2397 2398 if (ptl1_panic_xt_all_test) {
2398 2399 xt_some(other_cpus,
2399 2400 (xcfunc_t *)ptl1_panic_xt, 0, 0);
2400 2401 }
2401 2402 }
2402 2403 kpreempt_enable();
2403 2404 (void) timeout(ptl1_wakeup, NULL, hz);
2404 2405 (void) cv_wait(&ptl1_cv, &ptl1_mutex);
2405 2406 }
2406 2407 mutex_exit(&ptl1_mutex);
2407 2408 }
2408 2409
2409 2410 /*
2410 2411 * Called during early startup to create the ptl1_thread
2411 2412 */
2412 2413 void
2413 2414 init_ptl1_thread(void)
2414 2415 {
2415 2416 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2416 2417 &p0, TS_RUN, 0);
2417 2418 }
2418 2419 #endif /* PTL1_PANIC_DEBUG */
2419 2420
2420 2421
2421 2422 static void
2422 2423 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2423 2424 {
2424 2425 struct memlist *new;
2425 2426
2426 2427 new = *memlistp;
2427 2428 new->ml_address = start;
2428 2429 new->ml_size = len;
2429 2430 *memlistp = new + 1;
2430 2431 }
|
↓ open down ↓ |
1787 lines elided |
↑ open up ↑ |
2431 2432
2432 2433 /*
2433 2434 * Add to a memory list.
2434 2435 * start = start of new memory segment
2435 2436 * len = length of new memory segment in bytes
2436 2437 * memlistp = pointer to array of available memory segment structures
2437 2438 * curmemlistp = memory list to which to add segment.
2438 2439 */
2439 2440 static void
2440 2441 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2441 - struct memlist **curmemlistp)
2442 + struct memlist **curmemlistp)
2442 2443 {
2443 2444 struct memlist *new = *memlistp;
2444 2445
2445 2446 memlist_new(start, len, memlistp);
2446 2447 memlist_insert(new, curmemlistp);
2447 2448 }
2448 2449
2449 2450 static int
2450 2451 ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2451 2452 {
2452 2453 int nseg;
2453 2454 size_t memseg_sz;
2454 2455 struct memseg *msp;
2455 2456
2456 2457 /*
2457 2458 * The memseg list is for the chunks of physical memory that
2458 2459 * will be managed by the vm system. The number calculated is
2459 2460 * a guess as boot may fragment it more when memory allocations
2460 2461 * are made before kphysm_init().
2461 2462 */
2462 2463 memseg_sz = (avail + 10) * sizeof (struct memseg);
2463 2464 memseg_sz = roundup(memseg_sz, PAGESIZE);
2464 2465 nseg = memseg_sz / sizeof (struct memseg);
2465 2466 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2466 2467 if (msp == NULL)
2467 2468 return (1);
2468 2469 PRM_DEBUG(memseg_free);
2469 2470
2470 2471 while (nseg--) {
2471 2472 msp->next = memseg_free;
2472 2473 memseg_free = msp;
2473 2474 msp++;
2474 2475 }
2475 2476 return (0);
2476 2477 }
2477 2478
2478 2479 /*
2479 2480 * In the case of architectures that support dynamic addition of
2480 2481 * memory at run-time there are two cases where memsegs need to
2481 2482 * be initialized and added to the memseg list.
2482 2483 * 1) memsegs that are constructed at startup.
2483 2484 * 2) memsegs that are constructed at run-time on
2484 2485 * hot-plug capable architectures.
2485 2486 * This code was originally part of the function kphysm_init().
2486 2487 */
2487 2488
2488 2489 static void
2489 2490 memseg_list_add(struct memseg *memsegp)
2490 2491 {
2491 2492 struct memseg **prev_memsegp;
2492 2493 pgcnt_t num;
2493 2494
2494 2495 /* insert in memseg list, decreasing number of pages order */
2495 2496
2496 2497 num = MSEG_NPAGES(memsegp);
2497 2498
2498 2499 for (prev_memsegp = &memsegs; *prev_memsegp;
2499 2500 prev_memsegp = &((*prev_memsegp)->next)) {
2500 2501 if (num > MSEG_NPAGES(*prev_memsegp))
2501 2502 break;
2502 2503 }
2503 2504
2504 2505 memsegp->next = *prev_memsegp;
2505 2506 *prev_memsegp = memsegp;
2506 2507
2507 2508 if (kpm_enable) {
2508 2509 memsegp->nextpa = (memsegp->next) ?
2509 2510 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2510 2511
2511 2512 if (prev_memsegp != &memsegs) {
2512 2513 struct memseg *msp;
2513 2514 msp = (struct memseg *)((caddr_t)prev_memsegp -
2514 2515 offsetof(struct memseg, next));
2515 2516 msp->nextpa = va_to_pa(memsegp);
2516 2517 } else {
2517 2518 memsegspa = va_to_pa(memsegs);
2518 2519 }
2519 2520 }
2520 2521 }
2521 2522
2522 2523 /*
2523 2524 * PSM add_physmem_cb(). US-II and newer processors have some
2524 2525 * flavor of the prefetch capability implemented. We exploit
2525 2526 * this capability for optimum performance.
2526 2527 */
2527 2528 #define PREFETCH_BYTES 64
2528 2529
2529 2530 void
2530 2531 add_physmem_cb(page_t *pp, pfn_t pnum)
2531 2532 {
2532 2533 extern void prefetch_page_w(void *);
2533 2534
2534 2535 pp->p_pagenum = pnum;
2535 2536
2536 2537 /*
2537 2538 * Prefetch one more page_t into E$. To prevent future
2538 2539 * mishaps with the sizeof(page_t) changing on us, we
2539 2540 * catch this on debug kernels if we can't bring in the
2540 2541 * entire hpage with 2 PREFETCH_BYTES reads. See
2541 2542 * also, sun4u/cpu/cpu_module.c
2542 2543 */
2543 2544 /*LINTED*/
2544 2545 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2545 2546 prefetch_page_w((char *)pp);
2546 2547 }
2547 2548
2548 2549 /*
2549 2550 * Find memseg with given pfn
2550 2551 */
2551 2552 static struct memseg *
2552 2553 memseg_find(pfn_t base, pfn_t *next)
2553 2554 {
2554 2555 struct memseg *seg;
2555 2556
2556 2557 if (next != NULL)
2557 2558 *next = LONG_MAX;
2558 2559 for (seg = memsegs; seg != NULL; seg = seg->next) {
2559 2560 if (base >= seg->pages_base && base < seg->pages_end)
2560 2561 return (seg);
2561 2562 if (next != NULL && seg->pages_base > base &&
2562 2563 seg->pages_base < *next)
2563 2564 *next = seg->pages_base;
2564 2565 }
2565 2566 return (NULL);
2566 2567 }
2567 2568
2568 2569 /*
2569 2570 * Put page allocated by OBP on prom_ppages
2570 2571 */
2571 2572 static void
2572 2573 kphysm_erase(uint64_t addr, uint64_t len)
2573 2574 {
2574 2575 struct page *pp;
2575 2576 struct memseg *seg;
2576 2577 pfn_t base = btop(addr), next;
2577 2578 pgcnt_t num = btop(len);
2578 2579
2579 2580 while (num != 0) {
2580 2581 pgcnt_t off, left;
2581 2582
2582 2583 seg = memseg_find(base, &next);
2583 2584 if (seg == NULL) {
2584 2585 if (next == LONG_MAX)
2585 2586 break;
2586 2587 left = MIN(next - base, num);
2587 2588 base += left, num -= left;
2588 2589 continue;
2589 2590 }
2590 2591 off = base - seg->pages_base;
2591 2592 pp = seg->pages + off;
2592 2593 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2593 2594 while (num != left) {
2594 2595 /*
2595 2596 * init it, lock it, and hashin on prom_pages vp.
2596 2597 *
2597 2598 * Mark it as NONRELOC to let DR know the page
2598 2599 * is locked long term, otherwise DR hangs when
2599 2600 * trying to remove those pages.
2600 2601 *
2601 2602 * XXX vnode offsets on the prom_ppages vnode
2602 2603 * are page numbers (gack) for >32 bit
2603 2604 * physical memory machines.
2604 2605 */
2605 2606 PP_SETNORELOC(pp);
2606 2607 add_physmem_cb(pp, base);
2607 2608 if (page_trylock(pp, SE_EXCL) == 0)
2608 2609 cmn_err(CE_PANIC, "prom page locked");
2609 2610 (void) page_hashin(pp, &promvp,
2610 2611 (offset_t)base, NULL);
2611 2612 (void) page_pp_lock(pp, 0, 1);
2612 2613 pp++, base++, num--;
2613 2614 }
2614 2615 }
2615 2616 }
2616 2617
2617 2618 static page_t *ppnext;
2618 2619 static pgcnt_t ppleft;
2619 2620
2620 2621 static void *kpm_ppnext;
2621 2622 static pgcnt_t kpm_ppleft;
2622 2623
2623 2624 /*
2624 2625 * Create a memseg
2625 2626 */
2626 2627 static void
2627 2628 kphysm_memseg(uint64_t addr, uint64_t len)
2628 2629 {
2629 2630 pfn_t base = btop(addr);
2630 2631 pgcnt_t num = btop(len);
2631 2632 struct memseg *seg;
2632 2633
2633 2634 seg = memseg_free;
2634 2635 memseg_free = seg->next;
2635 2636 ASSERT(seg != NULL);
2636 2637
2637 2638 seg->pages = ppnext;
2638 2639 seg->epages = ppnext + num;
2639 2640 seg->pages_base = base;
2640 2641 seg->pages_end = base + num;
2641 2642 ppnext += num;
2642 2643 ppleft -= num;
2643 2644
2644 2645 if (kpm_enable) {
2645 2646 pgcnt_t kpnum = ptokpmpr(num);
2646 2647
2647 2648 if (kpnum > kpm_ppleft)
2648 2649 panic("kphysm_memseg: kpm_pp overflow");
2649 2650 seg->pagespa = va_to_pa(seg->pages);
2650 2651 seg->epagespa = va_to_pa(seg->epages);
2651 2652 seg->kpm_pbase = kpmptop(ptokpmp(base));
2652 2653 seg->kpm_nkpmpgs = kpnum;
2653 2654 /*
2654 2655 * In the kpm_smallpage case, the kpm array
2655 2656 * is 1-1 wrt the page array
2656 2657 */
2657 2658 if (kpm_smallpages) {
2658 2659 kpm_spage_t *kpm_pp = kpm_ppnext;
2659 2660
2660 2661 kpm_ppnext = kpm_pp + kpnum;
2661 2662 seg->kpm_spages = kpm_pp;
2662 2663 seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2663 2664 } else {
2664 2665 kpm_page_t *kpm_pp = kpm_ppnext;
2665 2666
2666 2667 kpm_ppnext = kpm_pp + kpnum;
2667 2668 seg->kpm_pages = kpm_pp;
2668 2669 seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2669 2670 /* ASSERT no kpm overlaps */
2670 2671 ASSERT(
2671 2672 memseg_find(base - pmodkpmp(base), NULL) == NULL);
2672 2673 ASSERT(memseg_find(
2673 2674 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2674 2675 }
2675 2676 kpm_ppleft -= kpnum;
2676 2677 }
2677 2678
2678 2679 memseg_list_add(seg);
2679 2680 }
2680 2681
2681 2682 /*
2682 2683 * Add range to free list
2683 2684 */
2684 2685 void
2685 2686 kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2686 2687 {
2687 2688 struct page *pp;
2688 2689 struct memseg *seg;
2689 2690 pfn_t base = btop(addr);
2690 2691 pgcnt_t num = btop(len);
2691 2692
2692 2693 seg = memseg_find(base, NULL);
2693 2694 ASSERT(seg != NULL);
2694 2695 pp = seg->pages + (base - seg->pages_base);
2695 2696
2696 2697 if (reclaim) {
2697 2698 struct page *rpp = pp;
2698 2699 struct page *lpp = pp + num;
2699 2700
2700 2701 /*
2701 2702 * page should be locked on prom_ppages
2702 2703 * unhash and unlock it
2703 2704 */
2704 2705 while (rpp < lpp) {
2705 2706 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2706 2707 ASSERT(PP_ISNORELOC(rpp));
2707 2708 PP_CLRNORELOC(rpp);
2708 2709 page_pp_unlock(rpp, 0, 1);
2709 2710 page_hashout(rpp, NULL);
2710 2711 page_unlock(rpp);
2711 2712 rpp++;
2712 2713 }
2713 2714 }
2714 2715
2715 2716 /*
2716 2717 * add_physmem() initializes the PSM part of the page
2717 2718 * struct by calling the PSM back with add_physmem_cb().
2718 2719 * In addition it coalesces pages into larger pages as
2719 2720 * it initializes them.
2720 2721 */
2721 2722 add_physmem(pp, num, base);
2722 2723 }
2723 2724
2724 2725 /*
2725 2726 * kphysm_init() tackles the problem of initializing physical memory.
2726 2727 */
2727 2728 static void
2728 2729 kphysm_init(void)
2729 2730 {
2730 2731 struct memlist *pmem;
2731 2732
2732 2733 ASSERT(page_hash != NULL && page_hashsz != 0);
2733 2734
2734 2735 ppnext = pp_base;
2735 2736 ppleft = npages;
2736 2737 kpm_ppnext = kpm_pp_base;
2737 2738 kpm_ppleft = kpm_npages;
2738 2739
2739 2740 /*
2740 2741 * installed pages not on nopp_memlist go in memseg list
2741 2742 */
2742 2743 diff_memlists(phys_install, nopp_list, kphysm_memseg);
2743 2744
2744 2745 /*
2745 2746 * Free the avail list
2746 2747 */
2747 2748 for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2748 2749 kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2749 2750
2750 2751 /*
2751 2752 * Erase pages that aren't available
2752 2753 */
2753 2754 diff_memlists(phys_install, phys_avail, kphysm_erase);
2754 2755
2755 2756 build_pfn_hash();
2756 2757 }
2757 2758
2758 2759 /*
2759 2760 * Kernel VM initialization.
2760 2761 * Assumptions about kernel address space ordering:
2761 2762 * (1) gap (user space)
2762 2763 * (2) kernel text
2763 2764 * (3) kernel data/bss
2764 2765 * (4) gap
2765 2766 * (5) kernel data structures
2766 2767 * (6) gap
2767 2768 * (7) debugger (optional)
2768 2769 * (8) monitor
2769 2770 * (9) gap (possibly null)
2770 2771 * (10) dvma
2771 2772 * (11) devices
2772 2773 */
2773 2774 static void
2774 2775 kvm_init(void)
2775 2776 {
2776 2777 /*
2777 2778 * Put the kernel segments in kernel address space.
2778 2779 */
2779 2780 rw_enter(&kas.a_lock, RW_WRITER);
2780 2781 as_avlinit(&kas);
2781 2782
2782 2783 (void) seg_attach(&kas, (caddr_t)KERNELBASE,
2783 2784 (size_t)(e_moddata - KERNELBASE), &ktextseg);
2784 2785 (void) segkmem_create(&ktextseg);
2785 2786
2786 2787 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2787 2788 (size_t)(MMU_PAGESIZE4M), &ktexthole);
2788 2789 (void) segkmem_create(&ktexthole);
2789 2790
2790 2791 (void) seg_attach(&kas, (caddr_t)valloc_base,
2791 2792 (size_t)(econtig32 - valloc_base), &kvalloc);
2792 2793 (void) segkmem_create(&kvalloc);
2793 2794
2794 2795 if (kmem64_base) {
2795 2796 (void) seg_attach(&kas, (caddr_t)kmem64_base,
2796 2797 (size_t)(kmem64_end - kmem64_base), &kmem64);
2797 2798 (void) segkmem_create(&kmem64);
2798 2799 }
2799 2800
2800 2801 /*
2801 2802 * We're about to map out /boot. This is the beginning of the
2802 2803 * system resource management transition. We can no longer
2803 2804 * call into /boot for I/O or memory allocations.
2804 2805 */
2805 2806 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2806 2807 (void) segkmem_create(&kvseg);
2807 2808 hblk_alloc_dynamic = 1;
2808 2809
2809 2810 /*
2810 2811 * we need to preallocate pages for DR operations before enabling large
2811 2812 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2812 2813 */
2813 2814 memseg_remap_init();
2814 2815
2815 2816 /* at this point we are ready to use large page heap */
2816 2817 segkmem_heap_lp_init();
2817 2818
2818 2819 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2819 2820 &kvseg32);
2820 2821 (void) segkmem_create(&kvseg32);
2821 2822
2822 2823 /*
2823 2824 * Create a segment for the debugger.
2824 2825 */
2825 2826 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2826 2827 (void) segkmem_create(&kdebugseg);
2827 2828
2828 2829 rw_exit(&kas.a_lock);
2829 2830 }
2830 2831
2831 2832 char obp_tte_str[] =
2832 2833 "h# %x constant MMU_PAGESHIFT "
2833 2834 "h# %x constant TTE8K "
2834 2835 "h# %x constant SFHME_SIZE "
2835 2836 "h# %x constant SFHME_TTE "
2836 2837 "h# %x constant HMEBLK_TAG "
2837 2838 "h# %x constant HMEBLK_NEXT "
2838 2839 "h# %x constant HMEBLK_MISC "
2839 2840 "h# %x constant HMEBLK_HME1 "
2840 2841 "h# %x constant NHMENTS "
2841 2842 "h# %x constant HBLK_SZMASK "
2842 2843 "h# %x constant HBLK_RANGE_SHIFT "
2843 2844 "h# %x constant HMEBP_HBLK "
2844 2845 "h# %x constant HMEBLK_ENDPA "
2845 2846 "h# %x constant HMEBUCKET_SIZE "
2846 2847 "h# %x constant HTAG_SFMMUPSZ "
2847 2848 "h# %x constant HTAG_BSPAGE_SHIFT "
2848 2849 "h# %x constant HTAG_REHASH_SHIFT "
2849 2850 "h# %x constant SFMMU_INVALID_SHMERID "
2850 2851 "h# %x constant mmu_hashcnt "
2851 2852 "h# %p constant uhme_hash "
2852 2853 "h# %p constant khme_hash "
2853 2854 "h# %x constant UHMEHASH_SZ "
2854 2855 "h# %x constant KHMEHASH_SZ "
2855 2856 "h# %p constant KCONTEXT "
2856 2857 "h# %p constant KHATID "
2857 2858 "h# %x constant ASI_MEM "
2858 2859
2859 2860 ": PHYS-X@ ( phys -- data ) "
2860 2861 " ASI_MEM spacex@ "
2861 2862 "; "
2862 2863
2863 2864 ": PHYS-W@ ( phys -- data ) "
2864 2865 " ASI_MEM spacew@ "
2865 2866 "; "
2866 2867
2867 2868 ": PHYS-L@ ( phys -- data ) "
2868 2869 " ASI_MEM spaceL@ "
2869 2870 "; "
2870 2871
2871 2872 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2872 2873 " 3 * MMU_PAGESHIFT + "
2873 2874 "; "
2874 2875
2875 2876 ": TTE_IS_VALID ( ttep -- flag ) "
2876 2877 " PHYS-X@ 0< "
2877 2878 "; "
2878 2879
2879 2880 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2880 2881 " dup TTE8K = if "
2881 2882 " drop HBLK_RANGE_SHIFT "
2882 2883 " else "
2883 2884 " TTE_PAGE_SHIFT "
2884 2885 " then "
2885 2886 "; "
2886 2887
2887 2888 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2888 2889 " tuck >> swap MMU_PAGESHIFT - << "
2889 2890 "; "
2890 2891
2891 2892 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2892 2893 " >> over xor swap ( hash sfmmup ) "
2893 2894 " KHATID <> if ( hash ) "
2894 2895 " UHMEHASH_SZ and ( bucket ) "
2895 2896 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) "
2896 2897 " else ( hash ) "
2897 2898 " KHMEHASH_SZ and ( bucket ) "
2898 2899 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) "
2899 2900 " then ( hmebp ) "
2900 2901 "; "
2901 2902
2902 2903 ": HME_HASH_TABLE_SEARCH "
2903 2904 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) "
2904 2905 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2905 2906 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) "
2906 2907 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) "
2907 2908 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if "
2908 2909 " true ( sfmmup hmeblkp true ) ( r: hblktag ) "
2909 2910 " else "
2910 2911 " hmeblk_next + phys-x@ false "
2911 2912 " ( sfmmup hmeblkp false ) ( r: hblktag ) "
2912 2913 " then "
2913 2914 " else "
2914 2915 " hmeblk_next + phys-x@ false "
2915 2916 " ( sfmmup hmeblkp false ) ( r: hblktag ) "
2916 2917 " then "
2917 2918 " else "
2918 2919 " drop 0 true "
2919 2920 " then "
2920 2921 " until r> drop "
2921 2922 "; "
2922 2923
2923 2924 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2924 2925 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) "
2925 2926 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )"
2926 2927 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )"
2927 2928 " SFMMU_INVALID_SHMERID or nip ( hblktag ) "
2928 2929 "; "
2929 2930
2930 2931 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2931 2932 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) "
2932 2933 " TTE8K = if ( hmeblkp addr ) "
2933 2934 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) "
2934 2935 " else ( hmeblkp addr ) "
2935 2936 " drop 0 ( hmeblkp 0 ) "
2936 2937 " then ( hmeblkp hme-index ) "
2937 2938 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) "
2938 2939 " SFHME_TTE + ( ttep ) "
2939 2940 "; "
2940 2941
2941 2942 ": unix-tte ( addr cnum -- false | tte-data true ) "
2942 2943 " KCONTEXT = if ( addr ) "
2943 2944 " KHATID ( addr khatid ) "
2944 2945 " else ( addr ) "
2945 2946 " drop false exit ( false ) "
2946 2947 " then "
2947 2948 " ( addr khatid ) "
2948 2949 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) "
2949 2950 " 2dup swap i HME_HASH_SHIFT "
2950 2951 "( addr sfmmup sfmmup addr hmeshift ) "
2951 2952 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) "
2952 2953 " over i 4 pick "
2953 2954 "( addr sfmmup hmebp sfmmup rehash addr ) "
2954 2955 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) "
2955 2956 " HME_HASH_TABLE_SEARCH "
2956 2957 "( addr sfmmup { null | hmeblkp } ) "
2957 2958 " ?dup if ( addr sfmmup hmeblkp ) "
2958 2959 " nip swap HBLK_TO_TTEP ( ttep ) "
2959 2960 " dup TTE_IS_VALID if ( valid-ttep ) "
2960 2961 " PHYS-X@ true ( tte-data true ) "
2961 2962 " else ( invalid-tte ) "
2962 2963 " drop false ( false ) "
2963 2964 " then ( false | tte-data true ) "
2964 2965 " unloop exit ( false | tte-data true ) "
2965 2966 " then ( addr sfmmup ) "
2966 2967 " loop ( addr sfmmup ) "
2967 2968 " 2drop false ( false ) "
2968 2969 "; "
2969 2970 ;
2970 2971
2971 2972 void
2972 2973 create_va_to_tte(void)
2973 2974 {
2974 2975 char *bp;
2975 2976 extern int khmehash_num, uhmehash_num;
2976 2977 extern struct hmehash_bucket *khme_hash, *uhme_hash;
2977 2978
2978 2979 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field))
2979 2980
2980 2981 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2981 2982
2982 2983 /*
2983 2984 * Teach obp how to parse our sw ttes.
2984 2985 */
2985 2986 (void) sprintf(bp, obp_tte_str,
2986 2987 MMU_PAGESHIFT,
2987 2988 TTE8K,
2988 2989 sizeof (struct sf_hment),
2989 2990 OFFSET(struct sf_hment, hme_tte),
2990 2991 OFFSET(struct hme_blk, hblk_tag),
2991 2992 OFFSET(struct hme_blk, hblk_nextpa),
2992 2993 OFFSET(struct hme_blk, hblk_misc),
2993 2994 OFFSET(struct hme_blk, hblk_hme),
2994 2995 NHMENTS,
2995 2996 HBLK_SZMASK,
2996 2997 HBLK_RANGE_SHIFT,
2997 2998 OFFSET(struct hmehash_bucket, hmeh_nextpa),
2998 2999 HMEBLK_ENDPA,
2999 3000 sizeof (struct hmehash_bucket),
3000 3001 HTAG_SFMMUPSZ,
3001 3002 HTAG_BSPAGE_SHIFT,
3002 3003 HTAG_REHASH_SHIFT,
3003 3004 SFMMU_INVALID_SHMERID,
3004 3005 mmu_hashcnt,
3005 3006 (caddr_t)va_to_pa((caddr_t)uhme_hash),
3006 3007 (caddr_t)va_to_pa((caddr_t)khme_hash),
3007 3008 UHMEHASH_SZ,
3008 3009 KHMEHASH_SZ,
3009 3010 KCONTEXT,
3010 3011 KHATID,
3011 3012 ASI_MEM);
3012 3013 prom_interpret(bp, 0, 0, 0, 0, 0);
3013 3014
3014 3015 kobj_free(bp, MMU_PAGESIZE);
3015 3016 }
3016 3017
3017 3018 void
3018 3019 install_va_to_tte(void)
3019 3020 {
3020 3021 /*
3021 3022 * advise prom that it can use unix-tte
3022 3023 */
3023 3024 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
3024 3025 }
3025 3026
3026 3027 /*
3027 3028 * Here we add "device-type=console" for /os-io node, for currently
3028 3029 * our kernel console output only supports displaying text and
3029 3030 * performing cursor-positioning operations (through kernel framebuffer
3030 3031 * driver) and it doesn't support other functionalities required for a
3031 3032 * standard "display" device as specified in 1275 spec. The main missing
3032 3033 * interface defined by the 1275 spec is "draw-logo".
3033 3034 * also see the comments above prom_stdout_is_framebuffer().
3034 3035 */
3035 3036 static char *create_node =
3036 3037 "\" /\" find-device "
3037 3038 "new-device "
3038 3039 "\" os-io\" device-name "
3039 3040 "\" "OBP_DISPLAY_CONSOLE"\" device-type "
3040 3041 ": cb-r/w ( adr,len method$ -- #read/#written ) "
3041 3042 " 2>r swap 2 2r> ['] $callback catch if "
3042 3043 " 2drop 3drop 0 "
3043 3044 " then "
3044 3045 "; "
3045 3046 ": read ( adr,len -- #read ) "
3046 3047 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then "
3047 3048 " ( retN ... ret1 N ) "
3048 3049 " ?dup if "
3049 3050 " swap >r 1- 0 ?do drop loop r> "
3050 3051 " else "
3051 3052 " -2 "
3052 3053 " then "
3053 3054 "; "
3054 3055 ": write ( adr,len -- #written ) "
3055 3056 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then "
3056 3057 " ( retN ... ret1 N ) "
3057 3058 " ?dup if "
3058 3059 " swap >r 1- 0 ?do drop loop r> "
3059 3060 " else "
3060 3061 " 0 "
3061 3062 " then "
3062 3063 "; "
3063 3064 ": poll-tty ( -- ) ; "
3064 3065 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; "
3065 3066 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; "
3066 3067 ": cb-give/take ( $method -- ) "
3067 3068 " 0 -rot ['] $callback catch ?dup if "
3068 3069 " >r 2drop 2drop r> throw "
3069 3070 " else "
3070 3071 " 0 ?do drop loop "
3071 3072 " then "
3072 3073 "; "
3073 3074 ": give ( -- ) \" exit-input\" cb-give/take ; "
3074 3075 ": take ( -- ) \" enter-input\" cb-give/take ; "
3075 3076 ": open ( -- ok? ) true ; "
3076 3077 ": close ( -- ) ; "
3077 3078 "finish-device "
3078 3079 "device-end ";
3079 3080
3080 3081 /*
3081 3082 * Create the OBP input/output node (FCode serial driver).
3082 3083 * It is needed for both USB console keyboard and for
3083 3084 * the kernel terminal emulator. It is too early to check for a
3084 3085 * kernel console compatible framebuffer now, so we create this
3085 3086 * so that we're ready if we need to enable kernel terminal emulation.
3086 3087 *
3087 3088 * When the USB software takes over the input device at the time
3088 3089 * consconfig runs, OBP's stdin is redirected to this node.
3089 3090 * Whenever the FORTH user interface is used after this switch,
3090 3091 * the node will call back into the kernel for console input.
3091 3092 * If a serial device such as ttya or a UART with a Type 5 keyboard
3092 3093 * attached is used, OBP takes over the serial device when the system
3093 3094 * goes to the debugger after the system is booted. This sharing
3094 3095 * of the relatively simple serial device is difficult but possible.
3095 3096 * Sharing the USB host controller is impossible due its complexity.
3096 3097 *
3097 3098 * Similarly to USB keyboard input redirection, after consconfig_dacf
3098 3099 * configures a kernel console framebuffer as the standard output
3099 3100 * device, OBP's stdout is switched to to vector through the
3100 3101 * /os-io node into the kernel terminal emulator.
3101 3102 */
3102 3103 static void
3103 3104 startup_create_io_node(void)
3104 3105 {
3105 3106 prom_interpret(create_node, 0, 0, 0, 0, 0);
3106 3107 }
3107 3108
3108 3109
3109 3110 static void
3110 3111 do_prom_version_check(void)
3111 3112 {
3112 3113 int i;
3113 3114 pnode_t node;
3114 3115 char buf[64];
3115 3116 static char drev[] = "Down-rev firmware detected%s\n"
3116 3117 "\tPlease upgrade to the following minimum version:\n"
3117 3118 "\t\t%s\n";
3118 3119
3119 3120 i = prom_version_check(buf, sizeof (buf), &node);
3120 3121
3121 3122 if (i == PROM_VER64_OK)
3122 3123 return;
3123 3124
3124 3125 if (i == PROM_VER64_UPGRADE) {
3125 3126 cmn_err(CE_WARN, drev, "", buf);
3126 3127
3127 3128 #ifdef DEBUG
3128 3129 prom_enter_mon(); /* Type 'go' to continue */
3129 3130 cmn_err(CE_WARN, "Booting with down-rev firmware\n");
3130 3131 return;
3131 3132 #else
3132 3133 halt(0);
3133 3134 #endif
3134 3135 }
3135 3136
3136 3137 /*
3137 3138 * The other possibility is that this is a server running
3138 3139 * good firmware, but down-rev firmware was detected on at
3139 3140 * least one other cpu board. We just complain if we see
3140 3141 * that.
3141 3142 */
3142 3143 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
3143 3144 }
3144 3145
3145 3146
3146 3147 /*
3147 3148 * Must be defined in platform dependent code.
3148 3149 */
3149 3150 extern caddr_t modtext;
3150 3151 extern size_t modtext_sz;
3151 3152 extern caddr_t moddata;
3152 3153
3153 3154 #define HEAPTEXT_ARENA(addr) \
3154 3155 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3155 3156 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3156 3157 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3157 3158
3158 3159 #define HEAPTEXT_OVERSIZED(addr) \
3159 3160 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3160 3161
3161 3162 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \
3162 3163 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3163 3164 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3164 3165
3165 3166 vmem_t *texthole_source[HEAPTEXT_NARENAS];
3166 3167 vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3167 3168 kmutex_t texthole_lock;
3168 3169
3169 3170 char kern_bootargs[OBP_MAXPATHLEN];
3170 3171 char kern_bootfile[OBP_MAXPATHLEN];
3171 3172
3172 3173 void
3173 3174 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3174 3175 {
3175 3176 uintptr_t addr, limit;
3176 3177
3177 3178 addr = HEAPTEXT_BASE;
3178 3179 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3179 3180
3180 3181 /*
3181 3182 * Before we initialize the text_arena, we want to punch holes in the
3182 3183 * underlying heaptext_arena. This guarantees that for any text
3183 3184 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3184 3185 */
3185 3186 for (; addr + HEAPTEXT_UNMAPPED <= limit;
3186 3187 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3187 3188 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3188 3189 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3189 3190 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3190 3191 }
3191 3192
3192 3193 /*
3193 3194 * Allocate one page at the oversize to break up the text region
3194 3195 * from the oversized region.
3195 3196 */
3196 3197 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3197 3198 (void *)limit, (void *)(limit + PAGESIZE),
3198 3199 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3199 3200
3200 3201 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3201 3202 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3202 3203 heaptext_arena, 0, VM_SLEEP);
3203 3204 *data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3204 3205 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3205 3206 }
3206 3207
3207 3208 caddr_t
3208 3209 kobj_text_alloc(vmem_t *arena, size_t size)
3209 3210 {
3210 3211 caddr_t rval, better;
3211 3212
3212 3213 /*
3213 3214 * First, try a sleeping allocation.
3214 3215 */
3215 3216 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3216 3217
3217 3218 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3218 3219 return (rval);
3219 3220
3220 3221 /*
3221 3222 * We didn't get the area that we wanted. We're going to try to do an
3222 3223 * allocation with explicit constraints.
3223 3224 */
3224 3225 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3225 3226 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3226 3227 VM_NOSLEEP | VM_BESTFIT);
3227 3228
3228 3229 if (better != NULL) {
3229 3230 /*
3230 3231 * That worked. Free our first attempt and return.
3231 3232 */
3232 3233 vmem_free(arena, rval, size);
3233 3234 return (better);
3234 3235 }
3235 3236
3236 3237 /*
3237 3238 * That didn't work; we'll have to return our first attempt.
3238 3239 */
3239 3240 return (rval);
3240 3241 }
3241 3242
3242 3243 caddr_t
3243 3244 kobj_texthole_alloc(caddr_t addr, size_t size)
3244 3245 {
3245 3246 int arena = HEAPTEXT_ARENA(addr);
3246 3247 char c[30];
3247 3248 uintptr_t base;
3248 3249
3249 3250 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3250 3251 /*
3251 3252 * If this is an oversized allocation or it is allocated in
3252 3253 * the nucleus data page, there is no text hole available for
3253 3254 * it; return NULL.
3254 3255 */
3255 3256 return (NULL);
3256 3257 }
3257 3258
3258 3259 mutex_enter(&texthole_lock);
3259 3260
3260 3261 if (texthole_arena[arena] == NULL) {
3261 3262 ASSERT(texthole_source[arena] == NULL);
3262 3263
3263 3264 if (arena == 0) {
3264 3265 texthole_source[0] = vmem_create("module_text_holesrc",
3265 3266 (void *)(KERNELBASE + MMU_PAGESIZE4M),
3266 3267 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3267 3268 0, VM_SLEEP);
3268 3269 } else {
3269 3270 base = HEAPTEXT_BASE +
3270 3271 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3271 3272
3272 3273 (void) snprintf(c, sizeof (c),
3273 3274 "heaptext_holesrc_%d", arena);
3274 3275
3275 3276 texthole_source[arena] = vmem_create(c, (void *)base,
3276 3277 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3277 3278 0, VM_SLEEP);
3278 3279 }
3279 3280
3280 3281 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3281 3282
3282 3283 texthole_arena[arena] = vmem_create(c, NULL, 0,
3283 3284 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3284 3285 texthole_source[arena], 0, VM_SLEEP);
3285 3286 }
3286 3287
3287 3288 mutex_exit(&texthole_lock);
3288 3289
3289 3290 ASSERT(texthole_arena[arena] != NULL);
3290 3291 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3291 3292 return (vmem_alloc(texthole_arena[arena], size,
3292 3293 VM_BESTFIT | VM_NOSLEEP));
3293 3294 }
3294 3295
3295 3296 void
3296 3297 kobj_texthole_free(caddr_t addr, size_t size)
3297 3298 {
3298 3299 int arena = HEAPTEXT_ARENA(addr);
3299 3300
3300 3301 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3301 3302 ASSERT(texthole_arena[arena] != NULL);
3302 3303 vmem_free(texthole_arena[arena], addr, size);
3303 3304 }
3304 3305
3305 3306 void
3306 3307 release_bootstrap(void)
3307 3308 {
3308 3309 if (&cif_init)
3309 3310 cif_init();
3310 3311 }
|
↓ open down ↓ |
859 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX