Print this page
NEX-5184 backport illumos 6065 page hash: use a static inline instead of a macro
Reviewed by: Jean McCormack <jean.mccormack@nexenta.com>
Reviewed by: Kevin Crowe <kevin.crowe@nexenta.com>
6065 page hash: use a static inline instead of a macro
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Robert Mustacchi <rm@joyent.com>
NEX-5164 backport illumos 6514 AS_* lock macros simplification
Reviewed by: Kevin Crowe <kevin.crowe@nexenta.com>
6514 AS_* lock macros simplification
Reviewed by: Piotr Jasiukajtis <estibi@me.com>
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
Reviewed by: Albert Lee <trisk@omniti.com>
Approved by: Dan McDonald <danmcd@omniti.com>
re #13613 rb4516 Tunables needs volatile keyword
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/vm_page.c
+++ new/usr/src/uts/common/vm/vm_page.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23 + * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
23 24 * Copyright (c) 2015, Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
24 25 * Copyright (c) 2015, 2016 by Delphix. All rights reserved.
25 26 */
26 27
27 28 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
28 29 /* All Rights Reserved */
29 30
30 31 /*
31 32 * University Copyright- Copyright (c) 1982, 1986, 1988
32 33 * The Regents of the University of California
33 34 * All Rights Reserved
34 35 *
35 36 * University Acknowledgment- Portions of this document are derived from
36 37 * software developed by the University of California, Berkeley, and its
37 38 * contributors.
38 39 */
39 40
40 41 /*
41 42 * VM - physical page management.
42 43 */
43 44
44 45 #include <sys/types.h>
45 46 #include <sys/t_lock.h>
46 47 #include <sys/param.h>
47 48 #include <sys/systm.h>
48 49 #include <sys/errno.h>
49 50 #include <sys/time.h>
50 51 #include <sys/vnode.h>
51 52 #include <sys/vm.h>
52 53 #include <sys/vtrace.h>
53 54 #include <sys/swap.h>
54 55 #include <sys/cmn_err.h>
55 56 #include <sys/tuneable.h>
56 57 #include <sys/sysmacros.h>
57 58 #include <sys/cpuvar.h>
58 59 #include <sys/callb.h>
59 60 #include <sys/debug.h>
60 61 #include <sys/tnf_probe.h>
61 62 #include <sys/condvar_impl.h>
62 63 #include <sys/mem_config.h>
63 64 #include <sys/mem_cage.h>
64 65 #include <sys/kmem.h>
65 66 #include <sys/atomic.h>
66 67 #include <sys/strlog.h>
67 68 #include <sys/mman.h>
68 69 #include <sys/ontrap.h>
69 70 #include <sys/lgrp.h>
70 71 #include <sys/vfs.h>
71 72
72 73 #include <vm/hat.h>
73 74 #include <vm/anon.h>
74 75 #include <vm/page.h>
75 76 #include <vm/seg.h>
76 77 #include <vm/pvn.h>
77 78 #include <vm/seg_kmem.h>
78 79 #include <vm/vm_dep.h>
79 80 #include <sys/vm_usage.h>
80 81 #include <fs/fs_subr.h>
81 82 #include <sys/ddi.h>
82 83 #include <sys/modctl.h>
83 84
84 85 static pgcnt_t max_page_get; /* max page_get request size in pages */
85 86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */
86 87
87 88 /*
88 89 * freemem_lock protects all freemem variables:
89 90 * availrmem. Also this lock protects the globals which track the
90 91 * availrmem changes for accurate kernel footprint calculation.
91 92 * See below for an explanation of these
92 93 * globals.
93 94 */
94 95 kmutex_t freemem_lock;
95 96 pgcnt_t availrmem;
96 97 pgcnt_t availrmem_initial;
97 98
98 99 /*
99 100 * These globals track availrmem changes to get a more accurate
100 101 * estimate of tke kernel size. Historically pp_kernel is used for
101 102 * kernel size and is based on availrmem. But availrmem is adjusted for
102 103 * locked pages in the system not just for kernel locked pages.
103 104 * These new counters will track the pages locked through segvn and
104 105 * by explicit user locking.
105 106 *
106 107 * pages_locked : How many pages are locked because of user specified
107 108 * locking through mlock or plock.
108 109 *
109 110 * pages_useclaim,pages_claimed : These two variables track the
110 111 * claim adjustments because of the protection changes on a segvn segment.
111 112 *
112 113 * All these globals are protected by the same lock which protects availrmem.
113 114 */
114 115 pgcnt_t pages_locked = 0;
115 116 pgcnt_t pages_useclaim = 0;
116 117 pgcnt_t pages_claimed = 0;
117 118
118 119
119 120 /*
120 121 * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
121 122 */
122 123 static kmutex_t new_freemem_lock;
123 124 static uint_t freemem_wait; /* someone waiting for freemem */
124 125 static kcondvar_t freemem_cv;
125 126
126 127 /*
127 128 * The logical page free list is maintained as two lists, the 'free'
128 129 * and the 'cache' lists.
129 130 * The free list contains those pages that should be reused first.
130 131 *
131 132 * The implementation of the lists is machine dependent.
132 133 * page_get_freelist(), page_get_cachelist(),
133 134 * page_list_sub(), and page_list_add()
134 135 * form the interface to the machine dependent implementation.
135 136 *
136 137 * Pages with p_free set are on the cache list.
137 138 * Pages with p_free and p_age set are on the free list,
138 139 *
139 140 * A page may be locked while on either list.
140 141 */
141 142
142 143 /*
143 144 * free list accounting stuff.
144 145 *
145 146 *
146 147 * Spread out the value for the number of pages on the
147 148 * page free and page cache lists. If there is just one
148 149 * value, then it must be under just one lock.
149 150 * The lock contention and cache traffic are a real bother.
150 151 *
151 152 * When we acquire and then drop a single pcf lock
152 153 * we can start in the middle of the array of pcf structures.
153 154 * If we acquire more than one pcf lock at a time, we need to
154 155 * start at the front to avoid deadlocking.
155 156 *
156 157 * pcf_count holds the number of pages in each pool.
157 158 *
158 159 * pcf_block is set when page_create_get_something() has asked the
159 160 * PSM page freelist and page cachelist routines without specifying
160 161 * a color and nothing came back. This is used to block anything
161 162 * else from moving pages from one list to the other while the
162 163 * lists are searched again. If a page is freeed while pcf_block is
163 164 * set, then pcf_reserve is incremented. pcgs_unblock() takes care
164 165 * of clearning pcf_block, doing the wakeups, etc.
165 166 */
166 167
167 168 #define MAX_PCF_FANOUT NCPU
168 169 static uint_t pcf_fanout = 1; /* Will get changed at boot time */
169 170 static uint_t pcf_fanout_mask = 0;
170 171
171 172 struct pcf {
172 173 kmutex_t pcf_lock; /* protects the structure */
173 174 uint_t pcf_count; /* page count */
174 175 uint_t pcf_wait; /* number of waiters */
175 176 uint_t pcf_block; /* pcgs flag to page_free() */
176 177 uint_t pcf_reserve; /* pages freed after pcf_block set */
177 178 uint_t pcf_fill[10]; /* to line up on the caches */
178 179 };
179 180
180 181 /*
181 182 * PCF_INDEX hash needs to be dynamic (every so often the hash changes where
182 183 * it will hash the cpu to). This is done to prevent a drain condition
183 184 * from happening. This drain condition will occur when pcf_count decrement
184 185 * occurs on cpu A and the increment of pcf_count always occurs on cpu B. An
185 186 * example of this shows up with device interrupts. The dma buffer is allocated
186 187 * by the cpu requesting the IO thus the pcf_count is decremented based on that.
187 188 * When the memory is returned by the interrupt thread, the pcf_count will be
188 189 * incremented based on the cpu servicing the interrupt.
189 190 */
190 191 static struct pcf pcf[MAX_PCF_FANOUT];
191 192 #define PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \
192 193 (randtick() >> 24)) & (pcf_fanout_mask))
193 194
194 195 static int pcf_decrement_bucket(pgcnt_t);
195 196 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int);
196 197
197 198 kmutex_t pcgs_lock; /* serializes page_create_get_ */
198 199 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */
199 200 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */
200 201 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */
201 202
202 203 #ifdef VM_STATS
203 204
204 205 /*
205 206 * No locks, but so what, they are only statistics.
206 207 */
207 208
208 209 static struct page_tcnt {
209 210 int pc_free_cache; /* free's into cache list */
210 211 int pc_free_dontneed; /* free's with dontneed */
211 212 int pc_free_pageout; /* free's from pageout */
212 213 int pc_free_free; /* free's into free list */
213 214 int pc_free_pages; /* free's into large page free list */
214 215 int pc_destroy_pages; /* large page destroy's */
215 216 int pc_get_cache; /* get's from cache list */
216 217 int pc_get_free; /* get's from free list */
217 218 int pc_reclaim; /* reclaim's */
218 219 int pc_abortfree; /* abort's of free pages */
219 220 int pc_find_hit; /* find's that find page */
220 221 int pc_find_miss; /* find's that don't find page */
221 222 int pc_destroy_free; /* # of free pages destroyed */
222 223 #define PC_HASH_CNT (4*PAGE_HASHAVELEN)
223 224 int pc_find_hashlen[PC_HASH_CNT+1];
224 225 int pc_addclaim_pages;
225 226 int pc_subclaim_pages;
226 227 int pc_free_replacement_page[2];
227 228 int pc_try_demote_pages[6];
228 229 int pc_demote_pages[2];
229 230 } pagecnt;
230 231
231 232 uint_t hashin_count;
232 233 uint_t hashin_not_held;
233 234 uint_t hashin_already;
234 235
235 236 uint_t hashout_count;
236 237 uint_t hashout_not_held;
237 238
238 239 uint_t page_create_count;
239 240 uint_t page_create_not_enough;
240 241 uint_t page_create_not_enough_again;
241 242 uint_t page_create_zero;
242 243 uint_t page_create_hashout;
243 244 uint_t page_create_page_lock_failed;
244 245 uint_t page_create_trylock_failed;
245 246 uint_t page_create_found_one;
246 247 uint_t page_create_hashin_failed;
247 248 uint_t page_create_dropped_phm;
248 249
249 250 uint_t page_create_new;
250 251 uint_t page_create_exists;
251 252 uint_t page_create_putbacks;
252 253 uint_t page_create_overshoot;
253 254
254 255 uint_t page_reclaim_zero;
255 256 uint_t page_reclaim_zero_locked;
256 257
257 258 uint_t page_rename_exists;
258 259 uint_t page_rename_count;
259 260
260 261 uint_t page_lookup_cnt[20];
261 262 uint_t page_lookup_nowait_cnt[10];
262 263 uint_t page_find_cnt;
263 264 uint_t page_exists_cnt;
264 265 uint_t page_exists_forreal_cnt;
265 266 uint_t page_lookup_dev_cnt;
266 267 uint_t get_cachelist_cnt;
267 268 uint_t page_create_cnt[10];
268 269 uint_t alloc_pages[9];
269 270 uint_t page_exphcontg[19];
270 271 uint_t page_create_large_cnt[10];
271 272
272 273 #endif
273 274
274 275 static inline page_t *
275 276 page_hash_search(ulong_t index, vnode_t *vnode, u_offset_t off)
276 277 {
277 278 uint_t mylen = 0;
278 279 page_t *page;
279 280
280 281 for (page = page_hash[index]; page; page = page->p_hash, mylen++)
281 282 if (page->p_vnode == vnode && page->p_offset == off)
282 283 break;
283 284
284 285 #ifdef VM_STATS
285 286 if (page != NULL)
286 287 pagecnt.pc_find_hit++;
287 288 else
288 289 pagecnt.pc_find_miss++;
289 290
290 291 pagecnt.pc_find_hashlen[MIN(mylen, PC_HASH_CNT)]++;
291 292 #endif
292 293
293 294 return (page);
294 295 }
295 296
296 297
297 298 #ifdef DEBUG
298 299 #define MEMSEG_SEARCH_STATS
299 300 #endif
300 301
301 302 #ifdef MEMSEG_SEARCH_STATS
302 303 struct memseg_stats {
303 304 uint_t nsearch;
304 305 uint_t nlastwon;
305 306 uint_t nhashwon;
306 307 uint_t nnotfound;
307 308 } memseg_stats;
308 309
309 310 #define MEMSEG_STAT_INCR(v) \
310 311 atomic_inc_32(&memseg_stats.v)
311 312 #else
312 313 #define MEMSEG_STAT_INCR(x)
313 314 #endif
314 315
315 316 struct memseg *memsegs; /* list of memory segments */
316 317
317 318 /*
318 319 * /etc/system tunable to control large page allocation hueristic.
319 320 *
320 321 * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup
321 322 * for large page allocation requests. If a large page is not readily
322 323 * avaliable on the local freelists we will go through additional effort
323 324 * to create a large page, potentially moving smaller pages around to coalesce
324 325 * larger pages in the local lgroup.
325 326 * Default value of LPAP_DEFAULT will go to remote freelists if large pages
326 327 * are not readily available in the local lgroup.
327 328 */
328 329 enum lpap {
329 330 LPAP_DEFAULT, /* default large page allocation policy */
330 331 LPAP_LOCAL /* local large page allocation policy */
331 332 };
332 333
333 334 enum lpap lpg_alloc_prefer = LPAP_DEFAULT;
334 335
335 336 static void page_init_mem_config(void);
336 337 static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
337 338 static void page_do_hashout(page_t *);
338 339 static void page_capture_init();
339 340 int page_capture_take_action(page_t *, uint_t, void *);
340 341
341 342 static void page_demote_vp_pages(page_t *);
342 343
343 344
344 345 void
345 346 pcf_init(void)
346 347 {
347 348 if (boot_ncpus != -1) {
348 349 pcf_fanout = boot_ncpus;
349 350 } else {
350 351 pcf_fanout = max_ncpus;
351 352 }
352 353 #ifdef sun4v
353 354 /*
354 355 * Force at least 4 buckets if possible for sun4v.
355 356 */
356 357 pcf_fanout = MAX(pcf_fanout, 4);
357 358 #endif /* sun4v */
358 359
359 360 /*
360 361 * Round up to the nearest power of 2.
361 362 */
362 363 pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT);
363 364 if (!ISP2(pcf_fanout)) {
364 365 pcf_fanout = 1 << highbit(pcf_fanout);
365 366
366 367 if (pcf_fanout > MAX_PCF_FANOUT) {
367 368 pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1);
368 369 }
369 370 }
370 371 pcf_fanout_mask = pcf_fanout - 1;
371 372 }
372 373
373 374 /*
374 375 * vm subsystem related initialization
375 376 */
376 377 void
377 378 vm_init(void)
378 379 {
379 380 boolean_t callb_vm_cpr(void *, int);
380 381
381 382 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
382 383 page_init_mem_config();
383 384 page_retire_init();
384 385 vm_usage_init();
385 386 page_capture_init();
386 387 }
387 388
388 389 /*
389 390 * This function is called at startup and when memory is added or deleted.
390 391 */
391 392 void
392 393 init_pages_pp_maximum()
393 394 {
394 395 static pgcnt_t p_min;
395 396 static pgcnt_t pages_pp_maximum_startup;
396 397 static pgcnt_t avrmem_delta;
397 398 static int init_done;
398 399 static int user_set; /* true if set in /etc/system */
399 400
400 401 if (init_done == 0) {
401 402
402 403 /* If the user specified a value, save it */
403 404 if (pages_pp_maximum != 0) {
404 405 user_set = 1;
405 406 pages_pp_maximum_startup = pages_pp_maximum;
406 407 }
407 408
408 409 /*
409 410 * Setting of pages_pp_maximum is based first time
410 411 * on the value of availrmem just after the start-up
411 412 * allocations. To preserve this relationship at run
412 413 * time, use a delta from availrmem_initial.
413 414 */
414 415 ASSERT(availrmem_initial >= availrmem);
415 416 avrmem_delta = availrmem_initial - availrmem;
416 417
417 418 /* The allowable floor of pages_pp_maximum */
418 419 p_min = tune.t_minarmem + 100;
419 420
420 421 /* Make sure we don't come through here again. */
421 422 init_done = 1;
422 423 }
423 424 /*
424 425 * Determine pages_pp_maximum, the number of currently available
425 426 * pages (availrmem) that can't be `locked'. If not set by
426 427 * the user, we set it to 4% of the currently available memory
427 428 * plus 4MB.
428 429 * But we also insist that it be greater than tune.t_minarmem;
429 430 * otherwise a process could lock down a lot of memory, get swapped
430 431 * out, and never have enough to get swapped back in.
431 432 */
432 433 if (user_set)
433 434 pages_pp_maximum = pages_pp_maximum_startup;
434 435 else
435 436 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
436 437 + btop(4 * 1024 * 1024);
437 438
438 439 if (pages_pp_maximum <= p_min) {
439 440 pages_pp_maximum = p_min;
440 441 }
441 442 }
442 443
443 444 void
444 445 set_max_page_get(pgcnt_t target_total_pages)
445 446 {
446 447 max_page_get = target_total_pages / 2;
447 448 }
448 449
449 450 static pgcnt_t pending_delete;
450 451
451 452 /*ARGSUSED*/
452 453 static void
453 454 page_mem_config_post_add(
454 455 void *arg,
455 456 pgcnt_t delta_pages)
456 457 {
457 458 set_max_page_get(total_pages - pending_delete);
458 459 init_pages_pp_maximum();
459 460 }
460 461
461 462 /*ARGSUSED*/
462 463 static int
463 464 page_mem_config_pre_del(
464 465 void *arg,
465 466 pgcnt_t delta_pages)
466 467 {
467 468 pgcnt_t nv;
468 469
469 470 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
470 471 set_max_page_get(total_pages - nv);
471 472 return (0);
472 473 }
473 474
474 475 /*ARGSUSED*/
475 476 static void
476 477 page_mem_config_post_del(
477 478 void *arg,
478 479 pgcnt_t delta_pages,
479 480 int cancelled)
480 481 {
481 482 pgcnt_t nv;
482 483
483 484 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
484 485 set_max_page_get(total_pages - nv);
485 486 if (!cancelled)
486 487 init_pages_pp_maximum();
487 488 }
488 489
489 490 static kphysm_setup_vector_t page_mem_config_vec = {
490 491 KPHYSM_SETUP_VECTOR_VERSION,
491 492 page_mem_config_post_add,
492 493 page_mem_config_pre_del,
493 494 page_mem_config_post_del,
494 495 };
495 496
496 497 static void
497 498 page_init_mem_config(void)
498 499 {
499 500 int ret;
500 501
501 502 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
502 503 ASSERT(ret == 0);
503 504 }
504 505
505 506 /*
506 507 * Evenly spread out the PCF counters for large free pages
507 508 */
508 509 static void
509 510 page_free_large_ctr(pgcnt_t npages)
510 511 {
511 512 static struct pcf *p = pcf;
512 513 pgcnt_t lump;
513 514
514 515 freemem += npages;
515 516
516 517 lump = roundup(npages, pcf_fanout) / pcf_fanout;
517 518
518 519 while (npages > 0) {
519 520
520 521 ASSERT(!p->pcf_block);
521 522
522 523 if (lump < npages) {
523 524 p->pcf_count += (uint_t)lump;
524 525 npages -= lump;
525 526 } else {
526 527 p->pcf_count += (uint_t)npages;
527 528 npages = 0;
528 529 }
529 530
530 531 ASSERT(!p->pcf_wait);
531 532
532 533 if (++p > &pcf[pcf_fanout - 1])
533 534 p = pcf;
534 535 }
535 536
536 537 ASSERT(npages == 0);
537 538 }
538 539
539 540 /*
540 541 * Add a physical chunk of memory to the system free lists during startup.
541 542 * Platform specific startup() allocates the memory for the page structs.
542 543 *
543 544 * num - number of page structures
544 545 * base - page number (pfn) to be associated with the first page.
545 546 *
546 547 * Since we are doing this during startup (ie. single threaded), we will
547 548 * use shortcut routines to avoid any locking overhead while putting all
548 549 * these pages on the freelists.
549 550 *
550 551 * NOTE: Any changes performed to page_free(), must also be performed to
551 552 * add_physmem() since this is how we initialize all page_t's at
552 553 * boot time.
553 554 */
554 555 void
555 556 add_physmem(
556 557 page_t *pp,
557 558 pgcnt_t num,
558 559 pfn_t pnum)
559 560 {
560 561 page_t *root = NULL;
561 562 uint_t szc = page_num_pagesizes() - 1;
562 563 pgcnt_t large = page_get_pagecnt(szc);
563 564 pgcnt_t cnt = 0;
564 565
565 566 TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
566 567 "add_physmem:pp %p num %lu", pp, num);
567 568
568 569 /*
569 570 * Arbitrarily limit the max page_get request
570 571 * to 1/2 of the page structs we have.
571 572 */
572 573 total_pages += num;
573 574 set_max_page_get(total_pages);
574 575
575 576 PLCNT_MODIFY_MAX(pnum, (long)num);
576 577
577 578 /*
578 579 * The physical space for the pages array
579 580 * representing ram pages has already been
580 581 * allocated. Here we initialize each lock
581 582 * in the page structure, and put each on
582 583 * the free list
583 584 */
584 585 for (; num; pp++, pnum++, num--) {
585 586
586 587 /*
587 588 * this needs to fill in the page number
588 589 * and do any other arch specific initialization
589 590 */
590 591 add_physmem_cb(pp, pnum);
591 592
592 593 pp->p_lckcnt = 0;
593 594 pp->p_cowcnt = 0;
594 595 pp->p_slckcnt = 0;
595 596
596 597 /*
597 598 * Initialize the page lock as unlocked, since nobody
598 599 * can see or access this page yet.
599 600 */
600 601 pp->p_selock = 0;
601 602
602 603 /*
603 604 * Initialize IO lock
604 605 */
605 606 page_iolock_init(pp);
606 607
607 608 /*
608 609 * initialize other fields in the page_t
609 610 */
610 611 PP_SETFREE(pp);
611 612 page_clr_all_props(pp);
612 613 PP_SETAGED(pp);
613 614 pp->p_offset = (u_offset_t)-1;
614 615 pp->p_next = pp;
615 616 pp->p_prev = pp;
616 617
617 618 /*
618 619 * Simple case: System doesn't support large pages.
619 620 */
620 621 if (szc == 0) {
621 622 pp->p_szc = 0;
622 623 page_free_at_startup(pp);
623 624 continue;
624 625 }
625 626
626 627 /*
627 628 * Handle unaligned pages, we collect them up onto
628 629 * the root page until we have a full large page.
629 630 */
630 631 if (!IS_P2ALIGNED(pnum, large)) {
631 632
632 633 /*
633 634 * If not in a large page,
634 635 * just free as small page.
635 636 */
636 637 if (root == NULL) {
637 638 pp->p_szc = 0;
638 639 page_free_at_startup(pp);
639 640 continue;
640 641 }
641 642
642 643 /*
643 644 * Link a constituent page into the large page.
644 645 */
645 646 pp->p_szc = szc;
646 647 page_list_concat(&root, &pp);
647 648
648 649 /*
649 650 * When large page is fully formed, free it.
650 651 */
651 652 if (++cnt == large) {
652 653 page_free_large_ctr(cnt);
653 654 page_list_add_pages(root, PG_LIST_ISINIT);
654 655 root = NULL;
655 656 cnt = 0;
656 657 }
657 658 continue;
658 659 }
659 660
660 661 /*
661 662 * At this point we have a page number which
662 663 * is aligned. We assert that we aren't already
663 664 * in a different large page.
664 665 */
665 666 ASSERT(IS_P2ALIGNED(pnum, large));
666 667 ASSERT(root == NULL && cnt == 0);
667 668
668 669 /*
669 670 * If insufficient number of pages left to form
670 671 * a large page, just free the small page.
671 672 */
672 673 if (num < large) {
673 674 pp->p_szc = 0;
674 675 page_free_at_startup(pp);
675 676 continue;
676 677 }
677 678
678 679 /*
679 680 * Otherwise start a new large page.
680 681 */
681 682 pp->p_szc = szc;
682 683 cnt++;
683 684 root = pp;
684 685 }
685 686 ASSERT(root == NULL && cnt == 0);
686 687 }
687 688
688 689 /*
689 690 * Find a page representing the specified [vp, offset].
690 691 * If we find the page but it is intransit coming in,
691 692 * it will have an "exclusive" lock and we wait for
692 693 * the i/o to complete. A page found on the free list
693 694 * is always reclaimed and then locked. On success, the page
694 695 * is locked, its data is valid and it isn't on the free
695 696 * list, while a NULL is returned if the page doesn't exist.
696 697 */
697 698 page_t *
698 699 page_lookup(vnode_t *vp, u_offset_t off, se_t se)
699 700 {
700 701 return (page_lookup_create(vp, off, se, NULL, NULL, 0));
701 702 }
702 703
703 704 /*
704 705 * Find a page representing the specified [vp, offset].
705 706 * We either return the one we found or, if passed in,
706 707 * create one with identity of [vp, offset] of the
707 708 * pre-allocated page. If we find existing page but it is
708 709 * intransit coming in, it will have an "exclusive" lock
709 710 * and we wait for the i/o to complete. A page found on
710 711 * the free list is always reclaimed and then locked.
711 712 * On success, the page is locked, its data is valid and
712 713 * it isn't on the free list, while a NULL is returned
713 714 * if the page doesn't exist and newpp is NULL;
714 715 */
715 716 page_t *
716 717 page_lookup_create(
717 718 vnode_t *vp,
718 719 u_offset_t off,
719 720 se_t se,
720 721 page_t *newpp,
721 722 spgcnt_t *nrelocp,
722 723 int flags)
723 724 {
724 725 page_t *pp;
725 726 kmutex_t *phm;
726 727 ulong_t index;
727 728 uint_t hash_locked;
728 729 uint_t es;
729 730
730 731 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
731 732 VM_STAT_ADD(page_lookup_cnt[0]);
732 733 ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
733 734
734 735 /*
735 736 * Acquire the appropriate page hash lock since
736 737 * we have to search the hash list. Pages that
737 738 * hash to this list can't change identity while
738 739 * this lock is held.
739 740 */
740 741 hash_locked = 0;
741 742 index = PAGE_HASH_FUNC(vp, off);
742 743 phm = NULL;
743 744 top:
744 745 pp = page_hash_search(index, vp, off);
745 746 if (pp != NULL) {
746 747 VM_STAT_ADD(page_lookup_cnt[1]);
747 748 es = (newpp != NULL) ? 1 : 0;
748 749 es |= flags;
749 750 if (!hash_locked) {
750 751 VM_STAT_ADD(page_lookup_cnt[2]);
751 752 if (!page_try_reclaim_lock(pp, se, es)) {
752 753 /*
753 754 * On a miss, acquire the phm. Then
754 755 * next time, page_lock() will be called,
755 756 * causing a wait if the page is busy.
756 757 * just looping with page_trylock() would
757 758 * get pretty boring.
758 759 */
759 760 VM_STAT_ADD(page_lookup_cnt[3]);
760 761 phm = PAGE_HASH_MUTEX(index);
761 762 mutex_enter(phm);
762 763 hash_locked = 1;
763 764 goto top;
764 765 }
765 766 } else {
766 767 VM_STAT_ADD(page_lookup_cnt[4]);
767 768 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
768 769 VM_STAT_ADD(page_lookup_cnt[5]);
769 770 goto top;
770 771 }
771 772 }
772 773
773 774 /*
774 775 * Since `pp' is locked it can not change identity now.
775 776 * Reconfirm we locked the correct page.
776 777 *
777 778 * Both the p_vnode and p_offset *must* be cast volatile
778 779 * to force a reload of their values: The page_hash_search
779 780 * function will have stuffed p_vnode and p_offset into
780 781 * registers before calling page_trylock(); another thread,
781 782 * actually holding the hash lock, could have changed the
782 783 * page's identity in memory, but our registers would not
783 784 * be changed, fooling the reconfirmation. If the hash
784 785 * lock was held during the search, the casting would
785 786 * not be needed.
786 787 */
787 788 VM_STAT_ADD(page_lookup_cnt[6]);
788 789 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
789 790 ((volatile u_offset_t)(pp->p_offset) != off)) {
790 791 VM_STAT_ADD(page_lookup_cnt[7]);
791 792 if (hash_locked) {
792 793 panic("page_lookup_create: lost page %p",
793 794 (void *)pp);
794 795 /*NOTREACHED*/
795 796 }
796 797 page_unlock(pp);
797 798 phm = PAGE_HASH_MUTEX(index);
798 799 mutex_enter(phm);
799 800 hash_locked = 1;
800 801 goto top;
801 802 }
802 803
803 804 /*
804 805 * If page_trylock() was called, then pp may still be on
805 806 * the cachelist (can't be on the free list, it would not
806 807 * have been found in the search). If it is on the
807 808 * cachelist it must be pulled now. To pull the page from
808 809 * the cachelist, it must be exclusively locked.
809 810 *
810 811 * The other big difference between page_trylock() and
811 812 * page_lock(), is that page_lock() will pull the
812 813 * page from whatever free list (the cache list in this
813 814 * case) the page is on. If page_trylock() was used
814 815 * above, then we have to do the reclaim ourselves.
815 816 */
816 817 if ((!hash_locked) && (PP_ISFREE(pp))) {
817 818 ASSERT(PP_ISAGED(pp) == 0);
818 819 VM_STAT_ADD(page_lookup_cnt[8]);
819 820
820 821 /*
821 822 * page_relcaim will insure that we
822 823 * have this page exclusively
823 824 */
824 825
825 826 if (!page_reclaim(pp, NULL)) {
826 827 /*
827 828 * Page_reclaim dropped whatever lock
828 829 * we held.
829 830 */
830 831 VM_STAT_ADD(page_lookup_cnt[9]);
831 832 phm = PAGE_HASH_MUTEX(index);
832 833 mutex_enter(phm);
833 834 hash_locked = 1;
834 835 goto top;
835 836 } else if (se == SE_SHARED && newpp == NULL) {
836 837 VM_STAT_ADD(page_lookup_cnt[10]);
837 838 page_downgrade(pp);
838 839 }
839 840 }
840 841
841 842 if (hash_locked) {
842 843 mutex_exit(phm);
843 844 }
844 845
845 846 if (newpp != NULL && pp->p_szc < newpp->p_szc &&
846 847 PAGE_EXCL(pp) && nrelocp != NULL) {
847 848 ASSERT(nrelocp != NULL);
848 849 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
849 850 NULL);
850 851 if (*nrelocp > 0) {
851 852 VM_STAT_COND_ADD(*nrelocp == 1,
852 853 page_lookup_cnt[11]);
853 854 VM_STAT_COND_ADD(*nrelocp > 1,
854 855 page_lookup_cnt[12]);
855 856 pp = newpp;
856 857 se = SE_EXCL;
857 858 } else {
858 859 if (se == SE_SHARED) {
859 860 page_downgrade(pp);
860 861 }
861 862 VM_STAT_ADD(page_lookup_cnt[13]);
862 863 }
863 864 } else if (newpp != NULL && nrelocp != NULL) {
864 865 if (PAGE_EXCL(pp) && se == SE_SHARED) {
865 866 page_downgrade(pp);
866 867 }
867 868 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
868 869 page_lookup_cnt[14]);
869 870 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
870 871 page_lookup_cnt[15]);
871 872 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
872 873 page_lookup_cnt[16]);
873 874 } else if (newpp != NULL && PAGE_EXCL(pp)) {
874 875 se = SE_EXCL;
875 876 }
876 877 } else if (!hash_locked) {
877 878 VM_STAT_ADD(page_lookup_cnt[17]);
878 879 phm = PAGE_HASH_MUTEX(index);
879 880 mutex_enter(phm);
880 881 hash_locked = 1;
881 882 goto top;
882 883 } else if (newpp != NULL) {
883 884 /*
884 885 * If we have a preallocated page then
885 886 * insert it now and basically behave like
886 887 * page_create.
887 888 */
888 889 VM_STAT_ADD(page_lookup_cnt[18]);
889 890 /*
890 891 * Since we hold the page hash mutex and
891 892 * just searched for this page, page_hashin
892 893 * had better not fail. If it does, that
893 894 * means some thread did not follow the
894 895 * page hash mutex rules. Panic now and
895 896 * get it over with. As usual, go down
896 897 * holding all the locks.
897 898 */
898 899 ASSERT(MUTEX_HELD(phm));
899 900 if (!page_hashin(newpp, vp, off, phm)) {
900 901 ASSERT(MUTEX_HELD(phm));
901 902 panic("page_lookup_create: hashin failed %p %p %llx %p",
902 903 (void *)newpp, (void *)vp, off, (void *)phm);
903 904 /*NOTREACHED*/
904 905 }
905 906 ASSERT(MUTEX_HELD(phm));
906 907 mutex_exit(phm);
907 908 phm = NULL;
908 909 page_set_props(newpp, P_REF);
909 910 page_io_lock(newpp);
910 911 pp = newpp;
911 912 se = SE_EXCL;
912 913 } else {
913 914 VM_STAT_ADD(page_lookup_cnt[19]);
914 915 mutex_exit(phm);
915 916 }
916 917
917 918 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
918 919
919 920 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);
920 921
921 922 return (pp);
922 923 }
923 924
924 925 /*
925 926 * Search the hash list for the page representing the
926 927 * specified [vp, offset] and return it locked. Skip
927 928 * free pages and pages that cannot be locked as requested.
928 929 * Used while attempting to kluster pages.
929 930 */
930 931 page_t *
931 932 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
932 933 {
933 934 page_t *pp;
934 935 kmutex_t *phm;
935 936 ulong_t index;
936 937 uint_t locked;
937 938
938 939 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
939 940 VM_STAT_ADD(page_lookup_nowait_cnt[0]);
940 941
941 942 index = PAGE_HASH_FUNC(vp, off);
942 943 pp = page_hash_search(index, vp, off);
943 944 locked = 0;
944 945 if (pp == NULL) {
945 946 top:
946 947 VM_STAT_ADD(page_lookup_nowait_cnt[1]);
947 948 locked = 1;
948 949 phm = PAGE_HASH_MUTEX(index);
949 950 mutex_enter(phm);
950 951 pp = page_hash_search(index, vp, off);
951 952 }
952 953
953 954 if (pp == NULL || PP_ISFREE(pp)) {
954 955 VM_STAT_ADD(page_lookup_nowait_cnt[2]);
955 956 pp = NULL;
956 957 } else {
957 958 if (!page_trylock(pp, se)) {
958 959 VM_STAT_ADD(page_lookup_nowait_cnt[3]);
959 960 pp = NULL;
960 961 } else {
961 962 VM_STAT_ADD(page_lookup_nowait_cnt[4]);
962 963 /*
963 964 * See the comment in page_lookup()
964 965 */
965 966 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
966 967 ((u_offset_t)(pp->p_offset) != off)) {
967 968 VM_STAT_ADD(page_lookup_nowait_cnt[5]);
968 969 if (locked) {
969 970 panic("page_lookup_nowait %p",
970 971 (void *)pp);
971 972 /*NOTREACHED*/
972 973 }
973 974 page_unlock(pp);
974 975 goto top;
975 976 }
976 977 if (PP_ISFREE(pp)) {
977 978 VM_STAT_ADD(page_lookup_nowait_cnt[6]);
978 979 page_unlock(pp);
979 980 pp = NULL;
980 981 }
981 982 }
982 983 }
983 984 if (locked) {
984 985 VM_STAT_ADD(page_lookup_nowait_cnt[7]);
985 986 mutex_exit(phm);
986 987 }
987 988
988 989 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
989 990
990 991 return (pp);
991 992 }
992 993
993 994 /*
994 995 * Search the hash list for a page with the specified [vp, off]
995 996 * that is known to exist and is already locked. This routine
996 997 * is typically used by segment SOFTUNLOCK routines.
997 998 */
998 999 page_t *
999 1000 page_find(vnode_t *vp, u_offset_t off)
1000 1001 {
1001 1002 page_t *pp;
1002 1003 kmutex_t *phm;
1003 1004 ulong_t index;
1004 1005
1005 1006 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1006 1007 VM_STAT_ADD(page_find_cnt);
1007 1008
1008 1009 index = PAGE_HASH_FUNC(vp, off);
1009 1010 phm = PAGE_HASH_MUTEX(index);
1010 1011
1011 1012 mutex_enter(phm);
1012 1013 pp = page_hash_search(index, vp, off);
1013 1014 mutex_exit(phm);
1014 1015
1015 1016 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
1016 1017 return (pp);
1017 1018 }
1018 1019
1019 1020 /*
1020 1021 * Determine whether a page with the specified [vp, off]
1021 1022 * currently exists in the system. Obviously this should
1022 1023 * only be considered as a hint since nothing prevents the
1023 1024 * page from disappearing or appearing immediately after
1024 1025 * the return from this routine. Subsequently, we don't
1025 1026 * even bother to lock the list.
1026 1027 */
1027 1028 page_t *
1028 1029 page_exists(vnode_t *vp, u_offset_t off)
1029 1030 {
1030 1031 ulong_t index;
1031 1032
1032 1033 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1033 1034 VM_STAT_ADD(page_exists_cnt);
1034 1035
1035 1036 index = PAGE_HASH_FUNC(vp, off);
1036 1037
1037 1038 return (page_hash_search(index, vp, off));
1038 1039 }
1039 1040
1040 1041 /*
1041 1042 * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1042 1043 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array
1043 1044 * with these pages locked SHARED. If necessary reclaim pages from
1044 1045 * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1045 1046 *
1046 1047 * If we fail to lock pages still return 1 if pages exist and contiguous.
1047 1048 * But in this case return value is just a hint. ppa array won't be filled.
1048 1049 * Caller should initialize ppa[0] as NULL to distinguish return value.
1049 1050 *
1050 1051 * Returns 0 if pages don't exist or not physically contiguous.
1051 1052 *
1052 1053 * This routine doesn't work for anonymous(swapfs) pages.
1053 1054 */
1054 1055 int
1055 1056 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1056 1057 {
1057 1058 pgcnt_t pages;
1058 1059 pfn_t pfn;
1059 1060 page_t *rootpp;
1060 1061 pgcnt_t i;
1061 1062 pgcnt_t j;
1062 1063 u_offset_t save_off = off;
1063 1064 ulong_t index;
1064 1065 kmutex_t *phm;
1065 1066 page_t *pp;
1066 1067 uint_t pszc;
1067 1068 int loopcnt = 0;
1068 1069
1069 1070 ASSERT(szc != 0);
1070 1071 ASSERT(vp != NULL);
1071 1072 ASSERT(!IS_SWAPFSVP(vp));
1072 1073 ASSERT(!VN_ISKAS(vp));
1073 1074
1074 1075 again:
1075 1076 if (++loopcnt > 3) {
1076 1077 VM_STAT_ADD(page_exphcontg[0]);
1077 1078 return (0);
1078 1079 }
1079 1080
1080 1081 index = PAGE_HASH_FUNC(vp, off);
1081 1082 phm = PAGE_HASH_MUTEX(index);
1082 1083
1083 1084 mutex_enter(phm);
1084 1085 pp = page_hash_search(index, vp, off);
1085 1086 mutex_exit(phm);
1086 1087
1087 1088 VM_STAT_ADD(page_exphcontg[1]);
1088 1089
1089 1090 if (pp == NULL) {
1090 1091 VM_STAT_ADD(page_exphcontg[2]);
1091 1092 return (0);
1092 1093 }
1093 1094
1094 1095 pages = page_get_pagecnt(szc);
1095 1096 rootpp = pp;
1096 1097 pfn = rootpp->p_pagenum;
1097 1098
1098 1099 if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1099 1100 VM_STAT_ADD(page_exphcontg[3]);
1100 1101 if (!page_trylock(pp, SE_SHARED)) {
1101 1102 VM_STAT_ADD(page_exphcontg[4]);
1102 1103 return (1);
1103 1104 }
1104 1105 /*
1105 1106 * Also check whether p_pagenum was modified by DR.
1106 1107 */
1107 1108 if (pp->p_szc != pszc || pp->p_vnode != vp ||
1108 1109 pp->p_offset != off || pp->p_pagenum != pfn) {
1109 1110 VM_STAT_ADD(page_exphcontg[5]);
1110 1111 page_unlock(pp);
1111 1112 off = save_off;
1112 1113 goto again;
1113 1114 }
1114 1115 /*
1115 1116 * szc was non zero and vnode and offset matched after we
1116 1117 * locked the page it means it can't become free on us.
1117 1118 */
1118 1119 ASSERT(!PP_ISFREE(pp));
1119 1120 if (!IS_P2ALIGNED(pfn, pages)) {
1120 1121 page_unlock(pp);
1121 1122 return (0);
1122 1123 }
1123 1124 ppa[0] = pp;
1124 1125 pp++;
1125 1126 off += PAGESIZE;
1126 1127 pfn++;
1127 1128 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1128 1129 if (!page_trylock(pp, SE_SHARED)) {
1129 1130 VM_STAT_ADD(page_exphcontg[6]);
1130 1131 pp--;
1131 1132 while (i-- > 0) {
1132 1133 page_unlock(pp);
1133 1134 pp--;
1134 1135 }
1135 1136 ppa[0] = NULL;
1136 1137 return (1);
1137 1138 }
1138 1139 if (pp->p_szc != pszc) {
1139 1140 VM_STAT_ADD(page_exphcontg[7]);
1140 1141 page_unlock(pp);
1141 1142 pp--;
1142 1143 while (i-- > 0) {
1143 1144 page_unlock(pp);
1144 1145 pp--;
1145 1146 }
1146 1147 ppa[0] = NULL;
1147 1148 off = save_off;
1148 1149 goto again;
1149 1150 }
1150 1151 /*
1151 1152 * szc the same as for previous already locked pages
1152 1153 * with right identity. Since this page had correct
1153 1154 * szc after we locked it can't get freed or destroyed
1154 1155 * and therefore must have the expected identity.
1155 1156 */
1156 1157 ASSERT(!PP_ISFREE(pp));
1157 1158 if (pp->p_vnode != vp ||
1158 1159 pp->p_offset != off) {
1159 1160 panic("page_exists_physcontig: "
1160 1161 "large page identity doesn't match");
1161 1162 }
1162 1163 ppa[i] = pp;
1163 1164 ASSERT(pp->p_pagenum == pfn);
1164 1165 }
1165 1166 VM_STAT_ADD(page_exphcontg[8]);
1166 1167 ppa[pages] = NULL;
1167 1168 return (1);
1168 1169 } else if (pszc >= szc) {
1169 1170 VM_STAT_ADD(page_exphcontg[9]);
1170 1171 if (!IS_P2ALIGNED(pfn, pages)) {
1171 1172 return (0);
1172 1173 }
1173 1174 return (1);
1174 1175 }
1175 1176
1176 1177 if (!IS_P2ALIGNED(pfn, pages)) {
1177 1178 VM_STAT_ADD(page_exphcontg[10]);
1178 1179 return (0);
1179 1180 }
1180 1181
1181 1182 if (page_numtomemseg_nolock(pfn) !=
1182 1183 page_numtomemseg_nolock(pfn + pages - 1)) {
1183 1184 VM_STAT_ADD(page_exphcontg[11]);
1184 1185 return (0);
1185 1186 }
1186 1187
1187 1188 /*
1188 1189 * We loop up 4 times across pages to promote page size.
1189 1190 * We're extra cautious to promote page size atomically with respect
1190 1191 * to everybody else. But we can probably optimize into 1 loop if
1191 1192 * this becomes an issue.
1192 1193 */
1193 1194
1194 1195 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1195 1196 if (!page_trylock(pp, SE_EXCL)) {
1196 1197 VM_STAT_ADD(page_exphcontg[12]);
1197 1198 break;
1198 1199 }
1199 1200 /*
1200 1201 * Check whether p_pagenum was modified by DR.
1201 1202 */
1202 1203 if (pp->p_pagenum != pfn) {
1203 1204 page_unlock(pp);
1204 1205 break;
1205 1206 }
1206 1207 if (pp->p_vnode != vp ||
1207 1208 pp->p_offset != off) {
1208 1209 VM_STAT_ADD(page_exphcontg[13]);
1209 1210 page_unlock(pp);
1210 1211 break;
1211 1212 }
1212 1213 if (pp->p_szc >= szc) {
1213 1214 ASSERT(i == 0);
1214 1215 page_unlock(pp);
1215 1216 off = save_off;
1216 1217 goto again;
1217 1218 }
1218 1219 }
1219 1220
1220 1221 if (i != pages) {
1221 1222 VM_STAT_ADD(page_exphcontg[14]);
1222 1223 --pp;
1223 1224 while (i-- > 0) {
1224 1225 page_unlock(pp);
1225 1226 --pp;
1226 1227 }
1227 1228 return (0);
1228 1229 }
1229 1230
1230 1231 pp = rootpp;
1231 1232 for (i = 0; i < pages; i++, pp++) {
1232 1233 if (PP_ISFREE(pp)) {
1233 1234 VM_STAT_ADD(page_exphcontg[15]);
1234 1235 ASSERT(!PP_ISAGED(pp));
1235 1236 ASSERT(pp->p_szc == 0);
1236 1237 if (!page_reclaim(pp, NULL)) {
1237 1238 break;
1238 1239 }
1239 1240 } else {
1240 1241 ASSERT(pp->p_szc < szc);
1241 1242 VM_STAT_ADD(page_exphcontg[16]);
1242 1243 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
1243 1244 }
1244 1245 }
1245 1246 if (i < pages) {
1246 1247 VM_STAT_ADD(page_exphcontg[17]);
1247 1248 /*
1248 1249 * page_reclaim failed because we were out of memory.
1249 1250 * drop the rest of the locks and return because this page
1250 1251 * must be already reallocated anyway.
1251 1252 */
1252 1253 pp = rootpp;
1253 1254 for (j = 0; j < pages; j++, pp++) {
1254 1255 if (j != i) {
1255 1256 page_unlock(pp);
1256 1257 }
1257 1258 }
1258 1259 return (0);
1259 1260 }
1260 1261
1261 1262 off = save_off;
1262 1263 pp = rootpp;
1263 1264 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) {
1264 1265 ASSERT(PAGE_EXCL(pp));
1265 1266 ASSERT(!PP_ISFREE(pp));
1266 1267 ASSERT(!hat_page_is_mapped(pp));
1267 1268 ASSERT(pp->p_vnode == vp);
1268 1269 ASSERT(pp->p_offset == off);
1269 1270 pp->p_szc = szc;
1270 1271 }
1271 1272 pp = rootpp;
1272 1273 for (i = 0; i < pages; i++, pp++) {
1273 1274 if (ppa == NULL) {
1274 1275 page_unlock(pp);
1275 1276 } else {
1276 1277 ppa[i] = pp;
1277 1278 page_downgrade(ppa[i]);
1278 1279 }
1279 1280 }
1280 1281 if (ppa != NULL) {
1281 1282 ppa[pages] = NULL;
1282 1283 }
1283 1284 VM_STAT_ADD(page_exphcontg[18]);
1284 1285 ASSERT(vp->v_pages != NULL);
1285 1286 return (1);
1286 1287 }
1287 1288
1288 1289 /*
1289 1290 * Determine whether a page with the specified [vp, off]
1290 1291 * currently exists in the system and if so return its
1291 1292 * size code. Obviously this should only be considered as
1292 1293 * a hint since nothing prevents the page from disappearing
1293 1294 * or appearing immediately after the return from this routine.
1294 1295 */
1295 1296 int
1296 1297 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1297 1298 {
1298 1299 page_t *pp;
1299 1300 kmutex_t *phm;
1300 1301 ulong_t index;
1301 1302 int rc = 0;
1302 1303
1303 1304 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1304 1305 ASSERT(szc != NULL);
1305 1306 VM_STAT_ADD(page_exists_forreal_cnt);
1306 1307
1307 1308 index = PAGE_HASH_FUNC(vp, off);
1308 1309 phm = PAGE_HASH_MUTEX(index);
1309 1310
1310 1311 mutex_enter(phm);
1311 1312 pp = page_hash_search(index, vp, off);
1312 1313 if (pp != NULL) {
1313 1314 *szc = pp->p_szc;
1314 1315 rc = 1;
1315 1316 }
1316 1317 mutex_exit(phm);
1317 1318 return (rc);
1318 1319 }
1319 1320
1320 1321 /* wakeup threads waiting for pages in page_create_get_something() */
1321 1322 void
1322 1323 wakeup_pcgs(void)
1323 1324 {
1324 1325 if (!CV_HAS_WAITERS(&pcgs_cv))
1325 1326 return;
1326 1327 cv_broadcast(&pcgs_cv);
1327 1328 }
1328 1329
1329 1330 /*
1330 1331 * 'freemem' is used all over the kernel as an indication of how many
1331 1332 * pages are free (either on the cache list or on the free page list)
1332 1333 * in the system. In very few places is a really accurate 'freemem'
1333 1334 * needed. To avoid contention of the lock protecting a the
1334 1335 * single freemem, it was spread out into NCPU buckets. Set_freemem
1335 1336 * sets freemem to the total of all NCPU buckets. It is called from
1336 1337 * clock() on each TICK.
1337 1338 */
1338 1339 void
1339 1340 set_freemem()
1340 1341 {
1341 1342 struct pcf *p;
1342 1343 ulong_t t;
1343 1344 uint_t i;
1344 1345
1345 1346 t = 0;
1346 1347 p = pcf;
1347 1348 for (i = 0; i < pcf_fanout; i++) {
1348 1349 t += p->pcf_count;
1349 1350 p++;
1350 1351 }
1351 1352 freemem = t;
1352 1353
1353 1354 /*
1354 1355 * Don't worry about grabbing mutex. It's not that
1355 1356 * critical if we miss a tick or two. This is
1356 1357 * where we wakeup possible delayers in
1357 1358 * page_create_get_something().
1358 1359 */
1359 1360 wakeup_pcgs();
1360 1361 }
1361 1362
1362 1363 ulong_t
1363 1364 get_freemem()
1364 1365 {
1365 1366 struct pcf *p;
1366 1367 ulong_t t;
1367 1368 uint_t i;
1368 1369
1369 1370 t = 0;
1370 1371 p = pcf;
1371 1372 for (i = 0; i < pcf_fanout; i++) {
1372 1373 t += p->pcf_count;
1373 1374 p++;
1374 1375 }
1375 1376 /*
1376 1377 * We just calculated it, might as well set it.
1377 1378 */
1378 1379 freemem = t;
1379 1380 return (t);
1380 1381 }
1381 1382
1382 1383 /*
1383 1384 * Acquire all of the page cache & free (pcf) locks.
1384 1385 */
1385 1386 void
1386 1387 pcf_acquire_all()
1387 1388 {
1388 1389 struct pcf *p;
1389 1390 uint_t i;
1390 1391
1391 1392 p = pcf;
1392 1393 for (i = 0; i < pcf_fanout; i++) {
1393 1394 mutex_enter(&p->pcf_lock);
1394 1395 p++;
1395 1396 }
1396 1397 }
1397 1398
1398 1399 /*
1399 1400 * Release all the pcf_locks.
1400 1401 */
1401 1402 void
1402 1403 pcf_release_all()
1403 1404 {
1404 1405 struct pcf *p;
1405 1406 uint_t i;
1406 1407
1407 1408 p = pcf;
1408 1409 for (i = 0; i < pcf_fanout; i++) {
1409 1410 mutex_exit(&p->pcf_lock);
1410 1411 p++;
1411 1412 }
1412 1413 }
1413 1414
1414 1415 /*
1415 1416 * Inform the VM system that we need some pages freed up.
1416 1417 * Calls must be symmetric, e.g.:
1417 1418 *
1418 1419 * page_needfree(100);
1419 1420 * wait a bit;
1420 1421 * page_needfree(-100);
1421 1422 */
1422 1423 void
1423 1424 page_needfree(spgcnt_t npages)
1424 1425 {
1425 1426 mutex_enter(&new_freemem_lock);
1426 1427 needfree += npages;
1427 1428 mutex_exit(&new_freemem_lock);
1428 1429 }
1429 1430
1430 1431 /*
1431 1432 * Throttle for page_create(): try to prevent freemem from dropping
1432 1433 * below throttlefree. We can't provide a 100% guarantee because
1433 1434 * KM_NOSLEEP allocations, page_reclaim(), and various other things
1434 1435 * nibble away at the freelist. However, we can block all PG_WAIT
1435 1436 * allocations until memory becomes available. The motivation is
1436 1437 * that several things can fall apart when there's no free memory:
1437 1438 *
1438 1439 * (1) If pageout() needs memory to push a page, the system deadlocks.
1439 1440 *
1440 1441 * (2) By (broken) specification, timeout(9F) can neither fail nor
1441 1442 * block, so it has no choice but to panic the system if it
1442 1443 * cannot allocate a callout structure.
1443 1444 *
1444 1445 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
1445 1446 * it panics if it cannot allocate a callback structure.
1446 1447 *
1447 1448 * (4) Untold numbers of third-party drivers have not yet been hardened
1448 1449 * against KM_NOSLEEP and/or allocb() failures; they simply assume
1449 1450 * success and panic the system with a data fault on failure.
1450 1451 * (The long-term solution to this particular problem is to ship
1451 1452 * hostile fault-injecting DEBUG kernels with the DDK.)
1452 1453 *
1453 1454 * It is theoretically impossible to guarantee success of non-blocking
1454 1455 * allocations, but in practice, this throttle is very hard to break.
1455 1456 */
1456 1457 static int
1457 1458 page_create_throttle(pgcnt_t npages, int flags)
1458 1459 {
1459 1460 ulong_t fm;
1460 1461 uint_t i;
1461 1462 pgcnt_t tf; /* effective value of throttlefree */
1462 1463
1463 1464 /*
1464 1465 * Normal priority allocations.
1465 1466 */
1466 1467 if ((flags & (PG_WAIT | PG_NORMALPRI)) == PG_NORMALPRI) {
1467 1468 ASSERT(!(flags & (PG_PANIC | PG_PUSHPAGE)));
1468 1469 return (freemem >= npages + throttlefree);
1469 1470 }
1470 1471
1471 1472 /*
1472 1473 * Never deny pages when:
1473 1474 * - it's a thread that cannot block [NOMEMWAIT()]
1474 1475 * - the allocation cannot block and must not fail
1475 1476 * - the allocation cannot block and is pageout dispensated
1476 1477 */
1477 1478 if (NOMEMWAIT() ||
1478 1479 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) ||
1479 1480 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE))
1480 1481 return (1);
1481 1482
1482 1483 /*
1483 1484 * If the allocation can't block, we look favorably upon it
1484 1485 * unless we're below pageout_reserve. In that case we fail
1485 1486 * the allocation because we want to make sure there are a few
1486 1487 * pages available for pageout.
1487 1488 */
1488 1489 if ((flags & PG_WAIT) == 0)
1489 1490 return (freemem >= npages + pageout_reserve);
1490 1491
1491 1492 /* Calculate the effective throttlefree value */
1492 1493 tf = throttlefree -
1493 1494 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0);
1494 1495
1495 1496 cv_signal(&proc_pageout->p_cv);
1496 1497
1497 1498 for (;;) {
1498 1499 fm = 0;
1499 1500 pcf_acquire_all();
1500 1501 mutex_enter(&new_freemem_lock);
1501 1502 for (i = 0; i < pcf_fanout; i++) {
1502 1503 fm += pcf[i].pcf_count;
1503 1504 pcf[i].pcf_wait++;
1504 1505 mutex_exit(&pcf[i].pcf_lock);
1505 1506 }
1506 1507 freemem = fm;
1507 1508 if (freemem >= npages + tf) {
1508 1509 mutex_exit(&new_freemem_lock);
1509 1510 break;
1510 1511 }
1511 1512 needfree += npages;
1512 1513 freemem_wait++;
1513 1514 cv_wait(&freemem_cv, &new_freemem_lock);
1514 1515 freemem_wait--;
1515 1516 needfree -= npages;
1516 1517 mutex_exit(&new_freemem_lock);
1517 1518 }
1518 1519 return (1);
1519 1520 }
1520 1521
1521 1522 /*
1522 1523 * page_create_wait() is called to either coalesce pages from the
1523 1524 * different pcf buckets or to wait because there simply are not
1524 1525 * enough pages to satisfy the caller's request.
1525 1526 *
1526 1527 * Sadly, this is called from platform/vm/vm_machdep.c
1527 1528 */
1528 1529 int
1529 1530 page_create_wait(pgcnt_t npages, uint_t flags)
1530 1531 {
1531 1532 pgcnt_t total;
1532 1533 uint_t i;
1533 1534 struct pcf *p;
1534 1535
1535 1536 /*
1536 1537 * Wait until there are enough free pages to satisfy our
1537 1538 * entire request.
1538 1539 * We set needfree += npages before prodding pageout, to make sure
1539 1540 * it does real work when npages > lotsfree > freemem.
1540 1541 */
1541 1542 VM_STAT_ADD(page_create_not_enough);
1542 1543
1543 1544 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1);
1544 1545 checkagain:
1545 1546 if ((flags & PG_NORELOC) &&
1546 1547 kcage_freemem < kcage_throttlefree + npages)
1547 1548 (void) kcage_create_throttle(npages, flags);
1548 1549
1549 1550 if (freemem < npages + throttlefree)
1550 1551 if (!page_create_throttle(npages, flags))
1551 1552 return (0);
1552 1553
1553 1554 if (pcf_decrement_bucket(npages) ||
1554 1555 pcf_decrement_multiple(&total, npages, 0))
1555 1556 return (1);
1556 1557
1557 1558 /*
1558 1559 * All of the pcf locks are held, there are not enough pages
1559 1560 * to satisfy the request (npages < total).
1560 1561 * Be sure to acquire the new_freemem_lock before dropping
1561 1562 * the pcf locks. This prevents dropping wakeups in page_free().
1562 1563 * The order is always pcf_lock then new_freemem_lock.
1563 1564 *
1564 1565 * Since we hold all the pcf locks, it is a good time to set freemem.
1565 1566 *
1566 1567 * If the caller does not want to wait, return now.
1567 1568 * Else turn the pageout daemon loose to find something
1568 1569 * and wait till it does.
1569 1570 *
1570 1571 */
1571 1572 freemem = total;
1572 1573
1573 1574 if ((flags & PG_WAIT) == 0) {
1574 1575 pcf_release_all();
1575 1576
1576 1577 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM,
1577 1578 "page_create_nomem:npages %ld freemem %ld", npages, freemem);
1578 1579 return (0);
1579 1580 }
1580 1581
1581 1582 ASSERT(proc_pageout != NULL);
1582 1583 cv_signal(&proc_pageout->p_cv);
1583 1584
1584 1585 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START,
1585 1586 "page_create_sleep_start: freemem %ld needfree %ld",
1586 1587 freemem, needfree);
1587 1588
1588 1589 /*
1589 1590 * We are going to wait.
1590 1591 * We currently hold all of the pcf_locks,
1591 1592 * get the new_freemem_lock (it protects freemem_wait),
1592 1593 * before dropping the pcf_locks.
1593 1594 */
1594 1595 mutex_enter(&new_freemem_lock);
1595 1596
1596 1597 p = pcf;
1597 1598 for (i = 0; i < pcf_fanout; i++) {
1598 1599 p->pcf_wait++;
1599 1600 mutex_exit(&p->pcf_lock);
1600 1601 p++;
1601 1602 }
1602 1603
1603 1604 needfree += npages;
1604 1605 freemem_wait++;
1605 1606
1606 1607 cv_wait(&freemem_cv, &new_freemem_lock);
1607 1608
1608 1609 freemem_wait--;
1609 1610 needfree -= npages;
1610 1611
1611 1612 mutex_exit(&new_freemem_lock);
1612 1613
1613 1614 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END,
1614 1615 "page_create_sleep_end: freemem %ld needfree %ld",
1615 1616 freemem, needfree);
1616 1617
1617 1618 VM_STAT_ADD(page_create_not_enough_again);
1618 1619 goto checkagain;
1619 1620 }
1620 1621 /*
1621 1622 * A routine to do the opposite of page_create_wait().
1622 1623 */
1623 1624 void
1624 1625 page_create_putback(spgcnt_t npages)
1625 1626 {
1626 1627 struct pcf *p;
1627 1628 pgcnt_t lump;
1628 1629 uint_t *which;
1629 1630
1630 1631 /*
1631 1632 * When a contiguous lump is broken up, we have to
1632 1633 * deal with lots of pages (min 64) so lets spread
1633 1634 * the wealth around.
1634 1635 */
1635 1636 lump = roundup(npages, pcf_fanout) / pcf_fanout;
1636 1637 freemem += npages;
1637 1638
1638 1639 for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) {
1639 1640 which = &p->pcf_count;
1640 1641
1641 1642 mutex_enter(&p->pcf_lock);
1642 1643
1643 1644 if (p->pcf_block) {
1644 1645 which = &p->pcf_reserve;
1645 1646 }
1646 1647
1647 1648 if (lump < npages) {
1648 1649 *which += (uint_t)lump;
1649 1650 npages -= lump;
1650 1651 } else {
1651 1652 *which += (uint_t)npages;
1652 1653 npages = 0;
1653 1654 }
1654 1655
1655 1656 if (p->pcf_wait) {
1656 1657 mutex_enter(&new_freemem_lock);
1657 1658 /*
1658 1659 * Check to see if some other thread
1659 1660 * is actually waiting. Another bucket
1660 1661 * may have woken it up by now. If there
1661 1662 * are no waiters, then set our pcf_wait
1662 1663 * count to zero to avoid coming in here
1663 1664 * next time.
1664 1665 */
1665 1666 if (freemem_wait) {
1666 1667 if (npages > 1) {
1667 1668 cv_broadcast(&freemem_cv);
1668 1669 } else {
1669 1670 cv_signal(&freemem_cv);
1670 1671 }
1671 1672 p->pcf_wait--;
1672 1673 } else {
1673 1674 p->pcf_wait = 0;
1674 1675 }
1675 1676 mutex_exit(&new_freemem_lock);
1676 1677 }
1677 1678 mutex_exit(&p->pcf_lock);
1678 1679 }
1679 1680 ASSERT(npages == 0);
1680 1681 }
1681 1682
1682 1683 /*
1683 1684 * A helper routine for page_create_get_something.
1684 1685 * The indenting got to deep down there.
1685 1686 * Unblock the pcf counters. Any pages freed after
1686 1687 * pcf_block got set are moved to pcf_count and
1687 1688 * wakeups (cv_broadcast() or cv_signal()) are done as needed.
1688 1689 */
1689 1690 static void
1690 1691 pcgs_unblock(void)
1691 1692 {
1692 1693 int i;
1693 1694 struct pcf *p;
1694 1695
1695 1696 /* Update freemem while we're here. */
1696 1697 freemem = 0;
1697 1698 p = pcf;
1698 1699 for (i = 0; i < pcf_fanout; i++) {
1699 1700 mutex_enter(&p->pcf_lock);
1700 1701 ASSERT(p->pcf_count == 0);
1701 1702 p->pcf_count = p->pcf_reserve;
1702 1703 p->pcf_block = 0;
1703 1704 freemem += p->pcf_count;
1704 1705 if (p->pcf_wait) {
1705 1706 mutex_enter(&new_freemem_lock);
1706 1707 if (freemem_wait) {
1707 1708 if (p->pcf_reserve > 1) {
1708 1709 cv_broadcast(&freemem_cv);
1709 1710 p->pcf_wait = 0;
1710 1711 } else {
1711 1712 cv_signal(&freemem_cv);
1712 1713 p->pcf_wait--;
1713 1714 }
1714 1715 } else {
1715 1716 p->pcf_wait = 0;
1716 1717 }
1717 1718 mutex_exit(&new_freemem_lock);
1718 1719 }
1719 1720 p->pcf_reserve = 0;
1720 1721 mutex_exit(&p->pcf_lock);
1721 1722 p++;
1722 1723 }
1723 1724 }
1724 1725
1725 1726 /*
1726 1727 * Called from page_create_va() when both the cache and free lists
1727 1728 * have been checked once.
1728 1729 *
1729 1730 * Either returns a page or panics since the accounting was done
1730 1731 * way before we got here.
1731 1732 *
1732 1733 * We don't come here often, so leave the accounting on permanently.
1733 1734 */
1734 1735
1735 1736 #define MAX_PCGS 100
1736 1737
1737 1738 #ifdef DEBUG
1738 1739 #define PCGS_TRIES 100
1739 1740 #else /* DEBUG */
1740 1741 #define PCGS_TRIES 10
1741 1742 #endif /* DEBUG */
1742 1743
1743 1744 #ifdef VM_STATS
1744 1745 uint_t pcgs_counts[PCGS_TRIES];
1745 1746 uint_t pcgs_too_many;
1746 1747 uint_t pcgs_entered;
1747 1748 uint_t pcgs_entered_noreloc;
1748 1749 uint_t pcgs_locked;
1749 1750 uint_t pcgs_cagelocked;
1750 1751 #endif /* VM_STATS */
1751 1752
1752 1753 static page_t *
1753 1754 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg,
1754 1755 caddr_t vaddr, uint_t flags)
1755 1756 {
1756 1757 uint_t count;
1757 1758 page_t *pp;
1758 1759 uint_t locked, i;
1759 1760 struct pcf *p;
1760 1761 lgrp_t *lgrp;
1761 1762 int cagelocked = 0;
1762 1763
1763 1764 VM_STAT_ADD(pcgs_entered);
1764 1765
1765 1766 /*
1766 1767 * Tap any reserve freelists: if we fail now, we'll die
1767 1768 * since the page(s) we're looking for have already been
1768 1769 * accounted for.
1769 1770 */
1770 1771 flags |= PG_PANIC;
1771 1772
1772 1773 if ((flags & PG_NORELOC) != 0) {
1773 1774 VM_STAT_ADD(pcgs_entered_noreloc);
1774 1775 /*
1775 1776 * Requests for free pages from critical threads
1776 1777 * such as pageout still won't throttle here, but
1777 1778 * we must try again, to give the cageout thread
1778 1779 * another chance to catch up. Since we already
1779 1780 * accounted for the pages, we had better get them
1780 1781 * this time.
1781 1782 *
1782 1783 * N.B. All non-critical threads acquire the pcgs_cagelock
1783 1784 * to serialize access to the freelists. This implements a
1784 1785 * turnstile-type synchornization to avoid starvation of
1785 1786 * critical requests for PG_NORELOC memory by non-critical
1786 1787 * threads: all non-critical threads must acquire a 'ticket'
1787 1788 * before passing through, which entails making sure
1788 1789 * kcage_freemem won't fall below minfree prior to grabbing
1789 1790 * pages from the freelists.
1790 1791 */
1791 1792 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) {
1792 1793 mutex_enter(&pcgs_cagelock);
1793 1794 cagelocked = 1;
1794 1795 VM_STAT_ADD(pcgs_cagelocked);
1795 1796 }
1796 1797 }
1797 1798
1798 1799 /*
1799 1800 * Time to get serious.
1800 1801 * We failed to get a `correctly colored' page from both the
1801 1802 * free and cache lists.
1802 1803 * We escalate in stage.
1803 1804 *
1804 1805 * First try both lists without worring about color.
1805 1806 *
1806 1807 * Then, grab all page accounting locks (ie. pcf[]) and
1807 1808 * steal any pages that they have and set the pcf_block flag to
1808 1809 * stop deletions from the lists. This will help because
1809 1810 * a page can get added to the free list while we are looking
1810 1811 * at the cache list, then another page could be added to the cache
1811 1812 * list allowing the page on the free list to be removed as we
1812 1813 * move from looking at the cache list to the free list. This
1813 1814 * could happen over and over. We would never find the page
1814 1815 * we have accounted for.
1815 1816 *
1816 1817 * Noreloc pages are a subset of the global (relocatable) page pool.
1817 1818 * They are not tracked separately in the pcf bins, so it is
1818 1819 * impossible to know when doing pcf accounting if the available
1819 1820 * page(s) are noreloc pages or not. When looking for a noreloc page
1820 1821 * it is quite easy to end up here even if the global (relocatable)
1821 1822 * page pool has plenty of free pages but the noreloc pool is empty.
1822 1823 *
1823 1824 * When the noreloc pool is empty (or low), additional noreloc pages
1824 1825 * are created by converting pages from the global page pool. This
1825 1826 * process will stall during pcf accounting if the pcf bins are
1826 1827 * already locked. Such is the case when a noreloc allocation is
1827 1828 * looping here in page_create_get_something waiting for more noreloc
1828 1829 * pages to appear.
1829 1830 *
1830 1831 * Short of adding a new field to the pcf bins to accurately track
1831 1832 * the number of free noreloc pages, we instead do not grab the
1832 1833 * pcgs_lock, do not set the pcf blocks and do not timeout when
1833 1834 * allocating a noreloc page. This allows noreloc allocations to
1834 1835 * loop without blocking global page pool allocations.
1835 1836 *
1836 1837 * NOTE: the behaviour of page_create_get_something has not changed
1837 1838 * for the case of global page pool allocations.
1838 1839 */
1839 1840
1840 1841 flags &= ~PG_MATCH_COLOR;
1841 1842 locked = 0;
1842 1843 #if defined(__i386) || defined(__amd64)
1843 1844 flags = page_create_update_flags_x86(flags);
1844 1845 #endif
1845 1846
1846 1847 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
1847 1848
1848 1849 for (count = 0; kcage_on || count < MAX_PCGS; count++) {
1849 1850 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
1850 1851 flags, lgrp);
1851 1852 if (pp == NULL) {
1852 1853 pp = page_get_cachelist(vp, off, seg, vaddr,
1853 1854 flags, lgrp);
1854 1855 }
1855 1856 if (pp == NULL) {
1856 1857 /*
1857 1858 * Serialize. Don't fight with other pcgs().
1858 1859 */
1859 1860 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) {
1860 1861 mutex_enter(&pcgs_lock);
1861 1862 VM_STAT_ADD(pcgs_locked);
1862 1863 locked = 1;
1863 1864 p = pcf;
1864 1865 for (i = 0; i < pcf_fanout; i++) {
1865 1866 mutex_enter(&p->pcf_lock);
1866 1867 ASSERT(p->pcf_block == 0);
1867 1868 p->pcf_block = 1;
1868 1869 p->pcf_reserve = p->pcf_count;
1869 1870 p->pcf_count = 0;
1870 1871 mutex_exit(&p->pcf_lock);
1871 1872 p++;
1872 1873 }
1873 1874 freemem = 0;
1874 1875 }
1875 1876
1876 1877 if (count) {
1877 1878 /*
1878 1879 * Since page_free() puts pages on
1879 1880 * a list then accounts for it, we
1880 1881 * just have to wait for page_free()
1881 1882 * to unlock any page it was working
1882 1883 * with. The page_lock()-page_reclaim()
1883 1884 * path falls in the same boat.
1884 1885 *
1885 1886 * We don't need to check on the
1886 1887 * PG_WAIT flag, we have already
1887 1888 * accounted for the page we are
1888 1889 * looking for in page_create_va().
1889 1890 *
1890 1891 * We just wait a moment to let any
1891 1892 * locked pages on the lists free up,
1892 1893 * then continue around and try again.
1893 1894 *
1894 1895 * Will be awakened by set_freemem().
1895 1896 */
1896 1897 mutex_enter(&pcgs_wait_lock);
1897 1898 cv_wait(&pcgs_cv, &pcgs_wait_lock);
1898 1899 mutex_exit(&pcgs_wait_lock);
1899 1900 }
1900 1901 } else {
1901 1902 #ifdef VM_STATS
1902 1903 if (count >= PCGS_TRIES) {
1903 1904 VM_STAT_ADD(pcgs_too_many);
1904 1905 } else {
1905 1906 VM_STAT_ADD(pcgs_counts[count]);
1906 1907 }
1907 1908 #endif
1908 1909 if (locked) {
1909 1910 pcgs_unblock();
1910 1911 mutex_exit(&pcgs_lock);
1911 1912 }
1912 1913 if (cagelocked)
1913 1914 mutex_exit(&pcgs_cagelock);
1914 1915 return (pp);
1915 1916 }
1916 1917 }
1917 1918 /*
1918 1919 * we go down holding the pcf locks.
1919 1920 */
1920 1921 panic("no %spage found %d",
1921 1922 ((flags & PG_NORELOC) ? "non-reloc " : ""), count);
1922 1923 /*NOTREACHED*/
1923 1924 }
1924 1925
1925 1926 /*
1926 1927 * Create enough pages for "bytes" worth of data starting at
1927 1928 * "off" in "vp".
1928 1929 *
1929 1930 * Where flag must be one of:
1930 1931 *
1931 1932 * PG_EXCL: Exclusive create (fail if any page already
1932 1933 * exists in the page cache) which does not
1933 1934 * wait for memory to become available.
1934 1935 *
1935 1936 * PG_WAIT: Non-exclusive create which can wait for
1936 1937 * memory to become available.
1937 1938 *
1938 1939 * PG_PHYSCONTIG: Allocate physically contiguous pages.
1939 1940 * (Not Supported)
1940 1941 *
1941 1942 * A doubly linked list of pages is returned to the caller. Each page
1942 1943 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
1943 1944 * lock.
1944 1945 *
1945 1946 * Unable to change the parameters to page_create() in a minor release,
1946 1947 * we renamed page_create() to page_create_va(), changed all known calls
1947 1948 * from page_create() to page_create_va(), and created this wrapper.
1948 1949 *
1949 1950 * Upon a major release, we should break compatibility by deleting this
1950 1951 * wrapper, and replacing all the strings "page_create_va", with "page_create".
1951 1952 *
1952 1953 * NOTE: There is a copy of this interface as page_create_io() in
1953 1954 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied
1954 1955 * there.
1955 1956 */
1956 1957 page_t *
1957 1958 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags)
1958 1959 {
1959 1960 caddr_t random_vaddr;
1960 1961 struct seg kseg;
1961 1962
1962 1963 #ifdef DEBUG
1963 1964 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p",
1964 1965 (void *)caller());
1965 1966 #endif
1966 1967
1967 1968 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^
1968 1969 (uintptr_t)(off >> PAGESHIFT));
1969 1970 kseg.s_as = &kas;
1970 1971
1971 1972 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr));
1972 1973 }
1973 1974
1974 1975 #ifdef DEBUG
1975 1976 uint32_t pg_alloc_pgs_mtbf = 0;
1976 1977 #endif
1977 1978
1978 1979 /*
1979 1980 * Used for large page support. It will attempt to allocate
1980 1981 * a large page(s) off the freelist.
1981 1982 *
1982 1983 * Returns non zero on failure.
1983 1984 */
1984 1985 int
1985 1986 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr,
1986 1987 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags)
1987 1988 {
1988 1989 pgcnt_t npgs, curnpgs, totpgs;
1989 1990 size_t pgsz;
1990 1991 page_t *pplist = NULL, *pp;
1991 1992 int err = 0;
1992 1993 lgrp_t *lgrp;
1993 1994
1994 1995 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1));
1995 1996 ASSERT(pgflags == 0 || pgflags == PG_LOCAL);
1996 1997
1997 1998 /*
1998 1999 * Check if system heavily prefers local large pages over remote
1999 2000 * on systems with multiple lgroups.
2000 2001 */
2001 2002 if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) {
2002 2003 pgflags = PG_LOCAL;
2003 2004 }
2004 2005
2005 2006 VM_STAT_ADD(alloc_pages[0]);
2006 2007
2007 2008 #ifdef DEBUG
2008 2009 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) {
2009 2010 return (ENOMEM);
2010 2011 }
2011 2012 #endif
2012 2013
2013 2014 /*
2014 2015 * One must be NULL but not both.
2015 2016 * And one must be non NULL but not both.
2016 2017 */
2017 2018 ASSERT(basepp != NULL || ppa != NULL);
2018 2019 ASSERT(basepp == NULL || ppa == NULL);
2019 2020
2020 2021 #if defined(__i386) || defined(__amd64)
2021 2022 while (page_chk_freelist(szc) == 0) {
2022 2023 VM_STAT_ADD(alloc_pages[8]);
2023 2024 if (anypgsz == 0 || --szc == 0)
2024 2025 return (ENOMEM);
2025 2026 }
2026 2027 #endif
2027 2028
2028 2029 pgsz = page_get_pagesize(szc);
2029 2030 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT;
2030 2031
2031 2032 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0);
2032 2033
2033 2034 (void) page_create_wait(npgs, PG_WAIT);
2034 2035
2035 2036 while (npgs && szc) {
2036 2037 lgrp = lgrp_mem_choose(seg, addr, pgsz);
2037 2038 if (pgflags == PG_LOCAL) {
2038 2039 pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2039 2040 pgflags, lgrp);
2040 2041 if (pp == NULL) {
2041 2042 pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2042 2043 0, lgrp);
2043 2044 }
2044 2045 } else {
2045 2046 pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2046 2047 0, lgrp);
2047 2048 }
2048 2049 if (pp != NULL) {
2049 2050 VM_STAT_ADD(alloc_pages[1]);
2050 2051 page_list_concat(&pplist, &pp);
2051 2052 ASSERT(npgs >= curnpgs);
2052 2053 npgs -= curnpgs;
2053 2054 } else if (anypgsz) {
2054 2055 VM_STAT_ADD(alloc_pages[2]);
2055 2056 szc--;
2056 2057 pgsz = page_get_pagesize(szc);
2057 2058 curnpgs = pgsz >> PAGESHIFT;
2058 2059 } else {
2059 2060 VM_STAT_ADD(alloc_pages[3]);
2060 2061 ASSERT(npgs == totpgs);
2061 2062 page_create_putback(npgs);
2062 2063 return (ENOMEM);
2063 2064 }
2064 2065 }
2065 2066 if (szc == 0) {
2066 2067 VM_STAT_ADD(alloc_pages[4]);
2067 2068 ASSERT(npgs != 0);
2068 2069 page_create_putback(npgs);
2069 2070 err = ENOMEM;
2070 2071 } else if (basepp != NULL) {
2071 2072 ASSERT(npgs == 0);
2072 2073 ASSERT(ppa == NULL);
2073 2074 *basepp = pplist;
2074 2075 }
2075 2076
2076 2077 npgs = totpgs - npgs;
2077 2078 pp = pplist;
2078 2079
2079 2080 /*
2080 2081 * Clear the free and age bits. Also if we were passed in a ppa then
2081 2082 * fill it in with all the constituent pages from the large page. But
2082 2083 * if we failed to allocate all the pages just free what we got.
2083 2084 */
2084 2085 while (npgs != 0) {
2085 2086 ASSERT(PP_ISFREE(pp));
2086 2087 ASSERT(PP_ISAGED(pp));
2087 2088 if (ppa != NULL || err != 0) {
2088 2089 if (err == 0) {
2089 2090 VM_STAT_ADD(alloc_pages[5]);
2090 2091 PP_CLRFREE(pp);
2091 2092 PP_CLRAGED(pp);
2092 2093 page_sub(&pplist, pp);
2093 2094 *ppa++ = pp;
2094 2095 npgs--;
2095 2096 } else {
2096 2097 VM_STAT_ADD(alloc_pages[6]);
2097 2098 ASSERT(pp->p_szc != 0);
2098 2099 curnpgs = page_get_pagecnt(pp->p_szc);
2099 2100 page_list_break(&pp, &pplist, curnpgs);
2100 2101 page_list_add_pages(pp, 0);
2101 2102 page_create_putback(curnpgs);
2102 2103 ASSERT(npgs >= curnpgs);
2103 2104 npgs -= curnpgs;
2104 2105 }
2105 2106 pp = pplist;
2106 2107 } else {
2107 2108 VM_STAT_ADD(alloc_pages[7]);
2108 2109 PP_CLRFREE(pp);
2109 2110 PP_CLRAGED(pp);
2110 2111 pp = pp->p_next;
2111 2112 npgs--;
2112 2113 }
2113 2114 }
2114 2115 return (err);
2115 2116 }
2116 2117
2117 2118 /*
2118 2119 * Get a single large page off of the freelists, and set it up for use.
2119 2120 * Number of bytes requested must be a supported page size.
2120 2121 *
2121 2122 * Note that this call may fail even if there is sufficient
2122 2123 * memory available or PG_WAIT is set, so the caller must
2123 2124 * be willing to fallback on page_create_va(), block and retry,
2124 2125 * or fail the requester.
2125 2126 */
2126 2127 page_t *
2127 2128 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2128 2129 struct seg *seg, caddr_t vaddr, void *arg)
2129 2130 {
2130 2131 pgcnt_t npages;
2131 2132 page_t *pp;
2132 2133 page_t *rootpp;
2133 2134 lgrp_t *lgrp;
2134 2135 lgrp_id_t *lgrpid = (lgrp_id_t *)arg;
2135 2136
2136 2137 ASSERT(vp != NULL);
2137 2138
2138 2139 ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2139 2140 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0);
2140 2141 /* but no others */
2141 2142
2142 2143 ASSERT((flags & PG_EXCL) == PG_EXCL);
2143 2144
2144 2145 npages = btop(bytes);
2145 2146
2146 2147 if (!kcage_on || panicstr) {
2147 2148 /*
2148 2149 * Cage is OFF, or we are single threaded in
2149 2150 * panic, so make everything a RELOC request.
2150 2151 */
2151 2152 flags &= ~PG_NORELOC;
2152 2153 }
2153 2154
2154 2155 /*
2155 2156 * Make sure there's adequate physical memory available.
2156 2157 * Note: PG_WAIT is ignored here.
2157 2158 */
2158 2159 if (freemem <= throttlefree + npages) {
2159 2160 VM_STAT_ADD(page_create_large_cnt[1]);
2160 2161 return (NULL);
2161 2162 }
2162 2163
2163 2164 /*
2164 2165 * If cage is on, dampen draw from cage when available
2165 2166 * cage space is low.
2166 2167 */
2167 2168 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) &&
2168 2169 kcage_freemem < kcage_throttlefree + npages) {
2169 2170
2170 2171 /*
2171 2172 * The cage is on, the caller wants PG_NORELOC
2172 2173 * pages and available cage memory is very low.
2173 2174 * Call kcage_create_throttle() to attempt to
2174 2175 * control demand on the cage.
2175 2176 */
2176 2177 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) {
2177 2178 VM_STAT_ADD(page_create_large_cnt[2]);
2178 2179 return (NULL);
2179 2180 }
2180 2181 }
2181 2182
2182 2183 if (!pcf_decrement_bucket(npages) &&
2183 2184 !pcf_decrement_multiple(NULL, npages, 1)) {
2184 2185 VM_STAT_ADD(page_create_large_cnt[4]);
2185 2186 return (NULL);
2186 2187 }
2187 2188
2188 2189 /*
2189 2190 * This is where this function behaves fundamentally differently
2190 2191 * than page_create_va(); since we're intending to map the page
2191 2192 * with a single TTE, we have to get it as a physically contiguous
2192 2193 * hardware pagesize chunk. If we can't, we fail.
2193 2194 */
2194 2195 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max &&
2195 2196 LGRP_EXISTS(lgrp_table[*lgrpid]))
2196 2197 lgrp = lgrp_table[*lgrpid];
2197 2198 else
2198 2199 lgrp = lgrp_mem_choose(seg, vaddr, bytes);
2199 2200
2200 2201 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr,
2201 2202 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) {
2202 2203 page_create_putback(npages);
2203 2204 VM_STAT_ADD(page_create_large_cnt[5]);
2204 2205 return (NULL);
2205 2206 }
2206 2207
2207 2208 /*
2208 2209 * if we got the page with the wrong mtype give it back this is a
2209 2210 * workaround for CR 6249718. When CR 6249718 is fixed we never get
2210 2211 * inside "if" and the workaround becomes just a nop
2211 2212 */
2212 2213 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) {
2213 2214 page_list_add_pages(rootpp, 0);
2214 2215 page_create_putback(npages);
2215 2216 VM_STAT_ADD(page_create_large_cnt[6]);
2216 2217 return (NULL);
2217 2218 }
2218 2219
2219 2220 /*
2220 2221 * If satisfying this request has left us with too little
2221 2222 * memory, start the wheels turning to get some back. The
2222 2223 * first clause of the test prevents waking up the pageout
2223 2224 * daemon in situations where it would decide that there's
2224 2225 * nothing to do.
2225 2226 */
2226 2227 if (nscan < desscan && freemem < minfree) {
2227 2228 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2228 2229 "pageout_cv_signal:freemem %ld", freemem);
2229 2230 cv_signal(&proc_pageout->p_cv);
2230 2231 }
2231 2232
2232 2233 pp = rootpp;
2233 2234 while (npages--) {
2234 2235 ASSERT(PAGE_EXCL(pp));
2235 2236 ASSERT(pp->p_vnode == NULL);
2236 2237 ASSERT(!hat_page_is_mapped(pp));
2237 2238 PP_CLRFREE(pp);
2238 2239 PP_CLRAGED(pp);
2239 2240 if (!page_hashin(pp, vp, off, NULL))
2240 2241 panic("page_create_large: hashin failed: page %p",
2241 2242 (void *)pp);
2242 2243 page_io_lock(pp);
2243 2244 off += PAGESIZE;
2244 2245 pp = pp->p_next;
2245 2246 }
2246 2247
2247 2248 VM_STAT_ADD(page_create_large_cnt[0]);
2248 2249 return (rootpp);
2249 2250 }
2250 2251
2251 2252 page_t *
2252 2253 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2253 2254 struct seg *seg, caddr_t vaddr)
2254 2255 {
2255 2256 page_t *plist = NULL;
2256 2257 pgcnt_t npages;
2257 2258 pgcnt_t found_on_free = 0;
2258 2259 pgcnt_t pages_req;
2259 2260 page_t *npp = NULL;
2260 2261 struct pcf *p;
2261 2262 lgrp_t *lgrp;
2262 2263
2263 2264 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
2264 2265 "page_create_start:vp %p off %llx bytes %lu flags %x",
2265 2266 vp, off, bytes, flags);
2266 2267
2267 2268 ASSERT(bytes != 0 && vp != NULL);
2268 2269
2269 2270 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) {
2270 2271 panic("page_create: invalid flags");
2271 2272 /*NOTREACHED*/
2272 2273 }
2273 2274 ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2274 2275 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0);
2275 2276 /* but no others */
2276 2277
2277 2278 pages_req = npages = btopr(bytes);
2278 2279 /*
2279 2280 * Try to see whether request is too large to *ever* be
2280 2281 * satisfied, in order to prevent deadlock. We arbitrarily
2281 2282 * decide to limit maximum size requests to max_page_get.
2282 2283 */
2283 2284 if (npages >= max_page_get) {
2284 2285 if ((flags & PG_WAIT) == 0) {
2285 2286 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG,
2286 2287 "page_create_toobig:vp %p off %llx npages "
2287 2288 "%lu max_page_get %lu",
2288 2289 vp, off, npages, max_page_get);
2289 2290 return (NULL);
2290 2291 } else {
2291 2292 cmn_err(CE_WARN,
2292 2293 "Request for too much kernel memory "
2293 2294 "(%lu bytes), will hang forever", bytes);
2294 2295 for (;;)
2295 2296 delay(1000000000);
2296 2297 }
2297 2298 }
2298 2299
2299 2300 if (!kcage_on || panicstr) {
2300 2301 /*
2301 2302 * Cage is OFF, or we are single threaded in
2302 2303 * panic, so make everything a RELOC request.
2303 2304 */
2304 2305 flags &= ~PG_NORELOC;
2305 2306 }
2306 2307
2307 2308 if (freemem <= throttlefree + npages)
2308 2309 if (!page_create_throttle(npages, flags))
2309 2310 return (NULL);
2310 2311
2311 2312 /*
2312 2313 * If cage is on, dampen draw from cage when available
2313 2314 * cage space is low.
2314 2315 */
2315 2316 if ((flags & PG_NORELOC) &&
2316 2317 kcage_freemem < kcage_throttlefree + npages) {
2317 2318
2318 2319 /*
2319 2320 * The cage is on, the caller wants PG_NORELOC
2320 2321 * pages and available cage memory is very low.
2321 2322 * Call kcage_create_throttle() to attempt to
2322 2323 * control demand on the cage.
2323 2324 */
2324 2325 if (kcage_create_throttle(npages, flags) == KCT_FAILURE)
2325 2326 return (NULL);
2326 2327 }
2327 2328
2328 2329 VM_STAT_ADD(page_create_cnt[0]);
2329 2330
2330 2331 if (!pcf_decrement_bucket(npages)) {
2331 2332 /*
2332 2333 * Have to look harder. If npages is greater than
2333 2334 * one, then we might have to coalesce the counters.
2334 2335 *
2335 2336 * Go wait. We come back having accounted
2336 2337 * for the memory.
2337 2338 */
2338 2339 VM_STAT_ADD(page_create_cnt[1]);
2339 2340 if (!page_create_wait(npages, flags)) {
2340 2341 VM_STAT_ADD(page_create_cnt[2]);
2341 2342 return (NULL);
2342 2343 }
2343 2344 }
2344 2345
2345 2346 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
2346 2347 "page_create_success:vp %p off %llx", vp, off);
2347 2348
2348 2349 /*
2349 2350 * If satisfying this request has left us with too little
2350 2351 * memory, start the wheels turning to get some back. The
2351 2352 * first clause of the test prevents waking up the pageout
2352 2353 * daemon in situations where it would decide that there's
2353 2354 * nothing to do.
2354 2355 */
2355 2356 if (nscan < desscan && freemem < minfree) {
2356 2357 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2357 2358 "pageout_cv_signal:freemem %ld", freemem);
2358 2359 cv_signal(&proc_pageout->p_cv);
2359 2360 }
2360 2361
2361 2362 /*
2362 2363 * Loop around collecting the requested number of pages.
2363 2364 * Most of the time, we have to `create' a new page. With
2364 2365 * this in mind, pull the page off the free list before
2365 2366 * getting the hash lock. This will minimize the hash
2366 2367 * lock hold time, nesting, and the like. If it turns
2367 2368 * out we don't need the page, we put it back at the end.
2368 2369 */
2369 2370 while (npages--) {
2370 2371 page_t *pp;
2371 2372 kmutex_t *phm = NULL;
2372 2373 ulong_t index;
2373 2374
2374 2375 index = PAGE_HASH_FUNC(vp, off);
2375 2376 top:
2376 2377 ASSERT(phm == NULL);
2377 2378 ASSERT(index == PAGE_HASH_FUNC(vp, off));
2378 2379 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
2379 2380
2380 2381 if (npp == NULL) {
2381 2382 /*
2382 2383 * Try to get a page from the freelist (ie,
2383 2384 * a page with no [vp, off] tag). If that
2384 2385 * fails, use the cachelist.
2385 2386 *
2386 2387 * During the first attempt at both the free
2387 2388 * and cache lists we try for the correct color.
2388 2389 */
2389 2390 /*
2390 2391 * XXXX-how do we deal with virtual indexed
2391 2392 * caches and and colors?
2392 2393 */
2393 2394 VM_STAT_ADD(page_create_cnt[4]);
2394 2395 /*
2395 2396 * Get lgroup to allocate next page of shared memory
2396 2397 * from and use it to specify where to allocate
2397 2398 * the physical memory
2398 2399 */
2399 2400 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
2400 2401 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
2401 2402 flags | PG_MATCH_COLOR, lgrp);
2402 2403 if (npp == NULL) {
2403 2404 npp = page_get_cachelist(vp, off, seg,
2404 2405 vaddr, flags | PG_MATCH_COLOR, lgrp);
2405 2406 if (npp == NULL) {
2406 2407 npp = page_create_get_something(vp,
2407 2408 off, seg, vaddr,
2408 2409 flags & ~PG_MATCH_COLOR);
2409 2410 }
2410 2411
2411 2412 if (PP_ISAGED(npp) == 0) {
2412 2413 /*
2413 2414 * Since this page came from the
2414 2415 * cachelist, we must destroy the
2415 2416 * old vnode association.
2416 2417 */
2417 2418 page_hashout(npp, NULL);
2418 2419 }
2419 2420 }
2420 2421 }
2421 2422
2422 2423 /*
2423 2424 * We own this page!
2424 2425 */
2425 2426 ASSERT(PAGE_EXCL(npp));
2426 2427 ASSERT(npp->p_vnode == NULL);
2427 2428 ASSERT(!hat_page_is_mapped(npp));
2428 2429 PP_CLRFREE(npp);
2429 2430 PP_CLRAGED(npp);
2430 2431
2431 2432 /*
2432 2433 * Here we have a page in our hot little mits and are
2433 2434 * just waiting to stuff it on the appropriate lists.
2434 2435 * Get the mutex and check to see if it really does
2435 2436 * not exist.
2436 2437 */
2437 2438 phm = PAGE_HASH_MUTEX(index);
2438 2439 mutex_enter(phm);
2439 2440 pp = page_hash_search(index, vp, off);
2440 2441 if (pp == NULL) {
2441 2442 VM_STAT_ADD(page_create_new);
2442 2443 pp = npp;
2443 2444 npp = NULL;
2444 2445 if (!page_hashin(pp, vp, off, phm)) {
2445 2446 /*
2446 2447 * Since we hold the page hash mutex and
2447 2448 * just searched for this page, page_hashin
2448 2449 * had better not fail. If it does, that
2449 2450 * means somethread did not follow the
2450 2451 * page hash mutex rules. Panic now and
2451 2452 * get it over with. As usual, go down
2452 2453 * holding all the locks.
2453 2454 */
2454 2455 ASSERT(MUTEX_HELD(phm));
2455 2456 panic("page_create: "
2456 2457 "hashin failed %p %p %llx %p",
2457 2458 (void *)pp, (void *)vp, off, (void *)phm);
2458 2459 /*NOTREACHED*/
2459 2460 }
2460 2461 ASSERT(MUTEX_HELD(phm));
2461 2462 mutex_exit(phm);
2462 2463 phm = NULL;
2463 2464
2464 2465 /*
2465 2466 * Hat layer locking need not be done to set
2466 2467 * the following bits since the page is not hashed
2467 2468 * and was on the free list (i.e., had no mappings).
2468 2469 *
2469 2470 * Set the reference bit to protect
2470 2471 * against immediate pageout
2471 2472 *
2472 2473 * XXXmh modify freelist code to set reference
2473 2474 * bit so we don't have to do it here.
2474 2475 */
2475 2476 page_set_props(pp, P_REF);
2476 2477 found_on_free++;
2477 2478 } else {
2478 2479 VM_STAT_ADD(page_create_exists);
2479 2480 if (flags & PG_EXCL) {
2480 2481 /*
2481 2482 * Found an existing page, and the caller
2482 2483 * wanted all new pages. Undo all of the work
2483 2484 * we have done.
2484 2485 */
2485 2486 mutex_exit(phm);
2486 2487 phm = NULL;
2487 2488 while (plist != NULL) {
2488 2489 pp = plist;
2489 2490 page_sub(&plist, pp);
2490 2491 page_io_unlock(pp);
2491 2492 /* large pages should not end up here */
2492 2493 ASSERT(pp->p_szc == 0);
2493 2494 /*LINTED: constant in conditional ctx*/
2494 2495 VN_DISPOSE(pp, B_INVAL, 0, kcred);
2495 2496 }
2496 2497 VM_STAT_ADD(page_create_found_one);
2497 2498 goto fail;
2498 2499 }
2499 2500 ASSERT(flags & PG_WAIT);
2500 2501 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) {
2501 2502 /*
2502 2503 * Start all over again if we blocked trying
2503 2504 * to lock the page.
2504 2505 */
2505 2506 mutex_exit(phm);
2506 2507 VM_STAT_ADD(page_create_page_lock_failed);
2507 2508 phm = NULL;
2508 2509 goto top;
2509 2510 }
2510 2511 mutex_exit(phm);
2511 2512 phm = NULL;
2512 2513
2513 2514 if (PP_ISFREE(pp)) {
2514 2515 ASSERT(PP_ISAGED(pp) == 0);
2515 2516 VM_STAT_ADD(pagecnt.pc_get_cache);
2516 2517 page_list_sub(pp, PG_CACHE_LIST);
2517 2518 PP_CLRFREE(pp);
2518 2519 found_on_free++;
2519 2520 }
2520 2521 }
2521 2522
2522 2523 /*
2523 2524 * Got a page! It is locked. Acquire the i/o
2524 2525 * lock since we are going to use the p_next and
2525 2526 * p_prev fields to link the requested pages together.
2526 2527 */
2527 2528 page_io_lock(pp);
2528 2529 page_add(&plist, pp);
2529 2530 plist = plist->p_next;
2530 2531 off += PAGESIZE;
2531 2532 vaddr += PAGESIZE;
2532 2533 }
2533 2534
2534 2535 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1);
2535 2536 fail:
2536 2537 if (npp != NULL) {
2537 2538 /*
2538 2539 * Did not need this page after all.
2539 2540 * Put it back on the free list.
2540 2541 */
2541 2542 VM_STAT_ADD(page_create_putbacks);
2542 2543 PP_SETFREE(npp);
2543 2544 PP_SETAGED(npp);
2544 2545 npp->p_offset = (u_offset_t)-1;
2545 2546 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
2546 2547 page_unlock(npp);
2547 2548
2548 2549 }
2549 2550
2550 2551 ASSERT(pages_req >= found_on_free);
2551 2552
2552 2553 {
2553 2554 uint_t overshoot = (uint_t)(pages_req - found_on_free);
2554 2555
2555 2556 if (overshoot) {
2556 2557 VM_STAT_ADD(page_create_overshoot);
2557 2558 p = &pcf[PCF_INDEX()];
2558 2559 mutex_enter(&p->pcf_lock);
2559 2560 if (p->pcf_block) {
2560 2561 p->pcf_reserve += overshoot;
2561 2562 } else {
2562 2563 p->pcf_count += overshoot;
2563 2564 if (p->pcf_wait) {
2564 2565 mutex_enter(&new_freemem_lock);
2565 2566 if (freemem_wait) {
2566 2567 cv_signal(&freemem_cv);
2567 2568 p->pcf_wait--;
2568 2569 } else {
2569 2570 p->pcf_wait = 0;
2570 2571 }
2571 2572 mutex_exit(&new_freemem_lock);
2572 2573 }
2573 2574 }
2574 2575 mutex_exit(&p->pcf_lock);
2575 2576 /* freemem is approximate, so this test OK */
2576 2577 if (!p->pcf_block)
2577 2578 freemem += overshoot;
2578 2579 }
2579 2580 }
2580 2581
2581 2582 return (plist);
2582 2583 }
2583 2584
2584 2585 /*
2585 2586 * One or more constituent pages of this large page has been marked
2586 2587 * toxic. Simply demote the large page to PAGESIZE pages and let
2587 2588 * page_free() handle it. This routine should only be called by
2588 2589 * large page free routines (page_free_pages() and page_destroy_pages().
2589 2590 * All pages are locked SE_EXCL and have already been marked free.
2590 2591 */
2591 2592 static void
2592 2593 page_free_toxic_pages(page_t *rootpp)
2593 2594 {
2594 2595 page_t *tpp;
2595 2596 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc);
2596 2597 uint_t szc = rootpp->p_szc;
2597 2598
2598 2599 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) {
2599 2600 ASSERT(tpp->p_szc == szc);
2600 2601 ASSERT((PAGE_EXCL(tpp) &&
2601 2602 !page_iolock_assert(tpp)) || panicstr);
2602 2603 tpp->p_szc = 0;
2603 2604 }
2604 2605
2605 2606 while (rootpp != NULL) {
2606 2607 tpp = rootpp;
2607 2608 page_sub(&rootpp, tpp);
2608 2609 ASSERT(PP_ISFREE(tpp));
2609 2610 PP_CLRFREE(tpp);
2610 2611 page_free(tpp, 1);
2611 2612 }
2612 2613 }
2613 2614
2614 2615 /*
2615 2616 * Put page on the "free" list.
2616 2617 * The free list is really two lists maintained by
2617 2618 * the PSM of whatever machine we happen to be on.
2618 2619 */
2619 2620 void
2620 2621 page_free(page_t *pp, int dontneed)
2621 2622 {
2622 2623 struct pcf *p;
2623 2624 uint_t pcf_index;
2624 2625
2625 2626 ASSERT((PAGE_EXCL(pp) &&
2626 2627 !page_iolock_assert(pp)) || panicstr);
2627 2628
2628 2629 if (PP_ISFREE(pp)) {
2629 2630 panic("page_free: page %p is free", (void *)pp);
2630 2631 }
2631 2632
2632 2633 if (pp->p_szc != 0) {
2633 2634 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
2634 2635 PP_ISKAS(pp)) {
2635 2636 panic("page_free: anon or kernel "
2636 2637 "or no vnode large page %p", (void *)pp);
2637 2638 }
2638 2639 page_demote_vp_pages(pp);
2639 2640 ASSERT(pp->p_szc == 0);
2640 2641 }
2641 2642
2642 2643 /*
2643 2644 * The page_struct_lock need not be acquired to examine these
2644 2645 * fields since the page has an "exclusive" lock.
2645 2646 */
2646 2647 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
2647 2648 pp->p_slckcnt != 0) {
2648 2649 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
2649 2650 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt,
2650 2651 pp->p_cowcnt, pp->p_slckcnt);
2651 2652 /*NOTREACHED*/
2652 2653 }
2653 2654
2654 2655 ASSERT(!hat_page_getshare(pp));
2655 2656
2656 2657 PP_SETFREE(pp);
2657 2658 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) ||
2658 2659 !hat_ismod(pp));
2659 2660 page_clr_all_props(pp);
2660 2661 ASSERT(!hat_page_getshare(pp));
2661 2662
2662 2663 /*
2663 2664 * Now we add the page to the head of the free list.
2664 2665 * But if this page is associated with a paged vnode
2665 2666 * then we adjust the head forward so that the page is
2666 2667 * effectively at the end of the list.
2667 2668 */
2668 2669 if (pp->p_vnode == NULL) {
2669 2670 /*
2670 2671 * Page has no identity, put it on the free list.
2671 2672 */
2672 2673 PP_SETAGED(pp);
2673 2674 pp->p_offset = (u_offset_t)-1;
2674 2675 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
2675 2676 VM_STAT_ADD(pagecnt.pc_free_free);
2676 2677 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2677 2678 "page_free_free:pp %p", pp);
2678 2679 } else {
2679 2680 PP_CLRAGED(pp);
2680 2681
2681 2682 if (!dontneed) {
2682 2683 /* move it to the tail of the list */
2683 2684 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL);
2684 2685
2685 2686 VM_STAT_ADD(pagecnt.pc_free_cache);
2686 2687 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL,
2687 2688 "page_free_cache_tail:pp %p", pp);
2688 2689 } else {
2689 2690 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD);
2690 2691
2691 2692 VM_STAT_ADD(pagecnt.pc_free_dontneed);
2692 2693 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD,
2693 2694 "page_free_cache_head:pp %p", pp);
2694 2695 }
2695 2696 }
2696 2697 page_unlock(pp);
2697 2698
2698 2699 /*
2699 2700 * Now do the `freemem' accounting.
2700 2701 */
2701 2702 pcf_index = PCF_INDEX();
2702 2703 p = &pcf[pcf_index];
2703 2704
2704 2705 mutex_enter(&p->pcf_lock);
2705 2706 if (p->pcf_block) {
2706 2707 p->pcf_reserve += 1;
2707 2708 } else {
2708 2709 p->pcf_count += 1;
2709 2710 if (p->pcf_wait) {
2710 2711 mutex_enter(&new_freemem_lock);
2711 2712 /*
2712 2713 * Check to see if some other thread
2713 2714 * is actually waiting. Another bucket
2714 2715 * may have woken it up by now. If there
2715 2716 * are no waiters, then set our pcf_wait
2716 2717 * count to zero to avoid coming in here
2717 2718 * next time. Also, since only one page
2718 2719 * was put on the free list, just wake
2719 2720 * up one waiter.
2720 2721 */
2721 2722 if (freemem_wait) {
2722 2723 cv_signal(&freemem_cv);
2723 2724 p->pcf_wait--;
2724 2725 } else {
2725 2726 p->pcf_wait = 0;
2726 2727 }
2727 2728 mutex_exit(&new_freemem_lock);
2728 2729 }
2729 2730 }
2730 2731 mutex_exit(&p->pcf_lock);
2731 2732
2732 2733 /* freemem is approximate, so this test OK */
2733 2734 if (!p->pcf_block)
2734 2735 freemem += 1;
2735 2736 }
2736 2737
2737 2738 /*
2738 2739 * Put page on the "free" list during intial startup.
2739 2740 * This happens during initial single threaded execution.
2740 2741 */
2741 2742 void
2742 2743 page_free_at_startup(page_t *pp)
2743 2744 {
2744 2745 struct pcf *p;
2745 2746 uint_t pcf_index;
2746 2747
2747 2748 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT);
2748 2749 VM_STAT_ADD(pagecnt.pc_free_free);
2749 2750
2750 2751 /*
2751 2752 * Now do the `freemem' accounting.
2752 2753 */
2753 2754 pcf_index = PCF_INDEX();
2754 2755 p = &pcf[pcf_index];
2755 2756
2756 2757 ASSERT(p->pcf_block == 0);
2757 2758 ASSERT(p->pcf_wait == 0);
2758 2759 p->pcf_count += 1;
2759 2760
2760 2761 /* freemem is approximate, so this is OK */
2761 2762 freemem += 1;
2762 2763 }
2763 2764
2764 2765 void
2765 2766 page_free_pages(page_t *pp)
2766 2767 {
2767 2768 page_t *tpp, *rootpp = NULL;
2768 2769 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc);
2769 2770 pgcnt_t i;
2770 2771 uint_t szc = pp->p_szc;
2771 2772
2772 2773 VM_STAT_ADD(pagecnt.pc_free_pages);
2773 2774 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2774 2775 "page_free_free:pp %p", pp);
2775 2776
2776 2777 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
2777 2778 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
2778 2779 panic("page_free_pages: not root page %p", (void *)pp);
2779 2780 /*NOTREACHED*/
2780 2781 }
2781 2782
2782 2783 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
2783 2784 ASSERT((PAGE_EXCL(tpp) &&
2784 2785 !page_iolock_assert(tpp)) || panicstr);
2785 2786 if (PP_ISFREE(tpp)) {
2786 2787 panic("page_free_pages: page %p is free", (void *)tpp);
2787 2788 /*NOTREACHED*/
2788 2789 }
2789 2790 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 ||
2790 2791 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) {
2791 2792 panic("page_free_pages %p", (void *)tpp);
2792 2793 /*NOTREACHED*/
2793 2794 }
2794 2795
2795 2796 ASSERT(!hat_page_getshare(tpp));
2796 2797 ASSERT(tpp->p_vnode == NULL);
2797 2798 ASSERT(tpp->p_szc == szc);
2798 2799
2799 2800 PP_SETFREE(tpp);
2800 2801 page_clr_all_props(tpp);
2801 2802 PP_SETAGED(tpp);
2802 2803 tpp->p_offset = (u_offset_t)-1;
2803 2804 ASSERT(tpp->p_next == tpp);
2804 2805 ASSERT(tpp->p_prev == tpp);
2805 2806 page_list_concat(&rootpp, &tpp);
2806 2807 }
2807 2808 ASSERT(rootpp == pp);
2808 2809
2809 2810 page_list_add_pages(rootpp, 0);
2810 2811 page_create_putback(pgcnt);
2811 2812 }
2812 2813
2813 2814 int free_pages = 1;
2814 2815
2815 2816 /*
2816 2817 * This routine attempts to return pages to the cachelist via page_release().
2817 2818 * It does not *have* to be successful in all cases, since the pageout scanner
2818 2819 * will catch any pages it misses. It does need to be fast and not introduce
2819 2820 * too much overhead.
2820 2821 *
2821 2822 * If a page isn't found on the unlocked sweep of the page_hash bucket, we
2822 2823 * don't lock and retry. This is ok, since the page scanner will eventually
2823 2824 * find any page we miss in free_vp_pages().
2824 2825 */
2825 2826 void
2826 2827 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len)
2827 2828 {
2828 2829 page_t *pp;
2829 2830 u_offset_t eoff;
2830 2831 extern int swap_in_range(vnode_t *, u_offset_t, size_t);
2831 2832
2832 2833 eoff = off + len;
2833 2834
2834 2835 if (free_pages == 0)
2835 2836 return;
2836 2837 if (swap_in_range(vp, off, len))
2837 2838 return;
2838 2839
2839 2840 for (; off < eoff; off += PAGESIZE) {
2840 2841
2841 2842 /*
2842 2843 * find the page using a fast, but inexact search. It'll be OK
2843 2844 * if a few pages slip through the cracks here.
2844 2845 */
2845 2846 pp = page_exists(vp, off);
2846 2847
2847 2848 /*
2848 2849 * If we didn't find the page (it may not exist), the page
2849 2850 * is free, looks still in use (shared), or we can't lock it,
2850 2851 * just give up.
2851 2852 */
2852 2853 if (pp == NULL ||
2853 2854 PP_ISFREE(pp) ||
2854 2855 page_share_cnt(pp) > 0 ||
2855 2856 !page_trylock(pp, SE_EXCL))
2856 2857 continue;
2857 2858
2858 2859 /*
2859 2860 * Once we have locked pp, verify that it's still the
2860 2861 * correct page and not already free
2861 2862 */
2862 2863 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL));
2863 2864 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) {
2864 2865 page_unlock(pp);
2865 2866 continue;
2866 2867 }
2867 2868
2868 2869 /*
2869 2870 * try to release the page...
2870 2871 */
2871 2872 (void) page_release(pp, 1);
2872 2873 }
2873 2874 }
2874 2875
2875 2876 /*
2876 2877 * Reclaim the given page from the free list.
2877 2878 * If pp is part of a large pages, only the given constituent page is reclaimed
2878 2879 * and the large page it belonged to will be demoted. This can only happen
2879 2880 * if the page is not on the cachelist.
2880 2881 *
2881 2882 * Returns 1 on success or 0 on failure.
2882 2883 *
2883 2884 * The page is unlocked if it can't be reclaimed (when freemem == 0).
2884 2885 * If `lock' is non-null, it will be dropped and re-acquired if
2885 2886 * the routine must wait while freemem is 0.
2886 2887 *
2887 2888 * As it turns out, boot_getpages() does this. It picks a page,
2888 2889 * based on where OBP mapped in some address, gets its pfn, searches
2889 2890 * the memsegs, locks the page, then pulls it off the free list!
2890 2891 */
2891 2892 int
2892 2893 page_reclaim(page_t *pp, kmutex_t *lock)
2893 2894 {
2894 2895 struct pcf *p;
2895 2896 struct cpu *cpup;
2896 2897 int enough;
2897 2898 uint_t i;
2898 2899
2899 2900 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
2900 2901 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
2901 2902
2902 2903 /*
2903 2904 * If `freemem' is 0, we cannot reclaim this page from the
2904 2905 * freelist, so release every lock we might hold: the page,
2905 2906 * and the `lock' before blocking.
2906 2907 *
2907 2908 * The only way `freemem' can become 0 while there are pages
2908 2909 * marked free (have their p->p_free bit set) is when the
2909 2910 * system is low on memory and doing a page_create(). In
2910 2911 * order to guarantee that once page_create() starts acquiring
2911 2912 * pages it will be able to get all that it needs since `freemem'
2912 2913 * was decreased by the requested amount. So, we need to release
2913 2914 * this page, and let page_create() have it.
2914 2915 *
2915 2916 * Since `freemem' being zero is not supposed to happen, just
2916 2917 * use the usual hash stuff as a starting point. If that bucket
2917 2918 * is empty, then assume the worst, and start at the beginning
2918 2919 * of the pcf array. If we always start at the beginning
2919 2920 * when acquiring more than one pcf lock, there won't be any
2920 2921 * deadlock problems.
2921 2922 */
2922 2923
2923 2924 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
2924 2925
2925 2926 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) {
2926 2927 pcf_acquire_all();
2927 2928 goto page_reclaim_nomem;
2928 2929 }
2929 2930
2930 2931 enough = pcf_decrement_bucket(1);
2931 2932
2932 2933 if (!enough) {
2933 2934 VM_STAT_ADD(page_reclaim_zero);
2934 2935 /*
2935 2936 * Check again. Its possible that some other thread
2936 2937 * could have been right behind us, and added one
2937 2938 * to a list somewhere. Acquire each of the pcf locks
2938 2939 * until we find a page.
2939 2940 */
2940 2941 p = pcf;
2941 2942 for (i = 0; i < pcf_fanout; i++) {
2942 2943 mutex_enter(&p->pcf_lock);
2943 2944 if (p->pcf_count >= 1) {
2944 2945 p->pcf_count -= 1;
2945 2946 /*
2946 2947 * freemem is not protected by any lock. Thus,
2947 2948 * we cannot have any assertion containing
2948 2949 * freemem here.
2949 2950 */
2950 2951 freemem -= 1;
2951 2952 enough = 1;
2952 2953 break;
2953 2954 }
2954 2955 p++;
2955 2956 }
2956 2957
2957 2958 if (!enough) {
2958 2959 page_reclaim_nomem:
2959 2960 /*
2960 2961 * We really can't have page `pp'.
2961 2962 * Time for the no-memory dance with
2962 2963 * page_free(). This is just like
2963 2964 * page_create_wait(). Plus the added
2964 2965 * attraction of releasing whatever mutex
2965 2966 * we held when we were called with in `lock'.
2966 2967 * Page_unlock() will wakeup any thread
2967 2968 * waiting around for this page.
2968 2969 */
2969 2970 if (lock) {
2970 2971 VM_STAT_ADD(page_reclaim_zero_locked);
2971 2972 mutex_exit(lock);
2972 2973 }
2973 2974 page_unlock(pp);
2974 2975
2975 2976 /*
2976 2977 * get this before we drop all the pcf locks.
2977 2978 */
2978 2979 mutex_enter(&new_freemem_lock);
2979 2980
2980 2981 p = pcf;
2981 2982 for (i = 0; i < pcf_fanout; i++) {
2982 2983 p->pcf_wait++;
2983 2984 mutex_exit(&p->pcf_lock);
2984 2985 p++;
2985 2986 }
2986 2987
2987 2988 freemem_wait++;
2988 2989 cv_wait(&freemem_cv, &new_freemem_lock);
2989 2990 freemem_wait--;
2990 2991
2991 2992 mutex_exit(&new_freemem_lock);
2992 2993
2993 2994 if (lock) {
2994 2995 mutex_enter(lock);
2995 2996 }
2996 2997 return (0);
2997 2998 }
2998 2999
2999 3000 /*
3000 3001 * The pcf accounting has been done,
3001 3002 * though none of the pcf_wait flags have been set,
3002 3003 * drop the locks and continue on.
3003 3004 */
3004 3005 while (p >= pcf) {
3005 3006 mutex_exit(&p->pcf_lock);
3006 3007 p--;
3007 3008 }
3008 3009 }
3009 3010
3010 3011
3011 3012 VM_STAT_ADD(pagecnt.pc_reclaim);
3012 3013
3013 3014 /*
3014 3015 * page_list_sub will handle the case where pp is a large page.
3015 3016 * It's possible that the page was promoted while on the freelist
3016 3017 */
3017 3018 if (PP_ISAGED(pp)) {
3018 3019 page_list_sub(pp, PG_FREE_LIST);
3019 3020 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
3020 3021 "page_reclaim_free:pp %p", pp);
3021 3022 } else {
3022 3023 page_list_sub(pp, PG_CACHE_LIST);
3023 3024 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
3024 3025 "page_reclaim_cache:pp %p", pp);
3025 3026 }
3026 3027
3027 3028 /*
3028 3029 * clear the p_free & p_age bits since this page is no longer
3029 3030 * on the free list. Notice that there was a brief time where
3030 3031 * a page is marked as free, but is not on the list.
3031 3032 *
3032 3033 * Set the reference bit to protect against immediate pageout.
3033 3034 */
3034 3035 PP_CLRFREE(pp);
3035 3036 PP_CLRAGED(pp);
3036 3037 page_set_props(pp, P_REF);
3037 3038
3038 3039 CPU_STATS_ENTER_K();
3039 3040 cpup = CPU; /* get cpup now that CPU cannot change */
3040 3041 CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
3041 3042 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
3042 3043 CPU_STATS_EXIT_K();
3043 3044 ASSERT(pp->p_szc == 0);
3044 3045
3045 3046 return (1);
3046 3047 }
3047 3048
3048 3049 /*
3049 3050 * Destroy identity of the page and put it back on
3050 3051 * the page free list. Assumes that the caller has
3051 3052 * acquired the "exclusive" lock on the page.
3052 3053 */
3053 3054 void
3054 3055 page_destroy(page_t *pp, int dontfree)
3055 3056 {
3056 3057 ASSERT((PAGE_EXCL(pp) &&
3057 3058 !page_iolock_assert(pp)) || panicstr);
3058 3059 ASSERT(pp->p_slckcnt == 0 || panicstr);
3059 3060
3060 3061 if (pp->p_szc != 0) {
3061 3062 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
3062 3063 PP_ISKAS(pp)) {
3063 3064 panic("page_destroy: anon or kernel or no vnode "
3064 3065 "large page %p", (void *)pp);
3065 3066 }
3066 3067 page_demote_vp_pages(pp);
3067 3068 ASSERT(pp->p_szc == 0);
3068 3069 }
3069 3070
3070 3071 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp);
3071 3072
3072 3073 /*
3073 3074 * Unload translations, if any, then hash out the
3074 3075 * page to erase its identity.
3075 3076 */
3076 3077 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3077 3078 page_hashout(pp, NULL);
3078 3079
3079 3080 if (!dontfree) {
3080 3081 /*
3081 3082 * Acquire the "freemem_lock" for availrmem.
3082 3083 * The page_struct_lock need not be acquired for lckcnt
3083 3084 * and cowcnt since the page has an "exclusive" lock.
3084 3085 * We are doing a modified version of page_pp_unlock here.
3085 3086 */
3086 3087 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
3087 3088 mutex_enter(&freemem_lock);
3088 3089 if (pp->p_lckcnt != 0) {
3089 3090 availrmem++;
3090 3091 pages_locked--;
3091 3092 pp->p_lckcnt = 0;
3092 3093 }
3093 3094 if (pp->p_cowcnt != 0) {
3094 3095 availrmem += pp->p_cowcnt;
3095 3096 pages_locked -= pp->p_cowcnt;
3096 3097 pp->p_cowcnt = 0;
3097 3098 }
3098 3099 mutex_exit(&freemem_lock);
3099 3100 }
3100 3101 /*
3101 3102 * Put the page on the "free" list.
3102 3103 */
3103 3104 page_free(pp, 0);
3104 3105 }
3105 3106 }
3106 3107
3107 3108 void
3108 3109 page_destroy_pages(page_t *pp)
3109 3110 {
3110 3111
3111 3112 page_t *tpp, *rootpp = NULL;
3112 3113 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc);
3113 3114 pgcnt_t i, pglcks = 0;
3114 3115 uint_t szc = pp->p_szc;
3115 3116
3116 3117 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
3117 3118
3118 3119 VM_STAT_ADD(pagecnt.pc_destroy_pages);
3119 3120
3120 3121 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp);
3121 3122
3122 3123 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
3123 3124 panic("page_destroy_pages: not root page %p", (void *)pp);
3124 3125 /*NOTREACHED*/
3125 3126 }
3126 3127
3127 3128 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
3128 3129 ASSERT((PAGE_EXCL(tpp) &&
3129 3130 !page_iolock_assert(tpp)) || panicstr);
3130 3131 ASSERT(tpp->p_slckcnt == 0 || panicstr);
3131 3132 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
3132 3133 page_hashout(tpp, NULL);
3133 3134 ASSERT(tpp->p_offset == (u_offset_t)-1);
3134 3135 if (tpp->p_lckcnt != 0) {
3135 3136 pglcks++;
3136 3137 tpp->p_lckcnt = 0;
3137 3138 } else if (tpp->p_cowcnt != 0) {
3138 3139 pglcks += tpp->p_cowcnt;
3139 3140 tpp->p_cowcnt = 0;
3140 3141 }
3141 3142 ASSERT(!hat_page_getshare(tpp));
3142 3143 ASSERT(tpp->p_vnode == NULL);
3143 3144 ASSERT(tpp->p_szc == szc);
3144 3145
3145 3146 PP_SETFREE(tpp);
3146 3147 page_clr_all_props(tpp);
3147 3148 PP_SETAGED(tpp);
3148 3149 ASSERT(tpp->p_next == tpp);
3149 3150 ASSERT(tpp->p_prev == tpp);
3150 3151 page_list_concat(&rootpp, &tpp);
3151 3152 }
3152 3153
3153 3154 ASSERT(rootpp == pp);
3154 3155 if (pglcks != 0) {
3155 3156 mutex_enter(&freemem_lock);
3156 3157 availrmem += pglcks;
3157 3158 mutex_exit(&freemem_lock);
3158 3159 }
3159 3160
3160 3161 page_list_add_pages(rootpp, 0);
3161 3162 page_create_putback(pgcnt);
3162 3163 }
3163 3164
3164 3165 /*
3165 3166 * Similar to page_destroy(), but destroys pages which are
3166 3167 * locked and known to be on the page free list. Since
3167 3168 * the page is known to be free and locked, no one can access
3168 3169 * it.
3169 3170 *
3170 3171 * Also, the number of free pages does not change.
3171 3172 */
3172 3173 void
3173 3174 page_destroy_free(page_t *pp)
3174 3175 {
3175 3176 ASSERT(PAGE_EXCL(pp));
3176 3177 ASSERT(PP_ISFREE(pp));
3177 3178 ASSERT(pp->p_vnode);
3178 3179 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0);
3179 3180 ASSERT(!hat_page_is_mapped(pp));
3180 3181 ASSERT(PP_ISAGED(pp) == 0);
3181 3182 ASSERT(pp->p_szc == 0);
3182 3183
3183 3184 VM_STAT_ADD(pagecnt.pc_destroy_free);
3184 3185 page_list_sub(pp, PG_CACHE_LIST);
3185 3186
3186 3187 page_hashout(pp, NULL);
3187 3188 ASSERT(pp->p_vnode == NULL);
3188 3189 ASSERT(pp->p_offset == (u_offset_t)-1);
3189 3190 ASSERT(pp->p_hash == NULL);
3190 3191
3191 3192 PP_SETAGED(pp);
3192 3193 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
3193 3194 page_unlock(pp);
3194 3195
3195 3196 mutex_enter(&new_freemem_lock);
3196 3197 if (freemem_wait) {
3197 3198 cv_signal(&freemem_cv);
3198 3199 }
3199 3200 mutex_exit(&new_freemem_lock);
3200 3201 }
3201 3202
3202 3203 /*
3203 3204 * Rename the page "opp" to have an identity specified
3204 3205 * by [vp, off]. If a page already exists with this name
3205 3206 * it is locked and destroyed. Note that the page's
3206 3207 * translations are not unloaded during the rename.
3207 3208 *
3208 3209 * This routine is used by the anon layer to "steal" the
3209 3210 * original page and is not unlike destroying a page and
3210 3211 * creating a new page using the same page frame.
3211 3212 *
3212 3213 * XXX -- Could deadlock if caller 1 tries to rename A to B while
3213 3214 * caller 2 tries to rename B to A.
3214 3215 */
3215 3216 void
3216 3217 page_rename(page_t *opp, vnode_t *vp, u_offset_t off)
3217 3218 {
3218 3219 page_t *pp;
3219 3220 int olckcnt = 0;
3220 3221 int ocowcnt = 0;
3221 3222 kmutex_t *phm;
3222 3223 ulong_t index;
3223 3224
3224 3225 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp));
3225 3226 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3226 3227 ASSERT(PP_ISFREE(opp) == 0);
3227 3228
3228 3229 VM_STAT_ADD(page_rename_count);
3229 3230
3230 3231 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME,
3231 3232 "page rename:pp %p vp %p off %llx", opp, vp, off);
3232 3233
3233 3234 /*
3234 3235 * CacheFS may call page_rename for a large NFS page
3235 3236 * when both CacheFS and NFS mount points are used
3236 3237 * by applications. Demote this large page before
3237 3238 * renaming it, to ensure that there are no "partial"
3238 3239 * large pages left lying around.
3239 3240 */
3240 3241 if (opp->p_szc != 0) {
3241 3242 vnode_t *ovp = opp->p_vnode;
3242 3243 ASSERT(ovp != NULL);
3243 3244 ASSERT(!IS_SWAPFSVP(ovp));
3244 3245 ASSERT(!VN_ISKAS(ovp));
3245 3246 page_demote_vp_pages(opp);
3246 3247 ASSERT(opp->p_szc == 0);
3247 3248 }
3248 3249
3249 3250 page_hashout(opp, NULL);
3250 3251 PP_CLRAGED(opp);
3251 3252
3252 3253 /*
3253 3254 * Acquire the appropriate page hash lock, since
3254 3255 * we're going to rename the page.
3255 3256 */
3256 3257 index = PAGE_HASH_FUNC(vp, off);
3257 3258 phm = PAGE_HASH_MUTEX(index);
3258 3259 mutex_enter(phm);
3259 3260 top:
3260 3261 /*
3261 3262 * Look for an existing page with this name and destroy it if found.
3262 3263 * By holding the page hash lock all the way to the page_hashin()
3263 3264 * call, we are assured that no page can be created with this
3264 3265 * identity. In the case when the phm lock is dropped to undo any
3265 3266 * hat layer mappings, the existing page is held with an "exclusive"
3266 3267 * lock, again preventing another page from being created with
3267 3268 * this identity.
3268 3269 */
3269 3270 pp = page_hash_search(index, vp, off);
3270 3271 if (pp != NULL) {
3271 3272 VM_STAT_ADD(page_rename_exists);
3272 3273
3273 3274 /*
3274 3275 * As it turns out, this is one of only two places where
3275 3276 * page_lock() needs to hold the passed in lock in the
3276 3277 * successful case. In all of the others, the lock could
3277 3278 * be dropped as soon as the attempt is made to lock
3278 3279 * the page. It is tempting to add yet another arguement,
3279 3280 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3280 3281 */
3281 3282 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3282 3283 /*
3283 3284 * Went to sleep because the page could not
3284 3285 * be locked. We were woken up when the page
3285 3286 * was unlocked, or when the page was destroyed.
3286 3287 * In either case, `phm' was dropped while we
3287 3288 * slept. Hence we should not just roar through
3288 3289 * this loop.
3289 3290 */
3290 3291 goto top;
3291 3292 }
3292 3293
3293 3294 /*
3294 3295 * If an existing page is a large page, then demote
3295 3296 * it to ensure that no "partial" large pages are
3296 3297 * "created" after page_rename. An existing page
3297 3298 * can be a CacheFS page, and can't belong to swapfs.
3298 3299 */
3299 3300 if (hat_page_is_mapped(pp)) {
3300 3301 /*
3301 3302 * Unload translations. Since we hold the
3302 3303 * exclusive lock on this page, the page
3303 3304 * can not be changed while we drop phm.
3304 3305 * This is also not a lock protocol violation,
3305 3306 * but rather the proper way to do things.
3306 3307 */
3307 3308 mutex_exit(phm);
3308 3309 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3309 3310 if (pp->p_szc != 0) {
3310 3311 ASSERT(!IS_SWAPFSVP(vp));
3311 3312 ASSERT(!VN_ISKAS(vp));
3312 3313 page_demote_vp_pages(pp);
3313 3314 ASSERT(pp->p_szc == 0);
3314 3315 }
3315 3316 mutex_enter(phm);
3316 3317 } else if (pp->p_szc != 0) {
3317 3318 ASSERT(!IS_SWAPFSVP(vp));
3318 3319 ASSERT(!VN_ISKAS(vp));
3319 3320 mutex_exit(phm);
3320 3321 page_demote_vp_pages(pp);
3321 3322 ASSERT(pp->p_szc == 0);
3322 3323 mutex_enter(phm);
3323 3324 }
3324 3325 page_hashout(pp, phm);
3325 3326 }
3326 3327 /*
3327 3328 * Hash in the page with the new identity.
3328 3329 */
3329 3330 if (!page_hashin(opp, vp, off, phm)) {
3330 3331 /*
3331 3332 * We were holding phm while we searched for [vp, off]
3332 3333 * and only dropped phm if we found and locked a page.
3333 3334 * If we can't create this page now, then some thing
3334 3335 * is really broken.
3335 3336 */
3336 3337 panic("page_rename: Can't hash in page: %p", (void *)pp);
3337 3338 /*NOTREACHED*/
3338 3339 }
3339 3340
3340 3341 ASSERT(MUTEX_HELD(phm));
3341 3342 mutex_exit(phm);
3342 3343
3343 3344 /*
3344 3345 * Now that we have dropped phm, lets get around to finishing up
3345 3346 * with pp.
3346 3347 */
3347 3348 if (pp != NULL) {
3348 3349 ASSERT(!hat_page_is_mapped(pp));
3349 3350 /* for now large pages should not end up here */
3350 3351 ASSERT(pp->p_szc == 0);
3351 3352 /*
3352 3353 * Save the locks for transfer to the new page and then
3353 3354 * clear them so page_free doesn't think they're important.
3354 3355 * The page_struct_lock need not be acquired for lckcnt and
3355 3356 * cowcnt since the page has an "exclusive" lock.
3356 3357 */
3357 3358 olckcnt = pp->p_lckcnt;
3358 3359 ocowcnt = pp->p_cowcnt;
3359 3360 pp->p_lckcnt = pp->p_cowcnt = 0;
3360 3361
3361 3362 /*
3362 3363 * Put the page on the "free" list after we drop
3363 3364 * the lock. The less work under the lock the better.
3364 3365 */
3365 3366 /*LINTED: constant in conditional context*/
3366 3367 VN_DISPOSE(pp, B_FREE, 0, kcred);
3367 3368 }
3368 3369
3369 3370 /*
3370 3371 * Transfer the lock count from the old page (if any).
3371 3372 * The page_struct_lock need not be acquired for lckcnt and
3372 3373 * cowcnt since the page has an "exclusive" lock.
3373 3374 */
3374 3375 opp->p_lckcnt += olckcnt;
3375 3376 opp->p_cowcnt += ocowcnt;
3376 3377 }
3377 3378
3378 3379 /*
3379 3380 * low level routine to add page `pp' to the hash and vp chains for [vp, offset]
3380 3381 *
3381 3382 * Pages are normally inserted at the start of a vnode's v_pages list.
3382 3383 * If the vnode is VMODSORT and the page is modified, it goes at the end.
3383 3384 * This can happen when a modified page is relocated for DR.
3384 3385 *
3385 3386 * Returns 1 on success and 0 on failure.
3386 3387 */
3387 3388 static int
3388 3389 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset)
3389 3390 {
3390 3391 page_t **listp;
3391 3392 page_t *tp;
3392 3393 ulong_t index;
3393 3394
3394 3395 ASSERT(PAGE_EXCL(pp));
3395 3396 ASSERT(vp != NULL);
3396 3397 ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3397 3398
3398 3399 /*
3399 3400 * Be sure to set these up before the page is inserted on the hash
3400 3401 * list. As soon as the page is placed on the list some other
3401 3402 * thread might get confused and wonder how this page could
3402 3403 * possibly hash to this list.
3403 3404 */
3404 3405 pp->p_vnode = vp;
3405 3406 pp->p_offset = offset;
3406 3407
3407 3408 /*
3408 3409 * record if this page is on a swap vnode
3409 3410 */
3410 3411 if ((vp->v_flag & VISSWAP) != 0)
3411 3412 PP_SETSWAP(pp);
3412 3413
3413 3414 index = PAGE_HASH_FUNC(vp, offset);
3414 3415 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index)));
3415 3416 listp = &page_hash[index];
3416 3417
3417 3418 /*
3418 3419 * If this page is already hashed in, fail this attempt to add it.
3419 3420 */
3420 3421 for (tp = *listp; tp != NULL; tp = tp->p_hash) {
3421 3422 if (tp->p_vnode == vp && tp->p_offset == offset) {
3422 3423 pp->p_vnode = NULL;
3423 3424 pp->p_offset = (u_offset_t)(-1);
3424 3425 return (0);
3425 3426 }
3426 3427 }
3427 3428 pp->p_hash = *listp;
3428 3429 *listp = pp;
3429 3430
3430 3431 /*
3431 3432 * Add the page to the vnode's list of pages
3432 3433 */
3433 3434 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp))
3434 3435 listp = &vp->v_pages->p_vpprev->p_vpnext;
3435 3436 else
3436 3437 listp = &vp->v_pages;
3437 3438
3438 3439 page_vpadd(listp, pp);
3439 3440
3440 3441 return (1);
3441 3442 }
3442 3443
3443 3444 /*
3444 3445 * Add page `pp' to both the hash and vp chains for [vp, offset].
3445 3446 *
3446 3447 * Returns 1 on success and 0 on failure.
3447 3448 * If hold is passed in, it is not dropped.
3448 3449 */
3449 3450 int
3450 3451 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold)
3451 3452 {
3452 3453 kmutex_t *phm = NULL;
3453 3454 kmutex_t *vphm;
3454 3455 int rc;
3455 3456
3456 3457 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3457 3458 ASSERT(pp->p_fsdata == 0 || panicstr);
3458 3459
3459 3460 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN,
3460 3461 "page_hashin:pp %p vp %p offset %llx",
3461 3462 pp, vp, offset);
3462 3463
3463 3464 VM_STAT_ADD(hashin_count);
3464 3465
3465 3466 if (hold != NULL)
3466 3467 phm = hold;
3467 3468 else {
3468 3469 VM_STAT_ADD(hashin_not_held);
3469 3470 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset));
3470 3471 mutex_enter(phm);
3471 3472 }
3472 3473
3473 3474 vphm = page_vnode_mutex(vp);
3474 3475 mutex_enter(vphm);
3475 3476 rc = page_do_hashin(pp, vp, offset);
3476 3477 mutex_exit(vphm);
3477 3478 if (hold == NULL)
3478 3479 mutex_exit(phm);
3479 3480 if (rc == 0)
3480 3481 VM_STAT_ADD(hashin_already);
3481 3482 return (rc);
3482 3483 }
3483 3484
3484 3485 /*
3485 3486 * Remove page ``pp'' from the hash and vp chains and remove vp association.
3486 3487 * All mutexes must be held
3487 3488 */
3488 3489 static void
3489 3490 page_do_hashout(page_t *pp)
3490 3491 {
3491 3492 page_t **hpp;
3492 3493 page_t *hp;
3493 3494 vnode_t *vp = pp->p_vnode;
3494 3495
3495 3496 ASSERT(vp != NULL);
3496 3497 ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3497 3498
3498 3499 /*
3499 3500 * First, take pp off of its hash chain.
3500 3501 */
3501 3502 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)];
3502 3503
3503 3504 for (;;) {
3504 3505 hp = *hpp;
3505 3506 if (hp == pp)
3506 3507 break;
3507 3508 if (hp == NULL) {
3508 3509 panic("page_do_hashout");
3509 3510 /*NOTREACHED*/
3510 3511 }
3511 3512 hpp = &hp->p_hash;
3512 3513 }
3513 3514 *hpp = pp->p_hash;
3514 3515
3515 3516 /*
3516 3517 * Now remove it from its associated vnode.
3517 3518 */
3518 3519 if (vp->v_pages)
3519 3520 page_vpsub(&vp->v_pages, pp);
3520 3521
3521 3522 pp->p_hash = NULL;
3522 3523 page_clr_all_props(pp);
3523 3524 PP_CLRSWAP(pp);
3524 3525 pp->p_vnode = NULL;
3525 3526 pp->p_offset = (u_offset_t)-1;
3526 3527 pp->p_fsdata = 0;
3527 3528 }
3528 3529
3529 3530 /*
3530 3531 * Remove page ``pp'' from the hash and vp chains and remove vp association.
3531 3532 *
3532 3533 * When `phm' is non-NULL it contains the address of the mutex protecting the
3533 3534 * hash list pp is on. It is not dropped.
3534 3535 */
3535 3536 void
3536 3537 page_hashout(page_t *pp, kmutex_t *phm)
3537 3538 {
3538 3539 vnode_t *vp;
3539 3540 ulong_t index;
3540 3541 kmutex_t *nphm;
3541 3542 kmutex_t *vphm;
3542 3543 kmutex_t *sep;
3543 3544
3544 3545 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1);
3545 3546 ASSERT(pp->p_vnode != NULL);
3546 3547 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr);
3547 3548 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode)));
3548 3549
3549 3550 vp = pp->p_vnode;
3550 3551
3551 3552 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT,
3552 3553 "page_hashout:pp %p vp %p", pp, vp);
3553 3554
3554 3555 /* Kernel probe */
3555 3556 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */,
3556 3557 tnf_opaque, vnode, vp,
3557 3558 tnf_offset, offset, pp->p_offset);
3558 3559
3559 3560 /*
3560 3561 *
3561 3562 */
3562 3563 VM_STAT_ADD(hashout_count);
3563 3564 index = PAGE_HASH_FUNC(vp, pp->p_offset);
3564 3565 if (phm == NULL) {
3565 3566 VM_STAT_ADD(hashout_not_held);
3566 3567 nphm = PAGE_HASH_MUTEX(index);
3567 3568 mutex_enter(nphm);
3568 3569 }
3569 3570 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1);
3570 3571
3571 3572
3572 3573 /*
3573 3574 * grab page vnode mutex and remove it...
3574 3575 */
3575 3576 vphm = page_vnode_mutex(vp);
3576 3577 mutex_enter(vphm);
3577 3578
3578 3579 page_do_hashout(pp);
3579 3580
3580 3581 mutex_exit(vphm);
3581 3582 if (phm == NULL)
3582 3583 mutex_exit(nphm);
3583 3584
3584 3585 /*
3585 3586 * Wake up processes waiting for this page. The page's
3586 3587 * identity has been changed, and is probably not the
3587 3588 * desired page any longer.
3588 3589 */
3589 3590 sep = page_se_mutex(pp);
3590 3591 mutex_enter(sep);
3591 3592 pp->p_selock &= ~SE_EWANTED;
3592 3593 if (CV_HAS_WAITERS(&pp->p_cv))
3593 3594 cv_broadcast(&pp->p_cv);
3594 3595 mutex_exit(sep);
3595 3596 }
3596 3597
3597 3598 /*
3598 3599 * Add the page to the front of a linked list of pages
3599 3600 * using the p_next & p_prev pointers for the list.
3600 3601 * The caller is responsible for protecting the list pointers.
3601 3602 */
3602 3603 void
3603 3604 page_add(page_t **ppp, page_t *pp)
3604 3605 {
3605 3606 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3606 3607
3607 3608 page_add_common(ppp, pp);
3608 3609 }
3609 3610
3610 3611
3611 3612
3612 3613 /*
3613 3614 * Common code for page_add() and mach_page_add()
3614 3615 */
3615 3616 void
3616 3617 page_add_common(page_t **ppp, page_t *pp)
3617 3618 {
3618 3619 if (*ppp == NULL) {
3619 3620 pp->p_next = pp->p_prev = pp;
3620 3621 } else {
3621 3622 pp->p_next = *ppp;
3622 3623 pp->p_prev = (*ppp)->p_prev;
3623 3624 (*ppp)->p_prev = pp;
3624 3625 pp->p_prev->p_next = pp;
3625 3626 }
3626 3627 *ppp = pp;
3627 3628 }
3628 3629
3629 3630
3630 3631 /*
3631 3632 * Remove this page from a linked list of pages
3632 3633 * using the p_next & p_prev pointers for the list.
3633 3634 *
3634 3635 * The caller is responsible for protecting the list pointers.
3635 3636 */
3636 3637 void
3637 3638 page_sub(page_t **ppp, page_t *pp)
3638 3639 {
3639 3640 ASSERT((PP_ISFREE(pp)) ? 1 :
3640 3641 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3641 3642
3642 3643 if (*ppp == NULL || pp == NULL) {
3643 3644 panic("page_sub: bad arg(s): pp %p, *ppp %p",
3644 3645 (void *)pp, (void *)(*ppp));
3645 3646 /*NOTREACHED*/
3646 3647 }
3647 3648
3648 3649 page_sub_common(ppp, pp);
3649 3650 }
3650 3651
3651 3652
3652 3653 /*
3653 3654 * Common code for page_sub() and mach_page_sub()
3654 3655 */
3655 3656 void
3656 3657 page_sub_common(page_t **ppp, page_t *pp)
3657 3658 {
3658 3659 if (*ppp == pp)
3659 3660 *ppp = pp->p_next; /* go to next page */
3660 3661
3661 3662 if (*ppp == pp)
3662 3663 *ppp = NULL; /* page list is gone */
3663 3664 else {
3664 3665 pp->p_prev->p_next = pp->p_next;
3665 3666 pp->p_next->p_prev = pp->p_prev;
3666 3667 }
3667 3668 pp->p_prev = pp->p_next = pp; /* make pp a list of one */
3668 3669 }
3669 3670
3670 3671
3671 3672 /*
3672 3673 * Break page list cppp into two lists with npages in the first list.
3673 3674 * The tail is returned in nppp.
3674 3675 */
3675 3676 void
3676 3677 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages)
3677 3678 {
3678 3679 page_t *s1pp = *oppp;
3679 3680 page_t *s2pp;
3680 3681 page_t *e1pp, *e2pp;
3681 3682 long n = 0;
3682 3683
3683 3684 if (s1pp == NULL) {
3684 3685 *nppp = NULL;
3685 3686 return;
3686 3687 }
3687 3688 if (npages == 0) {
3688 3689 *nppp = s1pp;
3689 3690 *oppp = NULL;
3690 3691 return;
3691 3692 }
3692 3693 for (n = 0, s2pp = *oppp; n < npages; n++) {
3693 3694 s2pp = s2pp->p_next;
3694 3695 }
3695 3696 /* Fix head and tail of new lists */
3696 3697 e1pp = s2pp->p_prev;
3697 3698 e2pp = s1pp->p_prev;
3698 3699 s1pp->p_prev = e1pp;
3699 3700 e1pp->p_next = s1pp;
3700 3701 s2pp->p_prev = e2pp;
3701 3702 e2pp->p_next = s2pp;
3702 3703
3703 3704 /* second list empty */
3704 3705 if (s2pp == s1pp) {
3705 3706 *oppp = s1pp;
3706 3707 *nppp = NULL;
3707 3708 } else {
3708 3709 *oppp = s1pp;
3709 3710 *nppp = s2pp;
3710 3711 }
3711 3712 }
3712 3713
3713 3714 /*
3714 3715 * Concatenate page list nppp onto the end of list ppp.
3715 3716 */
3716 3717 void
3717 3718 page_list_concat(page_t **ppp, page_t **nppp)
3718 3719 {
3719 3720 page_t *s1pp, *s2pp, *e1pp, *e2pp;
3720 3721
3721 3722 if (*nppp == NULL) {
3722 3723 return;
3723 3724 }
3724 3725 if (*ppp == NULL) {
3725 3726 *ppp = *nppp;
3726 3727 return;
3727 3728 }
3728 3729 s1pp = *ppp;
3729 3730 e1pp = s1pp->p_prev;
3730 3731 s2pp = *nppp;
3731 3732 e2pp = s2pp->p_prev;
3732 3733 s1pp->p_prev = e2pp;
3733 3734 e2pp->p_next = s1pp;
3734 3735 e1pp->p_next = s2pp;
3735 3736 s2pp->p_prev = e1pp;
3736 3737 }
3737 3738
3738 3739 /*
3739 3740 * return the next page in the page list
3740 3741 */
3741 3742 page_t *
3742 3743 page_list_next(page_t *pp)
3743 3744 {
3744 3745 return (pp->p_next);
3745 3746 }
3746 3747
3747 3748
3748 3749 /*
3749 3750 * Add the page to the front of the linked list of pages
3750 3751 * using p_vpnext/p_vpprev pointers for the list.
3751 3752 *
3752 3753 * The caller is responsible for protecting the lists.
3753 3754 */
3754 3755 void
3755 3756 page_vpadd(page_t **ppp, page_t *pp)
3756 3757 {
3757 3758 if (*ppp == NULL) {
3758 3759 pp->p_vpnext = pp->p_vpprev = pp;
3759 3760 } else {
3760 3761 pp->p_vpnext = *ppp;
3761 3762 pp->p_vpprev = (*ppp)->p_vpprev;
3762 3763 (*ppp)->p_vpprev = pp;
3763 3764 pp->p_vpprev->p_vpnext = pp;
3764 3765 }
3765 3766 *ppp = pp;
3766 3767 }
3767 3768
3768 3769 /*
3769 3770 * Remove this page from the linked list of pages
3770 3771 * using p_vpnext/p_vpprev pointers for the list.
3771 3772 *
3772 3773 * The caller is responsible for protecting the lists.
3773 3774 */
3774 3775 void
3775 3776 page_vpsub(page_t **ppp, page_t *pp)
3776 3777 {
3777 3778 if (*ppp == NULL || pp == NULL) {
3778 3779 panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
3779 3780 (void *)pp, (void *)(*ppp));
3780 3781 /*NOTREACHED*/
3781 3782 }
3782 3783
3783 3784 if (*ppp == pp)
3784 3785 *ppp = pp->p_vpnext; /* go to next page */
3785 3786
3786 3787 if (*ppp == pp)
3787 3788 *ppp = NULL; /* page list is gone */
3788 3789 else {
3789 3790 pp->p_vpprev->p_vpnext = pp->p_vpnext;
3790 3791 pp->p_vpnext->p_vpprev = pp->p_vpprev;
3791 3792 }
3792 3793 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */
3793 3794 }
3794 3795
3795 3796 /*
3796 3797 * Lock a physical page into memory "long term". Used to support "lock
3797 3798 * in memory" functions. Accepts the page to be locked, and a cow variable
3798 3799 * to indicate whether a the lock will travel to the new page during
3799 3800 * a potential copy-on-write.
3800 3801 */
3801 3802 int
3802 3803 page_pp_lock(
3803 3804 page_t *pp, /* page to be locked */
3804 3805 int cow, /* cow lock */
3805 3806 int kernel) /* must succeed -- ignore checking */
3806 3807 {
3807 3808 int r = 0; /* result -- assume failure */
3808 3809
3809 3810 ASSERT(PAGE_LOCKED(pp));
3810 3811
3811 3812 page_struct_lock(pp);
3812 3813 /*
3813 3814 * Acquire the "freemem_lock" for availrmem.
3814 3815 */
3815 3816 if (cow) {
3816 3817 mutex_enter(&freemem_lock);
3817 3818 if ((availrmem > pages_pp_maximum) &&
3818 3819 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
3819 3820 availrmem--;
3820 3821 pages_locked++;
3821 3822 mutex_exit(&freemem_lock);
3822 3823 r = 1;
3823 3824 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
3824 3825 cmn_err(CE_WARN,
3825 3826 "COW lock limit reached on pfn 0x%lx",
3826 3827 page_pptonum(pp));
3827 3828 }
3828 3829 } else
3829 3830 mutex_exit(&freemem_lock);
3830 3831 } else {
3831 3832 if (pp->p_lckcnt) {
3832 3833 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
3833 3834 r = 1;
3834 3835 if (++pp->p_lckcnt ==
3835 3836 (ushort_t)PAGE_LOCK_MAXIMUM) {
3836 3837 cmn_err(CE_WARN, "Page lock limit "
3837 3838 "reached on pfn 0x%lx",
3838 3839 page_pptonum(pp));
3839 3840 }
3840 3841 }
3841 3842 } else {
3842 3843 if (kernel) {
3843 3844 /* availrmem accounting done by caller */
3844 3845 ++pp->p_lckcnt;
3845 3846 r = 1;
3846 3847 } else {
3847 3848 mutex_enter(&freemem_lock);
3848 3849 if (availrmem > pages_pp_maximum) {
3849 3850 availrmem--;
3850 3851 pages_locked++;
3851 3852 ++pp->p_lckcnt;
3852 3853 r = 1;
3853 3854 }
3854 3855 mutex_exit(&freemem_lock);
3855 3856 }
3856 3857 }
3857 3858 }
3858 3859 page_struct_unlock(pp);
3859 3860 return (r);
3860 3861 }
3861 3862
3862 3863 /*
3863 3864 * Decommit a lock on a physical page frame. Account for cow locks if
3864 3865 * appropriate.
3865 3866 */
3866 3867 void
3867 3868 page_pp_unlock(
3868 3869 page_t *pp, /* page to be unlocked */
3869 3870 int cow, /* expect cow lock */
3870 3871 int kernel) /* this was a kernel lock */
3871 3872 {
3872 3873 ASSERT(PAGE_LOCKED(pp));
3873 3874
3874 3875 page_struct_lock(pp);
3875 3876 /*
3876 3877 * Acquire the "freemem_lock" for availrmem.
3877 3878 * If cowcnt or lcknt is already 0 do nothing; i.e., we
3878 3879 * could be called to unlock even if nothing is locked. This could
3879 3880 * happen if locked file pages were truncated (removing the lock)
3880 3881 * and the file was grown again and new pages faulted in; the new
3881 3882 * pages are unlocked but the segment still thinks they're locked.
3882 3883 */
3883 3884 if (cow) {
3884 3885 if (pp->p_cowcnt) {
3885 3886 mutex_enter(&freemem_lock);
3886 3887 pp->p_cowcnt--;
3887 3888 availrmem++;
3888 3889 pages_locked--;
3889 3890 mutex_exit(&freemem_lock);
3890 3891 }
3891 3892 } else {
3892 3893 if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
3893 3894 if (!kernel) {
3894 3895 mutex_enter(&freemem_lock);
3895 3896 availrmem++;
3896 3897 pages_locked--;
3897 3898 mutex_exit(&freemem_lock);
3898 3899 }
3899 3900 }
3900 3901 }
3901 3902 page_struct_unlock(pp);
3902 3903 }
3903 3904
3904 3905 /*
3905 3906 * This routine reserves availrmem for npages;
3906 3907 * flags: KM_NOSLEEP or KM_SLEEP
3907 3908 * returns 1 on success or 0 on failure
3908 3909 */
3909 3910 int
3910 3911 page_resv(pgcnt_t npages, uint_t flags)
3911 3912 {
3912 3913 mutex_enter(&freemem_lock);
3913 3914 while (availrmem < tune.t_minarmem + npages) {
3914 3915 if (flags & KM_NOSLEEP) {
3915 3916 mutex_exit(&freemem_lock);
3916 3917 return (0);
3917 3918 }
3918 3919 mutex_exit(&freemem_lock);
3919 3920 page_needfree(npages);
3920 3921 kmem_reap();
3921 3922 delay(hz >> 2);
3922 3923 page_needfree(-(spgcnt_t)npages);
3923 3924 mutex_enter(&freemem_lock);
3924 3925 }
3925 3926 availrmem -= npages;
3926 3927 mutex_exit(&freemem_lock);
3927 3928 return (1);
3928 3929 }
3929 3930
3930 3931 /*
3931 3932 * This routine unreserves availrmem for npages;
3932 3933 */
3933 3934 void
3934 3935 page_unresv(pgcnt_t npages)
3935 3936 {
3936 3937 mutex_enter(&freemem_lock);
3937 3938 availrmem += npages;
3938 3939 mutex_exit(&freemem_lock);
3939 3940 }
3940 3941
3941 3942 /*
3942 3943 * See Statement at the beginning of segvn_lockop() regarding
3943 3944 * the way we handle cowcnts and lckcnts.
3944 3945 *
3945 3946 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
3946 3947 * that breaks COW has PROT_WRITE.
3947 3948 *
3948 3949 * Note that, we may also break COW in case we are softlocking
3949 3950 * on read access during physio;
3950 3951 * in this softlock case, the vpage may not have PROT_WRITE.
3951 3952 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
3952 3953 * if the vpage doesn't have PROT_WRITE.
3953 3954 *
3954 3955 * This routine is never called if we are stealing a page
3955 3956 * in anon_private.
3956 3957 *
3957 3958 * The caller subtracted from availrmem for read only mapping.
3958 3959 * if lckcnt is 1 increment availrmem.
3959 3960 */
3960 3961 void
3961 3962 page_pp_useclaim(
3962 3963 page_t *opp, /* original page frame losing lock */
3963 3964 page_t *npp, /* new page frame gaining lock */
3964 3965 uint_t write_perm) /* set if vpage has PROT_WRITE */
3965 3966 {
3966 3967 int payback = 0;
3967 3968 int nidx, oidx;
3968 3969
3969 3970 ASSERT(PAGE_LOCKED(opp));
3970 3971 ASSERT(PAGE_LOCKED(npp));
3971 3972
3972 3973 /*
3973 3974 * Since we have two pages we probably have two locks. We need to take
3974 3975 * them in a defined order to avoid deadlocks. It's also possible they
3975 3976 * both hash to the same lock in which case this is a non-issue.
3976 3977 */
3977 3978 nidx = PAGE_LLOCK_HASH(PP_PAGEROOT(npp));
3978 3979 oidx = PAGE_LLOCK_HASH(PP_PAGEROOT(opp));
3979 3980 if (nidx < oidx) {
3980 3981 page_struct_lock(npp);
3981 3982 page_struct_lock(opp);
3982 3983 } else if (oidx < nidx) {
3983 3984 page_struct_lock(opp);
3984 3985 page_struct_lock(npp);
3985 3986 } else { /* The pages hash to the same lock */
3986 3987 page_struct_lock(npp);
3987 3988 }
3988 3989
3989 3990 ASSERT(npp->p_cowcnt == 0);
3990 3991 ASSERT(npp->p_lckcnt == 0);
3991 3992
3992 3993 /* Don't use claim if nothing is locked (see page_pp_unlock above) */
3993 3994 if ((write_perm && opp->p_cowcnt != 0) ||
3994 3995 (!write_perm && opp->p_lckcnt != 0)) {
3995 3996
3996 3997 if (write_perm) {
3997 3998 npp->p_cowcnt++;
3998 3999 ASSERT(opp->p_cowcnt != 0);
3999 4000 opp->p_cowcnt--;
4000 4001 } else {
4001 4002
4002 4003 ASSERT(opp->p_lckcnt != 0);
4003 4004
4004 4005 /*
4005 4006 * We didn't need availrmem decremented if p_lckcnt on
4006 4007 * original page is 1. Here, we are unlocking
4007 4008 * read-only copy belonging to original page and
4008 4009 * are locking a copy belonging to new page.
4009 4010 */
4010 4011 if (opp->p_lckcnt == 1)
4011 4012 payback = 1;
4012 4013
4013 4014 npp->p_lckcnt++;
4014 4015 opp->p_lckcnt--;
4015 4016 }
4016 4017 }
4017 4018 if (payback) {
4018 4019 mutex_enter(&freemem_lock);
4019 4020 availrmem++;
4020 4021 pages_useclaim--;
4021 4022 mutex_exit(&freemem_lock);
4022 4023 }
4023 4024
4024 4025 if (nidx < oidx) {
4025 4026 page_struct_unlock(opp);
4026 4027 page_struct_unlock(npp);
4027 4028 } else if (oidx < nidx) {
4028 4029 page_struct_unlock(npp);
4029 4030 page_struct_unlock(opp);
4030 4031 } else { /* The pages hash to the same lock */
4031 4032 page_struct_unlock(npp);
4032 4033 }
4033 4034 }
4034 4035
4035 4036 /*
4036 4037 * Simple claim adjust functions -- used to support changes in
4037 4038 * claims due to changes in access permissions. Used by segvn_setprot().
4038 4039 */
4039 4040 int
4040 4041 page_addclaim(page_t *pp)
4041 4042 {
4042 4043 int r = 0; /* result */
4043 4044
4044 4045 ASSERT(PAGE_LOCKED(pp));
4045 4046
4046 4047 page_struct_lock(pp);
4047 4048 ASSERT(pp->p_lckcnt != 0);
4048 4049
4049 4050 if (pp->p_lckcnt == 1) {
4050 4051 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4051 4052 --pp->p_lckcnt;
4052 4053 r = 1;
4053 4054 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4054 4055 cmn_err(CE_WARN,
4055 4056 "COW lock limit reached on pfn 0x%lx",
4056 4057 page_pptonum(pp));
4057 4058 }
4058 4059 }
4059 4060 } else {
4060 4061 mutex_enter(&freemem_lock);
4061 4062 if ((availrmem > pages_pp_maximum) &&
4062 4063 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
4063 4064 --availrmem;
4064 4065 ++pages_claimed;
4065 4066 mutex_exit(&freemem_lock);
4066 4067 --pp->p_lckcnt;
4067 4068 r = 1;
4068 4069 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4069 4070 cmn_err(CE_WARN,
4070 4071 "COW lock limit reached on pfn 0x%lx",
4071 4072 page_pptonum(pp));
4072 4073 }
4073 4074 } else
4074 4075 mutex_exit(&freemem_lock);
4075 4076 }
4076 4077 page_struct_unlock(pp);
4077 4078 return (r);
4078 4079 }
4079 4080
4080 4081 int
4081 4082 page_subclaim(page_t *pp)
4082 4083 {
4083 4084 int r = 0;
4084 4085
4085 4086 ASSERT(PAGE_LOCKED(pp));
4086 4087
4087 4088 page_struct_lock(pp);
4088 4089 ASSERT(pp->p_cowcnt != 0);
4089 4090
4090 4091 if (pp->p_lckcnt) {
4091 4092 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4092 4093 r = 1;
4093 4094 /*
4094 4095 * for availrmem
4095 4096 */
4096 4097 mutex_enter(&freemem_lock);
4097 4098 availrmem++;
4098 4099 pages_claimed--;
4099 4100 mutex_exit(&freemem_lock);
4100 4101
4101 4102 pp->p_cowcnt--;
4102 4103
4103 4104 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4104 4105 cmn_err(CE_WARN,
4105 4106 "Page lock limit reached on pfn 0x%lx",
4106 4107 page_pptonum(pp));
4107 4108 }
4108 4109 }
4109 4110 } else {
4110 4111 r = 1;
4111 4112 pp->p_cowcnt--;
4112 4113 pp->p_lckcnt++;
4113 4114 }
4114 4115 page_struct_unlock(pp);
4115 4116 return (r);
4116 4117 }
4117 4118
4118 4119 /*
4119 4120 * Variant of page_addclaim(), where ppa[] contains the pages of a single large
4120 4121 * page.
4121 4122 */
4122 4123 int
4123 4124 page_addclaim_pages(page_t **ppa)
4124 4125 {
4125 4126 pgcnt_t lckpgs = 0, pg_idx;
4126 4127
4127 4128 VM_STAT_ADD(pagecnt.pc_addclaim_pages);
4128 4129
4129 4130 /*
4130 4131 * Only need to take the page struct lock on the large page root.
4131 4132 */
4132 4133 page_struct_lock(ppa[0]);
4133 4134 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4134 4135
4135 4136 ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4136 4137 ASSERT(ppa[pg_idx]->p_lckcnt != 0);
4137 4138 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4138 4139 page_struct_unlock(ppa[0]);
4139 4140 return (0);
4140 4141 }
4141 4142 if (ppa[pg_idx]->p_lckcnt > 1)
4142 4143 lckpgs++;
4143 4144 }
4144 4145
4145 4146 if (lckpgs != 0) {
4146 4147 mutex_enter(&freemem_lock);
4147 4148 if (availrmem >= pages_pp_maximum + lckpgs) {
4148 4149 availrmem -= lckpgs;
4149 4150 pages_claimed += lckpgs;
4150 4151 } else {
4151 4152 mutex_exit(&freemem_lock);
4152 4153 page_struct_unlock(ppa[0]);
4153 4154 return (0);
4154 4155 }
4155 4156 mutex_exit(&freemem_lock);
4156 4157 }
4157 4158
4158 4159 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4159 4160 ppa[pg_idx]->p_lckcnt--;
4160 4161 ppa[pg_idx]->p_cowcnt++;
4161 4162 }
4162 4163 page_struct_unlock(ppa[0]);
4163 4164 return (1);
4164 4165 }
4165 4166
4166 4167 /*
4167 4168 * Variant of page_subclaim(), where ppa[] contains the pages of a single large
4168 4169 * page.
4169 4170 */
4170 4171 int
4171 4172 page_subclaim_pages(page_t **ppa)
4172 4173 {
4173 4174 pgcnt_t ulckpgs = 0, pg_idx;
4174 4175
4175 4176 VM_STAT_ADD(pagecnt.pc_subclaim_pages);
4176 4177
4177 4178 /*
4178 4179 * Only need to take the page struct lock on the large page root.
4179 4180 */
4180 4181 page_struct_lock(ppa[0]);
4181 4182 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4182 4183
4183 4184 ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4184 4185 ASSERT(ppa[pg_idx]->p_cowcnt != 0);
4185 4186 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4186 4187 page_struct_unlock(ppa[0]);
4187 4188 return (0);
4188 4189 }
4189 4190 if (ppa[pg_idx]->p_lckcnt != 0)
4190 4191 ulckpgs++;
4191 4192 }
4192 4193
4193 4194 if (ulckpgs != 0) {
4194 4195 mutex_enter(&freemem_lock);
4195 4196 availrmem += ulckpgs;
4196 4197 pages_claimed -= ulckpgs;
4197 4198 mutex_exit(&freemem_lock);
4198 4199 }
4199 4200
4200 4201 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4201 4202 ppa[pg_idx]->p_cowcnt--;
4202 4203 ppa[pg_idx]->p_lckcnt++;
4203 4204
4204 4205 }
4205 4206 page_struct_unlock(ppa[0]);
4206 4207 return (1);
4207 4208 }
4208 4209
4209 4210 page_t *
4210 4211 page_numtopp(pfn_t pfnum, se_t se)
4211 4212 {
4212 4213 page_t *pp;
4213 4214
4214 4215 retry:
4215 4216 pp = page_numtopp_nolock(pfnum);
4216 4217 if (pp == NULL) {
4217 4218 return ((page_t *)NULL);
4218 4219 }
4219 4220
4220 4221 /*
4221 4222 * Acquire the appropriate lock on the page.
4222 4223 */
4223 4224 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) {
4224 4225 if (page_pptonum(pp) != pfnum)
4225 4226 goto retry;
4226 4227 continue;
4227 4228 }
4228 4229
4229 4230 if (page_pptonum(pp) != pfnum) {
4230 4231 page_unlock(pp);
4231 4232 goto retry;
4232 4233 }
4233 4234
4234 4235 return (pp);
4235 4236 }
4236 4237
4237 4238 page_t *
4238 4239 page_numtopp_noreclaim(pfn_t pfnum, se_t se)
4239 4240 {
4240 4241 page_t *pp;
4241 4242
4242 4243 retry:
4243 4244 pp = page_numtopp_nolock(pfnum);
4244 4245 if (pp == NULL) {
4245 4246 return ((page_t *)NULL);
4246 4247 }
4247 4248
4248 4249 /*
4249 4250 * Acquire the appropriate lock on the page.
4250 4251 */
4251 4252 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) {
4252 4253 if (page_pptonum(pp) != pfnum)
4253 4254 goto retry;
4254 4255 continue;
4255 4256 }
4256 4257
4257 4258 if (page_pptonum(pp) != pfnum) {
4258 4259 page_unlock(pp);
4259 4260 goto retry;
4260 4261 }
4261 4262
4262 4263 return (pp);
4263 4264 }
4264 4265
4265 4266 /*
4266 4267 * This routine is like page_numtopp, but will only return page structs
4267 4268 * for pages which are ok for loading into hardware using the page struct.
4268 4269 */
4269 4270 page_t *
4270 4271 page_numtopp_nowait(pfn_t pfnum, se_t se)
4271 4272 {
4272 4273 page_t *pp;
4273 4274
4274 4275 retry:
4275 4276 pp = page_numtopp_nolock(pfnum);
4276 4277 if (pp == NULL) {
4277 4278 return ((page_t *)NULL);
4278 4279 }
4279 4280
4280 4281 /*
4281 4282 * Try to acquire the appropriate lock on the page.
4282 4283 */
4283 4284 if (PP_ISFREE(pp))
4284 4285 pp = NULL;
4285 4286 else {
4286 4287 if (!page_trylock(pp, se))
4287 4288 pp = NULL;
4288 4289 else {
4289 4290 if (page_pptonum(pp) != pfnum) {
4290 4291 page_unlock(pp);
4291 4292 goto retry;
4292 4293 }
4293 4294 if (PP_ISFREE(pp)) {
4294 4295 page_unlock(pp);
4295 4296 pp = NULL;
4296 4297 }
4297 4298 }
4298 4299 }
4299 4300 return (pp);
4300 4301 }
4301 4302
4302 4303 /*
4303 4304 * Returns a count of dirty pages that are in the process
4304 4305 * of being written out. If 'cleanit' is set, try to push the page.
4305 4306 */
4306 4307 pgcnt_t
4307 4308 page_busy(int cleanit)
4308 4309 {
4309 4310 page_t *page0 = page_first();
4310 4311 page_t *pp = page0;
4311 4312 pgcnt_t nppbusy = 0;
4312 4313 u_offset_t off;
4313 4314
4314 4315 do {
4315 4316 vnode_t *vp = pp->p_vnode;
4316 4317 /*
4317 4318 * A page is a candidate for syncing if it is:
4318 4319 *
4319 4320 * (a) On neither the freelist nor the cachelist
4320 4321 * (b) Hashed onto a vnode
4321 4322 * (c) Not a kernel page
4322 4323 * (d) Dirty
4323 4324 * (e) Not part of a swapfile
4324 4325 * (f) a page which belongs to a real vnode; eg has a non-null
4325 4326 * v_vfsp pointer.
4326 4327 * (g) Backed by a filesystem which doesn't have a
4327 4328 * stubbed-out sync operation
4328 4329 */
4329 4330 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) &&
4330 4331 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL &&
4331 4332 vfs_can_sync(vp->v_vfsp)) {
4332 4333 nppbusy++;
4333 4334
4334 4335 if (!cleanit)
4335 4336 continue;
4336 4337 if (!page_trylock(pp, SE_EXCL))
4337 4338 continue;
4338 4339
4339 4340 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) ||
4340 4341 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
4341 4342 !(hat_pagesync(pp,
4342 4343 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) {
4343 4344 page_unlock(pp);
4344 4345 continue;
4345 4346 }
4346 4347 off = pp->p_offset;
4347 4348 VN_HOLD(vp);
4348 4349 page_unlock(pp);
4349 4350 (void) VOP_PUTPAGE(vp, off, PAGESIZE,
4350 4351 B_ASYNC | B_FREE, kcred, NULL);
4351 4352 VN_RELE(vp);
4352 4353 }
4353 4354 } while ((pp = page_next(pp)) != page0);
4354 4355
4355 4356 return (nppbusy);
4356 4357 }
4357 4358
4358 4359 void page_invalidate_pages(void);
4359 4360
4360 4361 /*
4361 4362 * callback handler to vm sub-system
4362 4363 *
4363 4364 * callers make sure no recursive entries to this func.
4364 4365 */
4365 4366 /*ARGSUSED*/
4366 4367 boolean_t
4367 4368 callb_vm_cpr(void *arg, int code)
4368 4369 {
4369 4370 if (code == CB_CODE_CPR_CHKPT)
4370 4371 page_invalidate_pages();
4371 4372 return (B_TRUE);
4372 4373 }
4373 4374
4374 4375 /*
4375 4376 * Invalidate all pages of the system.
4376 4377 * It shouldn't be called until all user page activities are all stopped.
4377 4378 */
4378 4379 void
4379 4380 page_invalidate_pages()
4380 4381 {
4381 4382 page_t *pp;
4382 4383 page_t *page0;
4383 4384 pgcnt_t nbusypages;
4384 4385 int retry = 0;
4385 4386 const int MAXRETRIES = 4;
4386 4387 top:
4387 4388 /*
4388 4389 * Flush dirty pages and destroy the clean ones.
4389 4390 */
4390 4391 nbusypages = 0;
4391 4392
4392 4393 pp = page0 = page_first();
4393 4394 do {
4394 4395 struct vnode *vp;
4395 4396 u_offset_t offset;
4396 4397 int mod;
4397 4398
4398 4399 /*
4399 4400 * skip the page if it has no vnode or the page associated
4400 4401 * with the kernel vnode or prom allocated kernel mem.
4401 4402 */
4402 4403 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp))
4403 4404 continue;
4404 4405
4405 4406 /*
4406 4407 * skip the page which is already free invalidated.
4407 4408 */
4408 4409 if (PP_ISFREE(pp) && PP_ISAGED(pp))
4409 4410 continue;
4410 4411
4411 4412 /*
4412 4413 * skip pages that are already locked or can't be "exclusively"
4413 4414 * locked or are already free. After we lock the page, check
4414 4415 * the free and age bits again to be sure it's not destroyed
4415 4416 * yet.
4416 4417 * To achieve max. parallelization, we use page_trylock instead
4417 4418 * of page_lock so that we don't get block on individual pages
4418 4419 * while we have thousands of other pages to process.
4419 4420 */
4420 4421 if (!page_trylock(pp, SE_EXCL)) {
4421 4422 nbusypages++;
4422 4423 continue;
4423 4424 } else if (PP_ISFREE(pp)) {
4424 4425 if (!PP_ISAGED(pp)) {
4425 4426 page_destroy_free(pp);
4426 4427 } else {
4427 4428 page_unlock(pp);
4428 4429 }
4429 4430 continue;
4430 4431 }
4431 4432 /*
4432 4433 * Is this page involved in some I/O? shared?
4433 4434 *
4434 4435 * The page_struct_lock need not be acquired to
4435 4436 * examine these fields since the page has an
4436 4437 * "exclusive" lock.
4437 4438 */
4438 4439 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
4439 4440 page_unlock(pp);
4440 4441 continue;
4441 4442 }
4442 4443
4443 4444 if (vp->v_type == VCHR) {
4444 4445 panic("vp->v_type == VCHR");
4445 4446 /*NOTREACHED*/
4446 4447 }
4447 4448
4448 4449 if (!page_try_demote_pages(pp)) {
4449 4450 page_unlock(pp);
4450 4451 continue;
4451 4452 }
4452 4453
4453 4454 /*
4454 4455 * Check the modified bit. Leave the bits alone in hardware
4455 4456 * (they will be modified if we do the putpage).
4456 4457 */
4457 4458 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD)
4458 4459 & P_MOD);
4459 4460 if (mod) {
4460 4461 offset = pp->p_offset;
4461 4462 /*
4462 4463 * Hold the vnode before releasing the page lock
4463 4464 * to prevent it from being freed and re-used by
4464 4465 * some other thread.
4465 4466 */
4466 4467 VN_HOLD(vp);
4467 4468 page_unlock(pp);
4468 4469 /*
4469 4470 * No error return is checked here. Callers such as
4470 4471 * cpr deals with the dirty pages at the dump time
4471 4472 * if this putpage fails.
4472 4473 */
4473 4474 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL,
4474 4475 kcred, NULL);
4475 4476 VN_RELE(vp);
4476 4477 } else {
4477 4478 /*LINTED: constant in conditional context*/
4478 4479 VN_DISPOSE(pp, B_INVAL, 0, kcred);
4479 4480 }
4480 4481 } while ((pp = page_next(pp)) != page0);
4481 4482 if (nbusypages && retry++ < MAXRETRIES) {
4482 4483 delay(1);
4483 4484 goto top;
4484 4485 }
4485 4486 }
4486 4487
4487 4488 /*
4488 4489 * Replace the page "old" with the page "new" on the page hash and vnode lists
4489 4490 *
4490 4491 * the replacement must be done in place, ie the equivalent sequence:
4491 4492 *
4492 4493 * vp = old->p_vnode;
4493 4494 * off = old->p_offset;
4494 4495 * page_do_hashout(old)
4495 4496 * page_do_hashin(new, vp, off)
4496 4497 *
4497 4498 * doesn't work, since
4498 4499 * 1) if old is the only page on the vnode, the v_pages list has a window
4499 4500 * where it looks empty. This will break file system assumptions.
4500 4501 * and
4501 4502 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
4502 4503 */
4503 4504 static void
4504 4505 page_do_relocate_hash(page_t *new, page_t *old)
4505 4506 {
4506 4507 page_t **hash_list;
4507 4508 vnode_t *vp = old->p_vnode;
4508 4509 kmutex_t *sep;
4509 4510
4510 4511 ASSERT(PAGE_EXCL(old));
4511 4512 ASSERT(PAGE_EXCL(new));
4512 4513 ASSERT(vp != NULL);
4513 4514 ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
4514 4515 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset))));
4515 4516
4516 4517 /*
4517 4518 * First find old page on the page hash list
4518 4519 */
4519 4520 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)];
4520 4521
4521 4522 for (;;) {
4522 4523 if (*hash_list == old)
4523 4524 break;
4524 4525 if (*hash_list == NULL) {
4525 4526 panic("page_do_hashout");
4526 4527 /*NOTREACHED*/
4527 4528 }
4528 4529 hash_list = &(*hash_list)->p_hash;
4529 4530 }
4530 4531
4531 4532 /*
4532 4533 * update new and replace old with new on the page hash list
4533 4534 */
4534 4535 new->p_vnode = old->p_vnode;
4535 4536 new->p_offset = old->p_offset;
4536 4537 new->p_hash = old->p_hash;
4537 4538 *hash_list = new;
4538 4539
4539 4540 if ((new->p_vnode->v_flag & VISSWAP) != 0)
4540 4541 PP_SETSWAP(new);
4541 4542
4542 4543 /*
4543 4544 * replace old with new on the vnode's page list
4544 4545 */
4545 4546 if (old->p_vpnext == old) {
4546 4547 new->p_vpnext = new;
4547 4548 new->p_vpprev = new;
4548 4549 } else {
4549 4550 new->p_vpnext = old->p_vpnext;
4550 4551 new->p_vpprev = old->p_vpprev;
4551 4552 new->p_vpnext->p_vpprev = new;
4552 4553 new->p_vpprev->p_vpnext = new;
4553 4554 }
4554 4555 if (vp->v_pages == old)
4555 4556 vp->v_pages = new;
4556 4557
4557 4558 /*
4558 4559 * clear out the old page
4559 4560 */
4560 4561 old->p_hash = NULL;
4561 4562 old->p_vpnext = NULL;
4562 4563 old->p_vpprev = NULL;
4563 4564 old->p_vnode = NULL;
4564 4565 PP_CLRSWAP(old);
4565 4566 old->p_offset = (u_offset_t)-1;
4566 4567 page_clr_all_props(old);
4567 4568
4568 4569 /*
4569 4570 * Wake up processes waiting for this page. The page's
4570 4571 * identity has been changed, and is probably not the
4571 4572 * desired page any longer.
4572 4573 */
4573 4574 sep = page_se_mutex(old);
4574 4575 mutex_enter(sep);
4575 4576 old->p_selock &= ~SE_EWANTED;
4576 4577 if (CV_HAS_WAITERS(&old->p_cv))
4577 4578 cv_broadcast(&old->p_cv);
4578 4579 mutex_exit(sep);
4579 4580 }
4580 4581
4581 4582 /*
4582 4583 * This function moves the identity of page "pp_old" to page "pp_new".
4583 4584 * Both pages must be locked on entry. "pp_new" is free, has no identity,
4584 4585 * and need not be hashed out from anywhere.
4585 4586 */
4586 4587 void
4587 4588 page_relocate_hash(page_t *pp_new, page_t *pp_old)
4588 4589 {
4589 4590 vnode_t *vp = pp_old->p_vnode;
4590 4591 u_offset_t off = pp_old->p_offset;
4591 4592 kmutex_t *phm, *vphm;
4592 4593
4593 4594 /*
4594 4595 * Rehash two pages
4595 4596 */
4596 4597 ASSERT(PAGE_EXCL(pp_old));
4597 4598 ASSERT(PAGE_EXCL(pp_new));
4598 4599 ASSERT(vp != NULL);
4599 4600 ASSERT(pp_new->p_vnode == NULL);
4600 4601
4601 4602 /*
4602 4603 * hashout then hashin while holding the mutexes
4603 4604 */
4604 4605 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off));
4605 4606 mutex_enter(phm);
4606 4607 vphm = page_vnode_mutex(vp);
4607 4608 mutex_enter(vphm);
4608 4609
4609 4610 page_do_relocate_hash(pp_new, pp_old);
4610 4611
4611 4612 /* The following comment preserved from page_flip(). */
4612 4613 pp_new->p_fsdata = pp_old->p_fsdata;
4613 4614 pp_old->p_fsdata = 0;
4614 4615 mutex_exit(vphm);
4615 4616 mutex_exit(phm);
4616 4617
4617 4618 /*
4618 4619 * The page_struct_lock need not be acquired for lckcnt and
4619 4620 * cowcnt since the page has an "exclusive" lock.
4620 4621 */
4621 4622 ASSERT(pp_new->p_lckcnt == 0);
4622 4623 ASSERT(pp_new->p_cowcnt == 0);
4623 4624 pp_new->p_lckcnt = pp_old->p_lckcnt;
4624 4625 pp_new->p_cowcnt = pp_old->p_cowcnt;
4625 4626 pp_old->p_lckcnt = pp_old->p_cowcnt = 0;
4626 4627
4627 4628 }
4628 4629
4629 4630 /*
4630 4631 * Helper routine used to lock all remaining members of a
4631 4632 * large page. The caller is responsible for passing in a locked
4632 4633 * pp. If pp is a large page, then it succeeds in locking all the
4633 4634 * remaining constituent pages or it returns with only the
4634 4635 * original page locked.
4635 4636 *
4636 4637 * Returns 1 on success, 0 on failure.
4637 4638 *
4638 4639 * If success is returned this routine guarantees p_szc for all constituent
4639 4640 * pages of a large page pp belongs to can't change. To achieve this we
4640 4641 * recheck szc of pp after locking all constituent pages and retry if szc
4641 4642 * changed (it could only decrease). Since hat_page_demote() needs an EXCL
4642 4643 * lock on one of constituent pages it can't be running after all constituent
4643 4644 * pages are locked. hat_page_demote() with a lock on a constituent page
4644 4645 * outside of this large page (i.e. pp belonged to a larger large page) is
4645 4646 * already done with all constituent pages of pp since the root's p_szc is
4646 4647 * changed last. Therefore no need to synchronize with hat_page_demote() that
4647 4648 * locked a constituent page outside of pp's current large page.
4648 4649 */
4649 4650 #ifdef DEBUG
4650 4651 uint32_t gpg_trylock_mtbf = 0;
4651 4652 #endif
4652 4653
4653 4654 int
4654 4655 group_page_trylock(page_t *pp, se_t se)
4655 4656 {
4656 4657 page_t *tpp;
4657 4658 pgcnt_t npgs, i, j;
4658 4659 uint_t pszc = pp->p_szc;
4659 4660
4660 4661 #ifdef DEBUG
4661 4662 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) {
4662 4663 return (0);
4663 4664 }
4664 4665 #endif
4665 4666
4666 4667 if (pp != PP_GROUPLEADER(pp, pszc)) {
4667 4668 return (0);
4668 4669 }
4669 4670
4670 4671 retry:
4671 4672 ASSERT(PAGE_LOCKED_SE(pp, se));
4672 4673 ASSERT(!PP_ISFREE(pp));
4673 4674 if (pszc == 0) {
4674 4675 return (1);
4675 4676 }
4676 4677 npgs = page_get_pagecnt(pszc);
4677 4678 tpp = pp + 1;
4678 4679 for (i = 1; i < npgs; i++, tpp++) {
4679 4680 if (!page_trylock(tpp, se)) {
4680 4681 tpp = pp + 1;
4681 4682 for (j = 1; j < i; j++, tpp++) {
4682 4683 page_unlock(tpp);
4683 4684 }
4684 4685 return (0);
4685 4686 }
4686 4687 }
4687 4688 if (pp->p_szc != pszc) {
4688 4689 ASSERT(pp->p_szc < pszc);
4689 4690 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) &&
4690 4691 !IS_SWAPFSVP(pp->p_vnode));
4691 4692 tpp = pp + 1;
4692 4693 for (i = 1; i < npgs; i++, tpp++) {
4693 4694 page_unlock(tpp);
4694 4695 }
4695 4696 pszc = pp->p_szc;
4696 4697 goto retry;
4697 4698 }
4698 4699 return (1);
4699 4700 }
4700 4701
4701 4702 void
4702 4703 group_page_unlock(page_t *pp)
4703 4704 {
4704 4705 page_t *tpp;
4705 4706 pgcnt_t npgs, i;
4706 4707
4707 4708 ASSERT(PAGE_LOCKED(pp));
4708 4709 ASSERT(!PP_ISFREE(pp));
4709 4710 ASSERT(pp == PP_PAGEROOT(pp));
4710 4711 npgs = page_get_pagecnt(pp->p_szc);
4711 4712 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) {
4712 4713 page_unlock(tpp);
4713 4714 }
4714 4715 }
4715 4716
4716 4717 /*
4717 4718 * returns
4718 4719 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages
4719 4720 * ERANGE : this is not a base page
4720 4721 * EBUSY : failure to get locks on the page/pages
4721 4722 * ENOMEM : failure to obtain replacement pages
4722 4723 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel
4723 4724 * EIO : An error occurred while trying to copy the page data
4724 4725 *
4725 4726 * Return with all constituent members of target and replacement
4726 4727 * SE_EXCL locked. It is the callers responsibility to drop the
4727 4728 * locks.
4728 4729 */
4729 4730 int
4730 4731 do_page_relocate(
4731 4732 page_t **target,
4732 4733 page_t **replacement,
4733 4734 int grouplock,
4734 4735 spgcnt_t *nrelocp,
4735 4736 lgrp_t *lgrp)
4736 4737 {
4737 4738 page_t *first_repl;
4738 4739 page_t *repl;
4739 4740 page_t *targ;
4740 4741 page_t *pl = NULL;
4741 4742 uint_t ppattr;
4742 4743 pfn_t pfn, repl_pfn;
4743 4744 uint_t szc;
4744 4745 spgcnt_t npgs, i;
4745 4746 int repl_contig = 0;
4746 4747 uint_t flags = 0;
4747 4748 spgcnt_t dofree = 0;
4748 4749
4749 4750 *nrelocp = 0;
4750 4751
4751 4752 #if defined(__sparc)
4752 4753 /*
4753 4754 * We need to wait till OBP has completed
4754 4755 * its boot-time handoff of its resources to the kernel
4755 4756 * before we allow page relocation
4756 4757 */
4757 4758 if (page_relocate_ready == 0) {
4758 4759 return (EAGAIN);
4759 4760 }
4760 4761 #endif
4761 4762
4762 4763 /*
4763 4764 * If this is not a base page,
4764 4765 * just return with 0x0 pages relocated.
4765 4766 */
4766 4767 targ = *target;
4767 4768 ASSERT(PAGE_EXCL(targ));
4768 4769 ASSERT(!PP_ISFREE(targ));
4769 4770 szc = targ->p_szc;
4770 4771 ASSERT(szc < mmu_page_sizes);
4771 4772 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4772 4773 pfn = targ->p_pagenum;
4773 4774 if (pfn != PFN_BASE(pfn, szc)) {
4774 4775 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]);
4775 4776 return (ERANGE);
4776 4777 }
4777 4778
4778 4779 if ((repl = *replacement) != NULL && repl->p_szc >= szc) {
4779 4780 repl_pfn = repl->p_pagenum;
4780 4781 if (repl_pfn != PFN_BASE(repl_pfn, szc)) {
4781 4782 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]);
4782 4783 return (ERANGE);
4783 4784 }
4784 4785 repl_contig = 1;
4785 4786 }
4786 4787
4787 4788 /*
4788 4789 * We must lock all members of this large page or we cannot
4789 4790 * relocate any part of it.
4790 4791 */
4791 4792 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) {
4792 4793 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]);
4793 4794 return (EBUSY);
4794 4795 }
4795 4796
4796 4797 /*
4797 4798 * reread szc it could have been decreased before
4798 4799 * group_page_trylock() was done.
4799 4800 */
4800 4801 szc = targ->p_szc;
4801 4802 ASSERT(szc < mmu_page_sizes);
4802 4803 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4803 4804 ASSERT(pfn == PFN_BASE(pfn, szc));
4804 4805
4805 4806 npgs = page_get_pagecnt(targ->p_szc);
4806 4807
4807 4808 if (repl == NULL) {
4808 4809 dofree = npgs; /* Size of target page in MMU pages */
4809 4810 if (!page_create_wait(dofree, 0)) {
4810 4811 if (grouplock != 0) {
4811 4812 group_page_unlock(targ);
4812 4813 }
4813 4814 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4814 4815 return (ENOMEM);
4815 4816 }
4816 4817
4817 4818 /*
4818 4819 * seg kmem pages require that the target and replacement
4819 4820 * page be the same pagesize.
4820 4821 */
4821 4822 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0;
4822 4823 repl = page_get_replacement_page(targ, lgrp, flags);
4823 4824 if (repl == NULL) {
4824 4825 if (grouplock != 0) {
4825 4826 group_page_unlock(targ);
4826 4827 }
4827 4828 page_create_putback(dofree);
4828 4829 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4829 4830 return (ENOMEM);
4830 4831 }
4831 4832 }
4832 4833 #ifdef DEBUG
4833 4834 else {
4834 4835 ASSERT(PAGE_LOCKED(repl));
4835 4836 }
4836 4837 #endif /* DEBUG */
4837 4838
4838 4839 #if defined(__sparc)
4839 4840 /*
4840 4841 * Let hat_page_relocate() complete the relocation if it's kernel page
4841 4842 */
4842 4843 if (VN_ISKAS(targ->p_vnode)) {
4843 4844 *replacement = repl;
4844 4845 if (hat_page_relocate(target, replacement, nrelocp) != 0) {
4845 4846 if (grouplock != 0) {
4846 4847 group_page_unlock(targ);
4847 4848 }
4848 4849 if (dofree) {
4849 4850 *replacement = NULL;
4850 4851 page_free_replacement_page(repl);
4851 4852 page_create_putback(dofree);
4852 4853 }
4853 4854 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]);
4854 4855 return (EAGAIN);
4855 4856 }
4856 4857 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4857 4858 return (0);
4858 4859 }
4859 4860 #else
4860 4861 #if defined(lint)
4861 4862 dofree = dofree;
4862 4863 #endif
4863 4864 #endif
4864 4865
4865 4866 first_repl = repl;
4866 4867
4867 4868 for (i = 0; i < npgs; i++) {
4868 4869 ASSERT(PAGE_EXCL(targ));
4869 4870 ASSERT(targ->p_slckcnt == 0);
4870 4871 ASSERT(repl->p_slckcnt == 0);
4871 4872
4872 4873 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD);
4873 4874
4874 4875 ASSERT(hat_page_getshare(targ) == 0);
4875 4876 ASSERT(!PP_ISFREE(targ));
4876 4877 ASSERT(targ->p_pagenum == (pfn + i));
4877 4878 ASSERT(repl_contig == 0 ||
4878 4879 repl->p_pagenum == (repl_pfn + i));
4879 4880
4880 4881 /*
4881 4882 * Copy the page contents and attributes then
4882 4883 * relocate the page in the page hash.
4883 4884 */
4884 4885 if (ppcopy(targ, repl) == 0) {
4885 4886 targ = *target;
4886 4887 repl = first_repl;
4887 4888 VM_STAT_ADD(vmm_vmstats.ppr_copyfail);
4888 4889 if (grouplock != 0) {
4889 4890 group_page_unlock(targ);
4890 4891 }
4891 4892 if (dofree) {
4892 4893 *replacement = NULL;
4893 4894 page_free_replacement_page(repl);
4894 4895 page_create_putback(dofree);
4895 4896 }
4896 4897 return (EIO);
4897 4898 }
4898 4899
4899 4900 targ++;
4900 4901 if (repl_contig != 0) {
4901 4902 repl++;
4902 4903 } else {
4903 4904 repl = repl->p_next;
4904 4905 }
4905 4906 }
4906 4907
4907 4908 repl = first_repl;
4908 4909 targ = *target;
4909 4910
4910 4911 for (i = 0; i < npgs; i++) {
4911 4912 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO));
4912 4913 page_clr_all_props(repl);
4913 4914 page_set_props(repl, ppattr);
4914 4915 page_relocate_hash(repl, targ);
4915 4916
4916 4917 ASSERT(hat_page_getshare(targ) == 0);
4917 4918 ASSERT(hat_page_getshare(repl) == 0);
4918 4919 /*
4919 4920 * Now clear the props on targ, after the
4920 4921 * page_relocate_hash(), they no longer
4921 4922 * have any meaning.
4922 4923 */
4923 4924 page_clr_all_props(targ);
4924 4925 ASSERT(targ->p_next == targ);
4925 4926 ASSERT(targ->p_prev == targ);
4926 4927 page_list_concat(&pl, &targ);
4927 4928
4928 4929 targ++;
4929 4930 if (repl_contig != 0) {
4930 4931 repl++;
4931 4932 } else {
4932 4933 repl = repl->p_next;
4933 4934 }
4934 4935 }
4935 4936 /* assert that we have come full circle with repl */
4936 4937 ASSERT(repl_contig == 1 || first_repl == repl);
4937 4938
4938 4939 *target = pl;
4939 4940 if (*replacement == NULL) {
4940 4941 ASSERT(first_repl == repl);
4941 4942 *replacement = repl;
4942 4943 }
4943 4944 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4944 4945 *nrelocp = npgs;
4945 4946 return (0);
4946 4947 }
4947 4948 /*
4948 4949 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
4949 4950 */
4950 4951 int
4951 4952 page_relocate(
4952 4953 page_t **target,
4953 4954 page_t **replacement,
4954 4955 int grouplock,
4955 4956 int freetarget,
4956 4957 spgcnt_t *nrelocp,
4957 4958 lgrp_t *lgrp)
4958 4959 {
4959 4960 spgcnt_t ret;
4960 4961
4961 4962 /* do_page_relocate returns 0 on success or errno value */
4962 4963 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp);
4963 4964
4964 4965 if (ret != 0 || freetarget == 0) {
4965 4966 return (ret);
4966 4967 }
4967 4968 if (*nrelocp == 1) {
4968 4969 ASSERT(*target != NULL);
4969 4970 page_free(*target, 1);
4970 4971 } else {
4971 4972 page_t *tpp = *target;
4972 4973 uint_t szc = tpp->p_szc;
4973 4974 pgcnt_t npgs = page_get_pagecnt(szc);
4974 4975 ASSERT(npgs > 1);
4975 4976 ASSERT(szc != 0);
4976 4977 do {
4977 4978 ASSERT(PAGE_EXCL(tpp));
4978 4979 ASSERT(!hat_page_is_mapped(tpp));
4979 4980 ASSERT(tpp->p_szc == szc);
4980 4981 PP_SETFREE(tpp);
4981 4982 PP_SETAGED(tpp);
4982 4983 npgs--;
4983 4984 } while ((tpp = tpp->p_next) != *target);
4984 4985 ASSERT(npgs == 0);
4985 4986 page_list_add_pages(*target, 0);
4986 4987 npgs = page_get_pagecnt(szc);
4987 4988 page_create_putback(npgs);
4988 4989 }
4989 4990 return (ret);
4990 4991 }
4991 4992
4992 4993 /*
4993 4994 * it is up to the caller to deal with pcf accounting.
4994 4995 */
4995 4996 void
4996 4997 page_free_replacement_page(page_t *pplist)
4997 4998 {
4998 4999 page_t *pp;
4999 5000
5000 5001 while (pplist != NULL) {
5001 5002 /*
5002 5003 * pp_targ is a linked list.
5003 5004 */
5004 5005 pp = pplist;
5005 5006 if (pp->p_szc == 0) {
5006 5007 page_sub(&pplist, pp);
5007 5008 page_clr_all_props(pp);
5008 5009 PP_SETFREE(pp);
5009 5010 PP_SETAGED(pp);
5010 5011 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
5011 5012 page_unlock(pp);
5012 5013 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]);
5013 5014 } else {
5014 5015 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc);
5015 5016 page_t *tpp;
5016 5017 page_list_break(&pp, &pplist, curnpgs);
5017 5018 tpp = pp;
5018 5019 do {
5019 5020 ASSERT(PAGE_EXCL(tpp));
5020 5021 ASSERT(!hat_page_is_mapped(tpp));
5021 5022 page_clr_all_props(tpp);
5022 5023 PP_SETFREE(tpp);
5023 5024 PP_SETAGED(tpp);
5024 5025 } while ((tpp = tpp->p_next) != pp);
5025 5026 page_list_add_pages(pp, 0);
5026 5027 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]);
5027 5028 }
5028 5029 }
5029 5030 }
5030 5031
5031 5032 /*
5032 5033 * Relocate target to non-relocatable replacement page.
5033 5034 */
5034 5035 int
5035 5036 page_relocate_cage(page_t **target, page_t **replacement)
5036 5037 {
5037 5038 page_t *tpp, *rpp;
5038 5039 spgcnt_t pgcnt, npgs;
5039 5040 int result;
5040 5041
5041 5042 tpp = *target;
5042 5043
5043 5044 ASSERT(PAGE_EXCL(tpp));
5044 5045 ASSERT(tpp->p_szc == 0);
5045 5046
5046 5047 pgcnt = btop(page_get_pagesize(tpp->p_szc));
5047 5048
5048 5049 do {
5049 5050 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC);
5050 5051 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC);
5051 5052 if (rpp == NULL) {
5052 5053 page_create_putback(pgcnt);
5053 5054 kcage_cageout_wakeup();
5054 5055 }
5055 5056 } while (rpp == NULL);
5056 5057
5057 5058 ASSERT(PP_ISNORELOC(rpp));
5058 5059
5059 5060 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL);
5060 5061
5061 5062 if (result == 0) {
5062 5063 *replacement = rpp;
5063 5064 if (pgcnt != npgs)
5064 5065 panic("page_relocate_cage: partial relocation");
5065 5066 }
5066 5067
5067 5068 return (result);
5068 5069 }
5069 5070
5070 5071 /*
5071 5072 * Release the page lock on a page, place on cachelist
5072 5073 * tail if no longer mapped. Caller can let us know if
5073 5074 * the page is known to be clean.
5074 5075 */
5075 5076 int
5076 5077 page_release(page_t *pp, int checkmod)
5077 5078 {
5078 5079 int status;
5079 5080
5080 5081 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) &&
5081 5082 (pp->p_vnode != NULL));
5082 5083
5083 5084 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) &&
5084 5085 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) &&
5085 5086 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 &&
5086 5087 !hat_page_is_mapped(pp)) {
5087 5088
5088 5089 /*
5089 5090 * If page is modified, unlock it
5090 5091 *
5091 5092 * (p_nrm & P_MOD) bit has the latest stuff because:
5092 5093 * (1) We found that this page doesn't have any mappings
5093 5094 * _after_ holding SE_EXCL and
5094 5095 * (2) We didn't drop SE_EXCL lock after the check in (1)
5095 5096 */
5096 5097 if (checkmod && hat_ismod(pp)) {
5097 5098 page_unlock(pp);
5098 5099 status = PGREL_MOD;
5099 5100 } else {
5100 5101 /*LINTED: constant in conditional context*/
5101 5102 VN_DISPOSE(pp, B_FREE, 0, kcred);
5102 5103 status = PGREL_CLEAN;
5103 5104 }
5104 5105 } else {
5105 5106 page_unlock(pp);
5106 5107 status = PGREL_NOTREL;
5107 5108 }
5108 5109 return (status);
5109 5110 }
5110 5111
5111 5112 /*
5112 5113 * Given a constituent page, try to demote the large page on the freelist.
5113 5114 *
5114 5115 * Returns nonzero if the page could be demoted successfully. Returns with
5115 5116 * the constituent page still locked.
5116 5117 */
5117 5118 int
5118 5119 page_try_demote_free_pages(page_t *pp)
5119 5120 {
5120 5121 page_t *rootpp = pp;
5121 5122 pfn_t pfn = page_pptonum(pp);
5122 5123 spgcnt_t npgs;
5123 5124 uint_t szc = pp->p_szc;
5124 5125
5125 5126 ASSERT(PP_ISFREE(pp));
5126 5127 ASSERT(PAGE_EXCL(pp));
5127 5128
5128 5129 /*
5129 5130 * Adjust rootpp and lock it, if `pp' is not the base
5130 5131 * constituent page.
5131 5132 */
5132 5133 npgs = page_get_pagecnt(pp->p_szc);
5133 5134 if (npgs == 1) {
5134 5135 return (0);
5135 5136 }
5136 5137
5137 5138 if (!IS_P2ALIGNED(pfn, npgs)) {
5138 5139 pfn = P2ALIGN(pfn, npgs);
5139 5140 rootpp = page_numtopp_nolock(pfn);
5140 5141 }
5141 5142
5142 5143 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) {
5143 5144 return (0);
5144 5145 }
5145 5146
5146 5147 if (rootpp->p_szc != szc) {
5147 5148 if (pp != rootpp)
5148 5149 page_unlock(rootpp);
5149 5150 return (0);
5150 5151 }
5151 5152
5152 5153 page_demote_free_pages(rootpp);
5153 5154
5154 5155 if (pp != rootpp)
5155 5156 page_unlock(rootpp);
5156 5157
5157 5158 ASSERT(PP_ISFREE(pp));
5158 5159 ASSERT(PAGE_EXCL(pp));
5159 5160 return (1);
5160 5161 }
5161 5162
5162 5163 /*
5163 5164 * Given a constituent page, try to demote the large page.
5164 5165 *
5165 5166 * Returns nonzero if the page could be demoted successfully. Returns with
5166 5167 * the constituent page still locked.
5167 5168 */
5168 5169 int
5169 5170 page_try_demote_pages(page_t *pp)
5170 5171 {
5171 5172 page_t *tpp, *rootpp = pp;
5172 5173 pfn_t pfn = page_pptonum(pp);
5173 5174 spgcnt_t i, npgs;
5174 5175 uint_t szc = pp->p_szc;
5175 5176 vnode_t *vp = pp->p_vnode;
5176 5177
5177 5178 ASSERT(PAGE_EXCL(pp));
5178 5179
5179 5180 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]);
5180 5181
5181 5182 if (pp->p_szc == 0) {
5182 5183 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]);
5183 5184 return (1);
5184 5185 }
5185 5186
5186 5187 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) {
5187 5188 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]);
5188 5189 page_demote_vp_pages(pp);
5189 5190 ASSERT(pp->p_szc == 0);
5190 5191 return (1);
5191 5192 }
5192 5193
5193 5194 /*
5194 5195 * Adjust rootpp if passed in is not the base
5195 5196 * constituent page.
5196 5197 */
5197 5198 npgs = page_get_pagecnt(pp->p_szc);
5198 5199 ASSERT(npgs > 1);
5199 5200 if (!IS_P2ALIGNED(pfn, npgs)) {
5200 5201 pfn = P2ALIGN(pfn, npgs);
5201 5202 rootpp = page_numtopp_nolock(pfn);
5202 5203 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]);
5203 5204 ASSERT(rootpp->p_vnode != NULL);
5204 5205 ASSERT(rootpp->p_szc == szc);
5205 5206 }
5206 5207
5207 5208 /*
5208 5209 * We can't demote kernel pages since we can't hat_unload()
5209 5210 * the mappings.
5210 5211 */
5211 5212 if (VN_ISKAS(rootpp->p_vnode))
5212 5213 return (0);
5213 5214
5214 5215 /*
5215 5216 * Attempt to lock all constituent pages except the page passed
5216 5217 * in since it's already locked.
5217 5218 */
5218 5219 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5219 5220 ASSERT(!PP_ISFREE(tpp));
5220 5221 ASSERT(tpp->p_vnode != NULL);
5221 5222
5222 5223 if (tpp != pp && !page_trylock(tpp, SE_EXCL))
5223 5224 break;
5224 5225 ASSERT(tpp->p_szc == rootpp->p_szc);
5225 5226 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i);
5226 5227 }
5227 5228
5228 5229 /*
5229 5230 * If we failed to lock them all then unlock what we have
5230 5231 * locked so far and bail.
5231 5232 */
5232 5233 if (i < npgs) {
5233 5234 tpp = rootpp;
5234 5235 while (i-- > 0) {
5235 5236 if (tpp != pp)
5236 5237 page_unlock(tpp);
5237 5238 tpp++;
5238 5239 }
5239 5240 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]);
5240 5241 return (0);
5241 5242 }
5242 5243
5243 5244 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5244 5245 ASSERT(PAGE_EXCL(tpp));
5245 5246 ASSERT(tpp->p_slckcnt == 0);
5246 5247 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
5247 5248 tpp->p_szc = 0;
5248 5249 }
5249 5250
5250 5251 /*
5251 5252 * Unlock all pages except the page passed in.
5252 5253 */
5253 5254 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5254 5255 ASSERT(!hat_page_is_mapped(tpp));
5255 5256 if (tpp != pp)
5256 5257 page_unlock(tpp);
5257 5258 }
5258 5259
5259 5260 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]);
5260 5261 return (1);
5261 5262 }
5262 5263
5263 5264 /*
5264 5265 * Called by page_free() and page_destroy() to demote the page size code
5265 5266 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
5266 5267 * p_szc on free list, neither can we just clear p_szc of a single page_t
5267 5268 * within a large page since it will break other code that relies on p_szc
5268 5269 * being the same for all page_t's of a large page). Anonymous pages should
5269 5270 * never end up here because anon_map_getpages() cannot deal with p_szc
5270 5271 * changes after a single constituent page is locked. While anonymous or
5271 5272 * kernel large pages are demoted or freed the entire large page at a time
5272 5273 * with all constituent pages locked EXCL for the file system pages we
5273 5274 * have to be able to demote a large page (i.e. decrease all constituent pages
5274 5275 * p_szc) with only just an EXCL lock on one of constituent pages. The reason
5275 5276 * we can easily deal with anonymous page demotion the entire large page at a
5276 5277 * time is that those operation originate at address space level and concern
5277 5278 * the entire large page region with actual demotion only done when pages are
5278 5279 * not shared with any other processes (therefore we can always get EXCL lock
5279 5280 * on all anonymous constituent pages after clearing segment page
5280 5281 * cache). However file system pages can be truncated or invalidated at a
5281 5282 * PAGESIZE level from the file system side and end up in page_free() or
5282 5283 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed
5283 5284 * and therefore pageout should be able to demote a large page by EXCL locking
5284 5285 * any constituent page that is not under SOFTLOCK). In those cases we cannot
5285 5286 * rely on being able to lock EXCL all constituent pages.
5286 5287 *
5287 5288 * To prevent szc changes on file system pages one has to lock all constituent
5288 5289 * pages at least SHARED (or call page_szc_lock()). The only subsystem that
5289 5290 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to
5290 5291 * prevent szc changes is hat layer that uses its own page level mlist
5291 5292 * locks. hat assumes that szc doesn't change after mlist lock for a page is
5292 5293 * taken. Therefore we need to change szc under hat level locks if we only
5293 5294 * have an EXCL lock on a single constituent page and hat still references any
5294 5295 * of constituent pages. (Note we can't "ignore" hat layer by simply
5295 5296 * hat_pageunload() all constituent pages without having EXCL locks on all of
5296 5297 * constituent pages). We use hat_page_demote() call to safely demote szc of
5297 5298 * all constituent pages under hat locks when we only have an EXCL lock on one
5298 5299 * of constituent pages.
5299 5300 *
5300 5301 * This routine calls page_szc_lock() before calling hat_page_demote() to
5301 5302 * allow segvn in one special case not to lock all constituent pages SHARED
5302 5303 * before calling hat_memload_array() that relies on p_szc not changing even
5303 5304 * before hat level mlist lock is taken. In that case segvn uses
5304 5305 * page_szc_lock() to prevent hat_page_demote() changing p_szc values.
5305 5306 *
5306 5307 * Anonymous or kernel page demotion still has to lock all pages exclusively
5307 5308 * and do hat_pageunload() on all constituent pages before demoting the page
5308 5309 * therefore there's no need for anonymous or kernel page demotion to use
5309 5310 * hat_page_demote() mechanism.
5310 5311 *
5311 5312 * hat_page_demote() removes all large mappings that map pp and then decreases
5312 5313 * p_szc starting from the last constituent page of the large page. By working
5313 5314 * from the tail of a large page in pfn decreasing order allows one looking at
5314 5315 * the root page to know that hat_page_demote() is done for root's szc area.
5315 5316 * e.g. if a root page has szc 1 one knows it only has to lock all constituent
5316 5317 * pages within szc 1 area to prevent szc changes because hat_page_demote()
5317 5318 * that started on this page when it had szc > 1 is done for this szc 1 area.
5318 5319 *
5319 5320 * We are guaranteed that all constituent pages of pp's large page belong to
5320 5321 * the same vnode with the consecutive offsets increasing in the direction of
5321 5322 * the pfn i.e. the identity of constituent pages can't change until their
5322 5323 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
5323 5324 * large mappings to pp even though we don't lock any constituent page except
5324 5325 * pp (i.e. we won't unload e.g. kernel locked page).
5325 5326 */
5326 5327 static void
5327 5328 page_demote_vp_pages(page_t *pp)
5328 5329 {
5329 5330 kmutex_t *mtx;
5330 5331
5331 5332 ASSERT(PAGE_EXCL(pp));
5332 5333 ASSERT(!PP_ISFREE(pp));
5333 5334 ASSERT(pp->p_vnode != NULL);
5334 5335 ASSERT(!IS_SWAPFSVP(pp->p_vnode));
5335 5336 ASSERT(!PP_ISKAS(pp));
5336 5337
5337 5338 VM_STAT_ADD(pagecnt.pc_demote_pages[0]);
5338 5339
5339 5340 mtx = page_szc_lock(pp);
5340 5341 if (mtx != NULL) {
5341 5342 hat_page_demote(pp);
5342 5343 mutex_exit(mtx);
5343 5344 }
5344 5345 ASSERT(pp->p_szc == 0);
5345 5346 }
5346 5347
5347 5348 /*
5348 5349 * Mark any existing pages for migration in the given range
5349 5350 */
5350 5351 void
5351 5352 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len,
5352 5353 struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
5353 5354 u_offset_t vnoff, int rflag)
5354 5355 {
5355 5356 struct anon *ap;
5356 5357 vnode_t *curvp;
5357 5358 lgrp_t *from;
5358 5359 pgcnt_t nlocked;
5359 5360 u_offset_t off;
5360 5361 pfn_t pfn;
5361 5362 size_t pgsz;
5362 5363 size_t segpgsz;
5363 5364 pgcnt_t pages;
5364 5365 uint_t pszc;
5365 5366 page_t *pp0, *pp;
5366 5367 caddr_t va;
5367 5368 ulong_t an_idx;
5368 5369 anon_sync_obj_t cookie;
5369 5370
5370 5371 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
5371 5372
5372 5373 /*
5373 5374 * Don't do anything if don't need to do lgroup optimizations
5374 5375 * on this system
5375 5376 */
5376 5377 if (!lgrp_optimizations())
5377 5378 return;
5378 5379
5379 5380 /*
5380 5381 * Align address and length to (potentially large) page boundary
5381 5382 */
5382 5383 segpgsz = page_get_pagesize(seg->s_szc);
5383 5384 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz);
5384 5385 if (rflag)
5385 5386 len = P2ROUNDUP(len, segpgsz);
5386 5387
5387 5388 /*
5388 5389 * Do one (large) page at a time
5389 5390 */
5390 5391 va = addr;
5391 5392 while (va < addr + len) {
5392 5393 /*
5393 5394 * Lookup (root) page for vnode and offset corresponding to
5394 5395 * this virtual address
5395 5396 * Try anonmap first since there may be copy-on-write
5396 5397 * pages, but initialize vnode pointer and offset using
5397 5398 * vnode arguments just in case there isn't an amp.
5398 5399 */
5399 5400 curvp = vp;
5400 5401 off = vnoff + va - seg->s_base;
5401 5402 if (amp) {
5402 5403 ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
5403 5404 an_idx = anon_index + seg_page(seg, va);
5404 5405 anon_array_enter(amp, an_idx, &cookie);
5405 5406 ap = anon_get_ptr(amp->ahp, an_idx);
5406 5407 if (ap)
5407 5408 swap_xlate(ap, &curvp, &off);
5408 5409 anon_array_exit(&cookie);
5409 5410 ANON_LOCK_EXIT(&->a_rwlock);
5410 5411 }
5411 5412
5412 5413 pp = NULL;
5413 5414 if (curvp)
5414 5415 pp = page_lookup(curvp, off, SE_SHARED);
5415 5416
5416 5417 /*
5417 5418 * If there isn't a page at this virtual address,
5418 5419 * skip to next page
5419 5420 */
5420 5421 if (pp == NULL) {
5421 5422 va += PAGESIZE;
5422 5423 continue;
5423 5424 }
5424 5425
5425 5426 /*
5426 5427 * Figure out which lgroup this page is in for kstats
5427 5428 */
5428 5429 pfn = page_pptonum(pp);
5429 5430 from = lgrp_pfn_to_lgrp(pfn);
5430 5431
5431 5432 /*
5432 5433 * Get page size, and round up and skip to next page boundary
5433 5434 * if unaligned address
5434 5435 */
5435 5436 pszc = pp->p_szc;
5436 5437 pgsz = page_get_pagesize(pszc);
5437 5438 pages = btop(pgsz);
5438 5439 if (!IS_P2ALIGNED(va, pgsz) ||
5439 5440 !IS_P2ALIGNED(pfn, pages) ||
5440 5441 pgsz > segpgsz) {
5441 5442 pgsz = MIN(pgsz, segpgsz);
5442 5443 page_unlock(pp);
5443 5444 pages = btop(P2END((uintptr_t)va, pgsz) -
5444 5445 (uintptr_t)va);
5445 5446 va = (caddr_t)P2END((uintptr_t)va, pgsz);
5446 5447 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, pages);
5447 5448 continue;
5448 5449 }
5449 5450
5450 5451 /*
5451 5452 * Upgrade to exclusive lock on page
5452 5453 */
5453 5454 if (!page_tryupgrade(pp)) {
5454 5455 page_unlock(pp);
5455 5456 va += pgsz;
5456 5457 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5457 5458 btop(pgsz));
5458 5459 continue;
5459 5460 }
5460 5461
5461 5462 pp0 = pp++;
5462 5463 nlocked = 1;
5463 5464
5464 5465 /*
5465 5466 * Lock constituent pages if this is large page
5466 5467 */
5467 5468 if (pages > 1) {
5468 5469 /*
5469 5470 * Lock all constituents except root page, since it
5470 5471 * should be locked already.
5471 5472 */
5472 5473 for (; nlocked < pages; nlocked++) {
5473 5474 if (!page_trylock(pp, SE_EXCL)) {
5474 5475 break;
5475 5476 }
5476 5477 if (PP_ISFREE(pp) ||
5477 5478 pp->p_szc != pszc) {
5478 5479 /*
5479 5480 * hat_page_demote() raced in with us.
5480 5481 */
5481 5482 ASSERT(!IS_SWAPFSVP(curvp));
5482 5483 page_unlock(pp);
5483 5484 break;
5484 5485 }
5485 5486 pp++;
5486 5487 }
5487 5488 }
5488 5489
5489 5490 /*
5490 5491 * If all constituent pages couldn't be locked,
5491 5492 * unlock pages locked so far and skip to next page.
5492 5493 */
5493 5494 if (nlocked < pages) {
5494 5495 while (pp0 < pp) {
5495 5496 page_unlock(pp0++);
5496 5497 }
5497 5498 va += pgsz;
5498 5499 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5499 5500 btop(pgsz));
5500 5501 continue;
5501 5502 }
5502 5503
5503 5504 /*
5504 5505 * hat_page_demote() can no longer happen
5505 5506 * since last cons page had the right p_szc after
5506 5507 * all cons pages were locked. all cons pages
5507 5508 * should now have the same p_szc.
5508 5509 */
5509 5510
5510 5511 /*
5511 5512 * All constituent pages locked successfully, so mark
5512 5513 * large page for migration and unload the mappings of
5513 5514 * constituent pages, so a fault will occur on any part of the
5514 5515 * large page
5515 5516 */
5516 5517 PP_SETMIGRATE(pp0);
5517 5518 while (pp0 < pp) {
5518 5519 (void) hat_pageunload(pp0, HAT_FORCE_PGUNLOAD);
5519 5520 ASSERT(hat_page_getshare(pp0) == 0);
5520 5521 page_unlock(pp0++);
5521 5522 }
5522 5523 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked);
5523 5524
5524 5525 va += pgsz;
5525 5526 }
5526 5527 }
5527 5528
5528 5529 /*
5529 5530 * Migrate any pages that have been marked for migration in the given range
5530 5531 */
5531 5532 void
5532 5533 page_migrate(
5533 5534 struct seg *seg,
5534 5535 caddr_t addr,
5535 5536 page_t **ppa,
5536 5537 pgcnt_t npages)
5537 5538 {
5538 5539 lgrp_t *from;
5539 5540 lgrp_t *to;
5540 5541 page_t *newpp;
5541 5542 page_t *pp;
5542 5543 pfn_t pfn;
5543 5544 size_t pgsz;
5544 5545 spgcnt_t page_cnt;
5545 5546 spgcnt_t i;
5546 5547 uint_t pszc;
5547 5548
5548 5549 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
5549 5550
5550 5551 while (npages > 0) {
5551 5552 pp = *ppa;
5552 5553 pszc = pp->p_szc;
5553 5554 pgsz = page_get_pagesize(pszc);
5554 5555 page_cnt = btop(pgsz);
5555 5556
5556 5557 /*
5557 5558 * Check to see whether this page is marked for migration
5558 5559 *
5559 5560 * Assume that root page of large page is marked for
5560 5561 * migration and none of the other constituent pages
5561 5562 * are marked. This really simplifies clearing the
5562 5563 * migrate bit by not having to clear it from each
5563 5564 * constituent page.
5564 5565 *
5565 5566 * note we don't want to relocate an entire large page if
5566 5567 * someone is only using one subpage.
5567 5568 */
5568 5569 if (npages < page_cnt)
5569 5570 break;
5570 5571
5571 5572 /*
5572 5573 * Is it marked for migration?
5573 5574 */
5574 5575 if (!PP_ISMIGRATE(pp))
5575 5576 goto next;
5576 5577
5577 5578 /*
5578 5579 * Determine lgroups that page is being migrated between
5579 5580 */
5580 5581 pfn = page_pptonum(pp);
5581 5582 if (!IS_P2ALIGNED(pfn, page_cnt)) {
5582 5583 break;
5583 5584 }
5584 5585 from = lgrp_pfn_to_lgrp(pfn);
5585 5586 to = lgrp_mem_choose(seg, addr, pgsz);
5586 5587
5587 5588 /*
5588 5589 * Need to get exclusive lock's to migrate
5589 5590 */
5590 5591 for (i = 0; i < page_cnt; i++) {
5591 5592 ASSERT(PAGE_LOCKED(ppa[i]));
5592 5593 if (page_pptonum(ppa[i]) != pfn + i ||
5593 5594 ppa[i]->p_szc != pszc) {
5594 5595 break;
5595 5596 }
5596 5597 if (!page_tryupgrade(ppa[i])) {
5597 5598 lgrp_stat_add(from->lgrp_id,
5598 5599 LGRP_PM_FAIL_LOCK_PGS,
5599 5600 page_cnt);
5600 5601 break;
5601 5602 }
5602 5603
5603 5604 /*
5604 5605 * Check to see whether we are trying to migrate
5605 5606 * page to lgroup where it is allocated already.
5606 5607 * If so, clear the migrate bit and skip to next
5607 5608 * page.
5608 5609 */
5609 5610 if (i == 0 && to == from) {
5610 5611 PP_CLRMIGRATE(ppa[0]);
5611 5612 page_downgrade(ppa[0]);
5612 5613 goto next;
5613 5614 }
5614 5615 }
5615 5616
5616 5617 /*
5617 5618 * If all constituent pages couldn't be locked,
5618 5619 * unlock pages locked so far and skip to next page.
5619 5620 */
5620 5621 if (i != page_cnt) {
5621 5622 while (--i != -1) {
5622 5623 page_downgrade(ppa[i]);
5623 5624 }
5624 5625 goto next;
5625 5626 }
5626 5627
5627 5628 (void) page_create_wait(page_cnt, PG_WAIT);
5628 5629 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC);
5629 5630 if (newpp == NULL) {
5630 5631 page_create_putback(page_cnt);
5631 5632 for (i = 0; i < page_cnt; i++) {
5632 5633 page_downgrade(ppa[i]);
5633 5634 }
5634 5635 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS,
5635 5636 page_cnt);
5636 5637 goto next;
5637 5638 }
5638 5639 ASSERT(newpp->p_szc == pszc);
5639 5640 /*
5640 5641 * Clear migrate bit and relocate page
5641 5642 */
5642 5643 PP_CLRMIGRATE(pp);
5643 5644 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) {
5644 5645 panic("page_migrate: page_relocate failed");
5645 5646 }
5646 5647 ASSERT(page_cnt * PAGESIZE == pgsz);
5647 5648
5648 5649 /*
5649 5650 * Keep stats for number of pages migrated from and to
5650 5651 * each lgroup
5651 5652 */
5652 5653 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt);
5653 5654 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt);
5654 5655 /*
5655 5656 * update the page_t array we were passed in and
5656 5657 * unlink constituent pages of a large page.
5657 5658 */
5658 5659 for (i = 0; i < page_cnt; ++i, ++pp) {
5659 5660 ASSERT(PAGE_EXCL(newpp));
5660 5661 ASSERT(newpp->p_szc == pszc);
5661 5662 ppa[i] = newpp;
5662 5663 pp = newpp;
5663 5664 page_sub(&newpp, pp);
5664 5665 page_downgrade(pp);
5665 5666 }
5666 5667 ASSERT(newpp == NULL);
5667 5668 next:
5668 5669 addr += pgsz;
5669 5670 ppa += page_cnt;
5670 5671 npages -= page_cnt;
5671 5672 }
5672 5673 }
5673 5674
5674 5675 uint_t page_reclaim_maxcnt = 60; /* max total iterations */
5675 5676 uint_t page_reclaim_nofree_maxcnt = 3; /* max iterations without progress */
5676 5677 /*
5677 5678 * Reclaim/reserve availrmem for npages.
5678 5679 * If there is not enough memory start reaping seg, kmem caches.
5679 5680 * Start pageout scanner (via page_needfree()).
5680 5681 * Exit after ~ MAX_CNT s regardless of how much memory has been released.
5681 5682 * Note: There is no guarantee that any availrmem will be freed as
5682 5683 * this memory typically is locked (kernel heap) or reserved for swap.
5683 5684 * Also due to memory fragmentation kmem allocator may not be able
5684 5685 * to free any memory (single user allocated buffer will prevent
5685 5686 * freeing slab or a page).
5686 5687 */
5687 5688 int
5688 5689 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
5689 5690 {
5690 5691 int i = 0;
5691 5692 int i_nofree = 0;
5692 5693 int ret = 0;
5693 5694 pgcnt_t deficit;
5694 5695 pgcnt_t old_availrmem = 0;
5695 5696
5696 5697 mutex_enter(&freemem_lock);
5697 5698 while (availrmem < tune.t_minarmem + npages + epages &&
5698 5699 i++ < page_reclaim_maxcnt) {
5699 5700 /* ensure we made some progress in the last few iterations */
5700 5701 if (old_availrmem < availrmem) {
5701 5702 old_availrmem = availrmem;
5702 5703 i_nofree = 0;
5703 5704 } else if (i_nofree++ >= page_reclaim_nofree_maxcnt) {
5704 5705 break;
5705 5706 }
5706 5707
5707 5708 deficit = tune.t_minarmem + npages + epages - availrmem;
5708 5709 mutex_exit(&freemem_lock);
5709 5710 page_needfree(deficit);
5710 5711 kmem_reap();
5711 5712 delay(hz);
5712 5713 page_needfree(-(spgcnt_t)deficit);
5713 5714 mutex_enter(&freemem_lock);
5714 5715 }
5715 5716
5716 5717 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) {
5717 5718 availrmem -= npages;
5718 5719 ret = 1;
5719 5720 }
5720 5721
5721 5722 mutex_exit(&freemem_lock);
5722 5723
5723 5724 return (ret);
5724 5725 }
5725 5726
5726 5727 /*
5727 5728 * Search the memory segments to locate the desired page. Within a
5728 5729 * segment, pages increase linearly with one page structure per
5729 5730 * physical page frame (size PAGESIZE). The search begins
5730 5731 * with the segment that was accessed last, to take advantage of locality.
5731 5732 * If the hint misses, we start from the beginning of the sorted memseg list
5732 5733 */
5733 5734
5734 5735
5735 5736 /*
5736 5737 * Some data structures for pfn to pp lookup.
5737 5738 */
5738 5739 ulong_t mhash_per_slot;
5739 5740 struct memseg *memseg_hash[N_MEM_SLOTS];
5740 5741
5741 5742 page_t *
5742 5743 page_numtopp_nolock(pfn_t pfnum)
5743 5744 {
5744 5745 struct memseg *seg;
5745 5746 page_t *pp;
5746 5747 vm_cpu_data_t *vc;
5747 5748
5748 5749 /*
5749 5750 * We need to disable kernel preemption while referencing the
5750 5751 * cpu_vm_data field in order to prevent us from being switched to
5751 5752 * another cpu and trying to reference it after it has been freed.
5752 5753 * This will keep us on cpu and prevent it from being removed while
5753 5754 * we are still on it.
5754 5755 *
5755 5756 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5756 5757 * which is being resued by DR who will flush those references
5757 5758 * before modifying the reused memseg. See memseg_cpu_vm_flush().
5758 5759 */
5759 5760 kpreempt_disable();
5760 5761 vc = CPU->cpu_vm_data;
5761 5762 ASSERT(vc != NULL);
5762 5763
5763 5764 MEMSEG_STAT_INCR(nsearch);
5764 5765
5765 5766 /* Try last winner first */
5766 5767 if (((seg = vc->vc_pnum_memseg) != NULL) &&
5767 5768 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5768 5769 MEMSEG_STAT_INCR(nlastwon);
5769 5770 pp = seg->pages + (pfnum - seg->pages_base);
5770 5771 if (pp->p_pagenum == pfnum) {
5771 5772 kpreempt_enable();
5772 5773 return ((page_t *)pp);
5773 5774 }
5774 5775 }
5775 5776
5776 5777 /* Else Try hash */
5777 5778 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5778 5779 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5779 5780 MEMSEG_STAT_INCR(nhashwon);
5780 5781 vc->vc_pnum_memseg = seg;
5781 5782 pp = seg->pages + (pfnum - seg->pages_base);
5782 5783 if (pp->p_pagenum == pfnum) {
5783 5784 kpreempt_enable();
5784 5785 return ((page_t *)pp);
5785 5786 }
5786 5787 }
5787 5788
5788 5789 /* Else Brute force */
5789 5790 for (seg = memsegs; seg != NULL; seg = seg->next) {
5790 5791 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5791 5792 vc->vc_pnum_memseg = seg;
5792 5793 pp = seg->pages + (pfnum - seg->pages_base);
5793 5794 if (pp->p_pagenum == pfnum) {
5794 5795 kpreempt_enable();
5795 5796 return ((page_t *)pp);
5796 5797 }
5797 5798 }
5798 5799 }
5799 5800 vc->vc_pnum_memseg = NULL;
5800 5801 kpreempt_enable();
5801 5802 MEMSEG_STAT_INCR(nnotfound);
5802 5803 return ((page_t *)NULL);
5803 5804
5804 5805 }
5805 5806
5806 5807 struct memseg *
5807 5808 page_numtomemseg_nolock(pfn_t pfnum)
5808 5809 {
5809 5810 struct memseg *seg;
5810 5811 page_t *pp;
5811 5812
5812 5813 /*
5813 5814 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5814 5815 * which is being resued by DR who will flush those references
5815 5816 * before modifying the reused memseg. See memseg_cpu_vm_flush().
5816 5817 */
5817 5818 kpreempt_disable();
5818 5819 /* Try hash */
5819 5820 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5820 5821 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5821 5822 pp = seg->pages + (pfnum - seg->pages_base);
5822 5823 if (pp->p_pagenum == pfnum) {
5823 5824 kpreempt_enable();
5824 5825 return (seg);
5825 5826 }
5826 5827 }
5827 5828
5828 5829 /* Else Brute force */
5829 5830 for (seg = memsegs; seg != NULL; seg = seg->next) {
5830 5831 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5831 5832 pp = seg->pages + (pfnum - seg->pages_base);
5832 5833 if (pp->p_pagenum == pfnum) {
5833 5834 kpreempt_enable();
5834 5835 return (seg);
5835 5836 }
5836 5837 }
5837 5838 }
5838 5839 kpreempt_enable();
5839 5840 return ((struct memseg *)NULL);
5840 5841 }
5841 5842
5842 5843 /*
5843 5844 * Given a page and a count return the page struct that is
5844 5845 * n structs away from the current one in the global page
5845 5846 * list.
5846 5847 *
5847 5848 * This function wraps to the first page upon
5848 5849 * reaching the end of the memseg list.
5849 5850 */
5850 5851 page_t *
5851 5852 page_nextn(page_t *pp, ulong_t n)
5852 5853 {
5853 5854 struct memseg *seg;
5854 5855 page_t *ppn;
5855 5856 vm_cpu_data_t *vc;
5856 5857
5857 5858 /*
5858 5859 * We need to disable kernel preemption while referencing the
5859 5860 * cpu_vm_data field in order to prevent us from being switched to
5860 5861 * another cpu and trying to reference it after it has been freed.
5861 5862 * This will keep us on cpu and prevent it from being removed while
5862 5863 * we are still on it.
5863 5864 *
5864 5865 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5865 5866 * which is being resued by DR who will flush those references
5866 5867 * before modifying the reused memseg. See memseg_cpu_vm_flush().
5867 5868 */
5868 5869 kpreempt_disable();
5869 5870 vc = (vm_cpu_data_t *)CPU->cpu_vm_data;
5870 5871
5871 5872 ASSERT(vc != NULL);
5872 5873
5873 5874 if (((seg = vc->vc_pnext_memseg) == NULL) ||
5874 5875 (seg->pages_base == seg->pages_end) ||
5875 5876 !(pp >= seg->pages && pp < seg->epages)) {
5876 5877
5877 5878 for (seg = memsegs; seg; seg = seg->next) {
5878 5879 if (pp >= seg->pages && pp < seg->epages)
5879 5880 break;
5880 5881 }
5881 5882
5882 5883 if (seg == NULL) {
5883 5884 /* Memory delete got in, return something valid. */
5884 5885 /* TODO: fix me. */
5885 5886 seg = memsegs;
5886 5887 pp = seg->pages;
5887 5888 }
5888 5889 }
5889 5890
5890 5891 /* check for wraparound - possible if n is large */
5891 5892 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) {
5892 5893 n -= seg->epages - pp;
5893 5894 seg = seg->next;
5894 5895 if (seg == NULL)
5895 5896 seg = memsegs;
5896 5897 pp = seg->pages;
5897 5898 }
5898 5899 vc->vc_pnext_memseg = seg;
5899 5900 kpreempt_enable();
5900 5901 return (ppn);
5901 5902 }
5902 5903
5903 5904 /*
5904 5905 * Initialize for a loop using page_next_scan_large().
5905 5906 */
5906 5907 page_t *
5907 5908 page_next_scan_init(void **cookie)
5908 5909 {
5909 5910 ASSERT(cookie != NULL);
5910 5911 *cookie = (void *)memsegs;
5911 5912 return ((page_t *)memsegs->pages);
5912 5913 }
5913 5914
5914 5915 /*
5915 5916 * Return the next page in a scan of page_t's, assuming we want
5916 5917 * to skip over sub-pages within larger page sizes.
5917 5918 *
5918 5919 * The cookie is used to keep track of the current memseg.
5919 5920 */
5920 5921 page_t *
5921 5922 page_next_scan_large(
5922 5923 page_t *pp,
5923 5924 ulong_t *n,
5924 5925 void **cookie)
5925 5926 {
5926 5927 struct memseg *seg = (struct memseg *)*cookie;
5927 5928 page_t *new_pp;
5928 5929 ulong_t cnt;
5929 5930 pfn_t pfn;
5930 5931
5931 5932
5932 5933 /*
5933 5934 * get the count of page_t's to skip based on the page size
5934 5935 */
5935 5936 ASSERT(pp != NULL);
5936 5937 if (pp->p_szc == 0) {
5937 5938 cnt = 1;
5938 5939 } else {
5939 5940 pfn = page_pptonum(pp);
5940 5941 cnt = page_get_pagecnt(pp->p_szc);
5941 5942 cnt -= pfn & (cnt - 1);
5942 5943 }
5943 5944 *n += cnt;
5944 5945 new_pp = pp + cnt;
5945 5946
5946 5947 /*
5947 5948 * Catch if we went past the end of the current memory segment. If so,
5948 5949 * just move to the next segment with pages.
5949 5950 */
5950 5951 if (new_pp >= seg->epages || seg->pages_base == seg->pages_end) {
5951 5952 do {
5952 5953 seg = seg->next;
5953 5954 if (seg == NULL)
5954 5955 seg = memsegs;
5955 5956 } while (seg->pages_base == seg->pages_end);
5956 5957 new_pp = seg->pages;
5957 5958 *cookie = (void *)seg;
5958 5959 }
5959 5960
5960 5961 return (new_pp);
5961 5962 }
5962 5963
5963 5964
5964 5965 /*
5965 5966 * Returns next page in list. Note: this function wraps
5966 5967 * to the first page in the list upon reaching the end
5967 5968 * of the list. Callers should be aware of this fact.
5968 5969 */
5969 5970
5970 5971 /* We should change this be a #define */
5971 5972
5972 5973 page_t *
5973 5974 page_next(page_t *pp)
5974 5975 {
5975 5976 return (page_nextn(pp, 1));
5976 5977 }
5977 5978
5978 5979 page_t *
5979 5980 page_first()
5980 5981 {
5981 5982 return ((page_t *)memsegs->pages);
5982 5983 }
5983 5984
5984 5985
5985 5986 /*
5986 5987 * This routine is called at boot with the initial memory configuration
5987 5988 * and when memory is added or removed.
5988 5989 */
5989 5990 void
5990 5991 build_pfn_hash()
5991 5992 {
5992 5993 pfn_t cur;
5993 5994 pgcnt_t index;
5994 5995 struct memseg *pseg;
5995 5996 int i;
5996 5997
5997 5998 /*
5998 5999 * Clear memseg_hash array.
5999 6000 * Since memory add/delete is designed to operate concurrently
6000 6001 * with normal operation, the hash rebuild must be able to run
6001 6002 * concurrently with page_numtopp_nolock(). To support this
6002 6003 * functionality, assignments to memseg_hash array members must
6003 6004 * be done atomically.
6004 6005 *
6005 6006 * NOTE: bzero() does not currently guarantee this for kernel
6006 6007 * threads, and cannot be used here.
6007 6008 */
6008 6009 for (i = 0; i < N_MEM_SLOTS; i++)
6009 6010 memseg_hash[i] = NULL;
6010 6011
6011 6012 hat_kpm_mseghash_clear(N_MEM_SLOTS);
6012 6013
6013 6014 /*
6014 6015 * Physmax is the last valid pfn.
6015 6016 */
6016 6017 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT;
6017 6018 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) {
6018 6019 index = MEMSEG_PFN_HASH(pseg->pages_base);
6019 6020 cur = pseg->pages_base;
6020 6021 do {
6021 6022 if (index >= N_MEM_SLOTS)
6022 6023 index = MEMSEG_PFN_HASH(cur);
6023 6024
6024 6025 if (memseg_hash[index] == NULL ||
6025 6026 memseg_hash[index]->pages_base > pseg->pages_base) {
6026 6027 memseg_hash[index] = pseg;
6027 6028 hat_kpm_mseghash_update(index, pseg);
6028 6029 }
6029 6030 cur += mhash_per_slot;
6030 6031 index++;
6031 6032 } while (cur < pseg->pages_end);
6032 6033 }
6033 6034 }
6034 6035
6035 6036 /*
6036 6037 * Return the pagenum for the pp
6037 6038 */
6038 6039 pfn_t
6039 6040 page_pptonum(page_t *pp)
6040 6041 {
6041 6042 return (pp->p_pagenum);
6042 6043 }
6043 6044
6044 6045 /*
6045 6046 * interface to the referenced and modified etc bits
6046 6047 * in the PSM part of the page struct
6047 6048 * when no locking is desired.
6048 6049 */
6049 6050 void
6050 6051 page_set_props(page_t *pp, uint_t flags)
6051 6052 {
6052 6053 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0);
6053 6054 pp->p_nrm |= (uchar_t)flags;
6054 6055 }
6055 6056
6056 6057 void
6057 6058 page_clr_all_props(page_t *pp)
6058 6059 {
6059 6060 pp->p_nrm = 0;
6060 6061 }
6061 6062
6062 6063 /*
6063 6064 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
6064 6065 */
6065 6066 int
6066 6067 page_clear_lck_cow(page_t *pp, int adjust)
6067 6068 {
6068 6069 int f_amount;
6069 6070
6070 6071 ASSERT(PAGE_EXCL(pp));
6071 6072
6072 6073 /*
6073 6074 * The page_struct_lock need not be acquired here since
6074 6075 * we require the caller hold the page exclusively locked.
6075 6076 */
6076 6077 f_amount = 0;
6077 6078 if (pp->p_lckcnt) {
6078 6079 f_amount = 1;
6079 6080 pp->p_lckcnt = 0;
6080 6081 }
6081 6082 if (pp->p_cowcnt) {
6082 6083 f_amount += pp->p_cowcnt;
6083 6084 pp->p_cowcnt = 0;
6084 6085 }
6085 6086
6086 6087 if (adjust && f_amount) {
6087 6088 mutex_enter(&freemem_lock);
6088 6089 availrmem += f_amount;
6089 6090 mutex_exit(&freemem_lock);
6090 6091 }
6091 6092
6092 6093 return (f_amount);
6093 6094 }
6094 6095
6095 6096 /*
6096 6097 * The following functions is called from free_vp_pages()
6097 6098 * for an inexact estimate of a newly free'd page...
6098 6099 */
6099 6100 ulong_t
6100 6101 page_share_cnt(page_t *pp)
6101 6102 {
6102 6103 return (hat_page_getshare(pp));
6103 6104 }
6104 6105
6105 6106 int
6106 6107 page_isshared(page_t *pp)
6107 6108 {
6108 6109 return (hat_page_checkshare(pp, 1));
6109 6110 }
6110 6111
6111 6112 int
6112 6113 page_isfree(page_t *pp)
6113 6114 {
6114 6115 return (PP_ISFREE(pp));
6115 6116 }
6116 6117
6117 6118 int
6118 6119 page_isref(page_t *pp)
6119 6120 {
6120 6121 return (hat_page_getattr(pp, P_REF));
6121 6122 }
6122 6123
6123 6124 int
6124 6125 page_ismod(page_t *pp)
6125 6126 {
6126 6127 return (hat_page_getattr(pp, P_MOD));
6127 6128 }
6128 6129
6129 6130 /*
6130 6131 * The following code all currently relates to the page capture logic:
6131 6132 *
6132 6133 * This logic is used for cases where there is a desire to claim a certain
6133 6134 * physical page in the system for the caller. As it may not be possible
6134 6135 * to capture the page immediately, the p_toxic bits are used in the page
6135 6136 * structure to indicate that someone wants to capture this page. When the
6136 6137 * page gets unlocked, the toxic flag will be noted and an attempt to capture
6137 6138 * the page will be made. If it is successful, the original callers callback
6138 6139 * will be called with the page to do with it what they please.
6139 6140 *
6140 6141 * There is also an async thread which wakes up to attempt to capture
6141 6142 * pages occasionally which have the capture bit set. All of the pages which
6142 6143 * need to be captured asynchronously have been inserted into the
6143 6144 * page_capture_hash and thus this thread walks that hash list. Items in the
6144 6145 * hash have an expiration time so this thread handles that as well by removing
6145 6146 * the item from the hash if it has expired.
6146 6147 *
6147 6148 * Some important things to note are:
6148 6149 * - if the PR_CAPTURE bit is set on a page, then the page is in the
6149 6150 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed
6150 6151 * to set and clear this bit, and while the lock is held is the only time
6151 6152 * you can add or remove an entry from the hash.
6152 6153 * - the PR_CAPTURE bit can only be set and cleared while holding the
6153 6154 * page_capture_hash_head.pchh_mutex
6154 6155 * - the t_flag field of the thread struct is used with the T_CAPTURING
6155 6156 * flag to prevent recursion while dealing with large pages.
6156 6157 * - pages which need to be retired never expire on the page_capture_hash.
6157 6158 */
6158 6159
6159 6160 static void page_capture_thread(void);
6160 6161 static kthread_t *pc_thread_id;
6161 6162 kcondvar_t pc_cv;
6162 6163 static kmutex_t pc_thread_mutex;
6163 6164 static clock_t pc_thread_shortwait;
6164 6165 static clock_t pc_thread_longwait;
6165 6166 static int pc_thread_retry;
6166 6167
6167 6168 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS];
6168 6169
6169 6170 /* Note that this is a circular linked list */
6170 6171 typedef struct page_capture_hash_bucket {
6171 6172 page_t *pp;
6172 6173 uchar_t szc;
6173 6174 uchar_t pri;
6174 6175 uint_t flags;
6175 6176 clock_t expires; /* lbolt at which this request expires. */
6176 6177 void *datap; /* Cached data passed in for callback */
6177 6178 struct page_capture_hash_bucket *next;
6178 6179 struct page_capture_hash_bucket *prev;
6179 6180 } page_capture_hash_bucket_t;
6180 6181
6181 6182 #define PC_PRI_HI 0 /* capture now */
6182 6183 #define PC_PRI_LO 1 /* capture later */
6183 6184 #define PC_NUM_PRI 2
6184 6185
6185 6186 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI)
6186 6187
6187 6188
6188 6189 /*
6189 6190 * Each hash bucket will have it's own mutex and two lists which are:
6190 6191 * active (0): represents requests which have not been processed by
6191 6192 * the page_capture async thread yet.
6192 6193 * walked (1): represents requests which have been processed by the
6193 6194 * page_capture async thread within it's given walk of this bucket.
6194 6195 *
6195 6196 * These are all needed so that we can synchronize all async page_capture
6196 6197 * events. When the async thread moves to a new bucket, it will append the
6197 6198 * walked list to the active list and walk each item one at a time, moving it
6198 6199 * from the active list to the walked list. Thus if there is an async request
6199 6200 * outstanding for a given page, it will always be in one of the two lists.
6200 6201 * New requests will always be added to the active list.
6201 6202 * If we were not able to capture a page before the request expired, we'd free
6202 6203 * up the request structure which would indicate to page_capture that there is
6203 6204 * no longer a need for the given page, and clear the PR_CAPTURE flag if
6204 6205 * possible.
6205 6206 */
6206 6207 typedef struct page_capture_hash_head {
6207 6208 kmutex_t pchh_mutex;
6208 6209 uint_t num_pages[PC_NUM_PRI];
6209 6210 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */
6210 6211 } page_capture_hash_head_t;
6211 6212
6212 6213 #ifdef DEBUG
6213 6214 #define NUM_PAGE_CAPTURE_BUCKETS 4
|
↓ open down ↓ |
6181 lines elided |
↑ open up ↑ |
6214 6215 #else
6215 6216 #define NUM_PAGE_CAPTURE_BUCKETS 64
6216 6217 #endif
6217 6218
6218 6219 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS];
6219 6220
6220 6221 /* for now use a very simple hash based upon the size of a page struct */
6221 6222 #define PAGE_CAPTURE_HASH(pp) \
6222 6223 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
6223 6224
6224 -extern pgcnt_t swapfs_minfree;
6225 -
6226 6225 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
6227 6226
6228 6227 /*
6229 6228 * a callback function is required for page capture requests.
6230 6229 */
6231 6230 void
6232 6231 page_capture_register_callback(uint_t index, clock_t duration,
6233 6232 int (*cb_func)(page_t *, void *, uint_t))
6234 6233 {
6235 6234 ASSERT(pc_cb[index].cb_active == 0);
6236 6235 ASSERT(cb_func != NULL);
6237 6236 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6238 6237 pc_cb[index].duration = duration;
6239 6238 pc_cb[index].cb_func = cb_func;
6240 6239 pc_cb[index].cb_active = 1;
6241 6240 rw_exit(&pc_cb[index].cb_rwlock);
6242 6241 }
6243 6242
6244 6243 void
6245 6244 page_capture_unregister_callback(uint_t index)
6246 6245 {
6247 6246 int i, j;
6248 6247 struct page_capture_hash_bucket *bp1;
6249 6248 struct page_capture_hash_bucket *bp2;
6250 6249 struct page_capture_hash_bucket *head = NULL;
6251 6250 uint_t flags = (1 << index);
6252 6251
6253 6252 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6254 6253 ASSERT(pc_cb[index].cb_active == 1);
6255 6254 pc_cb[index].duration = 0; /* Paranoia */
6256 6255 pc_cb[index].cb_func = NULL; /* Paranoia */
6257 6256 pc_cb[index].cb_active = 0;
6258 6257 rw_exit(&pc_cb[index].cb_rwlock);
6259 6258
6260 6259 /*
6261 6260 * Just move all the entries to a private list which we can walk
6262 6261 * through without the need to hold any locks.
6263 6262 * No more requests can get added to the hash lists for this consumer
6264 6263 * as the cb_active field for the callback has been cleared.
6265 6264 */
6266 6265 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
6267 6266 mutex_enter(&page_capture_hash[i].pchh_mutex);
6268 6267 for (j = 0; j < 2; j++) {
6269 6268 bp1 = page_capture_hash[i].lists[j].next;
6270 6269 /* walk through all but first (sentinel) element */
6271 6270 while (bp1 != &page_capture_hash[i].lists[j]) {
6272 6271 bp2 = bp1;
6273 6272 if (bp2->flags & flags) {
6274 6273 bp1 = bp2->next;
6275 6274 bp1->prev = bp2->prev;
6276 6275 bp2->prev->next = bp1;
6277 6276 bp2->next = head;
6278 6277 head = bp2;
6279 6278 /*
6280 6279 * Clear the PR_CAPTURE bit as we
6281 6280 * hold appropriate locks here.
6282 6281 */
6283 6282 page_clrtoxic(head->pp, PR_CAPTURE);
6284 6283 page_capture_hash[i].
6285 6284 num_pages[bp2->pri]--;
6286 6285 continue;
6287 6286 }
6288 6287 bp1 = bp1->next;
6289 6288 }
6290 6289 }
6291 6290 mutex_exit(&page_capture_hash[i].pchh_mutex);
6292 6291 }
6293 6292
6294 6293 while (head != NULL) {
6295 6294 bp1 = head;
6296 6295 head = head->next;
6297 6296 kmem_free(bp1, sizeof (*bp1));
6298 6297 }
6299 6298 }
6300 6299
6301 6300
6302 6301 /*
6303 6302 * Find pp in the active list and move it to the walked list if it
6304 6303 * exists.
6305 6304 * Note that most often pp should be at the front of the active list
6306 6305 * as it is currently used and thus there is no other sort of optimization
6307 6306 * being done here as this is a linked list data structure.
6308 6307 * Returns 1 on successful move or 0 if page could not be found.
6309 6308 */
6310 6309 static int
6311 6310 page_capture_move_to_walked(page_t *pp)
6312 6311 {
6313 6312 page_capture_hash_bucket_t *bp;
6314 6313 int index;
6315 6314
6316 6315 index = PAGE_CAPTURE_HASH(pp);
6317 6316
6318 6317 mutex_enter(&page_capture_hash[index].pchh_mutex);
6319 6318 bp = page_capture_hash[index].lists[0].next;
6320 6319 while (bp != &page_capture_hash[index].lists[0]) {
6321 6320 if (bp->pp == pp) {
6322 6321 /* Remove from old list */
6323 6322 bp->next->prev = bp->prev;
6324 6323 bp->prev->next = bp->next;
6325 6324
6326 6325 /* Add to new list */
6327 6326 bp->next = page_capture_hash[index].lists[1].next;
6328 6327 bp->prev = &page_capture_hash[index].lists[1];
6329 6328 page_capture_hash[index].lists[1].next = bp;
6330 6329 bp->next->prev = bp;
6331 6330
6332 6331 /*
6333 6332 * There is a small probability of page on a free
6334 6333 * list being retired while being allocated
6335 6334 * and before P_RAF is set on it. The page may
6336 6335 * end up marked as high priority request instead
6337 6336 * of low priority request.
6338 6337 * If P_RAF page is not marked as low priority request
6339 6338 * change it to low priority request.
6340 6339 */
6341 6340 page_capture_hash[index].num_pages[bp->pri]--;
6342 6341 bp->pri = PAGE_CAPTURE_PRIO(pp);
6343 6342 page_capture_hash[index].num_pages[bp->pri]++;
6344 6343 mutex_exit(&page_capture_hash[index].pchh_mutex);
6345 6344 return (1);
6346 6345 }
6347 6346 bp = bp->next;
6348 6347 }
6349 6348 mutex_exit(&page_capture_hash[index].pchh_mutex);
6350 6349 return (0);
6351 6350 }
6352 6351
6353 6352 /*
6354 6353 * Add a new entry to the page capture hash. The only case where a new
6355 6354 * entry is not added is when the page capture consumer is no longer registered.
6356 6355 * In this case, we'll silently not add the page to the hash. We know that
6357 6356 * page retire will always be registered for the case where we are currently
6358 6357 * unretiring a page and thus there are no conflicts.
6359 6358 */
6360 6359 static void
6361 6360 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap)
6362 6361 {
6363 6362 page_capture_hash_bucket_t *bp1;
6364 6363 page_capture_hash_bucket_t *bp2;
6365 6364 int index;
6366 6365 int cb_index;
6367 6366 int i;
6368 6367 uchar_t pri;
6369 6368 #ifdef DEBUG
6370 6369 page_capture_hash_bucket_t *tp1;
6371 6370 int l;
6372 6371 #endif
6373 6372
6374 6373 ASSERT(!(flags & CAPTURE_ASYNC));
6375 6374
6376 6375 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP);
6377 6376
6378 6377 bp1->pp = pp;
6379 6378 bp1->szc = szc;
6380 6379 bp1->flags = flags;
6381 6380 bp1->datap = datap;
6382 6381
6383 6382 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6384 6383 if ((flags >> cb_index) & 1) {
6385 6384 break;
6386 6385 }
6387 6386 }
6388 6387
6389 6388 ASSERT(cb_index != PC_NUM_CALLBACKS);
6390 6389
6391 6390 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6392 6391 if (pc_cb[cb_index].cb_active) {
6393 6392 if (pc_cb[cb_index].duration == -1) {
6394 6393 bp1->expires = (clock_t)-1;
6395 6394 } else {
6396 6395 bp1->expires = ddi_get_lbolt() +
6397 6396 pc_cb[cb_index].duration;
6398 6397 }
6399 6398 } else {
6400 6399 /* There's no callback registered so don't add to the hash */
6401 6400 rw_exit(&pc_cb[cb_index].cb_rwlock);
6402 6401 kmem_free(bp1, sizeof (*bp1));
6403 6402 return;
6404 6403 }
6405 6404
6406 6405 index = PAGE_CAPTURE_HASH(pp);
6407 6406
6408 6407 /*
6409 6408 * Only allow capture flag to be modified under this mutex.
6410 6409 * Prevents multiple entries for same page getting added.
6411 6410 */
6412 6411 mutex_enter(&page_capture_hash[index].pchh_mutex);
6413 6412
6414 6413 /*
6415 6414 * if not already on the hash, set capture bit and add to the hash
6416 6415 */
6417 6416 if (!(pp->p_toxic & PR_CAPTURE)) {
6418 6417 #ifdef DEBUG
6419 6418 /* Check for duplicate entries */
6420 6419 for (l = 0; l < 2; l++) {
6421 6420 tp1 = page_capture_hash[index].lists[l].next;
6422 6421 while (tp1 != &page_capture_hash[index].lists[l]) {
6423 6422 if (tp1->pp == pp) {
6424 6423 panic("page pp 0x%p already on hash "
6425 6424 "at 0x%p\n",
6426 6425 (void *)pp, (void *)tp1);
6427 6426 }
6428 6427 tp1 = tp1->next;
6429 6428 }
6430 6429 }
6431 6430
6432 6431 #endif
6433 6432 page_settoxic(pp, PR_CAPTURE);
6434 6433 pri = PAGE_CAPTURE_PRIO(pp);
6435 6434 bp1->pri = pri;
6436 6435 bp1->next = page_capture_hash[index].lists[0].next;
6437 6436 bp1->prev = &page_capture_hash[index].lists[0];
6438 6437 bp1->next->prev = bp1;
6439 6438 page_capture_hash[index].lists[0].next = bp1;
6440 6439 page_capture_hash[index].num_pages[pri]++;
6441 6440 if (flags & CAPTURE_RETIRE) {
6442 6441 page_retire_incr_pend_count(datap);
6443 6442 }
6444 6443 mutex_exit(&page_capture_hash[index].pchh_mutex);
6445 6444 rw_exit(&pc_cb[cb_index].cb_rwlock);
6446 6445 cv_signal(&pc_cv);
6447 6446 return;
6448 6447 }
6449 6448
6450 6449 /*
6451 6450 * A page retire request will replace any other request.
6452 6451 * A second physmem request which is for a different process than
6453 6452 * the currently registered one will be dropped as there is
6454 6453 * no way to hold the private data for both calls.
6455 6454 * In the future, once there are more callers, this will have to
6456 6455 * be worked out better as there needs to be private storage for
6457 6456 * at least each type of caller (maybe have datap be an array of
6458 6457 * *void's so that we can index based upon callers index).
6459 6458 */
6460 6459
6461 6460 /* walk hash list to update expire time */
6462 6461 for (i = 0; i < 2; i++) {
6463 6462 bp2 = page_capture_hash[index].lists[i].next;
6464 6463 while (bp2 != &page_capture_hash[index].lists[i]) {
6465 6464 if (bp2->pp == pp) {
6466 6465 if (flags & CAPTURE_RETIRE) {
6467 6466 if (!(bp2->flags & CAPTURE_RETIRE)) {
6468 6467 page_retire_incr_pend_count(
6469 6468 datap);
6470 6469 bp2->flags = flags;
6471 6470 bp2->expires = bp1->expires;
6472 6471 bp2->datap = datap;
6473 6472 }
6474 6473 } else {
6475 6474 ASSERT(flags & CAPTURE_PHYSMEM);
6476 6475 if (!(bp2->flags & CAPTURE_RETIRE) &&
6477 6476 (datap == bp2->datap)) {
6478 6477 bp2->expires = bp1->expires;
6479 6478 }
6480 6479 }
6481 6480 mutex_exit(&page_capture_hash[index].
6482 6481 pchh_mutex);
6483 6482 rw_exit(&pc_cb[cb_index].cb_rwlock);
6484 6483 kmem_free(bp1, sizeof (*bp1));
6485 6484 return;
6486 6485 }
6487 6486 bp2 = bp2->next;
6488 6487 }
6489 6488 }
6490 6489
6491 6490 /*
6492 6491 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes
6493 6492 * and thus it either has to be set or not set and can't change
6494 6493 * while holding the mutex above.
6495 6494 */
6496 6495 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n",
6497 6496 (void *)pp);
6498 6497 }
6499 6498
6500 6499 /*
6501 6500 * We have a page in our hands, lets try and make it ours by turning
6502 6501 * it into a clean page like it had just come off the freelists.
6503 6502 *
6504 6503 * Returns 0 on success, with the page still EXCL locked.
6505 6504 * On failure, the page will be unlocked, and returns EAGAIN
6506 6505 */
6507 6506 static int
6508 6507 page_capture_clean_page(page_t *pp)
6509 6508 {
6510 6509 page_t *newpp;
6511 6510 int skip_unlock = 0;
6512 6511 spgcnt_t count;
6513 6512 page_t *tpp;
6514 6513 int ret = 0;
6515 6514 int extra;
6516 6515
6517 6516 ASSERT(PAGE_EXCL(pp));
6518 6517 ASSERT(!PP_RETIRED(pp));
6519 6518 ASSERT(curthread->t_flag & T_CAPTURING);
6520 6519
6521 6520 if (PP_ISFREE(pp)) {
6522 6521 if (!page_reclaim(pp, NULL)) {
6523 6522 skip_unlock = 1;
6524 6523 ret = EAGAIN;
6525 6524 goto cleanup;
6526 6525 }
6527 6526 ASSERT(pp->p_szc == 0);
6528 6527 if (pp->p_vnode != NULL) {
6529 6528 /*
6530 6529 * Since this page came from the
6531 6530 * cachelist, we must destroy the
6532 6531 * old vnode association.
6533 6532 */
6534 6533 page_hashout(pp, NULL);
6535 6534 }
6536 6535 goto cleanup;
6537 6536 }
6538 6537
6539 6538 /*
6540 6539 * If we know page_relocate will fail, skip it
6541 6540 * It could still fail due to a UE on another page but we
6542 6541 * can't do anything about that.
6543 6542 */
6544 6543 if (pp->p_toxic & PR_UE) {
6545 6544 goto skip_relocate;
6546 6545 }
6547 6546
6548 6547 /*
6549 6548 * It's possible that pages can not have a vnode as fsflush comes
6550 6549 * through and cleans up these pages. It's ugly but that's how it is.
6551 6550 */
6552 6551 if (pp->p_vnode == NULL) {
6553 6552 goto skip_relocate;
6554 6553 }
6555 6554
6556 6555 /*
6557 6556 * Page was not free, so lets try to relocate it.
6558 6557 * page_relocate only works with root pages, so if this is not a root
6559 6558 * page, we need to demote it to try and relocate it.
6560 6559 * Unfortunately this is the best we can do right now.
6561 6560 */
6562 6561 newpp = NULL;
6563 6562 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) {
6564 6563 if (page_try_demote_pages(pp) == 0) {
6565 6564 ret = EAGAIN;
6566 6565 goto cleanup;
6567 6566 }
6568 6567 }
6569 6568 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL);
6570 6569 if (ret == 0) {
6571 6570 page_t *npp;
6572 6571 /* unlock the new page(s) */
6573 6572 while (count-- > 0) {
6574 6573 ASSERT(newpp != NULL);
6575 6574 npp = newpp;
6576 6575 page_sub(&newpp, npp);
6577 6576 page_unlock(npp);
6578 6577 }
6579 6578 ASSERT(newpp == NULL);
6580 6579 /*
6581 6580 * Check to see if the page we have is too large.
6582 6581 * If so, demote it freeing up the extra pages.
6583 6582 */
6584 6583 if (pp->p_szc > 0) {
6585 6584 /* For now demote extra pages to szc == 0 */
6586 6585 extra = page_get_pagecnt(pp->p_szc) - 1;
6587 6586 while (extra > 0) {
6588 6587 tpp = pp->p_next;
6589 6588 page_sub(&pp, tpp);
6590 6589 tpp->p_szc = 0;
6591 6590 page_free(tpp, 1);
6592 6591 extra--;
6593 6592 }
6594 6593 /* Make sure to set our page to szc 0 as well */
6595 6594 ASSERT(pp->p_next == pp && pp->p_prev == pp);
6596 6595 pp->p_szc = 0;
6597 6596 }
6598 6597 goto cleanup;
6599 6598 } else if (ret == EIO) {
6600 6599 ret = EAGAIN;
6601 6600 goto cleanup;
6602 6601 } else {
6603 6602 /*
6604 6603 * Need to reset return type as we failed to relocate the page
6605 6604 * but that does not mean that some of the next steps will not
6606 6605 * work.
6607 6606 */
6608 6607 ret = 0;
6609 6608 }
6610 6609
6611 6610 skip_relocate:
6612 6611
6613 6612 if (pp->p_szc > 0) {
6614 6613 if (page_try_demote_pages(pp) == 0) {
6615 6614 ret = EAGAIN;
6616 6615 goto cleanup;
6617 6616 }
6618 6617 }
6619 6618
6620 6619 ASSERT(pp->p_szc == 0);
6621 6620
6622 6621 if (hat_ismod(pp)) {
6623 6622 ret = EAGAIN;
6624 6623 goto cleanup;
6625 6624 }
6626 6625 if (PP_ISKAS(pp)) {
6627 6626 ret = EAGAIN;
6628 6627 goto cleanup;
6629 6628 }
6630 6629 if (pp->p_lckcnt || pp->p_cowcnt) {
6631 6630 ret = EAGAIN;
6632 6631 goto cleanup;
6633 6632 }
6634 6633
6635 6634 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
6636 6635 ASSERT(!hat_page_is_mapped(pp));
6637 6636
6638 6637 if (hat_ismod(pp)) {
6639 6638 /*
6640 6639 * This is a semi-odd case as the page is now modified but not
6641 6640 * mapped as we just unloaded the mappings above.
6642 6641 */
6643 6642 ret = EAGAIN;
6644 6643 goto cleanup;
6645 6644 }
6646 6645 if (pp->p_vnode != NULL) {
6647 6646 page_hashout(pp, NULL);
6648 6647 }
6649 6648
6650 6649 /*
6651 6650 * At this point, the page should be in a clean state and
6652 6651 * we can do whatever we want with it.
6653 6652 */
6654 6653
6655 6654 cleanup:
6656 6655 if (ret != 0) {
6657 6656 if (!skip_unlock) {
6658 6657 page_unlock(pp);
6659 6658 }
6660 6659 } else {
6661 6660 ASSERT(pp->p_szc == 0);
6662 6661 ASSERT(PAGE_EXCL(pp));
6663 6662
6664 6663 pp->p_next = pp;
6665 6664 pp->p_prev = pp;
6666 6665 }
6667 6666 return (ret);
6668 6667 }
6669 6668
6670 6669 /*
6671 6670 * Various callers of page_trycapture() can have different restrictions upon
6672 6671 * what memory they have access to.
6673 6672 * Returns 0 on success, with the following error codes on failure:
6674 6673 * EPERM - The requested page is long term locked, and thus repeated
6675 6674 * requests to capture this page will likely fail.
6676 6675 * ENOMEM - There was not enough free memory in the system to safely
6677 6676 * map the requested page.
6678 6677 * ENOENT - The requested page was inside the kernel cage, and the
6679 6678 * PHYSMEM_CAGE flag was not set.
6680 6679 */
6681 6680 int
6682 6681 page_capture_pre_checks(page_t *pp, uint_t flags)
6683 6682 {
6684 6683 ASSERT(pp != NULL);
6685 6684
6686 6685 #if defined(__sparc)
6687 6686 if (pp->p_vnode == &promvp) {
6688 6687 return (EPERM);
6689 6688 }
6690 6689
6691 6690 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) &&
6692 6691 (flags & CAPTURE_PHYSMEM)) {
6693 6692 return (ENOENT);
6694 6693 }
6695 6694
6696 6695 if (PP_ISNORELOCKERNEL(pp)) {
6697 6696 return (EPERM);
6698 6697 }
6699 6698 #else
6700 6699 if (PP_ISKAS(pp)) {
6701 6700 return (EPERM);
6702 6701 }
6703 6702 #endif /* __sparc */
6704 6703
6705 6704 /* only physmem currently has the restrictions checked below */
6706 6705 if (!(flags & CAPTURE_PHYSMEM)) {
6707 6706 return (0);
6708 6707 }
6709 6708
6710 6709 if (availrmem < swapfs_minfree) {
6711 6710 /*
6712 6711 * We won't try to capture this page as we are
6713 6712 * running low on memory.
6714 6713 */
6715 6714 return (ENOMEM);
6716 6715 }
6717 6716 return (0);
6718 6717 }
6719 6718
6720 6719 /*
6721 6720 * Once we have a page in our mits, go ahead and complete the capture
6722 6721 * operation.
6723 6722 * Returns 1 on failure where page is no longer needed
6724 6723 * Returns 0 on success
6725 6724 * Returns -1 if there was a transient failure.
6726 6725 * Failure cases must release the SE_EXCL lock on pp (usually via page_free).
6727 6726 */
6728 6727 int
6729 6728 page_capture_take_action(page_t *pp, uint_t flags, void *datap)
6730 6729 {
6731 6730 int cb_index;
6732 6731 int ret = 0;
6733 6732 page_capture_hash_bucket_t *bp1;
6734 6733 page_capture_hash_bucket_t *bp2;
6735 6734 int index;
6736 6735 int found = 0;
6737 6736 int i;
6738 6737
6739 6738 ASSERT(PAGE_EXCL(pp));
6740 6739 ASSERT(curthread->t_flag & T_CAPTURING);
6741 6740
6742 6741 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6743 6742 if ((flags >> cb_index) & 1) {
6744 6743 break;
6745 6744 }
6746 6745 }
6747 6746 ASSERT(cb_index < PC_NUM_CALLBACKS);
6748 6747
6749 6748 /*
6750 6749 * Remove the entry from the page_capture hash, but don't free it yet
6751 6750 * as we may need to put it back.
6752 6751 * Since we own the page at this point in time, we should find it
6753 6752 * in the hash if this is an ASYNC call. If we don't it's likely
6754 6753 * that the page_capture_async() thread decided that this request
6755 6754 * had expired, in which case we just continue on.
6756 6755 */
6757 6756 if (flags & CAPTURE_ASYNC) {
6758 6757
6759 6758 index = PAGE_CAPTURE_HASH(pp);
6760 6759
6761 6760 mutex_enter(&page_capture_hash[index].pchh_mutex);
6762 6761 for (i = 0; i < 2 && !found; i++) {
6763 6762 bp1 = page_capture_hash[index].lists[i].next;
6764 6763 while (bp1 != &page_capture_hash[index].lists[i]) {
6765 6764 if (bp1->pp == pp) {
6766 6765 bp1->next->prev = bp1->prev;
6767 6766 bp1->prev->next = bp1->next;
6768 6767 page_capture_hash[index].
6769 6768 num_pages[bp1->pri]--;
6770 6769 page_clrtoxic(pp, PR_CAPTURE);
6771 6770 found = 1;
6772 6771 break;
6773 6772 }
6774 6773 bp1 = bp1->next;
6775 6774 }
6776 6775 }
6777 6776 mutex_exit(&page_capture_hash[index].pchh_mutex);
6778 6777 }
6779 6778
6780 6779 /* Synchronize with the unregister func. */
6781 6780 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6782 6781 if (!pc_cb[cb_index].cb_active) {
6783 6782 page_free(pp, 1);
6784 6783 rw_exit(&pc_cb[cb_index].cb_rwlock);
6785 6784 if (found) {
6786 6785 kmem_free(bp1, sizeof (*bp1));
6787 6786 }
6788 6787 return (1);
6789 6788 }
6790 6789
6791 6790 /*
6792 6791 * We need to remove the entry from the page capture hash and turn off
6793 6792 * the PR_CAPTURE bit before calling the callback. We'll need to cache
6794 6793 * the entry here, and then based upon the return value, cleanup
6795 6794 * appropriately or re-add it to the hash, making sure that someone else
6796 6795 * hasn't already done so.
6797 6796 * It should be rare for the callback to fail and thus it's ok for
6798 6797 * the failure path to be a bit complicated as the success path is
6799 6798 * cleaner and the locking rules are easier to follow.
6800 6799 */
6801 6800
6802 6801 ret = pc_cb[cb_index].cb_func(pp, datap, flags);
6803 6802
6804 6803 rw_exit(&pc_cb[cb_index].cb_rwlock);
6805 6804
6806 6805 /*
6807 6806 * If this was an ASYNC request, we need to cleanup the hash if the
6808 6807 * callback was successful or if the request was no longer valid.
6809 6808 * For non-ASYNC requests, we return failure to map and the caller
6810 6809 * will take care of adding the request to the hash.
6811 6810 * Note also that the callback itself is responsible for the page
6812 6811 * at this point in time in terms of locking ... The most common
6813 6812 * case for the failure path should just be a page_free.
6814 6813 */
6815 6814 if (ret >= 0) {
6816 6815 if (found) {
6817 6816 if (bp1->flags & CAPTURE_RETIRE) {
6818 6817 page_retire_decr_pend_count(datap);
6819 6818 }
6820 6819 kmem_free(bp1, sizeof (*bp1));
6821 6820 }
6822 6821 return (ret);
6823 6822 }
6824 6823 if (!found) {
6825 6824 return (ret);
6826 6825 }
6827 6826
6828 6827 ASSERT(flags & CAPTURE_ASYNC);
6829 6828
6830 6829 /*
6831 6830 * Check for expiration time first as we can just free it up if it's
6832 6831 * expired.
6833 6832 */
6834 6833 if (ddi_get_lbolt() > bp1->expires && bp1->expires != -1) {
6835 6834 kmem_free(bp1, sizeof (*bp1));
6836 6835 return (ret);
6837 6836 }
6838 6837
6839 6838 /*
6840 6839 * The callback failed and there used to be an entry in the hash for
6841 6840 * this page, so we need to add it back to the hash.
6842 6841 */
6843 6842 mutex_enter(&page_capture_hash[index].pchh_mutex);
6844 6843 if (!(pp->p_toxic & PR_CAPTURE)) {
6845 6844 /* just add bp1 back to head of walked list */
6846 6845 page_settoxic(pp, PR_CAPTURE);
6847 6846 bp1->next = page_capture_hash[index].lists[1].next;
6848 6847 bp1->prev = &page_capture_hash[index].lists[1];
6849 6848 bp1->next->prev = bp1;
6850 6849 bp1->pri = PAGE_CAPTURE_PRIO(pp);
6851 6850 page_capture_hash[index].lists[1].next = bp1;
6852 6851 page_capture_hash[index].num_pages[bp1->pri]++;
6853 6852 mutex_exit(&page_capture_hash[index].pchh_mutex);
6854 6853 return (ret);
6855 6854 }
6856 6855
6857 6856 /*
6858 6857 * Otherwise there was a new capture request added to list
6859 6858 * Need to make sure that our original data is represented if
6860 6859 * appropriate.
6861 6860 */
6862 6861 for (i = 0; i < 2; i++) {
6863 6862 bp2 = page_capture_hash[index].lists[i].next;
6864 6863 while (bp2 != &page_capture_hash[index].lists[i]) {
6865 6864 if (bp2->pp == pp) {
6866 6865 if (bp1->flags & CAPTURE_RETIRE) {
6867 6866 if (!(bp2->flags & CAPTURE_RETIRE)) {
6868 6867 bp2->szc = bp1->szc;
6869 6868 bp2->flags = bp1->flags;
6870 6869 bp2->expires = bp1->expires;
6871 6870 bp2->datap = bp1->datap;
6872 6871 }
6873 6872 } else {
6874 6873 ASSERT(bp1->flags & CAPTURE_PHYSMEM);
6875 6874 if (!(bp2->flags & CAPTURE_RETIRE)) {
6876 6875 bp2->szc = bp1->szc;
6877 6876 bp2->flags = bp1->flags;
6878 6877 bp2->expires = bp1->expires;
6879 6878 bp2->datap = bp1->datap;
6880 6879 }
6881 6880 }
6882 6881 page_capture_hash[index].num_pages[bp2->pri]--;
6883 6882 bp2->pri = PAGE_CAPTURE_PRIO(pp);
6884 6883 page_capture_hash[index].num_pages[bp2->pri]++;
6885 6884 mutex_exit(&page_capture_hash[index].
6886 6885 pchh_mutex);
6887 6886 kmem_free(bp1, sizeof (*bp1));
6888 6887 return (ret);
6889 6888 }
6890 6889 bp2 = bp2->next;
6891 6890 }
6892 6891 }
6893 6892 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp);
6894 6893 /*NOTREACHED*/
6895 6894 }
6896 6895
6897 6896 /*
6898 6897 * Try to capture the given page for the caller specified in the flags
6899 6898 * parameter. The page will either be captured and handed over to the
6900 6899 * appropriate callback, or will be queued up in the page capture hash
6901 6900 * to be captured asynchronously.
6902 6901 * If the current request is due to an async capture, the page must be
6903 6902 * exclusively locked before calling this function.
6904 6903 * Currently szc must be 0 but in the future this should be expandable to
6905 6904 * other page sizes.
6906 6905 * Returns 0 on success, with the following error codes on failure:
6907 6906 * EPERM - The requested page is long term locked, and thus repeated
6908 6907 * requests to capture this page will likely fail.
6909 6908 * ENOMEM - There was not enough free memory in the system to safely
6910 6909 * map the requested page.
6911 6910 * ENOENT - The requested page was inside the kernel cage, and the
6912 6911 * CAPTURE_GET_CAGE flag was not set.
6913 6912 * EAGAIN - The requested page could not be capturead at this point in
6914 6913 * time but future requests will likely work.
6915 6914 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag
6916 6915 * was not set.
6917 6916 */
6918 6917 int
6919 6918 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
6920 6919 {
6921 6920 int ret;
6922 6921 int cb_index;
6923 6922
6924 6923 if (flags & CAPTURE_ASYNC) {
6925 6924 ASSERT(PAGE_EXCL(pp));
6926 6925 goto async;
6927 6926 }
6928 6927
6929 6928 /* Make sure there's enough availrmem ... */
6930 6929 ret = page_capture_pre_checks(pp, flags);
6931 6930 if (ret != 0) {
6932 6931 return (ret);
6933 6932 }
6934 6933
6935 6934 if (!page_trylock(pp, SE_EXCL)) {
6936 6935 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6937 6936 if ((flags >> cb_index) & 1) {
6938 6937 break;
6939 6938 }
6940 6939 }
6941 6940 ASSERT(cb_index < PC_NUM_CALLBACKS);
6942 6941 ret = EAGAIN;
6943 6942 /* Special case for retired pages */
6944 6943 if (PP_RETIRED(pp)) {
6945 6944 if (flags & CAPTURE_GET_RETIRED) {
6946 6945 if (!page_unretire_pp(pp, PR_UNR_TEMP)) {
6947 6946 /*
6948 6947 * Need to set capture bit and add to
6949 6948 * hash so that the page will be
6950 6949 * retired when freed.
6951 6950 */
6952 6951 page_capture_add_hash(pp, szc,
6953 6952 CAPTURE_RETIRE, NULL);
6954 6953 ret = 0;
6955 6954 goto own_page;
6956 6955 }
6957 6956 } else {
6958 6957 return (EBUSY);
6959 6958 }
6960 6959 }
6961 6960 page_capture_add_hash(pp, szc, flags, datap);
6962 6961 return (ret);
6963 6962 }
6964 6963
6965 6964 async:
6966 6965 ASSERT(PAGE_EXCL(pp));
6967 6966
6968 6967 /* Need to check for physmem async requests that availrmem is sane */
6969 6968 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) ==
6970 6969 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) &&
6971 6970 (availrmem < swapfs_minfree)) {
6972 6971 page_unlock(pp);
6973 6972 return (ENOMEM);
6974 6973 }
6975 6974
6976 6975 ret = page_capture_clean_page(pp);
6977 6976
6978 6977 if (ret != 0) {
6979 6978 /* We failed to get the page, so lets add it to the hash */
6980 6979 if (!(flags & CAPTURE_ASYNC)) {
6981 6980 page_capture_add_hash(pp, szc, flags, datap);
6982 6981 }
6983 6982 return (ret);
6984 6983 }
6985 6984
6986 6985 own_page:
6987 6986 ASSERT(PAGE_EXCL(pp));
6988 6987 ASSERT(pp->p_szc == 0);
6989 6988
6990 6989 /* Call the callback */
6991 6990 ret = page_capture_take_action(pp, flags, datap);
6992 6991
6993 6992 if (ret == 0) {
6994 6993 return (0);
6995 6994 }
6996 6995
6997 6996 /*
6998 6997 * Note that in the failure cases from page_capture_take_action, the
6999 6998 * EXCL lock will have already been dropped.
7000 6999 */
7001 7000 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) {
7002 7001 page_capture_add_hash(pp, szc, flags, datap);
7003 7002 }
7004 7003 return (EAGAIN);
7005 7004 }
7006 7005
7007 7006 int
7008 7007 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
7009 7008 {
7010 7009 int ret;
7011 7010
7012 7011 curthread->t_flag |= T_CAPTURING;
7013 7012 ret = page_itrycapture(pp, szc, flags, datap);
7014 7013 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */
7015 7014 return (ret);
7016 7015 }
7017 7016
7018 7017 /*
7019 7018 * When unlocking a page which has the PR_CAPTURE bit set, this routine
7020 7019 * gets called to try and capture the page.
7021 7020 */
7022 7021 void
7023 7022 page_unlock_capture(page_t *pp)
7024 7023 {
7025 7024 page_capture_hash_bucket_t *bp;
7026 7025 int index;
7027 7026 int i;
7028 7027 uint_t szc;
7029 7028 uint_t flags = 0;
7030 7029 void *datap;
7031 7030 kmutex_t *mp;
7032 7031 extern vnode_t retired_pages;
7033 7032
7034 7033 /*
7035 7034 * We need to protect against a possible deadlock here where we own
7036 7035 * the vnode page hash mutex and want to acquire it again as there
7037 7036 * are locations in the code, where we unlock a page while holding
7038 7037 * the mutex which can lead to the page being captured and eventually
7039 7038 * end up here. As we may be hashing out the old page and hashing into
7040 7039 * the retire vnode, we need to make sure we don't own them.
7041 7040 * Other callbacks who do hash operations also need to make sure that
7042 7041 * before they hashin to a vnode that they do not currently own the
7043 7042 * vphm mutex otherwise there will be a panic.
7044 7043 */
7045 7044 if (mutex_owned(page_vnode_mutex(&retired_pages))) {
7046 7045 page_unlock_nocapture(pp);
7047 7046 return;
7048 7047 }
7049 7048 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) {
7050 7049 page_unlock_nocapture(pp);
7051 7050 return;
7052 7051 }
7053 7052
7054 7053 index = PAGE_CAPTURE_HASH(pp);
7055 7054
7056 7055 mp = &page_capture_hash[index].pchh_mutex;
7057 7056 mutex_enter(mp);
7058 7057 for (i = 0; i < 2; i++) {
7059 7058 bp = page_capture_hash[index].lists[i].next;
7060 7059 while (bp != &page_capture_hash[index].lists[i]) {
7061 7060 if (bp->pp == pp) {
7062 7061 szc = bp->szc;
7063 7062 flags = bp->flags | CAPTURE_ASYNC;
7064 7063 datap = bp->datap;
7065 7064 mutex_exit(mp);
7066 7065 (void) page_trycapture(pp, szc, flags, datap);
7067 7066 return;
7068 7067 }
7069 7068 bp = bp->next;
7070 7069 }
7071 7070 }
7072 7071
7073 7072 /* Failed to find page in hash so clear flags and unlock it. */
7074 7073 page_clrtoxic(pp, PR_CAPTURE);
7075 7074 page_unlock(pp);
7076 7075
7077 7076 mutex_exit(mp);
7078 7077 }
7079 7078
7080 7079 void
7081 7080 page_capture_init()
7082 7081 {
7083 7082 int i;
7084 7083 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7085 7084 page_capture_hash[i].lists[0].next =
7086 7085 &page_capture_hash[i].lists[0];
7087 7086 page_capture_hash[i].lists[0].prev =
7088 7087 &page_capture_hash[i].lists[0];
7089 7088 page_capture_hash[i].lists[1].next =
7090 7089 &page_capture_hash[i].lists[1];
7091 7090 page_capture_hash[i].lists[1].prev =
7092 7091 &page_capture_hash[i].lists[1];
7093 7092 }
7094 7093
7095 7094 pc_thread_shortwait = 23 * hz;
7096 7095 pc_thread_longwait = 1201 * hz;
7097 7096 pc_thread_retry = 3;
7098 7097 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL);
7099 7098 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL);
7100 7099 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0,
7101 7100 TS_RUN, minclsyspri);
7102 7101 }
7103 7102
7104 7103 /*
7105 7104 * It is necessary to scrub any failing pages prior to reboot in order to
7106 7105 * prevent a latent error trap from occurring on the next boot.
7107 7106 */
7108 7107 void
7109 7108 page_retire_mdboot()
7110 7109 {
7111 7110 page_t *pp;
7112 7111 int i, j;
7113 7112 page_capture_hash_bucket_t *bp;
7114 7113 uchar_t pri;
7115 7114
7116 7115 /* walk lists looking for pages to scrub */
7117 7116 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7118 7117 for (pri = 0; pri < PC_NUM_PRI; pri++) {
7119 7118 if (page_capture_hash[i].num_pages[pri] != 0) {
7120 7119 break;
7121 7120 }
7122 7121 }
7123 7122 if (pri == PC_NUM_PRI)
7124 7123 continue;
7125 7124
7126 7125 mutex_enter(&page_capture_hash[i].pchh_mutex);
7127 7126
7128 7127 for (j = 0; j < 2; j++) {
7129 7128 bp = page_capture_hash[i].lists[j].next;
7130 7129 while (bp != &page_capture_hash[i].lists[j]) {
7131 7130 pp = bp->pp;
7132 7131 if (PP_TOXIC(pp)) {
7133 7132 if (page_trylock(pp, SE_EXCL)) {
7134 7133 PP_CLRFREE(pp);
7135 7134 pagescrub(pp, 0, PAGESIZE);
7136 7135 page_unlock(pp);
7137 7136 }
7138 7137 }
7139 7138 bp = bp->next;
7140 7139 }
7141 7140 }
7142 7141 mutex_exit(&page_capture_hash[i].pchh_mutex);
7143 7142 }
7144 7143 }
7145 7144
7146 7145 /*
7147 7146 * Walk the page_capture_hash trying to capture pages and also cleanup old
7148 7147 * entries which have expired.
7149 7148 */
7150 7149 void
7151 7150 page_capture_async()
7152 7151 {
7153 7152 page_t *pp;
7154 7153 int i;
7155 7154 int ret;
7156 7155 page_capture_hash_bucket_t *bp1, *bp2;
7157 7156 uint_t szc;
7158 7157 uint_t flags;
7159 7158 void *datap;
7160 7159 uchar_t pri;
7161 7160
7162 7161 /* If there are outstanding pages to be captured, get to work */
7163 7162 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7164 7163 for (pri = 0; pri < PC_NUM_PRI; pri++) {
7165 7164 if (page_capture_hash[i].num_pages[pri] != 0)
7166 7165 break;
7167 7166 }
7168 7167 if (pri == PC_NUM_PRI)
7169 7168 continue;
7170 7169
7171 7170 /* Append list 1 to list 0 and then walk through list 0 */
7172 7171 mutex_enter(&page_capture_hash[i].pchh_mutex);
7173 7172 bp1 = &page_capture_hash[i].lists[1];
7174 7173 bp2 = bp1->next;
7175 7174 if (bp1 != bp2) {
7176 7175 bp1->prev->next = page_capture_hash[i].lists[0].next;
7177 7176 bp2->prev = &page_capture_hash[i].lists[0];
7178 7177 page_capture_hash[i].lists[0].next->prev = bp1->prev;
7179 7178 page_capture_hash[i].lists[0].next = bp2;
7180 7179 bp1->next = bp1;
7181 7180 bp1->prev = bp1;
7182 7181 }
7183 7182
7184 7183 /* list[1] will be empty now */
7185 7184
7186 7185 bp1 = page_capture_hash[i].lists[0].next;
7187 7186 while (bp1 != &page_capture_hash[i].lists[0]) {
7188 7187 /* Check expiration time */
7189 7188 if ((ddi_get_lbolt() > bp1->expires &&
7190 7189 bp1->expires != -1) ||
7191 7190 page_deleted(bp1->pp)) {
7192 7191 page_capture_hash[i].lists[0].next = bp1->next;
7193 7192 bp1->next->prev =
7194 7193 &page_capture_hash[i].lists[0];
7195 7194 page_capture_hash[i].num_pages[bp1->pri]--;
7196 7195
7197 7196 /*
7198 7197 * We can safely remove the PR_CAPTURE bit
7199 7198 * without holding the EXCL lock on the page
7200 7199 * as the PR_CAPTURE bit requres that the
7201 7200 * page_capture_hash[].pchh_mutex be held
7202 7201 * to modify it.
7203 7202 */
7204 7203 page_clrtoxic(bp1->pp, PR_CAPTURE);
7205 7204 mutex_exit(&page_capture_hash[i].pchh_mutex);
7206 7205 kmem_free(bp1, sizeof (*bp1));
7207 7206 mutex_enter(&page_capture_hash[i].pchh_mutex);
7208 7207 bp1 = page_capture_hash[i].lists[0].next;
7209 7208 continue;
7210 7209 }
7211 7210 pp = bp1->pp;
7212 7211 szc = bp1->szc;
7213 7212 flags = bp1->flags;
7214 7213 datap = bp1->datap;
7215 7214 mutex_exit(&page_capture_hash[i].pchh_mutex);
7216 7215 if (page_trylock(pp, SE_EXCL)) {
7217 7216 ret = page_trycapture(pp, szc,
7218 7217 flags | CAPTURE_ASYNC, datap);
7219 7218 } else {
7220 7219 ret = 1; /* move to walked hash */
7221 7220 }
7222 7221
7223 7222 if (ret != 0) {
7224 7223 /* Move to walked hash */
7225 7224 (void) page_capture_move_to_walked(pp);
7226 7225 }
7227 7226 mutex_enter(&page_capture_hash[i].pchh_mutex);
7228 7227 bp1 = page_capture_hash[i].lists[0].next;
7229 7228 }
7230 7229
7231 7230 mutex_exit(&page_capture_hash[i].pchh_mutex);
7232 7231 }
7233 7232 }
7234 7233
7235 7234 /*
7236 7235 * This function is called by the page_capture_thread, and is needed in
7237 7236 * in order to initiate aio cleanup, so that pages used in aio
7238 7237 * will be unlocked and subsequently retired by page_capture_thread.
7239 7238 */
7240 7239 static int
7241 7240 do_aio_cleanup(void)
7242 7241 {
7243 7242 proc_t *procp;
7244 7243 int (*aio_cleanup_dr_delete_memory)(proc_t *);
7245 7244 int cleaned = 0;
7246 7245
7247 7246 if (modload("sys", "kaio") == -1) {
7248 7247 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio");
7249 7248 return (0);
7250 7249 }
7251 7250 /*
7252 7251 * We use the aio_cleanup_dr_delete_memory function to
7253 7252 * initiate the actual clean up; this function will wake
7254 7253 * up the per-process aio_cleanup_thread.
7255 7254 */
7256 7255 aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
7257 7256 modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
7258 7257 if (aio_cleanup_dr_delete_memory == NULL) {
7259 7258 cmn_err(CE_WARN,
7260 7259 "aio_cleanup_dr_delete_memory not found in kaio");
7261 7260 return (0);
7262 7261 }
7263 7262 mutex_enter(&pidlock);
7264 7263 for (procp = practive; (procp != NULL); procp = procp->p_next) {
7265 7264 mutex_enter(&procp->p_lock);
7266 7265 if (procp->p_aio != NULL) {
7267 7266 /* cleanup proc's outstanding kaio */
7268 7267 cleaned += (*aio_cleanup_dr_delete_memory)(procp);
7269 7268 }
7270 7269 mutex_exit(&procp->p_lock);
7271 7270 }
7272 7271 mutex_exit(&pidlock);
7273 7272 return (cleaned);
7274 7273 }
7275 7274
7276 7275 /*
7277 7276 * helper function for page_capture_thread
7278 7277 */
7279 7278 static void
7280 7279 page_capture_handle_outstanding(void)
7281 7280 {
7282 7281 int ntry;
7283 7282
7284 7283 /* Reap pages before attempting capture pages */
7285 7284 kmem_reap();
7286 7285
7287 7286 if ((page_retire_pend_count() > page_retire_pend_kas_count()) &&
7288 7287 hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
7289 7288 /*
7290 7289 * Note: Purging only for platforms that support
7291 7290 * ISM hat_pageunload() - mainly SPARC. On x86/x64
7292 7291 * platforms ISM pages SE_SHARED locked until destroyed.
7293 7292 */
7294 7293
7295 7294 /* disable and purge seg_pcache */
7296 7295 (void) seg_p_disable();
7297 7296 for (ntry = 0; ntry < pc_thread_retry; ntry++) {
7298 7297 if (!page_retire_pend_count())
7299 7298 break;
7300 7299 if (do_aio_cleanup()) {
7301 7300 /*
7302 7301 * allow the apps cleanup threads
7303 7302 * to run
7304 7303 */
7305 7304 delay(pc_thread_shortwait);
7306 7305 }
7307 7306 page_capture_async();
7308 7307 }
7309 7308 /* reenable seg_pcache */
7310 7309 seg_p_enable();
7311 7310
7312 7311 /* completed what can be done. break out */
7313 7312 return;
7314 7313 }
7315 7314
7316 7315 /*
7317 7316 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap
7318 7317 * and then attempt to capture.
7319 7318 */
7320 7319 seg_preap();
7321 7320 page_capture_async();
7322 7321 }
7323 7322
7324 7323 /*
7325 7324 * The page_capture_thread loops forever, looking to see if there are
7326 7325 * pages still waiting to be captured.
7327 7326 */
7328 7327 static void
7329 7328 page_capture_thread(void)
7330 7329 {
7331 7330 callb_cpr_t c;
7332 7331 int i;
7333 7332 int high_pri_pages;
7334 7333 int low_pri_pages;
7335 7334 clock_t timeout;
7336 7335
7337 7336 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture");
7338 7337
7339 7338 mutex_enter(&pc_thread_mutex);
7340 7339 for (;;) {
7341 7340 high_pri_pages = 0;
7342 7341 low_pri_pages = 0;
7343 7342 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7344 7343 high_pri_pages +=
7345 7344 page_capture_hash[i].num_pages[PC_PRI_HI];
7346 7345 low_pri_pages +=
7347 7346 page_capture_hash[i].num_pages[PC_PRI_LO];
7348 7347 }
7349 7348
7350 7349 timeout = pc_thread_longwait;
7351 7350 if (high_pri_pages != 0) {
7352 7351 timeout = pc_thread_shortwait;
7353 7352 page_capture_handle_outstanding();
7354 7353 } else if (low_pri_pages != 0) {
7355 7354 page_capture_async();
7356 7355 }
7357 7356 CALLB_CPR_SAFE_BEGIN(&c);
7358 7357 (void) cv_reltimedwait(&pc_cv, &pc_thread_mutex,
7359 7358 timeout, TR_CLOCK_TICK);
7360 7359 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
7361 7360 }
7362 7361 /*NOTREACHED*/
7363 7362 }
7364 7363 /*
7365 7364 * Attempt to locate a bucket that has enough pages to satisfy the request.
7366 7365 * The initial check is done without the lock to avoid unneeded contention.
7367 7366 * The function returns 1 if enough pages were found, else 0 if it could not
7368 7367 * find enough pages in a bucket.
7369 7368 */
7370 7369 static int
7371 7370 pcf_decrement_bucket(pgcnt_t npages)
7372 7371 {
7373 7372 struct pcf *p;
7374 7373 struct pcf *q;
7375 7374 int i;
7376 7375
7377 7376 p = &pcf[PCF_INDEX()];
7378 7377 q = &pcf[pcf_fanout];
7379 7378 for (i = 0; i < pcf_fanout; i++) {
7380 7379 if (p->pcf_count > npages) {
7381 7380 /*
7382 7381 * a good one to try.
7383 7382 */
7384 7383 mutex_enter(&p->pcf_lock);
7385 7384 if (p->pcf_count > npages) {
7386 7385 p->pcf_count -= (uint_t)npages;
7387 7386 /*
7388 7387 * freemem is not protected by any lock.
7389 7388 * Thus, we cannot have any assertion
7390 7389 * containing freemem here.
7391 7390 */
7392 7391 freemem -= npages;
7393 7392 mutex_exit(&p->pcf_lock);
7394 7393 return (1);
7395 7394 }
7396 7395 mutex_exit(&p->pcf_lock);
7397 7396 }
7398 7397 p++;
7399 7398 if (p >= q) {
7400 7399 p = pcf;
7401 7400 }
7402 7401 }
7403 7402 return (0);
7404 7403 }
7405 7404
7406 7405 /*
7407 7406 * Arguments:
7408 7407 * pcftotal_ret: If the value is not NULL and we have walked all the
7409 7408 * buckets but did not find enough pages then it will
7410 7409 * be set to the total number of pages in all the pcf
7411 7410 * buckets.
7412 7411 * npages: Is the number of pages we have been requested to
7413 7412 * find.
7414 7413 * unlock: If set to 0 we will leave the buckets locked if the
7415 7414 * requested number of pages are not found.
7416 7415 *
7417 7416 * Go and try to satisfy the page request from any number of buckets.
7418 7417 * This can be a very expensive operation as we have to lock the buckets
7419 7418 * we are checking (and keep them locked), starting at bucket 0.
7420 7419 *
7421 7420 * The function returns 1 if enough pages were found, else 0 if it could not
7422 7421 * find enough pages in the buckets.
7423 7422 *
7424 7423 */
7425 7424 static int
7426 7425 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock)
7427 7426 {
7428 7427 struct pcf *p;
7429 7428 pgcnt_t pcftotal;
7430 7429 int i;
7431 7430
7432 7431 p = pcf;
7433 7432 /* try to collect pages from several pcf bins */
7434 7433 for (pcftotal = 0, i = 0; i < pcf_fanout; i++) {
7435 7434 mutex_enter(&p->pcf_lock);
7436 7435 pcftotal += p->pcf_count;
7437 7436 if (pcftotal >= npages) {
7438 7437 /*
7439 7438 * Wow! There are enough pages laying around
7440 7439 * to satisfy the request. Do the accounting,
7441 7440 * drop the locks we acquired, and go back.
7442 7441 *
7443 7442 * freemem is not protected by any lock. So,
7444 7443 * we cannot have any assertion containing
7445 7444 * freemem.
7446 7445 */
7447 7446 freemem -= npages;
7448 7447 while (p >= pcf) {
7449 7448 if (p->pcf_count <= npages) {
7450 7449 npages -= p->pcf_count;
7451 7450 p->pcf_count = 0;
7452 7451 } else {
7453 7452 p->pcf_count -= (uint_t)npages;
7454 7453 npages = 0;
7455 7454 }
7456 7455 mutex_exit(&p->pcf_lock);
7457 7456 p--;
7458 7457 }
7459 7458 ASSERT(npages == 0);
7460 7459 return (1);
7461 7460 }
7462 7461 p++;
7463 7462 }
7464 7463 if (unlock) {
7465 7464 /* failed to collect pages - release the locks */
7466 7465 while (--p >= pcf) {
7467 7466 mutex_exit(&p->pcf_lock);
7468 7467 }
7469 7468 }
7470 7469 if (pcftotal_ret != NULL)
7471 7470 *pcftotal_ret = pcftotal;
7472 7471 return (0);
7473 7472 }
|
↓ open down ↓ |
1238 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX