1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2015, Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
24 * Copyright (c) 2015, 2016 by Delphix. All rights reserved.
25 * Copyright 2018 Joyent, Inc.
26 */
27
28 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
29 /* All Rights Reserved */
30
31 /*
32 * University Copyright- Copyright (c) 1982, 1986, 1988
33 * The Regents of the University of California
34 * All Rights Reserved
35 *
36 * University Acknowledgment- Portions of this document are derived from
37 * software developed by the University of California, Berkeley, and its
38 * contributors.
39 */
40
41 /*
42 * VM - physical page management.
43 */
44
45 #include <sys/types.h>
46 #include <sys/t_lock.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/errno.h>
50 #include <sys/time.h>
51 #include <sys/vnode.h>
52 #include <sys/vm.h>
53 #include <sys/vtrace.h>
54 #include <sys/swap.h>
55 #include <sys/cmn_err.h>
56 #include <sys/tuneable.h>
57 #include <sys/sysmacros.h>
58 #include <sys/cpuvar.h>
59 #include <sys/callb.h>
60 #include <sys/debug.h>
61 #include <sys/tnf_probe.h>
62 #include <sys/condvar_impl.h>
63 #include <sys/mem_config.h>
64 #include <sys/mem_cage.h>
65 #include <sys/kmem.h>
66 #include <sys/atomic.h>
67 #include <sys/strlog.h>
68 #include <sys/mman.h>
69 #include <sys/ontrap.h>
70 #include <sys/lgrp.h>
71 #include <sys/vfs.h>
72
73 #include <vm/hat.h>
74 #include <vm/anon.h>
75 #include <vm/page.h>
76 #include <vm/seg.h>
77 #include <vm/pvn.h>
78 #include <vm/seg_kmem.h>
79 #include <vm/vm_dep.h>
80 #include <sys/vm_usage.h>
81 #include <fs/fs_subr.h>
82 #include <sys/ddi.h>
83 #include <sys/modctl.h>
84
85 static pgcnt_t max_page_get; /* max page_get request size in pages */
86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */
87
88 /*
89 * freemem_lock protects all freemem variables:
90 * availrmem. Also this lock protects the globals which track the
91 * availrmem changes for accurate kernel footprint calculation.
92 * See below for an explanation of these
93 * globals.
94 */
95 kmutex_t freemem_lock;
96 pgcnt_t availrmem;
97 pgcnt_t availrmem_initial;
98
99 /*
100 * These globals track availrmem changes to get a more accurate
101 * estimate of tke kernel size. Historically pp_kernel is used for
102 * kernel size and is based on availrmem. But availrmem is adjusted for
103 * locked pages in the system not just for kernel locked pages.
104 * These new counters will track the pages locked through segvn and
105 * by explicit user locking.
106 *
107 * pages_locked : How many pages are locked because of user specified
108 * locking through mlock or plock.
109 *
110 * pages_useclaim,pages_claimed : These two variables track the
111 * claim adjustments because of the protection changes on a segvn segment.
112 *
113 * All these globals are protected by the same lock which protects availrmem.
114 */
115 pgcnt_t pages_locked = 0;
116 pgcnt_t pages_useclaim = 0;
117 pgcnt_t pages_claimed = 0;
118
119
120 /*
121 * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
122 */
123 static kmutex_t new_freemem_lock;
124 static uint_t freemem_wait; /* someone waiting for freemem */
125 static kcondvar_t freemem_cv;
126
127 /*
128 * The logical page free list is maintained as two lists, the 'free'
129 * and the 'cache' lists.
130 * The free list contains those pages that should be reused first.
131 *
132 * The implementation of the lists is machine dependent.
133 * page_get_freelist(), page_get_cachelist(),
134 * page_list_sub(), and page_list_add()
135 * form the interface to the machine dependent implementation.
136 *
137 * Pages with p_free set are on the cache list.
138 * Pages with p_free and p_age set are on the free list,
139 *
140 * A page may be locked while on either list.
141 */
142
143 /*
144 * free list accounting stuff.
145 *
146 *
147 * Spread out the value for the number of pages on the
148 * page free and page cache lists. If there is just one
149 * value, then it must be under just one lock.
150 * The lock contention and cache traffic are a real bother.
151 *
152 * When we acquire and then drop a single pcf lock
153 * we can start in the middle of the array of pcf structures.
154 * If we acquire more than one pcf lock at a time, we need to
155 * start at the front to avoid deadlocking.
156 *
157 * pcf_count holds the number of pages in each pool.
158 *
159 * pcf_block is set when page_create_get_something() has asked the
160 * PSM page freelist and page cachelist routines without specifying
161 * a color and nothing came back. This is used to block anything
162 * else from moving pages from one list to the other while the
163 * lists are searched again. If a page is freeed while pcf_block is
164 * set, then pcf_reserve is incremented. pcgs_unblock() takes care
165 * of clearning pcf_block, doing the wakeups, etc.
166 */
167
168 #define MAX_PCF_FANOUT NCPU
169 static uint_t pcf_fanout = 1; /* Will get changed at boot time */
170 static uint_t pcf_fanout_mask = 0;
171
172 struct pcf {
173 kmutex_t pcf_lock; /* protects the structure */
174 uint_t pcf_count; /* page count */
175 uint_t pcf_wait; /* number of waiters */
176 uint_t pcf_block; /* pcgs flag to page_free() */
177 uint_t pcf_reserve; /* pages freed after pcf_block set */
178 uint_t pcf_fill[10]; /* to line up on the caches */
179 };
180
181 /*
182 * PCF_INDEX hash needs to be dynamic (every so often the hash changes where
183 * it will hash the cpu to). This is done to prevent a drain condition
184 * from happening. This drain condition will occur when pcf_count decrement
185 * occurs on cpu A and the increment of pcf_count always occurs on cpu B. An
186 * example of this shows up with device interrupts. The dma buffer is allocated
187 * by the cpu requesting the IO thus the pcf_count is decremented based on that.
188 * When the memory is returned by the interrupt thread, the pcf_count will be
189 * incremented based on the cpu servicing the interrupt.
190 */
191 static struct pcf pcf[MAX_PCF_FANOUT];
192 #define PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \
193 (randtick() >> 24)) & (pcf_fanout_mask))
194
195 static int pcf_decrement_bucket(pgcnt_t);
196 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int);
197
198 kmutex_t pcgs_lock; /* serializes page_create_get_ */
199 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */
200 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */
201 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */
202
203 #ifdef VM_STATS
204
205 /*
206 * No locks, but so what, they are only statistics.
207 */
208
209 static struct page_tcnt {
210 int pc_free_cache; /* free's into cache list */
211 int pc_free_dontneed; /* free's with dontneed */
212 int pc_free_pageout; /* free's from pageout */
213 int pc_free_free; /* free's into free list */
214 int pc_free_pages; /* free's into large page free list */
215 int pc_destroy_pages; /* large page destroy's */
216 int pc_get_cache; /* get's from cache list */
217 int pc_get_free; /* get's from free list */
218 int pc_reclaim; /* reclaim's */
219 int pc_abortfree; /* abort's of free pages */
220 int pc_find_hit; /* find's that find page */
221 int pc_find_miss; /* find's that don't find page */
222 int pc_destroy_free; /* # of free pages destroyed */
223 #define PC_HASH_CNT (4*PAGE_HASHAVELEN)
224 int pc_find_hashlen[PC_HASH_CNT+1];
225 int pc_addclaim_pages;
226 int pc_subclaim_pages;
227 int pc_free_replacement_page[2];
228 int pc_try_demote_pages[6];
229 int pc_demote_pages[2];
230 } pagecnt;
231
232 uint_t hashin_count;
233 uint_t hashin_not_held;
234 uint_t hashin_already;
235
236 uint_t hashout_count;
237 uint_t hashout_not_held;
238
239 uint_t page_create_count;
240 uint_t page_create_not_enough;
241 uint_t page_create_not_enough_again;
242 uint_t page_create_zero;
243 uint_t page_create_hashout;
244 uint_t page_create_page_lock_failed;
245 uint_t page_create_trylock_failed;
246 uint_t page_create_found_one;
247 uint_t page_create_hashin_failed;
248 uint_t page_create_dropped_phm;
249
250 uint_t page_create_new;
251 uint_t page_create_exists;
252 uint_t page_create_putbacks;
253 uint_t page_create_overshoot;
254
255 uint_t page_reclaim_zero;
256 uint_t page_reclaim_zero_locked;
257
258 uint_t page_rename_exists;
259 uint_t page_rename_count;
260
261 uint_t page_lookup_cnt[20];
262 uint_t page_lookup_nowait_cnt[10];
263 uint_t page_find_cnt;
264 uint_t page_exists_cnt;
265 uint_t page_exists_forreal_cnt;
266 uint_t page_lookup_dev_cnt;
267 uint_t get_cachelist_cnt;
268 uint_t page_create_cnt[10];
269 uint_t alloc_pages[9];
270 uint_t page_exphcontg[19];
271 uint_t page_create_large_cnt[10];
272
273 #endif
274
275 static inline page_t *
276 page_hash_search(ulong_t index, vnode_t *vnode, u_offset_t off)
277 {
278 uint_t mylen = 0;
279 page_t *page;
280
281 for (page = page_hash[index]; page; page = page->p_hash, mylen++)
282 if (page->p_vnode == vnode && page->p_offset == off)
283 break;
284
285 #ifdef VM_STATS
286 if (page != NULL)
287 pagecnt.pc_find_hit++;
288 else
289 pagecnt.pc_find_miss++;
290
291 pagecnt.pc_find_hashlen[MIN(mylen, PC_HASH_CNT)]++;
292 #endif
293
294 return (page);
295 }
296
297
298 #ifdef DEBUG
299 #define MEMSEG_SEARCH_STATS
300 #endif
301
302 #ifdef MEMSEG_SEARCH_STATS
303 struct memseg_stats {
304 uint_t nsearch;
305 uint_t nlastwon;
306 uint_t nhashwon;
307 uint_t nnotfound;
308 } memseg_stats;
309
310 #define MEMSEG_STAT_INCR(v) \
311 atomic_inc_32(&memseg_stats.v)
312 #else
313 #define MEMSEG_STAT_INCR(x)
314 #endif
315
316 struct memseg *memsegs; /* list of memory segments */
317
318 /*
319 * /etc/system tunable to control large page allocation hueristic.
320 *
321 * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup
322 * for large page allocation requests. If a large page is not readily
323 * avaliable on the local freelists we will go through additional effort
324 * to create a large page, potentially moving smaller pages around to coalesce
325 * larger pages in the local lgroup.
326 * Default value of LPAP_DEFAULT will go to remote freelists if large pages
327 * are not readily available in the local lgroup.
328 */
329 enum lpap {
330 LPAP_DEFAULT, /* default large page allocation policy */
331 LPAP_LOCAL /* local large page allocation policy */
332 };
333
334 enum lpap lpg_alloc_prefer = LPAP_DEFAULT;
335
336 static void page_init_mem_config(void);
337 static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
338 static void page_do_hashout(page_t *);
339 static void page_capture_init();
340 int page_capture_take_action(page_t *, uint_t, void *);
341
342 static void page_demote_vp_pages(page_t *);
343
344
345 void
346 pcf_init(void)
347 {
348 if (boot_ncpus != -1) {
349 pcf_fanout = boot_ncpus;
350 } else {
351 pcf_fanout = max_ncpus;
352 }
353 #ifdef sun4v
354 /*
355 * Force at least 4 buckets if possible for sun4v.
356 */
357 pcf_fanout = MAX(pcf_fanout, 4);
358 #endif /* sun4v */
359
360 /*
361 * Round up to the nearest power of 2.
362 */
363 pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT);
364 if (!ISP2(pcf_fanout)) {
365 pcf_fanout = 1 << highbit(pcf_fanout);
366
367 if (pcf_fanout > MAX_PCF_FANOUT) {
368 pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1);
369 }
370 }
371 pcf_fanout_mask = pcf_fanout - 1;
372 }
373
374 /*
375 * vm subsystem related initialization
376 */
377 void
378 vm_init(void)
379 {
380 boolean_t callb_vm_cpr(void *, int);
381
382 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
383 page_init_mem_config();
384 page_retire_init();
385 vm_usage_init();
386 page_capture_init();
387 }
388
389 /*
390 * This function is called at startup and when memory is added or deleted.
391 */
392 void
393 init_pages_pp_maximum()
394 {
395 static pgcnt_t p_min;
396 static pgcnt_t pages_pp_maximum_startup;
397 static pgcnt_t avrmem_delta;
398 static int init_done;
399 static int user_set; /* true if set in /etc/system */
400
401 if (init_done == 0) {
402
403 /* If the user specified a value, save it */
404 if (pages_pp_maximum != 0) {
405 user_set = 1;
406 pages_pp_maximum_startup = pages_pp_maximum;
407 }
408
409 /*
410 * Setting of pages_pp_maximum is based first time
411 * on the value of availrmem just after the start-up
412 * allocations. To preserve this relationship at run
413 * time, use a delta from availrmem_initial.
414 */
415 ASSERT(availrmem_initial >= availrmem);
416 avrmem_delta = availrmem_initial - availrmem;
417
418 /* The allowable floor of pages_pp_maximum */
419 p_min = tune.t_minarmem + 100;
420
421 /* Make sure we don't come through here again. */
422 init_done = 1;
423 }
424 /*
425 * Determine pages_pp_maximum, the number of currently available
426 * pages (availrmem) that can't be `locked'. If not set by
427 * the user, we set it to 4% of the currently available memory
428 * plus 4MB.
429 * But we also insist that it be greater than tune.t_minarmem;
430 * otherwise a process could lock down a lot of memory, get swapped
431 * out, and never have enough to get swapped back in.
432 */
433 if (user_set)
434 pages_pp_maximum = pages_pp_maximum_startup;
435 else
436 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
437 + btop(4 * 1024 * 1024);
438
439 if (pages_pp_maximum <= p_min) {
440 pages_pp_maximum = p_min;
441 }
442 }
443
444 /*
445 * In the past, we limited the maximum pages that could be gotten to essentially
446 * 1/2 of the total pages on the system. However, this is too conservative for
447 * some cases. For example, if we want to host a large virtual machine which
448 * needs to use a significant portion of the system's memory. In practice,
449 * allowing more than 1/2 of the total pages is fine, but becomes problematic
450 * as we approach or exceed 75% of the pages on the system. Thus, we limit the
451 * maximum to 23/32 of the total pages, which is ~72%.
452 */
453 void
454 set_max_page_get(pgcnt_t target_total_pages)
455 {
456 max_page_get = (target_total_pages >> 5) * 23;
457 ASSERT3U(max_page_get, >, 0);
458 }
459
460 pgcnt_t
461 get_max_page_get()
462 {
463 return (max_page_get);
464 }
465
466 static pgcnt_t pending_delete;
467
468 /*ARGSUSED*/
469 static void
470 page_mem_config_post_add(
471 void *arg,
472 pgcnt_t delta_pages)
473 {
474 set_max_page_get(total_pages - pending_delete);
475 init_pages_pp_maximum();
476 }
477
478 /*ARGSUSED*/
479 static int
480 page_mem_config_pre_del(
481 void *arg,
482 pgcnt_t delta_pages)
483 {
484 pgcnt_t nv;
485
486 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
487 set_max_page_get(total_pages - nv);
488 return (0);
489 }
490
491 /*ARGSUSED*/
492 static void
493 page_mem_config_post_del(
494 void *arg,
495 pgcnt_t delta_pages,
496 int cancelled)
497 {
498 pgcnt_t nv;
499
500 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
501 set_max_page_get(total_pages - nv);
502 if (!cancelled)
503 init_pages_pp_maximum();
504 }
505
506 static kphysm_setup_vector_t page_mem_config_vec = {
507 KPHYSM_SETUP_VECTOR_VERSION,
508 page_mem_config_post_add,
509 page_mem_config_pre_del,
510 page_mem_config_post_del,
511 };
512
513 static void
514 page_init_mem_config(void)
515 {
516 int ret;
517
518 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
519 ASSERT(ret == 0);
520 }
521
522 /*
523 * Evenly spread out the PCF counters for large free pages
524 */
525 static void
526 page_free_large_ctr(pgcnt_t npages)
527 {
528 static struct pcf *p = pcf;
529 pgcnt_t lump;
530
531 freemem += npages;
532
533 lump = roundup(npages, pcf_fanout) / pcf_fanout;
534
535 while (npages > 0) {
536
537 ASSERT(!p->pcf_block);
538
539 if (lump < npages) {
540 p->pcf_count += (uint_t)lump;
541 npages -= lump;
542 } else {
543 p->pcf_count += (uint_t)npages;
544 npages = 0;
545 }
546
547 ASSERT(!p->pcf_wait);
548
549 if (++p > &pcf[pcf_fanout - 1])
550 p = pcf;
551 }
552
553 ASSERT(npages == 0);
554 }
555
556 /*
557 * Add a physical chunk of memory to the system free lists during startup.
558 * Platform specific startup() allocates the memory for the page structs.
559 *
560 * num - number of page structures
561 * base - page number (pfn) to be associated with the first page.
562 *
563 * Since we are doing this during startup (ie. single threaded), we will
564 * use shortcut routines to avoid any locking overhead while putting all
565 * these pages on the freelists.
566 *
567 * NOTE: Any changes performed to page_free(), must also be performed to
568 * add_physmem() since this is how we initialize all page_t's at
569 * boot time.
570 */
571 void
572 add_physmem(
573 page_t *pp,
574 pgcnt_t num,
575 pfn_t pnum)
576 {
577 page_t *root = NULL;
578 uint_t szc = page_num_pagesizes() - 1;
579 pgcnt_t large = page_get_pagecnt(szc);
580 pgcnt_t cnt = 0;
581
582 TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
583 "add_physmem:pp %p num %lu", pp, num);
584
585 /*
586 * Arbitrarily limit the max page_get request
587 * to 1/2 of the page structs we have.
588 */
589 total_pages += num;
590 set_max_page_get(total_pages);
591
592 PLCNT_MODIFY_MAX(pnum, (long)num);
593
594 /*
595 * The physical space for the pages array
596 * representing ram pages has already been
597 * allocated. Here we initialize each lock
598 * in the page structure, and put each on
599 * the free list
600 */
601 for (; num; pp++, pnum++, num--) {
602
603 /*
604 * this needs to fill in the page number
605 * and do any other arch specific initialization
606 */
607 add_physmem_cb(pp, pnum);
608
609 pp->p_lckcnt = 0;
610 pp->p_cowcnt = 0;
611 pp->p_slckcnt = 0;
612
613 /*
614 * Initialize the page lock as unlocked, since nobody
615 * can see or access this page yet.
616 */
617 pp->p_selock = 0;
618
619 /*
620 * Initialize IO lock
621 */
622 page_iolock_init(pp);
623
624 /*
625 * initialize other fields in the page_t
626 */
627 PP_SETFREE(pp);
628 page_clr_all_props(pp);
629 PP_SETAGED(pp);
630 pp->p_offset = (u_offset_t)-1;
631 pp->p_next = pp;
632 pp->p_prev = pp;
633
634 /*
635 * Simple case: System doesn't support large pages.
636 */
637 if (szc == 0) {
638 pp->p_szc = 0;
639 page_free_at_startup(pp);
640 continue;
641 }
642
643 /*
644 * Handle unaligned pages, we collect them up onto
645 * the root page until we have a full large page.
646 */
647 if (!IS_P2ALIGNED(pnum, large)) {
648
649 /*
650 * If not in a large page,
651 * just free as small page.
652 */
653 if (root == NULL) {
654 pp->p_szc = 0;
655 page_free_at_startup(pp);
656 continue;
657 }
658
659 /*
660 * Link a constituent page into the large page.
661 */
662 pp->p_szc = szc;
663 page_list_concat(&root, &pp);
664
665 /*
666 * When large page is fully formed, free it.
667 */
668 if (++cnt == large) {
669 page_free_large_ctr(cnt);
670 page_list_add_pages(root, PG_LIST_ISINIT);
671 root = NULL;
672 cnt = 0;
673 }
674 continue;
675 }
676
677 /*
678 * At this point we have a page number which
679 * is aligned. We assert that we aren't already
680 * in a different large page.
681 */
682 ASSERT(IS_P2ALIGNED(pnum, large));
683 ASSERT(root == NULL && cnt == 0);
684
685 /*
686 * If insufficient number of pages left to form
687 * a large page, just free the small page.
688 */
689 if (num < large) {
690 pp->p_szc = 0;
691 page_free_at_startup(pp);
692 continue;
693 }
694
695 /*
696 * Otherwise start a new large page.
697 */
698 pp->p_szc = szc;
699 cnt++;
700 root = pp;
701 }
702 ASSERT(root == NULL && cnt == 0);
703 }
704
705 /*
706 * Find a page representing the specified [vp, offset].
707 * If we find the page but it is intransit coming in,
708 * it will have an "exclusive" lock and we wait for
709 * the i/o to complete. A page found on the free list
710 * is always reclaimed and then locked. On success, the page
711 * is locked, its data is valid and it isn't on the free
712 * list, while a NULL is returned if the page doesn't exist.
713 */
714 page_t *
715 page_lookup(vnode_t *vp, u_offset_t off, se_t se)
716 {
717 return (page_lookup_create(vp, off, se, NULL, NULL, 0));
718 }
719
720 /*
721 * Find a page representing the specified [vp, offset].
722 * We either return the one we found or, if passed in,
723 * create one with identity of [vp, offset] of the
724 * pre-allocated page. If we find existing page but it is
725 * intransit coming in, it will have an "exclusive" lock
726 * and we wait for the i/o to complete. A page found on
727 * the free list is always reclaimed and then locked.
728 * On success, the page is locked, its data is valid and
729 * it isn't on the free list, while a NULL is returned
730 * if the page doesn't exist and newpp is NULL;
731 */
732 page_t *
733 page_lookup_create(
734 vnode_t *vp,
735 u_offset_t off,
736 se_t se,
737 page_t *newpp,
738 spgcnt_t *nrelocp,
739 int flags)
740 {
741 page_t *pp;
742 kmutex_t *phm;
743 ulong_t index;
744 uint_t hash_locked;
745 uint_t es;
746
747 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
748 VM_STAT_ADD(page_lookup_cnt[0]);
749 ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
750
751 /*
752 * Acquire the appropriate page hash lock since
753 * we have to search the hash list. Pages that
754 * hash to this list can't change identity while
755 * this lock is held.
756 */
757 hash_locked = 0;
758 index = PAGE_HASH_FUNC(vp, off);
759 phm = NULL;
760 top:
761 pp = page_hash_search(index, vp, off);
762 if (pp != NULL) {
763 VM_STAT_ADD(page_lookup_cnt[1]);
764 es = (newpp != NULL) ? 1 : 0;
765 es |= flags;
766 if (!hash_locked) {
767 VM_STAT_ADD(page_lookup_cnt[2]);
768 if (!page_try_reclaim_lock(pp, se, es)) {
769 /*
770 * On a miss, acquire the phm. Then
771 * next time, page_lock() will be called,
772 * causing a wait if the page is busy.
773 * just looping with page_trylock() would
774 * get pretty boring.
775 */
776 VM_STAT_ADD(page_lookup_cnt[3]);
777 phm = PAGE_HASH_MUTEX(index);
778 mutex_enter(phm);
779 hash_locked = 1;
780 goto top;
781 }
782 } else {
783 VM_STAT_ADD(page_lookup_cnt[4]);
784 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
785 VM_STAT_ADD(page_lookup_cnt[5]);
786 goto top;
787 }
788 }
789
790 /*
791 * Since `pp' is locked it can not change identity now.
792 * Reconfirm we locked the correct page.
793 *
794 * Both the p_vnode and p_offset *must* be cast volatile
795 * to force a reload of their values: The page_hash_search
796 * function will have stuffed p_vnode and p_offset into
797 * registers before calling page_trylock(); another thread,
798 * actually holding the hash lock, could have changed the
799 * page's identity in memory, but our registers would not
800 * be changed, fooling the reconfirmation. If the hash
801 * lock was held during the search, the casting would
802 * not be needed.
803 */
804 VM_STAT_ADD(page_lookup_cnt[6]);
805 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
806 ((volatile u_offset_t)(pp->p_offset) != off)) {
807 VM_STAT_ADD(page_lookup_cnt[7]);
808 if (hash_locked) {
809 panic("page_lookup_create: lost page %p",
810 (void *)pp);
811 /*NOTREACHED*/
812 }
813 page_unlock(pp);
814 phm = PAGE_HASH_MUTEX(index);
815 mutex_enter(phm);
816 hash_locked = 1;
817 goto top;
818 }
819
820 /*
821 * If page_trylock() was called, then pp may still be on
822 * the cachelist (can't be on the free list, it would not
823 * have been found in the search). If it is on the
824 * cachelist it must be pulled now. To pull the page from
825 * the cachelist, it must be exclusively locked.
826 *
827 * The other big difference between page_trylock() and
828 * page_lock(), is that page_lock() will pull the
829 * page from whatever free list (the cache list in this
830 * case) the page is on. If page_trylock() was used
831 * above, then we have to do the reclaim ourselves.
832 */
833 if ((!hash_locked) && (PP_ISFREE(pp))) {
834 ASSERT(PP_ISAGED(pp) == 0);
835 VM_STAT_ADD(page_lookup_cnt[8]);
836
837 /*
838 * page_relcaim will insure that we
839 * have this page exclusively
840 */
841
842 if (!page_reclaim(pp, NULL)) {
843 /*
844 * Page_reclaim dropped whatever lock
845 * we held.
846 */
847 VM_STAT_ADD(page_lookup_cnt[9]);
848 phm = PAGE_HASH_MUTEX(index);
849 mutex_enter(phm);
850 hash_locked = 1;
851 goto top;
852 } else if (se == SE_SHARED && newpp == NULL) {
853 VM_STAT_ADD(page_lookup_cnt[10]);
854 page_downgrade(pp);
855 }
856 }
857
858 if (hash_locked) {
859 mutex_exit(phm);
860 }
861
862 if (newpp != NULL && pp->p_szc < newpp->p_szc &&
863 PAGE_EXCL(pp) && nrelocp != NULL) {
864 ASSERT(nrelocp != NULL);
865 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
866 NULL);
867 if (*nrelocp > 0) {
868 VM_STAT_COND_ADD(*nrelocp == 1,
869 page_lookup_cnt[11]);
870 VM_STAT_COND_ADD(*nrelocp > 1,
871 page_lookup_cnt[12]);
872 pp = newpp;
873 se = SE_EXCL;
874 } else {
875 if (se == SE_SHARED) {
876 page_downgrade(pp);
877 }
878 VM_STAT_ADD(page_lookup_cnt[13]);
879 }
880 } else if (newpp != NULL && nrelocp != NULL) {
881 if (PAGE_EXCL(pp) && se == SE_SHARED) {
882 page_downgrade(pp);
883 }
884 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
885 page_lookup_cnt[14]);
886 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
887 page_lookup_cnt[15]);
888 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
889 page_lookup_cnt[16]);
890 } else if (newpp != NULL && PAGE_EXCL(pp)) {
891 se = SE_EXCL;
892 }
893 } else if (!hash_locked) {
894 VM_STAT_ADD(page_lookup_cnt[17]);
895 phm = PAGE_HASH_MUTEX(index);
896 mutex_enter(phm);
897 hash_locked = 1;
898 goto top;
899 } else if (newpp != NULL) {
900 /*
901 * If we have a preallocated page then
902 * insert it now and basically behave like
903 * page_create.
904 */
905 VM_STAT_ADD(page_lookup_cnt[18]);
906 /*
907 * Since we hold the page hash mutex and
908 * just searched for this page, page_hashin
909 * had better not fail. If it does, that
910 * means some thread did not follow the
911 * page hash mutex rules. Panic now and
912 * get it over with. As usual, go down
913 * holding all the locks.
914 */
915 ASSERT(MUTEX_HELD(phm));
916 if (!page_hashin(newpp, vp, off, phm)) {
917 ASSERT(MUTEX_HELD(phm));
918 panic("page_lookup_create: hashin failed %p %p %llx %p",
919 (void *)newpp, (void *)vp, off, (void *)phm);
920 /*NOTREACHED*/
921 }
922 ASSERT(MUTEX_HELD(phm));
923 mutex_exit(phm);
924 phm = NULL;
925 page_set_props(newpp, P_REF);
926 page_io_lock(newpp);
927 pp = newpp;
928 se = SE_EXCL;
929 } else {
930 VM_STAT_ADD(page_lookup_cnt[19]);
931 mutex_exit(phm);
932 }
933
934 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
935
936 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);
937
938 return (pp);
939 }
940
941 /*
942 * Search the hash list for the page representing the
943 * specified [vp, offset] and return it locked. Skip
944 * free pages and pages that cannot be locked as requested.
945 * Used while attempting to kluster pages.
946 */
947 page_t *
948 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
949 {
950 page_t *pp;
951 kmutex_t *phm;
952 ulong_t index;
953 uint_t locked;
954
955 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
956 VM_STAT_ADD(page_lookup_nowait_cnt[0]);
957
958 index = PAGE_HASH_FUNC(vp, off);
959 pp = page_hash_search(index, vp, off);
960 locked = 0;
961 if (pp == NULL) {
962 top:
963 VM_STAT_ADD(page_lookup_nowait_cnt[1]);
964 locked = 1;
965 phm = PAGE_HASH_MUTEX(index);
966 mutex_enter(phm);
967 pp = page_hash_search(index, vp, off);
968 }
969
970 if (pp == NULL || PP_ISFREE(pp)) {
971 VM_STAT_ADD(page_lookup_nowait_cnt[2]);
972 pp = NULL;
973 } else {
974 if (!page_trylock(pp, se)) {
975 VM_STAT_ADD(page_lookup_nowait_cnt[3]);
976 pp = NULL;
977 } else {
978 VM_STAT_ADD(page_lookup_nowait_cnt[4]);
979 /*
980 * See the comment in page_lookup()
981 */
982 if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
983 ((u_offset_t)(pp->p_offset) != off)) {
984 VM_STAT_ADD(page_lookup_nowait_cnt[5]);
985 if (locked) {
986 panic("page_lookup_nowait %p",
987 (void *)pp);
988 /*NOTREACHED*/
989 }
990 page_unlock(pp);
991 goto top;
992 }
993 if (PP_ISFREE(pp)) {
994 VM_STAT_ADD(page_lookup_nowait_cnt[6]);
995 page_unlock(pp);
996 pp = NULL;
997 }
998 }
999 }
1000 if (locked) {
1001 VM_STAT_ADD(page_lookup_nowait_cnt[7]);
1002 mutex_exit(phm);
1003 }
1004
1005 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
1006
1007 return (pp);
1008 }
1009
1010 /*
1011 * Search the hash list for a page with the specified [vp, off]
1012 * that is known to exist and is already locked. This routine
1013 * is typically used by segment SOFTUNLOCK routines.
1014 */
1015 page_t *
1016 page_find(vnode_t *vp, u_offset_t off)
1017 {
1018 page_t *pp;
1019 kmutex_t *phm;
1020 ulong_t index;
1021
1022 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1023 VM_STAT_ADD(page_find_cnt);
1024
1025 index = PAGE_HASH_FUNC(vp, off);
1026 phm = PAGE_HASH_MUTEX(index);
1027
1028 mutex_enter(phm);
1029 pp = page_hash_search(index, vp, off);
1030 mutex_exit(phm);
1031
1032 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
1033 return (pp);
1034 }
1035
1036 /*
1037 * Determine whether a page with the specified [vp, off]
1038 * currently exists in the system. Obviously this should
1039 * only be considered as a hint since nothing prevents the
1040 * page from disappearing or appearing immediately after
1041 * the return from this routine. Subsequently, we don't
1042 * even bother to lock the list.
1043 */
1044 page_t *
1045 page_exists(vnode_t *vp, u_offset_t off)
1046 {
1047 ulong_t index;
1048
1049 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1050 VM_STAT_ADD(page_exists_cnt);
1051
1052 index = PAGE_HASH_FUNC(vp, off);
1053
1054 return (page_hash_search(index, vp, off));
1055 }
1056
1057 /*
1058 * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1059 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array
1060 * with these pages locked SHARED. If necessary reclaim pages from
1061 * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1062 *
1063 * If we fail to lock pages still return 1 if pages exist and contiguous.
1064 * But in this case return value is just a hint. ppa array won't be filled.
1065 * Caller should initialize ppa[0] as NULL to distinguish return value.
1066 *
1067 * Returns 0 if pages don't exist or not physically contiguous.
1068 *
1069 * This routine doesn't work for anonymous(swapfs) pages.
1070 */
1071 int
1072 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1073 {
1074 pgcnt_t pages;
1075 pfn_t pfn;
1076 page_t *rootpp;
1077 pgcnt_t i;
1078 pgcnt_t j;
1079 u_offset_t save_off = off;
1080 ulong_t index;
1081 kmutex_t *phm;
1082 page_t *pp;
1083 uint_t pszc;
1084 int loopcnt = 0;
1085
1086 ASSERT(szc != 0);
1087 ASSERT(vp != NULL);
1088 ASSERT(!IS_SWAPFSVP(vp));
1089 ASSERT(!VN_ISKAS(vp));
1090
1091 again:
1092 if (++loopcnt > 3) {
1093 VM_STAT_ADD(page_exphcontg[0]);
1094 return (0);
1095 }
1096
1097 index = PAGE_HASH_FUNC(vp, off);
1098 phm = PAGE_HASH_MUTEX(index);
1099
1100 mutex_enter(phm);
1101 pp = page_hash_search(index, vp, off);
1102 mutex_exit(phm);
1103
1104 VM_STAT_ADD(page_exphcontg[1]);
1105
1106 if (pp == NULL) {
1107 VM_STAT_ADD(page_exphcontg[2]);
1108 return (0);
1109 }
1110
1111 pages = page_get_pagecnt(szc);
1112 rootpp = pp;
1113 pfn = rootpp->p_pagenum;
1114
1115 if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1116 VM_STAT_ADD(page_exphcontg[3]);
1117 if (!page_trylock(pp, SE_SHARED)) {
1118 VM_STAT_ADD(page_exphcontg[4]);
1119 return (1);
1120 }
1121 /*
1122 * Also check whether p_pagenum was modified by DR.
1123 */
1124 if (pp->p_szc != pszc || pp->p_vnode != vp ||
1125 pp->p_offset != off || pp->p_pagenum != pfn) {
1126 VM_STAT_ADD(page_exphcontg[5]);
1127 page_unlock(pp);
1128 off = save_off;
1129 goto again;
1130 }
1131 /*
1132 * szc was non zero and vnode and offset matched after we
1133 * locked the page it means it can't become free on us.
1134 */
1135 ASSERT(!PP_ISFREE(pp));
1136 if (!IS_P2ALIGNED(pfn, pages)) {
1137 page_unlock(pp);
1138 return (0);
1139 }
1140 ppa[0] = pp;
1141 pp++;
1142 off += PAGESIZE;
1143 pfn++;
1144 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1145 if (!page_trylock(pp, SE_SHARED)) {
1146 VM_STAT_ADD(page_exphcontg[6]);
1147 pp--;
1148 while (i-- > 0) {
1149 page_unlock(pp);
1150 pp--;
1151 }
1152 ppa[0] = NULL;
1153 return (1);
1154 }
1155 if (pp->p_szc != pszc) {
1156 VM_STAT_ADD(page_exphcontg[7]);
1157 page_unlock(pp);
1158 pp--;
1159 while (i-- > 0) {
1160 page_unlock(pp);
1161 pp--;
1162 }
1163 ppa[0] = NULL;
1164 off = save_off;
1165 goto again;
1166 }
1167 /*
1168 * szc the same as for previous already locked pages
1169 * with right identity. Since this page had correct
1170 * szc after we locked it can't get freed or destroyed
1171 * and therefore must have the expected identity.
1172 */
1173 ASSERT(!PP_ISFREE(pp));
1174 if (pp->p_vnode != vp ||
1175 pp->p_offset != off) {
1176 panic("page_exists_physcontig: "
1177 "large page identity doesn't match");
1178 }
1179 ppa[i] = pp;
1180 ASSERT(pp->p_pagenum == pfn);
1181 }
1182 VM_STAT_ADD(page_exphcontg[8]);
1183 ppa[pages] = NULL;
1184 return (1);
1185 } else if (pszc >= szc) {
1186 VM_STAT_ADD(page_exphcontg[9]);
1187 if (!IS_P2ALIGNED(pfn, pages)) {
1188 return (0);
1189 }
1190 return (1);
1191 }
1192
1193 if (!IS_P2ALIGNED(pfn, pages)) {
1194 VM_STAT_ADD(page_exphcontg[10]);
1195 return (0);
1196 }
1197
1198 if (page_numtomemseg_nolock(pfn) !=
1199 page_numtomemseg_nolock(pfn + pages - 1)) {
1200 VM_STAT_ADD(page_exphcontg[11]);
1201 return (0);
1202 }
1203
1204 /*
1205 * We loop up 4 times across pages to promote page size.
1206 * We're extra cautious to promote page size atomically with respect
1207 * to everybody else. But we can probably optimize into 1 loop if
1208 * this becomes an issue.
1209 */
1210
1211 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1212 if (!page_trylock(pp, SE_EXCL)) {
1213 VM_STAT_ADD(page_exphcontg[12]);
1214 break;
1215 }
1216 /*
1217 * Check whether p_pagenum was modified by DR.
1218 */
1219 if (pp->p_pagenum != pfn) {
1220 page_unlock(pp);
1221 break;
1222 }
1223 if (pp->p_vnode != vp ||
1224 pp->p_offset != off) {
1225 VM_STAT_ADD(page_exphcontg[13]);
1226 page_unlock(pp);
1227 break;
1228 }
1229 if (pp->p_szc >= szc) {
1230 ASSERT(i == 0);
1231 page_unlock(pp);
1232 off = save_off;
1233 goto again;
1234 }
1235 }
1236
1237 if (i != pages) {
1238 VM_STAT_ADD(page_exphcontg[14]);
1239 --pp;
1240 while (i-- > 0) {
1241 page_unlock(pp);
1242 --pp;
1243 }
1244 return (0);
1245 }
1246
1247 pp = rootpp;
1248 for (i = 0; i < pages; i++, pp++) {
1249 if (PP_ISFREE(pp)) {
1250 VM_STAT_ADD(page_exphcontg[15]);
1251 ASSERT(!PP_ISAGED(pp));
1252 ASSERT(pp->p_szc == 0);
1253 if (!page_reclaim(pp, NULL)) {
1254 break;
1255 }
1256 } else {
1257 ASSERT(pp->p_szc < szc);
1258 VM_STAT_ADD(page_exphcontg[16]);
1259 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
1260 }
1261 }
1262 if (i < pages) {
1263 VM_STAT_ADD(page_exphcontg[17]);
1264 /*
1265 * page_reclaim failed because we were out of memory.
1266 * drop the rest of the locks and return because this page
1267 * must be already reallocated anyway.
1268 */
1269 pp = rootpp;
1270 for (j = 0; j < pages; j++, pp++) {
1271 if (j != i) {
1272 page_unlock(pp);
1273 }
1274 }
1275 return (0);
1276 }
1277
1278 off = save_off;
1279 pp = rootpp;
1280 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) {
1281 ASSERT(PAGE_EXCL(pp));
1282 ASSERT(!PP_ISFREE(pp));
1283 ASSERT(!hat_page_is_mapped(pp));
1284 ASSERT(pp->p_vnode == vp);
1285 ASSERT(pp->p_offset == off);
1286 pp->p_szc = szc;
1287 }
1288 pp = rootpp;
1289 for (i = 0; i < pages; i++, pp++) {
1290 if (ppa == NULL) {
1291 page_unlock(pp);
1292 } else {
1293 ppa[i] = pp;
1294 page_downgrade(ppa[i]);
1295 }
1296 }
1297 if (ppa != NULL) {
1298 ppa[pages] = NULL;
1299 }
1300 VM_STAT_ADD(page_exphcontg[18]);
1301 ASSERT(vp->v_pages != NULL);
1302 return (1);
1303 }
1304
1305 /*
1306 * Determine whether a page with the specified [vp, off]
1307 * currently exists in the system and if so return its
1308 * size code. Obviously this should only be considered as
1309 * a hint since nothing prevents the page from disappearing
1310 * or appearing immediately after the return from this routine.
1311 */
1312 int
1313 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1314 {
1315 page_t *pp;
1316 kmutex_t *phm;
1317 ulong_t index;
1318 int rc = 0;
1319
1320 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1321 ASSERT(szc != NULL);
1322 VM_STAT_ADD(page_exists_forreal_cnt);
1323
1324 index = PAGE_HASH_FUNC(vp, off);
1325 phm = PAGE_HASH_MUTEX(index);
1326
1327 mutex_enter(phm);
1328 pp = page_hash_search(index, vp, off);
1329 if (pp != NULL) {
1330 *szc = pp->p_szc;
1331 rc = 1;
1332 }
1333 mutex_exit(phm);
1334 return (rc);
1335 }
1336
1337 /* wakeup threads waiting for pages in page_create_get_something() */
1338 void
1339 wakeup_pcgs(void)
1340 {
1341 if (!CV_HAS_WAITERS(&pcgs_cv))
1342 return;
1343 cv_broadcast(&pcgs_cv);
1344 }
1345
1346 /*
1347 * 'freemem' is used all over the kernel as an indication of how many
1348 * pages are free (either on the cache list or on the free page list)
1349 * in the system. In very few places is a really accurate 'freemem'
1350 * needed. To avoid contention of the lock protecting a the
1351 * single freemem, it was spread out into NCPU buckets. Set_freemem
1352 * sets freemem to the total of all NCPU buckets. It is called from
1353 * clock() on each TICK.
1354 */
1355 void
1356 set_freemem(void)
1357 {
1358 struct pcf *p;
1359 ulong_t t;
1360 uint_t i;
1361
1362 t = 0;
1363 p = pcf;
1364 for (i = 0; i < pcf_fanout; i++) {
1365 t += p->pcf_count;
1366 p++;
1367 }
1368 freemem = t;
1369
1370 /*
1371 * Don't worry about grabbing mutex. It's not that
1372 * critical if we miss a tick or two. This is
1373 * where we wakeup possible delayers in
1374 * page_create_get_something().
1375 */
1376 wakeup_pcgs();
1377 }
1378
1379 ulong_t
1380 get_freemem()
1381 {
1382 struct pcf *p;
1383 ulong_t t;
1384 uint_t i;
1385
1386 t = 0;
1387 p = pcf;
1388 for (i = 0; i < pcf_fanout; i++) {
1389 t += p->pcf_count;
1390 p++;
1391 }
1392 /*
1393 * We just calculated it, might as well set it.
1394 */
1395 freemem = t;
1396 return (t);
1397 }
1398
1399 /*
1400 * Acquire all of the page cache & free (pcf) locks.
1401 */
1402 void
1403 pcf_acquire_all()
1404 {
1405 struct pcf *p;
1406 uint_t i;
1407
1408 p = pcf;
1409 for (i = 0; i < pcf_fanout; i++) {
1410 mutex_enter(&p->pcf_lock);
1411 p++;
1412 }
1413 }
1414
1415 /*
1416 * Release all the pcf_locks.
1417 */
1418 void
1419 pcf_release_all()
1420 {
1421 struct pcf *p;
1422 uint_t i;
1423
1424 p = pcf;
1425 for (i = 0; i < pcf_fanout; i++) {
1426 mutex_exit(&p->pcf_lock);
1427 p++;
1428 }
1429 }
1430
1431 /*
1432 * Inform the VM system that we need some pages freed up.
1433 * Calls must be symmetric, e.g.:
1434 *
1435 * page_needfree(100);
1436 * wait a bit;
1437 * page_needfree(-100);
1438 */
1439 void
1440 page_needfree(spgcnt_t npages)
1441 {
1442 mutex_enter(&new_freemem_lock);
1443 needfree += npages;
1444 mutex_exit(&new_freemem_lock);
1445 }
1446
1447 /*
1448 * Throttle for page_create(): try to prevent freemem from dropping
1449 * below throttlefree. We can't provide a 100% guarantee because
1450 * KM_NOSLEEP allocations, page_reclaim(), and various other things
1451 * nibble away at the freelist. However, we can block all PG_WAIT
1452 * allocations until memory becomes available. The motivation is
1453 * that several things can fall apart when there's no free memory:
1454 *
1455 * (1) If pageout() needs memory to push a page, the system deadlocks.
1456 *
1457 * (2) By (broken) specification, timeout(9F) can neither fail nor
1458 * block, so it has no choice but to panic the system if it
1459 * cannot allocate a callout structure.
1460 *
1461 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
1462 * it panics if it cannot allocate a callback structure.
1463 *
1464 * (4) Untold numbers of third-party drivers have not yet been hardened
1465 * against KM_NOSLEEP and/or allocb() failures; they simply assume
1466 * success and panic the system with a data fault on failure.
1467 * (The long-term solution to this particular problem is to ship
1468 * hostile fault-injecting DEBUG kernels with the DDK.)
1469 *
1470 * It is theoretically impossible to guarantee success of non-blocking
1471 * allocations, but in practice, this throttle is very hard to break.
1472 */
1473 static int
1474 page_create_throttle(pgcnt_t npages, int flags)
1475 {
1476 ulong_t fm;
1477 uint_t i;
1478 pgcnt_t tf; /* effective value of throttlefree */
1479
1480 atomic_inc_64(&n_throttle);
1481
1482 /*
1483 * Normal priority allocations.
1484 */
1485 if ((flags & (PG_WAIT | PG_NORMALPRI)) == PG_NORMALPRI) {
1486 ASSERT(!(flags & (PG_PANIC | PG_PUSHPAGE)));
1487 return (freemem >= npages + throttlefree);
1488 }
1489
1490 /*
1491 * Never deny pages when:
1492 * - it's a thread that cannot block [NOMEMWAIT()]
1493 * - the allocation cannot block and must not fail
1494 * - the allocation cannot block and is pageout dispensated
1495 */
1496 if (NOMEMWAIT() ||
1497 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) ||
1498 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE))
1499 return (1);
1500
1501 /*
1502 * If the allocation can't block, we look favorably upon it
1503 * unless we're below pageout_reserve. In that case we fail
1504 * the allocation because we want to make sure there are a few
1505 * pages available for pageout.
1506 */
1507 if ((flags & PG_WAIT) == 0)
1508 return (freemem >= npages + pageout_reserve);
1509
1510 /* Calculate the effective throttlefree value */
1511 tf = throttlefree -
1512 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0);
1513
1514 WAKE_PAGEOUT_SCANNER();
1515
1516 for (;;) {
1517 fm = 0;
1518 pcf_acquire_all();
1519 mutex_enter(&new_freemem_lock);
1520 for (i = 0; i < pcf_fanout; i++) {
1521 fm += pcf[i].pcf_count;
1522 pcf[i].pcf_wait++;
1523 mutex_exit(&pcf[i].pcf_lock);
1524 }
1525 freemem = fm;
1526 if (freemem >= npages + tf) {
1527 mutex_exit(&new_freemem_lock);
1528 break;
1529 }
1530 needfree += npages;
1531 freemem_wait++;
1532 cv_wait(&freemem_cv, &new_freemem_lock);
1533 freemem_wait--;
1534 needfree -= npages;
1535 mutex_exit(&new_freemem_lock);
1536 }
1537 return (1);
1538 }
1539
1540 /*
1541 * page_create_wait() is called to either coalesce pages from the
1542 * different pcf buckets or to wait because there simply are not
1543 * enough pages to satisfy the caller's request.
1544 *
1545 * Sadly, this is called from platform/vm/vm_machdep.c
1546 */
1547 int
1548 page_create_wait(pgcnt_t npages, uint_t flags)
1549 {
1550 pgcnt_t total;
1551 uint_t i;
1552 struct pcf *p;
1553
1554 /*
1555 * Wait until there are enough free pages to satisfy our
1556 * entire request.
1557 * We set needfree += npages before prodding pageout, to make sure
1558 * it does real work when npages > lotsfree > freemem.
1559 */
1560 VM_STAT_ADD(page_create_not_enough);
1561
1562 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1);
1563 checkagain:
1564 if ((flags & PG_NORELOC) &&
1565 kcage_freemem < kcage_throttlefree + npages)
1566 (void) kcage_create_throttle(npages, flags);
1567
1568 if (freemem < npages + throttlefree)
1569 if (!page_create_throttle(npages, flags))
1570 return (0);
1571
1572 if (pcf_decrement_bucket(npages) ||
1573 pcf_decrement_multiple(&total, npages, 0))
1574 return (1);
1575
1576 /*
1577 * All of the pcf locks are held, there are not enough pages
1578 * to satisfy the request (npages < total).
1579 * Be sure to acquire the new_freemem_lock before dropping
1580 * the pcf locks. This prevents dropping wakeups in page_free().
1581 * The order is always pcf_lock then new_freemem_lock.
1582 *
1583 * Since we hold all the pcf locks, it is a good time to set freemem.
1584 *
1585 * If the caller does not want to wait, return now.
1586 * Else turn the pageout daemon loose to find something
1587 * and wait till it does.
1588 *
1589 */
1590 freemem = total;
1591
1592 if ((flags & PG_WAIT) == 0) {
1593 pcf_release_all();
1594
1595 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM,
1596 "page_create_nomem:npages %ld freemem %ld", npages, freemem);
1597 return (0);
1598 }
1599
1600 ASSERT(proc_pageout != NULL);
1601 WAKE_PAGEOUT_SCANNER();
1602
1603 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START,
1604 "page_create_sleep_start: freemem %ld needfree %ld",
1605 freemem, needfree);
1606
1607 /*
1608 * We are going to wait.
1609 * We currently hold all of the pcf_locks,
1610 * get the new_freemem_lock (it protects freemem_wait),
1611 * before dropping the pcf_locks.
1612 */
1613 mutex_enter(&new_freemem_lock);
1614
1615 p = pcf;
1616 for (i = 0; i < pcf_fanout; i++) {
1617 p->pcf_wait++;
1618 mutex_exit(&p->pcf_lock);
1619 p++;
1620 }
1621
1622 needfree += npages;
1623 freemem_wait++;
1624
1625 cv_wait(&freemem_cv, &new_freemem_lock);
1626
1627 freemem_wait--;
1628 needfree -= npages;
1629
1630 mutex_exit(&new_freemem_lock);
1631
1632 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END,
1633 "page_create_sleep_end: freemem %ld needfree %ld",
1634 freemem, needfree);
1635
1636 VM_STAT_ADD(page_create_not_enough_again);
1637 goto checkagain;
1638 }
1639 /*
1640 * A routine to do the opposite of page_create_wait().
1641 */
1642 void
1643 page_create_putback(spgcnt_t npages)
1644 {
1645 struct pcf *p;
1646 pgcnt_t lump;
1647 uint_t *which;
1648
1649 /*
1650 * When a contiguous lump is broken up, we have to
1651 * deal with lots of pages (min 64) so lets spread
1652 * the wealth around.
1653 */
1654 lump = roundup(npages, pcf_fanout) / pcf_fanout;
1655 freemem += npages;
1656
1657 for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) {
1658 which = &p->pcf_count;
1659
1660 mutex_enter(&p->pcf_lock);
1661
1662 if (p->pcf_block) {
1663 which = &p->pcf_reserve;
1664 }
1665
1666 if (lump < npages) {
1667 *which += (uint_t)lump;
1668 npages -= lump;
1669 } else {
1670 *which += (uint_t)npages;
1671 npages = 0;
1672 }
1673
1674 if (p->pcf_wait) {
1675 mutex_enter(&new_freemem_lock);
1676 /*
1677 * Check to see if some other thread
1678 * is actually waiting. Another bucket
1679 * may have woken it up by now. If there
1680 * are no waiters, then set our pcf_wait
1681 * count to zero to avoid coming in here
1682 * next time.
1683 */
1684 if (freemem_wait) {
1685 if (npages > 1) {
1686 cv_broadcast(&freemem_cv);
1687 } else {
1688 cv_signal(&freemem_cv);
1689 }
1690 p->pcf_wait--;
1691 } else {
1692 p->pcf_wait = 0;
1693 }
1694 mutex_exit(&new_freemem_lock);
1695 }
1696 mutex_exit(&p->pcf_lock);
1697 }
1698 ASSERT(npages == 0);
1699 }
1700
1701 /*
1702 * A helper routine for page_create_get_something.
1703 * The indenting got to deep down there.
1704 * Unblock the pcf counters. Any pages freed after
1705 * pcf_block got set are moved to pcf_count and
1706 * wakeups (cv_broadcast() or cv_signal()) are done as needed.
1707 */
1708 static void
1709 pcgs_unblock(void)
1710 {
1711 int i;
1712 struct pcf *p;
1713
1714 /* Update freemem while we're here. */
1715 freemem = 0;
1716 p = pcf;
1717 for (i = 0; i < pcf_fanout; i++) {
1718 mutex_enter(&p->pcf_lock);
1719 ASSERT(p->pcf_count == 0);
1720 p->pcf_count = p->pcf_reserve;
1721 p->pcf_block = 0;
1722 freemem += p->pcf_count;
1723 if (p->pcf_wait) {
1724 mutex_enter(&new_freemem_lock);
1725 if (freemem_wait) {
1726 if (p->pcf_reserve > 1) {
1727 cv_broadcast(&freemem_cv);
1728 p->pcf_wait = 0;
1729 } else {
1730 cv_signal(&freemem_cv);
1731 p->pcf_wait--;
1732 }
1733 } else {
1734 p->pcf_wait = 0;
1735 }
1736 mutex_exit(&new_freemem_lock);
1737 }
1738 p->pcf_reserve = 0;
1739 mutex_exit(&p->pcf_lock);
1740 p++;
1741 }
1742 }
1743
1744 /*
1745 * Called from page_create_va() when both the cache and free lists
1746 * have been checked once.
1747 *
1748 * Either returns a page or panics since the accounting was done
1749 * way before we got here.
1750 *
1751 * We don't come here often, so leave the accounting on permanently.
1752 */
1753
1754 #define MAX_PCGS 100
1755
1756 #ifdef DEBUG
1757 #define PCGS_TRIES 100
1758 #else /* DEBUG */
1759 #define PCGS_TRIES 10
1760 #endif /* DEBUG */
1761
1762 #ifdef VM_STATS
1763 uint_t pcgs_counts[PCGS_TRIES];
1764 uint_t pcgs_too_many;
1765 uint_t pcgs_entered;
1766 uint_t pcgs_entered_noreloc;
1767 uint_t pcgs_locked;
1768 uint_t pcgs_cagelocked;
1769 #endif /* VM_STATS */
1770
1771 static page_t *
1772 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg,
1773 caddr_t vaddr, uint_t flags)
1774 {
1775 uint_t count;
1776 page_t *pp;
1777 uint_t locked, i;
1778 struct pcf *p;
1779 lgrp_t *lgrp;
1780 int cagelocked = 0;
1781
1782 VM_STAT_ADD(pcgs_entered);
1783
1784 /*
1785 * Tap any reserve freelists: if we fail now, we'll die
1786 * since the page(s) we're looking for have already been
1787 * accounted for.
1788 */
1789 flags |= PG_PANIC;
1790
1791 if ((flags & PG_NORELOC) != 0) {
1792 VM_STAT_ADD(pcgs_entered_noreloc);
1793 /*
1794 * Requests for free pages from critical threads
1795 * such as pageout still won't throttle here, but
1796 * we must try again, to give the cageout thread
1797 * another chance to catch up. Since we already
1798 * accounted for the pages, we had better get them
1799 * this time.
1800 *
1801 * N.B. All non-critical threads acquire the pcgs_cagelock
1802 * to serialize access to the freelists. This implements a
1803 * turnstile-type synchornization to avoid starvation of
1804 * critical requests for PG_NORELOC memory by non-critical
1805 * threads: all non-critical threads must acquire a 'ticket'
1806 * before passing through, which entails making sure
1807 * kcage_freemem won't fall below minfree prior to grabbing
1808 * pages from the freelists.
1809 */
1810 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) {
1811 mutex_enter(&pcgs_cagelock);
1812 cagelocked = 1;
1813 VM_STAT_ADD(pcgs_cagelocked);
1814 }
1815 }
1816
1817 /*
1818 * Time to get serious.
1819 * We failed to get a `correctly colored' page from both the
1820 * free and cache lists.
1821 * We escalate in stage.
1822 *
1823 * First try both lists without worring about color.
1824 *
1825 * Then, grab all page accounting locks (ie. pcf[]) and
1826 * steal any pages that they have and set the pcf_block flag to
1827 * stop deletions from the lists. This will help because
1828 * a page can get added to the free list while we are looking
1829 * at the cache list, then another page could be added to the cache
1830 * list allowing the page on the free list to be removed as we
1831 * move from looking at the cache list to the free list. This
1832 * could happen over and over. We would never find the page
1833 * we have accounted for.
1834 *
1835 * Noreloc pages are a subset of the global (relocatable) page pool.
1836 * They are not tracked separately in the pcf bins, so it is
1837 * impossible to know when doing pcf accounting if the available
1838 * page(s) are noreloc pages or not. When looking for a noreloc page
1839 * it is quite easy to end up here even if the global (relocatable)
1840 * page pool has plenty of free pages but the noreloc pool is empty.
1841 *
1842 * When the noreloc pool is empty (or low), additional noreloc pages
1843 * are created by converting pages from the global page pool. This
1844 * process will stall during pcf accounting if the pcf bins are
1845 * already locked. Such is the case when a noreloc allocation is
1846 * looping here in page_create_get_something waiting for more noreloc
1847 * pages to appear.
1848 *
1849 * Short of adding a new field to the pcf bins to accurately track
1850 * the number of free noreloc pages, we instead do not grab the
1851 * pcgs_lock, do not set the pcf blocks and do not timeout when
1852 * allocating a noreloc page. This allows noreloc allocations to
1853 * loop without blocking global page pool allocations.
1854 *
1855 * NOTE: the behaviour of page_create_get_something has not changed
1856 * for the case of global page pool allocations.
1857 */
1858
1859 flags &= ~PG_MATCH_COLOR;
1860 locked = 0;
1861 #if defined(__i386) || defined(__amd64)
1862 flags = page_create_update_flags_x86(flags);
1863 #endif
1864
1865 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
1866
1867 for (count = 0; kcage_on || count < MAX_PCGS; count++) {
1868 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
1869 flags, lgrp);
1870 if (pp == NULL) {
1871 pp = page_get_cachelist(vp, off, seg, vaddr,
1872 flags, lgrp);
1873 }
1874 if (pp == NULL) {
1875 /*
1876 * Serialize. Don't fight with other pcgs().
1877 */
1878 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) {
1879 mutex_enter(&pcgs_lock);
1880 VM_STAT_ADD(pcgs_locked);
1881 locked = 1;
1882 p = pcf;
1883 for (i = 0; i < pcf_fanout; i++) {
1884 mutex_enter(&p->pcf_lock);
1885 ASSERT(p->pcf_block == 0);
1886 p->pcf_block = 1;
1887 p->pcf_reserve = p->pcf_count;
1888 p->pcf_count = 0;
1889 mutex_exit(&p->pcf_lock);
1890 p++;
1891 }
1892 freemem = 0;
1893 }
1894
1895 if (count) {
1896 /*
1897 * Since page_free() puts pages on
1898 * a list then accounts for it, we
1899 * just have to wait for page_free()
1900 * to unlock any page it was working
1901 * with. The page_lock()-page_reclaim()
1902 * path falls in the same boat.
1903 *
1904 * We don't need to check on the
1905 * PG_WAIT flag, we have already
1906 * accounted for the page we are
1907 * looking for in page_create_va().
1908 *
1909 * We just wait a moment to let any
1910 * locked pages on the lists free up,
1911 * then continue around and try again.
1912 *
1913 * Will be awakened by set_freemem().
1914 */
1915 mutex_enter(&pcgs_wait_lock);
1916 cv_wait(&pcgs_cv, &pcgs_wait_lock);
1917 mutex_exit(&pcgs_wait_lock);
1918 }
1919 } else {
1920 #ifdef VM_STATS
1921 if (count >= PCGS_TRIES) {
1922 VM_STAT_ADD(pcgs_too_many);
1923 } else {
1924 VM_STAT_ADD(pcgs_counts[count]);
1925 }
1926 #endif
1927 if (locked) {
1928 pcgs_unblock();
1929 mutex_exit(&pcgs_lock);
1930 }
1931 if (cagelocked)
1932 mutex_exit(&pcgs_cagelock);
1933 return (pp);
1934 }
1935 }
1936 /*
1937 * we go down holding the pcf locks.
1938 */
1939 panic("no %spage found %d",
1940 ((flags & PG_NORELOC) ? "non-reloc " : ""), count);
1941 /*NOTREACHED*/
1942 }
1943
1944 /*
1945 * Create enough pages for "bytes" worth of data starting at
1946 * "off" in "vp".
1947 *
1948 * Where flag must be one of:
1949 *
1950 * PG_EXCL: Exclusive create (fail if any page already
1951 * exists in the page cache) which does not
1952 * wait for memory to become available.
1953 *
1954 * PG_WAIT: Non-exclusive create which can wait for
1955 * memory to become available.
1956 *
1957 * PG_PHYSCONTIG: Allocate physically contiguous pages.
1958 * (Not Supported)
1959 *
1960 * A doubly linked list of pages is returned to the caller. Each page
1961 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
1962 * lock.
1963 *
1964 * Unable to change the parameters to page_create() in a minor release,
1965 * we renamed page_create() to page_create_va(), changed all known calls
1966 * from page_create() to page_create_va(), and created this wrapper.
1967 *
1968 * Upon a major release, we should break compatibility by deleting this
1969 * wrapper, and replacing all the strings "page_create_va", with "page_create".
1970 *
1971 * NOTE: There is a copy of this interface as page_create_io() in
1972 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied
1973 * there.
1974 */
1975 page_t *
1976 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags)
1977 {
1978 caddr_t random_vaddr;
1979 struct seg kseg;
1980
1981 #ifdef DEBUG
1982 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p",
1983 (void *)caller());
1984 #endif
1985
1986 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^
1987 (uintptr_t)(off >> PAGESHIFT));
1988 kseg.s_as = &kas;
1989
1990 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr));
1991 }
1992
1993 #ifdef DEBUG
1994 uint32_t pg_alloc_pgs_mtbf = 0;
1995 #endif
1996
1997 /*
1998 * Used for large page support. It will attempt to allocate
1999 * a large page(s) off the freelist.
2000 *
2001 * Returns non zero on failure.
2002 */
2003 int
2004 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr,
2005 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags)
2006 {
2007 pgcnt_t npgs, curnpgs, totpgs;
2008 size_t pgsz;
2009 page_t *pplist = NULL, *pp;
2010 int err = 0;
2011 lgrp_t *lgrp;
2012
2013 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1));
2014 ASSERT(pgflags == 0 || pgflags == PG_LOCAL);
2015
2016 /*
2017 * Check if system heavily prefers local large pages over remote
2018 * on systems with multiple lgroups.
2019 */
2020 if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) {
2021 pgflags = PG_LOCAL;
2022 }
2023
2024 VM_STAT_ADD(alloc_pages[0]);
2025
2026 #ifdef DEBUG
2027 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) {
2028 return (ENOMEM);
2029 }
2030 #endif
2031
2032 /*
2033 * One must be NULL but not both.
2034 * And one must be non NULL but not both.
2035 */
2036 ASSERT(basepp != NULL || ppa != NULL);
2037 ASSERT(basepp == NULL || ppa == NULL);
2038
2039 #if defined(__i386) || defined(__amd64)
2040 while (page_chk_freelist(szc) == 0) {
2041 VM_STAT_ADD(alloc_pages[8]);
2042 if (anypgsz == 0 || --szc == 0)
2043 return (ENOMEM);
2044 }
2045 #endif
2046
2047 pgsz = page_get_pagesize(szc);
2048 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT;
2049
2050 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0);
2051
2052 (void) page_create_wait(npgs, PG_WAIT);
2053
2054 while (npgs && szc) {
2055 lgrp = lgrp_mem_choose(seg, addr, pgsz);
2056 if (pgflags == PG_LOCAL) {
2057 pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2058 pgflags, lgrp);
2059 if (pp == NULL) {
2060 pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2061 0, lgrp);
2062 }
2063 } else {
2064 pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2065 0, lgrp);
2066 }
2067 if (pp != NULL) {
2068 VM_STAT_ADD(alloc_pages[1]);
2069 page_list_concat(&pplist, &pp);
2070 ASSERT(npgs >= curnpgs);
2071 npgs -= curnpgs;
2072 } else if (anypgsz) {
2073 VM_STAT_ADD(alloc_pages[2]);
2074 szc--;
2075 pgsz = page_get_pagesize(szc);
2076 curnpgs = pgsz >> PAGESHIFT;
2077 } else {
2078 VM_STAT_ADD(alloc_pages[3]);
2079 ASSERT(npgs == totpgs);
2080 page_create_putback(npgs);
2081 return (ENOMEM);
2082 }
2083 }
2084 if (szc == 0) {
2085 VM_STAT_ADD(alloc_pages[4]);
2086 ASSERT(npgs != 0);
2087 page_create_putback(npgs);
2088 err = ENOMEM;
2089 } else if (basepp != NULL) {
2090 ASSERT(npgs == 0);
2091 ASSERT(ppa == NULL);
2092 *basepp = pplist;
2093 }
2094
2095 npgs = totpgs - npgs;
2096 pp = pplist;
2097
2098 /*
2099 * Clear the free and age bits. Also if we were passed in a ppa then
2100 * fill it in with all the constituent pages from the large page. But
2101 * if we failed to allocate all the pages just free what we got.
2102 */
2103 while (npgs != 0) {
2104 ASSERT(PP_ISFREE(pp));
2105 ASSERT(PP_ISAGED(pp));
2106 if (ppa != NULL || err != 0) {
2107 if (err == 0) {
2108 VM_STAT_ADD(alloc_pages[5]);
2109 PP_CLRFREE(pp);
2110 PP_CLRAGED(pp);
2111 page_sub(&pplist, pp);
2112 *ppa++ = pp;
2113 npgs--;
2114 } else {
2115 VM_STAT_ADD(alloc_pages[6]);
2116 ASSERT(pp->p_szc != 0);
2117 curnpgs = page_get_pagecnt(pp->p_szc);
2118 page_list_break(&pp, &pplist, curnpgs);
2119 page_list_add_pages(pp, 0);
2120 page_create_putback(curnpgs);
2121 ASSERT(npgs >= curnpgs);
2122 npgs -= curnpgs;
2123 }
2124 pp = pplist;
2125 } else {
2126 VM_STAT_ADD(alloc_pages[7]);
2127 PP_CLRFREE(pp);
2128 PP_CLRAGED(pp);
2129 pp = pp->p_next;
2130 npgs--;
2131 }
2132 }
2133 return (err);
2134 }
2135
2136 /*
2137 * Get a single large page off of the freelists, and set it up for use.
2138 * Number of bytes requested must be a supported page size.
2139 *
2140 * Note that this call may fail even if there is sufficient
2141 * memory available or PG_WAIT is set, so the caller must
2142 * be willing to fallback on page_create_va(), block and retry,
2143 * or fail the requester.
2144 */
2145 page_t *
2146 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2147 struct seg *seg, caddr_t vaddr, void *arg)
2148 {
2149 pgcnt_t npages;
2150 page_t *pp;
2151 page_t *rootpp;
2152 lgrp_t *lgrp;
2153 lgrp_id_t *lgrpid = (lgrp_id_t *)arg;
2154
2155 ASSERT(vp != NULL);
2156
2157 ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2158 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0);
2159 /* but no others */
2160
2161 ASSERT((flags & PG_EXCL) == PG_EXCL);
2162
2163 npages = btop(bytes);
2164
2165 if (!kcage_on || panicstr) {
2166 /*
2167 * Cage is OFF, or we are single threaded in
2168 * panic, so make everything a RELOC request.
2169 */
2170 flags &= ~PG_NORELOC;
2171 }
2172
2173 /*
2174 * Make sure there's adequate physical memory available.
2175 * Note: PG_WAIT is ignored here.
2176 */
2177 if (freemem <= throttlefree + npages) {
2178 VM_STAT_ADD(page_create_large_cnt[1]);
2179 return (NULL);
2180 }
2181
2182 /*
2183 * If cage is on, dampen draw from cage when available
2184 * cage space is low.
2185 */
2186 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) &&
2187 kcage_freemem < kcage_throttlefree + npages) {
2188
2189 /*
2190 * The cage is on, the caller wants PG_NORELOC
2191 * pages and available cage memory is very low.
2192 * Call kcage_create_throttle() to attempt to
2193 * control demand on the cage.
2194 */
2195 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) {
2196 VM_STAT_ADD(page_create_large_cnt[2]);
2197 return (NULL);
2198 }
2199 }
2200
2201 if (!pcf_decrement_bucket(npages) &&
2202 !pcf_decrement_multiple(NULL, npages, 1)) {
2203 VM_STAT_ADD(page_create_large_cnt[4]);
2204 return (NULL);
2205 }
2206
2207 /*
2208 * This is where this function behaves fundamentally differently
2209 * than page_create_va(); since we're intending to map the page
2210 * with a single TTE, we have to get it as a physically contiguous
2211 * hardware pagesize chunk. If we can't, we fail.
2212 */
2213 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max &&
2214 LGRP_EXISTS(lgrp_table[*lgrpid]))
2215 lgrp = lgrp_table[*lgrpid];
2216 else
2217 lgrp = lgrp_mem_choose(seg, vaddr, bytes);
2218
2219 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr,
2220 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) {
2221 page_create_putback(npages);
2222 VM_STAT_ADD(page_create_large_cnt[5]);
2223 return (NULL);
2224 }
2225
2226 /*
2227 * if we got the page with the wrong mtype give it back this is a
2228 * workaround for CR 6249718. When CR 6249718 is fixed we never get
2229 * inside "if" and the workaround becomes just a nop
2230 */
2231 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) {
2232 page_list_add_pages(rootpp, 0);
2233 page_create_putback(npages);
2234 VM_STAT_ADD(page_create_large_cnt[6]);
2235 return (NULL);
2236 }
2237
2238 /*
2239 * If satisfying this request has left us with too little
2240 * memory, start the wheels turning to get some back. The
2241 * first clause of the test prevents waking up the pageout
2242 * daemon in situations where it would decide that there's
2243 * nothing to do.
2244 */
2245 if (nscan < desscan && freemem < minfree) {
2246 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2247 "pageout_cv_signal:freemem %ld", freemem);
2248 WAKE_PAGEOUT_SCANNER();
2249 }
2250
2251 pp = rootpp;
2252 while (npages--) {
2253 ASSERT(PAGE_EXCL(pp));
2254 ASSERT(pp->p_vnode == NULL);
2255 ASSERT(!hat_page_is_mapped(pp));
2256 PP_CLRFREE(pp);
2257 PP_CLRAGED(pp);
2258 if (!page_hashin(pp, vp, off, NULL))
2259 panic("page_create_large: hashin failed: page %p",
2260 (void *)pp);
2261 page_io_lock(pp);
2262 off += PAGESIZE;
2263 pp = pp->p_next;
2264 }
2265
2266 VM_STAT_ADD(page_create_large_cnt[0]);
2267 return (rootpp);
2268 }
2269
2270 page_t *
2271 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2272 struct seg *seg, caddr_t vaddr)
2273 {
2274 page_t *plist = NULL;
2275 pgcnt_t npages;
2276 pgcnt_t found_on_free = 0;
2277 pgcnt_t pages_req;
2278 page_t *npp = NULL;
2279 struct pcf *p;
2280 lgrp_t *lgrp;
2281
2282 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
2283 "page_create_start:vp %p off %llx bytes %lu flags %x",
2284 vp, off, bytes, flags);
2285
2286 ASSERT(bytes != 0 && vp != NULL);
2287
2288 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) {
2289 panic("page_create: invalid flags");
2290 /*NOTREACHED*/
2291 }
2292 ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2293 PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0);
2294 /* but no others */
2295
2296 pages_req = npages = btopr(bytes);
2297 /*
2298 * Try to see whether request is too large to *ever* be
2299 * satisfied, in order to prevent deadlock. We arbitrarily
2300 * decide to limit maximum size requests to max_page_get.
2301 */
2302 if (npages >= max_page_get) {
2303 if ((flags & PG_WAIT) == 0) {
2304 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG,
2305 "page_create_toobig:vp %p off %llx npages "
2306 "%lu max_page_get %lu",
2307 vp, off, npages, max_page_get);
2308 return (NULL);
2309 } else {
2310 cmn_err(CE_WARN,
2311 "Request for too much kernel memory "
2312 "(%lu bytes), will hang forever", bytes);
2313 for (;;)
2314 delay(1000000000);
2315 }
2316 }
2317
2318 if (!kcage_on || panicstr) {
2319 /*
2320 * Cage is OFF, or we are single threaded in
2321 * panic, so make everything a RELOC request.
2322 */
2323 flags &= ~PG_NORELOC;
2324 }
2325
2326 if (freemem <= throttlefree + npages)
2327 if (!page_create_throttle(npages, flags))
2328 return (NULL);
2329
2330 /*
2331 * If cage is on, dampen draw from cage when available
2332 * cage space is low.
2333 */
2334 if ((flags & PG_NORELOC) &&
2335 kcage_freemem < kcage_throttlefree + npages) {
2336
2337 /*
2338 * The cage is on, the caller wants PG_NORELOC
2339 * pages and available cage memory is very low.
2340 * Call kcage_create_throttle() to attempt to
2341 * control demand on the cage.
2342 */
2343 if (kcage_create_throttle(npages, flags) == KCT_FAILURE)
2344 return (NULL);
2345 }
2346
2347 VM_STAT_ADD(page_create_cnt[0]);
2348
2349 if (!pcf_decrement_bucket(npages)) {
2350 /*
2351 * Have to look harder. If npages is greater than
2352 * one, then we might have to coalesce the counters.
2353 *
2354 * Go wait. We come back having accounted
2355 * for the memory.
2356 */
2357 VM_STAT_ADD(page_create_cnt[1]);
2358 if (!page_create_wait(npages, flags)) {
2359 VM_STAT_ADD(page_create_cnt[2]);
2360 return (NULL);
2361 }
2362 }
2363
2364 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
2365 "page_create_success:vp %p off %llx", vp, off);
2366
2367 /*
2368 * If satisfying this request has left us with too little
2369 * memory, start the wheels turning to get some back. The
2370 * first clause of the test prevents waking up the pageout
2371 * daemon in situations where it would decide that there's
2372 * nothing to do.
2373 */
2374 if (nscan < desscan && freemem < minfree) {
2375 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2376 "pageout_cv_signal:freemem %ld", freemem);
2377 WAKE_PAGEOUT_SCANNER();
2378 }
2379
2380 /*
2381 * Loop around collecting the requested number of pages.
2382 * Most of the time, we have to `create' a new page. With
2383 * this in mind, pull the page off the free list before
2384 * getting the hash lock. This will minimize the hash
2385 * lock hold time, nesting, and the like. If it turns
2386 * out we don't need the page, we put it back at the end.
2387 */
2388 while (npages--) {
2389 page_t *pp;
2390 kmutex_t *phm = NULL;
2391 ulong_t index;
2392
2393 index = PAGE_HASH_FUNC(vp, off);
2394 top:
2395 ASSERT(phm == NULL);
2396 ASSERT(index == PAGE_HASH_FUNC(vp, off));
2397 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
2398
2399 if (npp == NULL) {
2400 /*
2401 * Try to get a page from the freelist (ie,
2402 * a page with no [vp, off] tag). If that
2403 * fails, use the cachelist.
2404 *
2405 * During the first attempt at both the free
2406 * and cache lists we try for the correct color.
2407 */
2408 /*
2409 * XXXX-how do we deal with virtual indexed
2410 * caches and and colors?
2411 */
2412 VM_STAT_ADD(page_create_cnt[4]);
2413 /*
2414 * Get lgroup to allocate next page of shared memory
2415 * from and use it to specify where to allocate
2416 * the physical memory
2417 */
2418 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
2419 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
2420 flags | PG_MATCH_COLOR, lgrp);
2421 if (npp == NULL) {
2422 npp = page_get_cachelist(vp, off, seg,
2423 vaddr, flags | PG_MATCH_COLOR, lgrp);
2424 if (npp == NULL) {
2425 npp = page_create_get_something(vp,
2426 off, seg, vaddr,
2427 flags & ~PG_MATCH_COLOR);
2428 }
2429
2430 if (PP_ISAGED(npp) == 0) {
2431 /*
2432 * Since this page came from the
2433 * cachelist, we must destroy the
2434 * old vnode association.
2435 */
2436 page_hashout(npp, NULL);
2437 }
2438 }
2439 }
2440
2441 /*
2442 * We own this page!
2443 */
2444 ASSERT(PAGE_EXCL(npp));
2445 ASSERT(npp->p_vnode == NULL);
2446 ASSERT(!hat_page_is_mapped(npp));
2447 PP_CLRFREE(npp);
2448 PP_CLRAGED(npp);
2449
2450 /*
2451 * Here we have a page in our hot little mits and are
2452 * just waiting to stuff it on the appropriate lists.
2453 * Get the mutex and check to see if it really does
2454 * not exist.
2455 */
2456 phm = PAGE_HASH_MUTEX(index);
2457 mutex_enter(phm);
2458 pp = page_hash_search(index, vp, off);
2459 if (pp == NULL) {
2460 VM_STAT_ADD(page_create_new);
2461 pp = npp;
2462 npp = NULL;
2463 if (!page_hashin(pp, vp, off, phm)) {
2464 /*
2465 * Since we hold the page hash mutex and
2466 * just searched for this page, page_hashin
2467 * had better not fail. If it does, that
2468 * means somethread did not follow the
2469 * page hash mutex rules. Panic now and
2470 * get it over with. As usual, go down
2471 * holding all the locks.
2472 */
2473 ASSERT(MUTEX_HELD(phm));
2474 panic("page_create: "
2475 "hashin failed %p %p %llx %p",
2476 (void *)pp, (void *)vp, off, (void *)phm);
2477 /*NOTREACHED*/
2478 }
2479 ASSERT(MUTEX_HELD(phm));
2480 mutex_exit(phm);
2481 phm = NULL;
2482
2483 /*
2484 * Hat layer locking need not be done to set
2485 * the following bits since the page is not hashed
2486 * and was on the free list (i.e., had no mappings).
2487 *
2488 * Set the reference bit to protect
2489 * against immediate pageout
2490 *
2491 * XXXmh modify freelist code to set reference
2492 * bit so we don't have to do it here.
2493 */
2494 page_set_props(pp, P_REF);
2495 found_on_free++;
2496 } else {
2497 VM_STAT_ADD(page_create_exists);
2498 if (flags & PG_EXCL) {
2499 /*
2500 * Found an existing page, and the caller
2501 * wanted all new pages. Undo all of the work
2502 * we have done.
2503 */
2504 mutex_exit(phm);
2505 phm = NULL;
2506 while (plist != NULL) {
2507 pp = plist;
2508 page_sub(&plist, pp);
2509 page_io_unlock(pp);
2510 /* large pages should not end up here */
2511 ASSERT(pp->p_szc == 0);
2512 /*LINTED: constant in conditional ctx*/
2513 VN_DISPOSE(pp, B_INVAL, 0, kcred);
2514 }
2515 VM_STAT_ADD(page_create_found_one);
2516 goto fail;
2517 }
2518 ASSERT(flags & PG_WAIT);
2519 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) {
2520 /*
2521 * Start all over again if we blocked trying
2522 * to lock the page.
2523 */
2524 mutex_exit(phm);
2525 VM_STAT_ADD(page_create_page_lock_failed);
2526 phm = NULL;
2527 goto top;
2528 }
2529 mutex_exit(phm);
2530 phm = NULL;
2531
2532 if (PP_ISFREE(pp)) {
2533 ASSERT(PP_ISAGED(pp) == 0);
2534 VM_STAT_ADD(pagecnt.pc_get_cache);
2535 page_list_sub(pp, PG_CACHE_LIST);
2536 PP_CLRFREE(pp);
2537 found_on_free++;
2538 }
2539 }
2540
2541 /*
2542 * Got a page! It is locked. Acquire the i/o
2543 * lock since we are going to use the p_next and
2544 * p_prev fields to link the requested pages together.
2545 */
2546 page_io_lock(pp);
2547 page_add(&plist, pp);
2548 plist = plist->p_next;
2549 off += PAGESIZE;
2550 vaddr += PAGESIZE;
2551 }
2552
2553 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1);
2554 fail:
2555 if (npp != NULL) {
2556 /*
2557 * Did not need this page after all.
2558 * Put it back on the free list.
2559 */
2560 VM_STAT_ADD(page_create_putbacks);
2561 PP_SETFREE(npp);
2562 PP_SETAGED(npp);
2563 npp->p_offset = (u_offset_t)-1;
2564 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
2565 page_unlock(npp);
2566
2567 }
2568
2569 ASSERT(pages_req >= found_on_free);
2570
2571 {
2572 uint_t overshoot = (uint_t)(pages_req - found_on_free);
2573
2574 if (overshoot) {
2575 VM_STAT_ADD(page_create_overshoot);
2576 p = &pcf[PCF_INDEX()];
2577 mutex_enter(&p->pcf_lock);
2578 if (p->pcf_block) {
2579 p->pcf_reserve += overshoot;
2580 } else {
2581 p->pcf_count += overshoot;
2582 if (p->pcf_wait) {
2583 mutex_enter(&new_freemem_lock);
2584 if (freemem_wait) {
2585 cv_signal(&freemem_cv);
2586 p->pcf_wait--;
2587 } else {
2588 p->pcf_wait = 0;
2589 }
2590 mutex_exit(&new_freemem_lock);
2591 }
2592 }
2593 mutex_exit(&p->pcf_lock);
2594 /* freemem is approximate, so this test OK */
2595 if (!p->pcf_block)
2596 freemem += overshoot;
2597 }
2598 }
2599
2600 return (plist);
2601 }
2602
2603 /*
2604 * One or more constituent pages of this large page has been marked
2605 * toxic. Simply demote the large page to PAGESIZE pages and let
2606 * page_free() handle it. This routine should only be called by
2607 * large page free routines (page_free_pages() and page_destroy_pages().
2608 * All pages are locked SE_EXCL and have already been marked free.
2609 */
2610 static void
2611 page_free_toxic_pages(page_t *rootpp)
2612 {
2613 page_t *tpp;
2614 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc);
2615 uint_t szc = rootpp->p_szc;
2616
2617 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) {
2618 ASSERT(tpp->p_szc == szc);
2619 ASSERT((PAGE_EXCL(tpp) &&
2620 !page_iolock_assert(tpp)) || panicstr);
2621 tpp->p_szc = 0;
2622 }
2623
2624 while (rootpp != NULL) {
2625 tpp = rootpp;
2626 page_sub(&rootpp, tpp);
2627 ASSERT(PP_ISFREE(tpp));
2628 PP_CLRFREE(tpp);
2629 page_free(tpp, 1);
2630 }
2631 }
2632
2633 /*
2634 * Put page on the "free" list.
2635 * The free list is really two lists maintained by
2636 * the PSM of whatever machine we happen to be on.
2637 */
2638 void
2639 page_free(page_t *pp, int dontneed)
2640 {
2641 struct pcf *p;
2642 uint_t pcf_index;
2643
2644 ASSERT((PAGE_EXCL(pp) &&
2645 !page_iolock_assert(pp)) || panicstr);
2646
2647 if (PP_ISFREE(pp)) {
2648 panic("page_free: page %p is free", (void *)pp);
2649 }
2650
2651 if (pp->p_szc != 0) {
2652 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
2653 PP_ISKAS(pp)) {
2654 panic("page_free: anon or kernel "
2655 "or no vnode large page %p", (void *)pp);
2656 }
2657 page_demote_vp_pages(pp);
2658 ASSERT(pp->p_szc == 0);
2659 }
2660
2661 /*
2662 * The page_struct_lock need not be acquired to examine these
2663 * fields since the page has an "exclusive" lock.
2664 */
2665 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
2666 pp->p_slckcnt != 0) {
2667 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
2668 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt,
2669 pp->p_cowcnt, pp->p_slckcnt);
2670 /*NOTREACHED*/
2671 }
2672
2673 ASSERT(!hat_page_getshare(pp));
2674
2675 PP_SETFREE(pp);
2676 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) ||
2677 !hat_ismod(pp));
2678 page_clr_all_props(pp);
2679 ASSERT(!hat_page_getshare(pp));
2680
2681 /*
2682 * Now we add the page to the head of the free list.
2683 * But if this page is associated with a paged vnode
2684 * then we adjust the head forward so that the page is
2685 * effectively at the end of the list.
2686 */
2687 if (pp->p_vnode == NULL) {
2688 /*
2689 * Page has no identity, put it on the free list.
2690 */
2691 PP_SETAGED(pp);
2692 pp->p_offset = (u_offset_t)-1;
2693 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
2694 VM_STAT_ADD(pagecnt.pc_free_free);
2695 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2696 "page_free_free:pp %p", pp);
2697 } else {
2698 PP_CLRAGED(pp);
2699
2700 if (!dontneed) {
2701 /* move it to the tail of the list */
2702 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL);
2703
2704 VM_STAT_ADD(pagecnt.pc_free_cache);
2705 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL,
2706 "page_free_cache_tail:pp %p", pp);
2707 } else {
2708 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD);
2709
2710 VM_STAT_ADD(pagecnt.pc_free_dontneed);
2711 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD,
2712 "page_free_cache_head:pp %p", pp);
2713 }
2714 }
2715 page_unlock(pp);
2716
2717 /*
2718 * Now do the `freemem' accounting.
2719 */
2720 pcf_index = PCF_INDEX();
2721 p = &pcf[pcf_index];
2722
2723 mutex_enter(&p->pcf_lock);
2724 if (p->pcf_block) {
2725 p->pcf_reserve += 1;
2726 } else {
2727 p->pcf_count += 1;
2728 if (p->pcf_wait) {
2729 mutex_enter(&new_freemem_lock);
2730 /*
2731 * Check to see if some other thread
2732 * is actually waiting. Another bucket
2733 * may have woken it up by now. If there
2734 * are no waiters, then set our pcf_wait
2735 * count to zero to avoid coming in here
2736 * next time. Also, since only one page
2737 * was put on the free list, just wake
2738 * up one waiter.
2739 */
2740 if (freemem_wait) {
2741 cv_signal(&freemem_cv);
2742 p->pcf_wait--;
2743 } else {
2744 p->pcf_wait = 0;
2745 }
2746 mutex_exit(&new_freemem_lock);
2747 }
2748 }
2749 mutex_exit(&p->pcf_lock);
2750
2751 /* freemem is approximate, so this test OK */
2752 if (!p->pcf_block)
2753 freemem += 1;
2754 }
2755
2756 /*
2757 * Put page on the "free" list during intial startup.
2758 * This happens during initial single threaded execution.
2759 */
2760 void
2761 page_free_at_startup(page_t *pp)
2762 {
2763 struct pcf *p;
2764 uint_t pcf_index;
2765
2766 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT);
2767 VM_STAT_ADD(pagecnt.pc_free_free);
2768
2769 /*
2770 * Now do the `freemem' accounting.
2771 */
2772 pcf_index = PCF_INDEX();
2773 p = &pcf[pcf_index];
2774
2775 ASSERT(p->pcf_block == 0);
2776 ASSERT(p->pcf_wait == 0);
2777 p->pcf_count += 1;
2778
2779 /* freemem is approximate, so this is OK */
2780 freemem += 1;
2781 }
2782
2783 void
2784 page_free_pages(page_t *pp)
2785 {
2786 page_t *tpp, *rootpp = NULL;
2787 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc);
2788 pgcnt_t i;
2789 uint_t szc = pp->p_szc;
2790
2791 VM_STAT_ADD(pagecnt.pc_free_pages);
2792 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2793 "page_free_free:pp %p", pp);
2794
2795 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
2796 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
2797 panic("page_free_pages: not root page %p", (void *)pp);
2798 /*NOTREACHED*/
2799 }
2800
2801 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
2802 ASSERT((PAGE_EXCL(tpp) &&
2803 !page_iolock_assert(tpp)) || panicstr);
2804 if (PP_ISFREE(tpp)) {
2805 panic("page_free_pages: page %p is free", (void *)tpp);
2806 /*NOTREACHED*/
2807 }
2808 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 ||
2809 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) {
2810 panic("page_free_pages %p", (void *)tpp);
2811 /*NOTREACHED*/
2812 }
2813
2814 ASSERT(!hat_page_getshare(tpp));
2815 ASSERT(tpp->p_vnode == NULL);
2816 ASSERT(tpp->p_szc == szc);
2817
2818 PP_SETFREE(tpp);
2819 page_clr_all_props(tpp);
2820 PP_SETAGED(tpp);
2821 tpp->p_offset = (u_offset_t)-1;
2822 ASSERT(tpp->p_next == tpp);
2823 ASSERT(tpp->p_prev == tpp);
2824 page_list_concat(&rootpp, &tpp);
2825 }
2826 ASSERT(rootpp == pp);
2827
2828 page_list_add_pages(rootpp, 0);
2829 page_create_putback(pgcnt);
2830 }
2831
2832 int free_pages = 1;
2833
2834 /*
2835 * This routine attempts to return pages to the cachelist via page_release().
2836 * It does not *have* to be successful in all cases, since the pageout scanner
2837 * will catch any pages it misses. It does need to be fast and not introduce
2838 * too much overhead.
2839 *
2840 * If a page isn't found on the unlocked sweep of the page_hash bucket, we
2841 * don't lock and retry. This is ok, since the page scanner will eventually
2842 * find any page we miss in free_vp_pages().
2843 */
2844 void
2845 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len)
2846 {
2847 page_t *pp;
2848 u_offset_t eoff;
2849 extern int swap_in_range(vnode_t *, u_offset_t, size_t);
2850
2851 eoff = off + len;
2852
2853 if (free_pages == 0)
2854 return;
2855 if (swap_in_range(vp, off, len))
2856 return;
2857
2858 for (; off < eoff; off += PAGESIZE) {
2859
2860 /*
2861 * find the page using a fast, but inexact search. It'll be OK
2862 * if a few pages slip through the cracks here.
2863 */
2864 pp = page_exists(vp, off);
2865
2866 /*
2867 * If we didn't find the page (it may not exist), the page
2868 * is free, looks still in use (shared), or we can't lock it,
2869 * just give up.
2870 */
2871 if (pp == NULL ||
2872 PP_ISFREE(pp) ||
2873 page_share_cnt(pp) > 0 ||
2874 !page_trylock(pp, SE_EXCL))
2875 continue;
2876
2877 /*
2878 * Once we have locked pp, verify that it's still the
2879 * correct page and not already free
2880 */
2881 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL));
2882 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) {
2883 page_unlock(pp);
2884 continue;
2885 }
2886
2887 /*
2888 * try to release the page...
2889 */
2890 (void) page_release(pp, 1);
2891 }
2892 }
2893
2894 /*
2895 * Reclaim the given page from the free list.
2896 * If pp is part of a large pages, only the given constituent page is reclaimed
2897 * and the large page it belonged to will be demoted. This can only happen
2898 * if the page is not on the cachelist.
2899 *
2900 * Returns 1 on success or 0 on failure.
2901 *
2902 * The page is unlocked if it can't be reclaimed (when freemem == 0).
2903 * If `lock' is non-null, it will be dropped and re-acquired if
2904 * the routine must wait while freemem is 0.
2905 *
2906 * As it turns out, boot_getpages() does this. It picks a page,
2907 * based on where OBP mapped in some address, gets its pfn, searches
2908 * the memsegs, locks the page, then pulls it off the free list!
2909 */
2910 int
2911 page_reclaim(page_t *pp, kmutex_t *lock)
2912 {
2913 struct pcf *p;
2914 struct cpu *cpup;
2915 int enough;
2916 uint_t i;
2917
2918 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
2919 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
2920
2921 /*
2922 * If `freemem' is 0, we cannot reclaim this page from the
2923 * freelist, so release every lock we might hold: the page,
2924 * and the `lock' before blocking.
2925 *
2926 * The only way `freemem' can become 0 while there are pages
2927 * marked free (have their p->p_free bit set) is when the
2928 * system is low on memory and doing a page_create(). In
2929 * order to guarantee that once page_create() starts acquiring
2930 * pages it will be able to get all that it needs since `freemem'
2931 * was decreased by the requested amount. So, we need to release
2932 * this page, and let page_create() have it.
2933 *
2934 * Since `freemem' being zero is not supposed to happen, just
2935 * use the usual hash stuff as a starting point. If that bucket
2936 * is empty, then assume the worst, and start at the beginning
2937 * of the pcf array. If we always start at the beginning
2938 * when acquiring more than one pcf lock, there won't be any
2939 * deadlock problems.
2940 */
2941
2942 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
2943
2944 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) {
2945 pcf_acquire_all();
2946 goto page_reclaim_nomem;
2947 }
2948
2949 enough = pcf_decrement_bucket(1);
2950
2951 if (!enough) {
2952 VM_STAT_ADD(page_reclaim_zero);
2953 /*
2954 * Check again. Its possible that some other thread
2955 * could have been right behind us, and added one
2956 * to a list somewhere. Acquire each of the pcf locks
2957 * until we find a page.
2958 */
2959 p = pcf;
2960 for (i = 0; i < pcf_fanout; i++) {
2961 mutex_enter(&p->pcf_lock);
2962 if (p->pcf_count >= 1) {
2963 p->pcf_count -= 1;
2964 /*
2965 * freemem is not protected by any lock. Thus,
2966 * we cannot have any assertion containing
2967 * freemem here.
2968 */
2969 freemem -= 1;
2970 enough = 1;
2971 break;
2972 }
2973 p++;
2974 }
2975
2976 if (!enough) {
2977 page_reclaim_nomem:
2978 /*
2979 * We really can't have page `pp'.
2980 * Time for the no-memory dance with
2981 * page_free(). This is just like
2982 * page_create_wait(). Plus the added
2983 * attraction of releasing whatever mutex
2984 * we held when we were called with in `lock'.
2985 * Page_unlock() will wakeup any thread
2986 * waiting around for this page.
2987 */
2988 if (lock) {
2989 VM_STAT_ADD(page_reclaim_zero_locked);
2990 mutex_exit(lock);
2991 }
2992 page_unlock(pp);
2993
2994 /*
2995 * get this before we drop all the pcf locks.
2996 */
2997 mutex_enter(&new_freemem_lock);
2998
2999 p = pcf;
3000 for (i = 0; i < pcf_fanout; i++) {
3001 p->pcf_wait++;
3002 mutex_exit(&p->pcf_lock);
3003 p++;
3004 }
3005
3006 freemem_wait++;
3007 cv_wait(&freemem_cv, &new_freemem_lock);
3008 freemem_wait--;
3009
3010 mutex_exit(&new_freemem_lock);
3011
3012 if (lock) {
3013 mutex_enter(lock);
3014 }
3015 return (0);
3016 }
3017
3018 /*
3019 * The pcf accounting has been done,
3020 * though none of the pcf_wait flags have been set,
3021 * drop the locks and continue on.
3022 */
3023 while (p >= pcf) {
3024 mutex_exit(&p->pcf_lock);
3025 p--;
3026 }
3027 }
3028
3029
3030 VM_STAT_ADD(pagecnt.pc_reclaim);
3031
3032 /*
3033 * page_list_sub will handle the case where pp is a large page.
3034 * It's possible that the page was promoted while on the freelist
3035 */
3036 if (PP_ISAGED(pp)) {
3037 page_list_sub(pp, PG_FREE_LIST);
3038 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
3039 "page_reclaim_free:pp %p", pp);
3040 } else {
3041 page_list_sub(pp, PG_CACHE_LIST);
3042 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
3043 "page_reclaim_cache:pp %p", pp);
3044 }
3045
3046 /*
3047 * clear the p_free & p_age bits since this page is no longer
3048 * on the free list. Notice that there was a brief time where
3049 * a page is marked as free, but is not on the list.
3050 *
3051 * Set the reference bit to protect against immediate pageout.
3052 */
3053 PP_CLRFREE(pp);
3054 PP_CLRAGED(pp);
3055 page_set_props(pp, P_REF);
3056
3057 CPU_STATS_ENTER_K();
3058 cpup = CPU; /* get cpup now that CPU cannot change */
3059 CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
3060 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
3061 CPU_STATS_EXIT_K();
3062 ASSERT(pp->p_szc == 0);
3063
3064 return (1);
3065 }
3066
3067 /*
3068 * Destroy identity of the page and put it back on
3069 * the page free list. Assumes that the caller has
3070 * acquired the "exclusive" lock on the page.
3071 */
3072 void
3073 page_destroy(page_t *pp, int dontfree)
3074 {
3075 ASSERT((PAGE_EXCL(pp) &&
3076 !page_iolock_assert(pp)) || panicstr);
3077 ASSERT(pp->p_slckcnt == 0 || panicstr);
3078
3079 if (pp->p_szc != 0) {
3080 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
3081 PP_ISKAS(pp)) {
3082 panic("page_destroy: anon or kernel or no vnode "
3083 "large page %p", (void *)pp);
3084 }
3085 page_demote_vp_pages(pp);
3086 ASSERT(pp->p_szc == 0);
3087 }
3088
3089 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp);
3090
3091 /*
3092 * Unload translations, if any, then hash out the
3093 * page to erase its identity.
3094 */
3095 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3096 page_hashout(pp, NULL);
3097
3098 if (!dontfree) {
3099 /*
3100 * Acquire the "freemem_lock" for availrmem.
3101 * The page_struct_lock need not be acquired for lckcnt
3102 * and cowcnt since the page has an "exclusive" lock.
3103 * We are doing a modified version of page_pp_unlock here.
3104 */
3105 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
3106 mutex_enter(&freemem_lock);
3107 if (pp->p_lckcnt != 0) {
3108 availrmem++;
3109 pages_locked--;
3110 pp->p_lckcnt = 0;
3111 }
3112 if (pp->p_cowcnt != 0) {
3113 availrmem += pp->p_cowcnt;
3114 pages_locked -= pp->p_cowcnt;
3115 pp->p_cowcnt = 0;
3116 }
3117 mutex_exit(&freemem_lock);
3118 }
3119 /*
3120 * Put the page on the "free" list.
3121 */
3122 page_free(pp, 0);
3123 }
3124 }
3125
3126 void
3127 page_destroy_pages(page_t *pp)
3128 {
3129
3130 page_t *tpp, *rootpp = NULL;
3131 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc);
3132 pgcnt_t i, pglcks = 0;
3133 uint_t szc = pp->p_szc;
3134
3135 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
3136
3137 VM_STAT_ADD(pagecnt.pc_destroy_pages);
3138
3139 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp);
3140
3141 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
3142 panic("page_destroy_pages: not root page %p", (void *)pp);
3143 /*NOTREACHED*/
3144 }
3145
3146 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
3147 ASSERT((PAGE_EXCL(tpp) &&
3148 !page_iolock_assert(tpp)) || panicstr);
3149 ASSERT(tpp->p_slckcnt == 0 || panicstr);
3150 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
3151 page_hashout(tpp, NULL);
3152 ASSERT(tpp->p_offset == (u_offset_t)-1);
3153 if (tpp->p_lckcnt != 0) {
3154 pglcks++;
3155 tpp->p_lckcnt = 0;
3156 } else if (tpp->p_cowcnt != 0) {
3157 pglcks += tpp->p_cowcnt;
3158 tpp->p_cowcnt = 0;
3159 }
3160 ASSERT(!hat_page_getshare(tpp));
3161 ASSERT(tpp->p_vnode == NULL);
3162 ASSERT(tpp->p_szc == szc);
3163
3164 PP_SETFREE(tpp);
3165 page_clr_all_props(tpp);
3166 PP_SETAGED(tpp);
3167 ASSERT(tpp->p_next == tpp);
3168 ASSERT(tpp->p_prev == tpp);
3169 page_list_concat(&rootpp, &tpp);
3170 }
3171
3172 ASSERT(rootpp == pp);
3173 if (pglcks != 0) {
3174 mutex_enter(&freemem_lock);
3175 availrmem += pglcks;
3176 mutex_exit(&freemem_lock);
3177 }
3178
3179 page_list_add_pages(rootpp, 0);
3180 page_create_putback(pgcnt);
3181 }
3182
3183 /*
3184 * Similar to page_destroy(), but destroys pages which are
3185 * locked and known to be on the page free list. Since
3186 * the page is known to be free and locked, no one can access
3187 * it.
3188 *
3189 * Also, the number of free pages does not change.
3190 */
3191 void
3192 page_destroy_free(page_t *pp)
3193 {
3194 ASSERT(PAGE_EXCL(pp));
3195 ASSERT(PP_ISFREE(pp));
3196 ASSERT(pp->p_vnode);
3197 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0);
3198 ASSERT(!hat_page_is_mapped(pp));
3199 ASSERT(PP_ISAGED(pp) == 0);
3200 ASSERT(pp->p_szc == 0);
3201
3202 VM_STAT_ADD(pagecnt.pc_destroy_free);
3203 page_list_sub(pp, PG_CACHE_LIST);
3204
3205 page_hashout(pp, NULL);
3206 ASSERT(pp->p_vnode == NULL);
3207 ASSERT(pp->p_offset == (u_offset_t)-1);
3208 ASSERT(pp->p_hash == NULL);
3209
3210 PP_SETAGED(pp);
3211 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
3212 page_unlock(pp);
3213
3214 mutex_enter(&new_freemem_lock);
3215 if (freemem_wait) {
3216 cv_signal(&freemem_cv);
3217 }
3218 mutex_exit(&new_freemem_lock);
3219 }
3220
3221 /*
3222 * Rename the page "opp" to have an identity specified
3223 * by [vp, off]. If a page already exists with this name
3224 * it is locked and destroyed. Note that the page's
3225 * translations are not unloaded during the rename.
3226 *
3227 * This routine is used by the anon layer to "steal" the
3228 * original page and is not unlike destroying a page and
3229 * creating a new page using the same page frame.
3230 *
3231 * XXX -- Could deadlock if caller 1 tries to rename A to B while
3232 * caller 2 tries to rename B to A.
3233 */
3234 void
3235 page_rename(page_t *opp, vnode_t *vp, u_offset_t off)
3236 {
3237 page_t *pp;
3238 int olckcnt = 0;
3239 int ocowcnt = 0;
3240 kmutex_t *phm;
3241 ulong_t index;
3242
3243 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp));
3244 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3245 ASSERT(PP_ISFREE(opp) == 0);
3246
3247 VM_STAT_ADD(page_rename_count);
3248
3249 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME,
3250 "page rename:pp %p vp %p off %llx", opp, vp, off);
3251
3252 /*
3253 * CacheFS may call page_rename for a large NFS page
3254 * when both CacheFS and NFS mount points are used
3255 * by applications. Demote this large page before
3256 * renaming it, to ensure that there are no "partial"
3257 * large pages left lying around.
3258 */
3259 if (opp->p_szc != 0) {
3260 vnode_t *ovp = opp->p_vnode;
3261 ASSERT(ovp != NULL);
3262 ASSERT(!IS_SWAPFSVP(ovp));
3263 ASSERT(!VN_ISKAS(ovp));
3264 page_demote_vp_pages(opp);
3265 ASSERT(opp->p_szc == 0);
3266 }
3267
3268 page_hashout(opp, NULL);
3269 PP_CLRAGED(opp);
3270
3271 /*
3272 * Acquire the appropriate page hash lock, since
3273 * we're going to rename the page.
3274 */
3275 index = PAGE_HASH_FUNC(vp, off);
3276 phm = PAGE_HASH_MUTEX(index);
3277 mutex_enter(phm);
3278 top:
3279 /*
3280 * Look for an existing page with this name and destroy it if found.
3281 * By holding the page hash lock all the way to the page_hashin()
3282 * call, we are assured that no page can be created with this
3283 * identity. In the case when the phm lock is dropped to undo any
3284 * hat layer mappings, the existing page is held with an "exclusive"
3285 * lock, again preventing another page from being created with
3286 * this identity.
3287 */
3288 pp = page_hash_search(index, vp, off);
3289 if (pp != NULL) {
3290 VM_STAT_ADD(page_rename_exists);
3291
3292 /*
3293 * As it turns out, this is one of only two places where
3294 * page_lock() needs to hold the passed in lock in the
3295 * successful case. In all of the others, the lock could
3296 * be dropped as soon as the attempt is made to lock
3297 * the page. It is tempting to add yet another arguement,
3298 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3299 */
3300 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3301 /*
3302 * Went to sleep because the page could not
3303 * be locked. We were woken up when the page
3304 * was unlocked, or when the page was destroyed.
3305 * In either case, `phm' was dropped while we
3306 * slept. Hence we should not just roar through
3307 * this loop.
3308 */
3309 goto top;
3310 }
3311
3312 /*
3313 * If an existing page is a large page, then demote
3314 * it to ensure that no "partial" large pages are
3315 * "created" after page_rename. An existing page
3316 * can be a CacheFS page, and can't belong to swapfs.
3317 */
3318 if (hat_page_is_mapped(pp)) {
3319 /*
3320 * Unload translations. Since we hold the
3321 * exclusive lock on this page, the page
3322 * can not be changed while we drop phm.
3323 * This is also not a lock protocol violation,
3324 * but rather the proper way to do things.
3325 */
3326 mutex_exit(phm);
3327 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3328 if (pp->p_szc != 0) {
3329 ASSERT(!IS_SWAPFSVP(vp));
3330 ASSERT(!VN_ISKAS(vp));
3331 page_demote_vp_pages(pp);
3332 ASSERT(pp->p_szc == 0);
3333 }
3334 mutex_enter(phm);
3335 } else if (pp->p_szc != 0) {
3336 ASSERT(!IS_SWAPFSVP(vp));
3337 ASSERT(!VN_ISKAS(vp));
3338 mutex_exit(phm);
3339 page_demote_vp_pages(pp);
3340 ASSERT(pp->p_szc == 0);
3341 mutex_enter(phm);
3342 }
3343 page_hashout(pp, phm);
3344 }
3345 /*
3346 * Hash in the page with the new identity.
3347 */
3348 if (!page_hashin(opp, vp, off, phm)) {
3349 /*
3350 * We were holding phm while we searched for [vp, off]
3351 * and only dropped phm if we found and locked a page.
3352 * If we can't create this page now, then some thing
3353 * is really broken.
3354 */
3355 panic("page_rename: Can't hash in page: %p", (void *)pp);
3356 /*NOTREACHED*/
3357 }
3358
3359 ASSERT(MUTEX_HELD(phm));
3360 mutex_exit(phm);
3361
3362 /*
3363 * Now that we have dropped phm, lets get around to finishing up
3364 * with pp.
3365 */
3366 if (pp != NULL) {
3367 ASSERT(!hat_page_is_mapped(pp));
3368 /* for now large pages should not end up here */
3369 ASSERT(pp->p_szc == 0);
3370 /*
3371 * Save the locks for transfer to the new page and then
3372 * clear them so page_free doesn't think they're important.
3373 * The page_struct_lock need not be acquired for lckcnt and
3374 * cowcnt since the page has an "exclusive" lock.
3375 */
3376 olckcnt = pp->p_lckcnt;
3377 ocowcnt = pp->p_cowcnt;
3378 pp->p_lckcnt = pp->p_cowcnt = 0;
3379
3380 /*
3381 * Put the page on the "free" list after we drop
3382 * the lock. The less work under the lock the better.
3383 */
3384 /*LINTED: constant in conditional context*/
3385 VN_DISPOSE(pp, B_FREE, 0, kcred);
3386 }
3387
3388 /*
3389 * Transfer the lock count from the old page (if any).
3390 * The page_struct_lock need not be acquired for lckcnt and
3391 * cowcnt since the page has an "exclusive" lock.
3392 */
3393 opp->p_lckcnt += olckcnt;
3394 opp->p_cowcnt += ocowcnt;
3395 }
3396
3397 /*
3398 * low level routine to add page `pp' to the hash and vp chains for [vp, offset]
3399 *
3400 * Pages are normally inserted at the start of a vnode's v_pages list.
3401 * If the vnode is VMODSORT and the page is modified, it goes at the end.
3402 * This can happen when a modified page is relocated for DR.
3403 *
3404 * Returns 1 on success and 0 on failure.
3405 */
3406 static int
3407 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset)
3408 {
3409 page_t **listp;
3410 page_t *tp;
3411 ulong_t index;
3412
3413 ASSERT(PAGE_EXCL(pp));
3414 ASSERT(vp != NULL);
3415 ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3416
3417 /*
3418 * Be sure to set these up before the page is inserted on the hash
3419 * list. As soon as the page is placed on the list some other
3420 * thread might get confused and wonder how this page could
3421 * possibly hash to this list.
3422 */
3423 pp->p_vnode = vp;
3424 pp->p_offset = offset;
3425
3426 /*
3427 * record if this page is on a swap vnode
3428 */
3429 if ((vp->v_flag & VISSWAP) != 0)
3430 PP_SETSWAP(pp);
3431
3432 index = PAGE_HASH_FUNC(vp, offset);
3433 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index)));
3434 listp = &page_hash[index];
3435
3436 /*
3437 * If this page is already hashed in, fail this attempt to add it.
3438 */
3439 for (tp = *listp; tp != NULL; tp = tp->p_hash) {
3440 if (tp->p_vnode == vp && tp->p_offset == offset) {
3441 pp->p_vnode = NULL;
3442 pp->p_offset = (u_offset_t)(-1);
3443 return (0);
3444 }
3445 }
3446 pp->p_hash = *listp;
3447 *listp = pp;
3448
3449 /*
3450 * Add the page to the vnode's list of pages
3451 */
3452 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp))
3453 listp = &vp->v_pages->p_vpprev->p_vpnext;
3454 else
3455 listp = &vp->v_pages;
3456
3457 page_vpadd(listp, pp);
3458
3459 return (1);
3460 }
3461
3462 /*
3463 * Add page `pp' to both the hash and vp chains for [vp, offset].
3464 *
3465 * Returns 1 on success and 0 on failure.
3466 * If hold is passed in, it is not dropped.
3467 */
3468 int
3469 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold)
3470 {
3471 kmutex_t *phm = NULL;
3472 kmutex_t *vphm;
3473 int rc;
3474
3475 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3476 ASSERT(pp->p_fsdata == 0 || panicstr);
3477
3478 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN,
3479 "page_hashin:pp %p vp %p offset %llx",
3480 pp, vp, offset);
3481
3482 VM_STAT_ADD(hashin_count);
3483
3484 if (hold != NULL)
3485 phm = hold;
3486 else {
3487 VM_STAT_ADD(hashin_not_held);
3488 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset));
3489 mutex_enter(phm);
3490 }
3491
3492 vphm = page_vnode_mutex(vp);
3493 mutex_enter(vphm);
3494 rc = page_do_hashin(pp, vp, offset);
3495 mutex_exit(vphm);
3496 if (hold == NULL)
3497 mutex_exit(phm);
3498 if (rc == 0)
3499 VM_STAT_ADD(hashin_already);
3500 return (rc);
3501 }
3502
3503 /*
3504 * Remove page ``pp'' from the hash and vp chains and remove vp association.
3505 * All mutexes must be held
3506 */
3507 static void
3508 page_do_hashout(page_t *pp)
3509 {
3510 page_t **hpp;
3511 page_t *hp;
3512 vnode_t *vp = pp->p_vnode;
3513
3514 ASSERT(vp != NULL);
3515 ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3516
3517 /*
3518 * First, take pp off of its hash chain.
3519 */
3520 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)];
3521
3522 for (;;) {
3523 hp = *hpp;
3524 if (hp == pp)
3525 break;
3526 if (hp == NULL) {
3527 panic("page_do_hashout");
3528 /*NOTREACHED*/
3529 }
3530 hpp = &hp->p_hash;
3531 }
3532 *hpp = pp->p_hash;
3533
3534 /*
3535 * Now remove it from its associated vnode.
3536 */
3537 if (vp->v_pages)
3538 page_vpsub(&vp->v_pages, pp);
3539
3540 pp->p_hash = NULL;
3541 page_clr_all_props(pp);
3542 PP_CLRSWAP(pp);
3543 pp->p_vnode = NULL;
3544 pp->p_offset = (u_offset_t)-1;
3545 pp->p_fsdata = 0;
3546 }
3547
3548 /*
3549 * Remove page ``pp'' from the hash and vp chains and remove vp association.
3550 *
3551 * When `phm' is non-NULL it contains the address of the mutex protecting the
3552 * hash list pp is on. It is not dropped.
3553 */
3554 void
3555 page_hashout(page_t *pp, kmutex_t *phm)
3556 {
3557 vnode_t *vp;
3558 ulong_t index;
3559 kmutex_t *nphm;
3560 kmutex_t *vphm;
3561 kmutex_t *sep;
3562
3563 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1);
3564 ASSERT(pp->p_vnode != NULL);
3565 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr);
3566 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode)));
3567
3568 vp = pp->p_vnode;
3569
3570 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT,
3571 "page_hashout:pp %p vp %p", pp, vp);
3572
3573 /* Kernel probe */
3574 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */,
3575 tnf_opaque, vnode, vp,
3576 tnf_offset, offset, pp->p_offset);
3577
3578 /*
3579 *
3580 */
3581 VM_STAT_ADD(hashout_count);
3582 index = PAGE_HASH_FUNC(vp, pp->p_offset);
3583 if (phm == NULL) {
3584 VM_STAT_ADD(hashout_not_held);
3585 nphm = PAGE_HASH_MUTEX(index);
3586 mutex_enter(nphm);
3587 }
3588 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1);
3589
3590
3591 /*
3592 * grab page vnode mutex and remove it...
3593 */
3594 vphm = page_vnode_mutex(vp);
3595 mutex_enter(vphm);
3596
3597 page_do_hashout(pp);
3598
3599 mutex_exit(vphm);
3600 if (phm == NULL)
3601 mutex_exit(nphm);
3602
3603 /*
3604 * Wake up processes waiting for this page. The page's
3605 * identity has been changed, and is probably not the
3606 * desired page any longer.
3607 */
3608 sep = page_se_mutex(pp);
3609 mutex_enter(sep);
3610 pp->p_selock &= ~SE_EWANTED;
3611 if (CV_HAS_WAITERS(&pp->p_cv))
3612 cv_broadcast(&pp->p_cv);
3613 mutex_exit(sep);
3614 }
3615
3616 /*
3617 * Add the page to the front of a linked list of pages
3618 * using the p_next & p_prev pointers for the list.
3619 * The caller is responsible for protecting the list pointers.
3620 */
3621 void
3622 page_add(page_t **ppp, page_t *pp)
3623 {
3624 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3625
3626 page_add_common(ppp, pp);
3627 }
3628
3629
3630
3631 /*
3632 * Common code for page_add() and mach_page_add()
3633 */
3634 void
3635 page_add_common(page_t **ppp, page_t *pp)
3636 {
3637 if (*ppp == NULL) {
3638 pp->p_next = pp->p_prev = pp;
3639 } else {
3640 pp->p_next = *ppp;
3641 pp->p_prev = (*ppp)->p_prev;
3642 (*ppp)->p_prev = pp;
3643 pp->p_prev->p_next = pp;
3644 }
3645 *ppp = pp;
3646 }
3647
3648
3649 /*
3650 * Remove this page from a linked list of pages
3651 * using the p_next & p_prev pointers for the list.
3652 *
3653 * The caller is responsible for protecting the list pointers.
3654 */
3655 void
3656 page_sub(page_t **ppp, page_t *pp)
3657 {
3658 ASSERT((PP_ISFREE(pp)) ? 1 :
3659 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3660
3661 if (*ppp == NULL || pp == NULL) {
3662 panic("page_sub: bad arg(s): pp %p, *ppp %p",
3663 (void *)pp, (void *)(*ppp));
3664 /*NOTREACHED*/
3665 }
3666
3667 page_sub_common(ppp, pp);
3668 }
3669
3670
3671 /*
3672 * Common code for page_sub() and mach_page_sub()
3673 */
3674 void
3675 page_sub_common(page_t **ppp, page_t *pp)
3676 {
3677 if (*ppp == pp)
3678 *ppp = pp->p_next; /* go to next page */
3679
3680 if (*ppp == pp)
3681 *ppp = NULL; /* page list is gone */
3682 else {
3683 pp->p_prev->p_next = pp->p_next;
3684 pp->p_next->p_prev = pp->p_prev;
3685 }
3686 pp->p_prev = pp->p_next = pp; /* make pp a list of one */
3687 }
3688
3689
3690 /*
3691 * Break page list cppp into two lists with npages in the first list.
3692 * The tail is returned in nppp.
3693 */
3694 void
3695 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages)
3696 {
3697 page_t *s1pp = *oppp;
3698 page_t *s2pp;
3699 page_t *e1pp, *e2pp;
3700 long n = 0;
3701
3702 if (s1pp == NULL) {
3703 *nppp = NULL;
3704 return;
3705 }
3706 if (npages == 0) {
3707 *nppp = s1pp;
3708 *oppp = NULL;
3709 return;
3710 }
3711 for (n = 0, s2pp = *oppp; n < npages; n++) {
3712 s2pp = s2pp->p_next;
3713 }
3714 /* Fix head and tail of new lists */
3715 e1pp = s2pp->p_prev;
3716 e2pp = s1pp->p_prev;
3717 s1pp->p_prev = e1pp;
3718 e1pp->p_next = s1pp;
3719 s2pp->p_prev = e2pp;
3720 e2pp->p_next = s2pp;
3721
3722 /* second list empty */
3723 if (s2pp == s1pp) {
3724 *oppp = s1pp;
3725 *nppp = NULL;
3726 } else {
3727 *oppp = s1pp;
3728 *nppp = s2pp;
3729 }
3730 }
3731
3732 /*
3733 * Concatenate page list nppp onto the end of list ppp.
3734 */
3735 void
3736 page_list_concat(page_t **ppp, page_t **nppp)
3737 {
3738 page_t *s1pp, *s2pp, *e1pp, *e2pp;
3739
3740 if (*nppp == NULL) {
3741 return;
3742 }
3743 if (*ppp == NULL) {
3744 *ppp = *nppp;
3745 return;
3746 }
3747 s1pp = *ppp;
3748 e1pp = s1pp->p_prev;
3749 s2pp = *nppp;
3750 e2pp = s2pp->p_prev;
3751 s1pp->p_prev = e2pp;
3752 e2pp->p_next = s1pp;
3753 e1pp->p_next = s2pp;
3754 s2pp->p_prev = e1pp;
3755 }
3756
3757 /*
3758 * return the next page in the page list
3759 */
3760 page_t *
3761 page_list_next(page_t *pp)
3762 {
3763 return (pp->p_next);
3764 }
3765
3766
3767 /*
3768 * Add the page to the front of the linked list of pages
3769 * using p_vpnext/p_vpprev pointers for the list.
3770 *
3771 * The caller is responsible for protecting the lists.
3772 */
3773 void
3774 page_vpadd(page_t **ppp, page_t *pp)
3775 {
3776 if (*ppp == NULL) {
3777 pp->p_vpnext = pp->p_vpprev = pp;
3778 } else {
3779 pp->p_vpnext = *ppp;
3780 pp->p_vpprev = (*ppp)->p_vpprev;
3781 (*ppp)->p_vpprev = pp;
3782 pp->p_vpprev->p_vpnext = pp;
3783 }
3784 *ppp = pp;
3785 }
3786
3787 /*
3788 * Remove this page from the linked list of pages
3789 * using p_vpnext/p_vpprev pointers for the list.
3790 *
3791 * The caller is responsible for protecting the lists.
3792 */
3793 void
3794 page_vpsub(page_t **ppp, page_t *pp)
3795 {
3796 if (*ppp == NULL || pp == NULL) {
3797 panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
3798 (void *)pp, (void *)(*ppp));
3799 /*NOTREACHED*/
3800 }
3801
3802 if (*ppp == pp)
3803 *ppp = pp->p_vpnext; /* go to next page */
3804
3805 if (*ppp == pp)
3806 *ppp = NULL; /* page list is gone */
3807 else {
3808 pp->p_vpprev->p_vpnext = pp->p_vpnext;
3809 pp->p_vpnext->p_vpprev = pp->p_vpprev;
3810 }
3811 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */
3812 }
3813
3814 /*
3815 * Lock a physical page into memory "long term". Used to support "lock
3816 * in memory" functions. Accepts the page to be locked, and a cow variable
3817 * to indicate whether a the lock will travel to the new page during
3818 * a potential copy-on-write.
3819 */
3820 int
3821 page_pp_lock(
3822 page_t *pp, /* page to be locked */
3823 int cow, /* cow lock */
3824 int kernel) /* must succeed -- ignore checking */
3825 {
3826 int r = 0; /* result -- assume failure */
3827
3828 ASSERT(PAGE_LOCKED(pp));
3829
3830 page_struct_lock(pp);
3831 /*
3832 * Acquire the "freemem_lock" for availrmem.
3833 */
3834 if (cow) {
3835 mutex_enter(&freemem_lock);
3836 if ((availrmem > pages_pp_maximum) &&
3837 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
3838 availrmem--;
3839 pages_locked++;
3840 mutex_exit(&freemem_lock);
3841 r = 1;
3842 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
3843 cmn_err(CE_WARN,
3844 "COW lock limit reached on pfn 0x%lx",
3845 page_pptonum(pp));
3846 }
3847 } else
3848 mutex_exit(&freemem_lock);
3849 } else {
3850 if (pp->p_lckcnt) {
3851 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
3852 r = 1;
3853 if (++pp->p_lckcnt ==
3854 (ushort_t)PAGE_LOCK_MAXIMUM) {
3855 cmn_err(CE_WARN, "Page lock limit "
3856 "reached on pfn 0x%lx",
3857 page_pptonum(pp));
3858 }
3859 }
3860 } else {
3861 if (kernel) {
3862 /* availrmem accounting done by caller */
3863 ++pp->p_lckcnt;
3864 r = 1;
3865 } else {
3866 mutex_enter(&freemem_lock);
3867 if (availrmem > pages_pp_maximum) {
3868 availrmem--;
3869 pages_locked++;
3870 ++pp->p_lckcnt;
3871 r = 1;
3872 }
3873 mutex_exit(&freemem_lock);
3874 }
3875 }
3876 }
3877 page_struct_unlock(pp);
3878 return (r);
3879 }
3880
3881 /*
3882 * Decommit a lock on a physical page frame. Account for cow locks if
3883 * appropriate.
3884 */
3885 void
3886 page_pp_unlock(
3887 page_t *pp, /* page to be unlocked */
3888 int cow, /* expect cow lock */
3889 int kernel) /* this was a kernel lock */
3890 {
3891 ASSERT(PAGE_LOCKED(pp));
3892
3893 page_struct_lock(pp);
3894 /*
3895 * Acquire the "freemem_lock" for availrmem.
3896 * If cowcnt or lcknt is already 0 do nothing; i.e., we
3897 * could be called to unlock even if nothing is locked. This could
3898 * happen if locked file pages were truncated (removing the lock)
3899 * and the file was grown again and new pages faulted in; the new
3900 * pages are unlocked but the segment still thinks they're locked.
3901 */
3902 if (cow) {
3903 if (pp->p_cowcnt) {
3904 mutex_enter(&freemem_lock);
3905 pp->p_cowcnt--;
3906 availrmem++;
3907 pages_locked--;
3908 mutex_exit(&freemem_lock);
3909 }
3910 } else {
3911 if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
3912 if (!kernel) {
3913 mutex_enter(&freemem_lock);
3914 availrmem++;
3915 pages_locked--;
3916 mutex_exit(&freemem_lock);
3917 }
3918 }
3919 }
3920 page_struct_unlock(pp);
3921 }
3922
3923 /*
3924 * This routine reserves availrmem for npages;
3925 * flags: KM_NOSLEEP or KM_SLEEP
3926 * returns 1 on success or 0 on failure
3927 */
3928 int
3929 page_resv(pgcnt_t npages, uint_t flags)
3930 {
3931 mutex_enter(&freemem_lock);
3932 while (availrmem < tune.t_minarmem + npages) {
3933 if (flags & KM_NOSLEEP) {
3934 mutex_exit(&freemem_lock);
3935 return (0);
3936 }
3937 mutex_exit(&freemem_lock);
3938 page_needfree(npages);
3939 kmem_reap();
3940 delay(hz >> 2);
3941 page_needfree(-(spgcnt_t)npages);
3942 mutex_enter(&freemem_lock);
3943 }
3944 availrmem -= npages;
3945 mutex_exit(&freemem_lock);
3946 return (1);
3947 }
3948
3949 /*
3950 * This routine unreserves availrmem for npages;
3951 */
3952 void
3953 page_unresv(pgcnt_t npages)
3954 {
3955 mutex_enter(&freemem_lock);
3956 availrmem += npages;
3957 mutex_exit(&freemem_lock);
3958 }
3959
3960 /*
3961 * See Statement at the beginning of segvn_lockop() regarding
3962 * the way we handle cowcnts and lckcnts.
3963 *
3964 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
3965 * that breaks COW has PROT_WRITE.
3966 *
3967 * Note that, we may also break COW in case we are softlocking
3968 * on read access during physio;
3969 * in this softlock case, the vpage may not have PROT_WRITE.
3970 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
3971 * if the vpage doesn't have PROT_WRITE.
3972 *
3973 * This routine is never called if we are stealing a page
3974 * in anon_private.
3975 *
3976 * The caller subtracted from availrmem for read only mapping.
3977 * if lckcnt is 1 increment availrmem.
3978 */
3979 void
3980 page_pp_useclaim(
3981 page_t *opp, /* original page frame losing lock */
3982 page_t *npp, /* new page frame gaining lock */
3983 uint_t write_perm) /* set if vpage has PROT_WRITE */
3984 {
3985 int payback = 0;
3986 int nidx, oidx;
3987
3988 ASSERT(PAGE_LOCKED(opp));
3989 ASSERT(PAGE_LOCKED(npp));
3990
3991 /*
3992 * Since we have two pages we probably have two locks. We need to take
3993 * them in a defined order to avoid deadlocks. It's also possible they
3994 * both hash to the same lock in which case this is a non-issue.
3995 */
3996 nidx = PAGE_LLOCK_HASH(PP_PAGEROOT(npp));
3997 oidx = PAGE_LLOCK_HASH(PP_PAGEROOT(opp));
3998 if (nidx < oidx) {
3999 page_struct_lock(npp);
4000 page_struct_lock(opp);
4001 } else if (oidx < nidx) {
4002 page_struct_lock(opp);
4003 page_struct_lock(npp);
4004 } else { /* The pages hash to the same lock */
4005 page_struct_lock(npp);
4006 }
4007
4008 ASSERT(npp->p_cowcnt == 0);
4009 ASSERT(npp->p_lckcnt == 0);
4010
4011 /* Don't use claim if nothing is locked (see page_pp_unlock above) */
4012 if ((write_perm && opp->p_cowcnt != 0) ||
4013 (!write_perm && opp->p_lckcnt != 0)) {
4014
4015 if (write_perm) {
4016 npp->p_cowcnt++;
4017 ASSERT(opp->p_cowcnt != 0);
4018 opp->p_cowcnt--;
4019 } else {
4020
4021 ASSERT(opp->p_lckcnt != 0);
4022
4023 /*
4024 * We didn't need availrmem decremented if p_lckcnt on
4025 * original page is 1. Here, we are unlocking
4026 * read-only copy belonging to original page and
4027 * are locking a copy belonging to new page.
4028 */
4029 if (opp->p_lckcnt == 1)
4030 payback = 1;
4031
4032 npp->p_lckcnt++;
4033 opp->p_lckcnt--;
4034 }
4035 }
4036 if (payback) {
4037 mutex_enter(&freemem_lock);
4038 availrmem++;
4039 pages_useclaim--;
4040 mutex_exit(&freemem_lock);
4041 }
4042
4043 if (nidx < oidx) {
4044 page_struct_unlock(opp);
4045 page_struct_unlock(npp);
4046 } else if (oidx < nidx) {
4047 page_struct_unlock(npp);
4048 page_struct_unlock(opp);
4049 } else { /* The pages hash to the same lock */
4050 page_struct_unlock(npp);
4051 }
4052 }
4053
4054 /*
4055 * Simple claim adjust functions -- used to support changes in
4056 * claims due to changes in access permissions. Used by segvn_setprot().
4057 */
4058 int
4059 page_addclaim(page_t *pp)
4060 {
4061 int r = 0; /* result */
4062
4063 ASSERT(PAGE_LOCKED(pp));
4064
4065 page_struct_lock(pp);
4066 ASSERT(pp->p_lckcnt != 0);
4067
4068 if (pp->p_lckcnt == 1) {
4069 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4070 --pp->p_lckcnt;
4071 r = 1;
4072 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4073 cmn_err(CE_WARN,
4074 "COW lock limit reached on pfn 0x%lx",
4075 page_pptonum(pp));
4076 }
4077 }
4078 } else {
4079 mutex_enter(&freemem_lock);
4080 if ((availrmem > pages_pp_maximum) &&
4081 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
4082 --availrmem;
4083 ++pages_claimed;
4084 mutex_exit(&freemem_lock);
4085 --pp->p_lckcnt;
4086 r = 1;
4087 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4088 cmn_err(CE_WARN,
4089 "COW lock limit reached on pfn 0x%lx",
4090 page_pptonum(pp));
4091 }
4092 } else
4093 mutex_exit(&freemem_lock);
4094 }
4095 page_struct_unlock(pp);
4096 return (r);
4097 }
4098
4099 int
4100 page_subclaim(page_t *pp)
4101 {
4102 int r = 0;
4103
4104 ASSERT(PAGE_LOCKED(pp));
4105
4106 page_struct_lock(pp);
4107 ASSERT(pp->p_cowcnt != 0);
4108
4109 if (pp->p_lckcnt) {
4110 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4111 r = 1;
4112 /*
4113 * for availrmem
4114 */
4115 mutex_enter(&freemem_lock);
4116 availrmem++;
4117 pages_claimed--;
4118 mutex_exit(&freemem_lock);
4119
4120 pp->p_cowcnt--;
4121
4122 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4123 cmn_err(CE_WARN,
4124 "Page lock limit reached on pfn 0x%lx",
4125 page_pptonum(pp));
4126 }
4127 }
4128 } else {
4129 r = 1;
4130 pp->p_cowcnt--;
4131 pp->p_lckcnt++;
4132 }
4133 page_struct_unlock(pp);
4134 return (r);
4135 }
4136
4137 /*
4138 * Variant of page_addclaim(), where ppa[] contains the pages of a single large
4139 * page.
4140 */
4141 int
4142 page_addclaim_pages(page_t **ppa)
4143 {
4144 pgcnt_t lckpgs = 0, pg_idx;
4145
4146 VM_STAT_ADD(pagecnt.pc_addclaim_pages);
4147
4148 /*
4149 * Only need to take the page struct lock on the large page root.
4150 */
4151 page_struct_lock(ppa[0]);
4152 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4153
4154 ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4155 ASSERT(ppa[pg_idx]->p_lckcnt != 0);
4156 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4157 page_struct_unlock(ppa[0]);
4158 return (0);
4159 }
4160 if (ppa[pg_idx]->p_lckcnt > 1)
4161 lckpgs++;
4162 }
4163
4164 if (lckpgs != 0) {
4165 mutex_enter(&freemem_lock);
4166 if (availrmem >= pages_pp_maximum + lckpgs) {
4167 availrmem -= lckpgs;
4168 pages_claimed += lckpgs;
4169 } else {
4170 mutex_exit(&freemem_lock);
4171 page_struct_unlock(ppa[0]);
4172 return (0);
4173 }
4174 mutex_exit(&freemem_lock);
4175 }
4176
4177 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4178 ppa[pg_idx]->p_lckcnt--;
4179 ppa[pg_idx]->p_cowcnt++;
4180 }
4181 page_struct_unlock(ppa[0]);
4182 return (1);
4183 }
4184
4185 /*
4186 * Variant of page_subclaim(), where ppa[] contains the pages of a single large
4187 * page.
4188 */
4189 int
4190 page_subclaim_pages(page_t **ppa)
4191 {
4192 pgcnt_t ulckpgs = 0, pg_idx;
4193
4194 VM_STAT_ADD(pagecnt.pc_subclaim_pages);
4195
4196 /*
4197 * Only need to take the page struct lock on the large page root.
4198 */
4199 page_struct_lock(ppa[0]);
4200 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4201
4202 ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4203 ASSERT(ppa[pg_idx]->p_cowcnt != 0);
4204 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4205 page_struct_unlock(ppa[0]);
4206 return (0);
4207 }
4208 if (ppa[pg_idx]->p_lckcnt != 0)
4209 ulckpgs++;
4210 }
4211
4212 if (ulckpgs != 0) {
4213 mutex_enter(&freemem_lock);
4214 availrmem += ulckpgs;
4215 pages_claimed -= ulckpgs;
4216 mutex_exit(&freemem_lock);
4217 }
4218
4219 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4220 ppa[pg_idx]->p_cowcnt--;
4221 ppa[pg_idx]->p_lckcnt++;
4222
4223 }
4224 page_struct_unlock(ppa[0]);
4225 return (1);
4226 }
4227
4228 page_t *
4229 page_numtopp(pfn_t pfnum, se_t se)
4230 {
4231 page_t *pp;
4232
4233 retry:
4234 pp = page_numtopp_nolock(pfnum);
4235 if (pp == NULL) {
4236 return ((page_t *)NULL);
4237 }
4238
4239 /*
4240 * Acquire the appropriate lock on the page.
4241 */
4242 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) {
4243 if (page_pptonum(pp) != pfnum)
4244 goto retry;
4245 continue;
4246 }
4247
4248 if (page_pptonum(pp) != pfnum) {
4249 page_unlock(pp);
4250 goto retry;
4251 }
4252
4253 return (pp);
4254 }
4255
4256 page_t *
4257 page_numtopp_noreclaim(pfn_t pfnum, se_t se)
4258 {
4259 page_t *pp;
4260
4261 retry:
4262 pp = page_numtopp_nolock(pfnum);
4263 if (pp == NULL) {
4264 return ((page_t *)NULL);
4265 }
4266
4267 /*
4268 * Acquire the appropriate lock on the page.
4269 */
4270 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) {
4271 if (page_pptonum(pp) != pfnum)
4272 goto retry;
4273 continue;
4274 }
4275
4276 if (page_pptonum(pp) != pfnum) {
4277 page_unlock(pp);
4278 goto retry;
4279 }
4280
4281 return (pp);
4282 }
4283
4284 /*
4285 * This routine is like page_numtopp, but will only return page structs
4286 * for pages which are ok for loading into hardware using the page struct.
4287 */
4288 page_t *
4289 page_numtopp_nowait(pfn_t pfnum, se_t se)
4290 {
4291 page_t *pp;
4292
4293 retry:
4294 pp = page_numtopp_nolock(pfnum);
4295 if (pp == NULL) {
4296 return ((page_t *)NULL);
4297 }
4298
4299 /*
4300 * Try to acquire the appropriate lock on the page.
4301 */
4302 if (PP_ISFREE(pp))
4303 pp = NULL;
4304 else {
4305 if (!page_trylock(pp, se))
4306 pp = NULL;
4307 else {
4308 if (page_pptonum(pp) != pfnum) {
4309 page_unlock(pp);
4310 goto retry;
4311 }
4312 if (PP_ISFREE(pp)) {
4313 page_unlock(pp);
4314 pp = NULL;
4315 }
4316 }
4317 }
4318 return (pp);
4319 }
4320
4321 /*
4322 * Returns a count of dirty pages that are in the process
4323 * of being written out. If 'cleanit' is set, try to push the page.
4324 */
4325 pgcnt_t
4326 page_busy(int cleanit)
4327 {
4328 page_t *page0 = page_first();
4329 page_t *pp = page0;
4330 pgcnt_t nppbusy = 0;
4331 u_offset_t off;
4332
4333 do {
4334 vnode_t *vp = pp->p_vnode;
4335 /*
4336 * A page is a candidate for syncing if it is:
4337 *
4338 * (a) On neither the freelist nor the cachelist
4339 * (b) Hashed onto a vnode
4340 * (c) Not a kernel page
4341 * (d) Dirty
4342 * (e) Not part of a swapfile
4343 * (f) a page which belongs to a real vnode; eg has a non-null
4344 * v_vfsp pointer.
4345 * (g) Backed by a filesystem which doesn't have a
4346 * stubbed-out sync operation
4347 */
4348 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) &&
4349 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL &&
4350 vfs_can_sync(vp->v_vfsp)) {
4351 nppbusy++;
4352
4353 if (!cleanit)
4354 continue;
4355 if (!page_trylock(pp, SE_EXCL))
4356 continue;
4357
4358 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) ||
4359 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
4360 !(hat_pagesync(pp,
4361 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) {
4362 page_unlock(pp);
4363 continue;
4364 }
4365 off = pp->p_offset;
4366 VN_HOLD(vp);
4367 page_unlock(pp);
4368 (void) VOP_PUTPAGE(vp, off, PAGESIZE,
4369 B_ASYNC | B_FREE, kcred, NULL);
4370 VN_RELE(vp);
4371 }
4372 } while ((pp = page_next(pp)) != page0);
4373
4374 return (nppbusy);
4375 }
4376
4377 void page_invalidate_pages(void);
4378
4379 /*
4380 * callback handler to vm sub-system
4381 *
4382 * callers make sure no recursive entries to this func.
4383 */
4384 /*ARGSUSED*/
4385 boolean_t
4386 callb_vm_cpr(void *arg, int code)
4387 {
4388 if (code == CB_CODE_CPR_CHKPT)
4389 page_invalidate_pages();
4390 return (B_TRUE);
4391 }
4392
4393 /*
4394 * Invalidate all pages of the system.
4395 * It shouldn't be called until all user page activities are all stopped.
4396 */
4397 void
4398 page_invalidate_pages()
4399 {
4400 page_t *pp;
4401 page_t *page0;
4402 pgcnt_t nbusypages;
4403 int retry = 0;
4404 const int MAXRETRIES = 4;
4405 top:
4406 /*
4407 * Flush dirty pages and destroy the clean ones.
4408 */
4409 nbusypages = 0;
4410
4411 pp = page0 = page_first();
4412 do {
4413 struct vnode *vp;
4414 u_offset_t offset;
4415 int mod;
4416
4417 /*
4418 * skip the page if it has no vnode or the page associated
4419 * with the kernel vnode or prom allocated kernel mem.
4420 */
4421 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp))
4422 continue;
4423
4424 /*
4425 * skip the page which is already free invalidated.
4426 */
4427 if (PP_ISFREE(pp) && PP_ISAGED(pp))
4428 continue;
4429
4430 /*
4431 * skip pages that are already locked or can't be "exclusively"
4432 * locked or are already free. After we lock the page, check
4433 * the free and age bits again to be sure it's not destroyed
4434 * yet.
4435 * To achieve max. parallelization, we use page_trylock instead
4436 * of page_lock so that we don't get block on individual pages
4437 * while we have thousands of other pages to process.
4438 */
4439 if (!page_trylock(pp, SE_EXCL)) {
4440 nbusypages++;
4441 continue;
4442 } else if (PP_ISFREE(pp)) {
4443 if (!PP_ISAGED(pp)) {
4444 page_destroy_free(pp);
4445 } else {
4446 page_unlock(pp);
4447 }
4448 continue;
4449 }
4450 /*
4451 * Is this page involved in some I/O? shared?
4452 *
4453 * The page_struct_lock need not be acquired to
4454 * examine these fields since the page has an
4455 * "exclusive" lock.
4456 */
4457 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
4458 page_unlock(pp);
4459 continue;
4460 }
4461
4462 if (vp->v_type == VCHR) {
4463 panic("vp->v_type == VCHR");
4464 /*NOTREACHED*/
4465 }
4466
4467 if (!page_try_demote_pages(pp)) {
4468 page_unlock(pp);
4469 continue;
4470 }
4471
4472 /*
4473 * Check the modified bit. Leave the bits alone in hardware
4474 * (they will be modified if we do the putpage).
4475 */
4476 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD)
4477 & P_MOD);
4478 if (mod) {
4479 offset = pp->p_offset;
4480 /*
4481 * Hold the vnode before releasing the page lock
4482 * to prevent it from being freed and re-used by
4483 * some other thread.
4484 */
4485 VN_HOLD(vp);
4486 page_unlock(pp);
4487 /*
4488 * No error return is checked here. Callers such as
4489 * cpr deals with the dirty pages at the dump time
4490 * if this putpage fails.
4491 */
4492 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL,
4493 kcred, NULL);
4494 VN_RELE(vp);
4495 } else {
4496 /*LINTED: constant in conditional context*/
4497 VN_DISPOSE(pp, B_INVAL, 0, kcred);
4498 }
4499 } while ((pp = page_next(pp)) != page0);
4500 if (nbusypages && retry++ < MAXRETRIES) {
4501 delay(1);
4502 goto top;
4503 }
4504 }
4505
4506 /*
4507 * Replace the page "old" with the page "new" on the page hash and vnode lists
4508 *
4509 * the replacement must be done in place, ie the equivalent sequence:
4510 *
4511 * vp = old->p_vnode;
4512 * off = old->p_offset;
4513 * page_do_hashout(old)
4514 * page_do_hashin(new, vp, off)
4515 *
4516 * doesn't work, since
4517 * 1) if old is the only page on the vnode, the v_pages list has a window
4518 * where it looks empty. This will break file system assumptions.
4519 * and
4520 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
4521 */
4522 static void
4523 page_do_relocate_hash(page_t *new, page_t *old)
4524 {
4525 page_t **hash_list;
4526 vnode_t *vp = old->p_vnode;
4527 kmutex_t *sep;
4528
4529 ASSERT(PAGE_EXCL(old));
4530 ASSERT(PAGE_EXCL(new));
4531 ASSERT(vp != NULL);
4532 ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
4533 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset))));
4534
4535 /*
4536 * First find old page on the page hash list
4537 */
4538 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)];
4539
4540 for (;;) {
4541 if (*hash_list == old)
4542 break;
4543 if (*hash_list == NULL) {
4544 panic("page_do_hashout");
4545 /*NOTREACHED*/
4546 }
4547 hash_list = &(*hash_list)->p_hash;
4548 }
4549
4550 /*
4551 * update new and replace old with new on the page hash list
4552 */
4553 new->p_vnode = old->p_vnode;
4554 new->p_offset = old->p_offset;
4555 new->p_hash = old->p_hash;
4556 *hash_list = new;
4557
4558 if ((new->p_vnode->v_flag & VISSWAP) != 0)
4559 PP_SETSWAP(new);
4560
4561 /*
4562 * replace old with new on the vnode's page list
4563 */
4564 if (old->p_vpnext == old) {
4565 new->p_vpnext = new;
4566 new->p_vpprev = new;
4567 } else {
4568 new->p_vpnext = old->p_vpnext;
4569 new->p_vpprev = old->p_vpprev;
4570 new->p_vpnext->p_vpprev = new;
4571 new->p_vpprev->p_vpnext = new;
4572 }
4573 if (vp->v_pages == old)
4574 vp->v_pages = new;
4575
4576 /*
4577 * clear out the old page
4578 */
4579 old->p_hash = NULL;
4580 old->p_vpnext = NULL;
4581 old->p_vpprev = NULL;
4582 old->p_vnode = NULL;
4583 PP_CLRSWAP(old);
4584 old->p_offset = (u_offset_t)-1;
4585 page_clr_all_props(old);
4586
4587 /*
4588 * Wake up processes waiting for this page. The page's
4589 * identity has been changed, and is probably not the
4590 * desired page any longer.
4591 */
4592 sep = page_se_mutex(old);
4593 mutex_enter(sep);
4594 old->p_selock &= ~SE_EWANTED;
4595 if (CV_HAS_WAITERS(&old->p_cv))
4596 cv_broadcast(&old->p_cv);
4597 mutex_exit(sep);
4598 }
4599
4600 /*
4601 * This function moves the identity of page "pp_old" to page "pp_new".
4602 * Both pages must be locked on entry. "pp_new" is free, has no identity,
4603 * and need not be hashed out from anywhere.
4604 */
4605 void
4606 page_relocate_hash(page_t *pp_new, page_t *pp_old)
4607 {
4608 vnode_t *vp = pp_old->p_vnode;
4609 u_offset_t off = pp_old->p_offset;
4610 kmutex_t *phm, *vphm;
4611
4612 /*
4613 * Rehash two pages
4614 */
4615 ASSERT(PAGE_EXCL(pp_old));
4616 ASSERT(PAGE_EXCL(pp_new));
4617 ASSERT(vp != NULL);
4618 ASSERT(pp_new->p_vnode == NULL);
4619
4620 /*
4621 * hashout then hashin while holding the mutexes
4622 */
4623 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off));
4624 mutex_enter(phm);
4625 vphm = page_vnode_mutex(vp);
4626 mutex_enter(vphm);
4627
4628 page_do_relocate_hash(pp_new, pp_old);
4629
4630 /* The following comment preserved from page_flip(). */
4631 pp_new->p_fsdata = pp_old->p_fsdata;
4632 pp_old->p_fsdata = 0;
4633 mutex_exit(vphm);
4634 mutex_exit(phm);
4635
4636 /*
4637 * The page_struct_lock need not be acquired for lckcnt and
4638 * cowcnt since the page has an "exclusive" lock.
4639 */
4640 ASSERT(pp_new->p_lckcnt == 0);
4641 ASSERT(pp_new->p_cowcnt == 0);
4642 pp_new->p_lckcnt = pp_old->p_lckcnt;
4643 pp_new->p_cowcnt = pp_old->p_cowcnt;
4644 pp_old->p_lckcnt = pp_old->p_cowcnt = 0;
4645
4646 }
4647
4648 /*
4649 * Helper routine used to lock all remaining members of a
4650 * large page. The caller is responsible for passing in a locked
4651 * pp. If pp is a large page, then it succeeds in locking all the
4652 * remaining constituent pages or it returns with only the
4653 * original page locked.
4654 *
4655 * Returns 1 on success, 0 on failure.
4656 *
4657 * If success is returned this routine guarantees p_szc for all constituent
4658 * pages of a large page pp belongs to can't change. To achieve this we
4659 * recheck szc of pp after locking all constituent pages and retry if szc
4660 * changed (it could only decrease). Since hat_page_demote() needs an EXCL
4661 * lock on one of constituent pages it can't be running after all constituent
4662 * pages are locked. hat_page_demote() with a lock on a constituent page
4663 * outside of this large page (i.e. pp belonged to a larger large page) is
4664 * already done with all constituent pages of pp since the root's p_szc is
4665 * changed last. Therefore no need to synchronize with hat_page_demote() that
4666 * locked a constituent page outside of pp's current large page.
4667 */
4668 #ifdef DEBUG
4669 uint32_t gpg_trylock_mtbf = 0;
4670 #endif
4671
4672 int
4673 group_page_trylock(page_t *pp, se_t se)
4674 {
4675 page_t *tpp;
4676 pgcnt_t npgs, i, j;
4677 uint_t pszc = pp->p_szc;
4678
4679 #ifdef DEBUG
4680 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) {
4681 return (0);
4682 }
4683 #endif
4684
4685 if (pp != PP_GROUPLEADER(pp, pszc)) {
4686 return (0);
4687 }
4688
4689 retry:
4690 ASSERT(PAGE_LOCKED_SE(pp, se));
4691 ASSERT(!PP_ISFREE(pp));
4692 if (pszc == 0) {
4693 return (1);
4694 }
4695 npgs = page_get_pagecnt(pszc);
4696 tpp = pp + 1;
4697 for (i = 1; i < npgs; i++, tpp++) {
4698 if (!page_trylock(tpp, se)) {
4699 tpp = pp + 1;
4700 for (j = 1; j < i; j++, tpp++) {
4701 page_unlock(tpp);
4702 }
4703 return (0);
4704 }
4705 }
4706 if (pp->p_szc != pszc) {
4707 ASSERT(pp->p_szc < pszc);
4708 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) &&
4709 !IS_SWAPFSVP(pp->p_vnode));
4710 tpp = pp + 1;
4711 for (i = 1; i < npgs; i++, tpp++) {
4712 page_unlock(tpp);
4713 }
4714 pszc = pp->p_szc;
4715 goto retry;
4716 }
4717 return (1);
4718 }
4719
4720 void
4721 group_page_unlock(page_t *pp)
4722 {
4723 page_t *tpp;
4724 pgcnt_t npgs, i;
4725
4726 ASSERT(PAGE_LOCKED(pp));
4727 ASSERT(!PP_ISFREE(pp));
4728 ASSERT(pp == PP_PAGEROOT(pp));
4729 npgs = page_get_pagecnt(pp->p_szc);
4730 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) {
4731 page_unlock(tpp);
4732 }
4733 }
4734
4735 /*
4736 * returns
4737 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages
4738 * ERANGE : this is not a base page
4739 * EBUSY : failure to get locks on the page/pages
4740 * ENOMEM : failure to obtain replacement pages
4741 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel
4742 * EIO : An error occurred while trying to copy the page data
4743 *
4744 * Return with all constituent members of target and replacement
4745 * SE_EXCL locked. It is the callers responsibility to drop the
4746 * locks.
4747 */
4748 int
4749 do_page_relocate(
4750 page_t **target,
4751 page_t **replacement,
4752 int grouplock,
4753 spgcnt_t *nrelocp,
4754 lgrp_t *lgrp)
4755 {
4756 page_t *first_repl;
4757 page_t *repl;
4758 page_t *targ;
4759 page_t *pl = NULL;
4760 uint_t ppattr;
4761 pfn_t pfn, repl_pfn;
4762 uint_t szc;
4763 spgcnt_t npgs, i;
4764 int repl_contig = 0;
4765 uint_t flags = 0;
4766 spgcnt_t dofree = 0;
4767
4768 *nrelocp = 0;
4769
4770 #if defined(__sparc)
4771 /*
4772 * We need to wait till OBP has completed
4773 * its boot-time handoff of its resources to the kernel
4774 * before we allow page relocation
4775 */
4776 if (page_relocate_ready == 0) {
4777 return (EAGAIN);
4778 }
4779 #endif
4780
4781 /*
4782 * If this is not a base page,
4783 * just return with 0x0 pages relocated.
4784 */
4785 targ = *target;
4786 ASSERT(PAGE_EXCL(targ));
4787 ASSERT(!PP_ISFREE(targ));
4788 szc = targ->p_szc;
4789 ASSERT(szc < mmu_page_sizes);
4790 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4791 pfn = targ->p_pagenum;
4792 if (pfn != PFN_BASE(pfn, szc)) {
4793 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]);
4794 return (ERANGE);
4795 }
4796
4797 if ((repl = *replacement) != NULL && repl->p_szc >= szc) {
4798 repl_pfn = repl->p_pagenum;
4799 if (repl_pfn != PFN_BASE(repl_pfn, szc)) {
4800 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]);
4801 return (ERANGE);
4802 }
4803 repl_contig = 1;
4804 }
4805
4806 /*
4807 * We must lock all members of this large page or we cannot
4808 * relocate any part of it.
4809 */
4810 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) {
4811 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]);
4812 return (EBUSY);
4813 }
4814
4815 /*
4816 * reread szc it could have been decreased before
4817 * group_page_trylock() was done.
4818 */
4819 szc = targ->p_szc;
4820 ASSERT(szc < mmu_page_sizes);
4821 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4822 ASSERT(pfn == PFN_BASE(pfn, szc));
4823
4824 npgs = page_get_pagecnt(targ->p_szc);
4825
4826 if (repl == NULL) {
4827 dofree = npgs; /* Size of target page in MMU pages */
4828 if (!page_create_wait(dofree, 0)) {
4829 if (grouplock != 0) {
4830 group_page_unlock(targ);
4831 }
4832 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4833 return (ENOMEM);
4834 }
4835
4836 /*
4837 * seg kmem pages require that the target and replacement
4838 * page be the same pagesize.
4839 */
4840 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0;
4841 repl = page_get_replacement_page(targ, lgrp, flags);
4842 if (repl == NULL) {
4843 if (grouplock != 0) {
4844 group_page_unlock(targ);
4845 }
4846 page_create_putback(dofree);
4847 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4848 return (ENOMEM);
4849 }
4850 }
4851 #ifdef DEBUG
4852 else {
4853 ASSERT(PAGE_LOCKED(repl));
4854 }
4855 #endif /* DEBUG */
4856
4857 #if defined(__sparc)
4858 /*
4859 * Let hat_page_relocate() complete the relocation if it's kernel page
4860 */
4861 if (VN_ISKAS(targ->p_vnode)) {
4862 *replacement = repl;
4863 if (hat_page_relocate(target, replacement, nrelocp) != 0) {
4864 if (grouplock != 0) {
4865 group_page_unlock(targ);
4866 }
4867 if (dofree) {
4868 *replacement = NULL;
4869 page_free_replacement_page(repl);
4870 page_create_putback(dofree);
4871 }
4872 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]);
4873 return (EAGAIN);
4874 }
4875 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4876 return (0);
4877 }
4878 #else
4879 #if defined(lint)
4880 dofree = dofree;
4881 #endif
4882 #endif
4883
4884 first_repl = repl;
4885
4886 for (i = 0; i < npgs; i++) {
4887 ASSERT(PAGE_EXCL(targ));
4888 ASSERT(targ->p_slckcnt == 0);
4889 ASSERT(repl->p_slckcnt == 0);
4890
4891 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD);
4892
4893 ASSERT(hat_page_getshare(targ) == 0);
4894 ASSERT(!PP_ISFREE(targ));
4895 ASSERT(targ->p_pagenum == (pfn + i));
4896 ASSERT(repl_contig == 0 ||
4897 repl->p_pagenum == (repl_pfn + i));
4898
4899 /*
4900 * Copy the page contents and attributes then
4901 * relocate the page in the page hash.
4902 */
4903 if (ppcopy(targ, repl) == 0) {
4904 targ = *target;
4905 repl = first_repl;
4906 VM_STAT_ADD(vmm_vmstats.ppr_copyfail);
4907 if (grouplock != 0) {
4908 group_page_unlock(targ);
4909 }
4910 if (dofree) {
4911 *replacement = NULL;
4912 page_free_replacement_page(repl);
4913 page_create_putback(dofree);
4914 }
4915 return (EIO);
4916 }
4917
4918 targ++;
4919 if (repl_contig != 0) {
4920 repl++;
4921 } else {
4922 repl = repl->p_next;
4923 }
4924 }
4925
4926 repl = first_repl;
4927 targ = *target;
4928
4929 for (i = 0; i < npgs; i++) {
4930 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO));
4931 page_clr_all_props(repl);
4932 page_set_props(repl, ppattr);
4933 page_relocate_hash(repl, targ);
4934
4935 ASSERT(hat_page_getshare(targ) == 0);
4936 ASSERT(hat_page_getshare(repl) == 0);
4937 /*
4938 * Now clear the props on targ, after the
4939 * page_relocate_hash(), they no longer
4940 * have any meaning.
4941 */
4942 page_clr_all_props(targ);
4943 ASSERT(targ->p_next == targ);
4944 ASSERT(targ->p_prev == targ);
4945 page_list_concat(&pl, &targ);
4946
4947 targ++;
4948 if (repl_contig != 0) {
4949 repl++;
4950 } else {
4951 repl = repl->p_next;
4952 }
4953 }
4954 /* assert that we have come full circle with repl */
4955 ASSERT(repl_contig == 1 || first_repl == repl);
4956
4957 *target = pl;
4958 if (*replacement == NULL) {
4959 ASSERT(first_repl == repl);
4960 *replacement = repl;
4961 }
4962 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4963 *nrelocp = npgs;
4964 return (0);
4965 }
4966 /*
4967 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
4968 */
4969 int
4970 page_relocate(
4971 page_t **target,
4972 page_t **replacement,
4973 int grouplock,
4974 int freetarget,
4975 spgcnt_t *nrelocp,
4976 lgrp_t *lgrp)
4977 {
4978 spgcnt_t ret;
4979
4980 /* do_page_relocate returns 0 on success or errno value */
4981 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp);
4982
4983 if (ret != 0 || freetarget == 0) {
4984 return (ret);
4985 }
4986 if (*nrelocp == 1) {
4987 ASSERT(*target != NULL);
4988 page_free(*target, 1);
4989 } else {
4990 page_t *tpp = *target;
4991 uint_t szc = tpp->p_szc;
4992 pgcnt_t npgs = page_get_pagecnt(szc);
4993 ASSERT(npgs > 1);
4994 ASSERT(szc != 0);
4995 do {
4996 ASSERT(PAGE_EXCL(tpp));
4997 ASSERT(!hat_page_is_mapped(tpp));
4998 ASSERT(tpp->p_szc == szc);
4999 PP_SETFREE(tpp);
5000 PP_SETAGED(tpp);
5001 npgs--;
5002 } while ((tpp = tpp->p_next) != *target);
5003 ASSERT(npgs == 0);
5004 page_list_add_pages(*target, 0);
5005 npgs = page_get_pagecnt(szc);
5006 page_create_putback(npgs);
5007 }
5008 return (ret);
5009 }
5010
5011 /*
5012 * it is up to the caller to deal with pcf accounting.
5013 */
5014 void
5015 page_free_replacement_page(page_t *pplist)
5016 {
5017 page_t *pp;
5018
5019 while (pplist != NULL) {
5020 /*
5021 * pp_targ is a linked list.
5022 */
5023 pp = pplist;
5024 if (pp->p_szc == 0) {
5025 page_sub(&pplist, pp);
5026 page_clr_all_props(pp);
5027 PP_SETFREE(pp);
5028 PP_SETAGED(pp);
5029 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
5030 page_unlock(pp);
5031 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]);
5032 } else {
5033 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc);
5034 page_t *tpp;
5035 page_list_break(&pp, &pplist, curnpgs);
5036 tpp = pp;
5037 do {
5038 ASSERT(PAGE_EXCL(tpp));
5039 ASSERT(!hat_page_is_mapped(tpp));
5040 page_clr_all_props(tpp);
5041 PP_SETFREE(tpp);
5042 PP_SETAGED(tpp);
5043 } while ((tpp = tpp->p_next) != pp);
5044 page_list_add_pages(pp, 0);
5045 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]);
5046 }
5047 }
5048 }
5049
5050 /*
5051 * Relocate target to non-relocatable replacement page.
5052 */
5053 int
5054 page_relocate_cage(page_t **target, page_t **replacement)
5055 {
5056 page_t *tpp, *rpp;
5057 spgcnt_t pgcnt, npgs;
5058 int result;
5059
5060 tpp = *target;
5061
5062 ASSERT(PAGE_EXCL(tpp));
5063 ASSERT(tpp->p_szc == 0);
5064
5065 pgcnt = btop(page_get_pagesize(tpp->p_szc));
5066
5067 do {
5068 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC);
5069 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC);
5070 if (rpp == NULL) {
5071 page_create_putback(pgcnt);
5072 kcage_cageout_wakeup();
5073 }
5074 } while (rpp == NULL);
5075
5076 ASSERT(PP_ISNORELOC(rpp));
5077
5078 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL);
5079
5080 if (result == 0) {
5081 *replacement = rpp;
5082 if (pgcnt != npgs)
5083 panic("page_relocate_cage: partial relocation");
5084 }
5085
5086 return (result);
5087 }
5088
5089 /*
5090 * Release the page lock on a page, place on cachelist
5091 * tail if no longer mapped. Caller can let us know if
5092 * the page is known to be clean.
5093 */
5094 int
5095 page_release(page_t *pp, int checkmod)
5096 {
5097 int status;
5098
5099 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) &&
5100 (pp->p_vnode != NULL));
5101
5102 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) &&
5103 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) &&
5104 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 &&
5105 !hat_page_is_mapped(pp)) {
5106
5107 /*
5108 * If page is modified, unlock it
5109 *
5110 * (p_nrm & P_MOD) bit has the latest stuff because:
5111 * (1) We found that this page doesn't have any mappings
5112 * _after_ holding SE_EXCL and
5113 * (2) We didn't drop SE_EXCL lock after the check in (1)
5114 */
5115 if (checkmod && hat_ismod(pp)) {
5116 page_unlock(pp);
5117 status = PGREL_MOD;
5118 } else {
5119 /*LINTED: constant in conditional context*/
5120 VN_DISPOSE(pp, B_FREE, 0, kcred);
5121 status = PGREL_CLEAN;
5122 }
5123 } else {
5124 page_unlock(pp);
5125 status = PGREL_NOTREL;
5126 }
5127 return (status);
5128 }
5129
5130 /*
5131 * Given a constituent page, try to demote the large page on the freelist.
5132 *
5133 * Returns nonzero if the page could be demoted successfully. Returns with
5134 * the constituent page still locked.
5135 */
5136 int
5137 page_try_demote_free_pages(page_t *pp)
5138 {
5139 page_t *rootpp = pp;
5140 pfn_t pfn = page_pptonum(pp);
5141 spgcnt_t npgs;
5142 uint_t szc = pp->p_szc;
5143
5144 ASSERT(PP_ISFREE(pp));
5145 ASSERT(PAGE_EXCL(pp));
5146
5147 /*
5148 * Adjust rootpp and lock it, if `pp' is not the base
5149 * constituent page.
5150 */
5151 npgs = page_get_pagecnt(pp->p_szc);
5152 if (npgs == 1) {
5153 return (0);
5154 }
5155
5156 if (!IS_P2ALIGNED(pfn, npgs)) {
5157 pfn = P2ALIGN(pfn, npgs);
5158 rootpp = page_numtopp_nolock(pfn);
5159 }
5160
5161 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) {
5162 return (0);
5163 }
5164
5165 if (rootpp->p_szc != szc) {
5166 if (pp != rootpp)
5167 page_unlock(rootpp);
5168 return (0);
5169 }
5170
5171 page_demote_free_pages(rootpp);
5172
5173 if (pp != rootpp)
5174 page_unlock(rootpp);
5175
5176 ASSERT(PP_ISFREE(pp));
5177 ASSERT(PAGE_EXCL(pp));
5178 return (1);
5179 }
5180
5181 /*
5182 * Given a constituent page, try to demote the large page.
5183 *
5184 * Returns nonzero if the page could be demoted successfully. Returns with
5185 * the constituent page still locked.
5186 */
5187 int
5188 page_try_demote_pages(page_t *pp)
5189 {
5190 page_t *tpp, *rootpp = pp;
5191 pfn_t pfn = page_pptonum(pp);
5192 spgcnt_t i, npgs;
5193 uint_t szc = pp->p_szc;
5194 vnode_t *vp = pp->p_vnode;
5195
5196 ASSERT(PAGE_EXCL(pp));
5197
5198 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]);
5199
5200 if (pp->p_szc == 0) {
5201 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]);
5202 return (1);
5203 }
5204
5205 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) {
5206 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]);
5207 page_demote_vp_pages(pp);
5208 ASSERT(pp->p_szc == 0);
5209 return (1);
5210 }
5211
5212 /*
5213 * Adjust rootpp if passed in is not the base
5214 * constituent page.
5215 */
5216 npgs = page_get_pagecnt(pp->p_szc);
5217 ASSERT(npgs > 1);
5218 if (!IS_P2ALIGNED(pfn, npgs)) {
5219 pfn = P2ALIGN(pfn, npgs);
5220 rootpp = page_numtopp_nolock(pfn);
5221 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]);
5222 ASSERT(rootpp->p_vnode != NULL);
5223 ASSERT(rootpp->p_szc == szc);
5224 }
5225
5226 /*
5227 * We can't demote kernel pages since we can't hat_unload()
5228 * the mappings.
5229 */
5230 if (VN_ISKAS(rootpp->p_vnode))
5231 return (0);
5232
5233 /*
5234 * Attempt to lock all constituent pages except the page passed
5235 * in since it's already locked.
5236 */
5237 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5238 ASSERT(!PP_ISFREE(tpp));
5239 ASSERT(tpp->p_vnode != NULL);
5240
5241 if (tpp != pp && !page_trylock(tpp, SE_EXCL))
5242 break;
5243 ASSERT(tpp->p_szc == rootpp->p_szc);
5244 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i);
5245 }
5246
5247 /*
5248 * If we failed to lock them all then unlock what we have
5249 * locked so far and bail.
5250 */
5251 if (i < npgs) {
5252 tpp = rootpp;
5253 while (i-- > 0) {
5254 if (tpp != pp)
5255 page_unlock(tpp);
5256 tpp++;
5257 }
5258 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]);
5259 return (0);
5260 }
5261
5262 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5263 ASSERT(PAGE_EXCL(tpp));
5264 ASSERT(tpp->p_slckcnt == 0);
5265 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
5266 tpp->p_szc = 0;
5267 }
5268
5269 /*
5270 * Unlock all pages except the page passed in.
5271 */
5272 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5273 ASSERT(!hat_page_is_mapped(tpp));
5274 if (tpp != pp)
5275 page_unlock(tpp);
5276 }
5277
5278 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]);
5279 return (1);
5280 }
5281
5282 /*
5283 * Called by page_free() and page_destroy() to demote the page size code
5284 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
5285 * p_szc on free list, neither can we just clear p_szc of a single page_t
5286 * within a large page since it will break other code that relies on p_szc
5287 * being the same for all page_t's of a large page). Anonymous pages should
5288 * never end up here because anon_map_getpages() cannot deal with p_szc
5289 * changes after a single constituent page is locked. While anonymous or
5290 * kernel large pages are demoted or freed the entire large page at a time
5291 * with all constituent pages locked EXCL for the file system pages we
5292 * have to be able to demote a large page (i.e. decrease all constituent pages
5293 * p_szc) with only just an EXCL lock on one of constituent pages. The reason
5294 * we can easily deal with anonymous page demotion the entire large page at a
5295 * time is that those operation originate at address space level and concern
5296 * the entire large page region with actual demotion only done when pages are
5297 * not shared with any other processes (therefore we can always get EXCL lock
5298 * on all anonymous constituent pages after clearing segment page
5299 * cache). However file system pages can be truncated or invalidated at a
5300 * PAGESIZE level from the file system side and end up in page_free() or
5301 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed
5302 * and therefore pageout should be able to demote a large page by EXCL locking
5303 * any constituent page that is not under SOFTLOCK). In those cases we cannot
5304 * rely on being able to lock EXCL all constituent pages.
5305 *
5306 * To prevent szc changes on file system pages one has to lock all constituent
5307 * pages at least SHARED (or call page_szc_lock()). The only subsystem that
5308 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to
5309 * prevent szc changes is hat layer that uses its own page level mlist
5310 * locks. hat assumes that szc doesn't change after mlist lock for a page is
5311 * taken. Therefore we need to change szc under hat level locks if we only
5312 * have an EXCL lock on a single constituent page and hat still references any
5313 * of constituent pages. (Note we can't "ignore" hat layer by simply
5314 * hat_pageunload() all constituent pages without having EXCL locks on all of
5315 * constituent pages). We use hat_page_demote() call to safely demote szc of
5316 * all constituent pages under hat locks when we only have an EXCL lock on one
5317 * of constituent pages.
5318 *
5319 * This routine calls page_szc_lock() before calling hat_page_demote() to
5320 * allow segvn in one special case not to lock all constituent pages SHARED
5321 * before calling hat_memload_array() that relies on p_szc not changing even
5322 * before hat level mlist lock is taken. In that case segvn uses
5323 * page_szc_lock() to prevent hat_page_demote() changing p_szc values.
5324 *
5325 * Anonymous or kernel page demotion still has to lock all pages exclusively
5326 * and do hat_pageunload() on all constituent pages before demoting the page
5327 * therefore there's no need for anonymous or kernel page demotion to use
5328 * hat_page_demote() mechanism.
5329 *
5330 * hat_page_demote() removes all large mappings that map pp and then decreases
5331 * p_szc starting from the last constituent page of the large page. By working
5332 * from the tail of a large page in pfn decreasing order allows one looking at
5333 * the root page to know that hat_page_demote() is done for root's szc area.
5334 * e.g. if a root page has szc 1 one knows it only has to lock all constituent
5335 * pages within szc 1 area to prevent szc changes because hat_page_demote()
5336 * that started on this page when it had szc > 1 is done for this szc 1 area.
5337 *
5338 * We are guaranteed that all constituent pages of pp's large page belong to
5339 * the same vnode with the consecutive offsets increasing in the direction of
5340 * the pfn i.e. the identity of constituent pages can't change until their
5341 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
5342 * large mappings to pp even though we don't lock any constituent page except
5343 * pp (i.e. we won't unload e.g. kernel locked page).
5344 */
5345 static void
5346 page_demote_vp_pages(page_t *pp)
5347 {
5348 kmutex_t *mtx;
5349
5350 ASSERT(PAGE_EXCL(pp));
5351 ASSERT(!PP_ISFREE(pp));
5352 ASSERT(pp->p_vnode != NULL);
5353 ASSERT(!IS_SWAPFSVP(pp->p_vnode));
5354 ASSERT(!PP_ISKAS(pp));
5355
5356 VM_STAT_ADD(pagecnt.pc_demote_pages[0]);
5357
5358 mtx = page_szc_lock(pp);
5359 if (mtx != NULL) {
5360 hat_page_demote(pp);
5361 mutex_exit(mtx);
5362 }
5363 ASSERT(pp->p_szc == 0);
5364 }
5365
5366 /*
5367 * Mark any existing pages for migration in the given range
5368 */
5369 void
5370 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len,
5371 struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
5372 u_offset_t vnoff, int rflag)
5373 {
5374 struct anon *ap;
5375 vnode_t *curvp;
5376 lgrp_t *from;
5377 pgcnt_t nlocked;
5378 u_offset_t off;
5379 pfn_t pfn;
5380 size_t pgsz;
5381 size_t segpgsz;
5382 pgcnt_t pages;
5383 uint_t pszc;
5384 page_t *pp0, *pp;
5385 caddr_t va;
5386 ulong_t an_idx;
5387 anon_sync_obj_t cookie;
5388
5389 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
5390
5391 /*
5392 * Don't do anything if don't need to do lgroup optimizations
5393 * on this system
5394 */
5395 if (!lgrp_optimizations())
5396 return;
5397
5398 /*
5399 * Align address and length to (potentially large) page boundary
5400 */
5401 segpgsz = page_get_pagesize(seg->s_szc);
5402 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz);
5403 if (rflag)
5404 len = P2ROUNDUP(len, segpgsz);
5405
5406 /*
5407 * Do one (large) page at a time
5408 */
5409 va = addr;
5410 while (va < addr + len) {
5411 /*
5412 * Lookup (root) page for vnode and offset corresponding to
5413 * this virtual address
5414 * Try anonmap first since there may be copy-on-write
5415 * pages, but initialize vnode pointer and offset using
5416 * vnode arguments just in case there isn't an amp.
5417 */
5418 curvp = vp;
5419 off = vnoff + va - seg->s_base;
5420 if (amp) {
5421 ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
5422 an_idx = anon_index + seg_page(seg, va);
5423 anon_array_enter(amp, an_idx, &cookie);
5424 ap = anon_get_ptr(amp->ahp, an_idx);
5425 if (ap)
5426 swap_xlate(ap, &curvp, &off);
5427 anon_array_exit(&cookie);
5428 ANON_LOCK_EXIT(&->a_rwlock);
5429 }
5430
5431 pp = NULL;
5432 if (curvp)
5433 pp = page_lookup(curvp, off, SE_SHARED);
5434
5435 /*
5436 * If there isn't a page at this virtual address,
5437 * skip to next page
5438 */
5439 if (pp == NULL) {
5440 va += PAGESIZE;
5441 continue;
5442 }
5443
5444 /*
5445 * Figure out which lgroup this page is in for kstats
5446 */
5447 pfn = page_pptonum(pp);
5448 from = lgrp_pfn_to_lgrp(pfn);
5449
5450 /*
5451 * Get page size, and round up and skip to next page boundary
5452 * if unaligned address
5453 */
5454 pszc = pp->p_szc;
5455 pgsz = page_get_pagesize(pszc);
5456 pages = btop(pgsz);
5457 if (!IS_P2ALIGNED(va, pgsz) ||
5458 !IS_P2ALIGNED(pfn, pages) ||
5459 pgsz > segpgsz) {
5460 pgsz = MIN(pgsz, segpgsz);
5461 page_unlock(pp);
5462 pages = btop(P2END((uintptr_t)va, pgsz) -
5463 (uintptr_t)va);
5464 va = (caddr_t)P2END((uintptr_t)va, pgsz);
5465 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, pages);
5466 continue;
5467 }
5468
5469 /*
5470 * Upgrade to exclusive lock on page
5471 */
5472 if (!page_tryupgrade(pp)) {
5473 page_unlock(pp);
5474 va += pgsz;
5475 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5476 btop(pgsz));
5477 continue;
5478 }
5479
5480 pp0 = pp++;
5481 nlocked = 1;
5482
5483 /*
5484 * Lock constituent pages if this is large page
5485 */
5486 if (pages > 1) {
5487 /*
5488 * Lock all constituents except root page, since it
5489 * should be locked already.
5490 */
5491 for (; nlocked < pages; nlocked++) {
5492 if (!page_trylock(pp, SE_EXCL)) {
5493 break;
5494 }
5495 if (PP_ISFREE(pp) ||
5496 pp->p_szc != pszc) {
5497 /*
5498 * hat_page_demote() raced in with us.
5499 */
5500 ASSERT(!IS_SWAPFSVP(curvp));
5501 page_unlock(pp);
5502 break;
5503 }
5504 pp++;
5505 }
5506 }
5507
5508 /*
5509 * If all constituent pages couldn't be locked,
5510 * unlock pages locked so far and skip to next page.
5511 */
5512 if (nlocked < pages) {
5513 while (pp0 < pp) {
5514 page_unlock(pp0++);
5515 }
5516 va += pgsz;
5517 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5518 btop(pgsz));
5519 continue;
5520 }
5521
5522 /*
5523 * hat_page_demote() can no longer happen
5524 * since last cons page had the right p_szc after
5525 * all cons pages were locked. all cons pages
5526 * should now have the same p_szc.
5527 */
5528
5529 /*
5530 * All constituent pages locked successfully, so mark
5531 * large page for migration and unload the mappings of
5532 * constituent pages, so a fault will occur on any part of the
5533 * large page
5534 */
5535 PP_SETMIGRATE(pp0);
5536 while (pp0 < pp) {
5537 (void) hat_pageunload(pp0, HAT_FORCE_PGUNLOAD);
5538 ASSERT(hat_page_getshare(pp0) == 0);
5539 page_unlock(pp0++);
5540 }
5541 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked);
5542
5543 va += pgsz;
5544 }
5545 }
5546
5547 /*
5548 * Migrate any pages that have been marked for migration in the given range
5549 */
5550 void
5551 page_migrate(
5552 struct seg *seg,
5553 caddr_t addr,
5554 page_t **ppa,
5555 pgcnt_t npages)
5556 {
5557 lgrp_t *from;
5558 lgrp_t *to;
5559 page_t *newpp;
5560 page_t *pp;
5561 pfn_t pfn;
5562 size_t pgsz;
5563 spgcnt_t page_cnt;
5564 spgcnt_t i;
5565 uint_t pszc;
5566
5567 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
5568
5569 while (npages > 0) {
5570 pp = *ppa;
5571 pszc = pp->p_szc;
5572 pgsz = page_get_pagesize(pszc);
5573 page_cnt = btop(pgsz);
5574
5575 /*
5576 * Check to see whether this page is marked for migration
5577 *
5578 * Assume that root page of large page is marked for
5579 * migration and none of the other constituent pages
5580 * are marked. This really simplifies clearing the
5581 * migrate bit by not having to clear it from each
5582 * constituent page.
5583 *
5584 * note we don't want to relocate an entire large page if
5585 * someone is only using one subpage.
5586 */
5587 if (npages < page_cnt)
5588 break;
5589
5590 /*
5591 * Is it marked for migration?
5592 */
5593 if (!PP_ISMIGRATE(pp))
5594 goto next;
5595
5596 /*
5597 * Determine lgroups that page is being migrated between
5598 */
5599 pfn = page_pptonum(pp);
5600 if (!IS_P2ALIGNED(pfn, page_cnt)) {
5601 break;
5602 }
5603 from = lgrp_pfn_to_lgrp(pfn);
5604 to = lgrp_mem_choose(seg, addr, pgsz);
5605
5606 /*
5607 * Need to get exclusive lock's to migrate
5608 */
5609 for (i = 0; i < page_cnt; i++) {
5610 ASSERT(PAGE_LOCKED(ppa[i]));
5611 if (page_pptonum(ppa[i]) != pfn + i ||
5612 ppa[i]->p_szc != pszc) {
5613 break;
5614 }
5615 if (!page_tryupgrade(ppa[i])) {
5616 lgrp_stat_add(from->lgrp_id,
5617 LGRP_PM_FAIL_LOCK_PGS,
5618 page_cnt);
5619 break;
5620 }
5621
5622 /*
5623 * Check to see whether we are trying to migrate
5624 * page to lgroup where it is allocated already.
5625 * If so, clear the migrate bit and skip to next
5626 * page.
5627 */
5628 if (i == 0 && to == from) {
5629 PP_CLRMIGRATE(ppa[0]);
5630 page_downgrade(ppa[0]);
5631 goto next;
5632 }
5633 }
5634
5635 /*
5636 * If all constituent pages couldn't be locked,
5637 * unlock pages locked so far and skip to next page.
5638 */
5639 if (i != page_cnt) {
5640 while (--i != -1) {
5641 page_downgrade(ppa[i]);
5642 }
5643 goto next;
5644 }
5645
5646 (void) page_create_wait(page_cnt, PG_WAIT);
5647 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC);
5648 if (newpp == NULL) {
5649 page_create_putback(page_cnt);
5650 for (i = 0; i < page_cnt; i++) {
5651 page_downgrade(ppa[i]);
5652 }
5653 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS,
5654 page_cnt);
5655 goto next;
5656 }
5657 ASSERT(newpp->p_szc == pszc);
5658 /*
5659 * Clear migrate bit and relocate page
5660 */
5661 PP_CLRMIGRATE(pp);
5662 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) {
5663 panic("page_migrate: page_relocate failed");
5664 }
5665 ASSERT(page_cnt * PAGESIZE == pgsz);
5666
5667 /*
5668 * Keep stats for number of pages migrated from and to
5669 * each lgroup
5670 */
5671 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt);
5672 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt);
5673 /*
5674 * update the page_t array we were passed in and
5675 * unlink constituent pages of a large page.
5676 */
5677 for (i = 0; i < page_cnt; ++i, ++pp) {
5678 ASSERT(PAGE_EXCL(newpp));
5679 ASSERT(newpp->p_szc == pszc);
5680 ppa[i] = newpp;
5681 pp = newpp;
5682 page_sub(&newpp, pp);
5683 page_downgrade(pp);
5684 }
5685 ASSERT(newpp == NULL);
5686 next:
5687 addr += pgsz;
5688 ppa += page_cnt;
5689 npages -= page_cnt;
5690 }
5691 }
5692
5693 uint_t page_reclaim_maxcnt = 60; /* max total iterations */
5694 uint_t page_reclaim_nofree_maxcnt = 3; /* max iterations without progress */
5695 /*
5696 * Reclaim/reserve availrmem for npages.
5697 * If there is not enough memory start reaping seg, kmem caches.
5698 * Start pageout scanner (via page_needfree()).
5699 * Exit after ~ MAX_CNT s regardless of how much memory has been released.
5700 * Note: There is no guarantee that any availrmem will be freed as
5701 * this memory typically is locked (kernel heap) or reserved for swap.
5702 * Also due to memory fragmentation kmem allocator may not be able
5703 * to free any memory (single user allocated buffer will prevent
5704 * freeing slab or a page).
5705 */
5706 int
5707 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
5708 {
5709 int i = 0;
5710 int i_nofree = 0;
5711 int ret = 0;
5712 pgcnt_t deficit;
5713 pgcnt_t old_availrmem = 0;
5714
5715 mutex_enter(&freemem_lock);
5716 while (availrmem < tune.t_minarmem + npages + epages &&
5717 i++ < page_reclaim_maxcnt) {
5718 /* ensure we made some progress in the last few iterations */
5719 if (old_availrmem < availrmem) {
5720 old_availrmem = availrmem;
5721 i_nofree = 0;
5722 } else if (i_nofree++ >= page_reclaim_nofree_maxcnt) {
5723 break;
5724 }
5725
5726 deficit = tune.t_minarmem + npages + epages - availrmem;
5727 mutex_exit(&freemem_lock);
5728 page_needfree(deficit);
5729 kmem_reap();
5730 delay(hz);
5731 page_needfree(-(spgcnt_t)deficit);
5732 mutex_enter(&freemem_lock);
5733 }
5734
5735 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) {
5736 availrmem -= npages;
5737 ret = 1;
5738 }
5739
5740 mutex_exit(&freemem_lock);
5741
5742 return (ret);
5743 }
5744
5745 /*
5746 * Search the memory segments to locate the desired page. Within a
5747 * segment, pages increase linearly with one page structure per
5748 * physical page frame (size PAGESIZE). The search begins
5749 * with the segment that was accessed last, to take advantage of locality.
5750 * If the hint misses, we start from the beginning of the sorted memseg list
5751 */
5752
5753
5754 /*
5755 * Some data structures for pfn to pp lookup.
5756 */
5757 ulong_t mhash_per_slot;
5758 struct memseg *memseg_hash[N_MEM_SLOTS];
5759
5760 page_t *
5761 page_numtopp_nolock(pfn_t pfnum)
5762 {
5763 struct memseg *seg;
5764 page_t *pp;
5765 vm_cpu_data_t *vc;
5766
5767 /*
5768 * We need to disable kernel preemption while referencing the
5769 * cpu_vm_data field in order to prevent us from being switched to
5770 * another cpu and trying to reference it after it has been freed.
5771 * This will keep us on cpu and prevent it from being removed while
5772 * we are still on it.
5773 *
5774 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5775 * which is being resued by DR who will flush those references
5776 * before modifying the reused memseg. See memseg_cpu_vm_flush().
5777 */
5778 kpreempt_disable();
5779 vc = CPU->cpu_vm_data;
5780 ASSERT(vc != NULL);
5781
5782 MEMSEG_STAT_INCR(nsearch);
5783
5784 /* Try last winner first */
5785 if (((seg = vc->vc_pnum_memseg) != NULL) &&
5786 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5787 MEMSEG_STAT_INCR(nlastwon);
5788 pp = seg->pages + (pfnum - seg->pages_base);
5789 if (pp->p_pagenum == pfnum) {
5790 kpreempt_enable();
5791 return ((page_t *)pp);
5792 }
5793 }
5794
5795 /* Else Try hash */
5796 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5797 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5798 MEMSEG_STAT_INCR(nhashwon);
5799 vc->vc_pnum_memseg = seg;
5800 pp = seg->pages + (pfnum - seg->pages_base);
5801 if (pp->p_pagenum == pfnum) {
5802 kpreempt_enable();
5803 return ((page_t *)pp);
5804 }
5805 }
5806
5807 /* Else Brute force */
5808 for (seg = memsegs; seg != NULL; seg = seg->next) {
5809 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5810 vc->vc_pnum_memseg = seg;
5811 pp = seg->pages + (pfnum - seg->pages_base);
5812 if (pp->p_pagenum == pfnum) {
5813 kpreempt_enable();
5814 return ((page_t *)pp);
5815 }
5816 }
5817 }
5818 vc->vc_pnum_memseg = NULL;
5819 kpreempt_enable();
5820 MEMSEG_STAT_INCR(nnotfound);
5821 return ((page_t *)NULL);
5822
5823 }
5824
5825 struct memseg *
5826 page_numtomemseg_nolock(pfn_t pfnum)
5827 {
5828 struct memseg *seg;
5829 page_t *pp;
5830
5831 /*
5832 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5833 * which is being resued by DR who will flush those references
5834 * before modifying the reused memseg. See memseg_cpu_vm_flush().
5835 */
5836 kpreempt_disable();
5837 /* Try hash */
5838 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5839 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5840 pp = seg->pages + (pfnum - seg->pages_base);
5841 if (pp->p_pagenum == pfnum) {
5842 kpreempt_enable();
5843 return (seg);
5844 }
5845 }
5846
5847 /* Else Brute force */
5848 for (seg = memsegs; seg != NULL; seg = seg->next) {
5849 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5850 pp = seg->pages + (pfnum - seg->pages_base);
5851 if (pp->p_pagenum == pfnum) {
5852 kpreempt_enable();
5853 return (seg);
5854 }
5855 }
5856 }
5857 kpreempt_enable();
5858 return ((struct memseg *)NULL);
5859 }
5860
5861 /*
5862 * Given a page and a count return the page struct that is
5863 * n structs away from the current one in the global page
5864 * list.
5865 *
5866 * This function wraps to the first page upon
5867 * reaching the end of the memseg list.
5868 */
5869 page_t *
5870 page_nextn(page_t *pp, ulong_t n)
5871 {
5872 struct memseg *seg;
5873 page_t *ppn;
5874 vm_cpu_data_t *vc;
5875
5876 /*
5877 * We need to disable kernel preemption while referencing the
5878 * cpu_vm_data field in order to prevent us from being switched to
5879 * another cpu and trying to reference it after it has been freed.
5880 * This will keep us on cpu and prevent it from being removed while
5881 * we are still on it.
5882 *
5883 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5884 * which is being resued by DR who will flush those references
5885 * before modifying the reused memseg. See memseg_cpu_vm_flush().
5886 */
5887 kpreempt_disable();
5888 vc = (vm_cpu_data_t *)CPU->cpu_vm_data;
5889
5890 ASSERT(vc != NULL);
5891
5892 if (((seg = vc->vc_pnext_memseg) == NULL) ||
5893 (seg->pages_base == seg->pages_end) ||
5894 !(pp >= seg->pages && pp < seg->epages)) {
5895
5896 for (seg = memsegs; seg; seg = seg->next) {
5897 if (pp >= seg->pages && pp < seg->epages)
5898 break;
5899 }
5900
5901 if (seg == NULL) {
5902 /* Memory delete got in, return something valid. */
5903 /* TODO: fix me. */
5904 seg = memsegs;
5905 pp = seg->pages;
5906 }
5907 }
5908
5909 /* check for wraparound - possible if n is large */
5910 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) {
5911 n -= seg->epages - pp;
5912 seg = seg->next;
5913 if (seg == NULL)
5914 seg = memsegs;
5915 pp = seg->pages;
5916 }
5917 vc->vc_pnext_memseg = seg;
5918 kpreempt_enable();
5919 return (ppn);
5920 }
5921
5922 /*
5923 * Initialize for a loop using page_next_scan_large().
5924 */
5925 page_t *
5926 page_next_scan_init(void **cookie)
5927 {
5928 ASSERT(cookie != NULL);
5929 *cookie = (void *)memsegs;
5930 return ((page_t *)memsegs->pages);
5931 }
5932
5933 /*
5934 * Return the next page in a scan of page_t's, assuming we want
5935 * to skip over sub-pages within larger page sizes.
5936 *
5937 * The cookie is used to keep track of the current memseg.
5938 */
5939 page_t *
5940 page_next_scan_large(
5941 page_t *pp,
5942 ulong_t *n,
5943 void **cookie)
5944 {
5945 struct memseg *seg = (struct memseg *)*cookie;
5946 page_t *new_pp;
5947 ulong_t cnt;
5948 pfn_t pfn;
5949
5950
5951 /*
5952 * get the count of page_t's to skip based on the page size
5953 */
5954 ASSERT(pp != NULL);
5955 if (pp->p_szc == 0) {
5956 cnt = 1;
5957 } else {
5958 pfn = page_pptonum(pp);
5959 cnt = page_get_pagecnt(pp->p_szc);
5960 cnt -= pfn & (cnt - 1);
5961 }
5962 *n += cnt;
5963 new_pp = pp + cnt;
5964
5965 /*
5966 * Catch if we went past the end of the current memory segment. If so,
5967 * just move to the next segment with pages.
5968 */
5969 if (new_pp >= seg->epages || seg->pages_base == seg->pages_end) {
5970 do {
5971 seg = seg->next;
5972 if (seg == NULL)
5973 seg = memsegs;
5974 } while (seg->pages_base == seg->pages_end);
5975 new_pp = seg->pages;
5976 *cookie = (void *)seg;
5977 }
5978
5979 return (new_pp);
5980 }
5981
5982
5983 /*
5984 * Returns next page in list. Note: this function wraps
5985 * to the first page in the list upon reaching the end
5986 * of the list. Callers should be aware of this fact.
5987 */
5988
5989 /* We should change this be a #define */
5990
5991 page_t *
5992 page_next(page_t *pp)
5993 {
5994 return (page_nextn(pp, 1));
5995 }
5996
5997 page_t *
5998 page_first()
5999 {
6000 return ((page_t *)memsegs->pages);
6001 }
6002
6003
6004 /*
6005 * This routine is called at boot with the initial memory configuration
6006 * and when memory is added or removed.
6007 */
6008 void
6009 build_pfn_hash()
6010 {
6011 pfn_t cur;
6012 pgcnt_t index;
6013 struct memseg *pseg;
6014 int i;
6015
6016 /*
6017 * Clear memseg_hash array.
6018 * Since memory add/delete is designed to operate concurrently
6019 * with normal operation, the hash rebuild must be able to run
6020 * concurrently with page_numtopp_nolock(). To support this
6021 * functionality, assignments to memseg_hash array members must
6022 * be done atomically.
6023 *
6024 * NOTE: bzero() does not currently guarantee this for kernel
6025 * threads, and cannot be used here.
6026 */
6027 for (i = 0; i < N_MEM_SLOTS; i++)
6028 memseg_hash[i] = NULL;
6029
6030 hat_kpm_mseghash_clear(N_MEM_SLOTS);
6031
6032 /*
6033 * Physmax is the last valid pfn.
6034 */
6035 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT;
6036 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) {
6037 index = MEMSEG_PFN_HASH(pseg->pages_base);
6038 cur = pseg->pages_base;
6039 do {
6040 if (index >= N_MEM_SLOTS)
6041 index = MEMSEG_PFN_HASH(cur);
6042
6043 if (memseg_hash[index] == NULL ||
6044 memseg_hash[index]->pages_base > pseg->pages_base) {
6045 memseg_hash[index] = pseg;
6046 hat_kpm_mseghash_update(index, pseg);
6047 }
6048 cur += mhash_per_slot;
6049 index++;
6050 } while (cur < pseg->pages_end);
6051 }
6052 }
6053
6054 /*
6055 * Return the pagenum for the pp
6056 */
6057 pfn_t
6058 page_pptonum(page_t *pp)
6059 {
6060 return (pp->p_pagenum);
6061 }
6062
6063 /*
6064 * interface to the referenced and modified etc bits
6065 * in the PSM part of the page struct
6066 * when no locking is desired.
6067 */
6068 void
6069 page_set_props(page_t *pp, uint_t flags)
6070 {
6071 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0);
6072 pp->p_nrm |= (uchar_t)flags;
6073 }
6074
6075 void
6076 page_clr_all_props(page_t *pp)
6077 {
6078 pp->p_nrm = 0;
6079 }
6080
6081 /*
6082 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
6083 */
6084 int
6085 page_clear_lck_cow(page_t *pp, int adjust)
6086 {
6087 int f_amount;
6088
6089 ASSERT(PAGE_EXCL(pp));
6090
6091 /*
6092 * The page_struct_lock need not be acquired here since
6093 * we require the caller hold the page exclusively locked.
6094 */
6095 f_amount = 0;
6096 if (pp->p_lckcnt) {
6097 f_amount = 1;
6098 pp->p_lckcnt = 0;
6099 }
6100 if (pp->p_cowcnt) {
6101 f_amount += pp->p_cowcnt;
6102 pp->p_cowcnt = 0;
6103 }
6104
6105 if (adjust && f_amount) {
6106 mutex_enter(&freemem_lock);
6107 availrmem += f_amount;
6108 mutex_exit(&freemem_lock);
6109 }
6110
6111 return (f_amount);
6112 }
6113
6114 /*
6115 * The following functions is called from free_vp_pages()
6116 * for an inexact estimate of a newly free'd page...
6117 */
6118 ulong_t
6119 page_share_cnt(page_t *pp)
6120 {
6121 return (hat_page_getshare(pp));
6122 }
6123
6124 int
6125 page_isshared(page_t *pp)
6126 {
6127 return (hat_page_checkshare(pp, 1));
6128 }
6129
6130 int
6131 page_isfree(page_t *pp)
6132 {
6133 return (PP_ISFREE(pp));
6134 }
6135
6136 int
6137 page_isref(page_t *pp)
6138 {
6139 return (hat_page_getattr(pp, P_REF));
6140 }
6141
6142 int
6143 page_ismod(page_t *pp)
6144 {
6145 return (hat_page_getattr(pp, P_MOD));
6146 }
6147
6148 /*
6149 * The following code all currently relates to the page capture logic:
6150 *
6151 * This logic is used for cases where there is a desire to claim a certain
6152 * physical page in the system for the caller. As it may not be possible
6153 * to capture the page immediately, the p_toxic bits are used in the page
6154 * structure to indicate that someone wants to capture this page. When the
6155 * page gets unlocked, the toxic flag will be noted and an attempt to capture
6156 * the page will be made. If it is successful, the original callers callback
6157 * will be called with the page to do with it what they please.
6158 *
6159 * There is also an async thread which wakes up to attempt to capture
6160 * pages occasionally which have the capture bit set. All of the pages which
6161 * need to be captured asynchronously have been inserted into the
6162 * page_capture_hash and thus this thread walks that hash list. Items in the
6163 * hash have an expiration time so this thread handles that as well by removing
6164 * the item from the hash if it has expired.
6165 *
6166 * Some important things to note are:
6167 * - if the PR_CAPTURE bit is set on a page, then the page is in the
6168 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed
6169 * to set and clear this bit, and while the lock is held is the only time
6170 * you can add or remove an entry from the hash.
6171 * - the PR_CAPTURE bit can only be set and cleared while holding the
6172 * page_capture_hash_head.pchh_mutex
6173 * - the t_flag field of the thread struct is used with the T_CAPTURING
6174 * flag to prevent recursion while dealing with large pages.
6175 * - pages which need to be retired never expire on the page_capture_hash.
6176 */
6177
6178 static void page_capture_thread(void);
6179 static kthread_t *pc_thread_id;
6180 kcondvar_t pc_cv;
6181 static kmutex_t pc_thread_mutex;
6182 static clock_t pc_thread_shortwait;
6183 static clock_t pc_thread_longwait;
6184 static int pc_thread_retry;
6185
6186 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS];
6187
6188 /* Note that this is a circular linked list */
6189 typedef struct page_capture_hash_bucket {
6190 page_t *pp;
6191 uchar_t szc;
6192 uchar_t pri;
6193 uint_t flags;
6194 clock_t expires; /* lbolt at which this request expires. */
6195 void *datap; /* Cached data passed in for callback */
6196 struct page_capture_hash_bucket *next;
6197 struct page_capture_hash_bucket *prev;
6198 } page_capture_hash_bucket_t;
6199
6200 #define PC_PRI_HI 0 /* capture now */
6201 #define PC_PRI_LO 1 /* capture later */
6202 #define PC_NUM_PRI 2
6203
6204 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI)
6205
6206
6207 /*
6208 * Each hash bucket will have it's own mutex and two lists which are:
6209 * active (0): represents requests which have not been processed by
6210 * the page_capture async thread yet.
6211 * walked (1): represents requests which have been processed by the
6212 * page_capture async thread within it's given walk of this bucket.
6213 *
6214 * These are all needed so that we can synchronize all async page_capture
6215 * events. When the async thread moves to a new bucket, it will append the
6216 * walked list to the active list and walk each item one at a time, moving it
6217 * from the active list to the walked list. Thus if there is an async request
6218 * outstanding for a given page, it will always be in one of the two lists.
6219 * New requests will always be added to the active list.
6220 * If we were not able to capture a page before the request expired, we'd free
6221 * up the request structure which would indicate to page_capture that there is
6222 * no longer a need for the given page, and clear the PR_CAPTURE flag if
6223 * possible.
6224 */
6225 typedef struct page_capture_hash_head {
6226 kmutex_t pchh_mutex;
6227 uint_t num_pages[PC_NUM_PRI];
6228 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */
6229 } page_capture_hash_head_t;
6230
6231 #ifdef DEBUG
6232 #define NUM_PAGE_CAPTURE_BUCKETS 4
6233 #else
6234 #define NUM_PAGE_CAPTURE_BUCKETS 64
6235 #endif
6236
6237 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS];
6238
6239 /* for now use a very simple hash based upon the size of a page struct */
6240 #define PAGE_CAPTURE_HASH(pp) \
6241 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
6242
6243 extern pgcnt_t swapfs_minfree;
6244
6245 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
6246
6247 /*
6248 * a callback function is required for page capture requests.
6249 */
6250 void
6251 page_capture_register_callback(uint_t index, clock_t duration,
6252 int (*cb_func)(page_t *, void *, uint_t))
6253 {
6254 ASSERT(pc_cb[index].cb_active == 0);
6255 ASSERT(cb_func != NULL);
6256 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6257 pc_cb[index].duration = duration;
6258 pc_cb[index].cb_func = cb_func;
6259 pc_cb[index].cb_active = 1;
6260 rw_exit(&pc_cb[index].cb_rwlock);
6261 }
6262
6263 void
6264 page_capture_unregister_callback(uint_t index)
6265 {
6266 int i, j;
6267 struct page_capture_hash_bucket *bp1;
6268 struct page_capture_hash_bucket *bp2;
6269 struct page_capture_hash_bucket *head = NULL;
6270 uint_t flags = (1 << index);
6271
6272 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6273 ASSERT(pc_cb[index].cb_active == 1);
6274 pc_cb[index].duration = 0; /* Paranoia */
6275 pc_cb[index].cb_func = NULL; /* Paranoia */
6276 pc_cb[index].cb_active = 0;
6277 rw_exit(&pc_cb[index].cb_rwlock);
6278
6279 /*
6280 * Just move all the entries to a private list which we can walk
6281 * through without the need to hold any locks.
6282 * No more requests can get added to the hash lists for this consumer
6283 * as the cb_active field for the callback has been cleared.
6284 */
6285 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
6286 mutex_enter(&page_capture_hash[i].pchh_mutex);
6287 for (j = 0; j < 2; j++) {
6288 bp1 = page_capture_hash[i].lists[j].next;
6289 /* walk through all but first (sentinel) element */
6290 while (bp1 != &page_capture_hash[i].lists[j]) {
6291 bp2 = bp1;
6292 if (bp2->flags & flags) {
6293 bp1 = bp2->next;
6294 bp1->prev = bp2->prev;
6295 bp2->prev->next = bp1;
6296 bp2->next = head;
6297 head = bp2;
6298 /*
6299 * Clear the PR_CAPTURE bit as we
6300 * hold appropriate locks here.
6301 */
6302 page_clrtoxic(head->pp, PR_CAPTURE);
6303 page_capture_hash[i].
6304 num_pages[bp2->pri]--;
6305 continue;
6306 }
6307 bp1 = bp1->next;
6308 }
6309 }
6310 mutex_exit(&page_capture_hash[i].pchh_mutex);
6311 }
6312
6313 while (head != NULL) {
6314 bp1 = head;
6315 head = head->next;
6316 kmem_free(bp1, sizeof (*bp1));
6317 }
6318 }
6319
6320
6321 /*
6322 * Find pp in the active list and move it to the walked list if it
6323 * exists.
6324 * Note that most often pp should be at the front of the active list
6325 * as it is currently used and thus there is no other sort of optimization
6326 * being done here as this is a linked list data structure.
6327 * Returns 1 on successful move or 0 if page could not be found.
6328 */
6329 static int
6330 page_capture_move_to_walked(page_t *pp)
6331 {
6332 page_capture_hash_bucket_t *bp;
6333 int index;
6334
6335 index = PAGE_CAPTURE_HASH(pp);
6336
6337 mutex_enter(&page_capture_hash[index].pchh_mutex);
6338 bp = page_capture_hash[index].lists[0].next;
6339 while (bp != &page_capture_hash[index].lists[0]) {
6340 if (bp->pp == pp) {
6341 /* Remove from old list */
6342 bp->next->prev = bp->prev;
6343 bp->prev->next = bp->next;
6344
6345 /* Add to new list */
6346 bp->next = page_capture_hash[index].lists[1].next;
6347 bp->prev = &page_capture_hash[index].lists[1];
6348 page_capture_hash[index].lists[1].next = bp;
6349 bp->next->prev = bp;
6350
6351 /*
6352 * There is a small probability of page on a free
6353 * list being retired while being allocated
6354 * and before P_RAF is set on it. The page may
6355 * end up marked as high priority request instead
6356 * of low priority request.
6357 * If P_RAF page is not marked as low priority request
6358 * change it to low priority request.
6359 */
6360 page_capture_hash[index].num_pages[bp->pri]--;
6361 bp->pri = PAGE_CAPTURE_PRIO(pp);
6362 page_capture_hash[index].num_pages[bp->pri]++;
6363 mutex_exit(&page_capture_hash[index].pchh_mutex);
6364 return (1);
6365 }
6366 bp = bp->next;
6367 }
6368 mutex_exit(&page_capture_hash[index].pchh_mutex);
6369 return (0);
6370 }
6371
6372 /*
6373 * Add a new entry to the page capture hash. The only case where a new
6374 * entry is not added is when the page capture consumer is no longer registered.
6375 * In this case, we'll silently not add the page to the hash. We know that
6376 * page retire will always be registered for the case where we are currently
6377 * unretiring a page and thus there are no conflicts.
6378 */
6379 static void
6380 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap)
6381 {
6382 page_capture_hash_bucket_t *bp1;
6383 page_capture_hash_bucket_t *bp2;
6384 int index;
6385 int cb_index;
6386 int i;
6387 uchar_t pri;
6388 #ifdef DEBUG
6389 page_capture_hash_bucket_t *tp1;
6390 int l;
6391 #endif
6392
6393 ASSERT(!(flags & CAPTURE_ASYNC));
6394
6395 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP);
6396
6397 bp1->pp = pp;
6398 bp1->szc = szc;
6399 bp1->flags = flags;
6400 bp1->datap = datap;
6401
6402 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6403 if ((flags >> cb_index) & 1) {
6404 break;
6405 }
6406 }
6407
6408 ASSERT(cb_index != PC_NUM_CALLBACKS);
6409
6410 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6411 if (pc_cb[cb_index].cb_active) {
6412 if (pc_cb[cb_index].duration == -1) {
6413 bp1->expires = (clock_t)-1;
6414 } else {
6415 bp1->expires = ddi_get_lbolt() +
6416 pc_cb[cb_index].duration;
6417 }
6418 } else {
6419 /* There's no callback registered so don't add to the hash */
6420 rw_exit(&pc_cb[cb_index].cb_rwlock);
6421 kmem_free(bp1, sizeof (*bp1));
6422 return;
6423 }
6424
6425 index = PAGE_CAPTURE_HASH(pp);
6426
6427 /*
6428 * Only allow capture flag to be modified under this mutex.
6429 * Prevents multiple entries for same page getting added.
6430 */
6431 mutex_enter(&page_capture_hash[index].pchh_mutex);
6432
6433 /*
6434 * if not already on the hash, set capture bit and add to the hash
6435 */
6436 if (!(pp->p_toxic & PR_CAPTURE)) {
6437 #ifdef DEBUG
6438 /* Check for duplicate entries */
6439 for (l = 0; l < 2; l++) {
6440 tp1 = page_capture_hash[index].lists[l].next;
6441 while (tp1 != &page_capture_hash[index].lists[l]) {
6442 if (tp1->pp == pp) {
6443 panic("page pp 0x%p already on hash "
6444 "at 0x%p\n",
6445 (void *)pp, (void *)tp1);
6446 }
6447 tp1 = tp1->next;
6448 }
6449 }
6450
6451 #endif
6452 page_settoxic(pp, PR_CAPTURE);
6453 pri = PAGE_CAPTURE_PRIO(pp);
6454 bp1->pri = pri;
6455 bp1->next = page_capture_hash[index].lists[0].next;
6456 bp1->prev = &page_capture_hash[index].lists[0];
6457 bp1->next->prev = bp1;
6458 page_capture_hash[index].lists[0].next = bp1;
6459 page_capture_hash[index].num_pages[pri]++;
6460 if (flags & CAPTURE_RETIRE) {
6461 page_retire_incr_pend_count(datap);
6462 }
6463 mutex_exit(&page_capture_hash[index].pchh_mutex);
6464 rw_exit(&pc_cb[cb_index].cb_rwlock);
6465 cv_signal(&pc_cv);
6466 return;
6467 }
6468
6469 /*
6470 * A page retire request will replace any other request.
6471 * A second physmem request which is for a different process than
6472 * the currently registered one will be dropped as there is
6473 * no way to hold the private data for both calls.
6474 * In the future, once there are more callers, this will have to
6475 * be worked out better as there needs to be private storage for
6476 * at least each type of caller (maybe have datap be an array of
6477 * *void's so that we can index based upon callers index).
6478 */
6479
6480 /* walk hash list to update expire time */
6481 for (i = 0; i < 2; i++) {
6482 bp2 = page_capture_hash[index].lists[i].next;
6483 while (bp2 != &page_capture_hash[index].lists[i]) {
6484 if (bp2->pp == pp) {
6485 if (flags & CAPTURE_RETIRE) {
6486 if (!(bp2->flags & CAPTURE_RETIRE)) {
6487 page_retire_incr_pend_count(
6488 datap);
6489 bp2->flags = flags;
6490 bp2->expires = bp1->expires;
6491 bp2->datap = datap;
6492 }
6493 } else {
6494 ASSERT(flags & CAPTURE_PHYSMEM);
6495 if (!(bp2->flags & CAPTURE_RETIRE) &&
6496 (datap == bp2->datap)) {
6497 bp2->expires = bp1->expires;
6498 }
6499 }
6500 mutex_exit(&page_capture_hash[index].
6501 pchh_mutex);
6502 rw_exit(&pc_cb[cb_index].cb_rwlock);
6503 kmem_free(bp1, sizeof (*bp1));
6504 return;
6505 }
6506 bp2 = bp2->next;
6507 }
6508 }
6509
6510 /*
6511 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes
6512 * and thus it either has to be set or not set and can't change
6513 * while holding the mutex above.
6514 */
6515 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n",
6516 (void *)pp);
6517 }
6518
6519 /*
6520 * We have a page in our hands, lets try and make it ours by turning
6521 * it into a clean page like it had just come off the freelists.
6522 *
6523 * Returns 0 on success, with the page still EXCL locked.
6524 * On failure, the page will be unlocked, and returns EAGAIN
6525 */
6526 static int
6527 page_capture_clean_page(page_t *pp)
6528 {
6529 page_t *newpp;
6530 int skip_unlock = 0;
6531 spgcnt_t count;
6532 page_t *tpp;
6533 int ret = 0;
6534 int extra;
6535
6536 ASSERT(PAGE_EXCL(pp));
6537 ASSERT(!PP_RETIRED(pp));
6538 ASSERT(curthread->t_flag & T_CAPTURING);
6539
6540 if (PP_ISFREE(pp)) {
6541 if (!page_reclaim(pp, NULL)) {
6542 skip_unlock = 1;
6543 ret = EAGAIN;
6544 goto cleanup;
6545 }
6546 ASSERT(pp->p_szc == 0);
6547 if (pp->p_vnode != NULL) {
6548 /*
6549 * Since this page came from the
6550 * cachelist, we must destroy the
6551 * old vnode association.
6552 */
6553 page_hashout(pp, NULL);
6554 }
6555 goto cleanup;
6556 }
6557
6558 /*
6559 * If we know page_relocate will fail, skip it
6560 * It could still fail due to a UE on another page but we
6561 * can't do anything about that.
6562 */
6563 if (pp->p_toxic & PR_UE) {
6564 goto skip_relocate;
6565 }
6566
6567 /*
6568 * It's possible that pages can not have a vnode as fsflush comes
6569 * through and cleans up these pages. It's ugly but that's how it is.
6570 */
6571 if (pp->p_vnode == NULL) {
6572 goto skip_relocate;
6573 }
6574
6575 /*
6576 * Page was not free, so lets try to relocate it.
6577 * page_relocate only works with root pages, so if this is not a root
6578 * page, we need to demote it to try and relocate it.
6579 * Unfortunately this is the best we can do right now.
6580 */
6581 newpp = NULL;
6582 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) {
6583 if (page_try_demote_pages(pp) == 0) {
6584 ret = EAGAIN;
6585 goto cleanup;
6586 }
6587 }
6588 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL);
6589 if (ret == 0) {
6590 page_t *npp;
6591 /* unlock the new page(s) */
6592 while (count-- > 0) {
6593 ASSERT(newpp != NULL);
6594 npp = newpp;
6595 page_sub(&newpp, npp);
6596 page_unlock(npp);
6597 }
6598 ASSERT(newpp == NULL);
6599 /*
6600 * Check to see if the page we have is too large.
6601 * If so, demote it freeing up the extra pages.
6602 */
6603 if (pp->p_szc > 0) {
6604 /* For now demote extra pages to szc == 0 */
6605 extra = page_get_pagecnt(pp->p_szc) - 1;
6606 while (extra > 0) {
6607 tpp = pp->p_next;
6608 page_sub(&pp, tpp);
6609 tpp->p_szc = 0;
6610 page_free(tpp, 1);
6611 extra--;
6612 }
6613 /* Make sure to set our page to szc 0 as well */
6614 ASSERT(pp->p_next == pp && pp->p_prev == pp);
6615 pp->p_szc = 0;
6616 }
6617 goto cleanup;
6618 } else if (ret == EIO) {
6619 ret = EAGAIN;
6620 goto cleanup;
6621 } else {
6622 /*
6623 * Need to reset return type as we failed to relocate the page
6624 * but that does not mean that some of the next steps will not
6625 * work.
6626 */
6627 ret = 0;
6628 }
6629
6630 skip_relocate:
6631
6632 if (pp->p_szc > 0) {
6633 if (page_try_demote_pages(pp) == 0) {
6634 ret = EAGAIN;
6635 goto cleanup;
6636 }
6637 }
6638
6639 ASSERT(pp->p_szc == 0);
6640
6641 if (hat_ismod(pp)) {
6642 ret = EAGAIN;
6643 goto cleanup;
6644 }
6645 if (PP_ISKAS(pp)) {
6646 ret = EAGAIN;
6647 goto cleanup;
6648 }
6649 if (pp->p_lckcnt || pp->p_cowcnt) {
6650 ret = EAGAIN;
6651 goto cleanup;
6652 }
6653
6654 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
6655 ASSERT(!hat_page_is_mapped(pp));
6656
6657 if (hat_ismod(pp)) {
6658 /*
6659 * This is a semi-odd case as the page is now modified but not
6660 * mapped as we just unloaded the mappings above.
6661 */
6662 ret = EAGAIN;
6663 goto cleanup;
6664 }
6665 if (pp->p_vnode != NULL) {
6666 page_hashout(pp, NULL);
6667 }
6668
6669 /*
6670 * At this point, the page should be in a clean state and
6671 * we can do whatever we want with it.
6672 */
6673
6674 cleanup:
6675 if (ret != 0) {
6676 if (!skip_unlock) {
6677 page_unlock(pp);
6678 }
6679 } else {
6680 ASSERT(pp->p_szc == 0);
6681 ASSERT(PAGE_EXCL(pp));
6682
6683 pp->p_next = pp;
6684 pp->p_prev = pp;
6685 }
6686 return (ret);
6687 }
6688
6689 /*
6690 * Various callers of page_trycapture() can have different restrictions upon
6691 * what memory they have access to.
6692 * Returns 0 on success, with the following error codes on failure:
6693 * EPERM - The requested page is long term locked, and thus repeated
6694 * requests to capture this page will likely fail.
6695 * ENOMEM - There was not enough free memory in the system to safely
6696 * map the requested page.
6697 * ENOENT - The requested page was inside the kernel cage, and the
6698 * PHYSMEM_CAGE flag was not set.
6699 */
6700 int
6701 page_capture_pre_checks(page_t *pp, uint_t flags)
6702 {
6703 ASSERT(pp != NULL);
6704
6705 #if defined(__sparc)
6706 if (pp->p_vnode == &promvp) {
6707 return (EPERM);
6708 }
6709
6710 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) &&
6711 (flags & CAPTURE_PHYSMEM)) {
6712 return (ENOENT);
6713 }
6714
6715 if (PP_ISNORELOCKERNEL(pp)) {
6716 return (EPERM);
6717 }
6718 #else
6719 if (PP_ISKAS(pp)) {
6720 return (EPERM);
6721 }
6722 #endif /* __sparc */
6723
6724 /* only physmem currently has the restrictions checked below */
6725 if (!(flags & CAPTURE_PHYSMEM)) {
6726 return (0);
6727 }
6728
6729 if (availrmem < swapfs_minfree) {
6730 /*
6731 * We won't try to capture this page as we are
6732 * running low on memory.
6733 */
6734 return (ENOMEM);
6735 }
6736 return (0);
6737 }
6738
6739 /*
6740 * Once we have a page in our mits, go ahead and complete the capture
6741 * operation.
6742 * Returns 1 on failure where page is no longer needed
6743 * Returns 0 on success
6744 * Returns -1 if there was a transient failure.
6745 * Failure cases must release the SE_EXCL lock on pp (usually via page_free).
6746 */
6747 int
6748 page_capture_take_action(page_t *pp, uint_t flags, void *datap)
6749 {
6750 int cb_index;
6751 int ret = 0;
6752 page_capture_hash_bucket_t *bp1;
6753 page_capture_hash_bucket_t *bp2;
6754 int index;
6755 int found = 0;
6756 int i;
6757
6758 ASSERT(PAGE_EXCL(pp));
6759 ASSERT(curthread->t_flag & T_CAPTURING);
6760
6761 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6762 if ((flags >> cb_index) & 1) {
6763 break;
6764 }
6765 }
6766 ASSERT(cb_index < PC_NUM_CALLBACKS);
6767
6768 /*
6769 * Remove the entry from the page_capture hash, but don't free it yet
6770 * as we may need to put it back.
6771 * Since we own the page at this point in time, we should find it
6772 * in the hash if this is an ASYNC call. If we don't it's likely
6773 * that the page_capture_async() thread decided that this request
6774 * had expired, in which case we just continue on.
6775 */
6776 if (flags & CAPTURE_ASYNC) {
6777
6778 index = PAGE_CAPTURE_HASH(pp);
6779
6780 mutex_enter(&page_capture_hash[index].pchh_mutex);
6781 for (i = 0; i < 2 && !found; i++) {
6782 bp1 = page_capture_hash[index].lists[i].next;
6783 while (bp1 != &page_capture_hash[index].lists[i]) {
6784 if (bp1->pp == pp) {
6785 bp1->next->prev = bp1->prev;
6786 bp1->prev->next = bp1->next;
6787 page_capture_hash[index].
6788 num_pages[bp1->pri]--;
6789 page_clrtoxic(pp, PR_CAPTURE);
6790 found = 1;
6791 break;
6792 }
6793 bp1 = bp1->next;
6794 }
6795 }
6796 mutex_exit(&page_capture_hash[index].pchh_mutex);
6797 }
6798
6799 /* Synchronize with the unregister func. */
6800 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6801 if (!pc_cb[cb_index].cb_active) {
6802 page_free(pp, 1);
6803 rw_exit(&pc_cb[cb_index].cb_rwlock);
6804 if (found) {
6805 kmem_free(bp1, sizeof (*bp1));
6806 }
6807 return (1);
6808 }
6809
6810 /*
6811 * We need to remove the entry from the page capture hash and turn off
6812 * the PR_CAPTURE bit before calling the callback. We'll need to cache
6813 * the entry here, and then based upon the return value, cleanup
6814 * appropriately or re-add it to the hash, making sure that someone else
6815 * hasn't already done so.
6816 * It should be rare for the callback to fail and thus it's ok for
6817 * the failure path to be a bit complicated as the success path is
6818 * cleaner and the locking rules are easier to follow.
6819 */
6820
6821 ret = pc_cb[cb_index].cb_func(pp, datap, flags);
6822
6823 rw_exit(&pc_cb[cb_index].cb_rwlock);
6824
6825 /*
6826 * If this was an ASYNC request, we need to cleanup the hash if the
6827 * callback was successful or if the request was no longer valid.
6828 * For non-ASYNC requests, we return failure to map and the caller
6829 * will take care of adding the request to the hash.
6830 * Note also that the callback itself is responsible for the page
6831 * at this point in time in terms of locking ... The most common
6832 * case for the failure path should just be a page_free.
6833 */
6834 if (ret >= 0) {
6835 if (found) {
6836 if (bp1->flags & CAPTURE_RETIRE) {
6837 page_retire_decr_pend_count(datap);
6838 }
6839 kmem_free(bp1, sizeof (*bp1));
6840 }
6841 return (ret);
6842 }
6843 if (!found) {
6844 return (ret);
6845 }
6846
6847 ASSERT(flags & CAPTURE_ASYNC);
6848
6849 /*
6850 * Check for expiration time first as we can just free it up if it's
6851 * expired.
6852 */
6853 if (ddi_get_lbolt() > bp1->expires && bp1->expires != -1) {
6854 kmem_free(bp1, sizeof (*bp1));
6855 return (ret);
6856 }
6857
6858 /*
6859 * The callback failed and there used to be an entry in the hash for
6860 * this page, so we need to add it back to the hash.
6861 */
6862 mutex_enter(&page_capture_hash[index].pchh_mutex);
6863 if (!(pp->p_toxic & PR_CAPTURE)) {
6864 /* just add bp1 back to head of walked list */
6865 page_settoxic(pp, PR_CAPTURE);
6866 bp1->next = page_capture_hash[index].lists[1].next;
6867 bp1->prev = &page_capture_hash[index].lists[1];
6868 bp1->next->prev = bp1;
6869 bp1->pri = PAGE_CAPTURE_PRIO(pp);
6870 page_capture_hash[index].lists[1].next = bp1;
6871 page_capture_hash[index].num_pages[bp1->pri]++;
6872 mutex_exit(&page_capture_hash[index].pchh_mutex);
6873 return (ret);
6874 }
6875
6876 /*
6877 * Otherwise there was a new capture request added to list
6878 * Need to make sure that our original data is represented if
6879 * appropriate.
6880 */
6881 for (i = 0; i < 2; i++) {
6882 bp2 = page_capture_hash[index].lists[i].next;
6883 while (bp2 != &page_capture_hash[index].lists[i]) {
6884 if (bp2->pp == pp) {
6885 if (bp1->flags & CAPTURE_RETIRE) {
6886 if (!(bp2->flags & CAPTURE_RETIRE)) {
6887 bp2->szc = bp1->szc;
6888 bp2->flags = bp1->flags;
6889 bp2->expires = bp1->expires;
6890 bp2->datap = bp1->datap;
6891 }
6892 } else {
6893 ASSERT(bp1->flags & CAPTURE_PHYSMEM);
6894 if (!(bp2->flags & CAPTURE_RETIRE)) {
6895 bp2->szc = bp1->szc;
6896 bp2->flags = bp1->flags;
6897 bp2->expires = bp1->expires;
6898 bp2->datap = bp1->datap;
6899 }
6900 }
6901 page_capture_hash[index].num_pages[bp2->pri]--;
6902 bp2->pri = PAGE_CAPTURE_PRIO(pp);
6903 page_capture_hash[index].num_pages[bp2->pri]++;
6904 mutex_exit(&page_capture_hash[index].
6905 pchh_mutex);
6906 kmem_free(bp1, sizeof (*bp1));
6907 return (ret);
6908 }
6909 bp2 = bp2->next;
6910 }
6911 }
6912 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp);
6913 /*NOTREACHED*/
6914 }
6915
6916 /*
6917 * Try to capture the given page for the caller specified in the flags
6918 * parameter. The page will either be captured and handed over to the
6919 * appropriate callback, or will be queued up in the page capture hash
6920 * to be captured asynchronously.
6921 * If the current request is due to an async capture, the page must be
6922 * exclusively locked before calling this function.
6923 * Currently szc must be 0 but in the future this should be expandable to
6924 * other page sizes.
6925 * Returns 0 on success, with the following error codes on failure:
6926 * EPERM - The requested page is long term locked, and thus repeated
6927 * requests to capture this page will likely fail.
6928 * ENOMEM - There was not enough free memory in the system to safely
6929 * map the requested page.
6930 * ENOENT - The requested page was inside the kernel cage, and the
6931 * CAPTURE_GET_CAGE flag was not set.
6932 * EAGAIN - The requested page could not be capturead at this point in
6933 * time but future requests will likely work.
6934 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag
6935 * was not set.
6936 */
6937 int
6938 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
6939 {
6940 int ret;
6941 int cb_index;
6942
6943 if (flags & CAPTURE_ASYNC) {
6944 ASSERT(PAGE_EXCL(pp));
6945 goto async;
6946 }
6947
6948 /* Make sure there's enough availrmem ... */
6949 ret = page_capture_pre_checks(pp, flags);
6950 if (ret != 0) {
6951 return (ret);
6952 }
6953
6954 if (!page_trylock(pp, SE_EXCL)) {
6955 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6956 if ((flags >> cb_index) & 1) {
6957 break;
6958 }
6959 }
6960 ASSERT(cb_index < PC_NUM_CALLBACKS);
6961 ret = EAGAIN;
6962 /* Special case for retired pages */
6963 if (PP_RETIRED(pp)) {
6964 if (flags & CAPTURE_GET_RETIRED) {
6965 if (!page_unretire_pp(pp, PR_UNR_TEMP)) {
6966 /*
6967 * Need to set capture bit and add to
6968 * hash so that the page will be
6969 * retired when freed.
6970 */
6971 page_capture_add_hash(pp, szc,
6972 CAPTURE_RETIRE, NULL);
6973 ret = 0;
6974 goto own_page;
6975 }
6976 } else {
6977 return (EBUSY);
6978 }
6979 }
6980 page_capture_add_hash(pp, szc, flags, datap);
6981 return (ret);
6982 }
6983
6984 async:
6985 ASSERT(PAGE_EXCL(pp));
6986
6987 /* Need to check for physmem async requests that availrmem is sane */
6988 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) ==
6989 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) &&
6990 (availrmem < swapfs_minfree)) {
6991 page_unlock(pp);
6992 return (ENOMEM);
6993 }
6994
6995 ret = page_capture_clean_page(pp);
6996
6997 if (ret != 0) {
6998 /* We failed to get the page, so lets add it to the hash */
6999 if (!(flags & CAPTURE_ASYNC)) {
7000 page_capture_add_hash(pp, szc, flags, datap);
7001 }
7002 return (ret);
7003 }
7004
7005 own_page:
7006 ASSERT(PAGE_EXCL(pp));
7007 ASSERT(pp->p_szc == 0);
7008
7009 /* Call the callback */
7010 ret = page_capture_take_action(pp, flags, datap);
7011
7012 if (ret == 0) {
7013 return (0);
7014 }
7015
7016 /*
7017 * Note that in the failure cases from page_capture_take_action, the
7018 * EXCL lock will have already been dropped.
7019 */
7020 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) {
7021 page_capture_add_hash(pp, szc, flags, datap);
7022 }
7023 return (EAGAIN);
7024 }
7025
7026 int
7027 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
7028 {
7029 int ret;
7030
7031 curthread->t_flag |= T_CAPTURING;
7032 ret = page_itrycapture(pp, szc, flags, datap);
7033 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */
7034 return (ret);
7035 }
7036
7037 /*
7038 * When unlocking a page which has the PR_CAPTURE bit set, this routine
7039 * gets called to try and capture the page.
7040 */
7041 void
7042 page_unlock_capture(page_t *pp)
7043 {
7044 page_capture_hash_bucket_t *bp;
7045 int index;
7046 int i;
7047 uint_t szc;
7048 uint_t flags = 0;
7049 void *datap;
7050 kmutex_t *mp;
7051 extern vnode_t retired_pages;
7052
7053 /*
7054 * We need to protect against a possible deadlock here where we own
7055 * the vnode page hash mutex and want to acquire it again as there
7056 * are locations in the code, where we unlock a page while holding
7057 * the mutex which can lead to the page being captured and eventually
7058 * end up here. As we may be hashing out the old page and hashing into
7059 * the retire vnode, we need to make sure we don't own them.
7060 * Other callbacks who do hash operations also need to make sure that
7061 * before they hashin to a vnode that they do not currently own the
7062 * vphm mutex otherwise there will be a panic.
7063 */
7064 if (mutex_owned(page_vnode_mutex(&retired_pages))) {
7065 page_unlock_nocapture(pp);
7066 return;
7067 }
7068 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) {
7069 page_unlock_nocapture(pp);
7070 return;
7071 }
7072
7073 index = PAGE_CAPTURE_HASH(pp);
7074
7075 mp = &page_capture_hash[index].pchh_mutex;
7076 mutex_enter(mp);
7077 for (i = 0; i < 2; i++) {
7078 bp = page_capture_hash[index].lists[i].next;
7079 while (bp != &page_capture_hash[index].lists[i]) {
7080 if (bp->pp == pp) {
7081 szc = bp->szc;
7082 flags = bp->flags | CAPTURE_ASYNC;
7083 datap = bp->datap;
7084 mutex_exit(mp);
7085 (void) page_trycapture(pp, szc, flags, datap);
7086 return;
7087 }
7088 bp = bp->next;
7089 }
7090 }
7091
7092 /* Failed to find page in hash so clear flags and unlock it. */
7093 page_clrtoxic(pp, PR_CAPTURE);
7094 page_unlock(pp);
7095
7096 mutex_exit(mp);
7097 }
7098
7099 void
7100 page_capture_init()
7101 {
7102 int i;
7103 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7104 page_capture_hash[i].lists[0].next =
7105 &page_capture_hash[i].lists[0];
7106 page_capture_hash[i].lists[0].prev =
7107 &page_capture_hash[i].lists[0];
7108 page_capture_hash[i].lists[1].next =
7109 &page_capture_hash[i].lists[1];
7110 page_capture_hash[i].lists[1].prev =
7111 &page_capture_hash[i].lists[1];
7112 }
7113
7114 pc_thread_shortwait = 23 * hz;
7115 pc_thread_longwait = 1201 * hz;
7116 pc_thread_retry = 3;
7117 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL);
7118 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL);
7119 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0,
7120 TS_RUN, minclsyspri);
7121 }
7122
7123 /*
7124 * It is necessary to scrub any failing pages prior to reboot in order to
7125 * prevent a latent error trap from occurring on the next boot.
7126 */
7127 void
7128 page_retire_mdboot()
7129 {
7130 page_t *pp;
7131 int i, j;
7132 page_capture_hash_bucket_t *bp;
7133 uchar_t pri;
7134
7135 /* walk lists looking for pages to scrub */
7136 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7137 for (pri = 0; pri < PC_NUM_PRI; pri++) {
7138 if (page_capture_hash[i].num_pages[pri] != 0) {
7139 break;
7140 }
7141 }
7142 if (pri == PC_NUM_PRI)
7143 continue;
7144
7145 mutex_enter(&page_capture_hash[i].pchh_mutex);
7146
7147 for (j = 0; j < 2; j++) {
7148 bp = page_capture_hash[i].lists[j].next;
7149 while (bp != &page_capture_hash[i].lists[j]) {
7150 pp = bp->pp;
7151 if (PP_TOXIC(pp)) {
7152 if (page_trylock(pp, SE_EXCL)) {
7153 PP_CLRFREE(pp);
7154 pagescrub(pp, 0, PAGESIZE);
7155 page_unlock(pp);
7156 }
7157 }
7158 bp = bp->next;
7159 }
7160 }
7161 mutex_exit(&page_capture_hash[i].pchh_mutex);
7162 }
7163 }
7164
7165 /*
7166 * Walk the page_capture_hash trying to capture pages and also cleanup old
7167 * entries which have expired.
7168 */
7169 void
7170 page_capture_async()
7171 {
7172 page_t *pp;
7173 int i;
7174 int ret;
7175 page_capture_hash_bucket_t *bp1, *bp2;
7176 uint_t szc;
7177 uint_t flags;
7178 void *datap;
7179 uchar_t pri;
7180
7181 /* If there are outstanding pages to be captured, get to work */
7182 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7183 for (pri = 0; pri < PC_NUM_PRI; pri++) {
7184 if (page_capture_hash[i].num_pages[pri] != 0)
7185 break;
7186 }
7187 if (pri == PC_NUM_PRI)
7188 continue;
7189
7190 /* Append list 1 to list 0 and then walk through list 0 */
7191 mutex_enter(&page_capture_hash[i].pchh_mutex);
7192 bp1 = &page_capture_hash[i].lists[1];
7193 bp2 = bp1->next;
7194 if (bp1 != bp2) {
7195 bp1->prev->next = page_capture_hash[i].lists[0].next;
7196 bp2->prev = &page_capture_hash[i].lists[0];
7197 page_capture_hash[i].lists[0].next->prev = bp1->prev;
7198 page_capture_hash[i].lists[0].next = bp2;
7199 bp1->next = bp1;
7200 bp1->prev = bp1;
7201 }
7202
7203 /* list[1] will be empty now */
7204
7205 bp1 = page_capture_hash[i].lists[0].next;
7206 while (bp1 != &page_capture_hash[i].lists[0]) {
7207 /* Check expiration time */
7208 if ((ddi_get_lbolt() > bp1->expires &&
7209 bp1->expires != -1) ||
7210 page_deleted(bp1->pp)) {
7211 page_capture_hash[i].lists[0].next = bp1->next;
7212 bp1->next->prev =
7213 &page_capture_hash[i].lists[0];
7214 page_capture_hash[i].num_pages[bp1->pri]--;
7215
7216 /*
7217 * We can safely remove the PR_CAPTURE bit
7218 * without holding the EXCL lock on the page
7219 * as the PR_CAPTURE bit requres that the
7220 * page_capture_hash[].pchh_mutex be held
7221 * to modify it.
7222 */
7223 page_clrtoxic(bp1->pp, PR_CAPTURE);
7224 mutex_exit(&page_capture_hash[i].pchh_mutex);
7225 kmem_free(bp1, sizeof (*bp1));
7226 mutex_enter(&page_capture_hash[i].pchh_mutex);
7227 bp1 = page_capture_hash[i].lists[0].next;
7228 continue;
7229 }
7230 pp = bp1->pp;
7231 szc = bp1->szc;
7232 flags = bp1->flags;
7233 datap = bp1->datap;
7234 mutex_exit(&page_capture_hash[i].pchh_mutex);
7235 if (page_trylock(pp, SE_EXCL)) {
7236 ret = page_trycapture(pp, szc,
7237 flags | CAPTURE_ASYNC, datap);
7238 } else {
7239 ret = 1; /* move to walked hash */
7240 }
7241
7242 if (ret != 0) {
7243 /* Move to walked hash */
7244 (void) page_capture_move_to_walked(pp);
7245 }
7246 mutex_enter(&page_capture_hash[i].pchh_mutex);
7247 bp1 = page_capture_hash[i].lists[0].next;
7248 }
7249
7250 mutex_exit(&page_capture_hash[i].pchh_mutex);
7251 }
7252 }
7253
7254 /*
7255 * This function is called by the page_capture_thread, and is needed in
7256 * in order to initiate aio cleanup, so that pages used in aio
7257 * will be unlocked and subsequently retired by page_capture_thread.
7258 */
7259 static int
7260 do_aio_cleanup(void)
7261 {
7262 proc_t *procp;
7263 int (*aio_cleanup_dr_delete_memory)(proc_t *);
7264 int cleaned = 0;
7265
7266 if (modload("sys", "kaio") == -1) {
7267 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio");
7268 return (0);
7269 }
7270 /*
7271 * We use the aio_cleanup_dr_delete_memory function to
7272 * initiate the actual clean up; this function will wake
7273 * up the per-process aio_cleanup_thread.
7274 */
7275 aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
7276 modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
7277 if (aio_cleanup_dr_delete_memory == NULL) {
7278 cmn_err(CE_WARN,
7279 "aio_cleanup_dr_delete_memory not found in kaio");
7280 return (0);
7281 }
7282 mutex_enter(&pidlock);
7283 for (procp = practive; (procp != NULL); procp = procp->p_next) {
7284 mutex_enter(&procp->p_lock);
7285 if (procp->p_aio != NULL) {
7286 /* cleanup proc's outstanding kaio */
7287 cleaned += (*aio_cleanup_dr_delete_memory)(procp);
7288 }
7289 mutex_exit(&procp->p_lock);
7290 }
7291 mutex_exit(&pidlock);
7292 return (cleaned);
7293 }
7294
7295 /*
7296 * helper function for page_capture_thread
7297 */
7298 static void
7299 page_capture_handle_outstanding(void)
7300 {
7301 int ntry;
7302
7303 /* Reap pages before attempting capture pages */
7304 kmem_reap();
7305
7306 if ((page_retire_pend_count() > page_retire_pend_kas_count()) &&
7307 hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
7308 /*
7309 * Note: Purging only for platforms that support
7310 * ISM hat_pageunload() - mainly SPARC. On x86/x64
7311 * platforms ISM pages SE_SHARED locked until destroyed.
7312 */
7313
7314 /* disable and purge seg_pcache */
7315 (void) seg_p_disable();
7316 for (ntry = 0; ntry < pc_thread_retry; ntry++) {
7317 if (!page_retire_pend_count())
7318 break;
7319 if (do_aio_cleanup()) {
7320 /*
7321 * allow the apps cleanup threads
7322 * to run
7323 */
7324 delay(pc_thread_shortwait);
7325 }
7326 page_capture_async();
7327 }
7328 /* reenable seg_pcache */
7329 seg_p_enable();
7330
7331 /* completed what can be done. break out */
7332 return;
7333 }
7334
7335 /*
7336 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap
7337 * and then attempt to capture.
7338 */
7339 seg_preap();
7340 page_capture_async();
7341 }
7342
7343 /*
7344 * The page_capture_thread loops forever, looking to see if there are
7345 * pages still waiting to be captured.
7346 */
7347 static void
7348 page_capture_thread(void)
7349 {
7350 callb_cpr_t c;
7351 int i;
7352 int high_pri_pages;
7353 int low_pri_pages;
7354 clock_t timeout;
7355
7356 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture");
7357
7358 mutex_enter(&pc_thread_mutex);
7359 for (;;) {
7360 high_pri_pages = 0;
7361 low_pri_pages = 0;
7362 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7363 high_pri_pages +=
7364 page_capture_hash[i].num_pages[PC_PRI_HI];
7365 low_pri_pages +=
7366 page_capture_hash[i].num_pages[PC_PRI_LO];
7367 }
7368
7369 timeout = pc_thread_longwait;
7370 if (high_pri_pages != 0) {
7371 timeout = pc_thread_shortwait;
7372 page_capture_handle_outstanding();
7373 } else if (low_pri_pages != 0) {
7374 page_capture_async();
7375 }
7376 CALLB_CPR_SAFE_BEGIN(&c);
7377 (void) cv_reltimedwait(&pc_cv, &pc_thread_mutex,
7378 timeout, TR_CLOCK_TICK);
7379 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
7380 }
7381 /*NOTREACHED*/
7382 }
7383 /*
7384 * Attempt to locate a bucket that has enough pages to satisfy the request.
7385 * The initial check is done without the lock to avoid unneeded contention.
7386 * The function returns 1 if enough pages were found, else 0 if it could not
7387 * find enough pages in a bucket.
7388 */
7389 static int
7390 pcf_decrement_bucket(pgcnt_t npages)
7391 {
7392 struct pcf *p;
7393 struct pcf *q;
7394 int i;
7395
7396 p = &pcf[PCF_INDEX()];
7397 q = &pcf[pcf_fanout];
7398 for (i = 0; i < pcf_fanout; i++) {
7399 if (p->pcf_count > npages) {
7400 /*
7401 * a good one to try.
7402 */
7403 mutex_enter(&p->pcf_lock);
7404 if (p->pcf_count > npages) {
7405 p->pcf_count -= (uint_t)npages;
7406 /*
7407 * freemem is not protected by any lock.
7408 * Thus, we cannot have any assertion
7409 * containing freemem here.
7410 */
7411 freemem -= npages;
7412 mutex_exit(&p->pcf_lock);
7413 return (1);
7414 }
7415 mutex_exit(&p->pcf_lock);
7416 }
7417 p++;
7418 if (p >= q) {
7419 p = pcf;
7420 }
7421 }
7422 return (0);
7423 }
7424
7425 /*
7426 * Arguments:
7427 * pcftotal_ret: If the value is not NULL and we have walked all the
7428 * buckets but did not find enough pages then it will
7429 * be set to the total number of pages in all the pcf
7430 * buckets.
7431 * npages: Is the number of pages we have been requested to
7432 * find.
7433 * unlock: If set to 0 we will leave the buckets locked if the
7434 * requested number of pages are not found.
7435 *
7436 * Go and try to satisfy the page request from any number of buckets.
7437 * This can be a very expensive operation as we have to lock the buckets
7438 * we are checking (and keep them locked), starting at bucket 0.
7439 *
7440 * The function returns 1 if enough pages were found, else 0 if it could not
7441 * find enough pages in the buckets.
7442 *
7443 */
7444 static int
7445 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock)
7446 {
7447 struct pcf *p;
7448 pgcnt_t pcftotal;
7449 int i;
7450
7451 p = pcf;
7452 /* try to collect pages from several pcf bins */
7453 for (pcftotal = 0, i = 0; i < pcf_fanout; i++) {
7454 mutex_enter(&p->pcf_lock);
7455 pcftotal += p->pcf_count;
7456 if (pcftotal >= npages) {
7457 /*
7458 * Wow! There are enough pages laying around
7459 * to satisfy the request. Do the accounting,
7460 * drop the locks we acquired, and go back.
7461 *
7462 * freemem is not protected by any lock. So,
7463 * we cannot have any assertion containing
7464 * freemem.
7465 */
7466 freemem -= npages;
7467 while (p >= pcf) {
7468 if (p->pcf_count <= npages) {
7469 npages -= p->pcf_count;
7470 p->pcf_count = 0;
7471 } else {
7472 p->pcf_count -= (uint_t)npages;
7473 npages = 0;
7474 }
7475 mutex_exit(&p->pcf_lock);
7476 p--;
7477 }
7478 ASSERT(npages == 0);
7479 return (1);
7480 }
7481 p++;
7482 }
7483 if (unlock) {
7484 /* failed to collect pages - release the locks */
7485 while (--p >= pcf) {
7486 mutex_exit(&p->pcf_lock);
7487 }
7488 }
7489 if (pcftotal_ret != NULL)
7490 *pcftotal_ret = pcftotal;
7491 return (0);
7492 }