1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 /*
28 * Copyright 2017 Joyent, Inc.
29 * Copyright 2017 James S Blachly, MD <james.blachly@gmail.com>
30 */
31
32 /*
33 * Memory special file
34 */
35
36 #include <sys/types.h>
37 #include <sys/param.h>
38 #include <sys/user.h>
39 #include <sys/buf.h>
40 #include <sys/systm.h>
41 #include <sys/cred.h>
42 #include <sys/vm.h>
43 #include <sys/uio.h>
44 #include <sys/mman.h>
45 #include <sys/kmem.h>
46 #include <vm/seg.h>
47 #include <vm/page.h>
48 #include <sys/stat.h>
49 #include <sys/vmem.h>
50 #include <sys/memlist.h>
51 #include <sys/bootconf.h>
52
53 #include <vm/seg_vn.h>
54 #include <vm/seg_dev.h>
55 #include <vm/seg_kmem.h>
56 #include <vm/seg_kp.h>
57 #include <vm/seg_kpm.h>
58 #include <vm/hat.h>
59
60 #include <sys/conf.h>
61 #include <sys/mem.h>
62 #include <sys/types.h>
63 #include <sys/conf.h>
64 #include <sys/param.h>
65 #include <sys/systm.h>
66 #include <sys/errno.h>
67 #include <sys/modctl.h>
68 #include <sys/memlist.h>
69 #include <sys/ddi.h>
70 #include <sys/sunddi.h>
71 #include <sys/debug.h>
72 #include <sys/fm/protocol.h>
73
74 #if defined(__sparc)
75 extern int cpu_get_mem_name(uint64_t, uint64_t *, uint64_t, char *, int, int *);
76 extern int cpu_get_mem_info(uint64_t, uint64_t, uint64_t *, uint64_t *,
77 uint64_t *, int *, int *, int *);
78 extern size_t cpu_get_name_bufsize(void);
79 extern int cpu_get_mem_sid(char *, char *, int, int *);
80 extern int cpu_get_mem_addr(char *, char *, uint64_t, uint64_t *);
81 #elif defined(__x86)
82 #include <sys/cpu_module.h>
83 #endif /* __sparc */
84
85 /*
86 * Turn a byte length into a pagecount. The DDI btop takes a
87 * 32-bit size on 32-bit machines, this handles 64-bit sizes for
88 * large physical-memory 32-bit machines.
89 */
90 #define BTOP(x) ((pgcnt_t)((x) >> _pageshift))
91
92 static kmutex_t mm_lock;
93 static caddr_t mm_map;
94
95 static dev_info_t *mm_dip; /* private copy of devinfo pointer */
96
97 static int mm_kmem_io_access;
98
99 static int mm_kstat_update(kstat_t *ksp, int rw);
100 static int mm_kstat_snapshot(kstat_t *ksp, void *buf, int rw);
101
102 static int mm_read_mem_name(intptr_t data, mem_name_t *mem_name);
103
104 #define MM_KMEMLOG_NENTRIES 64
105
106 static int mm_kmemlogent;
107 static mm_logentry_t mm_kmemlog[MM_KMEMLOG_NENTRIES];
108
109 /*
110 * On kmem/allmem writes, we log information that might be useful in the event
111 * that a write is errant (that is, due to operator error) and induces a later
112 * problem. Note that (in particular) in the event of such operator-induced
113 * corruption, a search over the kernel address space for the corrupted
114 * address will yield the ring buffer entry that recorded the write. And
115 * should it seem baroque or otherwise unnecessary, yes, we need this kind of
116 * auditing facility and yes, we learned that the hard way: disturbingly,
117 * there exist recommendations for "tuning" the system that involve writing to
118 * kernel memory addresses via the kernel debugger, and -- as we discovered --
119 * these can easily be applied incorrectly or unsafely, yielding an entirely
120 * undebuggable "can't happen" kind of panic.
121 */
122 static void
123 mm_logkmem(struct uio *uio)
124 {
125 mm_logentry_t *ent;
126 proc_t *p = curthread->t_procp;
127
128 mutex_enter(&mm_lock);
129
130 ent = &mm_kmemlog[mm_kmemlogent++];
131
132 if (mm_kmemlogent == MM_KMEMLOG_NENTRIES)
133 mm_kmemlogent = 0;
134
135 ent->mle_vaddr = (uintptr_t)uio->uio_loffset;
136 ent->mle_len = uio->uio_resid;
137 gethrestime(&ent->mle_hrestime);
138 ent->mle_hrtime = gethrtime();
139 ent->mle_pid = p->p_pidp->pid_id;
140
141 (void) strncpy(ent->mle_psargs,
142 p->p_user.u_psargs, sizeof (ent->mle_psargs));
143
144 mutex_exit(&mm_lock);
145 }
146
147 /*ARGSUSED1*/
148 static int
149 mm_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
150 {
151 int i;
152 struct mem_minor {
153 char *name;
154 minor_t minor;
155 int privonly;
156 const char *rdpriv;
157 const char *wrpriv;
158 mode_t priv_mode;
159 } mm[] = {
160 { "mem", M_MEM, 0, NULL, "all", 0640 },
161 { "kmem", M_KMEM, 0, NULL, "all", 0640 },
162 { "allkmem", M_ALLKMEM, 0, "all", "all", 0600 },
163 { "null", M_NULL, PRIVONLY_DEV, NULL, NULL, 0666 },
164 { "zero", M_ZERO, PRIVONLY_DEV, NULL, NULL, 0666 },
165 { "full", M_FULL, PRIVONLY_DEV, NULL, NULL, 0666 },
166 };
167 kstat_t *ksp;
168
169 mutex_init(&mm_lock, NULL, MUTEX_DEFAULT, NULL);
170 mm_map = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
171
172 for (i = 0; i < (sizeof (mm) / sizeof (mm[0])); i++) {
173 if (ddi_create_priv_minor_node(devi, mm[i].name, S_IFCHR,
174 mm[i].minor, DDI_PSEUDO, mm[i].privonly,
175 mm[i].rdpriv, mm[i].wrpriv, mm[i].priv_mode) ==
176 DDI_FAILURE) {
177 ddi_remove_minor_node(devi, NULL);
178 return (DDI_FAILURE);
179 }
180 }
181
182 mm_dip = devi;
183
184 ksp = kstat_create("mm", 0, "phys_installed", "misc",
185 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL);
186 if (ksp != NULL) {
187 ksp->ks_update = mm_kstat_update;
188 ksp->ks_snapshot = mm_kstat_snapshot;
189 ksp->ks_lock = &mm_lock; /* XXX - not really needed */
190 kstat_install(ksp);
191 }
192
193 mm_kmem_io_access = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
194 "kmem_io_access", 0);
195
196 return (DDI_SUCCESS);
197 }
198
199 /*ARGSUSED*/
200 static int
201 mm_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
202 {
203 register int error;
204
205 switch (infocmd) {
206 case DDI_INFO_DEVT2DEVINFO:
207 *result = (void *)mm_dip;
208 error = DDI_SUCCESS;
209 break;
210 case DDI_INFO_DEVT2INSTANCE:
211 *result = (void *)0;
212 error = DDI_SUCCESS;
213 break;
214 default:
215 error = DDI_FAILURE;
216 }
217 return (error);
218 }
219
220 /*ARGSUSED1*/
221 static int
222 mmopen(dev_t *devp, int flag, int typ, struct cred *cred)
223 {
224 switch (getminor(*devp)) {
225 case M_NULL:
226 case M_ZERO:
227 case M_FULL:
228 case M_MEM:
229 case M_KMEM:
230 case M_ALLKMEM:
231 /* standard devices */
232 break;
233
234 default:
235 /* Unsupported or unknown type */
236 return (EINVAL);
237 }
238 /* must be character device */
239 if (typ != OTYP_CHR)
240 return (EINVAL);
241 return (0);
242 }
243
244 struct pollhead mm_pollhd;
245
246 /*ARGSUSED*/
247 static int
248 mmchpoll(dev_t dev, short events, int anyyet, short *reventsp,
249 struct pollhead **phpp)
250 {
251 switch (getminor(dev)) {
252 case M_NULL:
253 case M_ZERO:
254 case M_FULL:
255 case M_MEM:
256 case M_KMEM:
257 case M_ALLKMEM:
258 *reventsp = events & (POLLIN | POLLOUT | POLLPRI | POLLRDNORM |
259 POLLWRNORM | POLLRDBAND | POLLWRBAND);
260 /*
261 * A non NULL pollhead pointer should be returned in case
262 * user polls for 0 events or is doing an edge-triggerd poll.
263 */
264 if ((!*reventsp && !anyyet) || (events & POLLET)) {
265 *phpp = &mm_pollhd;
266 }
267 return (0);
268 default:
269 /* no other devices currently support polling */
270 return (ENXIO);
271 }
272 }
273
274 static int
275 mmpropop(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int flags,
276 char *name, caddr_t valuep, int *lengthp)
277 {
278 /*
279 * implement zero size to reduce overhead (avoid two failing
280 * property lookups per stat).
281 */
282 return (ddi_prop_op_size(dev, dip, prop_op,
283 flags, name, valuep, lengthp, 0));
284 }
285
286 static int
287 mmio(struct uio *uio, enum uio_rw rw, pfn_t pfn, off_t pageoff, int allowio,
288 page_t *pp)
289 {
290 int error = 0;
291 int devload = 0;
292 int is_memory = pf_is_memory(pfn);
293 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
294 (size_t)uio->uio_iov->iov_len);
295 caddr_t va = NULL;
296
297 mutex_enter(&mm_lock);
298
299 if (is_memory && kpm_enable) {
300 if (pp)
301 va = hat_kpm_mapin(pp, NULL);
302 else
303 va = hat_kpm_mapin_pfn(pfn);
304 }
305
306 if (va == NULL) {
307 hat_devload(kas.a_hat, mm_map, PAGESIZE, pfn,
308 (uint_t)(rw == UIO_READ ? PROT_READ : PROT_READ|PROT_WRITE),
309 HAT_LOAD_NOCONSIST|HAT_LOAD_LOCK);
310 va = mm_map;
311 devload = 1;
312 }
313
314 if (!is_memory) {
315 if (allowio) {
316 size_t c = uio->uio_iov->iov_len;
317
318 if (ddi_peekpokeio(NULL, uio, rw,
319 (caddr_t)(uintptr_t)uio->uio_loffset, c,
320 sizeof (int32_t)) != DDI_SUCCESS)
321 error = EFAULT;
322 } else
323 error = EIO;
324 } else
325 error = uiomove(va + pageoff, nbytes, rw, uio);
326
327 if (devload)
328 hat_unload(kas.a_hat, mm_map, PAGESIZE, HAT_UNLOAD_UNLOCK);
329 else if (pp)
330 hat_kpm_mapout(pp, NULL, va);
331 else
332 hat_kpm_mapout_pfn(pfn);
333
334 mutex_exit(&mm_lock);
335 return (error);
336 }
337
338 static int
339 mmpagelock(struct as *as, caddr_t va)
340 {
341 struct seg *seg;
342 int i;
343
344 AS_LOCK_ENTER(as, RW_READER);
345 seg = as_segat(as, va);
346 i = (seg != NULL)? SEGOP_CAPABLE(seg, S_CAPABILITY_NOMINFLT) : 0;
347 AS_LOCK_EXIT(as);
348
349 return (i);
350 }
351
352 #ifdef __sparc
353
354 #define NEED_LOCK_KVADDR(kva) mmpagelock(&kas, kva)
355
356 #else /* __i386, __amd64 */
357
358 #define NEED_LOCK_KVADDR(va) 0
359
360 #endif /* __sparc */
361
362 /*ARGSUSED3*/
363 static int
364 mmrw(dev_t dev, struct uio *uio, enum uio_rw rw, cred_t *cred)
365 {
366 pfn_t v;
367 struct iovec *iov;
368 int error = 0;
369 size_t c;
370 ssize_t oresid = uio->uio_resid;
371 minor_t minor = getminor(dev);
372
373 while (uio->uio_resid > 0 && error == 0) {
374 iov = uio->uio_iov;
375 if (iov->iov_len == 0) {
376 uio->uio_iov++;
377 uio->uio_iovcnt--;
378 if (uio->uio_iovcnt < 0)
379 panic("mmrw");
380 continue;
381 }
382 switch (minor) {
383
384 case M_MEM:
385 memlist_read_lock();
386 if (!address_in_memlist(phys_install,
387 (uint64_t)uio->uio_loffset, 1)) {
388 memlist_read_unlock();
389 error = EFAULT;
390 break;
391 }
392 memlist_read_unlock();
393
394 v = BTOP((u_offset_t)uio->uio_loffset);
395 error = mmio(uio, rw, v,
396 uio->uio_loffset & PAGEOFFSET, 0, NULL);
397 break;
398
399 case M_KMEM:
400 case M_ALLKMEM:
401 {
402 page_t **ppp = NULL;
403 caddr_t vaddr = (caddr_t)uio->uio_offset;
404 int try_lock = NEED_LOCK_KVADDR(vaddr);
405 int locked = 0;
406
407 if ((error = plat_mem_do_mmio(uio, rw)) != ENOTSUP)
408 break;
409
410 if (rw == UIO_WRITE)
411 mm_logkmem(uio);
412
413 /*
414 * If vaddr does not map a valid page, as_pagelock()
415 * will return failure. Hence we can't check the
416 * return value and return EFAULT here as we'd like.
417 * seg_kp and seg_kpm do not properly support
418 * as_pagelock() for this context so we avoid it
419 * using the try_lock set check above. Some day when
420 * the kernel page locking gets redesigned all this
421 * muck can be cleaned up.
422 */
423 if (try_lock)
424 locked = (as_pagelock(&kas, &ppp, vaddr,
425 PAGESIZE, S_WRITE) == 0);
426
427 v = hat_getpfnum(kas.a_hat,
428 (caddr_t)(uintptr_t)uio->uio_loffset);
429 if (v == PFN_INVALID) {
430 if (locked)
431 as_pageunlock(&kas, ppp, vaddr,
432 PAGESIZE, S_WRITE);
433 error = EFAULT;
434 break;
435 }
436
437 error = mmio(uio, rw, v, uio->uio_loffset & PAGEOFFSET,
438 minor == M_ALLKMEM || mm_kmem_io_access,
439 (locked && ppp) ? *ppp : NULL);
440 if (locked)
441 as_pageunlock(&kas, ppp, vaddr, PAGESIZE,
442 S_WRITE);
443 }
444
445 break;
446
447 case M_FULL:
448 if (rw == UIO_WRITE) {
449 error = ENOSPC;
450 break;
451 }
452 /* else it's a read, fall through to zero case */
453 /*FALLTHROUGH*/
454
455 case M_ZERO:
456 if (rw == UIO_READ) {
457 label_t ljb;
458
459 if (on_fault(&ljb)) {
460 no_fault();
461 error = EFAULT;
462 break;
463 }
464 uzero(iov->iov_base, iov->iov_len);
465 no_fault();
466 uio->uio_resid -= iov->iov_len;
467 uio->uio_loffset += iov->iov_len;
468 break;
469 }
470 /* else it's a write, fall through to NULL case */
471 /*FALLTHROUGH*/
472
473 case M_NULL:
474 if (rw == UIO_READ)
475 return (0);
476 c = iov->iov_len;
477 iov->iov_base += c;
478 iov->iov_len -= c;
479 uio->uio_loffset += c;
480 uio->uio_resid -= c;
481 break;
482
483 }
484 }
485 return (uio->uio_resid == oresid ? error : 0);
486 }
487
488 static int
489 mmread(dev_t dev, struct uio *uio, cred_t *cred)
490 {
491 return (mmrw(dev, uio, UIO_READ, cred));
492 }
493
494 static int
495 mmwrite(dev_t dev, struct uio *uio, cred_t *cred)
496 {
497 return (mmrw(dev, uio, UIO_WRITE, cred));
498 }
499
500 /*
501 * Private ioctl for libkvm to support kvm_physaddr().
502 * Given an address space and a VA, compute the PA.
503 */
504 static int
505 mmioctl_vtop(intptr_t data)
506 {
507 #ifdef _SYSCALL32
508 mem_vtop32_t vtop32;
509 #endif
510 mem_vtop_t mem_vtop;
511 proc_t *p;
512 pfn_t pfn = (pfn_t)PFN_INVALID;
513 pid_t pid = 0;
514 struct as *as;
515 struct seg *seg;
516
517 if (get_udatamodel() == DATAMODEL_NATIVE) {
518 if (copyin((void *)data, &mem_vtop, sizeof (mem_vtop_t)))
519 return (EFAULT);
520 }
521 #ifdef _SYSCALL32
522 else {
523 if (copyin((void *)data, &vtop32, sizeof (mem_vtop32_t)))
524 return (EFAULT);
525 mem_vtop.m_as = (struct as *)(uintptr_t)vtop32.m_as;
526 mem_vtop.m_va = (void *)(uintptr_t)vtop32.m_va;
527
528 if (mem_vtop.m_as != NULL)
529 return (EINVAL);
530 }
531 #endif
532
533 if (mem_vtop.m_as == &kas) {
534 pfn = hat_getpfnum(kas.a_hat, mem_vtop.m_va);
535 } else {
536 if (mem_vtop.m_as == NULL) {
537 /*
538 * Assume the calling process's address space if the
539 * caller didn't specify one.
540 */
541 p = curthread->t_procp;
542 if (p == NULL)
543 return (EIO);
544 mem_vtop.m_as = p->p_as;
545 }
546
547 mutex_enter(&pidlock);
548 for (p = practive; p != NULL; p = p->p_next) {
549 if (p->p_as == mem_vtop.m_as) {
550 pid = p->p_pid;
551 break;
552 }
553 }
554 mutex_exit(&pidlock);
555 if (p == NULL)
556 return (EIO);
557 p = sprlock(pid);
558 if (p == NULL)
559 return (EIO);
560 as = p->p_as;
561 if (as == mem_vtop.m_as) {
562 mutex_exit(&p->p_lock);
563 AS_LOCK_ENTER(as, RW_READER);
564 for (seg = AS_SEGFIRST(as); seg != NULL;
565 seg = AS_SEGNEXT(as, seg))
566 if ((uintptr_t)mem_vtop.m_va -
567 (uintptr_t)seg->s_base < seg->s_size)
568 break;
569 if (seg != NULL)
570 pfn = hat_getpfnum(as->a_hat, mem_vtop.m_va);
571 AS_LOCK_EXIT(as);
572 mutex_enter(&p->p_lock);
573 }
574 sprunlock(p);
575 }
576 mem_vtop.m_pfn = pfn;
577 if (pfn == PFN_INVALID)
578 return (EIO);
579
580 if (get_udatamodel() == DATAMODEL_NATIVE) {
581 if (copyout(&mem_vtop, (void *)data, sizeof (mem_vtop_t)))
582 return (EFAULT);
583 }
584 #ifdef _SYSCALL32
585 else {
586 vtop32.m_pfn = mem_vtop.m_pfn;
587 if (copyout(&vtop32, (void *)data, sizeof (mem_vtop32_t)))
588 return (EFAULT);
589 }
590 #endif
591
592 return (0);
593 }
594
595 /*
596 * Given a PA, execute the given page retire command on it.
597 */
598 static int
599 mmioctl_page_retire(int cmd, intptr_t data)
600 {
601 extern int page_retire_test(void);
602 uint64_t pa;
603
604 if (copyin((void *)data, &pa, sizeof (uint64_t))) {
605 return (EFAULT);
606 }
607
608 switch (cmd) {
609 case MEM_PAGE_ISRETIRED:
610 return (page_retire_check(pa, NULL));
611
612 case MEM_PAGE_UNRETIRE:
613 return (page_unretire(pa));
614
615 case MEM_PAGE_RETIRE:
616 return (page_retire(pa, PR_FMA));
617
618 case MEM_PAGE_RETIRE_MCE:
619 return (page_retire(pa, PR_MCE));
620
621 case MEM_PAGE_RETIRE_UE:
622 return (page_retire(pa, PR_UE));
623
624 case MEM_PAGE_GETERRORS:
625 {
626 uint64_t page_errors;
627 int rc = page_retire_check(pa, &page_errors);
628 if (copyout(&page_errors, (void *)data,
629 sizeof (uint64_t))) {
630 return (EFAULT);
631 }
632 return (rc);
633 }
634
635 case MEM_PAGE_RETIRE_TEST:
636 return (page_retire_test());
637
638 }
639
640 return (EINVAL);
641 }
642
643 #ifdef __sparc
644 /*
645 * Given a syndrome, syndrome type, and address return the
646 * associated memory name in the provided data buffer.
647 */
648 static int
649 mmioctl_get_mem_name(intptr_t data)
650 {
651 mem_name_t mem_name;
652 void *buf;
653 size_t bufsize;
654 int len, err;
655
656 if ((bufsize = cpu_get_name_bufsize()) == 0)
657 return (ENOTSUP);
658
659 if ((err = mm_read_mem_name(data, &mem_name)) < 0)
660 return (err);
661
662 buf = kmem_alloc(bufsize, KM_SLEEP);
663
664 /*
665 * Call into cpu specific code to do the lookup.
666 */
667 if ((err = cpu_get_mem_name(mem_name.m_synd, mem_name.m_type,
668 mem_name.m_addr, buf, bufsize, &len)) != 0) {
669 kmem_free(buf, bufsize);
670 return (err);
671 }
672
673 if (len >= mem_name.m_namelen) {
674 kmem_free(buf, bufsize);
675 return (ENOSPC);
676 }
677
678 if (copyoutstr(buf, (char *)mem_name.m_name,
679 mem_name.m_namelen, NULL) != 0) {
680 kmem_free(buf, bufsize);
681 return (EFAULT);
682 }
683
684 kmem_free(buf, bufsize);
685 return (0);
686 }
687
688 /*
689 * Given a syndrome and address return information about the associated memory.
690 */
691 static int
692 mmioctl_get_mem_info(intptr_t data)
693 {
694 mem_info_t mem_info;
695 int err;
696
697 if (copyin((void *)data, &mem_info, sizeof (mem_info_t)))
698 return (EFAULT);
699
700 if ((err = cpu_get_mem_info(mem_info.m_synd, mem_info.m_addr,
701 &mem_info.m_mem_size, &mem_info.m_seg_size, &mem_info.m_bank_size,
702 &mem_info.m_segments, &mem_info.m_banks, &mem_info.m_mcid)) != 0)
703 return (err);
704
705 if (copyout(&mem_info, (void *)data, sizeof (mem_info_t)) != 0)
706 return (EFAULT);
707
708 return (0);
709 }
710
711 /*
712 * Given a memory name, return its associated serial id
713 */
714 static int
715 mmioctl_get_mem_sid(intptr_t data)
716 {
717 mem_name_t mem_name;
718 void *buf;
719 void *name;
720 size_t name_len;
721 size_t bufsize;
722 int len, err;
723
724 if ((bufsize = cpu_get_name_bufsize()) == 0)
725 return (ENOTSUP);
726
727 if ((err = mm_read_mem_name(data, &mem_name)) < 0)
728 return (err);
729
730 buf = kmem_alloc(bufsize, KM_SLEEP);
731
732 if (mem_name.m_namelen > 1024)
733 mem_name.m_namelen = 1024; /* cap at 1024 bytes */
734
735 name = kmem_alloc(mem_name.m_namelen, KM_SLEEP);
736
737 if ((err = copyinstr((char *)mem_name.m_name, (char *)name,
738 mem_name.m_namelen, &name_len)) != 0) {
739 kmem_free(buf, bufsize);
740 kmem_free(name, mem_name.m_namelen);
741 return (err);
742 }
743
744 /*
745 * Call into cpu specific code to do the lookup.
746 */
747 if ((err = cpu_get_mem_sid(name, buf, bufsize, &len)) != 0) {
748 kmem_free(buf, bufsize);
749 kmem_free(name, mem_name.m_namelen);
750 return (err);
751 }
752
753 if (len > mem_name.m_sidlen) {
754 kmem_free(buf, bufsize);
755 kmem_free(name, mem_name.m_namelen);
756 return (ENAMETOOLONG);
757 }
758
759 if (copyoutstr(buf, (char *)mem_name.m_sid,
760 mem_name.m_sidlen, NULL) != 0) {
761 kmem_free(buf, bufsize);
762 kmem_free(name, mem_name.m_namelen);
763 return (EFAULT);
764 }
765
766 kmem_free(buf, bufsize);
767 kmem_free(name, mem_name.m_namelen);
768 return (0);
769 }
770 #endif /* __sparc */
771
772 /*
773 * Private ioctls for
774 * libkvm to support kvm_physaddr().
775 * FMA support for page_retire() and memory attribute information.
776 */
777 /*ARGSUSED*/
778 static int
779 mmioctl(dev_t dev, int cmd, intptr_t data, int flag, cred_t *cred, int *rvalp)
780 {
781 if ((cmd == MEM_VTOP && getminor(dev) != M_KMEM) ||
782 (cmd != MEM_VTOP && getminor(dev) != M_MEM))
783 return (ENXIO);
784
785 switch (cmd) {
786 case MEM_VTOP:
787 return (mmioctl_vtop(data));
788
789 case MEM_PAGE_RETIRE:
790 case MEM_PAGE_ISRETIRED:
791 case MEM_PAGE_UNRETIRE:
792 case MEM_PAGE_RETIRE_MCE:
793 case MEM_PAGE_RETIRE_UE:
794 case MEM_PAGE_GETERRORS:
795 case MEM_PAGE_RETIRE_TEST:
796 return (mmioctl_page_retire(cmd, data));
797
798 #ifdef __sparc
799 case MEM_NAME:
800 return (mmioctl_get_mem_name(data));
801
802 case MEM_INFO:
803 return (mmioctl_get_mem_info(data));
804
805 case MEM_SID:
806 return (mmioctl_get_mem_sid(data));
807 #else
808 case MEM_NAME:
809 case MEM_INFO:
810 case MEM_SID:
811 return (ENOTSUP);
812 #endif /* __sparc */
813 }
814 return (ENXIO);
815 }
816
817 /*ARGSUSED2*/
818 static int
819 mmmmap(dev_t dev, off_t off, int prot)
820 {
821 pfn_t pf;
822 struct memlist *pmem;
823 minor_t minor = getminor(dev);
824
825 switch (minor) {
826 case M_MEM:
827 pf = btop(off);
828 memlist_read_lock();
829 for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
830 if (pf >= BTOP(pmem->ml_address) &&
831 pf < BTOP(pmem->ml_address + pmem->ml_size)) {
832 memlist_read_unlock();
833 return (impl_obmem_pfnum(pf));
834 }
835 }
836 memlist_read_unlock();
837 break;
838
839 case M_KMEM:
840 case M_ALLKMEM:
841 /* no longer supported with KPR */
842 return (-1);
843
844 case M_FULL:
845 case M_ZERO:
846 /*
847 * We shouldn't be mmap'ing to /dev/zero here as
848 * mmsegmap() should have already converted
849 * a mapping request for this device to a mapping
850 * using seg_vn for anonymous memory.
851 */
852 break;
853
854 }
855 return (-1);
856 }
857
858 /*
859 * This function is called when a memory device is mmap'ed.
860 * Set up the mapping to the correct device driver.
861 */
862 static int
863 mmsegmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len,
864 uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
865 {
866 struct segvn_crargs vn_a;
867 struct segdev_crargs dev_a;
868 int error;
869 minor_t minor;
870 off_t i;
871
872 minor = getminor(dev);
873
874 as_rangelock(as);
875 /*
876 * No need to worry about vac alignment on /dev/zero
877 * since this is a "clone" object that doesn't yet exist.
878 */
879 error = choose_addr(as, addrp, len, off,
880 (minor == M_MEM) || (minor == M_KMEM), flags);
881 if (error != 0) {
882 as_rangeunlock(as);
883 return (error);
884 }
885
886 switch (minor) {
887 case M_MEM:
888 /* /dev/mem cannot be mmap'ed with MAP_PRIVATE */
889 if ((flags & MAP_TYPE) != MAP_SHARED) {
890 as_rangeunlock(as);
891 return (EINVAL);
892 }
893
894 /*
895 * Check to ensure that the entire range is
896 * legal and we are not trying to map in
897 * more than the device will let us.
898 */
899 for (i = 0; i < len; i += PAGESIZE) {
900 if (mmmmap(dev, off + i, maxprot) == -1) {
901 as_rangeunlock(as);
902 return (ENXIO);
903 }
904 }
905
906 /*
907 * Use seg_dev segment driver for /dev/mem mapping.
908 */
909 dev_a.mapfunc = mmmmap;
910 dev_a.dev = dev;
911 dev_a.offset = off;
912 dev_a.type = (flags & MAP_TYPE);
913 dev_a.prot = (uchar_t)prot;
914 dev_a.maxprot = (uchar_t)maxprot;
915 dev_a.hat_attr = 0;
916
917 /*
918 * Make /dev/mem mappings non-consistent since we can't
919 * alias pages that don't have page structs behind them,
920 * such as kernel stack pages. If someone mmap()s a kernel
921 * stack page and if we give them a tte with cv, a line from
922 * that page can get into both pages of the spitfire d$.
923 * But snoop from another processor will only invalidate
924 * the first page. This later caused kernel (xc_attention)
925 * to go into an infinite loop at pil 13 and no interrupts
926 * could come in. See 1203630.
927 *
928 */
929 dev_a.hat_flags = HAT_LOAD_NOCONSIST;
930 dev_a.devmap_data = NULL;
931
932 error = as_map(as, *addrp, len, segdev_create, &dev_a);
933 break;
934
935 case M_ZERO:
936 /*
937 * Use seg_vn segment driver for /dev/zero mapping.
938 * Passing in a NULL amp gives us the "cloning" effect.
939 */
940 vn_a.vp = NULL;
941 vn_a.offset = 0;
942 vn_a.type = (flags & MAP_TYPE);
943 vn_a.prot = prot;
944 vn_a.maxprot = maxprot;
945 vn_a.flags = flags & ~MAP_TYPE;
946 vn_a.cred = cred;
947 vn_a.amp = NULL;
948 vn_a.szc = 0;
949 vn_a.lgrp_mem_policy_flags = 0;
950 error = as_map(as, *addrp, len, segvn_create, &vn_a);
951 break;
952
953 case M_KMEM:
954 case M_ALLKMEM:
955 /* No longer supported with KPR. */
956 error = ENXIO;
957 break;
958
959 case M_NULL:
960 /*
961 * Use seg_dev segment driver for /dev/null mapping.
962 */
963 dev_a.mapfunc = mmmmap;
964 dev_a.dev = dev;
965 dev_a.offset = off;
966 dev_a.type = 0; /* neither PRIVATE nor SHARED */
967 dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE;
968 dev_a.hat_attr = 0;
969 dev_a.hat_flags = 0;
970 error = as_map(as, *addrp, len, segdev_create, &dev_a);
971 break;
972
973 default:
974 error = ENXIO;
975 }
976
977 as_rangeunlock(as);
978 return (error);
979 }
980
981 static struct cb_ops mm_cb_ops = {
982 mmopen, /* open */
983 nulldev, /* close */
984 nodev, /* strategy */
985 nodev, /* print */
986 nodev, /* dump */
987 mmread, /* read */
988 mmwrite, /* write */
989 mmioctl, /* ioctl */
990 nodev, /* devmap */
991 mmmmap, /* mmap */
992 mmsegmap, /* segmap */
993 mmchpoll, /* poll */
994 mmpropop, /* prop_op */
995 0, /* streamtab */
996 D_NEW | D_MP | D_64BIT | D_U64BIT
997 };
998
999 static struct dev_ops mm_ops = {
1000 DEVO_REV, /* devo_rev, */
1001 0, /* refcnt */
1002 mm_info, /* get_dev_info */
1003 nulldev, /* identify */
1004 nulldev, /* probe */
1005 mm_attach, /* attach */
1006 nodev, /* detach */
1007 nodev, /* reset */
1008 &mm_cb_ops, /* driver operations */
1009 (struct bus_ops *)0, /* bus operations */
1010 NULL, /* power */
1011 ddi_quiesce_not_needed, /* quiesce */
1012 };
1013
1014 static struct modldrv modldrv = {
1015 &mod_driverops, "memory driver", &mm_ops,
1016 };
1017
1018 static struct modlinkage modlinkage = {
1019 MODREV_1, &modldrv, NULL
1020 };
1021
1022 int
1023 _init(void)
1024 {
1025 return (mod_install(&modlinkage));
1026 }
1027
1028 int
1029 _info(struct modinfo *modinfop)
1030 {
1031 return (mod_info(&modlinkage, modinfop));
1032 }
1033
1034 int
1035 _fini(void)
1036 {
1037 return (mod_remove(&modlinkage));
1038 }
1039
1040 static int
1041 mm_kstat_update(kstat_t *ksp, int rw)
1042 {
1043 struct memlist *pmem;
1044 uint_t count;
1045
1046 if (rw == KSTAT_WRITE)
1047 return (EACCES);
1048
1049 count = 0;
1050 memlist_read_lock();
1051 for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
1052 count++;
1053 }
1054 memlist_read_unlock();
1055
1056 ksp->ks_ndata = count;
1057 ksp->ks_data_size = count * 2 * sizeof (uint64_t);
1058
1059 return (0);
1060 }
1061
1062 static int
1063 mm_kstat_snapshot(kstat_t *ksp, void *buf, int rw)
1064 {
1065 struct memlist *pmem;
1066 struct memunit {
1067 uint64_t address;
1068 uint64_t size;
1069 } *kspmem;
1070
1071 if (rw == KSTAT_WRITE)
1072 return (EACCES);
1073
1074 ksp->ks_snaptime = gethrtime();
1075
1076 kspmem = (struct memunit *)buf;
1077 memlist_read_lock();
1078 for (pmem = phys_install; pmem != NULL;
1079 pmem = pmem->ml_next, kspmem++) {
1080 if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size)
1081 break;
1082 kspmem->address = pmem->ml_address;
1083 kspmem->size = pmem->ml_size;
1084 }
1085 memlist_read_unlock();
1086
1087 return (0);
1088 }
1089
1090 /*
1091 * Read a mem_name_t from user-space and store it in the mem_name_t
1092 * pointed to by the mem_name argument.
1093 */
1094 static int
1095 mm_read_mem_name(intptr_t data, mem_name_t *mem_name)
1096 {
1097 if (get_udatamodel() == DATAMODEL_NATIVE) {
1098 if (copyin((void *)data, mem_name, sizeof (mem_name_t)))
1099 return (EFAULT);
1100 }
1101 #ifdef _SYSCALL32
1102 else {
1103 mem_name32_t mem_name32;
1104
1105 if (copyin((void *)data, &mem_name32, sizeof (mem_name32_t)))
1106 return (EFAULT);
1107 mem_name->m_addr = mem_name32.m_addr;
1108 mem_name->m_synd = mem_name32.m_synd;
1109 mem_name->m_type[0] = mem_name32.m_type[0];
1110 mem_name->m_type[1] = mem_name32.m_type[1];
1111 mem_name->m_name = (caddr_t)(uintptr_t)mem_name32.m_name;
1112 mem_name->m_namelen = (size_t)mem_name32.m_namelen;
1113 mem_name->m_sid = (caddr_t)(uintptr_t)mem_name32.m_sid;
1114 mem_name->m_sidlen = (size_t)mem_name32.m_sidlen;
1115 }
1116 #endif /* _SYSCALL32 */
1117
1118 return (0);
1119 }