1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2016 Joyent, Inc.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 /*
31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 * The Regents of the University of California
33 * All Rights Reserved
34 *
35 * University Acknowledgment- Portions of this document are derived from
36 * software developed by the University of California, Berkeley, and its
37 * contributors.
38 */
39
40 /*
41 * Inter-Process Communication Shared Memory Facility.
42 *
43 * See os/ipc.c for a description of common IPC functionality.
44 *
45 * Resource controls
46 * -----------------
47 *
48 * Control: zone.max-shm-ids (rc_zone_shmmni)
49 * Description: Maximum number of shared memory ids allowed a zone.
50 *
51 * When shmget() is used to allocate a shared memory segment, one id
52 * is allocated. If the id allocation doesn't succeed, shmget()
53 * fails and errno is set to ENOSPC. Upon successful shmctl(,
54 * IPC_RMID) the id is deallocated.
55 *
56 * Control: project.max-shm-ids (rc_project_shmmni)
57 * Description: Maximum number of shared memory ids allowed a project.
58 *
59 * When shmget() is used to allocate a shared memory segment, one id
60 * is allocated. If the id allocation doesn't succeed, shmget()
61 * fails and errno is set to ENOSPC. Upon successful shmctl(,
62 * IPC_RMID) the id is deallocated.
63 *
64 * Control: zone.max-shm-memory (rc_zone_shmmax)
65 * Description: Total amount of shared memory allowed a zone.
66 *
67 * When shmget() is used to allocate a shared memory segment, the
68 * segment's size is allocated against this limit. If the space
69 * allocation doesn't succeed, shmget() fails and errno is set to
70 * EINVAL. The size will be deallocated once the last process has
71 * detached the segment and the segment has been successfully
72 * shmctl(, IPC_RMID)ed.
73 *
74 * Control: project.max-shm-memory (rc_project_shmmax)
75 * Description: Total amount of shared memory allowed a project.
76 *
77 * When shmget() is used to allocate a shared memory segment, the
78 * segment's size is allocated against this limit. If the space
79 * allocation doesn't succeed, shmget() fails and errno is set to
80 * EINVAL. The size will be deallocated once the last process has
81 * detached the segment and the segment has been successfully
82 * shmctl(, IPC_RMID)ed.
83 */
84
85 #include <sys/types.h>
86 #include <sys/param.h>
87 #include <sys/cred.h>
88 #include <sys/errno.h>
89 #include <sys/time.h>
90 #include <sys/kmem.h>
91 #include <sys/user.h>
92 #include <sys/proc.h>
93 #include <sys/systm.h>
94 #include <sys/prsystm.h>
95 #include <sys/sysmacros.h>
96 #include <sys/tuneable.h>
97 #include <sys/vm.h>
98 #include <sys/mman.h>
99 #include <sys/swap.h>
100 #include <sys/cmn_err.h>
101 #include <sys/debug.h>
102 #include <sys/lwpchan_impl.h>
103 #include <sys/avl.h>
104 #include <sys/modctl.h>
105 #include <sys/syscall.h>
106 #include <sys/task.h>
107 #include <sys/project.h>
108 #include <sys/policy.h>
109 #include <sys/zone.h>
110 #include <sys/rctl.h>
111
112 #include <sys/ipc.h>
113 #include <sys/ipc_impl.h>
114 #include <sys/shm.h>
115 #include <sys/shm_impl.h>
116
117 #include <vm/hat.h>
118 #include <vm/seg.h>
119 #include <vm/as.h>
120 #include <vm/seg_vn.h>
121 #include <vm/anon.h>
122 #include <vm/page.h>
123 #include <vm/vpage.h>
124 #include <vm/seg_spt.h>
125
126 #include <c2/audit.h>
127
128 static int shmem_lock(kshmid_t *sp, struct anon_map *amp);
129 static void shmem_unlock(kshmid_t *sp, struct anon_map *amp);
130 static void sa_add(struct proc *pp, caddr_t addr, size_t len, ulong_t flags,
131 kshmid_t *id);
132 static void shm_rm_amp(kshmid_t *sp);
133 static void shm_dtor(kipc_perm_t *);
134 static void shm_rmid(kipc_perm_t *);
135 static void shm_remove_zone(zoneid_t, void *);
136
137 /*
138 * Semantics for share_page_table and ism_off:
139 *
140 * These are hooks in /etc/system - only for internal testing purpose.
141 *
142 * Setting share_page_table automatically turns on the SHM_SHARE_MMU (ISM) flag
143 * in a call to shmat(2). In other words, with share_page_table set, you always
144 * get ISM, even if say, DISM is specified. It should really be called "ism_on".
145 *
146 * Setting ism_off turns off the SHM_SHARE_MMU flag from the flags passed to
147 * shmat(2).
148 *
149 * If both share_page_table and ism_off are set, share_page_table prevails.
150 *
151 * Although these tunables should probably be removed, they do have some
152 * external exposure; as long as they exist, they should at least work sensibly.
153 */
154
155 int share_page_table;
156 int ism_off;
157
158 /*
159 * The following tunables are obsolete. Though for compatibility we
160 * still read and interpret shminfo_shmmax and shminfo_shmmni (see
161 * os/project.c), the preferred mechanism for administrating the IPC
162 * Shared Memory facility is through the resource controls described at
163 * the top of this file.
164 */
165 size_t shminfo_shmmax = 0x800000; /* (obsolete) */
166 int shminfo_shmmni = 100; /* (obsolete) */
167 size_t shminfo_shmmin = 1; /* (obsolete) */
168 int shminfo_shmseg = 6; /* (obsolete) */
169
170 extern rctl_hndl_t rc_zone_shmmax;
171 extern rctl_hndl_t rc_zone_shmmni;
172 extern rctl_hndl_t rc_project_shmmax;
173 extern rctl_hndl_t rc_project_shmmni;
174 static ipc_service_t *shm_svc;
175 static zone_key_t shm_zone_key;
176
177 /*
178 * Module linkage information for the kernel.
179 */
180 static uintptr_t shmsys(int, uintptr_t, uintptr_t, uintptr_t);
181
182 static struct sysent ipcshm_sysent = {
183 4,
184 #ifdef _SYSCALL32_IMPL
185 SE_ARGC | SE_NOUNLOAD | SE_64RVAL,
186 #else /* _SYSCALL32_IMPL */
187 SE_ARGC | SE_NOUNLOAD | SE_32RVAL1,
188 #endif /* _SYSCALL32_IMPL */
189 (int (*)())shmsys
190 };
191
192 #ifdef _SYSCALL32_IMPL
193 static struct sysent ipcshm_sysent32 = {
194 4,
195 SE_ARGC | SE_NOUNLOAD | SE_32RVAL1,
196 (int (*)())shmsys
197 };
198 #endif /* _SYSCALL32_IMPL */
199
200 static struct modlsys modlsys = {
201 &mod_syscallops, "System V shared memory", &ipcshm_sysent
202 };
203
204 #ifdef _SYSCALL32_IMPL
205 static struct modlsys modlsys32 = {
206 &mod_syscallops32, "32-bit System V shared memory", &ipcshm_sysent32
207 };
208 #endif /* _SYSCALL32_IMPL */
209
210 static struct modlinkage modlinkage = {
211 MODREV_1,
212 &modlsys,
213 #ifdef _SYSCALL32_IMPL
214 &modlsys32,
215 #endif
216 NULL
217 };
218
219
220 int
221 _init(void)
222 {
223 int result;
224
225 shm_svc = ipcs_create("shmids", rc_project_shmmni, rc_zone_shmmni,
226 sizeof (kshmid_t), shm_dtor, shm_rmid, AT_IPC_SHM,
227 offsetof(ipc_rqty_t, ipcq_shmmni));
228 zone_key_create(&shm_zone_key, NULL, shm_remove_zone, NULL);
229
230 if ((result = mod_install(&modlinkage)) == 0)
231 return (0);
232
233 (void) zone_key_delete(shm_zone_key);
234 ipcs_destroy(shm_svc);
235
236 return (result);
237 }
238
239 int
240 _fini(void)
241 {
242 return (EBUSY);
243 }
244
245 int
246 _info(struct modinfo *modinfop)
247 {
248 return (mod_info(&modlinkage, modinfop));
249 }
250
251 /*
252 * Shmat (attach shared segment) system call.
253 */
254 static int
255 shmat(int shmid, caddr_t uaddr, int uflags, uintptr_t *rvp)
256 {
257 kshmid_t *sp; /* shared memory header ptr */
258 size_t size;
259 int error = 0;
260 proc_t *pp = curproc;
261 struct as *as = pp->p_as;
262 struct segvn_crargs crargs; /* segvn create arguments */
263 kmutex_t *lock;
264 struct seg *segspt = NULL;
265 caddr_t addr = uaddr;
266 int flags = (uflags & SHMAT_VALID_FLAGS_MASK);
267 int useISM;
268 uchar_t prot = PROT_ALL;
269 int result;
270
271 if ((lock = ipc_lookup(shm_svc, shmid, (kipc_perm_t **)&sp)) == NULL)
272 return (EINVAL);
273 if (error = ipcperm_access(&sp->shm_perm, SHM_R, CRED()))
274 goto errret;
275 if ((flags & SHM_RDONLY) == 0 &&
276 (error = ipcperm_access(&sp->shm_perm, SHM_W, CRED())))
277 goto errret;
278 if (spt_invalid(flags)) {
279 error = EINVAL;
280 goto errret;
281 }
282 if (ism_off)
283 flags = flags & ~SHM_SHARE_MMU;
284 if (share_page_table) {
285 flags = flags & ~SHM_PAGEABLE;
286 flags = flags | SHM_SHARE_MMU;
287 }
288 useISM = (spt_locked(flags) || spt_pageable(flags));
289 if (useISM && (error = ipcperm_access(&sp->shm_perm, SHM_W, CRED())))
290 goto errret;
291 if (useISM && isspt(sp)) {
292 uint_t newsptflags = flags | spt_flags(sp->shm_sptseg);
293 /*
294 * If trying to change an existing {D}ISM segment from ISM
295 * to DISM or vice versa, return error. Note that this
296 * validation of flags needs to be done after the effect of
297 * tunables such as ism_off and share_page_table, for
298 * semantics that are consistent with the tunables' settings.
299 */
300 if (spt_invalid(newsptflags)) {
301 error = EINVAL;
302 goto errret;
303 }
304 }
305 ANON_LOCK_ENTER(&sp->shm_amp->a_rwlock, RW_WRITER);
306 size = sp->shm_amp->size;
307 ANON_LOCK_EXIT(&sp->shm_amp->a_rwlock);
308
309 /* somewhere to record spt info for final detach */
310 if (sp->shm_sptinfo == NULL)
311 sp->shm_sptinfo = kmem_zalloc(sizeof (sptinfo_t), KM_SLEEP);
312
313 as_rangelock(as);
314
315 if (useISM) {
316 /*
317 * Handle ISM
318 */
319 uint_t share_szc;
320 size_t share_size;
321 struct shm_data ssd;
322 uintptr_t align_hint;
323 long curprot;
324
325 /*
326 * Pick a share pagesize to use, if (!isspt(sp)).
327 * Otherwise use the already chosen page size.
328 *
329 * For the initial shmat (!isspt(sp)), where sptcreate is
330 * called, map_pgsz is called to recommend a [D]ISM pagesize,
331 * important for systems which offer more than one potential
332 * [D]ISM pagesize.
333 * If the shmat is just to attach to an already created
334 * [D]ISM segment, then use the previously selected page size.
335 */
336 if (!isspt(sp)) {
337 share_size = map_pgsz(MAPPGSZ_ISM, pp, addr, size, 0);
338 if (share_size == 0) {
339 as_rangeunlock(as);
340 error = EINVAL;
341 goto errret;
342 }
343 share_szc = page_szc(share_size);
344 } else {
345 share_szc = sp->shm_sptseg->s_szc;
346 share_size = page_get_pagesize(share_szc);
347 }
348 size = P2ROUNDUP(size, share_size);
349
350 align_hint = share_size;
351 #if defined(__i386) || defined(__amd64)
352 /*
353 * For x86, we want to share as much of the page table tree
354 * as possible. We use a large align_hint at first, but
355 * if that fails, then the code below retries with align_hint
356 * set to share_size.
357 *
358 * The explicit extern here is due to the difficulties
359 * of getting to platform dependent includes. When/if the
360 * platform dependent bits of this function are cleaned up,
361 * another way of doing this should found.
362 */
363 {
364 extern uint_t ptes_per_table;
365
366 while (size >= ptes_per_table * (uint64_t)align_hint)
367 align_hint *= ptes_per_table;
368 }
369 #endif /* __i386 || __amd64 */
370
371 #if defined(__sparcv9)
372 if (addr == 0 &&
373 pp->p_model == DATAMODEL_LP64 && AS_TYPE_64BIT(as)) {
374 /*
375 * If no address has been passed in, and this is a
376 * 64-bit process, we'll try to find an address
377 * in the predict-ISM zone.
378 */
379 caddr_t predbase = (caddr_t)PREDISM_1T_BASE;
380 size_t len = PREDISM_BOUND - PREDISM_1T_BASE;
381
382 as_purge(as);
383 if (as_gap(as, size + share_size, &predbase, &len,
384 AH_LO, (caddr_t)NULL) != -1) {
385 /*
386 * We found an address which looks like a
387 * candidate. We want to round it up, and
388 * then check that it's a valid user range.
389 * This assures that we won't fail below.
390 */
391 addr = (caddr_t)P2ROUNDUP((uintptr_t)predbase,
392 share_size);
393
394 if (valid_usr_range(addr, size, prot,
395 as, as->a_userlimit) != RANGE_OKAY) {
396 addr = 0;
397 }
398 }
399 }
400 #endif /* __sparcv9 */
401
402 if (addr == 0) {
403 for (;;) {
404 addr = (caddr_t)align_hint;
405 map_addr(&addr, size, 0ll, 1, MAP_ALIGN);
406 if (addr != NULL || align_hint == share_size)
407 break;
408 align_hint = share_size;
409 }
410 if (addr == NULL) {
411 as_rangeunlock(as);
412 error = ENOMEM;
413 goto errret;
414 }
415 ASSERT(((uintptr_t)addr & (align_hint - 1)) == 0);
416 } else {
417 /* Use the user-supplied attach address */
418 caddr_t base;
419 size_t len;
420
421 /*
422 * Check that the address range
423 * 1) is properly aligned
424 * 2) is correct in unix terms
425 * 3) is within an unmapped address segment
426 */
427 base = addr;
428 len = size; /* use spt aligned size */
429 /* XXX - in SunOS, is sp->shm_segsz */
430 if ((uintptr_t)base & (share_size - 1)) {
431 error = EINVAL;
432 as_rangeunlock(as);
433 goto errret;
434 }
435 result = valid_usr_range(base, len, prot, as,
436 as->a_userlimit);
437 if (result == RANGE_BADPROT) {
438 /*
439 * We try to accomodate processors which
440 * may not support execute permissions on
441 * all ISM segments by trying the check
442 * again but without PROT_EXEC.
443 */
444 prot &= ~PROT_EXEC;
445 result = valid_usr_range(base, len, prot, as,
446 as->a_userlimit);
447 }
448 as_purge(as);
449 if (result != RANGE_OKAY ||
450 as_gap(as, len, &base, &len, AH_LO,
451 (caddr_t)NULL) != 0) {
452 error = EINVAL;
453 as_rangeunlock(as);
454 goto errret;
455 }
456 }
457
458 curprot = sp->shm_opts & SHM_PROT_MASK;
459 if (!isspt(sp)) {
460 error = sptcreate(size, &segspt, sp->shm_amp, prot,
461 flags, share_szc);
462 if (error) {
463 as_rangeunlock(as);
464 goto errret;
465 }
466 sp->shm_sptinfo->sptas = segspt->s_as;
467 sp->shm_sptseg = segspt;
468 sp->shm_opts = (sp->shm_opts & ~SHM_PROT_MASK) | prot;
469 } else if ((prot & curprot) != curprot) {
470 /*
471 * Ensure we're attaching to an ISM segment with
472 * fewer or equal permissions than what we're
473 * allowed. Fail if the segment has more
474 * permissions than what we're allowed.
475 */
476 error = EACCES;
477 as_rangeunlock(as);
478 goto errret;
479 }
480
481 ssd.shm_sptseg = sp->shm_sptseg;
482 ssd.shm_sptas = sp->shm_sptinfo->sptas;
483 ssd.shm_amp = sp->shm_amp;
484 error = as_map(as, addr, size, segspt_shmattach, &ssd);
485 if (error == 0)
486 sp->shm_ismattch++; /* keep count of ISM attaches */
487 } else {
488
489 /*
490 * Normal case.
491 */
492 if (flags & SHM_RDONLY)
493 prot &= ~PROT_WRITE;
494
495 if (addr == 0) {
496 /* Let the system pick the attach address */
497 map_addr(&addr, size, 0ll, 1, 0);
498 if (addr == NULL) {
499 as_rangeunlock(as);
500 error = ENOMEM;
501 goto errret;
502 }
503 } else {
504 /* Use the user-supplied attach address */
505 caddr_t base;
506 size_t len;
507
508 if (flags & SHM_RND)
509 addr = (caddr_t)((uintptr_t)addr &
510 ~(SHMLBA - 1));
511 /*
512 * Check that the address range
513 * 1) is properly aligned
514 * 2) is correct in unix terms
515 * 3) is within an unmapped address segment
516 */
517 base = addr;
518 len = size; /* use aligned size */
519 /* XXX - in SunOS, is sp->shm_segsz */
520 if ((uintptr_t)base & PAGEOFFSET) {
521 error = EINVAL;
522 as_rangeunlock(as);
523 goto errret;
524 }
525 result = valid_usr_range(base, len, prot, as,
526 as->a_userlimit);
527 if (result == RANGE_BADPROT) {
528 prot &= ~PROT_EXEC;
529 result = valid_usr_range(base, len, prot, as,
530 as->a_userlimit);
531 }
532 as_purge(as);
533 if (result != RANGE_OKAY ||
534 as_gap(as, len, &base, &len,
535 AH_LO, (caddr_t)NULL) != 0) {
536 error = EINVAL;
537 as_rangeunlock(as);
538 goto errret;
539 }
540 }
541
542 /* Initialize the create arguments and map the segment */
543 crargs = *(struct segvn_crargs *)zfod_argsp;
544 crargs.offset = 0;
545 crargs.type = MAP_SHARED;
546 crargs.amp = sp->shm_amp;
547 crargs.prot = prot;
548 crargs.maxprot = crargs.prot;
549 crargs.flags = 0;
550
551 error = as_map(as, addr, size, segvn_create, &crargs);
552 }
553
554 as_rangeunlock(as);
555 if (error)
556 goto errret;
557
558 /* record shmem range for the detach */
559 sa_add(pp, addr, (size_t)size, useISM ? SHMSA_ISM : 0, sp);
560 *rvp = (uintptr_t)addr;
561
562 sp->shm_atime = gethrestime_sec();
563 sp->shm_lpid = pp->p_pid;
564 ipc_hold(shm_svc, (kipc_perm_t *)sp);
565
566 /*
567 * Tell machine specific code that lwp has mapped shared memory
568 */
569 LWP_MMODEL_SHARED_AS(addr, size);
570
571 errret:
572 mutex_exit(lock);
573 return (error);
574 }
575
576 static void
577 shm_dtor(kipc_perm_t *perm)
578 {
579 kshmid_t *sp = (kshmid_t *)perm;
580 uint_t cnt;
581 size_t rsize;
582
583 ANON_LOCK_ENTER(&sp->shm_amp->a_rwlock, RW_WRITER);
584 anonmap_purge(sp->shm_amp);
585 ANON_LOCK_EXIT(&sp->shm_amp->a_rwlock);
586
587 if (sp->shm_sptinfo) {
588 if (isspt(sp)) {
589 sptdestroy(sp->shm_sptinfo->sptas, sp->shm_amp);
590 sp->shm_lkcnt = 0;
591 }
592 kmem_free(sp->shm_sptinfo, sizeof (sptinfo_t));
593 }
594
595 if (sp->shm_lkcnt > 0) {
596 shmem_unlock(sp, sp->shm_amp);
597 sp->shm_lkcnt = 0;
598 }
599
600 ANON_LOCK_ENTER(&sp->shm_amp->a_rwlock, RW_WRITER);
601 cnt = --sp->shm_amp->refcnt;
602 ANON_LOCK_EXIT(&sp->shm_amp->a_rwlock);
603 ASSERT(cnt == 0);
604 shm_rm_amp(sp);
605
606 if (sp->shm_perm.ipc_id != IPC_ID_INVAL) {
607 rsize = ptob(btopr(sp->shm_segsz));
608 ipcs_lock(shm_svc);
609 sp->shm_perm.ipc_proj->kpj_data.kpd_shmmax -= rsize;
610 sp->shm_perm.ipc_zone_ref.zref_zone->zone_shmmax -= rsize;
611 ipcs_unlock(shm_svc);
612 }
613 }
614
615 /* ARGSUSED */
616 static void
617 shm_rmid(kipc_perm_t *perm)
618 {
619 /* nothing to do */
620 }
621
622 /*
623 * Shmctl system call.
624 */
625 /* ARGSUSED */
626 static int
627 shmctl(int shmid, int cmd, void *arg)
628 {
629 kshmid_t *sp; /* shared memory header ptr */
630 STRUCT_DECL(shmid_ds, ds); /* for SVR4 IPC_SET */
631 int error = 0;
632 struct cred *cr = CRED();
633 kmutex_t *lock;
634 model_t mdl = get_udatamodel();
635 struct shmid_ds64 ds64;
636 shmatt_t nattch;
637
638 STRUCT_INIT(ds, mdl);
639
640 /*
641 * Perform pre- or non-lookup actions (e.g. copyins, RMID).
642 */
643 switch (cmd) {
644 case IPC_SET:
645 if (copyin(arg, STRUCT_BUF(ds), STRUCT_SIZE(ds)))
646 return (EFAULT);
647 break;
648
649 case IPC_SET64:
650 if (copyin(arg, &ds64, sizeof (struct shmid_ds64)))
651 return (EFAULT);
652 break;
653
654 case IPC_RMID:
655 return (ipc_rmid(shm_svc, shmid, cr));
656 }
657
658 if ((lock = ipc_lookup(shm_svc, shmid, (kipc_perm_t **)&sp)) == NULL)
659 return (EINVAL);
660
661 switch (cmd) {
662 /* Set ownership and permissions. */
663 case IPC_SET:
664 if (error = ipcperm_set(shm_svc, cr, &sp->shm_perm,
665 &STRUCT_BUF(ds)->shm_perm, mdl))
666 break;
667 sp->shm_ctime = gethrestime_sec();
668 break;
669
670 case IPC_STAT:
671 if (error = ipcperm_access(&sp->shm_perm, SHM_R, cr))
672 break;
673
674 nattch = sp->shm_perm.ipc_ref - 1;
675
676 ipcperm_stat(&STRUCT_BUF(ds)->shm_perm, &sp->shm_perm, mdl);
677 STRUCT_FSET(ds, shm_segsz, sp->shm_segsz);
678 STRUCT_FSETP(ds, shm_amp, NULL); /* kernel addr */
679 STRUCT_FSET(ds, shm_lkcnt, sp->shm_lkcnt);
680 STRUCT_FSET(ds, shm_lpid, sp->shm_lpid);
681 STRUCT_FSET(ds, shm_cpid, sp->shm_cpid);
682 STRUCT_FSET(ds, shm_nattch, nattch);
683 STRUCT_FSET(ds, shm_cnattch, sp->shm_ismattch);
684 STRUCT_FSET(ds, shm_atime, sp->shm_atime);
685 STRUCT_FSET(ds, shm_dtime, sp->shm_dtime);
686 STRUCT_FSET(ds, shm_ctime, sp->shm_ctime);
687
688 mutex_exit(lock);
689 if (copyout(STRUCT_BUF(ds), arg, STRUCT_SIZE(ds)))
690 return (EFAULT);
691
692 return (0);
693
694 case IPC_SET64:
695 if (error = ipcperm_set64(shm_svc, cr,
696 &sp->shm_perm, &ds64.shmx_perm))
697 break;
698 sp->shm_ctime = gethrestime_sec();
699 break;
700
701 case IPC_STAT64:
702 nattch = sp->shm_perm.ipc_ref - 1;
703
704 ipcperm_stat64(&ds64.shmx_perm, &sp->shm_perm);
705 ds64.shmx_segsz = sp->shm_segsz;
706 ds64.shmx_lkcnt = sp->shm_lkcnt;
707 ds64.shmx_lpid = sp->shm_lpid;
708 ds64.shmx_cpid = sp->shm_cpid;
709 ds64.shmx_nattch = nattch;
710 ds64.shmx_cnattch = sp->shm_ismattch;
711 ds64.shmx_atime = sp->shm_atime;
712 ds64.shmx_dtime = sp->shm_dtime;
713 ds64.shmx_ctime = sp->shm_ctime;
714
715 mutex_exit(lock);
716 if (copyout(&ds64, arg, sizeof (struct shmid_ds64)))
717 return (EFAULT);
718
719 return (0);
720
721 /* Lock segment in memory */
722 case SHM_LOCK:
723 if ((error = secpolicy_lock_memory(cr)) != 0)
724 break;
725
726 /* protect against overflow */
727 if (sp->shm_lkcnt >= USHRT_MAX) {
728 error = ENOMEM;
729 break;
730 }
731 if (!isspt(sp) && (sp->shm_lkcnt++ == 0)) {
732 if (error = shmem_lock(sp, sp->shm_amp)) {
733 ANON_LOCK_ENTER(&sp->shm_amp->a_rwlock,
734 RW_WRITER);
735 cmn_err(CE_NOTE, "shmctl - couldn't lock %ld"
736 " pages into memory", sp->shm_amp->size);
737 ANON_LOCK_EXIT(&sp->shm_amp->a_rwlock);
738 error = ENOMEM;
739 sp->shm_lkcnt--;
740 }
741 }
742 break;
743
744 /* Unlock segment */
745 case SHM_UNLOCK:
746 if ((error = secpolicy_lock_memory(cr)) != 0)
747 break;
748
749 if (sp->shm_lkcnt && (--sp->shm_lkcnt == 0)) {
750 shmem_unlock(sp, sp->shm_amp);
751 }
752 break;
753
754 /* Stage segment for removal, but don't remove until last detach */
755 case SHM_RMID:
756 if ((error = secpolicy_ipc_owner(cr, (kipc_perm_t *)sp)) != 0)
757 break;
758
759 /*
760 * If attached, just mark it as a pending remove, otherwise
761 * we must perform the normal ipc_rmid now.
762 */
763 if ((sp->shm_perm.ipc_ref - 1) > 0) {
764 sp->shm_opts |= SHM_RM_PENDING;
765 } else {
766 mutex_exit(lock);
767 return (ipc_rmid(shm_svc, shmid, cr));
768 }
769 break;
770
771 default:
772 error = EINVAL;
773 break;
774 }
775 mutex_exit(lock);
776 return (error);
777 }
778
779 static void
780 shm_detach(proc_t *pp, segacct_t *sap)
781 {
782 kshmid_t *sp = sap->sa_id;
783 size_t len = sap->sa_len;
784 caddr_t addr = sap->sa_addr;
785
786 /*
787 * Discard lwpchan mappings.
788 */
789 if (pp->p_lcp != NULL)
790 lwpchan_delete_mapping(pp, addr, addr + len);
791 (void) as_unmap(pp->p_as, addr, len);
792
793 /*
794 * Perform some detach-time accounting.
795 */
796 (void) ipc_lock(shm_svc, sp->shm_perm.ipc_id);
797 if (sap->sa_flags & SHMSA_ISM)
798 sp->shm_ismattch--;
799 sp->shm_dtime = gethrestime_sec();
800 sp->shm_lpid = pp->p_pid;
801 if ((sp->shm_opts & SHM_RM_PENDING) != 0 &&
802 sp->shm_perm.ipc_ref == 2) {
803 /*
804 * If this is the last detach of the segment across the whole
805 * system then now we can perform the delayed IPC_RMID.
806 * The ipc_ref count has 1 for the original 'get' and one for
807 * each 'attach' (see 'stat' handling in shmctl).
808 */
809 sp->shm_opts &= ~SHM_RM_PENDING;
810 mutex_enter(&shm_svc->ipcs_lock);
811 ipc_rmsvc(shm_svc, (kipc_perm_t *)sp); /* Drops lock */
812 ASSERT(!MUTEX_HELD(&shm_svc->ipcs_lock));
813 ASSERT(((kipc_perm_t *)sp)->ipc_ref > 0);
814
815 /* Lock was dropped, need to retake it for following rele. */
816 (void) ipc_lock(shm_svc, sp->shm_perm.ipc_id);
817 }
818 ipc_rele(shm_svc, (kipc_perm_t *)sp); /* Drops lock */
819
820 kmem_free(sap, sizeof (segacct_t));
821 }
822
823 static int
824 shmdt(caddr_t addr)
825 {
826 proc_t *pp = curproc;
827 segacct_t *sap, template;
828
829 mutex_enter(&pp->p_lock);
830 prbarrier(pp); /* block /proc. See shmgetid(). */
831
832 template.sa_addr = addr;
833 template.sa_len = 0;
834 if ((pp->p_segacct == NULL) ||
835 ((sap = avl_find(pp->p_segacct, &template, NULL)) == NULL)) {
836 mutex_exit(&pp->p_lock);
837 return (EINVAL);
838 }
839 if (sap->sa_addr != addr) {
840 mutex_exit(&pp->p_lock);
841 return (EINVAL);
842 }
843 avl_remove(pp->p_segacct, sap);
844 mutex_exit(&pp->p_lock);
845
846 shm_detach(pp, sap);
847
848 return (0);
849 }
850
851 /*
852 * Remove all shared memory segments associated with a given zone.
853 * Called by zone_shutdown when the zone is halted.
854 */
855 /*ARGSUSED1*/
856 static void
857 shm_remove_zone(zoneid_t zoneid, void *arg)
858 {
859 ipc_remove_zone(shm_svc, zoneid);
860 }
861
862 /*
863 * Shmget (create new shmem) system call.
864 */
865 static int
866 shmget(key_t key, size_t size, int shmflg, uintptr_t *rvp)
867 {
868 proc_t *pp = curproc;
869 kshmid_t *sp;
870 kmutex_t *lock;
871 int error;
872
873 top:
874 if (error = ipc_get(shm_svc, key, shmflg, (kipc_perm_t **)&sp, &lock))
875 return (error);
876
877 if (!IPC_FREE(&sp->shm_perm)) {
878 /*
879 * A segment with the requested key exists.
880 */
881 if (size > sp->shm_segsz) {
882 mutex_exit(lock);
883 return (EINVAL);
884 }
885 } else {
886 /*
887 * A new segment should be created.
888 */
889 size_t npages = btopr(size);
890 size_t rsize = ptob(npages);
891
892 /*
893 * Check rsize and the per-project and per-zone limit on
894 * shared memory. Checking rsize handles both the size == 0
895 * case and the size < ULONG_MAX & PAGEMASK case (i.e.
896 * rounding up wraps a size_t).
897 */
898 if (rsize == 0 ||
899 (rctl_test(rc_project_shmmax,
900 pp->p_task->tk_proj->kpj_rctls, pp, rsize,
901 RCA_SAFE) & RCT_DENY) ||
902 (rctl_test(rc_zone_shmmax,
903 pp->p_zone->zone_rctls, pp, rsize,
904 RCA_SAFE) & RCT_DENY)) {
905
906 mutex_exit(&pp->p_lock);
907 mutex_exit(lock);
908 ipc_cleanup(shm_svc, (kipc_perm_t *)sp);
909 return (EINVAL);
910 }
911 mutex_exit(&pp->p_lock);
912 mutex_exit(lock);
913
914 if (anon_resv(rsize) == 0) {
915 ipc_cleanup(shm_svc, (kipc_perm_t *)sp);
916 return (ENOMEM);
917 }
918
919 /*
920 * If any new failure points are introduced between the
921 * the above anon_resv() and the below ipc_commit_begin(),
922 * these failure points will need to unreserve the anon
923 * reserved using anon_unresv().
924 *
925 * Once ipc_commit_begin() is called, the anon reserved
926 * above will be automatically unreserved by future calls to
927 * ipcs_cleanup() -> shm_dtor() -> shm_rm_amp(). If
928 * ipc_commit_begin() fails, it internally calls shm_dtor(),
929 * unreserving the above anon, and freeing the below amp.
930 */
931
932 sp->shm_amp = anonmap_alloc(rsize, rsize, ANON_SLEEP);
933 sp->shm_amp->a_sp = sp;
934 /*
935 * Store the original user's requested size, in bytes,
936 * rather than the page-aligned size. The former is
937 * used for IPC_STAT and shmget() lookups. The latter
938 * is saved in the anon_map structure and is used for
939 * calls to the vm layer.
940 */
941 sp->shm_segsz = size;
942 sp->shm_atime = sp->shm_dtime = 0;
943 sp->shm_ctime = gethrestime_sec();
944 sp->shm_lpid = (pid_t)0;
945 sp->shm_cpid = curproc->p_pid;
946 sp->shm_ismattch = 0;
947 sp->shm_sptinfo = NULL;
948 /*
949 * Check limits one last time, push id into global
950 * visibility, and update resource usage counts.
951 */
952 if (error = ipc_commit_begin(shm_svc, key, shmflg,
953 (kipc_perm_t *)sp)) {
954 if (error == EAGAIN)
955 goto top;
956 return (error);
957 }
958
959 if ((rctl_test(rc_project_shmmax,
960 sp->shm_perm.ipc_proj->kpj_rctls, pp, rsize,
961 RCA_SAFE) & RCT_DENY) ||
962 (rctl_test(rc_zone_shmmax,
963 sp->shm_perm.ipc_zone_ref.zref_zone->zone_rctls, pp, rsize,
964 RCA_SAFE) & RCT_DENY)) {
965 ipc_cleanup(shm_svc, (kipc_perm_t *)sp);
966 return (EINVAL);
967 }
968 sp->shm_perm.ipc_proj->kpj_data.kpd_shmmax += rsize;
969 sp->shm_perm.ipc_zone_ref.zref_zone->zone_shmmax += rsize;
970
971 lock = ipc_commit_end(shm_svc, &sp->shm_perm);
972 }
973
974 if (AU_AUDITING())
975 audit_ipcget(AT_IPC_SHM, (void *)sp);
976
977 *rvp = (uintptr_t)(sp->shm_perm.ipc_id);
978
979 mutex_exit(lock);
980 return (0);
981 }
982
983 /*
984 * shmids system call.
985 */
986 static int
987 shmids(int *buf, uint_t nids, uint_t *pnids)
988 {
989 return (ipc_ids(shm_svc, buf, nids, pnids));
990 }
991
992 /*
993 * System entry point for shmat, shmctl, shmdt, and shmget system calls.
994 */
995 static uintptr_t
996 shmsys(int opcode, uintptr_t a0, uintptr_t a1, uintptr_t a2)
997 {
998 int error;
999 uintptr_t r_val = 0;
1000
1001 switch (opcode) {
1002 case SHMAT:
1003 error = shmat((int)a0, (caddr_t)a1, (int)a2, &r_val);
1004 break;
1005 case SHMCTL:
1006 error = shmctl((int)a0, (int)a1, (void *)a2);
1007 break;
1008 case SHMDT:
1009 error = shmdt((caddr_t)a0);
1010 break;
1011 case SHMGET:
1012 error = shmget((key_t)a0, (size_t)a1, (int)a2, &r_val);
1013 break;
1014 case SHMIDS:
1015 error = shmids((int *)a0, (uint_t)a1, (uint_t *)a2);
1016 break;
1017 default:
1018 error = EINVAL;
1019 break;
1020 }
1021
1022 if (error)
1023 return ((uintptr_t)set_errno(error));
1024
1025 return (r_val);
1026 }
1027
1028 /*
1029 * segacct_t comparator
1030 * This works as expected, with one minor change: the first of two real
1031 * segments with equal addresses is considered to be 'greater than' the
1032 * second. We only return equal when searching using a template, in
1033 * which case we explicitly set the template segment's length to 0
1034 * (which is invalid for a real segment).
1035 */
1036 static int
1037 shm_sacompar(const void *x, const void *y)
1038 {
1039 segacct_t *sa1 = (segacct_t *)x;
1040 segacct_t *sa2 = (segacct_t *)y;
1041
1042 if (sa1->sa_addr < sa2->sa_addr) {
1043 return (-1);
1044 } else if (sa2->sa_len != 0) {
1045 if (sa1->sa_addr >= sa2->sa_addr + sa2->sa_len) {
1046 return (1);
1047 } else if (sa1->sa_len != 0) {
1048 return (1);
1049 } else {
1050 return (0);
1051 }
1052 } else if (sa1->sa_addr > sa2->sa_addr) {
1053 return (1);
1054 } else {
1055 return (0);
1056 }
1057 }
1058
1059 /*
1060 * add this record to the segacct list.
1061 */
1062 static void
1063 sa_add(struct proc *pp, caddr_t addr, size_t len, ulong_t flags, kshmid_t *id)
1064 {
1065 segacct_t *nsap;
1066 avl_tree_t *tree = NULL;
1067 avl_index_t where;
1068
1069 nsap = kmem_alloc(sizeof (segacct_t), KM_SLEEP);
1070 nsap->sa_addr = addr;
1071 nsap->sa_len = len;
1072 nsap->sa_flags = flags;
1073 nsap->sa_id = id;
1074
1075 if (pp->p_segacct == NULL)
1076 tree = kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
1077
1078 mutex_enter(&pp->p_lock);
1079 prbarrier(pp); /* block /proc. See shmgetid(). */
1080
1081 if (pp->p_segacct == NULL) {
1082 avl_create(tree, shm_sacompar, sizeof (segacct_t),
1083 offsetof(segacct_t, sa_tree));
1084 pp->p_segacct = tree;
1085 } else if (tree) {
1086 kmem_free(tree, sizeof (avl_tree_t));
1087 }
1088
1089 /*
1090 * We can ignore the result of avl_find, as the comparator will
1091 * never return equal for segments with non-zero length. This
1092 * is a necessary hack to get around the fact that we do, in
1093 * fact, have duplicate keys.
1094 */
1095 (void) avl_find(pp->p_segacct, nsap, &where);
1096 avl_insert(pp->p_segacct, nsap, where);
1097
1098 mutex_exit(&pp->p_lock);
1099 }
1100
1101 /*
1102 * Duplicate parent's segacct records in child.
1103 */
1104 void
1105 shmfork(struct proc *ppp, struct proc *cpp)
1106 {
1107 segacct_t *sap;
1108 kshmid_t *sp;
1109 kmutex_t *mp;
1110
1111 ASSERT(ppp->p_segacct != NULL);
1112
1113 /*
1114 * We are the only lwp running in the parent so nobody can
1115 * mess with our p_segacct list. Thus it is safe to traverse
1116 * the list without holding p_lock. This is essential because
1117 * we can't hold p_lock during a KM_SLEEP allocation.
1118 */
1119 for (sap = (segacct_t *)avl_first(ppp->p_segacct); sap != NULL;
1120 sap = (segacct_t *)AVL_NEXT(ppp->p_segacct, sap)) {
1121 sa_add(cpp, sap->sa_addr, sap->sa_len, sap->sa_flags,
1122 sap->sa_id);
1123 sp = sap->sa_id;
1124 mp = ipc_lock(shm_svc, sp->shm_perm.ipc_id);
1125 if (sap->sa_flags & SHMSA_ISM)
1126 sp->shm_ismattch++;
1127 ipc_hold(shm_svc, (kipc_perm_t *)sp);
1128 mutex_exit(mp);
1129 }
1130 }
1131
1132 /*
1133 * Detach shared memory segments from exiting process.
1134 */
1135 void
1136 shmexit(struct proc *pp)
1137 {
1138 segacct_t *sap;
1139 avl_tree_t *tree;
1140 void *cookie = NULL;
1141
1142 ASSERT(pp->p_segacct != NULL);
1143
1144 mutex_enter(&pp->p_lock);
1145 prbarrier(pp);
1146 tree = pp->p_segacct;
1147 pp->p_segacct = NULL;
1148 mutex_exit(&pp->p_lock);
1149
1150 while ((sap = avl_destroy_nodes(tree, &cookie)) != NULL)
1151 (void) shm_detach(pp, sap);
1152
1153 avl_destroy(tree);
1154 kmem_free(tree, sizeof (avl_tree_t));
1155 }
1156
1157 /*
1158 * At this time pages should be in memory, so just lock them.
1159 */
1160 static void
1161 lock_again(size_t npages, kshmid_t *sp, struct anon_map *amp)
1162 {
1163 struct anon *ap;
1164 struct page *pp;
1165 struct vnode *vp;
1166 u_offset_t off;
1167 ulong_t anon_idx;
1168 anon_sync_obj_t cookie;
1169
1170 mutex_enter(&sp->shm_mlock);
1171 ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
1172 for (anon_idx = 0; npages != 0; anon_idx++, npages--) {
1173
1174 anon_array_enter(amp, anon_idx, &cookie);
1175 ap = anon_get_ptr(amp->ahp, anon_idx);
1176 ASSERT(ap != NULL);
1177 swap_xlate(ap, &vp, &off);
1178 anon_array_exit(&cookie);
1179
1180 pp = page_lookup(vp, off, SE_SHARED);
1181 if (pp == NULL) {
1182 panic("lock_again: page not in the system");
1183 /*NOTREACHED*/
1184 }
1185 /* page should already be locked by caller */
1186 ASSERT(pp->p_lckcnt > 0);
1187 (void) page_pp_lock(pp, 0, 0);
1188 page_unlock(pp);
1189 }
1190 ANON_LOCK_EXIT(&->a_rwlock);
1191 mutex_exit(&sp->shm_mlock);
1192 }
1193
1194 /*
1195 * Attach the shared memory segment to the process
1196 * address space and lock the pages.
1197 */
1198 static int
1199 shmem_lock(kshmid_t *sp, struct anon_map *amp)
1200 {
1201 size_t npages = btopr(amp->size);
1202 struct as *as;
1203 struct segvn_crargs crargs;
1204 uint_t error;
1205
1206 /*
1207 * A later ISM/DISM attach may increase the size of the amp, so
1208 * cache the number of pages locked for the future shmem_unlock()
1209 */
1210 sp->shm_lkpages = npages;
1211
1212 as = as_alloc();
1213 /* Initialize the create arguments and map the segment */
1214 crargs = *(struct segvn_crargs *)zfod_argsp; /* structure copy */
1215 crargs.offset = (u_offset_t)0;
1216 crargs.type = MAP_SHARED;
1217 crargs.amp = amp;
1218 crargs.prot = PROT_ALL;
1219 crargs.maxprot = crargs.prot;
1220 crargs.flags = 0;
1221 error = as_map(as, 0x0, amp->size, segvn_create, &crargs);
1222 if (!error) {
1223 if ((error = as_ctl(as, 0x0, amp->size, MC_LOCK, 0, 0,
1224 NULL, 0)) == 0) {
1225 lock_again(npages, sp, amp);
1226 }
1227 (void) as_unmap(as, 0x0, amp->size);
1228 }
1229 as_free(as);
1230 return (error);
1231 }
1232
1233
1234 /*
1235 * Unlock shared memory
1236 */
1237 static void
1238 shmem_unlock(kshmid_t *sp, struct anon_map *amp)
1239 {
1240 struct anon *ap;
1241 pgcnt_t npages = sp->shm_lkpages;
1242 struct vnode *vp;
1243 struct page *pp;
1244 u_offset_t off;
1245 ulong_t anon_idx;
1246 size_t unlocked_bytes = 0;
1247 kproject_t *proj;
1248 anon_sync_obj_t cookie;
1249
1250 proj = sp->shm_perm.ipc_proj;
1251 mutex_enter(&sp->shm_mlock);
1252 ANON_LOCK_ENTER(&->a_rwlock, RW_READER);
1253 for (anon_idx = 0; anon_idx < npages; anon_idx++) {
1254
1255 anon_array_enter(amp, anon_idx, &cookie);
1256 if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
1257 panic("shmem_unlock: null app");
1258 /*NOTREACHED*/
1259 }
1260 swap_xlate(ap, &vp, &off);
1261 anon_array_exit(&cookie);
1262 pp = page_lookup(vp, off, SE_SHARED);
1263 if (pp == NULL) {
1264 panic("shmem_unlock: page not in the system");
1265 /*NOTREACHED*/
1266 }
1267 /*
1268 * Page should at least have once lock from previous
1269 * shmem_lock
1270 */
1271 ASSERT(pp->p_lckcnt > 0);
1272 page_pp_unlock(pp, 0, 0);
1273 if (pp->p_lckcnt == 0)
1274 unlocked_bytes += PAGESIZE;
1275
1276 page_unlock(pp);
1277 }
1278
1279 if (unlocked_bytes > 0) {
1280 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
1281 }
1282
1283 ANON_LOCK_EXIT(&->a_rwlock);
1284 mutex_exit(&sp->shm_mlock);
1285 }
1286
1287 /*
1288 * We call this routine when we have removed all references to this
1289 * amp. This means all shmdt()s and the IPC_RMID have been done.
1290 */
1291 static void
1292 shm_rm_amp(kshmid_t *sp)
1293 {
1294 struct anon_map *amp = sp->shm_amp;
1295 zone_t *zone;
1296
1297 zone = sp->shm_perm.ipc_zone_ref.zref_zone;
1298 ASSERT(zone != NULL);
1299 /*
1300 * Free up the anon_map.
1301 */
1302 lgrp_shm_policy_fini(amp, NULL);
1303 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
1304 if (amp->a_szc != 0) {
1305 anon_shmap_free_pages(amp, 0, amp->size);
1306 } else {
1307 anon_free(amp->ahp, 0, amp->size);
1308 }
1309 ANON_LOCK_EXIT(&->a_rwlock);
1310 anon_unresv_zone(amp->swresv, zone);
1311 anonmap_free(amp);
1312 }
1313
1314 /*
1315 * Return the shared memory id for the process's virtual address.
1316 * Return SHMID_NONE if addr is not within a SysV shared memory segment.
1317 * Return SHMID_FREE if addr's SysV shared memory segment's id has been freed.
1318 *
1319 * shmgetid() is called from code in /proc with the process locked but
1320 * with pp->p_lock not held. The address space lock is held, so we
1321 * cannot grab pp->p_lock here due to lock-ordering constraints.
1322 * Because of all this, modifications to the p_segacct list must only
1323 * be made after calling prbarrier() to ensure the process is not locked.
1324 * See shmdt() and sa_add(), above. shmgetid() may also be called on a
1325 * thread's own process without the process locked.
1326 */
1327 int
1328 shmgetid(proc_t *pp, caddr_t addr)
1329 {
1330 segacct_t *sap, template;
1331
1332 ASSERT(MUTEX_NOT_HELD(&pp->p_lock));
1333 ASSERT((pp->p_proc_flag & P_PR_LOCK) || pp == curproc);
1334
1335 if (pp->p_segacct == NULL)
1336 return (SHMID_NONE);
1337
1338 template.sa_addr = addr;
1339 template.sa_len = 0;
1340 if ((sap = avl_find(pp->p_segacct, &template, NULL)) == NULL)
1341 return (SHMID_NONE);
1342
1343 if (IPC_FREE(&sap->sa_id->shm_perm))
1344 return (SHMID_FREE);
1345
1346 return (sap->sa_id->shm_perm.ipc_id);
1347 }