3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2020 Joyent, Inc.
24 * Copyright 2021 RackTop Systems, Inc.
25 */
26
27 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
29 /* All Rights Reserved */
30
31 /* Copyright (c) 1987, 1988 Microsoft Corporation */
32 /* All Rights Reserved */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 #include <sys/types.h>
40 #include <sys/param.h>
41 #include <sys/signal.h>
42 #include <sys/regset.h>
43 #include <sys/privregs.h>
653 cxs = cfp->fpu_regs.kfpu_u.kfpu_xs;
654 cfx = &cxs->xs_fxsave;
655
656 bcopy(&avx_initial, cxs, sizeof (*cxs));
657 cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS;
658 cfx->fx_fcw = fx->fx_fcw;
659 cxs->xs_xstate_bv |= (get_xcr(XFEATURE_ENABLED_MASK) &
660 XFEATURE_FP_INITIAL);
661 break;
662 default:
663 panic("Invalid fp_save_mech");
664 /*NOTREACHED*/
665 }
666
667 /*
668 * Mark that both the parent and child need to have the FPU cleaned up
669 * before returning to user land.
670 */
671
672 installctx(ct, cfp, fpsave_ctxt, fprestore_ctxt, fp_new_lwp,
673 fp_new_lwp, NULL, fp_free);
674 }
675
676 /*
677 * Free any state associated with floating point context.
678 * Fp_free can be called in three cases:
679 * 1) from reaper -> thread_free -> freectx-> fp_free
680 * fp context belongs to a thread on deathrow
681 * nothing to do, thread will never be resumed
682 * thread calling ctxfree is reaper
683 *
684 * 2) from exec -> freectx -> fp_free
685 * fp context belongs to the current thread
686 * must disable fpu, thread calling ctxfree is curthread
687 *
688 * 3) from restorecontext -> setfpregs -> fp_free
689 * we have a modified context in the memory (lwp->pcb_fpu)
690 * disable fpu and release the fp context for the CPU
691 *
692 */
693 /*ARGSUSED*/
779 xrestore(fp->fpu_regs.kfpu_u.kfpu_xs, fp->fpu_xsave_mask);
780 break;
781 default:
782 panic("Invalid fp_save_mech");
783 /*NOTREACHED*/
784 }
785
786 fp->fpu_flags &= ~FPU_VALID;
787 }
788
789 /*
790 * Reset the FPU such that it is in a valid state for a new thread that is
791 * coming out of exec. The FPU will be in a usable state at this point. At this
792 * point we know that the FPU state has already been allocated and if this
793 * wasn't an init process, then it will have had fp_free() previously called.
794 */
795 void
796 fp_exec(void)
797 {
798 struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;
799
800 if (fp_save_mech == FP_XSAVE) {
801 fp->fpu_xsave_mask = XFEATURE_FP_ALL;
802 }
803
804 /*
805 * Make sure that we're not preempted in the middle of initializing the
806 * FPU on CPU.
807 */
808 kpreempt_disable();
809 installctx(curthread, fp, fpsave_ctxt, fprestore_ctxt, fp_new_lwp,
810 fp_new_lwp, NULL, fp_free);
811 fpinit();
812 fp->fpu_flags = FPU_EN;
813 kpreempt_enable();
814 }
815
816
817 /*
818 * Seeds the initial state for the current thread. The possibilities are:
819 * 1. Another process has modified the FPU state before we have done any
820 * initialization: Load the FPU state from the LWP state.
821 * 2. The FPU state has not been externally modified: Load a clean state.
822 */
823 void
824 fp_seed(void)
825 {
826 struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;
827
828 ASSERT(curthread->t_preempt >= 1);
829 ASSERT((fp->fpu_flags & FPU_EN) == 0);
830
831 /*
832 * Always initialize a new context and initialize the hardware.
833 */
834 if (fp_save_mech == FP_XSAVE) {
835 fp->fpu_xsave_mask = XFEATURE_FP_ALL;
836 }
837
838 installctx(curthread, fp, fpsave_ctxt, fprestore_ctxt, fp_new_lwp,
839 fp_new_lwp, NULL, fp_free);
840 fpinit();
841
842 /*
843 * If FPU_VALID is set, it means someone has modified registers via
844 * /proc. In this case, restore the current lwp's state.
845 */
846 if (fp->fpu_flags & FPU_VALID)
847 fp_restore(fp);
848
849 ASSERT((fp->fpu_flags & FPU_VALID) == 0);
850 fp->fpu_flags = FPU_EN;
851 }
852
853 /*
854 * When using xsave/xrstor, these three functions are used by the lwp code to
855 * manage the memory for the xsave area.
856 */
857 void
858 fp_lwp_init(struct _klwp *lwp)
859 {
1301 fp_restore(pf);
1302 curthread->t_flag |= T_KFPU;
1303 }
1304
1305 /*
1306 * Validate that the thread is not switching off-cpu while actively using the
1307 * FPU within the kernel.
1308 */
1309 void
1310 kernel_fpu_no_swtch(void)
1311 {
1312 if ((curthread->t_flag & T_KFPU) != 0) {
1313 panic("curthread swtch-ing while the kernel is using the FPU");
1314 }
1315 }
1316
1317 void
1318 kernel_fpu_begin(kfpu_state_t *kfpu, uint_t flags)
1319 {
1320 klwp_t *pl = curthread->t_lwp;
1321
1322 if ((curthread->t_flag & T_KFPU) != 0) {
1323 panic("curthread attempting to nest kernel FPU states");
1324 }
1325
1326 /* KFPU_USE_LWP and KFPU_NO_STATE are mutually exclusive. */
1327 ASSERT((flags & (KFPU_USE_LWP | KFPU_NO_STATE)) !=
1328 (KFPU_USE_LWP | KFPU_NO_STATE));
1329
1330 if ((flags & KFPU_NO_STATE) == KFPU_NO_STATE) {
1331 /*
1332 * Since we don't have a kfpu_state or usable lwp pcb_fpu to
1333 * hold our kernel FPU context, we depend on the caller doing
1334 * kpreempt_disable for the duration of our FPU usage. This
1335 * should only be done for very short periods of time.
1336 */
1337 ASSERT(curthread->t_preempt > 0);
1338 ASSERT(kfpu == NULL);
1339
1340 if (pl != NULL) {
1363 panic("attempting to reuse kernel FPU state at %p when "
1364 "another thread already is using", kfpu);
1365
1366 if ((kfpu->kfpu_flags & KFPU_F_INITIALIZED) == 0)
1367 kernel_fpu_fpstate_init(kfpu);
1368
1369 kfpu->kfpu_curthread = curthread;
1370 }
1371
1372 /*
1373 * Not all threads may have an active LWP. If they do and we're not
1374 * going to re-use the LWP, then we should go ahead and save the state.
1375 * We must also note that the fpu is now being used by the kernel and
1376 * therefore we do not want to manage the fpu state via the user-level
1377 * thread's context handlers.
1378 *
1379 * We might have already saved once (due to a prior use of the kernel
1380 * FPU or another code path) so FPU_VALID could be set. This is handled
1381 * by fp_save, as is the FPU_EN check.
1382 */
1383 if (pl != NULL) {
1384 kpreempt_disable();
1385 if ((flags & KFPU_USE_LWP) == 0)
1386 fp_save(&pl->lwp_pcb.pcb_fpu);
1387 pl->lwp_pcb.pcb_fpu.fpu_flags |= FPU_KERNEL;
1388 kpreempt_enable();
1389 }
1390
1391 /*
1392 * Set the context operations for kernel FPU usage. Note that this
1393 * cannot be done with pre-emption and interrupts disabled, since
1394 * installctx does a sleeping allocation. We haven't finished
1395 * initializing our kernel FPU state yet, but in the rare case that we
1396 * happen to save/restore before that, no harm is done.
1397 */
1398 installctx(curthread, kfpu, kernel_fpu_ctx_save, kernel_fpu_ctx_restore,
1399 NULL, NULL, NULL, NULL);
1400
1401 curthread->t_flag |= T_KFPU;
1402
1403 if ((flags & KFPU_USE_LWP) == KFPU_USE_LWP) {
1404 /*
1405 * For pure kernel threads with an LWP, we can use the LWP's
1406 * pcb_fpu to save/restore context.
1407 */
1408 fpu_ctx_t *pf = &pl->lwp_pcb.pcb_fpu;
1409
1410 VERIFY(curthread->t_procp->p_flag & SSYS);
1411 VERIFY(kfpu == NULL);
1412 ASSERT((pf->fpu_flags & FPU_EN) == 0);
1413
1414 /* Always restore the fpu to the initial state. */
1415 if (fp_save_mech == FP_XSAVE)
1416 pf->fpu_xsave_mask = XFEATURE_FP_ALL;
1417 fpinit();
1418 pf->fpu_flags = FPU_EN | FPU_KERNEL;
1419 } else {
1420 /* initialize the kfpu state */
1421 kernel_fpu_ctx_restore(kfpu);
1422 }
1423 }
1424
1425 void
1426 kernel_fpu_end(kfpu_state_t *kfpu, uint_t flags)
1427 {
1428 ulong_t iflags;
1429
1430 if ((curthread->t_flag & T_KFPU) == 0) {
1431 panic("curthread attempting to clear kernel FPU state "
1432 "without using it");
1433 }
1434
1435 /*
1436 * General comments on why the rest of this function is structured the
1437 * way it is. Be aware that there is a lot of subtlety here.
1438 *
1439 * If a user-level thread ever uses the fpu while in the kernel, then
1440 * we cannot call fpdisable since that does STTS. That will set the
1441 * ts bit in %cr0 which will cause an exception if anything touches the
1442 * fpu. However, the user-level context switch handler (fpsave_ctxt)
|
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2021 Joyent, Inc.
24 * Copyright 2021 RackTop Systems, Inc.
25 */
26
27 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
29 /* All Rights Reserved */
30
31 /* Copyright (c) 1987, 1988 Microsoft Corporation */
32 /* All Rights Reserved */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 #include <sys/types.h>
40 #include <sys/param.h>
41 #include <sys/signal.h>
42 #include <sys/regset.h>
43 #include <sys/privregs.h>
653 cxs = cfp->fpu_regs.kfpu_u.kfpu_xs;
654 cfx = &cxs->xs_fxsave;
655
656 bcopy(&avx_initial, cxs, sizeof (*cxs));
657 cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS;
658 cfx->fx_fcw = fx->fx_fcw;
659 cxs->xs_xstate_bv |= (get_xcr(XFEATURE_ENABLED_MASK) &
660 XFEATURE_FP_INITIAL);
661 break;
662 default:
663 panic("Invalid fp_save_mech");
664 /*NOTREACHED*/
665 }
666
667 /*
668 * Mark that both the parent and child need to have the FPU cleaned up
669 * before returning to user land.
670 */
671
672 installctx(ct, cfp, fpsave_ctxt, fprestore_ctxt, fp_new_lwp,
673 fp_new_lwp, NULL, fp_free, NULL);
674 }
675
676 /*
677 * Free any state associated with floating point context.
678 * Fp_free can be called in three cases:
679 * 1) from reaper -> thread_free -> freectx-> fp_free
680 * fp context belongs to a thread on deathrow
681 * nothing to do, thread will never be resumed
682 * thread calling ctxfree is reaper
683 *
684 * 2) from exec -> freectx -> fp_free
685 * fp context belongs to the current thread
686 * must disable fpu, thread calling ctxfree is curthread
687 *
688 * 3) from restorecontext -> setfpregs -> fp_free
689 * we have a modified context in the memory (lwp->pcb_fpu)
690 * disable fpu and release the fp context for the CPU
691 *
692 */
693 /*ARGSUSED*/
779 xrestore(fp->fpu_regs.kfpu_u.kfpu_xs, fp->fpu_xsave_mask);
780 break;
781 default:
782 panic("Invalid fp_save_mech");
783 /*NOTREACHED*/
784 }
785
786 fp->fpu_flags &= ~FPU_VALID;
787 }
788
789 /*
790 * Reset the FPU such that it is in a valid state for a new thread that is
791 * coming out of exec. The FPU will be in a usable state at this point. At this
792 * point we know that the FPU state has already been allocated and if this
793 * wasn't an init process, then it will have had fp_free() previously called.
794 */
795 void
796 fp_exec(void)
797 {
798 struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;
799 struct ctxop *ctx = installctx_preallocate();
800
801 if (fp_save_mech == FP_XSAVE) {
802 fp->fpu_xsave_mask = XFEATURE_FP_ALL;
803 }
804
805 /*
806 * Make sure that we're not preempted in the middle of initializing the
807 * FPU on CPU.
808 */
809 kpreempt_disable();
810 installctx(curthread, fp, fpsave_ctxt, fprestore_ctxt, fp_new_lwp,
811 fp_new_lwp, NULL, fp_free, ctx);
812 fpinit();
813 fp->fpu_flags = FPU_EN;
814 kpreempt_enable();
815 }
816
817
818 /*
819 * Seeds the initial state for the current thread. The possibilities are:
820 * 1. Another process has modified the FPU state before we have done any
821 * initialization: Load the FPU state from the LWP state.
822 * 2. The FPU state has not been externally modified: Load a clean state.
823 */
824 void
825 fp_seed(void)
826 {
827 struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;
828
829 ASSERT(curthread->t_preempt >= 1);
830 ASSERT((fp->fpu_flags & FPU_EN) == 0);
831
832 /*
833 * Always initialize a new context and initialize the hardware.
834 */
835 if (fp_save_mech == FP_XSAVE) {
836 fp->fpu_xsave_mask = XFEATURE_FP_ALL;
837 }
838
839 installctx(curthread, fp, fpsave_ctxt, fprestore_ctxt, fp_new_lwp,
840 fp_new_lwp, NULL, fp_free, NULL);
841 fpinit();
842
843 /*
844 * If FPU_VALID is set, it means someone has modified registers via
845 * /proc. In this case, restore the current lwp's state.
846 */
847 if (fp->fpu_flags & FPU_VALID)
848 fp_restore(fp);
849
850 ASSERT((fp->fpu_flags & FPU_VALID) == 0);
851 fp->fpu_flags = FPU_EN;
852 }
853
854 /*
855 * When using xsave/xrstor, these three functions are used by the lwp code to
856 * manage the memory for the xsave area.
857 */
858 void
859 fp_lwp_init(struct _klwp *lwp)
860 {
1302 fp_restore(pf);
1303 curthread->t_flag |= T_KFPU;
1304 }
1305
1306 /*
1307 * Validate that the thread is not switching off-cpu while actively using the
1308 * FPU within the kernel.
1309 */
1310 void
1311 kernel_fpu_no_swtch(void)
1312 {
1313 if ((curthread->t_flag & T_KFPU) != 0) {
1314 panic("curthread swtch-ing while the kernel is using the FPU");
1315 }
1316 }
1317
1318 void
1319 kernel_fpu_begin(kfpu_state_t *kfpu, uint_t flags)
1320 {
1321 klwp_t *pl = curthread->t_lwp;
1322 struct ctxop *ctx;
1323
1324 if ((curthread->t_flag & T_KFPU) != 0) {
1325 panic("curthread attempting to nest kernel FPU states");
1326 }
1327
1328 /* KFPU_USE_LWP and KFPU_NO_STATE are mutually exclusive. */
1329 ASSERT((flags & (KFPU_USE_LWP | KFPU_NO_STATE)) !=
1330 (KFPU_USE_LWP | KFPU_NO_STATE));
1331
1332 if ((flags & KFPU_NO_STATE) == KFPU_NO_STATE) {
1333 /*
1334 * Since we don't have a kfpu_state or usable lwp pcb_fpu to
1335 * hold our kernel FPU context, we depend on the caller doing
1336 * kpreempt_disable for the duration of our FPU usage. This
1337 * should only be done for very short periods of time.
1338 */
1339 ASSERT(curthread->t_preempt > 0);
1340 ASSERT(kfpu == NULL);
1341
1342 if (pl != NULL) {
1365 panic("attempting to reuse kernel FPU state at %p when "
1366 "another thread already is using", kfpu);
1367
1368 if ((kfpu->kfpu_flags & KFPU_F_INITIALIZED) == 0)
1369 kernel_fpu_fpstate_init(kfpu);
1370
1371 kfpu->kfpu_curthread = curthread;
1372 }
1373
1374 /*
1375 * Not all threads may have an active LWP. If they do and we're not
1376 * going to re-use the LWP, then we should go ahead and save the state.
1377 * We must also note that the fpu is now being used by the kernel and
1378 * therefore we do not want to manage the fpu state via the user-level
1379 * thread's context handlers.
1380 *
1381 * We might have already saved once (due to a prior use of the kernel
1382 * FPU or another code path) so FPU_VALID could be set. This is handled
1383 * by fp_save, as is the FPU_EN check.
1384 */
1385 ctx = installctx_preallocate();
1386 kpreempt_disable();
1387 if (pl != NULL) {
1388 if ((flags & KFPU_USE_LWP) == 0)
1389 fp_save(&pl->lwp_pcb.pcb_fpu);
1390 pl->lwp_pcb.pcb_fpu.fpu_flags |= FPU_KERNEL;
1391 }
1392
1393 /*
1394 * Set the context operations for kernel FPU usage. Note that this is
1395 * done with a preallocated buffer and under kpreempt_disable because
1396 * without a preallocated buffer, installctx does a sleeping
1397 * allocation. We haven't finished initializing our kernel FPU state
1398 * yet, and in the rare case that we happen to save/restore just as
1399 * installctx() exits its own kpreempt_enable() internal call, we
1400 * guard against restoring an uninitialized buffer (0xbaddcafe).
1401 */
1402 installctx(curthread, kfpu, kernel_fpu_ctx_save, kernel_fpu_ctx_restore,
1403 NULL, NULL, NULL, NULL, ctx);
1404
1405 curthread->t_flag |= T_KFPU;
1406
1407 if ((flags & KFPU_USE_LWP) == KFPU_USE_LWP) {
1408 /*
1409 * For pure kernel threads with an LWP, we can use the LWP's
1410 * pcb_fpu to save/restore context.
1411 */
1412 fpu_ctx_t *pf = &pl->lwp_pcb.pcb_fpu;
1413
1414 VERIFY(curthread->t_procp->p_flag & SSYS);
1415 VERIFY(kfpu == NULL);
1416 ASSERT((pf->fpu_flags & FPU_EN) == 0);
1417
1418 /* Always restore the fpu to the initial state. */
1419 if (fp_save_mech == FP_XSAVE)
1420 pf->fpu_xsave_mask = XFEATURE_FP_ALL;
1421 fpinit();
1422 pf->fpu_flags = FPU_EN | FPU_KERNEL;
1423 } else {
1424 /* initialize the kfpu state */
1425 kernel_fpu_ctx_restore(kfpu);
1426 }
1427 kpreempt_enable();
1428 }
1429
1430 void
1431 kernel_fpu_end(kfpu_state_t *kfpu, uint_t flags)
1432 {
1433 ulong_t iflags;
1434
1435 if ((curthread->t_flag & T_KFPU) == 0) {
1436 panic("curthread attempting to clear kernel FPU state "
1437 "without using it");
1438 }
1439
1440 /*
1441 * General comments on why the rest of this function is structured the
1442 * way it is. Be aware that there is a lot of subtlety here.
1443 *
1444 * If a user-level thread ever uses the fpu while in the kernel, then
1445 * we cannot call fpdisable since that does STTS. That will set the
1446 * ts bit in %cr0 which will cause an exception if anything touches the
1447 * fpu. However, the user-level context switch handler (fpsave_ctxt)
|