`mlxcx` -> QSFP28 ================= [root@server4 (us-mobile-1) /zones/global/mlxcx]# iperf3 -c 10.100.4.2 -P 4 Connecting to host 10.100.4.2, port 5201 [ 5] local 10.100.4.1 port 55912 connected to 10.100.4.2 port 5201 [ 7] local 10.100.4.1 port 51135 connected to 10.100.4.2 port 5201 [ 9] local 10.100.4.1 port 36177 connected to 10.100.4.2 port 5201 [ 11] local 10.100.4.1 port 47296 connected to 10.100.4.2 port 5201 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.01 sec 143 MBytes 1.19 Gbits/sec [ 7] 0.00-1.01 sec 103 MBytes 860 Mbits/sec [ 9] 0.00-1.01 sec 145 MBytes 1.20 Gbits/sec [ 11] 0.00-1.01 sec 144 MBytes 1.20 Gbits/sec [SUM] 0.00-1.01 sec 536 MBytes 4.46 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 1.01-2.01 sec 147 MBytes 1.23 Gbits/sec [ 7] 1.01-2.01 sec 107 MBytes 900 Mbits/sec [ 9] 1.01-2.01 sec 145 MBytes 1.22 Gbits/sec [ 11] 1.01-2.01 sec 146 MBytes 1.23 Gbits/sec [SUM] 1.01-2.01 sec 545 MBytes 4.58 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 2.01-3.01 sec 146 MBytes 1.22 Gbits/sec [ 7] 2.01-3.01 sec 106 MBytes 884 Mbits/sec [ 9] 2.01-3.01 sec 146 MBytes 1.22 Gbits/sec [ 11] 2.01-3.01 sec 146 MBytes 1.22 Gbits/sec [SUM] 2.01-3.01 sec 544 MBytes 4.56 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 3.01-4.01 sec 138 MBytes 1.16 Gbits/sec [ 7] 3.01-4.01 sec 98.4 MBytes 825 Mbits/sec [ 9] 3.01-4.01 sec 139 MBytes 1.17 Gbits/sec [ 11] 3.01-4.01 sec 140 MBytes 1.17 Gbits/sec [SUM] 3.01-4.01 sec 516 MBytes 4.33 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 4.01-5.00 sec 139 MBytes 1.17 Gbits/sec [ 7] 4.01-5.00 sec 98.6 MBytes 831 Mbits/sec [ 9] 4.01-5.00 sec 139 MBytes 1.17 Gbits/sec [ 11] 4.01-5.00 sec 139 MBytes 1.17 Gbits/sec [SUM] 4.01-5.00 sec 515 MBytes 4.34 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 5.00-6.01 sec 142 MBytes 1.19 Gbits/sec [ 7] 5.00-6.01 sec 101 MBytes 843 Mbits/sec [ 9] 5.00-6.01 sec 142 MBytes 1.19 Gbits/sec [ 11] 5.00-6.01 sec 142 MBytes 1.19 Gbits/sec [SUM] 5.00-6.01 sec 528 MBytes 4.41 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 6.01-7.01 sec 142 MBytes 1.20 Gbits/sec [ 7] 6.01-7.01 sec 101 MBytes 850 Mbits/sec [ 9] 6.01-7.01 sec 142 MBytes 1.20 Gbits/sec [ 11] 6.01-7.01 sec 142 MBytes 1.20 Gbits/sec [SUM] 6.01-7.01 sec 528 MBytes 4.44 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 7.01-8.01 sec 144 MBytes 1.21 Gbits/sec [ 7] 7.01-8.01 sec 104 MBytes 868 Mbits/sec [ 9] 7.01-8.01 sec 144 MBytes 1.21 Gbits/sec [ 11] 7.01-8.01 sec 144 MBytes 1.21 Gbits/sec [SUM] 7.01-8.01 sec 535 MBytes 4.49 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 8.01-9.00 sec 144 MBytes 1.21 Gbits/sec [ 7] 8.01-9.00 sec 102 MBytes 859 Mbits/sec [ 9] 8.01-9.00 sec 144 MBytes 1.21 Gbits/sec [ 11] 8.01-9.00 sec 144 MBytes 1.21 Gbits/sec [SUM] 8.01-9.00 sec 533 MBytes 4.49 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 9.00-10.00 sec 145 MBytes 1.21 Gbits/sec [ 7] 9.00-10.00 sec 102 MBytes 855 Mbits/sec [ 9] 9.00-10.00 sec 145 MBytes 1.21 Gbits/sec [ 11] 9.00-10.00 sec 145 MBytes 1.21 Gbits/sec [SUM] 9.00-10.00 sec 537 MBytes 4.49 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.00 sec 1.40 GBytes 1.20 Gbits/sec sender [ 5] 0.00-10.01 sec 1.40 GBytes 1.20 Gbits/sec receiver [ 7] 0.00-10.00 sec 1023 MBytes 858 Mbits/sec sender [ 7] 0.00-10.01 sec 1023 MBytes 857 Mbits/sec receiver [ 9] 0.00-10.00 sec 1.40 GBytes 1.20 Gbits/sec sender [ 9] 0.00-10.01 sec 1.40 GBytes 1.20 Gbits/sec receiver [ 11] 0.00-10.00 sec 1.40 GBytes 1.20 Gbits/sec sender [ 11] 0.00-10.01 sec 1.40 GBytes 1.20 Gbits/sec receiver [SUM] 0.00-10.00 sec 5.19 GBytes 4.46 Gbits/sec sender [SUM] 0.00-10.01 sec 5.19 GBytes 4.46 Gbits/sec receiver iperf Done. `mlxcx` -> QSFP56 ================= [root@server4 (us-mobile-1) /zones/global/mlxcx]# iperf3 -c 10.100.2.2 -P 4 Connecting to host 10.100.2.2, port 5201 [ 5] local 10.100.2.1 port 46378 connected to 10.100.2.2 port 5201 [ 7] local 10.100.2.1 port 52520 connected to 10.100.2.2 port 5201 [ 9] local 10.100.2.1 port 64202 connected to 10.100.2.2 port 5201 [ 11] local 10.100.2.1 port 48983 connected to 10.100.2.2 port 5201 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.01 sec 132 MBytes 1.10 Gbits/sec [ 7] 0.00-1.01 sec 131 MBytes 1.09 Gbits/sec [ 9] 0.00-1.01 sec 132 MBytes 1.10 Gbits/sec [ 11] 0.00-1.01 sec 131 MBytes 1.09 Gbits/sec [SUM] 0.00-1.01 sec 526 MBytes 4.38 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 1.01-2.00 sec 130 MBytes 1.10 Gbits/sec [ 7] 1.01-2.00 sec 130 MBytes 1.10 Gbits/sec [ 9] 1.01-2.00 sec 130 MBytes 1.10 Gbits/sec [ 11] 1.01-2.00 sec 130 MBytes 1.10 Gbits/sec [SUM] 1.01-2.00 sec 520 MBytes 4.39 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 2.00-3.00 sec 130 MBytes 1.09 Gbits/sec [ 7] 2.00-3.00 sec 130 MBytes 1.09 Gbits/sec [ 9] 2.00-3.00 sec 130 MBytes 1.09 Gbits/sec [ 11] 2.00-3.00 sec 130 MBytes 1.09 Gbits/sec [SUM] 2.00-3.00 sec 520 MBytes 4.37 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 3.00-4.01 sec 129 MBytes 1.07 Gbits/sec [ 7] 3.00-4.01 sec 129 MBytes 1.07 Gbits/sec [ 9] 3.00-4.01 sec 129 MBytes 1.07 Gbits/sec [ 11] 3.00-4.01 sec 129 MBytes 1.07 Gbits/sec [SUM] 3.00-4.01 sec 515 MBytes 4.28 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [ 7] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [ 9] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [ 11] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [SUM] 4.01-5.00 sec 520 MBytes 4.39 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 5.00-6.00 sec 130 MBytes 1.09 Gbits/sec [ 7] 5.00-6.00 sec 130 MBytes 1.09 Gbits/sec [ 9] 5.00-6.00 sec 130 MBytes 1.09 Gbits/sec [ 11] 5.00-6.00 sec 130 MBytes 1.09 Gbits/sec [SUM] 5.00-6.00 sec 520 MBytes 4.35 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [ 7] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [ 9] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [ 11] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 6.00-7.00 sec 495 MBytes 4.15 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [ 7] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [ 9] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [ 11] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 7.00-8.00 sec 495 MBytes 4.16 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 8.00-9.01 sec 125 MBytes 1.04 Gbits/sec [ 7] 8.00-9.01 sec 125 MBytes 1.04 Gbits/sec [ 9] 8.00-9.01 sec 125 MBytes 1.04 Gbits/sec [ 11] 8.00-9.01 sec 125 MBytes 1.04 Gbits/sec [SUM] 8.00-9.01 sec 500 MBytes 4.17 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 9.01-10.01 sec 125 MBytes 1.05 Gbits/sec [ 7] 9.01-10.01 sec 125 MBytes 1.05 Gbits/sec [ 9] 9.01-10.01 sec 125 MBytes 1.05 Gbits/sec [ 11] 9.01-10.01 sec 125 MBytes 1.05 Gbits/sec [SUM] 9.01-10.01 sec 500 MBytes 4.19 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec sender [ 5] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec receiver [ 7] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec sender [ 7] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec receiver [ 9] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec sender [ 9] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec receiver [ 11] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec sender [ 11] 0.00-10.01 sec 1.25 GBytes 1.07 Gbits/sec receiver [SUM] 0.00-10.01 sec 4.99 GBytes 4.28 Gbits/sec sender [SUM] 0.00-10.01 sec 4.99 GBytes 4.28 Gbits/sec receiver iperf Done. `mlxcx` lockstat on QSFP56 [root@server4 (us-mobile-1) /zones/global/mlxcx]# lockstat iperf3 -c 10.100.2.2 -P 4 Connecting to host 10.100.2.2, port 5201 [ 6] local 10.100.2.1 port 59498 connected to 10.100.2.2 port 5201 [ 8] local 10.100.2.1 port 44827 connected to 10.100.2.2 port 5201 [ 10] local 10.100.2.1 port 62435 connected to 10.100.2.2 port 5201 [ 12] local 10.100.2.1 port 41605 connected to 10.100.2.2 port 5201 [ ID] Interval Transfer Bitrate [ 6] 0.00-1.00 sec 123 MBytes 1.03 Gbits/sec [ 8] 0.00-1.00 sec 125 MBytes 1.05 Gbits/sec [ 10] 0.00-1.00 sec 123 MBytes 1.03 Gbits/sec [ 12] 0.00-1.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 0.00-1.00 sec 496 MBytes 4.15 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 1.00-2.01 sec 126 MBytes 1.05 Gbits/sec [ 8] 1.00-2.01 sec 126 MBytes 1.05 Gbits/sec [ 10] 1.00-2.01 sec 126 MBytes 1.05 Gbits/sec [ 12] 1.00-2.01 sec 126 MBytes 1.05 Gbits/sec [SUM] 1.00-2.01 sec 505 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 2.01-3.01 sec 125 MBytes 1.05 Gbits/sec [ 8] 2.01-3.01 sec 125 MBytes 1.05 Gbits/sec [ 10] 2.01-3.01 sec 125 MBytes 1.05 Gbits/sec [ 12] 2.01-3.01 sec 125 MBytes 1.05 Gbits/sec [SUM] 2.01-3.01 sec 500 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 3.01-4.01 sec 124 MBytes 1.04 Gbits/sec [ 8] 3.01-4.01 sec 124 MBytes 1.04 Gbits/sec [ 10] 3.01-4.01 sec 124 MBytes 1.04 Gbits/sec [ 12] 3.01-4.01 sec 124 MBytes 1.04 Gbits/sec [SUM] 3.01-4.01 sec 495 MBytes 4.16 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 4.01-5.01 sec 129 MBytes 1.08 Gbits/sec [ 8] 4.01-5.01 sec 129 MBytes 1.08 Gbits/sec [ 10] 4.01-5.01 sec 129 MBytes 1.08 Gbits/sec [ 12] 4.01-5.01 sec 129 MBytes 1.08 Gbits/sec [SUM] 4.01-5.01 sec 515 MBytes 4.32 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 5.01-6.00 sec 124 MBytes 1.04 Gbits/sec [ 8] 5.01-6.00 sec 124 MBytes 1.04 Gbits/sec [ 10] 5.01-6.00 sec 124 MBytes 1.04 Gbits/sec [ 12] 5.01-6.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 5.01-6.00 sec 495 MBytes 4.16 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [ 8] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [ 10] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [ 12] 6.00-7.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 6.00-7.00 sec 495 MBytes 4.17 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 7.00-8.00 sec 125 MBytes 1.05 Gbits/sec [ 8] 7.00-8.00 sec 125 MBytes 1.05 Gbits/sec [ 10] 7.00-8.00 sec 125 MBytes 1.05 Gbits/sec [ 12] 7.00-8.00 sec 125 MBytes 1.05 Gbits/sec [SUM] 7.00-8.00 sec 500 MBytes 4.19 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 8.00-9.00 sec 125 MBytes 1.05 Gbits/sec [ 8] 8.00-9.00 sec 125 MBytes 1.05 Gbits/sec [ 10] 8.00-9.00 sec 125 MBytes 1.05 Gbits/sec [ 12] 8.00-9.00 sec 125 MBytes 1.05 Gbits/sec [SUM] 8.00-9.00 sec 500 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 9.00-10.00 sec 125 MBytes 1.05 Gbits/sec [ 8] 9.00-10.00 sec 125 MBytes 1.05 Gbits/sec [ 10] 9.00-10.00 sec 125 MBytes 1.05 Gbits/sec [ 12] 9.00-10.00 sec 125 MBytes 1.05 Gbits/sec [SUM] 9.00-10.00 sec 500 MBytes 4.19 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 6] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec sender [ 6] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec receiver [ 8] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec sender [ 8] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec receiver [ 10] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec sender [ 10] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec receiver [ 12] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec sender [ 12] 0.00-10.00 sec 1.22 GBytes 1.05 Gbits/sec receiver [SUM] 0.00-10.00 sec 4.88 GBytes 4.19 Gbits/sec sender [SUM] 0.00-10.00 sec 4.88 GBytes 4.19 Gbits/sec receiver iperf Done. Adaptive mutex spin: 376179 events in 10.655 seconds (35304 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 62268 17% 17% 0.00 684 0xfffffe235bd40b58 mlxcx_buf_return+0x8c 58814 16% 32% 0.00 641 0xfffffe2358117b58 mlxcx_buf_return+0x8c 58175 15% 48% 0.00 624 0xfffffe2358117558 mlxcx_buf_return+0x8c 51813 14% 61% 0.00 622 0xfffffe235bd40558 mlxcx_buf_return+0x8c 15538 4% 66% 0.00 613 0xfffffe235bd438f8 mlxcx_process_cq+0x207 15531 4% 70% 0.00 596 0xfffffe2357349638 mlxcx_process_cq+0x207 14774 4% 74% 0.00 619 0xfffffe235bd431f8 mlxcx_process_cq+0x207 14074 4% 77% 0.00 662 0xfffffe235a379bb8 mlxcx_process_cq+0x207 11959 3% 81% 0.00 479 0xfffffe2358117b58 mlxcx_buf_take+0x2f 11893 3% 84% 0.00 474 0xfffffe235bd40b58 mlxcx_buf_take+0x2f 9630 3% 86% 0.00 438 0xfffffe235bd40558 mlxcx_buf_take+0x2f 9458 3% 89% 0.00 456 0xfffffe2358117558 mlxcx_buf_take+0x2f 4992 1% 90% 0.00 598 0xfffffe235bd438f8 mlxcx_sq_add_buffer+0x469 4894 1% 91% 0.00 537 0xfffffe235a379bb8 mlxcx_sq_add_buffer+0x469 4519 1% 93% 0.00 572 0xfffffe2357349638 mlxcx_sq_add_buffer+0x469 4119 1% 94% 0.00 533 0xfffffe235bd431f8 mlxcx_sq_add_buffer+0x469 2510 1% 94% 0.00 809 0xfffffe22d6186680 hook_run+0x34 1739 0% 95% 0.00 1012 0xfffffe22e170b350 squeue_enter+0x37a 1695 0% 95% 0.00 1031 0xfffffe22e170b7d0 squeue_enter+0x37a 1572 0% 96% 0.00 999 0xfffffe22e170bb90 squeue_enter+0x37a 1541 0% 96% 0.00 954 0xfffffe22d6186680 hook_run+0xd1 1538 0% 97% 0.00 1028 0xfffffe2314a86e90 squeue_enter+0x37a 1088 0% 97% 0.00 99951 overlay_target_lock cv_timedwait_hires+0xf0 1060 0% 97% 0.00 991 0xfffffe22e170b7d0 squeue_drain+0x18b 1053 0% 97% 0.00 575 0xfffffe235d8fc358 mlxcx_buf_loan+0x75 905 0% 98% 0.00 1651 0xfffffe235d8fc358 mlxcx_buf_return+0x8c 791 0% 98% 0.00 51771 overlay_target_lock overlay_target_lookup_request+0x47 728 0% 98% 0.00 123837 0xfffffe2397bd99c8 cv_wait+0x78 664 0% 98% 0.00 39471 0xfffffe2397bd99c8 taskq_thread+0x2cd 582 0% 98% 0.00 856 0xfffffe22e170b7d0 squeue_enter+0x53 487 0% 98% 0.00 894 0xfffffe2314a86e90 squeue_enter+0x53 478 0% 99% 0.00 34456 0xfffffe23970c3c00 taskq_dispatch_ent+0x51 429 0% 99% 0.00 953 0xfffffe22e170bb90 squeue_enter+0x53 428 0% 99% 0.00 815 0xfffffe22e170b350 squeue_enter+0x53 369 0% 99% 0.00 24163 0xfffffe23970c3c00 cv_wait+0x78 333 0% 99% 0.00 13886 0xfffffe2397bd99c8 taskq_dispatch+0x174 327 0% 99% 0.00 27527 0xfffffe23970c3c00 taskq_thread+0x2cd 325 0% 99% 0.00 1108 0xfffffe2314a86e90 squeue_drain+0x18b 308 0% 99% 0.00 1927 0xfffffe26d44b5980 cv_wait+0x78 297 0% 99% 0.00 1019 0xfffffe22e170b350 squeue_drain+0x18b 254 0% 99% 0.00 913 0xfffffe22e170bb90 squeue_drain+0x18b 136 0% 99% 0.00 534 0xfffffe229dae01d0 kmem_depot_alloc+0x88 131 0% 99% 0.00 552 0xfffffe235e7abe18 mlxcx_buf_loan+0x75 110 0% 100% 0.00 418 0xfffffe26da21cf38 tcp_output+0x63 104 0% 100% 0.00 490 0xfffffe26d860bf38 tcp_output+0x63 96 0% 100% 0.00 444 0xfffffe26d81816f8 tcp_output+0x63 89 0% 100% 0.00 622 0xfffffe235d8fc718 mlxcx_buf_loan+0x75 81 0% 100% 0.00 494 0xfffffe23e0e32738 tcp_output+0x63 76 0% 100% 0.00 46472 0xfffffe2397bd99c8 taskq_ent_free+0x6d 64 0% 100% 0.00 1349 0xfffffe235e7abe18 mlxcx_buf_return+0x8c 63 0% 100% 0.00 434126 0xfffffe22d620c2b0 gcpu_ntv_mca_poll_wrapper+0x72 61 0% 100% 0.00 148486 0xfffffe22e1717da8 cv_timedwait_hires+0xf0 58 0% 100% 0.00 536 0xfffffe229dacc1d0 kmem_depot_alloc+0x88 51 0% 100% 0.00 2271 0xfffffe26d44b7180 cv_wait+0x78 49 0% 100% 0.00 1782 0xfffffe235d8fc718 mlxcx_buf_return+0x8c 47 0% 100% 0.00 1466 0xfffffe256a64b040 mac_rx_soft_ring_drain+0x115 44 0% 100% 0.00 8366 0xfffffe23976bde88 vdev_queue_io+0x66 43 0% 100% 0.00 8135 0xfffffe23970c3540 cv_wait+0x78 36 0% 100% 0.00 852 sqset_lock ip_squeue_random+0x22 36 0% 100% 0.00 6262 0xfffffe2397bd99c8 taskq_ent_alloc+0xe4 34 0% 100% 0.00 2407 0xfffffe257a19a740 cv_wait+0x78 32 0% 100% 0.00 1991 0xfffffe256a652200 cv_wait+0x78 32 0% 100% 0.00 6919 0xfffffe2397122e88 vdev_queue_io+0x66 27 0% 100% 0.00 1654 0xfffffe26d44b5980 mac_rx_soft_ring_process+0x53 24 0% 100% 0.00 1288 0xfffffe256a64b040 cv_wait+0x78 23 0% 100% 0.00 1201 0xfffffe26d81816f8 tcp_sendmsg+0x83 23 0% 100% 0.00 3455 0xfffffe23976bde88 vdev_queue_io_done+0x30 23 0% 100% 0.00 3232 0xfffffe2397122e88 vdev_queue_io_done+0x30 22 0% 100% 0.00 15493 0xfffffe23976bde88 vdev_queue_io_done+0x75 20 0% 100% 0.00 42366 0xfffffe2319ddb408 cv_wait+0x78 19 0% 100% 0.00 2245 0xfffffe26d44b2c40 cv_wait+0x78 19 0% 100% 0.00 2714 0xfffffe23970c3540 taskq_dispatch_ent+0x51 18 0% 100% 0.00 1954 0xfffffe22d6a50268 cv_wait+0x78 16 0% 100% 0.00 1778 0xfffffe26d8181000 squeue_enter+0x4cd 16 0% 100% 0.00 933 0xfffffe26d7d3f620 i_dls_head_hold+0x1f 16 0% 100% 0.00 3433 0xfffffe2397122e88 vdev_queue_io_done+0x75 16 0% 100% 0.00 12071 0xfffffe23970c3540 taskq_thread+0x2cd 16 0% 100% 0.00 17773 0xfffffe2319ddb408 taskq_thread+0x2cd 14 0% 100% 0.00 1041 0xfffffe2315918df0 vdev_queue_pending_remove+0x51 13 0% 100% 0.00 1108 zone_pdata zfs_zone_zio_done+0xa9 13 0% 100% 0.00 356 0xfffffe26da21c840 ipcl_classify_v4+0x25d 13 0% 100% 0.00 917 0xfffffe26d860bf38 tcp_sendmsg+0x83 13 0% 100% 0.00 1803 0xfffffe256a64b040 mac_rx_srs_process+0x5c 13 0% 100% 0.00 1411 0xfffffe22d6c82f80 stop+0x2eb 13 0% 100% 0.00 1904 0xfffffe229dacc240 kmem_cache_free+0x78 12 0% 100% 0.00 625 0xfffffe26da21cf38 tcp_sendmsg+0x83 12 0% 100% 0.00 473 0xfffffe23e0e32738 tcp_sendmsg+0x83 12 0% 100% 0.00 761 0xfffffe2315918df0 vdev_queue_io_remove+0x86 12 0% 100% 0.00 1553 0xfffffe22d6a50c88 taskq_thread+0x2cd 12 0% 100% 0.00 969 0xfffffe22d6a50c88 taskq_dispatch+0x174 11 0% 100% 0.00 6832 0xfffffe23976bde88 vdev_queue_aggregate+0x43c 10 0% 100% 0.00 472 0xfffffe23e0e32738 tcp_wput_data+0x401 10 0% 100% 0.00 820 0xfffffe2315918df0 vdev_queue_pending_add+0x51 10 0% 100% 0.00 2860 0xfffffe22d6a50268 taskq_dispatch_ent+0x51 10 0% 100% 0.00 554 0xfffffcc2673f05c8 page_ctr_sub+0x6b 9 0% 100% 0.00 676 0xfffffe26d860b840 squeue_enter+0x4cd 9 0% 100% 0.00 488 0xfffffe26d860b840 ipcl_classify_v4+0x25d 9 0% 100% 0.00 1018 0xfffffe26d7d3f620 i_dls_head_rele+0x18 8 0% 100% 0.00 553 0xfffffe23e0e32040 ipcl_classify_v4+0x25d 8 0% 100% 0.00 2553 0xfffffe22d6a50268 taskq_thread+0x2cd 7 0% 100% 0.00 631 0xfffffe26d8448bd0 ire_route_recursive_dstonly_v4+0x68 7 0% 100% 0.00 428 0xfffffe23e0e32040 tcp_sendmsg+0xb8 6 0% 100% 0.00 524 0xfffffe26da21cf38 tcp_wput_data+0x401 6 0% 100% 0.00 395 0xfffffe22e1ca2380 getf_gen+0x63 5 0% 100% 0.00 10295 zfs_disk_lock zfs_zone_zio_done+0xe1 5 0% 100% 0.00 2066 0xfffffe26d83c5c90 dls_accept_common+0x80 5 0% 100% 0.00 847 0xfffffe26d8181000 ipcl_classify_v4+0x25d 5 0% 100% 0.00 525 0xfffffe2397122f30 vdev_stat_update+0xab 5 0% 100% 0.00 4637 0xfffffe2397122e88 vdev_queue_aggregate+0x43c 4 0% 100% 0.00 273 0xfffffe26d860b840 tcp_sendmsg+0xb8 4 0% 100% 0.00 543 0xfffffe26d81816f8 tcp_wput_data+0x401 4 0% 100% 0.00 957 0xfffffe26d817d2a8 strread+0x2e4 4 0% 100% 0.00 4428 0xfffffe2397bd4000 metaslab_group_alloc_normal+0x15f 4 0% 100% 0.00 916 0xfffffe2319ebec98 bd_runq_exit+0x53 4 0% 100% 0.00 551 0xfffffe2315918df0 vdev_queue_io_add+0x86 4 0% 100% 0.00 3464 0xfffffe23155c2800 metaslab_group_alloc_normal+0x1b4 3 0% 100% 0.00 1128 zone_pdata add_zone_iop+0x2d 3 0% 100% 0.00 1225 0xfffffe26da21c840 tcp_sendmsg+0xb8 3 0% 100% 0.00 317 0xfffffe26da21c840 squeue_enter+0x4cd 3 0% 100% 0.00 805 0xfffffe26d860bf38 tcp_wput_data+0x401 3 0% 100% 0.00 367 0xfffffe26d8181000 tcp_sendmsg+0xb8 3 0% 100% 0.00 841 0xfffffe26d817d2a8 putnext+0x373 3 0% 100% 0.00 640 0xfffffe257a19a740 mac_rx_soft_ring_drain+0x115 3 0% 100% 0.00 243 0xfffffe23e0e32040 squeue_drain+0x255 3 0% 100% 0.00 354 0xfffffe23e0e32040 squeue_enter+0x4cd 3 0% 100% 0.00 1957 0xfffffe2397bd4000 metaslab_group_alloc+0x63 3 0% 100% 0.00 1345 0xfffffe231813c4f0 bge_chip_factotum+0x5e 3 0% 100% 0.00 968 0xfffffe231813c4f0 bge_intr+0xec 3 0% 100% 0.00 511 0xfffffe22e1ca2380 releasef+0x53 3 0% 100% 0.00 1970 0xfffffe22d6a50c88 cv_wait+0x78 3 0% 100% 0.00 891 0xfffffe229de0f1d0 kmem_depot_alloc+0x88 3 0% 100% 0.00 1379 0xfffffe229dae0240 kmem_cache_free+0x78 2 0% 100% 0.00 381 0xfffffe26da21c840 squeue_drain+0x255 2 0% 100% 0.00 826 0xfffffe26d44b7180 mac_rx_soft_ring_process+0x53 2 0% 100% 0.00 721 0xfffffe2397568a68 zio_remove_child+0x47 2 0% 100% 0.00 405 0xfffffe235e7bb608 mlxcx_intr_n+0x10b 2 0% 100% 0.00 4598 0xfffffe231c3eb4f0 bge_chip_factotum+0x5e 2 0% 100% 0.00 486 0xfffffe231c3eb4f0 bge_intr+0x32 2 0% 100% 0.00 711 0xfffffe2319ebec98 bd_submit+0x72 2 0% 100% 0.00 1006 0xfffffe2316f86c18 bd_runq_exit+0x53 2 0% 100% 0.00 641 0xfffffe2316f86c18 bd_submit+0x72 2 0% 100% 0.00 1242 0xfffffe231551f000 metaslab_group_alloc_normal+0x1b4 2 0% 100% 0.00 1397 0xfffffe229dacc2c0 kmem_cache_free+0x78 1 0% 100% 0.00 541 zone_pdata zfs_zone_zio_start+0x3c 1 0% 100% 0.00 9990 pidlock thread_exit+0x5c 1 0% 100% 0.00 1292 pidlock sigcld_repost+0x32 1 0% 100% 0.00 792 htable_mutex+0x158 htable_release+0x103 1 0% 100% 0.00 1333 pidlock cv_wait_sig_swap_core+0x174 1 0% 100% 0.00 370 0xfffffe26d860b840 squeue_drain+0x255 1 0% 100% 0.00 1793 0xfffffe256a64b040 mac_rx_srs_drain+0x22d 1 0% 100% 0.00 1062 0xfffffe24224025b0 lwp_unpark+0x54 1 0% 100% 0.00 822 0xfffffe2423a168d0 zio_remove_child+0x47 1 0% 100% 0.00 2274 0xfffffe2414be16d0 pollwakeup+0x193 1 0% 100% 0.00 191 0xfffffe256a652200 mac_rx_soft_ring_drain+0x115 1 0% 100% 0.00 972 0xfffffe2396ae7d40 multilist_sublist_lock+0x27 1 0% 100% 0.00 390 0xfffffe2396ae7d80 multilist_sublist_lock+0x27 1 0% 100% 0.00 371 0xfffffe235d8f6088 mlxcx_mac_ring_intr_enable+0x2b 1 0% 100% 0.00 441 0xfffffe235d8fc718 mlxcx_buf_take_n+0x36 1 0% 100% 0.00 221 0xfffffe23975798a0 zio_notify_parent+0x43 1 0% 100% 0.00 190 0xfffffe23976b75f0 zio_notify_parent+0x43 1 0% 100% 0.00 1532 0xfffffe2397461af0 zio_dva_throttle_done+0x52 1 0% 100% 0.00 952 0xfffffe23976bdf30 vdev_stat_update+0xab 1 0% 100% 0.00 190 0xfffffe23976cc3f8 dnode_diduse_space+0x35 1 0% 100% 0.00 461 0xfffffe2399fa34c0 zio_remove_child+0x47 1 0% 100% 0.00 330 0xfffffe2399f8c9b0 zio_remove_child+0x47 1 0% 100% 0.00 1252 0xfffffe22d6c82f80 syslwp_continue+0x28 1 0% 100% 0.00 4889 0xfffffe22d6c82940 cv_wait_sig+0x166 1 0% 100% 0.00 782 0xfffffe22d6c82880 lookuppnatcred+0x74 1 0% 100% 0.00 350 0xfffffe22d6c82680 clock_tick_process+0xd0 1 0% 100% 0.00 320 0xfffffe22d6c82940 stop+0x2eb 1 0% 100% 0.00 1112 0xfffffe22d6c82f80 clock_tick_process+0xd0 1 0% 100% 0.00 5451 0xfffffe22d6c82f80 cv_wait_sig+0x166 1 0% 100% 0.00 912 0xfffffe22d6a03ac0 anon_getpage+0x1ee 1 0% 100% 0.00 391 0xfffffe22d6a037c0 anon_getpage+0x1ee 1 0% 100% 0.00 451 0xfffffe229e460100 untimeout_generic+0x65 1 0% 100% 0.00 1162 0xfffffe229dadc1d0 kmem_depot_alloc+0x88 1 0% 100% 0.00 1113 0xfffffe229dada1d0 kmem_depot_alloc+0x88 1 0% 100% 0.00 861 0xfffffcc2673ed150 page_get_mnode_freelist+0x12b 1 0% 100% 0.00 871 0xfffffcc2673ed140 page_get_mnode_freelist+0x12b 1 0% 100% 0.00 361 0xfffffcc2673ed138 page_get_mnode_freelist+0x12b 1 0% 100% 0.00 1102 0xfffffcc2673ed190 page_get_mnode_freelist+0x12b 1 0% 100% 0.00 1483 0xfffffcc2673f0598 page_ctr_sub+0x6b 1 0% 100% 0.00 1113 0xfffffe229e1467c0 x86_hm_enter+0x34 1 0% 100% 0.00 441 0xfffffe229e145800 x86_hm_enter+0x34 1 0% 100% 0.00 1051 0xfffffe229dae01d0 kmem_depot_free+0x35 1 0% 100% 0.00 2535 0xfffffe229dae0380 kmem_cache_free+0x78 1 0% 100% 0.00 2134 0xfffffe229e148028 x86_hm_enter+0x34 1 0% 100% 0.00 922 0xfffffe229e146ab0 x86_hm_enter+0x34 1 0% 100% 0.00 561 0xfffffe229e146878 x86_hm_enter+0x34 1 0% 100% 0.00 401 0xfffffe229e1486c8 x86_hm_enter+0x34 1 0% 100% 0.00 421 0xfffffe229e148f28 x86_hm_enter+0x34 1 0% 100% 0.00 391 0xfffffe229e45e000 untimeout_generic+0x65 1 0% 100% 0.00 661 0xfffffe23155c2800 metaslab_free_concrete+0xc3 1 0% 100% 0.00 461 0xfffffe2315905dd0 zio_dva_throttle+0xa8 1 0% 100% 0.00 1323 0xfffffe2315539000 metaslab_group_alloc_normal+0x1b4 1 0% 100% 0.00 661 0xfffffe2315539000 metaslab_free_concrete+0xc3 1 0% 100% 0.00 8757 0xfffffe22e1717da8 compute_min_max_threads+0x14 1 0% 100% 0.00 651 0xfffffe22e29611d0 kmem_depot_alloc+0x88 1 0% 100% 0.00 16380 0xfffffe231382d000 metaslab_group_alloc_normal+0x1b4 1 0% 100% 0.00 190 0xfffffe2319ebec98 bd_sched+0xca 1 0% 100% 0.00 1102 0xfffffe23178789d8 zio_remove_child+0x47 1 0% 100% 0.00 461 0xfffffe2319c14d00 metaslab_class_throttle_unreserve+0x1d 1 0% 100% 0.00 1242 0xfffffe23581179d8 mlxcx_buf_take+0x2f 1 0% 100% 0.00 481 0xfffffe2358aae8e0 fifo_inactive+0xc2 ------------------------------------------------------------------------------- Adaptive mutex block: 1504 events in 10.655 seconds (141 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 301 20% 20% 0.00 13687 0xfffffe26d44b5980 cv_wait+0x78 183 12% 32% 0.00 13893 0xfffffe22d6186680 hook_run+0x34 174 12% 44% 0.00 3296 0xfffffe235bd438f8 mlxcx_process_cq+0x207 83 6% 49% 0.00 12246 0xfffffe2358117b58 mlxcx_buf_take+0x2f 77 5% 54% 0.00 13045 0xfffffe235bd40b58 mlxcx_buf_take+0x2f 58 4% 58% 0.00 11368 0xfffffe22d6186680 hook_run+0xd1 52 3% 62% 0.00 6659 0xfffffe2358117558 mlxcx_buf_return+0x8c 51 3% 65% 0.00 6714 0xfffffe26d44b7180 cv_wait+0x78 51 3% 68% 0.00 5455 0xfffffe235bd40b58 mlxcx_buf_return+0x8c 50 3% 72% 0.00 16371 0xfffffe2314a86e90 squeue_enter+0x37a 44 3% 75% 0.00 7430 0xfffffe235a379bb8 mlxcx_process_cq+0x207 34 2% 77% 0.00 6347 0xfffffe257a19a740 cv_wait+0x78 32 2% 79% 0.00 6250 0xfffffe256a652200 cv_wait+0x78 30 2% 81% 0.00 14349 0xfffffe235d8fc358 mlxcx_buf_return+0x8c 25 2% 83% 0.00 13274 0xfffffe2357349638 mlxcx_sq_add_buffer+0x469 24 2% 84% 0.00 11749 0xfffffe256a64b040 mac_rx_soft_ring_drain+0x115 23 2% 86% 0.00 12901 0xfffffe235bd438f8 mlxcx_sq_add_buffer+0x469 16 1% 87% 0.00 15031 0xfffffe2314a86e90 squeue_drain+0x18b 13 1% 88% 0.00 17230 0xfffffe23970c3c00 taskq_dispatch_ent+0x51 13 1% 89% 0.00 3095 0xfffffe229dacc240 kmem_cache_free+0x78 11 1% 89% 0.00 16395 0xfffffe26d8181000 squeue_enter+0x4cd 11 1% 90% 0.00 6613 0xfffffe26d44b2c40 cv_wait+0x78 11 1% 91% 0.00 15025 0xfffffe256a64b040 mac_rx_srs_process+0x5c 11 1% 92% 0.00 28920 0xfffffe2397bd99c8 cv_wait+0x78 8 1% 92% 0.00 17505 sqset_lock ip_squeue_random+0x22 8 1% 93% 0.00 16430 0xfffffe23970c3c00 cv_wait+0x78 8 1% 93% 0.00 7460 0xfffffe22d6a50268 taskq_thread+0x2cd 7 0% 94% 0.00 8520 0xfffffe235bd431f8 mlxcx_process_cq+0x207 6 0% 94% 0.00 32966 overlay_target_lock cv_timedwait_hires+0xf0 6 0% 94% 0.00 11463 0xfffffe26d7d3f620 i_dls_head_hold+0x1f 6 0% 95% 0.00 17827 0xfffffe2397bd99c8 taskq_thread+0x2cd 6 0% 95% 0.00 20639 0xfffffe23970c3c00 taskq_thread+0x2cd 5 0% 96% 0.00 19611 overlay_target_lock overlay_target_lookup_request+0x47 5 0% 96% 0.00 11493 0xfffffe26d83c5c90 dls_accept_common+0x80 5 0% 96% 0.00 11590 0xfffffe26d81816f8 tcp_sendmsg+0x83 5 0% 97% 0.00 12343 0xfffffe256a64b040 cv_wait+0x78 5 0% 97% 0.00 10838 0xfffffe2357349638 mlxcx_process_cq+0x207 4 0% 97% 0.00 19757 zfs_disk_lock zfs_zone_zio_done+0xe1 4 0% 97% 0.00 7033 0xfffffe235bd40558 mlxcx_buf_return+0x8c 4 0% 98% 0.00 10742 0xfffffe2358117b58 mlxcx_buf_return+0x8c 3 0% 98% 0.00 13412 0xfffffe23970c3540 cv_wait+0x78 3 0% 98% 0.00 12283 0xfffffe22e170b7d0 squeue_enter+0x37a 3 0% 98% 0.00 3072 0xfffffe229dae0240 kmem_cache_free+0x78 2 0% 98% 0.00 16866 0xfffffe26d7d3f620 i_dls_head_rele+0x18 2 0% 99% 0.00 16951 0xfffffe23976bde88 vdev_queue_io_done+0x75 2 0% 99% 0.00 17163 0xfffffe23976bde88 vdev_queue_io+0x66 2 0% 99% 0.00 11918 0xfffffe2314a86e90 squeue_enter+0x53 2 0% 99% 0.00 13555 0xfffffe22e170b7d0 squeue_drain+0x18b 2 0% 99% 0.00 3201 0xfffffe229dacc2c0 kmem_cache_free+0x78 1 0% 99% 0.00 12945 0xfffffe26d860bf38 tcp_sendmsg+0x83 1 0% 99% 0.00 20358 0xfffffe26d8448bd0 ire_route_recursive_dstonly_v4+0x68 1 0% 99% 0.00 13415 0xfffffe2397bd99c8 taskq_ent_free+0x6d 1 0% 99% 0.00 26821 0xfffffe2397bd4000 metaslab_group_alloc_normal+0x15f 1 0% 99% 0.00 20469 0xfffffe23976bde88 vdev_queue_io_done+0x30 1 0% 100% 0.00 8296 0xfffffe2316f86c18 bd_runq_exit+0x53 1 0% 100% 0.00 12223 0xfffffe231551f000 metaslab_group_alloc_normal+0x1b4 1 0% 100% 0.00 28464 0xfffffe23155c2800 metaslab_group_alloc_normal+0x1b4 1 0% 100% 0.00 9398 0xfffffe22d6c82940 cv_wait_sig+0x166 1 0% 100% 0.00 12674 0xfffffe22d6a50c88 cv_wait+0x78 1 0% 100% 0.00 5080 0xfffffe229dae0380 kmem_cache_free+0x78 1 0% 100% 0.00 17012 0xfffffe22d6c82f80 cv_wait_sig+0x166 1 0% 100% 0.00 18886 0xfffffe22e170bb90 squeue_enter+0x37a ------------------------------------------------------------------------------- Spin lock spin: 2782 events in 10.655 seconds (261 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 840 30% 30% 0.00 1242 cpu0_disp disp_lock_enter+0x26 464 17% 47% 0.00 4288 cp_default disp_lock_enter+0x26 195 7% 54% 0.00 703 0xfffffe2313615d78 disp_lock_enter+0x26 183 7% 60% 0.00 769 0xfffffe231325dcf0 disp_lock_enter+0x26 176 6% 67% 0.00 3376 0xfffffe2315b4b008 disp_lock_enter_high+0x11 124 4% 71% 0.00 1515 turnstile_table+0xd38 disp_lock_enter+0x26 110 4% 75% 0.00 6300 sleepq_head+0x1068 disp_lock_enter_high+0x11 87 3% 78% 0.00 1720 0xfffffe22e21b2980 disp_lock_enter+0x26 51 2% 80% 0.00 25531 turnstile_table+0xbe8 disp_lock_enter+0x26 50 2% 82% 0.00 2534 0xfffffe22e21b2018 disp_lock_enter+0x26 46 2% 84% 0.00 1398 0xfffffe22e1abdf70 disp_lock_enter+0x26 42 2% 85% 0.00 44728 turnstile_table+0xb48 disp_lock_enter+0x26 32 1% 86% 0.00 1271 0xfffffe2313df4800 disp_lock_enter+0x26 31 1% 87% 0.00 797 0xfffffe22e1e3adc8 disp_lock_enter+0x26 31 1% 88% 0.00 701 0xfffffe22e1abdb48 disp_lock_enter+0x26 28 1% 90% 0.00 25243 turnstile_table+0x9e8 disp_lock_enter+0x26 28 1% 91% 0.00 1223 0xfffffe2313e092b8 disp_lock_enter+0x26 23 1% 91% 0.00 1708 0xfffffe22e1aa76a8 disp_lock_enter+0x26 22 1% 92% 0.00 4190 0xfffffe2313f66798 disp_lock_enter+0x26 18 1% 93% 0.00 1351 0xfffffe22e21b2da8 disp_lock_enter+0x26 13 0% 93% 0.00 14043 cp_default disp_lock_enter_high+0x11 13 0% 94% 0.00 1031 0xfffffe2312c4dbe8 disp_lock_enter+0x26 12 0% 94% 0.00 1090 0xfffffe22e1ffe698 disp_lock_enter+0x26 10 0% 95% 0.00 572 turnstile_table+0xa68 disp_lock_enter+0x26 9 0% 95% 0.00 1593 turnstile_table+0xdb8 disp_lock_enter+0x26 9 0% 95% 0.00 3357 sleepq_head+0x6028 disp_lock_enter_high+0x11 9 0% 95% 0.00 1847 0xfffffe231326d578 disp_lock_enter+0x26 9 0% 96% 0.00 813 0xfffffe22e1e4cd18 disp_lock_enter+0x26 9 0% 96% 0.00 1361 0xfffffe22e1e3a150 disp_lock_enter+0x26 8 0% 96% 0.00 1093 0xfffffe22e1e3a1f8 disp_lock_enter+0x26 6 0% 97% 0.00 750 turnstile_table+0x908 disp_lock_enter+0x26 6 0% 97% 0.00 1066 0xfffffe23139aeba0 disp_lock_enter+0x26 6 0% 97% 0.00 951 0xfffffe2312c4d210 disp_lock_enter+0x26 5 0% 97% 0.00 1017 0xfffffe2312e97e60 disp_lock_enter+0x26 5 0% 97% 0.00 1919 0xfffffe2312e976f0 disp_lock_enter+0x26 4 0% 98% 0.00 8511 turnstile_table+0xfb8 disp_lock_enter+0x26 4 0% 98% 0.00 1355 0xfffffe231325dcf0 disp_lock_enter_high+0x11 3 0% 98% 0.00 6208 cpu0_disp disp_lock_enter_high+0x11 3 0% 98% 0.00 1720 0xfffffe23139bf620 disp_lock_enter+0x26 3 0% 98% 0.00 1286 0xfffffe22e27ab730 disp_lock_enter+0x26 3 0% 98% 0.00 928 0xfffffe22e21b2980 disp_lock_enter_high+0x11 3 0% 98% 0.00 968 0xfffffe22e1ffe970 disp_lock_enter+0x26 2 0% 98% 0.00 23414 turnstile_table+0xea8 disp_lock_enter+0x26 2 0% 98% 0.00 1763 turnstile_table+0xc98 disp_lock_enter+0x26 2 0% 98% 0.00 1067 turnstile_table+0xbb8 disp_lock_enter+0x26 2 0% 99% 0.00 5961 turnstile_table+0x918 disp_lock_enter+0x26 2 0% 99% 0.00 8907 sleepq_head+0x2558 disp_lock_enter+0x26 2 0% 99% 0.00 1052 0xfffffe2313f66798 disp_lock_enter_high+0x11 2 0% 99% 0.00 1503 0xfffffe2313e09cc8 disp_lock_enter+0x26 2 0% 99% 0.00 2715 0xfffffe2313df4800 disp_lock_enter_high+0x11 2 0% 99% 0.00 736 0xfffffe2313bdf7b8 disp_lock_enter+0x26 2 0% 99% 0.00 1097 0xfffffe231383b5d8 disp_lock_enter+0x26 2 0% 99% 0.00 556 0xfffffe2313615d78 disp_lock_enter_high+0x11 2 0% 99% 0.00 2249 0xfffffe22f2a42f28 disp_lock_enter+0x26 2 0% 99% 0.00 3531 0xfffffe22e1aa76a8 disp_lock_enter_high+0x11 1 0% 99% 0.00 961 turnstile_table+0xe88 disp_lock_enter+0x26 1 0% 99% 0.00 1122 turnstile_table+0xe68 disp_lock_enter+0x26 1 0% 99% 0.00 961 turnstile_table+0xda8 disp_lock_enter+0x26 1 0% 99% 0.00 481 turnstile_table+0xcf8 disp_lock_enter+0x26 1 0% 99% 0.00 1282 turnstile_table+0xad8 disp_lock_enter+0x26 1 0% 99% 0.00 1031 turnstile_table+0xb78 disp_lock_enter+0x26 1 0% 99% 0.00 1893 sleepq_head+0x2558 disp_lock_enter_high+0x11 1 0% 99% 0.00 731 turnstile_table+0x8c8 disp_lock_enter+0x26 1 0% 99% 0.00 2023 turnstile_table+0x838 disp_lock_enter+0x26 1 0% 100% 0.00 2505 turnstile_table+0x938 disp_lock_enter+0x26 1 0% 100% 0.00 1502 0xfffffe23139bf5e8 disp_lock_enter_high+0x11 1 0% 100% 0.00 4157 0xfffffe231325dba0 disp_lock_enter_high+0x11 1 0% 100% 0.00 721 0xfffffe2313464e88 disp_lock_enter_high+0x11 1 0% 100% 0.00 752 0xfffffe2313464e88 disp_lock_enter+0x26 1 0% 100% 0.00 461 0xfffffe2313e092b8 disp_lock_enter_high+0x11 1 0% 100% 0.00 1022 0xfffffe2313f66d80 disp_lock_enter+0x26 1 0% 100% 0.00 1053 0xfffffe231325d238 disp_lock_enter+0x26 1 0% 100% 0.00 782 0xfffffe2312e976f0 disp_lock_enter_high+0x11 1 0% 100% 0.00 1743 0xfffffe22e25578e8 disp_lock_enter+0x26 1 0% 100% 0.00 461 0xfffffe22e2557b18 disp_lock_enter+0x26 1 0% 100% 0.00 12033 0xfffffe22e1abdf70 disp_lock_enter_high+0x11 1 0% 100% 0.00 470 0xfffffe22e1ff1cc0 disp_lock_enter+0x26 1 0% 100% 0.00 1433 0xfffffe22e1e4cd18 disp_lock_enter_high+0x11 ------------------------------------------------------------------------------- Thread lock spin: 11964 events in 10.655 seconds (1123 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 10698 89% 89% 0.00 1958 sleepq_head+0x1068 setrun+0x18 1032 9% 98% 0.00 2597 sleepq_head+0x6028 setrun+0x18 106 1% 99% 0.00 384 cpu[59]+0xe8 fss_tick+0x24 34 0% 99% 0.00 1617 sleepq_head+0x1068 fss_tick+0x24 29 0% 99% 0.00 995 cpu[59]+0xe8 post_syscall+0x392 7 0% 100% 0.00 712 turnstile_table+0xdb8 fss_tick+0x24 5 0% 100% 0.00 446 turnstile_table+0x938 fss_tick+0x24 3 0% 100% 0.00 617 turnstile_table+0xda8 fss_tick+0x24 3 0% 100% 0.00 935 cpu[28]+0xe8 fss_tick+0x24 3 0% 100% 0.00 360 cpu[21]+0xe8 fss_tick+0x24 2 0% 100% 0.00 1047 cpu[59]+0xe8 preempt+0x47 2 0% 100% 0.00 370 cpu[45]+0xe8 fss_tick+0x24 2 0% 100% 0.00 2034 cpu[44]+0xe8 post_syscall+0x538 2 0% 100% 0.00 360 cpu[44]+0xe8 fss_tick+0x24 2 0% 100% 0.00 3692 cpu[9]+0xe8 post_syscall+0x538 2 0% 100% 0.00 907 cpu[10]+0xe8 post_syscall+0x538 2 0% 100% 0.00 1242 cpu[15]+0xe8 fss_tick+0x24 1 0% 100% 0.00 792 turnstile_table+0xd38 fss_tick+0x24 1 0% 100% 0.00 450 turnstile_table+0xd98 fss_tick+0x24 1 0% 100% 0.00 641 cpu[60]+0xe8 fss_tick+0x24 1 0% 100% 0.00 671 cpu[60]+0xe8 post_syscall+0x392 1 0% 100% 0.00 1794 cpu[44]+0xe8 preempt+0x47 1 0% 100% 0.00 421 cpu[28]+0xe8 post_syscall+0x392 1 0% 100% 0.00 3487 cpu[34]+0xe8 fss_tick+0x24 1 0% 100% 0.00 6292 cpu[47]+0xe8 cv_timedwait_hires+0xaa 1 0% 100% 0.00 260663 cpu[48]+0xe8 fss_tick+0x24 1 0% 100% 0.00 401 cpu[49]+0xe8 fss_tick+0x24 1 0% 100% 0.00 822 cpu[54]+0xe8 post_syscall+0x538 1 0% 100% 0.00 461 cpu[52]+0xe8 post_syscall+0x392 1 0% 100% 0.00 480 cpu[9]+0xe8 fss_tick+0x24 1 0% 100% 0.00 3376 cpu[10]+0xe8 fss_tick+0x24 1 0% 100% 0.00 911 cpu[15]+0xe8 cv_timedwait_hires+0xaa 1 0% 100% 0.00 330 cpu[5]+0xe8 post_syscall+0x392 1 0% 100% 0.00 531 cpu[7]+0xe8 fss_tick+0x24 1 0% 100% 0.00 171 cpu[2]+0xe8 post_syscall+0x538 1 0% 100% 0.00 2515 cpu[1]+0xe8 preempt+0x47 1 0% 100% 0.00 2605 cpu[1]+0xe8 post_syscall+0x392 1 0% 100% 0.00 370 0xfffffe22f2a425c0 sigtoproc+0x258 1 0% 100% 0.00 1954 cpu[6]+0xe8 turnstile_pi_waive+0x1b 1 0% 100% 0.00 2335 cpu[6]+0xe8 cv_timedwait_hires+0xaa 1 0% 100% 0.00 4970 cpu[8]+0xe8 fss_tick+0x24 1 0% 100% 0.00 2064 cpu[6]+0xe8 fss_tick+0x24 1 0% 100% 0.00 12504 cpu[24]+0xe8 cv_timedwait_hires+0xaa 1 0% 100% 0.00 431 cpu[24]+0xe8 fss_tick+0x24 1 0% 100% 0.00 1212 cpu[21]+0xe8 post_syscall+0x392 1 0% 100% 0.00 7463 cpu[30]+0xe8 fss_tick+0x24 1 0% 100% 0.00 521 cpu[30]+0xe8 cv_timedwait_hires+0xaa ------------------------------------------------------------------------------- Initiator netstat and rx queue drops [root@server4 (us-mobile-1) /zones/global/mlxcx]# netstat -s -P tcp TCP tcpRtoAlgorithm = 4 tcpRtoMin = 400 tcpRtoMax = 60000 tcpMaxConn = -1 tcpActiveOpens = 198 tcpPassiveOpens = 22 tcpAttemptFails = 2 tcpEstabResets = 0 tcpCurrEstab = 31 tcpOutSegs =22344324 tcpOutDataSegs =11234943 tcpOutDataBytes =3293532269 tcpRetransSegs = 10 tcpRetransBytes = 0 tcpOutAck = 581 tcpOutAckDelayed = 29 tcpOutUrg = 0 tcpOutWinUpdate = 0 tcpOutWinProbe = 0 tcpOutControl = 421 tcpOutRsts = 4 tcpOutFastRetrans = 0 tcpInSegs =1408575 tcpInAckSegs = 0 tcpInAckBytes =3293525310 tcpInDupAck = 511 tcpInAckUnsent = 0 tcpInInorderSegs = 2477 tcpInInorderBytes =218705 tcpInUnorderSegs = 0 tcpInUnorderBytes = 0 tcpInDupSegs = 0 tcpInDupBytes = 0 tcpInPartDupSegs = 0 tcpInPartDupBytes = 0 tcpInPastWinSegs = 0 tcpInPastWinBytes = 0 tcpInWinProbe = 0 tcpInWinUpdate = 0 tcpInClosed = 0 tcpRttNoUpdate = 0 tcpRttUpdate =1406612 tcpTimRetrans = 0 tcpTimRetransDrop = 0 tcpTimKeepalive = 648 tcpTimKeepaliveProbe= 374 tcpTimKeepaliveDrop = 0 tcpListenDrop = 0 tcpListenDropQ0 = 0 tcpHalfOpenDrop = 0 tcpOutSackRetrans = 0 [root@server4 (us-mobile-1) /zones/global/mlxcx]# mdb -k Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc apix scsi_vhci ufs ip hook neti sockfs arp usba xhci stmf_sbd stmf zfs mm lofs random idm sata cpc logindmux ptm sppp klmmod nfs vmm ] > mlxcx_softstate/p | ::softstate 0 | ::print mlxcx_t mlx_ports->mlp_stats.mlps_rx_drops mlx_ports->mlp_stats.mlps_rx_drops = 0 > mlxcx_softstate/p | ::softstate 1 | ::print mlxcx_t mlx_ports->mlp_stats.mlps_rx_drops mlx_ports->mlp_stats.mlps_rx_drops = 0 > Target netstat and rx queue drops [root@server5 (us-mobile-1) /zones/global/mlxcx]# netstat -s -P tcp TCP tcpRtoAlgorithm = 4 tcpRtoMin = 400 tcpRtoMax = 60000 tcpMaxConn = -1 tcpActiveOpens = 186 tcpPassiveOpens = 38 tcpAttemptFails = 2 tcpEstabResets = 0 tcpCurrEstab = 32 tcpOutSegs =1406258 tcpOutDataSegs = 1574 tcpOutDataBytes =142482 tcpRetransSegs = 8 tcpRetransBytes = 0 tcpOutAck =1404651 tcpOutAckDelayed = 30 tcpOutUrg = 0 tcpOutWinUpdate = 20 tcpOutWinProbe = 0 tcpOutControl = 426 tcpOutRsts = 4 tcpOutFastRetrans = 0 tcpInSegs =11235199 tcpInAckSegs = 0 tcpInAckBytes =135284 tcpInDupAck = 541 tcpInAckUnsent = 0 tcpInInorderSegs =11233527 tcpInInorderBytes =3293494170 tcpInUnorderSegs = 0 tcpInUnorderBytes = 0 tcpInDupSegs = 0 tcpInDupBytes = 0 tcpInPartDupSegs = 0 tcpInPartDupBytes = 0 tcpInPastWinSegs = 0 tcpInPastWinBytes = 0 tcpInWinProbe = 0 tcpInWinUpdate = 0 tcpInClosed = 0 tcpRttNoUpdate = 0 tcpRttUpdate = 1403 tcpTimRetrans = 0 tcpTimRetransDrop = 0 tcpTimKeepalive = 684 tcpTimKeepaliveProbe= 389 tcpTimKeepaliveDrop = 0 tcpListenDrop = 0 tcpListenDropQ0 = 0 tcpHalfOpenDrop = 0 tcpOutSackRetrans = 0 [root@server5 (us-mobile-1) /zones/global/mlxcx]# mdb -k Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc apix scsi_vhci ufs ip hook neti sockfs arp usba xhci stmf_sbd stmf zfs mm lofs random idm sata cpc logindmux ptm sppp klmmod nfs vmm ] > mlxcx_softstate/p | ::softstate 0 | ::print mlxcx_t mlx_ports->mlp_stats.mlps_rx_drops mlx_ports->mlp_stats.mlps_rx_drops = 0 > mlxcx_softstate/p | ::softstate 1 | ::print mlxcx_t mlx_ports->mlp_stats.mlps_rx_drops mlx_ports->mlp_stats.mlps_rx_drops = 0 >