`master` -> QSFP28 ================== [root@server4 (us-mobile-1) /zones/global/mlxcx]# iperf3 -c 10.100.4.2 -P 4 Connecting to host 10.100.4.2, port 5201 [ 5] local 10.100.4.1 port 47318 connected to 10.100.4.2 port 5201 [ 7] local 10.100.4.1 port 44956 connected to 10.100.4.2 port 5201 [ 9] local 10.100.4.1 port 57685 connected to 10.100.4.2 port 5201 [ 11] local 10.100.4.1 port 59511 connected to 10.100.4.2 port 5201 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.01 sec 124 MBytes 1.03 Gbits/sec [ 7] 0.00-1.01 sec 123 MBytes 1.03 Gbits/sec [ 9] 0.00-1.01 sec 122 MBytes 1.02 Gbits/sec [ 11] 0.00-1.01 sec 124 MBytes 1.03 Gbits/sec [SUM] 0.00-1.01 sec 493 MBytes 4.10 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 1.01-2.00 sec 126 MBytes 1.07 Gbits/sec [ 7] 1.01-2.00 sec 126 MBytes 1.07 Gbits/sec [ 9] 1.01-2.00 sec 126 MBytes 1.07 Gbits/sec [ 11] 1.01-2.00 sec 126 MBytes 1.07 Gbits/sec [SUM] 1.01-2.00 sec 505 MBytes 4.27 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 2.00-3.01 sec 126 MBytes 1.05 Gbits/sec [ 7] 2.00-3.01 sec 126 MBytes 1.05 Gbits/sec [ 9] 2.00-3.01 sec 126 MBytes 1.05 Gbits/sec [ 11] 2.00-3.01 sec 126 MBytes 1.05 Gbits/sec [SUM] 2.00-3.01 sec 505 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 3.01-4.01 sec 125 MBytes 1.05 Gbits/sec [ 7] 3.01-4.01 sec 125 MBytes 1.05 Gbits/sec [ 9] 3.01-4.01 sec 125 MBytes 1.05 Gbits/sec [ 11] 3.01-4.01 sec 125 MBytes 1.05 Gbits/sec [SUM] 3.01-4.01 sec 500 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 4.01-5.00 sec 126 MBytes 1.06 Gbits/sec [ 7] 4.01-5.00 sec 126 MBytes 1.06 Gbits/sec [ 9] 4.01-5.00 sec 126 MBytes 1.06 Gbits/sec [ 11] 4.01-5.00 sec 126 MBytes 1.06 Gbits/sec [SUM] 4.01-5.00 sec 505 MBytes 4.25 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 5.00-6.01 sec 124 MBytes 1.03 Gbits/sec [ 7] 5.00-6.01 sec 124 MBytes 1.03 Gbits/sec [ 9] 5.00-6.01 sec 124 MBytes 1.03 Gbits/sec [ 11] 5.00-6.01 sec 124 MBytes 1.03 Gbits/sec [SUM] 5.00-6.01 sec 495 MBytes 4.14 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 6.01-7.00 sec 121 MBytes 1.02 Gbits/sec [ 7] 6.01-7.00 sec 121 MBytes 1.02 Gbits/sec [ 9] 6.01-7.00 sec 121 MBytes 1.02 Gbits/sec [ 11] 6.01-7.00 sec 121 MBytes 1.02 Gbits/sec [SUM] 6.01-7.00 sec 485 MBytes 4.09 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [ 7] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [ 9] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [ 11] 7.00-8.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 7.00-8.00 sec 495 MBytes 4.14 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 8.00-9.00 sec 126 MBytes 1.06 Gbits/sec [ 7] 8.00-9.00 sec 126 MBytes 1.06 Gbits/sec [ 9] 8.00-9.00 sec 126 MBytes 1.06 Gbits/sec [ 11] 8.00-9.00 sec 126 MBytes 1.06 Gbits/sec [SUM] 8.00-9.00 sec 505 MBytes 4.24 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 9.00-10.01 sec 126 MBytes 1.05 Gbits/sec [ 7] 9.00-10.01 sec 126 MBytes 1.05 Gbits/sec [ 9] 9.00-10.01 sec 126 MBytes 1.05 Gbits/sec [ 11] 9.00-10.01 sec 126 MBytes 1.05 Gbits/sec [SUM] 9.00-10.01 sec 505 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec sender [ 5] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec receiver [ 7] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec sender [ 7] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec receiver [ 9] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec sender [ 9] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec receiver [ 11] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec sender [ 11] 0.00-10.01 sec 1.22 GBytes 1.05 Gbits/sec receiver [SUM] 0.00-10.01 sec 4.88 GBytes 4.18 Gbits/sec sender [SUM] 0.00-10.01 sec 4.88 GBytes 4.18 Gbits/sec receiver iperf Done. `master` -> QSFP56 ================== Connecting to host 10.100.2.2, port 5201 [ 5] local 10.100.2.1 port 51672 connected to 10.100.2.2 port 5201 [ 7] local 10.100.2.1 port 54580 connected to 10.100.2.2 port 5201 [ 9] local 10.100.2.1 port 52814 connected to 10.100.2.2 port 5201 [ 11] local 10.100.2.1 port 46076 connected to 10.100.2.2 port 5201 [ ID] Interval Transfer Bitrate [ 5] 0.00-1.01 sec 128 MBytes 1.06 Gbits/sec [ 7] 0.00-1.01 sec 129 MBytes 1.07 Gbits/sec [ 9] 0.00-1.01 sec 128 MBytes 1.06 Gbits/sec [ 11] 0.00-1.01 sec 129 MBytes 1.07 Gbits/sec [SUM] 0.00-1.01 sec 514 MBytes 4.28 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 1.01-2.01 sec 126 MBytes 1.06 Gbits/sec [ 7] 1.01-2.01 sec 126 MBytes 1.06 Gbits/sec [ 9] 1.01-2.01 sec 126 MBytes 1.06 Gbits/sec [ 11] 1.01-2.01 sec 126 MBytes 1.06 Gbits/sec [SUM] 1.01-2.01 sec 505 MBytes 4.24 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 2.01-3.00 sec 128 MBytes 1.07 Gbits/sec [ 7] 2.01-3.00 sec 128 MBytes 1.07 Gbits/sec [ 9] 2.01-3.00 sec 128 MBytes 1.07 Gbits/sec [ 11] 2.01-3.00 sec 128 MBytes 1.07 Gbits/sec [SUM] 2.01-3.00 sec 510 MBytes 4.29 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 3.00-4.01 sec 130 MBytes 1.08 Gbits/sec [ 7] 3.00-4.01 sec 130 MBytes 1.08 Gbits/sec [ 9] 3.00-4.01 sec 130 MBytes 1.08 Gbits/sec [ 11] 3.00-4.01 sec 130 MBytes 1.08 Gbits/sec [SUM] 3.00-4.01 sec 520 MBytes 4.33 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [ 7] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [ 9] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [ 11] 4.01-5.00 sec 130 MBytes 1.10 Gbits/sec [SUM] 4.01-5.00 sec 520 MBytes 4.40 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 5.00-6.00 sec 128 MBytes 1.07 Gbits/sec [ 7] 5.00-6.00 sec 128 MBytes 1.07 Gbits/sec [ 9] 5.00-6.00 sec 128 MBytes 1.07 Gbits/sec [ 11] 5.00-6.00 sec 128 MBytes 1.07 Gbits/sec [SUM] 5.00-6.00 sec 510 MBytes 4.27 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 6.00-7.01 sec 130 MBytes 1.09 Gbits/sec [ 7] 6.00-7.01 sec 130 MBytes 1.09 Gbits/sec [ 9] 6.00-7.01 sec 130 MBytes 1.09 Gbits/sec [ 11] 6.00-7.01 sec 130 MBytes 1.09 Gbits/sec [SUM] 6.00-7.01 sec 520 MBytes 4.35 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 7.01-8.01 sec 130 MBytes 1.09 Gbits/sec [ 7] 7.01-8.01 sec 130 MBytes 1.09 Gbits/sec [ 9] 7.01-8.01 sec 130 MBytes 1.09 Gbits/sec [ 11] 7.01-8.01 sec 130 MBytes 1.09 Gbits/sec [SUM] 7.01-8.01 sec 520 MBytes 4.35 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 8.01-9.00 sec 129 MBytes 1.08 Gbits/sec [ 7] 8.01-9.00 sec 129 MBytes 1.08 Gbits/sec [ 9] 8.01-9.00 sec 129 MBytes 1.08 Gbits/sec [ 11] 8.01-9.00 sec 129 MBytes 1.08 Gbits/sec [SUM] 8.01-9.00 sec 515 MBytes 4.34 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 5] 9.00-10.01 sec 129 MBytes 1.08 Gbits/sec [ 7] 9.00-10.01 sec 129 MBytes 1.08 Gbits/sec [ 9] 9.00-10.01 sec 129 MBytes 1.08 Gbits/sec [ 11] 9.00-10.01 sec 129 MBytes 1.08 Gbits/sec [SUM] 9.00-10.01 sec 515 MBytes 4.31 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 5] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec sender [ 5] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec receiver [ 7] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec sender [ 7] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec receiver [ 9] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec sender [ 9] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec receiver [ 11] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec sender [ 11] 0.00-10.01 sec 1.26 GBytes 1.08 Gbits/sec receiver [SUM] 0.00-10.01 sec 5.03 GBytes 4.32 Gbits/sec sender [SUM] 0.00-10.01 sec 5.03 GBytes 4.32 Gbits/sec receiver iperf Done. `master` lockstat on QSFP56 [root@server4 (us-mobile-1) /zones/global/mlxcx]# lockstat iperf3 -c 10.100.2.2 -P 4 Connecting to host 10.100.2.2, port 5201 [ 6] local 10.100.2.1 port 41036 connected to 10.100.2.2 port 5201 [ 8] local 10.100.2.1 port 64398 connected to 10.100.2.2 port 5201 [ 10] local 10.100.2.1 port 46961 connected to 10.100.2.2 port 5201 [ 12] local 10.100.2.1 port 38772 connected to 10.100.2.2 port 5201 [ ID] Interval Transfer Bitrate [ 6] 0.00-1.01 sec 128 MBytes 1.07 Gbits/sec [ 8] 0.00-1.01 sec 129 MBytes 1.08 Gbits/sec [ 10] 0.00-1.01 sec 129 MBytes 1.07 Gbits/sec [ 12] 0.00-1.01 sec 128 MBytes 1.07 Gbits/sec [SUM] 0.00-1.01 sec 514 MBytes 4.29 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 1.01-2.00 sec 125 MBytes 1.05 Gbits/sec [ 8] 1.01-2.00 sec 125 MBytes 1.05 Gbits/sec [ 10] 1.01-2.00 sec 125 MBytes 1.05 Gbits/sec [ 12] 1.01-2.00 sec 125 MBytes 1.05 Gbits/sec [SUM] 1.01-2.00 sec 500 MBytes 4.20 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 2.00-3.01 sec 121 MBytes 1.01 Gbits/sec [ 8] 2.00-3.01 sec 121 MBytes 1.01 Gbits/sec [ 10] 2.00-3.01 sec 121 MBytes 1.01 Gbits/sec [ 12] 2.00-3.01 sec 121 MBytes 1.01 Gbits/sec [SUM] 2.00-3.01 sec 485 MBytes 4.05 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 3.01-4.00 sec 122 MBytes 1.03 Gbits/sec [ 8] 3.01-4.00 sec 122 MBytes 1.03 Gbits/sec [ 10] 3.01-4.00 sec 122 MBytes 1.03 Gbits/sec [ 12] 3.01-4.00 sec 122 MBytes 1.03 Gbits/sec [SUM] 3.01-4.00 sec 490 MBytes 4.14 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 4.00-5.00 sec 124 MBytes 1.04 Gbits/sec [ 8] 4.00-5.00 sec 124 MBytes 1.04 Gbits/sec [ 10] 4.00-5.00 sec 124 MBytes 1.04 Gbits/sec [ 12] 4.00-5.00 sec 124 MBytes 1.04 Gbits/sec [SUM] 4.00-5.00 sec 495 MBytes 4.14 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 5.00-6.00 sec 122 MBytes 1.03 Gbits/sec [ 8] 5.00-6.00 sec 122 MBytes 1.03 Gbits/sec [ 10] 5.00-6.00 sec 122 MBytes 1.03 Gbits/sec [ 12] 5.00-6.00 sec 122 MBytes 1.03 Gbits/sec [SUM] 5.00-6.00 sec 490 MBytes 4.12 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 6.00-7.01 sec 122 MBytes 1.02 Gbits/sec [ 8] 6.00-7.01 sec 122 MBytes 1.02 Gbits/sec [ 10] 6.00-7.01 sec 122 MBytes 1.02 Gbits/sec [ 12] 6.00-7.01 sec 122 MBytes 1.02 Gbits/sec [SUM] 6.00-7.01 sec 490 MBytes 4.09 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 7.01-8.01 sec 122 MBytes 1.03 Gbits/sec [ 8] 7.01-8.01 sec 122 MBytes 1.03 Gbits/sec [ 10] 7.01-8.01 sec 122 MBytes 1.03 Gbits/sec [ 12] 7.01-8.01 sec 122 MBytes 1.03 Gbits/sec [SUM] 7.01-8.01 sec 490 MBytes 4.11 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 8.01-9.00 sec 121 MBytes 1.02 Gbits/sec [ 8] 8.01-9.00 sec 121 MBytes 1.02 Gbits/sec [ 10] 8.01-9.00 sec 121 MBytes 1.02 Gbits/sec [ 12] 8.01-9.00 sec 121 MBytes 1.02 Gbits/sec [SUM] 8.01-9.00 sec 485 MBytes 4.10 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ 6] 9.00-10.01 sec 124 MBytes 1.03 Gbits/sec [ 8] 9.00-10.01 sec 124 MBytes 1.03 Gbits/sec [ 10] 9.00-10.01 sec 124 MBytes 1.03 Gbits/sec [ 12] 9.00-10.01 sec 124 MBytes 1.03 Gbits/sec [SUM] 9.00-10.01 sec 495 MBytes 4.11 Gbits/sec - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate [ 6] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec sender [ 6] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec receiver [ 8] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec sender [ 8] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec receiver [ 10] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec sender [ 10] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec receiver [ 12] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec sender [ 12] 0.00-10.01 sec 1.20 GBytes 1.03 Gbits/sec receiver [SUM] 0.00-10.01 sec 4.82 GBytes 4.13 Gbits/sec sender [SUM] 0.00-10.01 sec 4.82 GBytes 4.13 Gbits/sec receiver iperf Done. Adaptive mutex spin: 364593 events in 10.669 seconds (34173 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 58963 16% 16% 0.00 583 0xfffffe235d62ad50 mlxcx_buf_return+0x8c 57707 16% 32% 0.00 547 0xfffffe235ba64658 mlxcx_buf_return+0x8c 56040 15% 47% 0.00 572 0xfffffe235ba64b58 mlxcx_buf_return+0x8c 47929 13% 61% 0.00 548 0xfffffe235ba4e340 mlxcx_buf_return+0x8c 16912 5% 65% 0.00 752 0xfffffe235d631478 mlxcx_process_cq+0x207 16150 4% 70% 0.00 750 0xfffffe235ba61378 mlxcx_process_cq+0x207 15581 4% 74% 0.00 770 0xfffffe235ba61a78 mlxcx_process_cq+0x207 14356 4% 78% 0.00 623 0xfffffe235c7995b8 mlxcx_process_cq+0x207 12014 3% 81% 0.00 526 0xfffffe235d62ad50 mlxcx_buf_take+0x2f 11008 3% 84% 0.00 575 0xfffffe235ba64b58 mlxcx_buf_take+0x2f 10290 3% 87% 0.00 515 0xfffffe235ba64658 mlxcx_buf_take+0x2f 7721 2% 89% 0.00 385 0xfffffe235ba4e340 mlxcx_buf_take+0x2f 5515 2% 91% 0.00 567 0xfffffe235d631478 mlxcx_sq_add_buffer+0x469 4856 1% 92% 0.00 515 0xfffffe235c7995b8 mlxcx_sq_add_buffer+0x469 4723 1% 93% 0.00 594 0xfffffe235ba61a78 mlxcx_sq_add_buffer+0x469 4624 1% 94% 0.00 552 0xfffffe235ba61378 mlxcx_sq_add_buffer+0x469 2122 1% 95% 0.00 512 0xfffffe22d6186680 hook_run+0x34 1955 1% 96% 0.00 983 0xfffffe2313c7b0d0 squeue_enter+0x37a 1778 0% 96% 0.00 942 0xfffffe22e170b710 squeue_enter+0x37a 1558 0% 96% 0.00 972 0xfffffe22e2c55d50 squeue_enter+0x37a 1547 0% 97% 0.00 982 0xfffffe22e2c55a50 squeue_enter+0x37a 1331 0% 97% 0.00 877 0xfffffe22d6186680 hook_run+0xd1 1020 0% 98% 0.00 74380 overlay_target_lock cv_timedwait_hires+0xf0 784 0% 98% 0.00 35409 overlay_target_lock overlay_target_lookup_request+0x47 595 0% 98% 0.00 887 0xfffffe2313c7b0d0 squeue_enter+0x53 531 0% 98% 0.00 958 0xfffffe22e170b710 squeue_enter+0x53 523 0% 98% 0.00 980 0xfffffe2313c7b0d0 squeue_drain+0x18b 412 0% 98% 0.00 915 0xfffffe22e170b710 squeue_drain+0x18b 394 0% 98% 0.00 886 0xfffffe22e2c55a50 squeue_enter+0x53 366 0% 99% 0.00 868 0xfffffe22e2c55d50 squeue_enter+0x53 357 0% 99% 0.00 154241 0xfffffe23970b0bf8 cv_wait+0x78 339 0% 99% 0.00 39370 0xfffffe23970b0bf8 taskq_thread+0x2cd 325 0% 99% 0.00 727 0xfffffe235d62a170 mlxcx_buf_loan+0x76 289 0% 99% 0.00 947 0xfffffe22e2c55d50 squeue_drain+0x18b 251 0% 99% 0.00 26562 0xfffffe231a28f2e8 taskq_dispatch_ent+0x51 245 0% 99% 0.00 662 0xfffffe235d62a030 mlxcx_buf_loan+0x76 239 0% 99% 0.00 4386 0xfffffe235d62a170 mlxcx_buf_return+0x8c 206 0% 99% 0.00 959 0xfffffe22e2c55a50 squeue_drain+0x18b 172 0% 99% 0.00 526 0xfffffe229dae01d0 kmem_depot_alloc+0x88 161 0% 99% 0.00 30349 0xfffffe231a28f2e8 taskq_thread+0x2cd 152 0% 99% 0.00 22790 0xfffffe231a28f2e8 cv_wait+0x78 146 0% 99% 0.00 619 0xfffffe235e4d6ab8 mlxcx_buf_loan+0x76 146 0% 99% 0.00 3476 0xfffffe235d62a030 mlxcx_buf_return+0x8c 141 0% 99% 0.00 210 sqset_lock ip_squeue_random+0x22 133 0% 99% 0.00 483 0xfffffe26d728af38 tcp_output+0x63 128 0% 99% 0.00 14058 0xfffffe23970b0bf8 taskq_dispatch+0x174 125 0% 100% 0.00 4185 0xfffffe235e4d6ab8 mlxcx_buf_return+0x8c 109 0% 100% 0.00 433 0xfffffe26d8ba9f38 tcp_output+0x63 102 0% 100% 0.00 484 0xfffffe26d72e2738 tcp_output+0x63 87 0% 100% 0.00 488 0xfffffe235e4d6dd8 mlxcx_buf_loan+0x76 76 0% 100% 0.00 316 0xfffffe25df0134b0 i_dls_head_hold+0x1f 75 0% 100% 0.00 2683 0xfffffe235e4d6dd8 mlxcx_buf_return+0x8c 71 0% 100% 0.00 511 0xfffffe26d8babef8 tcp_output+0x63 63 0% 100% 0.00 487604 0xfffffe22d620c2b0 gcpu_ntv_mca_poll_wrapper+0x72 59 0% 100% 0.00 89267 0xfffffe22e1717da8 cv_timedwait_hires+0xf0 38 0% 100% 0.00 2419 0xfffffe26d35fd600 cv_wait+0x78 38 0% 100% 0.00 23825 0xfffffe23970b0bf8 taskq_ent_free+0x6d 38 0% 100% 0.00 645 0xfffffe229dacc1d0 kmem_depot_alloc+0x88 35 0% 100% 0.00 2372 0xfffffe26d35fee00 cv_wait+0x78 35 0% 100% 0.00 2509 0xfffffe256a6a8b40 cv_wait+0x78 34 0% 100% 0.00 198 0xfffffe26d915f910 ire_route_recursive_dstonly_v4+0x68 33 0% 100% 0.00 22365 0xfffffe2397d66e88 vdev_queue_io+0x66 29 0% 100% 0.00 1765 0xfffffe256a6a4700 cv_wait+0x78 28 0% 100% 0.00 1789 0xfffffe22d6a50c88 taskq_dispatch+0x174 25 0% 100% 0.00 17894 0xfffffe2397d69e88 vdev_queue_io+0x66 24 0% 100% 0.00 2353 0xfffffe26d3600c40 cv_wait+0x78 23 0% 100% 0.00 2015 0xfffffe257a0e64c0 cv_wait+0x78 23 0% 100% 0.00 1995 0xfffffe256a6ab8c0 cv_wait+0x78 23 0% 100% 0.00 3046 0xfffffe2399491a00 getf_gen+0x63 23 0% 100% 0.00 20076 0xfffffe23970aac00 cv_wait+0x78 20 0% 100% 0.00 91287 0xfffffe231a28f408 cv_wait+0x78 20 0% 100% 0.00 1195 0xfffffe22d6a50c88 taskq_thread+0x2cd 17 0% 100% 0.00 2867 0xfffffe26d3601600 cv_wait+0x78 16 0% 100% 0.00 5818 0xfffffe2399491a00 releasef+0x53 16 0% 100% 0.00 12765 0xfffffe2397d66e88 vdev_queue_io_done+0x30 16 0% 100% 0.00 22478 0xfffffe23970b0bf8 taskq_ent_alloc+0xe4 15 0% 100% 0.00 8665 0xfffffe231a28f408 taskq_thread+0x2cd 14 0% 100% 0.00 11555 0xfffffe2397d69e88 vdev_queue_io_done+0x30 14 0% 100% 0.00 949 0xfffffe23968e0df0 vdev_queue_io_remove+0x86 13 0% 100% 0.00 719 0xfffffe23968e0df0 vdev_queue_io_add+0x86 11 0% 100% 0.00 501 0xfffffe26d728af38 tcp_sendmsg+0x83 11 0% 100% 0.00 3298 0xfffffe2397d66e88 vdev_queue_io_done+0x75 10 0% 100% 0.00 655 0xfffffe26d8ba9f38 tcp_sendmsg+0x83 10 0% 100% 0.00 650 0xfffffe26d72e2738 tcp_sendmsg+0x83 10 0% 100% 0.00 544 0xfffffe25df0134b0 i_dls_head_rele+0x18 10 0% 100% 0.00 865 0xfffffe23968e0df0 vdev_queue_pending_remove+0x51 10 0% 100% 0.00 1049 0xfffffe23968e0df0 vdev_queue_pending_add+0x51 9 0% 100% 0.00 585 0xfffffe26d8ba9f38 tcp_wput_data+0x401 9 0% 100% 0.00 3989 0xfffffe22d6a50268 taskq_thread+0x2cd 8 0% 100% 0.00 613 0xfffffe26d8babef8 tcp_sendmsg+0x83 8 0% 100% 0.00 405 0xfffffe26d8bab800 ipcl_classify_v4+0x25d 8 0% 100% 0.00 819 0xfffffe26d72e2738 tcp_wput_data+0x401 8 0% 100% 0.00 608 0xfffffe26d72e2040 ipcl_classify_v4+0x25d 8 0% 100% 0.00 474 0xfffffe26d728a840 ipcl_classify_v4+0x25d 8 0% 100% 0.00 4771 0xfffffe2397d69e88 vdev_queue_io_done+0x75 8 0% 100% 0.00 1039 0xfffffe2313a7f3b0 taskq_thread+0x2cd 8 0% 100% 0.00 830 0xfffffe229e460dc0 callout_list_expire+0xa1 7 0% 100% 0.00 1406 0xfffffe2396ed3640 metaslab_group_alloc+0x63 7 0% 100% 0.00 2191 0xfffffe23181354f0 bge_chip_factotum+0x5e 7 0% 100% 0.00 1794 0xfffffe2314dc2bb8 cv_wait+0x78 7 0% 100% 0.00 946 0xfffffe229e460dc0 callout_execute+0x18 6 0% 100% 0.00 607 0xfffffe26d8babef8 tcp_wput_data+0x401 6 0% 100% 0.00 698 0xfffffe26d8ba9840 ipcl_classify_v4+0x25d 6 0% 100% 0.00 6346 0xfffffe2397d69e88 vdev_queue_aggregate+0x43c 6 0% 100% 0.00 3202 0xfffffe2397d66e88 vdev_queue_aggregate+0x43c 6 0% 100% 0.00 517 0xfffffe2314dc2a98 cv_wait+0x78 6 0% 100% 0.00 791 0xfffffe2313a7f5f0 cv_wait+0x78 6 0% 100% 0.00 848 0xfffffe229e460f40 callout_execute+0x18 6 0% 100% 0.00 713 0xfffffe229e460f40 callout_list_expire+0xa1 5 0% 100% 0.00 450 0xfffffe26d8ba9840 tcp_sendmsg+0xb8 5 0% 100% 0.00 487 0xfffffe26d728a840 squeue_drain+0x255 5 0% 100% 0.00 4693 0xfffffe23970aac00 taskq_thread+0x2cd 5 0% 100% 0.00 2893 0xfffffe2319e614f0 bge_chip_factotum+0x5e 5 0% 100% 0.00 4937 0xfffffe2315605800 metaslab_group_alloc_normal+0x1b4 5 0% 100% 0.00 717 0xfffffe2314dc2a98 taskq_thread+0x2cd 5 0% 100% 0.00 2338 0xfffffe22d6a50c88 cv_wait+0x78 5 0% 100% 0.00 2266 0xfffffe22d6a50268 cv_wait+0x78 4 0% 100% 0.00 674 zone_pdata zfs_zone_zio_done+0xa9 4 0% 100% 0.00 521 0xfffffe26d8bab800 tcp_sendmsg+0xb8 4 0% 100% 0.00 343 0xfffffe26d8bab800 squeue_enter+0x4cd 4 0% 100% 0.00 4087 0xfffffe23970aac00 taskq_dispatch_ent+0x51 4 0% 100% 0.00 516 0xfffffe229de0f1d0 kmem_depot_alloc+0x88 4 0% 100% 0.00 1127 0xfffffe229dae01d0 kmem_depot_free+0x35 3 0% 100% 0.00 417 0xfffffe26d72e2040 tcp_sendmsg+0xb8 3 0% 100% 0.00 1289 0xfffffe2471952558 strread+0x2e4 3 0% 100% 0.00 935 0xfffffe235e4d6ab8 mlxcx_buf_take_n+0x36 3 0% 100% 0.00 390 0xfffffe235d62a030 mlxcx_buf_take_n+0x36 3 0% 100% 0.00 2681 0xfffffe2319e614f0 bge_intr+0xec 3 0% 100% 0.00 2592 0xfffffe2314dca3c8 taskq_thread+0x2cd 3 0% 100% 0.00 992 0xfffffe2314dca3c8 cv_wait+0x78 3 0% 100% 0.00 1980 0xfffffe2314dc7190 cv_wait+0x78 3 0% 100% 0.00 645 0xfffffe229e460dc0 timeout_generic+0x72 3 0% 100% 0.00 1249 0xfffffe229e460c40 callout_execute+0x18 3 0% 100% 0.00 1215 0xfffffe229e45bfc0 callout_list_expire+0xa1 3 0% 100% 0.00 567 0xfffffe229e45b3c0 callout_execute+0x18 3 0% 100% 0.00 1061 0xfffffe229e45b3c0 callout_list_expire+0xa1 3 0% 100% 0.00 2585 0xfffffe229e45b240 callout_list_expire+0xa1 3 0% 100% 0.00 657 0xfffffe229dada1d0 kmem_depot_alloc+0x88 2 0% 100% 0.00 285 zone_pdata zfs_zone_zio_start+0x3c 2 0% 100% 0.00 711 0xfffffe26d8bab800 squeue_drain+0x255 2 0% 100% 0.00 636 0xfffffe26d728af38 tcp_wput_data+0x401 2 0% 100% 0.00 851 0xfffffe256a6a8b40 mac_rx_soft_ring_drain+0x115 2 0% 100% 0.00 2094 0xfffffe2396ed3640 metaslab_group_alloc_normal+0x15f 2 0% 100% 0.00 846 0xfffffe235d62a170 mlxcx_buf_take_n+0x36 2 0% 100% 0.00 1121 0xfffffe2316f94788 zio_dva_throttle+0xa8 2 0% 100% 0.00 746 0xfffffe23168f4018 bd_submit+0x72 2 0% 100% 0.00 1248 0xfffffe2315bd9d00 metaslab_class_throttle_unreserve+0x1d 2 0% 100% 0.00 1849 0xfffffe231558b800 metaslab_group_alloc_normal+0x1b4 2 0% 100% 0.00 3175 0xfffffe2315505000 metaslab_group_alloc_normal+0x1b4 2 0% 100% 0.00 781 0xfffffe2314dd5a78 taskq_thread+0x2cd 2 0% 100% 0.00 1037 0xfffffe2314dce2a0 cv_wait+0x78 2 0% 100% 0.00 766 0xfffffe2314dc2978 cv_wait+0x78 2 0% 100% 0.00 6412 0xfffffe2313a7f3b0 cv_wait+0x78 2 0% 100% 0.00 1027 0xfffffe22e1bcd418 bd_submit+0x72 2 0% 100% 0.00 1112 0xfffffe229e460f40 timeout_generic+0x72 2 0% 100% 0.00 1092 0xfffffe229e460340 callout_execute+0x18 2 0% 100% 0.00 916 0xfffffe229e4601c0 callout_list_expire+0xa1 2 0% 100% 0.00 1352 0xfffffe229e45fec0 callout_list_expire+0xa1 2 0% 100% 0.00 616 0xfffffe229e45fec0 timeout_generic+0x72 2 0% 100% 0.00 836 0xfffffe229e45f980 callout_list_expire+0xa1 2 0% 100% 0.00 691 0xfffffe229e45efc0 callout_execute+0x18 2 0% 100% 0.00 636 0xfffffe229e45dc40 timeout_generic+0x72 2 0% 100% 0.00 1998 0xfffffe229e45c140 callout_list_expire+0xa1 2 0% 100% 0.00 1142 0xfffffe229e45bcc0 callout_execute+0x18 2 0% 100% 0.00 531 0xfffffe229e45b840 callout_execute+0x18 2 0% 100% 0.00 982 0xfffffe229e45b540 callout_execute+0x18 2 0% 100% 0.00 1673 0xfffffe229e45b3c0 timeout_generic+0x72 1 0% 100% 0.00 351 zfs_disk_lock zfs_zone_zio_done+0xe1 1 0% 100% 0.00 370 0xfffffe2715a1d2f8 zio_remove_child+0x47 1 0% 100% 0.00 180 0xfffffe2710b850e8 zio_remove_child+0x47 1 0% 100% 0.00 230 0xfffffe27107624b0 zio_remove_child+0x47 1 0% 100% 0.00 2485 0xfffffe2710b70558 zio_remove_child+0x47 1 0% 100% 0.00 371 0xfffffe26d72e2040 squeue_enter+0x4cd 1 0% 100% 0.00 150 0xfffffe26d8ba9840 squeue_drain+0x255 1 0% 100% 0.00 812 0xfffffe256a6a8b40 mac_rx_srs_drain+0x22d 1 0% 100% 0.00 2204 0xfffffe2471952558 strrput+0xac 1 0% 100% 0.00 1022 0xfffffe2471952558 putnext+0x373 1 0% 100% 0.00 180 0xfffffe256a6a4700 mac_rx_soft_ring_drain+0x115 1 0% 100% 0.00 2375 0xfffffe26d3600c40 mac_rx_soft_ring_process+0x53 1 0% 100% 0.00 201 0xfffffe26d35fee00 mac_rx_soft_ring_process+0x53 1 0% 100% 0.00 370 0xfffffe26d728a840 squeue_enter+0x4cd 1 0% 100% 0.00 181 0xfffffe26d728a840 tcp_sendmsg+0xb8 1 0% 100% 0.00 190 0xfffffe239ddc3b60 dls_accept_common+0x80 1 0% 100% 0.00 410 0xfffffe23970b0778 taskq_dispatch_ent+0x51 1 0% 100% 0.00 902 0xfffffe231550d800 metaslab_group_alloc_normal+0x1b4 1 0% 100% 0.00 2946 0xfffffe2315505000 metaslab_free_concrete+0xc3 1 0% 100% 0.00 381 0xfffffe2314dd5b98 cv_wait+0x78 1 0% 100% 0.00 190 0xfffffe2314dd5b98 taskq_thread+0x2cd 1 0% 100% 0.00 201 0xfffffe2314dcacc8 cv_wait+0x78 1 0% 100% 0.00 2073 0xfffffe2314dce3c0 cv_wait+0x78 1 0% 100% 0.00 2795 0xfffffe2314dd5a78 cv_wait+0x78 1 0% 100% 0.00 1092 0xfffffe2313e26000 metaslab_class_evict_old+0xec 1 0% 100% 0.00 190 0xfffffe2314dc7730 taskq_thread+0x2cd 1 0% 100% 0.00 2385 0xfffffe2314dca068 cv_wait+0x78 1 0% 100% 0.00 962 0xfffffe2314dca2a8 taskq_thread+0x2cd 1 0% 100% 0.00 1393 0xfffffe23197d9760 taskq_thread+0x2cd 1 0% 100% 0.00 2926 0xfffffe231704b000 metaslab_group_alloc_normal+0x1b4 1 0% 100% 0.00 1332 0xfffffe23168b2620 dsl_pool_undirty_space+0x4f 1 0% 100% 0.00 180 0xfffffe231558b800 metaslab_free_concrete+0xc3 1 0% 100% 0.00 401 0xfffffe231550d800 metaslab_free_concrete+0xc3 1 0% 100% 0.00 2204 0xfffffe2315605800 metaslab_free_concrete+0xc3 1 0% 100% 0.00 361 0xfffffe2319e614f0 bge_intr+0x32 1 0% 100% 0.00 671 0xfffffe235abbb178 mlxcx_sq_add_buffer+0x469 1 0% 100% 0.00 1403 0xfffffe229e45f2c0 callout_list_expire+0xa1 1 0% 100% 0.00 371 0xfffffe229e45f2c0 timeout_generic+0x72 1 0% 100% 0.00 1693 0xfffffe229e45f140 timeout_generic+0x72 1 0% 100% 0.00 2725 0xfffffe229e45f140 callout_list_expire+0xa1 1 0% 100% 0.00 461 0xfffffe229e45ec00 untimeout_generic+0x65 1 0% 100% 0.00 942 0xfffffe229e45efc0 callout_list_expire+0xa1 1 0% 100% 0.00 1443 0xfffffe229e45ddc0 callout_list_expire+0xa1 1 0% 100% 0.00 1233 0xfffffe229e45ddc0 callout_list_alloc+0x51 1 0% 100% 0.00 471 0xfffffe229e45dc40 callout_execute+0x18 1 0% 100% 0.00 210 0xfffffe229e45e240 timeout_generic+0x72 1 0% 100% 0.00 1042 0xfffffe229e45e240 callout_list_expire+0xa1 1 0% 100% 0.00 230 0xfffffe229e45e3c0 timeout_generic+0x72 1 0% 100% 0.00 1413 0xfffffe229e45bf00 untimeout_generic+0x65 1 0% 100% 0.00 1182 0xfffffe229e45bcc0 callout_list_expire+0xa1 1 0% 100% 0.00 1353 0xfffffe229e45bb40 callout_execute+0x18 1 0% 100% 0.00 220 0xfffffe229e45bc00 untimeout_generic+0x65 1 0% 100% 0.00 1342 0xfffffe229e45bcc0 timeout_generic+0x72 1 0% 100% 0.00 751 0xfffffe229e45c140 timeout_generic+0x72 1 0% 100% 0.00 391 0xfffffe229e45bfc0 callout_execute+0x18 1 0% 100% 0.00 451 0xfffffe229e45dc40 callout_list_expire+0xa1 1 0% 100% 0.00 1874 0xfffffe229dae0380 kmem_cache_free+0x78 1 0% 100% 0.00 832 0xfffffe229dacc1d0 kmem_depot_free+0x35 1 0% 100% 0.00 1242 0xfffffcc2673f05a8 page_ctr_sub+0x6b 1 0% 100% 0.00 411 0xfffffe229dad01d0 kmem_depot_alloc+0x88 1 0% 100% 0.00 370 0xfffffe229dadc1d0 kmem_depot_alloc+0x88 1 0% 100% 0.00 862 0xfffffe229e45b840 callout_list_expire+0xa1 1 0% 100% 0.00 320 0xfffffe229e45b840 timeout_generic+0x72 1 0% 100% 0.00 1042 0xfffffe229e45b540 callout_list_expire+0xa1 1 0% 100% 0.00 450 0xfffffe229e45b9c0 untimeout_generic+0x65 1 0% 100% 0.00 962 0xfffffe229e45b9c0 callout_list_alloc+0x51 1 0% 100% 0.00 1182 0xfffffe229e45b9c0 callout_list_expire+0xa1 1 0% 100% 0.00 510 0xfffffe229e45b9c0 callout_execute+0x18 1 0% 100% 0.00 1042 0xfffffe229e45bb40 callout_list_expire+0xa1 1 0% 100% 0.00 921 0xfffffe229e460e80 callout_list_expire+0xa1 1 0% 100% 0.00 211 0xfffffe229e4601c0 timeout_generic+0x72 1 0% 100% 0.00 881 0xfffffe229e45fec0 callout_execute+0x18 1 0% 100% 0.00 1033 0xfffffe229e45f440 callout_list_expire+0xa1 1 0% 100% 0.00 3356 0xfffffe229e460340 timeout_generic+0x72 1 0% 100% 0.00 1373 0xfffffe229e460880 timeout_generic+0x72 1 0% 100% 0.00 180 0xfffffe229e460c40 callout_list_expire+0xa1 1 0% 100% 0.00 180 0xfffffe22e1bcd418 bd_runq_exit+0x53 1 0% 100% 0.00 5780 0xfffffe22e1717da8 compute_min_max_threads+0x14 1 0% 100% 0.00 791 0xfffffe22d6c82880 pollsys+0x131 1 0% 100% 0.00 1733 0xfffffe2313a7f050 cv_wait+0x78 1 0% 100% 0.00 681 0xfffffe2313a7f170 cv_wait+0x78 1 0% 100% 0.00 831 0xfffffe2313a7f050 taskq_thread+0x2cd 1 0% 100% 0.00 1613 0xfffffe2313a7f4d0 cv_wait+0x78 1 0% 100% 0.00 180 0xfffffe2313a7f5f0 taskq_thread+0x2cd ------------------------------------------------------------------------------- Adaptive mutex block: 539 events in 10.669 seconds (51 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 86 16% 16% 0.00 12977 0xfffffe235d62ad50 mlxcx_buf_take+0x2f 86 16% 32% 0.00 14651 0xfffffe235ba64b58 mlxcx_buf_take+0x2f 37 7% 39% 0.00 6659 0xfffffe26d35fd600 cv_wait+0x78 35 6% 45% 0.00 7178 0xfffffe256a6a8b40 cv_wait+0x78 34 6% 52% 0.00 6037 0xfffffe26d35fee00 cv_wait+0x78 29 5% 57% 0.00 8044 0xfffffe256a6a4700 cv_wait+0x78 29 5% 62% 0.00 12164 0xfffffe235d631478 mlxcx_sq_add_buffer+0x469 26 5% 67% 0.00 13434 0xfffffe235ba61a78 mlxcx_sq_add_buffer+0x469 25 5% 72% 0.00 104108 overlay_target_lock cv_timedwait_hires+0xf0 24 4% 76% 0.00 6461 0xfffffe26d3600c40 cv_wait+0x78 23 4% 81% 0.00 6095 0xfffffe257a0e64c0 cv_wait+0x78 23 4% 85% 0.00 7192 0xfffffe256a6ab8c0 cv_wait+0x78 13 2% 87% 0.00 10066 0xfffffe235ba61378 mlxcx_process_cq+0x207 10 2% 89% 0.00 7186 0xfffffe26d3601600 cv_wait+0x78 9 2% 91% 0.00 145121 overlay_target_lock overlay_target_lookup_request+0x47 7 1% 92% 0.00 16392 0xfffffe22d6a50268 taskq_thread+0x2cd 7 1% 93% 0.00 11075 0xfffffe22d6186680 hook_run+0x34 6 1% 94% 0.00 12578 0xfffffe23970b0bf8 cv_wait+0x78 5 1% 95% 0.00 13786 0xfffffe23970b0bf8 taskq_thread+0x2cd 5 1% 96% 0.00 16156 0xfffffe235ba64658 mlxcx_buf_return+0x8c 3 1% 97% 0.00 9868 0xfffffe22d6a50c88 cv_wait+0x78 2 0% 97% 0.00 10179 0xfffffe235d631478 mlxcx_process_cq+0x207 1 0% 97% 0.00 7284 0xfffffe2710b70558 zio_remove_child+0x47 1 0% 98% 0.00 7424 0xfffffe26d3600c40 mac_rx_soft_ring_process+0x53 1 0% 98% 0.00 10299 0xfffffe235d62ad50 mlxcx_buf_return+0x8c 1 0% 98% 0.00 2915 0xfffffe235e4d6dd8 mlxcx_buf_loan+0x76 1 0% 98% 0.00 7935 0xfffffe23970b0bf8 taskq_ent_free+0x6d 1 0% 98% 0.00 32762 0xfffffe235ba61a78 mlxcx_process_cq+0x207 1 0% 99% 0.00 14358 0xfffffe231a28f2e8 cv_wait+0x78 1 0% 99% 0.00 24567 0xfffffe2315505000 metaslab_group_alloc_normal+0x1b4 1 0% 99% 0.00 17663 0xfffffe2314dca3c8 taskq_thread+0x2cd 1 0% 99% 0.00 7945 0xfffffe231a28f2e8 taskq_dispatch_ent+0x51 1 0% 99% 0.00 7605 0xfffffe231a28f2e8 taskq_thread+0x2cd 1 0% 99% 0.00 13926 0xfffffe235ba4e340 mlxcx_buf_return+0x8c 1 0% 100% 0.00 5280 0xfffffe229dae0380 kmem_cache_free+0x78 1 0% 100% 0.00 14236 0xfffffe229dacc1d0 kmem_depot_alloc+0x88 1 0% 100% 0.00 168801 0xfffffe2313c7b0d0 squeue_drain+0x18b ------------------------------------------------------------------------------- Spin lock spin: 2209 events in 10.669 seconds (207 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1202 54% 54% 0.00 6563 cp_default disp_lock_enter+0x26 238 11% 65% 0.00 2274 cp_default disp_lock_enter_high+0x11 194 9% 74% 0.00 2220 0xfffffe2315f13008 disp_lock_enter_high+0x11 90 4% 78% 0.00 5794 sleepq_head+0x1068 disp_lock_enter_high+0x11 59 3% 81% 0.00 45693 turnstile_table+0xb48 disp_lock_enter+0x26 41 2% 83% 0.00 1606 0xfffffe22e25d6120 disp_lock_enter+0x26 40 2% 84% 0.00 226 0xfffffe22e25d6510 disp_lock_enter+0x26 36 2% 86% 0.00 497 0xfffffe22e21ad0c0 disp_lock_enter+0x26 35 2% 88% 0.00 1345 0xfffffe22e25d6388 disp_lock_enter+0x26 30 1% 89% 0.00 956 0xfffffe22e1df9268 disp_lock_enter+0x26 24 1% 90% 0.00 1370 0xfffffe2313c65328 disp_lock_enter+0x26 24 1% 91% 0.00 2031 0xfffffe22e1fff778 disp_lock_enter+0x26 22 1% 92% 0.00 5530 0xfffffe231385acd8 disp_lock_enter+0x26 21 1% 93% 0.00 13863 turnstile_table+0x838 disp_lock_enter+0x26 16 1% 94% 0.00 5833 0xfffffe22e21ad980 disp_lock_enter+0x26 10 0% 94% 0.00 3859 sleepq_head+0x6028 disp_lock_enter_high+0x11 10 0% 95% 0.00 1262 0xfffffe22e1fffa18 disp_lock_enter+0x26 9 0% 95% 0.00 1318 0xfffffe2313faaf08 disp_lock_enter+0x26 9 0% 96% 0.00 911 0xfffffe23136b7368 disp_lock_enter+0x26 9 0% 96% 0.00 1250 0xfffffe2313032378 disp_lock_enter+0x26 9 0% 96% 0.00 1658 0xfffffe2312e1edb0 disp_lock_enter+0x26 7 0% 97% 0.00 1123 0xfffffe231328a850 disp_lock_enter+0x26 7 0% 97% 0.00 1312 0xfffffe22e1e0e5e0 disp_lock_enter+0x26 5 0% 97% 0.00 29035 turnstile_table+0xb38 disp_lock_enter+0x26 5 0% 97% 0.00 853 0xfffffe22e21ad018 disp_lock_enter+0x26 5 0% 98% 0.00 861 0xfffffe22e1e0e0a0 disp_lock_enter+0x26 4 0% 98% 0.00 849 turnstile_table+0x8b8 disp_lock_enter+0x26 4 0% 98% 0.00 889 0xfffffe22e1abdb48 disp_lock_enter+0x26 4 0% 98% 0.00 1026 0xfffffe22e1aa76a8 disp_lock_enter+0x26 3 0% 98% 0.00 762 0xfffffe2313e12870 disp_lock_enter+0x26 3 0% 98% 0.00 921 0xfffffe2312e1ebb8 disp_lock_enter+0x26 2 0% 99% 0.00 285 turnstile_table+0xd38 disp_lock_enter+0x26 2 0% 99% 0.00 787 turnstile_table+0xaa8 disp_lock_enter+0x26 2 0% 99% 0.00 1097 0xfffffe231327bd28 disp_lock_enter+0x26 2 0% 99% 0.00 2444 0xfffffe22e25d6388 disp_lock_enter_high+0x11 2 0% 99% 0.00 752 0xfffffe22e1e0ed18 disp_lock_enter+0x26 1 0% 99% 0.00 641 turnstile_table+0xf78 disp_lock_enter+0x26 1 0% 99% 0.00 1213 turnstile_table+0xa88 disp_lock_enter+0x26 1 0% 99% 0.00 581 turnstile_table+0x968 disp_lock_enter+0x26 1 0% 99% 0.00 2595 cpu0_disp disp_lock_enter+0x26 1 0% 99% 0.00 1433 cpu0_disp disp_lock_enter_high+0x11 1 0% 99% 0.00 1393 0xfffffe23136b7368 disp_lock_enter_high+0x11 1 0% 99% 0.00 381 0xfffffe231328a850 disp_lock_enter_high+0x11 1 0% 99% 0.00 1583 0xfffffe2313a88b28 disp_lock_enter_high+0x11 1 0% 99% 0.00 901 0xfffffe2313c65328 disp_lock_enter_high+0x11 1 0% 99% 0.00 461 0xfffffe2313c65d70 disp_lock_enter+0x26 1 0% 99% 0.00 1102 0xfffffe22e219fad8 disp_lock_enter+0x26 1 0% 99% 0.00 431 0xfffffe22e1e0e5e0 disp_lock_enter_high+0x11 1 0% 100% 0.00 1563 0xfffffe22e1e0e2d0 disp_lock_enter+0x26 1 0% 100% 0.00 1543 0xfffffe22e1e0e2d0 disp_lock_enter_high+0x11 1 0% 100% 0.00 1232 0xfffffe22e1ff0358 disp_lock_enter+0x26 1 0% 100% 0.00 1132 0xfffffe22e24e5ae0 disp_lock_enter+0x26 1 0% 100% 0.00 3617 0xfffffe22e25d6120 disp_lock_enter_high+0x11 1 0% 100% 0.00 10460 0xfffffe2312e10610 disp_lock_enter_high+0x11 1 0% 100% 0.00 350 0xfffffe22e2c616d8 disp_lock_enter_high+0x11 1 0% 100% 0.00 751 0xfffffe22e2c53c90 disp_lock_enter+0x26 1 0% 100% 0.00 881 0xfffffe2312e10610 disp_lock_enter+0x26 1 0% 100% 0.00 1192 0xfffffe2312e10f78 disp_lock_enter+0x26 1 0% 100% 0.00 731 0xfffffe22e1df9268 disp_lock_enter_high+0x11 1 0% 100% 0.00 421 0xfffffe22e1aa76a8 disp_lock_enter_high+0x11 ------------------------------------------------------------------------------- Thread lock spin: 8280 events in 10.669 seconds (776 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 7441 90% 90% 0.00 1962 sleepq_head+0x1068 setrun+0x18 667 8% 98% 0.00 2331 sleepq_head+0x6028 setrun+0x18 53 1% 99% 0.00 304 cpu[44]+0xe8 fss_tick+0x24 38 0% 99% 0.00 784 cpu[44]+0xe8 post_syscall+0x392 15 0% 99% 0.00 1487 sleepq_head+0x1068 fss_tick+0x24 11 0% 99% 0.00 570 turnstile_table+0xaa8 fss_tick+0x24 5 0% 99% 0.00 412 turnstile_table+0xfb8 fss_tick+0x24 3 0% 99% 0.00 825 cpu[11]+0xe8 post_syscall+0x538 2 0% 99% 0.00 295 turnstile_table+0xab8 fss_tick+0x24 2 0% 99% 0.00 626 turnstile_table+0xa88 fss_tick+0x24 2 0% 100% 0.00 2169 cpu[54]+0xe8 post_syscall+0x538 2 0% 100% 0.00 11482 cpu[49]+0xe8 fss_tick+0x24 2 0% 100% 0.00 2254 cpu[42]+0xe8 fss_tick+0x24 2 0% 100% 0.00 656 cpu[28]+0xe8 post_syscall+0x538 2 0% 100% 0.00 1347 cpu[24]+0xe8 post_syscall+0x538 2 0% 100% 0.00 756 cpu[8]+0xe8 fss_tick+0x24 2 0% 100% 0.00 747 cpu[11]+0xe8 fss_tick+0x24 2 0% 100% 0.00 691 cpu[6]+0xe8 fss_tick+0x24 2 0% 100% 0.00 21316 cpu[59]+0xe8 fss_tick+0x24 1 0% 100% 0.00 2004 cpu[51]+0xe8 fss_tick+0x24 1 0% 100% 0.00 401 cpu[54]+0xe8 fss_tick+0x24 1 0% 100% 0.00 481 cpu[50]+0xe8 fss_tick+0x24 1 0% 100% 0.00 962 cpu[50]+0xe8 post_syscall+0x538 1 0% 100% 0.00 671 cpu[56]+0xe8 post_syscall+0x538 1 0% 100% 0.00 330 cpu[28]+0xe8 fss_tick+0x24 1 0% 100% 0.00 3377 cpu[40]+0xe8 fss_tick+0x24 1 0% 100% 0.00 641 cpu[48]+0xe8 post_syscall+0x538 1 0% 100% 0.00 631 cpu[41]+0xe8 fss_tick+0x24 1 0% 100% 0.00 2394 cpu[47]+0xe8 fss_tick+0x24 1 0% 100% 0.00 8066 cpu[46]+0xe8 fss_tick+0x24 1 0% 100% 0.00 1583 cpu[55]+0xe8 preempt+0x47 1 0% 100% 0.00 321 cpu[10]+0xe8 post_syscall+0x538 1 0% 100% 0.00 631 cpu[11]+0xe8 post_syscall+0x392 1 0% 100% 0.00 311 cpu[1]+0xe8 post_syscall+0x538 1 0% 100% 0.00 1263 cpu[7]+0xe8 post_syscall+0x538 1 0% 100% 0.00 1232 cpu[8]+0xe8 cv_timedwait_hires+0xaa 1 0% 100% 0.00 1342 cpu[8]+0xe8 post_syscall+0x392 1 0% 100% 0.00 411 cpu[9]+0xe8 fss_tick+0x24 1 0% 100% 0.00 451 cpu[9]+0xe8 post_syscall+0x538 1 0% 100% 0.00 310 cpu[30]+0xe8 fss_tick+0x24 1 0% 100% 0.00 541 cpu[13]+0xe8 post_syscall+0x538 1 0% 100% 0.00 2545 cpu[30]+0xe8 post_syscall+0x392 1 0% 100% 0.00 1112 cpu[30]+0xe8 post_syscall+0x538 1 0% 100% 0.00 1172 cpu[30]+0xe8 cv_timedwait_hires+0xaa ------------------------------------------------------------------------------- Initiator netstat and rx queue drops [root@server4 (us-mobile-1) /zones/global/mlxcx]# netstat -s -P tcp TCP tcpRtoAlgorithm = 4 tcpRtoMin = 400 tcpRtoMax = 60000 tcpMaxConn = -1 tcpActiveOpens = 2499 tcpPassiveOpens = 139 tcpAttemptFails = 6 tcpEstabResets = 0 tcpCurrEstab = 31 tcpOutSegs =29418157 tcpOutDataSegs =14764317 tcpOutDataBytes =4061034655 tcpRetransSegs = 9 tcpRetransBytes = 0 tcpOutAck = 69450 tcpOutAckDelayed = 657 tcpOutUrg = 0 tcpOutWinUpdate = 0 tcpOutWinProbe = 0 tcpOutControl = 5256 tcpOutRsts = 4 tcpOutFastRetrans = 0 tcpInSegs =2385281 tcpInAckSegs = 0 tcpInAckBytes =4060998180 tcpInDupAck = 6589 tcpInAckUnsent = 0 tcpInInorderSegs =523397 tcpInInorderBytes =702442020 tcpInUnorderSegs = 0 tcpInUnorderBytes = 0 tcpInDupSegs = 0 tcpInDupBytes = 0 tcpInPartDupSegs = 0 tcpInPartDupBytes = 0 tcpInPastWinSegs = 0 tcpInPastWinBytes = 0 tcpInWinProbe = 0 tcpInWinUpdate = 0 tcpInClosed = 0 tcpRttNoUpdate = 0 tcpRttUpdate =1860746 tcpTimRetrans = 0 tcpTimRetransDrop = 0 tcpTimKeepalive = 9349 tcpTimKeepaliveProbe= 4506 tcpTimKeepaliveDrop = 0 tcpListenDrop = 0 tcpListenDropQ0 = 0 tcpHalfOpenDrop = 0 tcpOutSackRetrans = 0 [root@server4 (us-mobile-1) /zones/global/mlxcx]# mdb -k Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc apix scsi_vhci ufs ip hook neti sockfs arp usba xhci smbios stmf_sbd stmf zfs mm lofs random idm sata crypto fcp fctl cpc logindmux ptm kvm sppp nsmb smbsrv klmmod nfs vmm ] > ::prtconf !grep mlxcx fffffe22d6252e38 pciex15b3,101d, instance #0 (driver name: mlxcx) fffffe22d6252b68 pciex15b3,101d, instance #1 (driver name: mlxcx) > fffffe22d6252e38::print struct dev_info devi_driver_data |::print mlxcx_t mlx_ports |::print mlxcx_port_t mlp_stats mlp_stats = { mlp_stats.mlps_rx_drops = 0 } > fffffe22d6252b68::print struct dev_info devi_driver_data |::print mlxcx_t mlx_ports |::print mlxcx_port_t mlp_stats mlp_stats = { mlp_stats.mlps_rx_drops = 0 } > Target netstat and rx queue drops [root@server5 (us-mobile-1) /zones/global/mlxcx]# netstat -s -P tcp TCP tcpRtoAlgorithm = 4 tcpRtoMin = 400 tcpRtoMax = 60000 tcpMaxConn = -1 tcpActiveOpens = 2773 tcpPassiveOpens = 189 tcpAttemptFails = 12 tcpEstabResets = 0 tcpCurrEstab = 31 tcpOutSegs =1945675 tcpOutDataSegs = 30544 tcpOutDataBytes =2007006 tcpRetransSegs = 9 tcpRetransBytes = 0 tcpOutAck =1914803 tcpOutAckDelayed = 825 tcpOutUrg = 0 tcpOutWinUpdate = 6 tcpOutWinProbe = 0 tcpOutControl = 5904 tcpOutRsts = 5 tcpOutFastRetrans = 0 tcpInSegs =15305060 tcpInAckSegs = 0 tcpInAckBytes =1965850 tcpInDupAck = 7494 tcpInAckUnsent = 0 tcpInInorderSegs =15280822 tcpInInorderBytes =467548119 tcpInUnorderSegs = 0 tcpInUnorderBytes = 0 tcpInDupSegs = 1 tcpInDupBytes = 0 tcpInPartDupSegs = 0 tcpInPartDupBytes = 0 tcpInPastWinSegs = 0 tcpInPastWinBytes = 0 tcpInWinProbe = 0 tcpInWinUpdate = 0 tcpInClosed = 0 tcpRttNoUpdate = 0 tcpRttUpdate = 29035 tcpTimRetrans = 0 tcpTimRetransDrop = 0 tcpTimKeepalive = 10637 tcpTimKeepaliveProbe= 5116 tcpTimKeepaliveDrop = 0 tcpListenDrop = 0 tcpListenDropQ0 = 0 tcpHalfOpenDrop = 0 tcpOutSackRetrans = 0 [root@server5 (us-mobile-1) /zones/global/mlxcx]# mdb -k Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc apix scsi_vhci ufs ip hook neti sockfs arp usba xhci smbios stmf_sbd stmf zfs mm lofs random idm sata crypto fcp fctl cpc logindmux ptm kvm sppp nsmb smbsrv klmmod nfs vmm ] > mlxcx_softstate/p | ::softstate 0 | ::print mlxcx_t mlx_ports->mlp_stats.mlps_rx_drops mlx_ports->mlp_stats.mlps_rx_drops = 0 > mlxcx_softstate/p | ::softstate 1 | ::print mlxcx_t mlx_ports->mlp_stats.mlps_rx_drops mlx_ports->mlp_stats.mlps_rx_drops = 0 >