Print this page
MFV: illumos-joyent@61dc3dec4f82a3e13e94609a0a83d5f66c64e760
OS-6846 want i40e multi-group support
OS-7372 i40e_alloc_ring_mem() unwinds when it shouldn't
Reviewed by: Robert Mustacchi <rm@joyent.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Author: Ryan Zezeski <rpz@joyent.com>
NEX-13226 xvv710 25Gb NIC panics system under load
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
NEX-7822 40Gb Intel XL710 NIC performance data
Reviewed by: Steve Peng <steve.peng@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
NEX-6977 Ericsson hangs on reboot with Intel XL710 NICs
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/i40e/i40e_intr.c
+++ new/usr/src/uts/common/io/i40e/i40e_intr.c
1 1 /*
2 2 * This file and its contents are supplied under the terms of the
|
↓ open down ↓ |
2 lines elided |
↑ open up ↑ |
3 3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 4 * You may only use this file in accordance with the terms of version
5 5 * 1.0 of the CDDL.
6 6 *
7 7 * A full copy of the text of the CDDL should have accompanied this
8 8 * source. A copy of the CDDL is also available via the Internet at
9 9 * http://www.illumos.org/license/CDDL.
10 10 */
11 11
12 12 /*
13 - * Copyright (c) 2017, Joyent, Inc.
13 + * Copyright 2018 Joyent, Inc.
14 14 * Copyright 2017 Tegile Systems, Inc. All rights reserved.
15 15 */
16 16
17 17 /*
18 18 * -------------------------
19 19 * Interrupt Handling Theory
20 20 * -------------------------
21 21 *
22 22 * There are a couple different sets of interrupts that we need to worry about:
23 23 *
24 24 * - Interrupts from receive queues
25 25 * - Interrupts from transmit queues
26 26 * - 'Other Interrupts', such as the administrative queue
27 27 *
28 28 * 'Other Interrupts' are asynchronous events such as a link status change event
29 29 * being posted to the administrative queue, unrecoverable ECC errors, and more.
30 30 * If we have something being posted to the administrative queue, then we go
31 31 * through and process it, because it's generally enabled as a separate logical
32 32 * interrupt. Note, we may need to do more here eventually. To re-enable the
33 33 * interrupts from the 'Other Interrupts' section, we need to clear the PBA and
34 34 * write ENA to PFINT_ICR0.
35 35 *
36 36 * Interrupts from the transmit and receive queues indicates that our requests
37 37 * have been processed. In the rx case, it means that we have data that we
38 38 * should take a look at and send up the stack. In the tx case, it means that
39 39 * data which we got from MAC has now been sent out on the wire and we can free
40 40 * the associated data. Most of the logic for acting upon the presence of this
41 41 * data can be found in i40e_transciever.c which handles all of the DMA, rx, and
42 42 * tx operations. This file is dedicated to handling and dealing with interrupt
43 43 * processing.
44 44 *
45 45 * All devices supported by this driver support three kinds of interrupts:
46 46 *
47 47 * o Extended Message Signaled Interrupts (MSI-X)
48 48 * o Message Signaled Interrupts (MSI)
49 49 * o Legacy PCI interrupts (INTx)
50 50 *
51 51 * Generally speaking the hardware logically handles MSI and INTx the same and
52 52 * restricts us to only using a single interrupt, which isn't the interesting
53 53 * case. With MSI-X available, each physical function of the device provides the
54 54 * opportunity for multiple interrupts which is what we'll focus on.
55 55 *
56 56 * --------------------
57 57 * Interrupt Management
58 58 * --------------------
59 59 *
60 60 * By default, the admin queue, which consists of the asynchronous other
61 61 * interrupts is always bound to MSI-X vector zero. Next, we spread out all of
62 62 * the other interrupts that we have available to us over the remaining
63 63 * interrupt vectors.
64 64 *
65 65 * This means that there may be multiple queues, both tx and rx, which are
66 66 * mapped to the same interrupt. When the interrupt fires, we'll have to check
67 67 * all of them for servicing, before we go through and indicate that the
68 68 * interrupt is claimed.
69 69 *
70 70 * The hardware provides the means of mapping various queues to MSI-X interrupts
71 71 * by programming the I40E_QINT_RQCTL() and I4OE_QINT_TQCTL() registers. These
72 72 * registers can also be used to enable and disable whether or not the queue is
73 73 * a source of interrupts. As part of this, the hardware requires that we
74 74 * maintain a linked list of queues for each interrupt vector. While it may seem
75 75 * like this is only there for the purproses of ITRs, that's not the case. The
76 76 * first queue must be programmed in I40E_QINT_LNKLSTN(%vector) register. Each
77 77 * queue defines the next one in either the I40E_QINT_RQCTL or I40E_QINT_TQCTL
78 78 * register.
79 79 *
80 80 * Finally, the individual interrupt vector itself has the ability to be enabled
81 81 * and disabled. The overall interrupt is controlled through the
82 82 * I40E_PFINT_DYN_CTLN() register. This is used to turn on and off the interrupt
83 83 * as a whole.
84 84 *
85 85 * Note that this means that both the individual queue and the interrupt as a
86 86 * whole can be toggled and re-enabled.
87 87 *
88 88 * -------------------
89 89 * Non-MSIX Management
90 90 * -------------------
91 91 *
92 92 * We may have a case where the Operating System is unable to actually allocate
93 93 * any MSI-X to the system. In such a world, there is only one transmit/receive
94 94 * queue pair and it is bound to the same interrupt with index zero. The
95 95 * hardware doesn't allow us access to additional interrupt vectors in these
96 96 * modes. Note that technically we could support more transmit/receive queues if
97 97 * we wanted.
98 98 *
99 99 * In this world, because the interrupts for the admin queue and traffic are
100 100 * mixed together, we have to consult ICR0 to determine what has occurred. The
101 101 * QINT_TQCTL and QINT_RQCTL registers have a field, 'MSI-X 0 index' which
102 102 * allows us to set a specific bit in ICR0. There are up to seven such bits;
103 103 * however, we only use the bit 0 and 1 for the rx and tx queue respectively.
104 104 * These are contained by the I40E_INTR_NOTX_{R|T}X_QUEUE and
105 105 * I40E_INTR_NOTX_{R|T}X_MASK registers respectively.
106 106 *
107 107 * Unfortunately, these corresponding queue bits have no corresponding entry in
108 108 * the ICR0_ENA register. So instead, when enabling interrupts on the queues, we
109 109 * end up enabling it on the queue registers rather than on the MSI-X registers.
110 110 * In the MSI-X world, because they can be enabled and disabled, this is
111 111 * different and the queues can always be enabled and disabled, but the
112 112 * interrupts themselves are toggled (ignoring the question of interrupt
113 113 * blanking for polling on rings).
114 114 *
115 115 * Finally, we still have to set up the interrupt linked list, but the list is
116 116 * instead rooted at the register I40E_PFINT_LNKLST0, rather than being tied to
117 117 * one of the other MSI-X registers.
118 118 *
119 119 * --------------------
120 120 * Interrupt Moderation
121 121 * --------------------
122 122 *
123 123 * The XL710 hardware has three different interrupt moderation registers per
124 124 * interrupt. Unsurprisingly, we use these for:
125 125 *
126 126 * o RX interrupts
127 127 * o TX interrupts
128 128 * o 'Other interrupts' (link status change, admin queue, etc.)
129 129 *
130 130 * By default, we throttle 'other interrupts' the most, then TX interrupts, and
131 131 * then RX interrupts. The default values for these were based on trying to
132 132 * reason about both the importance and frequency of events. Generally speaking
133 133 * 'other interrupts' are not very frequent and they're not important for the
134 134 * I/O data path in and of itself (though they may indicate issues with the I/O
135 135 * data path).
136 136 *
137 137 * On the flip side, when we're not polling, RX interrupts are very important.
138 138 * The longer we wait for them, the more latency that we inject into the system.
139 139 * However, if we allow interrupts to occur too frequently, we risk a few
140 140 * problems:
141 141 *
142 142 * 1) Abusing system resources. Without proper interrupt blanking and polling,
143 143 * we can see upwards of 200k-300k interrupts per second on the system.
144 144 *
145 145 * 2) Not enough data coalescing to enable polling. In other words, the more
146 146 * data that we allow to build up, the more likely we'll be able to enable
147 147 * polling mode and allowing us to better handle bulk data.
148 148 *
149 149 * In-between the 'other interrupts' and the TX interrupts we have the
150 150 * reclamation of TX buffers. This operation is not quite as important as we
151 151 * generally size the ring large enough that we should be able to reclaim a
152 152 * substantial amount of the descriptors that we have used per interrupt. So
153 153 * while it's important that this interrupt occur, we don't necessarily need it
154 154 * firing as frequently as RX; it doesn't, on its own, induce additional latency
155 155 * into the system.
156 156 *
157 157 * Based on all this we currently assign static ITR values for the system. While
158 158 * we could move to a dynamic system (the hardware supports that), we'd want to
159 159 * make sure that we're seeing problems from this that we believe would be
160 160 * generally helped by the added complexity.
161 161 *
162 162 * Based on this, the default values that we have allow for the following
163 163 * interrupt thresholds:
164 164 *
165 165 * o 20k interrupts/s for RX
166 166 * o 5k interrupts/s for TX
167 167 * o 2k interupts/s for 'Other Interrupts'
168 168 */
169 169
170 170 #include "i40e_sw.h"
171 171
172 172 #define I40E_INTR_NOTX_QUEUE 0
173 173 #define I40E_INTR_NOTX_INTR 0
174 174 #define I40E_INTR_NOTX_RX_QUEUE 0
175 175 #define I40E_INTR_NOTX_RX_MASK (1 << I40E_PFINT_ICR0_QUEUE_0_SHIFT)
176 176 #define I40E_INTR_NOTX_TX_QUEUE 1
177 177 #define I40E_INTR_NOTX_TX_MASK (1 << I40E_PFINT_ICR0_QUEUE_1_SHIFT)
178 178
179 179 void
180 180 i40e_intr_set_itr(i40e_t *i40e, i40e_itr_index_t itr, uint_t val)
181 181 {
182 182 int i;
183 183 i40e_hw_t *hw = &i40e->i40e_hw_space;
184 184
185 185 VERIFY3U(val, <=, I40E_MAX_ITR);
186 186 VERIFY3U(itr, <, I40E_ITR_INDEX_NONE);
187 187
188 188 /*
189 189 * No matter the interrupt mode, the ITR for other interrupts is always
190 190 * on interrupt zero and the same is true if we're not using MSI-X.
191 191 */
192 192 if (itr == I40E_ITR_INDEX_OTHER ||
193 193 i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) {
194 194 I40E_WRITE_REG(hw, I40E_PFINT_ITR0(itr), val);
195 195 return;
196 196 }
197 197
198 198 for (i = 0; i < i40e->i40e_num_trqpairs; i++) {
199 199 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(itr, i), val);
200 200 }
201 201 }
202 202
203 203 /*
204 204 * Re-enable the adminq. Note that the adminq doesn't have a traditional queue
205 205 * associated with it from an interrupt perspective and just lives on ICR0.
206 206 * However when MSI-X interrupts are not being used, then this also enables and
207 207 * disables those interrupts.
208 208 */
209 209 static void
210 210 i40e_intr_adminq_enable(i40e_t *i40e)
211 211 {
212 212 i40e_hw_t *hw = &i40e->i40e_hw_space;
213 213 uint32_t reg;
214 214
215 215 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
216 216 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
217 217 (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
218 218 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg);
219 219 i40e_flush(hw);
220 220 }
221 221
|
↓ open down ↓ |
198 lines elided |
↑ open up ↑ |
222 222 static void
223 223 i40e_intr_adminq_disable(i40e_t *i40e)
224 224 {
225 225 i40e_hw_t *hw = &i40e->i40e_hw_space;
226 226 uint32_t reg;
227 227
228 228 reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
229 229 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg);
230 230 }
231 231
232 +/*
233 + * The next two functions enable/disable the reception of interrupts
234 + * on the given vector. Only vectors 1..N are programmed by these
235 + * functions; vector 0 is special and handled by a different register.
236 + * We must subtract one from the vector because i40e implicitly adds
237 + * one to the vector value. See section 10.2.2.10.13 for more details.
238 + */
232 239 static void
233 240 i40e_intr_io_enable(i40e_t *i40e, int vector)
234 241 {
235 242 uint32_t reg;
236 243 i40e_hw_t *hw = &i40e->i40e_hw_space;
237 244
245 + ASSERT3S(vector, >, 0);
238 246 reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
239 247 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
240 248 (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
241 249 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg);
242 250 }
243 251
244 252 static void
245 253 i40e_intr_io_disable(i40e_t *i40e, int vector)
246 254 {
247 255 uint32_t reg;
248 256 i40e_hw_t *hw = &i40e->i40e_hw_space;
249 257
258 + ASSERT3S(vector, >, 0);
250 259 reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
251 260 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg);
252 261 }
253 262
254 263 /*
255 264 * When MSI-X interrupts are being used, then we can enable the actual
256 265 * interrupts themselves. However, when they are not, we instead have to turn
257 266 * towards the queue's CAUSE_ENA bit and enable that.
258 267 */
259 268 void
260 269 i40e_intr_io_enable_all(i40e_t *i40e)
261 270 {
262 271 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
263 272 int i;
264 273
265 274 for (i = 1; i < i40e->i40e_intr_count; i++) {
266 275 i40e_intr_io_enable(i40e, i);
267 276 }
268 277 } else {
269 278 uint32_t reg;
270 279 i40e_hw_t *hw = &i40e->i40e_hw_space;
271 280
272 281 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE));
273 282 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
274 283 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg);
275 284
276 285 reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE));
277 286 reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
278 287 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg);
279 288 }
280 289 }
281 290
282 291 /*
283 292 * When MSI-X interrupts are being used, then we can disable the actual
284 293 * interrupts themselves. However, when they are not, we instead have to turn
285 294 * towards the queue's CAUSE_ENA bit and disable that.
286 295 */
287 296 void
288 297 i40e_intr_io_disable_all(i40e_t *i40e)
289 298 {
290 299 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
291 300 int i;
292 301
293 302 for (i = 1; i < i40e->i40e_intr_count; i++) {
294 303 i40e_intr_io_disable(i40e, i);
295 304 }
296 305 } else {
297 306 uint32_t reg;
298 307 i40e_hw_t *hw = &i40e->i40e_hw_space;
299 308
300 309 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE));
301 310 reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
302 311 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg);
303 312
304 313 reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE));
305 314 reg &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
306 315 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg);
307 316 }
308 317 }
309 318
310 319 /*
311 320 * As part of disabling the tx and rx queue's we're technically supposed to
312 321 * remove the linked list entries. The simplest way is to clear the LNKLSTN
313 322 * register by setting it to I40E_QUEUE_TYPE_EOL (0x7FF).
314 323 *
315 324 * Note all of the FM register access checks are performed by the caller.
316 325 */
317 326 void
318 327 i40e_intr_io_clear_cause(i40e_t *i40e)
319 328 {
320 329 int i;
321 330 i40e_hw_t *hw = &i40e->i40e_hw_space;
322 331
323 332 if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) {
324 333 uint32_t reg;
325 334 reg = I40E_QUEUE_TYPE_EOL;
326 335 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg);
327 336 return;
328 337 }
329 338
330 339 for (i = 0; i < i40e->i40e_num_trqpairs; i++) {
331 340 uint32_t reg;
332 341 #ifdef DEBUG
333 342 /*
334 343 * Verify that the interrupt in question is disabled. This is a
335 344 * prerequisite of modifying the data in question.
336 345 */
337 346 reg = I40E_READ_REG(hw, I40E_PFINT_DYN_CTLN(i));
338 347 VERIFY0(reg & I40E_PFINT_DYN_CTLN_INTENA_MASK);
339 348 #endif
340 349 reg = I40E_QUEUE_TYPE_EOL;
341 350 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(i), reg);
342 351 }
343 352
344 353 i40e_flush(hw);
345 354 }
346 355
347 356 /*
348 357 * Finalize interrupt handling. Mostly this disables the admin queue.
349 358 */
350 359 void
351 360 i40e_intr_chip_fini(i40e_t *i40e)
352 361 {
353 362 #ifdef DEBUG
354 363 int i;
355 364 uint32_t reg;
356 365
357 366 i40e_hw_t *hw = &i40e->i40e_hw_space;
358 367
359 368 /*
360 369 * Take a look and verify that all other interrupts have been disabled
361 370 * and the interrupt linked lists have been zeroed.
362 371 */
363 372 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
364 373 for (i = 0; i < i40e->i40e_num_trqpairs; i++) {
365 374 reg = I40E_READ_REG(hw, I40E_PFINT_DYN_CTLN(i));
366 375 VERIFY0(reg & I40E_PFINT_DYN_CTLN_INTENA_MASK);
367 376
|
↓ open down ↓ |
108 lines elided |
↑ open up ↑ |
368 377 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i));
369 378 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL);
370 379 }
371 380 }
372 381 #endif
373 382
374 383 i40e_intr_adminq_disable(i40e);
375 384 }
376 385
377 386 /*
378 - * Enable all of the queues and set the corresponding LNKLSTN registers. Note
379 - * that we always enable queues as interrupt sources, even though we don't
380 - * enable the MSI-X interrupt vectors.
387 + * Set the head of the interrupt linked list. The PFINT_LNKLSTN[N]
388 + * register actually refers to the 'N + 1' interrupt vector. E.g.,
389 + * PFINT_LNKLSTN[0] refers to interrupt vector 1.
381 390 */
382 391 static void
392 +i40e_set_lnklstn(i40e_t *i40e, uint_t vector, uint_t queue)
393 +{
394 + uint32_t reg;
395 + i40e_hw_t *hw = &i40e->i40e_hw_space;
396 +
397 + reg = (queue << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
398 + (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT);
399 +
400 + I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(vector), reg);
401 + DEBUGOUT2("PFINT_LNKLSTN[%u] = 0x%x", vector, reg);
402 +}
403 +
404 +/*
405 + * Set the QINT_RQCTL[queue] register. The next queue is always the Tx
406 + * queue associated with this Rx queue. Unlike PFINT_LNKLSTN, the
407 + * vector should be the actual vector this queue is on -- i.e., it
408 + * should be equal to itrq_rx_intrvec.
409 + */
410 +static void
411 +i40e_set_rqctl(i40e_t *i40e, uint_t vector, uint_t queue)
412 +{
413 + uint32_t reg;
414 + i40e_hw_t *hw = &i40e->i40e_hw_space;
415 +
416 + ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_rx_intrvec);
417 +
418 + reg = (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
419 + (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
420 + (queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
421 + (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
422 + I40E_QINT_RQCTL_CAUSE_ENA_MASK;
423 +
424 + I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg);
425 + DEBUGOUT2("QINT_RQCTL[%u] = 0x%x", queue, reg);
426 +}
427 +
428 +/*
429 + * Like i40e_set_rqctl(), but for QINT_TQCTL[queue]. The next queue is
430 + * either the Rx queue of another TRQP, or EOL.
431 + */
432 +static void
433 +i40e_set_tqctl(i40e_t *i40e, uint_t vector, uint_t queue, uint_t next_queue)
434 +{
435 + uint32_t reg;
436 + i40e_hw_t *hw = &i40e->i40e_hw_space;
437 +
438 + ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_tx_intrvec);
439 +
440 + reg = (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
441 + (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
442 + (next_queue << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
443 + (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) |
444 + I40E_QINT_TQCTL_CAUSE_ENA_MASK;
445 +
446 + I40E_WRITE_REG(hw, I40E_QINT_TQCTL(queue), reg);
447 + DEBUGOUT2("QINT_TQCTL[%u] = 0x%x", queue, reg);
448 +}
449 +
450 +/*
451 + * Program the interrupt linked list. Each vector has a linked list of
452 + * queues which act as event sources for that vector. When one of
453 + * those sources has an event the associated interrupt vector is
454 + * fired. This mapping must match the mapping found in
455 + * i40e_map_intrs_to_vectors().
456 + *
457 + * See section 7.5.3 for more information about the configuration of
458 + * the interrupt linked list.
459 + */
460 +static void
383 461 i40e_intr_init_queue_msix(i40e_t *i40e)
384 462 {
385 - i40e_hw_t *hw = &i40e->i40e_hw_space;
386 - uint32_t reg;
387 - int i;
463 + uint_t intr_count;
388 464
389 465 /*
390 - * Map queues to MSI-X interrupts. Queue i is mapped to vector i + 1.
391 - * Note that we skip the ITR logic for the moment, just to make our
392 - * lives as explicit and simple as possible.
466 + * The 0th vector is for 'Other Interrupts' only (subject to
467 + * change in the future).
393 468 */
394 - for (i = 0; i < i40e->i40e_num_trqpairs; i++) {
395 - i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i];
469 + intr_count = i40e->i40e_intr_count - 1;
396 470
397 - reg = (i << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
398 - (I40E_QUEUE_TYPE_RX <<
399 - I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT);
400 - I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(i), reg);
471 + for (uint_t vec = 0; vec < intr_count; vec++) {
472 + boolean_t head = B_TRUE;
401 473
402 - reg =
403 - (itrq->itrq_rx_intrvec << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
404 - (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
405 - (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
406 - (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
407 - I40E_QINT_RQCTL_CAUSE_ENA_MASK;
474 + for (uint_t qidx = vec; qidx < i40e->i40e_num_trqpairs;
475 + qidx += intr_count) {
476 + uint_t next_qidx = qidx + intr_count;
408 477
409 - I40E_WRITE_REG(hw, I40E_QINT_RQCTL(i), reg);
478 + next_qidx = (next_qidx > i40e->i40e_num_trqpairs) ?
479 + I40E_QUEUE_TYPE_EOL : next_qidx;
410 480
411 - reg =
412 - (itrq->itrq_tx_intrvec << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
413 - (I40E_ITR_INDEX_TX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
414 - (I40E_QUEUE_TYPE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
415 - (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) |
416 - I40E_QINT_TQCTL_CAUSE_ENA_MASK;
481 + if (head) {
482 + i40e_set_lnklstn(i40e, vec, qidx);
483 + head = B_FALSE;
484 + }
417 485
418 - I40E_WRITE_REG(hw, I40E_QINT_TQCTL(i), reg);
486 + i40e_set_rqctl(i40e, vec + 1, qidx);
487 + i40e_set_tqctl(i40e, vec + 1, qidx, next_qidx);
488 + }
419 489 }
420 -
421 490 }
422 491
423 492 /*
424 493 * Set up a single queue to share the admin queue interrupt in the non-MSI-X
425 494 * world. Note we do not enable the queue as an interrupt cause at this time. We
426 495 * don't have any other vector of control here, unlike with the MSI-X interrupt
427 496 * case.
428 497 */
429 498 static void
430 499 i40e_intr_init_queue_shared(i40e_t *i40e)
431 500 {
432 501 i40e_hw_t *hw = &i40e->i40e_hw_space;
433 502 uint32_t reg;
434 503
435 504 VERIFY(i40e->i40e_intr_type == DDI_INTR_TYPE_FIXED ||
436 505 i40e->i40e_intr_type == DDI_INTR_TYPE_MSI);
437 506
438 507 reg = (I40E_INTR_NOTX_QUEUE << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
439 508 (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT);
440 509 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg);
441 510
442 511 reg = (I40E_INTR_NOTX_INTR << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
443 512 (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
444 513 (I40E_INTR_NOTX_RX_QUEUE << I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) |
445 514 (I40E_INTR_NOTX_QUEUE << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
446 515 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
447 516
448 517 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg);
449 518
450 519 reg = (I40E_INTR_NOTX_INTR << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
451 520 (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
452 521 (I40E_INTR_NOTX_TX_QUEUE << I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) |
453 522 (I40E_QUEUE_TYPE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
454 523 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
455 524
456 525 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg);
457 526 }
458 527
459 528 /*
460 529 * Enable the specified queue as a valid source of interrupts. Note, this should
461 530 * only be used as part of the GLDv3's interrupt blanking routines. The debug
462 531 * build assertions are specific to that.
463 532 */
464 533 void
465 534 i40e_intr_rx_queue_enable(i40e_trqpair_t *itrq)
466 535 {
467 536 uint32_t reg;
468 537 uint_t queue = itrq->itrq_index;
469 538 i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space;
470 539
471 540 ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock));
472 541 ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs);
473 542
474 543 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue));
475 544 ASSERT0(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK);
476 545 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
477 546 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg);
478 547 }
479 548
480 549 /*
481 550 * Disable the specified queue as a valid source of interrupts. Note, this
482 551 * should only be used as part of the GLDv3's interrupt blanking routines. The
483 552 * debug build assertions are specific to that.
484 553 */
485 554 void
486 555 i40e_intr_rx_queue_disable(i40e_trqpair_t *itrq)
487 556 {
488 557 uint32_t reg;
489 558 uint_t queue = itrq->itrq_index;
490 559 i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space;
491 560
492 561 ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock));
493 562 ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs);
494 563
495 564 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue));
496 565 ASSERT3U(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK, ==,
497 566 I40E_QINT_RQCTL_CAUSE_ENA_MASK);
498 567 reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
499 568 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg);
500 569 }
501 570
502 571 /*
503 572 * Start up the various chip's interrupt handling. We not only configure the
504 573 * adminq here, but we also go through and configure all of the actual queues,
505 574 * the interrupt linked lists, and others.
506 575 */
507 576 void
508 577 i40e_intr_chip_init(i40e_t *i40e)
509 578 {
510 579 i40e_hw_t *hw = &i40e->i40e_hw_space;
511 580 uint32_t reg;
512 581
513 582 /*
514 583 * Ensure that all non adminq interrupts are disabled at the chip level.
515 584 */
516 585 i40e_intr_io_disable_all(i40e);
517 586
518 587 I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, 0);
519 588 (void) I40E_READ_REG(hw, I40E_PFINT_ICR0);
520 589
521 590 /*
522 591 * Always enable all of the other-class interrupts to be on their own
523 592 * ITR. This only needs to be set on interrupt zero, which has its own
524 593 * special setting.
525 594 */
526 595 reg = I40E_ITR_INDEX_OTHER << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT;
527 596 I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0, reg);
528 597
529 598 /*
530 599 * Enable interrupt types we expect to receive. At the moment, this
531 600 * is limited to the adminq; however, we'll want to review 11.2.2.9.22
532 601 * for more types here as we add support for detecting them, handling
533 602 * them, and resetting the device as appropriate.
534 603 */
535 604 reg = I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
536 605 I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg);
537 606
538 607 /*
539 608 * Always set the interrupt linked list to empty. We'll come back and
540 609 * change this if MSI-X are actually on the scene.
541 610 */
542 611 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_TYPE_EOL);
543 612
544 613 i40e_intr_adminq_enable(i40e);
545 614
546 615 /*
547 616 * Set up all of the queues and map them to interrupts based on the bit
548 617 * assignments.
549 618 */
550 619 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
551 620 i40e_intr_init_queue_msix(i40e);
552 621 } else {
553 622 i40e_intr_init_queue_shared(i40e);
554 623 }
555 624
556 625 /*
557 626 * Finally set all of the default ITRs for the interrupts. Note that the
558 627 * queues will have been set up above.
559 628 */
560 629 i40e_intr_set_itr(i40e, I40E_ITR_INDEX_RX, i40e->i40e_rx_itr);
561 630 i40e_intr_set_itr(i40e, I40E_ITR_INDEX_TX, i40e->i40e_tx_itr);
562 631 i40e_intr_set_itr(i40e, I40E_ITR_INDEX_OTHER, i40e->i40e_other_itr);
563 632 }
564 633
565 634 static void
566 635 i40e_intr_adminq_work(i40e_t *i40e)
567 636 {
568 637 struct i40e_hw *hw = &i40e->i40e_hw_space;
569 638 struct i40e_arq_event_info evt;
570 639 uint16_t remain = 1;
571 640
572 641 bzero(&evt, sizeof (struct i40e_arq_event_info));
573 642 evt.buf_len = I40E_ADMINQ_BUFSZ;
574 643 evt.msg_buf = i40e->i40e_aqbuf;
575 644
576 645 while (remain != 0) {
577 646 enum i40e_status_code ret;
578 647 uint16_t opcode;
579 648
580 649 /*
|
↓ open down ↓ |
150 lines elided |
↑ open up ↑ |
581 650 * At the moment, the only error code that seems to be returned
582 651 * is one saying that there's no work. In such a case we leave
583 652 * this be.
584 653 */
585 654 ret = i40e_clean_arq_element(hw, &evt, &remain);
586 655 if (ret != I40E_SUCCESS)
587 656 break;
588 657
589 658 opcode = LE_16(evt.desc.opcode);
590 659 switch (opcode) {
591 - case i40e_aqc_opc_get_link_status:
592 - mutex_enter(&i40e->i40e_general_lock);
593 - i40e_link_check(i40e);
594 - mutex_exit(&i40e->i40e_general_lock);
595 - break;
660 + /*
661 + * Disable link checks for NEX-6977. With the fibers unplugged
662 + * we can end up receiving too many link check interrupts,
663 + * saturating one CPU for each link. This can cause system hangs
664 + * at boot or shutdown when the system is running single-threaded.
665 + *
666 + * case i40e_aqc_opc_get_link_status:
667 + * mutex_enter(&i40e->i40e_general_lock);
668 + * i40e_link_check(i40e);
669 + * mutex_exit(&i40e->i40e_general_lock);
670 + * break;
671 + */
596 672 default:
597 673 /*
598 674 * Longer term we'll want to enable other causes here
599 675 * and get these cleaned up and doing something.
600 676 */
601 677 break;
602 678 }
603 679 }
604 680 }
605 681
606 682 static void
607 -i40e_intr_rx_work(i40e_t *i40e, int queue)
683 +i40e_intr_rx_work(i40e_t *i40e, i40e_trqpair_t *itrq)
608 684 {
609 685 mblk_t *mp = NULL;
610 - i40e_trqpair_t *itrq;
611 686
612 - ASSERT(queue < i40e->i40e_num_trqpairs);
613 - itrq = &i40e->i40e_trqpairs[queue];
614 -
615 687 mutex_enter(&itrq->itrq_rx_lock);
616 688 if (!itrq->itrq_intr_poll)
617 689 mp = i40e_ring_rx(itrq, I40E_POLL_NULL);
618 690 mutex_exit(&itrq->itrq_rx_lock);
619 691
620 - if (mp != NULL) {
621 - mac_rx_ring(i40e->i40e_mac_hdl, itrq->itrq_macrxring, mp,
622 - itrq->itrq_rxgen);
623 - }
692 + if (mp == NULL)
693 + return;
694 +
695 + mac_rx_ring(i40e->i40e_mac_hdl, itrq->itrq_macrxring, mp,
696 + itrq->itrq_rxgen);
624 697 }
625 698
699 +/* ARGSUSED */
626 700 static void
627 -i40e_intr_tx_work(i40e_t *i40e, int queue)
701 +i40e_intr_tx_work(i40e_t *i40e, i40e_trqpair_t *itrq)
628 702 {
629 - i40e_trqpair_t *itrq;
630 -
631 - itrq = &i40e->i40e_trqpairs[queue];
632 703 i40e_tx_recycle_ring(itrq);
633 704 }
634 705
635 706 /*
636 707 * At the moment, the only 'other' interrupt on ICR0 that we handle is the
637 708 * adminq. We should go through and support the other notifications at some
638 709 * point.
639 710 */
640 711 static void
641 712 i40e_intr_other_work(i40e_t *i40e)
642 713 {
643 714 struct i40e_hw *hw = &i40e->i40e_hw_space;
644 715 uint32_t reg;
645 716
646 717 reg = I40E_READ_REG(hw, I40E_PFINT_ICR0);
647 718 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) !=
648 719 DDI_FM_OK) {
649 720 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED);
650 721 atomic_or_32(&i40e->i40e_state, I40E_ERROR);
651 722 return;
652 723 }
653 724
654 725 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
655 726 i40e_intr_adminq_work(i40e);
656 727
657 728 /*
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
658 729 * Make sure that the adminq interrupt is not masked and then explicitly
659 730 * enable the adminq and thus the other interrupt.
660 731 */
661 732 reg = I40E_READ_REG(hw, I40E_PFINT_ICR0_ENA);
662 733 reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
663 734 I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg);
664 735
665 736 i40e_intr_adminq_enable(i40e);
666 737 }
667 738
739 +/*
740 + * Handle an MSI-X interrupt. See section 7.5.1.3 for an overview of
741 + * the MSI-X interrupt sequence.
742 + */
668 743 uint_t
669 744 i40e_intr_msix(void *arg1, void *arg2)
670 745 {
671 746 i40e_t *i40e = (i40e_t *)arg1;
672 - int vector_idx = (int)(uintptr_t)arg2;
747 + uint_t vector_idx = (uint_t)(uintptr_t)arg2;
673 748
749 + ASSERT3U(vector_idx, <, i40e->i40e_intr_count);
750 +
674 751 /*
675 752 * When using MSI-X interrupts, vector 0 is always reserved for the
676 753 * adminq at this time. Though longer term, we'll want to also bridge
677 754 * some I/O to them.
678 755 */
679 756 if (vector_idx == 0) {
680 757 i40e_intr_other_work(i40e);
681 758 return (DDI_INTR_CLAIMED);
682 759 }
683 760
684 - i40e_intr_rx_work(i40e, vector_idx - 1);
685 - i40e_intr_tx_work(i40e, vector_idx - 1);
686 - i40e_intr_io_enable(i40e, vector_idx);
761 + ASSERT3U(vector_idx, >, 0);
687 762
763 + /*
764 + * We determine the queue indexes via simple arithmetic (as
765 + * opposed to keeping explicit state like a bitmap). While
766 + * conveinent, it does mean that i40e_map_intrs_to_vectors(),
767 + * i40e_intr_init_queue_msix(), and this function must be
768 + * modified as a unit.
769 + *
770 + * We subtract 1 from the vector to offset the addition we
771 + * performed during i40e_map_intrs_to_vectors().
772 + */
773 + for (uint_t i = vector_idx - 1; i < i40e->i40e_num_trqpairs;
774 + i += (i40e->i40e_intr_count - 1)) {
775 + i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i];
776 +
777 + ASSERT3U(i, <, i40e->i40e_num_trqpairs);
778 + ASSERT3P(itrq, !=, NULL);
779 + i40e_intr_rx_work(i40e, itrq);
780 + i40e_intr_tx_work(i40e, itrq);
781 + }
782 +
783 + i40e_intr_io_enable(i40e, vector_idx);
688 784 return (DDI_INTR_CLAIMED);
689 785 }
690 786
691 787 static uint_t
692 788 i40e_intr_notx(i40e_t *i40e, boolean_t shared)
693 789 {
694 790 i40e_hw_t *hw = &i40e->i40e_hw_space;
695 791 uint32_t reg;
792 + i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[0];
696 793 int ret = DDI_INTR_CLAIMED;
697 794
698 795 if (shared == B_TRUE) {
699 796 mutex_enter(&i40e->i40e_general_lock);
700 797 if (i40e->i40e_state & I40E_SUSPENDED) {
701 798 mutex_exit(&i40e->i40e_general_lock);
702 799 return (DDI_INTR_UNCLAIMED);
703 800 }
704 801 mutex_exit(&i40e->i40e_general_lock);
705 802 }
706 803
707 804 reg = I40E_READ_REG(hw, I40E_PFINT_ICR0);
708 805 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) !=
709 806 DDI_FM_OK) {
710 807 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED);
711 808 atomic_or_32(&i40e->i40e_state, I40E_ERROR);
712 809 return (DDI_INTR_CLAIMED);
713 810 }
714 811
|
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
715 812 if (reg == 0) {
716 813 if (shared == B_TRUE)
717 814 ret = DDI_INTR_UNCLAIMED;
718 815 goto done;
719 816 }
720 817
721 818 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
722 819 i40e_intr_adminq_work(i40e);
723 820
724 821 if (reg & I40E_INTR_NOTX_RX_MASK)
725 - i40e_intr_rx_work(i40e, 0);
822 + i40e_intr_rx_work(i40e, itrq);
726 823
727 824 if (reg & I40E_INTR_NOTX_TX_MASK)
728 - i40e_intr_tx_work(i40e, 0);
825 + i40e_intr_tx_work(i40e, itrq);
729 826
730 827 done:
731 828 i40e_intr_adminq_enable(i40e);
732 829 return (ret);
733 830
734 831 }
735 832
736 833 /* ARGSUSED */
737 834 uint_t
738 835 i40e_intr_msi(void *arg1, void *arg2)
739 836 {
740 837 i40e_t *i40e = (i40e_t *)arg1;
741 838
742 839 return (i40e_intr_notx(i40e, B_FALSE));
743 840 }
744 841
745 842 /* ARGSUSED */
746 843 uint_t
747 844 i40e_intr_legacy(void *arg1, void *arg2)
748 845 {
749 846 i40e_t *i40e = (i40e_t *)arg1;
750 847
751 848 return (i40e_intr_notx(i40e, B_TRUE));
752 849 }
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX