1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2018, Joyent, Inc.
26 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
27 */
28
29 /*
30 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
31 *
32 * Power Management Support
33 * ------------------------
34 *
35 * At the moment, the ahci driver only implements suspend/resume to
36 * support Suspend to RAM on X86 feature. Device power management isn't
37 * implemented, link power management is disabled, and hot plug isn't
38 * allowed during the period from suspend to resume.
39 *
40 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
41 * and DDI_RESUME entries, and don't need to take care of new requests
42 * sent down after suspend because the target driver (sd) has already
43 * handled these conditions, and blocked these requests. For the detailed
44 * information, please check with sdopen, sdclose and sdioctl routines.
45 *
46 *
47 * Enclosure Management Support
48 * ----------------------------
49 *
50 * The ahci driver has basic support for AHCI Enclosure Management (EM)
51 * services. The AHCI specification provides an area in the primary ahci BAR for
52 * posting data to send out to the enclosure management and provides a register
53 * that provides both information and control about this. While the
54 * specification allows for multiple forms of enclosure management, the only
55 * supported, and commonly found form, is the AHCI specified LED format. The LED
56 * format is often implemented as a one-way communication mechanism. Software
57 * can write out what it cares about into the aforementioned data buffer and
58 * then we wait for the transmission to be sent.
59 *
60 * This has some drawbacks. It means that we cannot know whether or not it has
61 * succeeded. This means we cannot ask hardware what it thinks the LEDs are
62 * set to. There's also the added unfortunate reality that firmware on the
63 * microcontroller driving this will often not show the LEDs if no drive is
64 * present and that actions taken may potentially cause this to get out of sync
65 * with what we expect it to be. For example, the specification does not
66 * describe what should happen if a drive is removed from the enclosure while
67 * this is set and what should happen when it returns. We can only infer that it
68 * should be the same.
69 *
70 * Because only a single command can be sent at any time and we don't want to
71 * interfere with controller I/O, we create a taskq dedicated to this that has a
72 * single thread. Both resets (which occur on attach and resume) and normal
73 * changes to the LED state will be driven through this taskq. Because the taskq
74 * has a single thread, this guarantees serial processing.
75 *
76 * Each userland-submitted task (basically not resets) has a reference counted
77 * task structure. This allows the thread that called it to be cancelled and
78 * have the system clean itself up. The user thread in ioctl blocks on a CV that
79 * can receive signals as it waits for completion. Note, there is no guarantee
80 * provided by the kernel that the first thread to enter the kernel will be the
81 * first one to change state.
82 */
83
84 #include <sys/note.h>
85 #include <sys/scsi/scsi.h>
86 #include <sys/pci.h>
87 #include <sys/disp.h>
88 #include <sys/sata/sata_hba.h>
89 #include <sys/sata/adapters/ahci/ahcireg.h>
90 #include <sys/sata/adapters/ahci/ahcivar.h>
91
92 /*
93 * FMA header files
94 */
95 #include <sys/ddifm.h>
96 #include <sys/fm/protocol.h>
97 #include <sys/fm/util.h>
98 #include <sys/fm/io/ddi.h>
99
100 /*
101 * EM Control header files
102 */
103 #include <sys/types.h>
104 #include <sys/file.h>
105 #include <sys/errno.h>
106 #include <sys/open.h>
107 #include <sys/cred.h>
108 #include <sys/ddi.h>
109 #include <sys/sunddi.h>
110
111 /*
112 * This is the string displayed by modinfo, etc.
113 */
114 static char ahci_ident[] = "ahci driver";
115
116 /*
117 * Function prototypes for driver entry points
118 */
119 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
120 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
121 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
122 static int ahci_quiesce(dev_info_t *);
123
124 /*
125 * Function prototypes for SATA Framework interfaces
126 */
127 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
128 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
129
130 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
131 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
132 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
133 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
134 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
135 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
136 #if defined(__lock_lint)
137 static int ahci_selftest(dev_info_t *, sata_device_t *);
138 #endif
139
140 /*
141 * FMA Prototypes
142 */
143 static void ahci_fm_init(ahci_ctl_t *);
144 static void ahci_fm_fini(ahci_ctl_t *);
145 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
146 int ahci_check_acc_handle(ddi_acc_handle_t);
147 int ahci_check_dma_handle(ddi_dma_handle_t);
148 void ahci_fm_ereport(ahci_ctl_t *, char *);
149 static int ahci_check_all_handle(ahci_ctl_t *);
150 static int ahci_check_ctl_handle(ahci_ctl_t *);
151 static int ahci_check_port_handle(ahci_ctl_t *, int);
152 static int ahci_check_slot_handle(ahci_port_t *, int);
153
154 /*
155 * Local function prototypes
156 */
157 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
158 static int ahci_alloc_ports_state(ahci_ctl_t *);
159 static void ahci_dealloc_ports_state(ahci_ctl_t *);
160 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
161 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
162 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
163 static void ahci_dealloc_rcvd_fis(ahci_port_t *);
164 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
165 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
166 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
167 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
168 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
169 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
170
171 static int ahci_initialize_controller(ahci_ctl_t *);
172 static void ahci_uninitialize_controller(ahci_ctl_t *);
173 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
174 static int ahci_config_space_init(ahci_ctl_t *);
175 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
176
177 static void ahci_drain_ports_taskq(ahci_ctl_t *);
178 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
179 uint8_t);
180 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
181 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
182 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
183 sata_device_t *);
184 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
185 sata_pmult_gscr_t *);
186 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
187 sata_device_t *);
188 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
189 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
190 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
191 sata_device_t *);
192
193 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
194 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
195 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
196 ahci_addr_t *);
197 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
198 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
199 ahci_addr_t *, sata_pkt_t *);
200 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
201 ahci_addr_t *, sata_pkt_t *);
202 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
203 ahci_addr_t *, int);
204 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
205 static void ahci_copy_ncq_err_page(sata_cmd_t *,
206 struct sata_ncq_error_recovery_page *);
207 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
208 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
209 static void ahci_flush_doneq(ahci_port_t *);
210
211 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
212 static int ahci_hba_reset(ahci_ctl_t *);
213 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
214 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
215 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
216 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
217 ahci_addr_t *);
218 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
219 ahci_addr_t *);
220 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
221 ahci_addr_t *);
222 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
223 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
224 uint8_t);
225 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
226 uint8_t, int, int *);
227 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
228 uint32_t, uint32_t, uint32_t, uint32_t);
229 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
230 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
231 uint8_t, sata_pkt_t *);
232 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
233 ahci_addr_t *, uint32_t);
234 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
235 uint8_t, uint32_t);
236 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
237 uint8_t, uint32_t);
238 static void ahci_events_handler(void *);
239 static void ahci_watchdog_handler(ahci_ctl_t *);
240
241 static uint_t ahci_intr(caddr_t, caddr_t);
242 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
243 static int ahci_add_intrs(ahci_ctl_t *, int);
244 static void ahci_rem_intrs(ahci_ctl_t *);
245 static void ahci_enable_all_intrs(ahci_ctl_t *);
246 static void ahci_disable_all_intrs(ahci_ctl_t *);
247 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
248 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
249
250 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
251 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
252 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
253 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
254 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
255 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
256 ahci_port_t *, uint8_t);
257 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
258 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
259 uint8_t, uint32_t);
260 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
261 uint8_t, uint32_t);
262 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
263
264 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
265 static int ahci_get_num_implemented_ports(uint32_t);
266 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
267 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
268 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
269 #if AHCI_DEBUG
270 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
271 #endif
272
273 static boolean_t ahci_em_init(ahci_ctl_t *);
274 static void ahci_em_fini(ahci_ctl_t *);
275 static void ahci_em_suspend(ahci_ctl_t *);
276 static void ahci_em_resume(ahci_ctl_t *);
277 static int ahci_em_ioctl(dev_info_t *, int, intptr_t);
278
279
280 /*
281 * DMA attributes for the data buffer
282 *
283 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
284 * does not support 64-bit addressing
285 */
286 static ddi_dma_attr_t buffer_dma_attr = {
287 DMA_ATTR_V0, /* dma_attr_version */
288 0x0ull, /* dma_attr_addr_lo: lowest bus address */
289 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
290 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */
291 0x2ull, /* dma_attr_align: word aligned */
292 1, /* dma_attr_burstsizes */
293 1, /* dma_attr_minxfer */
294 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
295 0xffffffffull, /* dma_attr_seg */
296 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */
297 512, /* dma_attr_granular */
298 0, /* dma_attr_flags */
299 };
300
301 /*
302 * DMA attributes for the rcvd FIS
303 *
304 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
305 * does not support 64-bit addressing
306 */
307 static ddi_dma_attr_t rcvd_fis_dma_attr = {
308 DMA_ATTR_V0, /* dma_attr_version */
309 0x0ull, /* dma_attr_addr_lo: lowest bus address */
310 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
311 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
312 0x100ull, /* dma_attr_align: 256-byte aligned */
313 1, /* dma_attr_burstsizes */
314 1, /* dma_attr_minxfer */
315 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
316 0xffffffffull, /* dma_attr_seg */
317 1, /* dma_attr_sgllen */
318 1, /* dma_attr_granular */
319 0, /* dma_attr_flags */
320 };
321
322 /*
323 * DMA attributes for the command list
324 *
325 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
326 * does not support 64-bit addressing
327 */
328 static ddi_dma_attr_t cmd_list_dma_attr = {
329 DMA_ATTR_V0, /* dma_attr_version */
330 0x0ull, /* dma_attr_addr_lo: lowest bus address */
331 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
332 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
333 0x400ull, /* dma_attr_align: 1K-byte aligned */
334 1, /* dma_attr_burstsizes */
335 1, /* dma_attr_minxfer */
336 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
337 0xffffffffull, /* dma_attr_seg */
338 1, /* dma_attr_sgllen */
339 1, /* dma_attr_granular */
340 0, /* dma_attr_flags */
341 };
342
343 /*
344 * DMA attributes for cmd tables
345 *
346 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
347 * does not support 64-bit addressing
348 */
349 static ddi_dma_attr_t cmd_table_dma_attr = {
350 DMA_ATTR_V0, /* dma_attr_version */
351 0x0ull, /* dma_attr_addr_lo: lowest bus address */
352 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */
353 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */
354 0x80ull, /* dma_attr_align: 128-byte aligned */
355 1, /* dma_attr_burstsizes */
356 1, /* dma_attr_minxfer */
357 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */
358 0xffffffffull, /* dma_attr_seg */
359 1, /* dma_attr_sgllen */
360 1, /* dma_attr_granular */
361 0, /* dma_attr_flags */
362 };
363
364
365 /* Device access attributes */
366 static ddi_device_acc_attr_t accattr = {
367 DDI_DEVICE_ATTR_V1,
368 DDI_STRUCTURE_LE_ACC,
369 DDI_STRICTORDER_ACC,
370 DDI_DEFAULT_ACC
371 };
372
373 static struct dev_ops ahcictl_dev_ops = {
374 DEVO_REV, /* devo_rev */
375 0, /* refcnt */
376 ahci_getinfo, /* info */
377 nulldev, /* identify */
378 nulldev, /* probe */
379 ahci_attach, /* attach */
380 ahci_detach, /* detach */
381 nodev, /* no reset */
382 NULL, /* driver operations */
383 NULL, /* bus operations */
384 NULL, /* power */
385 ahci_quiesce, /* quiesce */
386 };
387
388 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
389 SATA_TRAN_HOTPLUG_OPS_REV_1,
390 ahci_tran_hotplug_port_activate,
391 ahci_tran_hotplug_port_deactivate
392 };
393
394 extern struct mod_ops mod_driverops;
395
396 static struct modldrv modldrv = {
397 &mod_driverops, /* driverops */
398 ahci_ident, /* short description */
399 &ahcictl_dev_ops, /* driver ops */
400 };
401
402 static struct modlinkage modlinkage = {
403 MODREV_1,
404 &modldrv,
405 NULL
406 };
407
408 /* The following variables are watchdog handler related */
409 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
410 static clock_t ahci_watchdog_tick;
411
412 /*
413 * This static variable indicates the size of command table,
414 * and it's changeable with prdt number, which ahci_dma_prdt_number
415 * indicates.
416 */
417 static size_t ahci_cmd_table_size;
418
419 /*
420 * The below global variables are tunable via /etc/system
421 *
422 * ahci_dma_prdt_number
423 * ahci_msi_enabled
424 * ahci_buf_64bit_dma
425 * ahci_commu_64bit_dma
426 */
427
428 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
429 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
430
431 /* AHCI MSI is tunable */
432 boolean_t ahci_msi_enabled = B_TRUE;
433
434 /*
435 * 64-bit dma addressing for data buffer is tunable
436 *
437 * The variable controls only the below value:
438 * DBAU (upper 32-bits physical address of data block)
439 */
440 boolean_t ahci_buf_64bit_dma = B_TRUE;
441
442 /*
443 * 64-bit dma addressing for communication system descriptors is tunable
444 *
445 * The variable controls the below three values:
446 *
447 * PxCLBU (upper 32-bits for the command list base physical address)
448 * PxFBU (upper 32-bits for the received FIS base physical address)
449 * CTBAU (upper 32-bits of command table base)
450 */
451 boolean_t ahci_commu_64bit_dma = B_TRUE;
452
453 /*
454 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
455 * chipset. If the users want to have a try with 64-bit dma, please change
456 * the below variable value to enable it.
457 */
458 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
459
460 /*
461 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
462 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
463 * please change the below value to enable it.
464 */
465 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
466
467 /*
468 * These values control the default delay and default number of times to wait
469 * for an enclosure message to complete.
470 */
471 uint_t ahci_em_reset_delay_ms = 1;
472 uint_t ahci_em_reset_delay_count = 1000;
473 uint_t ahci_em_tx_delay_ms = 1;
474 uint_t ahci_em_tx_delay_count = 1000;
475
476
477 /*
478 * End of global tunable variable definition
479 */
480
481 #if AHCI_DEBUG
482 uint32_t ahci_debug_flags = 0;
483 #else
484 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
485 #endif
486
487
488 #if AHCI_DEBUG
489 /* The following is needed for ahci_log() */
490 static kmutex_t ahci_log_mutex;
491 static char ahci_log_buf[512];
492 #endif
493
494 /* Opaque state pointer initialized by ddi_soft_state_init() */
495 static void *ahci_statep = NULL;
496
497 /*
498 * ahci module initialization.
499 */
500 int
501 _init(void)
502 {
503 int ret;
504
505 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
506 if (ret != 0) {
507 goto err_out;
508 }
509
510 #if AHCI_DEBUG
511 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
512 #endif
513
514 if ((ret = sata_hba_init(&modlinkage)) != 0) {
515 #if AHCI_DEBUG
516 mutex_destroy(&ahci_log_mutex);
517 #endif
518 ddi_soft_state_fini(&ahci_statep);
519 goto err_out;
520 }
521
522 /* watchdog tick */
523 ahci_watchdog_tick = drv_usectohz(
524 (clock_t)ahci_watchdog_timeout * 1000000);
525
526 ret = mod_install(&modlinkage);
527 if (ret != 0) {
528 sata_hba_fini(&modlinkage);
529 #if AHCI_DEBUG
530 mutex_destroy(&ahci_log_mutex);
531 #endif
532 ddi_soft_state_fini(&ahci_statep);
533 goto err_out;
534 }
535
536 return (ret);
537
538 err_out:
539 cmn_err(CE_WARN, "!ahci: Module init failed");
540 return (ret);
541 }
542
543 /*
544 * ahci module uninitialize.
545 */
546 int
547 _fini(void)
548 {
549 int ret;
550
551 ret = mod_remove(&modlinkage);
552 if (ret != 0) {
553 return (ret);
554 }
555
556 /* Remove the resources allocated in _init(). */
557 sata_hba_fini(&modlinkage);
558 #if AHCI_DEBUG
559 mutex_destroy(&ahci_log_mutex);
560 #endif
561 ddi_soft_state_fini(&ahci_statep);
562
563 return (ret);
564 }
565
566 /*
567 * _info entry point
568 */
569 int
570 _info(struct modinfo *modinfop)
571 {
572 return (mod_info(&modlinkage, modinfop));
573 }
574
575 /*
576 * The attach entry point for dev_ops.
577 */
578 static int
579 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
580 {
581 ahci_ctl_t *ahci_ctlp = NULL;
582 int instance = ddi_get_instance(dip);
583 int status;
584 int attach_state;
585 uint32_t cap_status, ahci_version;
586 uint32_t ghc_control;
587 int intr_types;
588 int i;
589 pci_regspec_t *regs;
590 int regs_length;
591 int rnumber;
592 #if AHCI_DEBUG
593 int speed;
594 #endif
595
596 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
597 NULL);
598
599 switch (cmd) {
600 case DDI_ATTACH:
601 break;
602
603 case DDI_RESUME:
604
605 /*
606 * During DDI_RESUME, the hardware state of the device
607 * (power may have been removed from the device) must be
608 * restored, allow pending requests to continue, and
609 * service new requests.
610 */
611 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
612 mutex_enter(&ahci_ctlp->ahcictl_mutex);
613
614 /*
615 * GHC.AE must be set to 1 before any other AHCI register
616 * is accessed
617 */
618 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
619 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
620 ghc_control |= AHCI_HBA_GHC_AE;
621 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
622 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
623
624 /* Restart watch thread */
625 if (ahci_ctlp->ahcictl_timeout_id == 0)
626 ahci_ctlp->ahcictl_timeout_id = timeout(
627 (void (*)(void *))ahci_watchdog_handler,
628 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
629
630 mutex_exit(&ahci_ctlp->ahcictl_mutex);
631
632 /*
633 * Re-initialize the controller and enable the interrupts and
634 * restart all the ports.
635 *
636 * Note that so far we don't support hot-plug during
637 * suspend/resume.
638 */
639 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
640 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
641 "Failed to initialize the controller "
642 "during DDI_RESUME", NULL);
643 return (DDI_FAILURE);
644 }
645
646 /*
647 * Reset the enclosure services.
648 */
649 ahci_em_resume(ahci_ctlp);
650
651 mutex_enter(&ahci_ctlp->ahcictl_mutex);
652 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
653 mutex_exit(&ahci_ctlp->ahcictl_mutex);
654
655 return (DDI_SUCCESS);
656
657 default:
658 return (DDI_FAILURE);
659 }
660
661 attach_state = AHCI_ATTACH_STATE_NONE;
662
663 /* Allocate soft state */
664 status = ddi_soft_state_zalloc(ahci_statep, instance);
665 if (status != DDI_SUCCESS) {
666 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
667 instance);
668 goto err_out;
669 }
670
671 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
672 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
673 ahci_ctlp->ahcictl_dip = dip;
674
675 /* Initialize the cport/port mapping */
676 for (i = 0; i < AHCI_MAX_PORTS; i++) {
677 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
678 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
679 }
680
681 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
682
683 /* Initialize FMA properties */
684 ahci_fm_init(ahci_ctlp);
685
686 attach_state |= AHCI_ATTACH_STATE_FMA;
687
688 /*
689 * Now map the AHCI base address; which includes global
690 * registers and port control registers
691 *
692 * According to the spec, the AHCI Base Address is BAR5,
693 * but BAR0-BAR4 are optional, so we need to check which
694 * rnumber is used for BAR5.
695 */
696
697 /*
698 * search through DDI "reg" property for the AHCI register set
699 */
700 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
701 DDI_PROP_DONTPASS, "reg", (int **)®s,
702 (uint_t *)®s_length) != DDI_PROP_SUCCESS) {
703 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
704 instance);
705 goto err_out;
706 }
707
708 /* AHCI Base Address is located at 0x24 offset */
709 for (rnumber = 0; rnumber < regs_length; ++rnumber) {
710 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
711 == AHCI_PCI_RNUM)
712 break;
713 }
714
715 ddi_prop_free(regs);
716
717 if (rnumber == regs_length) {
718 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
719 instance);
720 goto err_out;
721 }
722
723 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
724
725 status = ddi_regs_map_setup(dip,
726 rnumber,
727 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
728 0,
729 0,
730 &accattr,
731 &ahci_ctlp->ahcictl_ahci_acc_handle);
732 if (status != DDI_SUCCESS) {
733 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
734 instance);
735 goto err_out;
736 }
737
738 attach_state |= AHCI_ATTACH_STATE_REG_MAP;
739
740 /*
741 * GHC.AE must be set to 1 before any other AHCI register
742 * is accessed
743 */
744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
746 ghc_control |= AHCI_HBA_GHC_AE;
747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
748 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
749
750 /* Get the AHCI version information */
751 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
752 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
753
754 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
755 (ahci_version & 0xffff0000) >> 16,
756 ((ahci_version & 0x0000ff00) >> 4 |
757 (ahci_version & 0x000000ff)));
758
759 /* We don't support controllers whose versions are lower than 1.0 */
760 if (!(ahci_version & 0xffff0000)) {
761 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
762 "than version 1.0", instance);
763 goto err_out;
764 }
765
766 /* Get the HBA capabilities information */
767 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
768 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
769
770 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
771 cap_status);
772
773 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
774 if (ahci_version >= 0x00010200) {
775 uint32_t cap2_status;
776
777 /* Get the HBA capabilities extended information */
778 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
779 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
780
781 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
782 "hba capabilities extended = 0x%x", cap2_status);
783 }
784
785 if (cap_status & AHCI_HBA_CAP_EMS) {
786 ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS;
787 ahci_ctlp->ahcictl_em_loc =
788 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
789 (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp));
790 ahci_ctlp->ahcictl_em_ctl =
791 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
792 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
793 }
794
795 #if AHCI_DEBUG
796 /* Get the interface speed supported by the HBA */
797 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
798 if (speed == 0x01) {
799 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
800 "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
801 } else if (speed == 0x10) {
802 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
803 "hba interface speed support: Gen 2 (3 Gbps)", NULL);
804 } else if (speed == 0x11) {
805 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
806 "hba interface speed support: Gen 3 (6 Gbps)", NULL);
807 }
808 #endif
809
810 /* Get the number of command slots supported by the HBA */
811 ahci_ctlp->ahcictl_num_cmd_slots =
812 ((cap_status & AHCI_HBA_CAP_NCS) >>
813 AHCI_HBA_CAP_NCS_SHIFT) + 1;
814
815 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
816 ahci_ctlp->ahcictl_num_cmd_slots);
817
818 /* Get the bit map which indicates ports implemented by the HBA */
819 ahci_ctlp->ahcictl_ports_implemented =
820 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
821 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
822
823 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
824 ahci_ctlp->ahcictl_ports_implemented);
825
826 /* Max port number implemented */
827 ahci_ctlp->ahcictl_num_ports =
828 ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
829
830 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
831 (cap_status & AHCI_HBA_CAP_NP) + 1);
832
833 /* Get the number of implemented ports by the HBA */
834 ahci_ctlp->ahcictl_num_implemented_ports =
835 ahci_get_num_implemented_ports(
836 ahci_ctlp->ahcictl_ports_implemented);
837
838 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
839 "hba number of implemented ports: %d",
840 ahci_ctlp->ahcictl_num_implemented_ports);
841
842 /* Check whether HBA supports 64bit DMA addressing */
843 if (!(cap_status & AHCI_HBA_CAP_S64A)) {
844 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
845 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
846 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
847 "hba does not support 64-bit addressing", NULL);
848 }
849
850 /* Checking for the support of Port Multiplier */
851 if (cap_status & AHCI_HBA_CAP_SPM) {
852 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
853 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
854 "hba supports port multiplier (CBSS)", NULL);
855
856 /* Support FIS-based switching ? */
857 if (cap_status & AHCI_HBA_CAP_FBSS) {
858 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
859 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
860 "hba supports FIS-based switching (FBSS)", NULL);
861 }
862 }
863
864 /* Checking for Support Command List Override */
865 if (cap_status & AHCI_HBA_CAP_SCLO) {
866 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
867 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
868 "hba supports command list override.", NULL);
869 }
870
871 /* Checking for Asynchronous Notification */
872 if (cap_status & AHCI_HBA_CAP_SSNTF) {
873 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
874 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
875 "hba supports asynchronous notification.", NULL);
876 }
877
878 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
879 != DDI_SUCCESS) {
880 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
881 instance);
882 goto err_out;
883 }
884
885 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
886
887 /*
888 * Check the pci configuration space, and set caps. We also
889 * handle the hardware defect in this function.
890 *
891 * For example, force ATI SB600 to use 32-bit dma addressing
892 * since it doesn't support 64-bit dma though its CAP register
893 * declares it support.
894 */
895 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
896 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
897 instance);
898 goto err_out;
899 }
900
901 /*
902 * Disable the whole controller interrupts before adding
903 * interrupt handlers(s).
904 */
905 ahci_disable_all_intrs(ahci_ctlp);
906
907 /* Get supported interrupt types */
908 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
909 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
910 instance);
911 goto err_out;
912 }
913
914 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
915 "ddi_intr_get_supported_types() returned: 0x%x",
916 intr_types);
917
918 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
919 /*
920 * Try MSI first, but fall back to FIXED if failed
921 */
922 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
923 DDI_SUCCESS) {
924 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
925 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
926 "Using MSI interrupt type", NULL);
927 goto intr_done;
928 }
929
930 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
931 "MSI registration failed, "
932 "trying FIXED interrupts", NULL);
933 }
934
935 if (intr_types & DDI_INTR_TYPE_FIXED) {
936 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
937 DDI_SUCCESS) {
938 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
939 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
940 "Using FIXED interrupt type", NULL);
941 goto intr_done;
942 }
943
944 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
945 "FIXED interrupt registration failed", NULL);
946 }
947
948 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
949
950 goto err_out;
951
952 intr_done:
953
954 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
955
956 /* Initialize the controller mutex */
957 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
958 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
959
960 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
961
962 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
963 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
964 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
965 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
966 }
967
968 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
969 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
970 sizeof (ahci_prdt_item_t));
971
972 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
973 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
974 " ahci_cmd_table_size is 0x%x",
975 ahci_dma_prdt_number, ahci_cmd_table_size);
976
977 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
978 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
979 ahci_dma_prdt_number;
980
981 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
982 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
983 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
984 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
985
986 /*
987 * enable 64bit dma for data buffer for SB600 if
988 * sb600_buf_64bit_dma_disable is B_FALSE
989 */
990 if ((ahci_buf_64bit_dma == B_FALSE) ||
991 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
992 !(sb600_buf_64bit_dma_disable == B_FALSE &&
993 ahci_ctlp->ahcictl_venid == 0x1002 &&
994 ahci_ctlp->ahcictl_devid == 0x4380))) {
995 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
996 0xffffffffull;
997 }
998
999 /*
1000 * enable 64bit dma for command buffer for SB600/700/710/800
1001 * if sbxxx_commu_64bit_dma_disable is B_FALSE
1002 */
1003 if ((ahci_commu_64bit_dma == B_FALSE) ||
1004 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
1005 !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
1006 ahci_ctlp->ahcictl_venid == 0x1002 &&
1007 (ahci_ctlp->ahcictl_devid == 0x4380 ||
1008 ahci_ctlp->ahcictl_devid == 0x4391)))) {
1009 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
1010 0xffffffffull;
1011 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
1012 0xffffffffull;
1013 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
1014 0xffffffffull;
1015 }
1016
1017 /* Allocate the ports structure */
1018 status = ahci_alloc_ports_state(ahci_ctlp);
1019 if (status != AHCI_SUCCESS) {
1020 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
1021 instance);
1022 goto err_out;
1023 }
1024
1025 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
1026
1027 /*
1028 * Initialize the controller and ports.
1029 */
1030 status = ahci_initialize_controller(ahci_ctlp);
1031 if (status != AHCI_SUCCESS) {
1032 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
1033 instance);
1034 goto err_out;
1035 }
1036
1037 attach_state |= AHCI_ATTACH_STATE_HW_INIT;
1038
1039 /* Start one thread to check packet timeouts */
1040 ahci_ctlp->ahcictl_timeout_id = timeout(
1041 (void (*)(void *))ahci_watchdog_handler,
1042 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
1043
1044 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
1045
1046 if (!ahci_em_init(ahci_ctlp)) {
1047 cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure "
1048 "services", instance);
1049 goto err_out;
1050 }
1051 attach_state |= AHCI_ATTACH_STATE_ENCLOSURE;
1052
1053 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
1054 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
1055 instance);
1056 goto err_out;
1057 }
1058
1059 /* Check all handles at the end of the attach operation. */
1060 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
1061 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
1062 instance);
1063 goto err_out;
1064 }
1065
1066 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
1067
1068 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
1069
1070 return (DDI_SUCCESS);
1071
1072 err_out:
1073 /* FMA message */
1074 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
1075 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
1076
1077 if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) {
1078 ahci_em_fini(ahci_ctlp);
1079 }
1080
1081 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
1082 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1083 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1084 ahci_ctlp->ahcictl_timeout_id = 0;
1085 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1086 }
1087
1088 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1089 ahci_uninitialize_controller(ahci_ctlp);
1090 }
1091
1092 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1093 ahci_dealloc_ports_state(ahci_ctlp);
1094 }
1095
1096 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1097 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1098 }
1099
1100 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1101 ahci_rem_intrs(ahci_ctlp);
1102 }
1103
1104 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1105 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1106 }
1107
1108 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1109 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1110 }
1111
1112 if (attach_state & AHCI_ATTACH_STATE_FMA) {
1113 ahci_fm_fini(ahci_ctlp);
1114 }
1115
1116 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1117 ddi_soft_state_free(ahci_statep, instance);
1118 }
1119
1120 return (DDI_FAILURE);
1121 }
1122
1123 /*
1124 * The detach entry point for dev_ops.
1125 */
1126 static int
1127 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1128 {
1129 ahci_ctl_t *ahci_ctlp;
1130 int instance;
1131 int ret;
1132
1133 instance = ddi_get_instance(dip);
1134 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1135
1136 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1137
1138 switch (cmd) {
1139 case DDI_DETACH:
1140
1141 /* disable the interrupts for an uninterrupted detach */
1142 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1143 ahci_disable_all_intrs(ahci_ctlp);
1144 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1145
1146 /* unregister from the sata framework. */
1147 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1148 if (ret != AHCI_SUCCESS) {
1149 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1150 ahci_enable_all_intrs(ahci_ctlp);
1151 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1152 return (DDI_FAILURE);
1153 }
1154
1155 ahci_em_fini(ahci_ctlp);
1156
1157 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1158
1159 /* stop the watchdog handler */
1160 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1161 ahci_ctlp->ahcictl_timeout_id = 0;
1162
1163 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1164
1165 /* uninitialize the controller */
1166 ahci_uninitialize_controller(ahci_ctlp);
1167
1168 /* remove the interrupts */
1169 ahci_rem_intrs(ahci_ctlp);
1170
1171 /* deallocate the ports structures */
1172 ahci_dealloc_ports_state(ahci_ctlp);
1173
1174 /* destroy mutex */
1175 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1176
1177 /* teardown the pci config */
1178 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1179
1180 /* remove the reg maps. */
1181 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1182
1183 /* release fma resource */
1184 ahci_fm_fini(ahci_ctlp);
1185
1186 /* free the soft state. */
1187 ddi_soft_state_free(ahci_statep, instance);
1188
1189 return (DDI_SUCCESS);
1190
1191 case DDI_SUSPEND:
1192
1193 /*
1194 * The steps associated with suspension must include putting
1195 * the underlying device into a quiescent state so that it
1196 * will not generate interrupts or modify or access memory.
1197 */
1198 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1199 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1200 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1201 return (DDI_SUCCESS);
1202 }
1203
1204 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1205
1206 /* stop the watchdog handler */
1207 if (ahci_ctlp->ahcictl_timeout_id) {
1208 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1209 ahci_ctlp->ahcictl_timeout_id = 0;
1210 }
1211
1212 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1213
1214 ahci_em_suspend(ahci_ctlp);
1215
1216 /*
1217 * drain the taskq
1218 */
1219 ahci_drain_ports_taskq(ahci_ctlp);
1220
1221 /*
1222 * Disable the interrupts and stop all the ports.
1223 */
1224 ahci_uninitialize_controller(ahci_ctlp);
1225
1226 return (DDI_SUCCESS);
1227
1228 default:
1229 return (DDI_FAILURE);
1230 }
1231 }
1232
1233 /*
1234 * The info entry point for dev_ops.
1235 *
1236 */
1237 static int
1238 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1239 void *arg, void **result)
1240 {
1241 #ifndef __lock_lint
1242 _NOTE(ARGUNUSED(dip))
1243 #endif /* __lock_lint */
1244
1245 ahci_ctl_t *ahci_ctlp;
1246 int instance;
1247 dev_t dev;
1248
1249 dev = (dev_t)arg;
1250 instance = getminor(dev);
1251
1252 switch (infocmd) {
1253 case DDI_INFO_DEVT2DEVINFO:
1254 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1255 if (ahci_ctlp != NULL) {
1256 *result = ahci_ctlp->ahcictl_dip;
1257 return (DDI_SUCCESS);
1258 } else {
1259 *result = NULL;
1260 return (DDI_FAILURE);
1261 }
1262 case DDI_INFO_DEVT2INSTANCE:
1263 *(int *)result = instance;
1264 break;
1265 default:
1266 break;
1267 }
1268
1269 return (DDI_SUCCESS);
1270 }
1271
1272 /*
1273 * Registers the ahci with sata framework.
1274 */
1275 static int
1276 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1277 {
1278 struct sata_hba_tran *sata_hba_tran;
1279
1280 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1281 "ahci_register_sata_hba_tran enter", NULL);
1282
1283 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1284
1285 /* Allocate memory for the sata_hba_tran */
1286 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1287
1288 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1289 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1290 sata_hba_tran->sata_tran_hba_dma_attr =
1291 &ahci_ctlp->ahcictl_buffer_dma_attr;
1292
1293 /* Report the number of implemented ports */
1294 sata_hba_tran->sata_tran_hba_num_cports =
1295 ahci_ctlp->ahcictl_num_implemented_ports;
1296
1297 /* Support ATAPI device */
1298 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1299
1300 /* Get the data transfer capability for PIO command by the HBA */
1301 if (cap_status & AHCI_HBA_CAP_PMD) {
1302 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1303 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1304 "DRQ block data transfer for PIO command protocol", NULL);
1305 }
1306
1307 /*
1308 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1309 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1310 * QUEUED (EXT), and SERVICE commands). Queued operations are
1311 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1312 * QUEUED commands when the HBA and device support native command
1313 * queuing(NCQ).
1314 *
1315 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1316 * CAP register of the HBA indicates NCQ is supported.
1317 *
1318 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1319 * set because the previous register content of PxCI can be re-written
1320 * in the register write.
1321 */
1322 if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1323 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1324 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1325 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1326 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1327 "Command Queuing", NULL);
1328 }
1329
1330 /* Support port multiplier? */
1331 if (cap_status & AHCI_HBA_CAP_SPM) {
1332 sata_hba_tran->sata_tran_hba_features_support |=
1333 SATA_CTLF_PORT_MULTIPLIER;
1334
1335 /* Support FIS-based switching for port multiplier? */
1336 if (cap_status & AHCI_HBA_CAP_FBSS) {
1337 sata_hba_tran->sata_tran_hba_features_support |=
1338 SATA_CTLF_PMULT_FBS;
1339 }
1340 }
1341
1342 /* Report the number of command slots */
1343 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1344
1345 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1346 sata_hba_tran->sata_tran_start = ahci_tran_start;
1347 sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1348 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1349 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1350 #ifdef __lock_lint
1351 sata_hba_tran->sata_tran_selftest = ahci_selftest;
1352 #endif
1353 /*
1354 * When SATA framework adds support for pwrmgt the
1355 * pwrmgt_ops needs to be updated
1356 */
1357 sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1358 sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl;
1359
1360 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1361
1362 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1363
1364 /* Attach it to SATA framework */
1365 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1366 != DDI_SUCCESS) {
1367 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1368 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1369 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1370 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1371 return (AHCI_FAILURE);
1372 }
1373
1374 return (AHCI_SUCCESS);
1375 }
1376
1377 /*
1378 * Unregisters the ahci with sata framework.
1379 */
1380 static int
1381 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1382 {
1383 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1384 "ahci_unregister_sata_hba_tran enter", NULL);
1385
1386 /* Detach from the SATA framework. */
1387 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1388 DDI_SUCCESS) {
1389 return (AHCI_FAILURE);
1390 }
1391
1392 /* Deallocate sata_hba_tran. */
1393 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1394 sizeof (sata_hba_tran_t));
1395
1396 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1397 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1398 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1399
1400 return (AHCI_SUCCESS);
1401 }
1402
1403 #define SET_PORTSTR(str, addrp) \
1404 if (AHCI_ADDR_IS_PORT(addrp)) \
1405 (void) sprintf((str), "%d", (addrp)->aa_port); \
1406 else if (AHCI_ADDR_IS_PMULT(addrp)) \
1407 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \
1408 else \
1409 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \
1410 (addrp)->aa_pmport);
1411
1412 /*
1413 * ahci_tran_probe_port is called by SATA framework. It returns port state,
1414 * port status registers and an attached device type via sata_device
1415 * structure.
1416 *
1417 * We return the cached information from a previous hardware probe. The
1418 * actual hardware probing itself was done either from within
1419 * ahci_initialize_controller() during the driver attach or from a phy
1420 * ready change interrupt handler.
1421 */
1422 static int
1423 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1424 {
1425 ahci_ctl_t *ahci_ctlp;
1426 ahci_port_t *ahci_portp;
1427 ahci_addr_t addr, pmult_addr;
1428 uint8_t cport = sd->satadev_addr.cport;
1429 char portstr[10];
1430 uint8_t device_type;
1431 uint32_t port_state;
1432 uint8_t port;
1433 int rval = SATA_SUCCESS, rval_init;
1434
1435 port_state = 0;
1436 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1437 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1438
1439 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1440
1441 mutex_enter(&ahci_portp->ahciport_mutex);
1442
1443 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1444 ASSERT(AHCI_ADDR_IS_VALID(&addr));
1445 SET_PORTSTR(portstr, &addr);
1446
1447 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1448 "ahci_tran_probe_port enter: port %s", portstr);
1449
1450 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1451 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1452 ahci_portp->ahciport_pmult_info == NULL)) {
1453 /* port mutliplier is removed. */
1454 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1455 "ahci_tran_probe_port: "
1456 "pmult is removed from port %s", portstr);
1457 mutex_exit(&ahci_portp->ahciport_mutex);
1458 return (SATA_FAILURE);
1459 }
1460
1461 /*
1462 * The sata_device may refer to
1463 * 1. A controller port.
1464 * A controller port should be ready here.
1465 * 2. A port multiplier.
1466 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1467 * it and register the port multiplier to the framework.
1468 * SATA_ADDR_PMULT - check the status of all its device ports.
1469 * 3. A port multiplier port.
1470 * If it has not been initialized, initialized it.
1471 *
1472 * A port multiplier or a port multiplier port may require some
1473 * initialization because we cannot do these time-consuming jobs in an
1474 * interrupt context.
1475 */
1476 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1477 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1478 /* Initialize registers on a port multiplier */
1479 rval_init = ahci_initialize_pmult(ahci_ctlp,
1480 ahci_portp, &pmult_addr, sd);
1481 if (rval_init != AHCI_SUCCESS) {
1482 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1483 "ahci_tran_probe_port: "
1484 "pmult initialization failed.", NULL);
1485 mutex_exit(&ahci_portp->ahciport_mutex);
1486 return (SATA_FAILURE);
1487 }
1488 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1489 /* Check pmports hotplug events */
1490 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1491 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1492 SATA_ADDR_DPMPORT)) {
1493 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1494 &addr, sd) != AHCI_SUCCESS) {
1495 rval = SATA_FAILURE;
1496 goto out;
1497 }
1498 }
1499
1500 /* Update port state and device type */
1501 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1502
1503 switch (port_state) {
1504
1505 case SATA_PSTATE_FAILED:
1506 sd->satadev_state = SATA_PSTATE_FAILED;
1507 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1508 "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1509 goto out;
1510
1511 case SATA_PSTATE_SHUTDOWN:
1512 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1513 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1514 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1515 goto out;
1516
1517 case SATA_PSTATE_PWROFF:
1518 sd->satadev_state = SATA_PSTATE_PWROFF;
1519 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1520 "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1521 goto out;
1522
1523 case SATA_PSTATE_PWRON:
1524 sd->satadev_state = SATA_PSTATE_PWRON;
1525 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1526 "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1527 break;
1528
1529 default:
1530 sd->satadev_state = port_state;
1531 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1532 "ahci_tran_probe_port: port %s PORT NORMAL %x",
1533 portstr, port_state);
1534 break;
1535 }
1536
1537 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1538
1539 switch (device_type) {
1540
1541 case SATA_DTYPE_ATADISK:
1542 sd->satadev_type = SATA_DTYPE_ATADISK;
1543 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1544 "ahci_tran_probe_port: port %s DISK found", portstr);
1545 break;
1546
1547 case SATA_DTYPE_ATAPI:
1548 /*
1549 * HBA driver only knows it's an ATAPI device, and don't know
1550 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1551 * type need to be determined by checking IDENTIFY PACKET
1552 * DEVICE data
1553 */
1554 sd->satadev_type = SATA_DTYPE_ATAPI;
1555 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1556 "ahci_tran_probe_port: port %s ATAPI found", portstr);
1557 break;
1558
1559 case SATA_DTYPE_PMULT:
1560 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1561 sd->satadev_type = SATA_DTYPE_PMULT;
1562
1563 /* Update the number of pmports. */
1564 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1565 sd->satadev_add_info = ahci_portp->
1566 ahciport_pmult_info->ahcipmi_num_dev_ports;
1567
1568 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1569 "ahci_tran_probe_port: port %s Port Multiplier found",
1570 portstr);
1571 break;
1572
1573 case SATA_DTYPE_UNKNOWN:
1574 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1575 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1576 "ahci_tran_probe_port: port %s Unknown device found",
1577 portstr);
1578 break;
1579
1580 default:
1581 /* we don't support any other device types */
1582 sd->satadev_type = SATA_DTYPE_NONE;
1583 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1584 "ahci_tran_probe_port: port %s No device found", portstr);
1585 break;
1586 }
1587
1588 out:
1589 /* Register update only fails while probing a pmult/pmport */
1590 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1591 ahci_update_sata_registers(ahci_ctlp, port, sd);
1592 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1593 if (port_state & SATA_STATE_READY)
1594 if (ahci_update_pmult_pscr(ahci_ctlp,
1595 &addr, sd) != AHCI_SUCCESS)
1596 rval = SATA_FAILURE;
1597 }
1598
1599 /* Check handles for the sata registers access */
1600 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1601 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1602 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1603 DDI_SERVICE_UNAFFECTED);
1604 rval = SATA_FAILURE;
1605 }
1606
1607 mutex_exit(&ahci_portp->ahciport_mutex);
1608 return (rval);
1609 }
1610
1611 /*
1612 * There are four operation modes in sata framework:
1613 * SATA_OPMODE_INTERRUPTS
1614 * SATA_OPMODE_POLLING
1615 * SATA_OPMODE_ASYNCH
1616 * SATA_OPMODE_SYNCH
1617 *
1618 * Their combined meanings as following:
1619 *
1620 * SATA_OPMODE_SYNCH
1621 * The command has to be completed before sata_tran_start functions returns.
1622 * Either interrupts or polling could be used - it's up to the driver.
1623 * Mode used currently for internal, sata-module initiated operations.
1624 *
1625 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1626 * It is the same as the one above.
1627 *
1628 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1629 * The command has to be completed before sata_tran_start function returns.
1630 * No interrupt used, polling only. This should be the mode used for scsi
1631 * packets with FLAG_NOINTR.
1632 *
1633 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1634 * The command may be queued (callback function specified). Interrupts could
1635 * be used. It's normal operation mode.
1636 */
1637 /*
1638 * Called by sata framework to transport a sata packet down stream.
1639 */
1640 static int
1641 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1642 {
1643 ahci_ctl_t *ahci_ctlp;
1644 ahci_port_t *ahci_portp;
1645 ahci_addr_t addr;
1646 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1647 uint8_t port;
1648 char portstr[10];
1649
1650 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1651 port = ahci_ctlp->ahcictl_cport_to_port[cport];
1652
1653 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1654 "ahci_tran_start enter: cport %d satapkt 0x%p",
1655 cport, (void *)spkt);
1656
1657 ahci_portp = ahci_ctlp->ahcictl_ports[port];
1658
1659 mutex_enter(&ahci_portp->ahciport_mutex);
1660 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1661 SET_PORTSTR(portstr, &addr);
1662
1663 /* Sanity check */
1664 if (AHCI_ADDR_IS_PMPORT(&addr)) {
1665 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1666 ahci_portp->ahciport_pmult_info == NULL) {
1667
1668 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1669 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1670 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1671 ahci_update_sata_registers(ahci_ctlp, port,
1672 &spkt->satapkt_device);
1673 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1674 "ahci_tran_start returning PORT_ERROR while "
1675 "pmult removed: port: %s", portstr);
1676 mutex_exit(&ahci_portp->ahciport_mutex);
1677 return (SATA_TRAN_PORT_ERROR);
1678 }
1679
1680 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1681 SATA_STATE_READY)) {
1682 if (!ddi_in_panic() ||
1683 ahci_initialize_pmport(ahci_ctlp,
1684 ahci_portp, &addr) != AHCI_SUCCESS) {
1685 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1686 spkt->satapkt_device.satadev_type =
1687 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1688 spkt->satapkt_device.satadev_state =
1689 AHCIPORT_GET_STATE(ahci_portp, &addr);
1690 ahci_update_sata_registers(ahci_ctlp, port,
1691 &spkt->satapkt_device);
1692 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1693 "ahci_tran_start returning PORT_ERROR "
1694 "while sub-link is not initialized "
1695 "at port: %s", portstr);
1696 mutex_exit(&ahci_portp->ahciport_mutex);
1697 return (SATA_TRAN_PORT_ERROR);
1698 }
1699 }
1700 }
1701
1702 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1703 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1704 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1705 /*
1706 * In case the target driver would send the packet before
1707 * sata framework can have the opportunity to process those
1708 * event reports.
1709 */
1710 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1711 spkt->satapkt_device.satadev_state =
1712 ahci_portp->ahciport_port_state;
1713 ahci_update_sata_registers(ahci_ctlp, port,
1714 &spkt->satapkt_device);
1715 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1716 "ahci_tran_start returning PORT_ERROR while "
1717 "port in FAILED/SHUTDOWN/PWROFF state: "
1718 "port: %s", portstr);
1719 mutex_exit(&ahci_portp->ahciport_mutex);
1720 return (SATA_TRAN_PORT_ERROR);
1721 }
1722
1723 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1724 /*
1725 * ahci_intr_phyrdy_change() may have rendered it to
1726 * SATA_DTYPE_NONE.
1727 */
1728 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1729 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1730 spkt->satapkt_device.satadev_state =
1731 ahci_portp->ahciport_port_state;
1732 ahci_update_sata_registers(ahci_ctlp, port,
1733 &spkt->satapkt_device);
1734 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1735 "ahci_tran_start returning PORT_ERROR while "
1736 "no device attached: port: %s", portstr);
1737 mutex_exit(&ahci_portp->ahciport_mutex);
1738 return (SATA_TRAN_PORT_ERROR);
1739 }
1740
1741 /* R/W PMULT command will occupy the whole HBA port */
1742 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1743 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1744 "ahci_tran_start returning BUSY while "
1745 "executing READ/WRITE PORT-MULT command: "
1746 "port: %s", portstr);
1747 spkt->satapkt_reason = SATA_PKT_BUSY;
1748 mutex_exit(&ahci_portp->ahciport_mutex);
1749 return (SATA_TRAN_BUSY);
1750 }
1751
1752 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1753 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1754 "ahci_tran_start returning BUSY while "
1755 "hot-plug in progress: port: %s", portstr);
1756 spkt->satapkt_reason = SATA_PKT_BUSY;
1757 mutex_exit(&ahci_portp->ahciport_mutex);
1758 return (SATA_TRAN_BUSY);
1759 }
1760
1761 /*
1762 * SATA HBA driver should remember that a device was reset and it
1763 * is supposed to reject any packets which do not specify either
1764 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1765 *
1766 * This is to prevent a race condition when a device was arbitrarily
1767 * reset by the HBA driver (and lost it's setting) and a target
1768 * driver sending some commands to a device before the sata framework
1769 * has a chance to restore the device setting (such as cache enable/
1770 * disable or other resettable stuff).
1771 */
1772 /*
1773 * It is unnecessary to use specific flags to indicate
1774 * reset_in_progress for a pmport. While mopping, all command will be
1775 * mopped so that the entire HBA port is being dealt as a single
1776 * object.
1777 */
1778 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1779 ahci_portp->ahciport_reset_in_progress = 0;
1780 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1781 "ahci_tran_start [CLEAR] the "
1782 "reset_in_progress for port: %d", port);
1783 }
1784
1785 if (ahci_portp->ahciport_reset_in_progress &&
1786 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1787 ! ddi_in_panic()) {
1788 spkt->satapkt_reason = SATA_PKT_BUSY;
1789 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1790 "ahci_tran_start returning BUSY while "
1791 "reset in progress: port: %d", port);
1792 mutex_exit(&ahci_portp->ahciport_mutex);
1793 return (SATA_TRAN_BUSY);
1794 }
1795
1796 #ifdef AHCI_DEBUG
1797 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1798 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1799 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1800 spkt, port);
1801 }
1802 #endif
1803
1804 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1805 spkt->satapkt_reason = SATA_PKT_BUSY;
1806 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1807 "ahci_tran_start returning BUSY while "
1808 "mopping in progress: port: %d", port);
1809 mutex_exit(&ahci_portp->ahciport_mutex);
1810 return (SATA_TRAN_BUSY);
1811 }
1812
1813 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1814 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1815 DDI_SERVICE_UNAFFECTED);
1816 mutex_exit(&ahci_portp->ahciport_mutex);
1817 return (SATA_TRAN_BUSY);
1818 }
1819
1820 if (spkt->satapkt_op_mode &
1821 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1822 /*
1823 * If a SYNC command to be executed in interrupt context,
1824 * bounce it back to sata module.
1825 */
1826 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1827 servicing_interrupt()) {
1828 spkt->satapkt_reason = SATA_PKT_BUSY;
1829 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1830 "ahci_tran_start returning BUSY while "
1831 "sending SYNC mode under interrupt context: "
1832 "port : %d", port);
1833 mutex_exit(&ahci_portp->ahciport_mutex);
1834 return (SATA_TRAN_BUSY);
1835 }
1836
1837 /* We need to do the sync start now */
1838 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1839 spkt) == AHCI_FAILURE) {
1840 goto fail_out;
1841 }
1842 } else {
1843 /* Async start, using interrupt */
1844 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1845 == AHCI_FAILURE) {
1846 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1847 goto fail_out;
1848 }
1849 }
1850
1851 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1852 "sata tran accepted: port %s", portstr);
1853
1854 mutex_exit(&ahci_portp->ahciport_mutex);
1855 return (SATA_TRAN_ACCEPTED);
1856
1857 fail_out:
1858 /*
1859 * Failed to deliver packet to the controller.
1860 * Check if it's caused by invalid handles.
1861 */
1862 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1863 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1864 spkt->satapkt_device.satadev_type =
1865 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1866 spkt->satapkt_device.satadev_state =
1867 AHCIPORT_GET_STATE(ahci_portp, &addr);
1868 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1869 mutex_exit(&ahci_portp->ahciport_mutex);
1870 return (SATA_TRAN_PORT_ERROR);
1871 }
1872
1873 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1874 "return QUEUE_FULL: port %d", port);
1875 mutex_exit(&ahci_portp->ahciport_mutex);
1876 return (SATA_TRAN_QUEUE_FULL);
1877 }
1878
1879 /*
1880 * SATA_OPMODE_SYNCH flag is set
1881 *
1882 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1883 * without interrupt, otherwise we can still use the interrupt.
1884 */
1885 static int
1886 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1887 ahci_addr_t *addrp, sata_pkt_t *spkt)
1888 {
1889 int pkt_timeout_ticks;
1890 uint32_t timeout_tags;
1891 int rval;
1892 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1893 uint8_t port = addrp->aa_port;
1894
1895 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1896
1897 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1898 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1899
1900 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1901 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1902 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1903 addrp, spkt)) == AHCI_FAILURE) {
1904 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1905 return (rval);
1906 }
1907
1908 pkt_timeout_ticks =
1909 drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1910
1911 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1912 /* Simulate the interrupt */
1913 mutex_exit(&ahci_portp->ahciport_mutex);
1914 ahci_port_intr(ahci_ctlp, ahci_portp, port);
1915 mutex_enter(&ahci_portp->ahciport_mutex);
1916
1917 if (spkt->satapkt_reason != SATA_PKT_BUSY)
1918 break;
1919
1920 mutex_exit(&ahci_portp->ahciport_mutex);
1921 drv_usecwait(AHCI_1MS_USECS);
1922 mutex_enter(&ahci_portp->ahciport_mutex);
1923
1924 pkt_timeout_ticks -= AHCI_1MS_TICKS;
1925 if (pkt_timeout_ticks < 0) {
1926 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1927 "port %d satapkt 0x%p timed out\n",
1928 instance, port, (void *)spkt);
1929 timeout_tags = (0x1 << rval);
1930 mutex_exit(&ahci_portp->ahciport_mutex);
1931 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1932 port, timeout_tags);
1933 mutex_enter(&ahci_portp->ahciport_mutex);
1934 }
1935 }
1936
1937 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1938 return (AHCI_SUCCESS);
1939
1940 } else {
1941 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1942 addrp, spkt)) == AHCI_FAILURE)
1943 return (rval);
1944
1945 #if AHCI_DEBUG
1946 /*
1947 * Note that the driver always uses the slot 0 to deliver
1948 * REQUEST SENSE or READ LOG EXT command
1949 */
1950 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1951 ASSERT(rval == 0);
1952 #endif
1953
1954 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1955 cv_wait(&ahci_portp->ahciport_cv,
1956 &ahci_portp->ahciport_mutex);
1957
1958 return (AHCI_SUCCESS);
1959 }
1960 }
1961
1962 /*
1963 * Searches for and claims a free command slot.
1964 *
1965 * Returns value:
1966 *
1967 * AHCI_FAILURE returned only if
1968 * 1. No empty slot left
1969 * 2. Non-queued command requested while queued command(s) is outstanding
1970 * 3. Queued command requested while non-queued command(s) is outstanding
1971 * 4. HBA doesn't support multiple-use of command list while already a
1972 * non-queued command is oustanding
1973 * 5. Queued command requested while some queued command(s) has been
1974 * outstanding on a different port multiplier port. (AHCI spec 1.2,
1975 * 9.1.2)
1976 *
1977 * claimed slot number returned if succeeded
1978 *
1979 * NOTE: it will always return slot 0 for following commands to simplify the
1980 * algorithm.
1981 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process
1982 * 2. READ/WRITE PORTMULT command
1983 */
1984 static int
1985 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1986 ahci_addr_t *addrp, int command_type)
1987 {
1988 uint32_t port_cmd_issue;
1989 uint32_t free_slots;
1990 int slot;
1991
1992 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1993
1994 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1995 "ahciport_pending_tags = 0x%x "
1996 "ahciport_pending_ncq_tags = 0x%x",
1997 ahci_portp->ahciport_pending_tags,
1998 ahci_portp->ahciport_pending_ncq_tags);
1999
2000 free_slots = 0;
2001 /*
2002 * According to the AHCI spec, system software is responsible to
2003 * ensure that queued and non-queued commands are not mixed in
2004 * the command list.
2005 */
2006 if (command_type == AHCI_NON_NCQ_CMD) {
2007 /* Non-NCQ command request */
2008 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2009 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2010 "ahci_claim_free_slot: there is still pending "
2011 "queued command(s) in the command list, "
2012 "so no available slot for the non-queued "
2013 "command", NULL);
2014 return (AHCI_FAILURE);
2015 }
2016 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
2017 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2018 "ahci_claim_free_slot: there is still pending "
2019 "read/write port-mult command(s) in command list, "
2020 "so no available slot for the non-queued command",
2021 NULL);
2022 return (AHCI_FAILURE);
2023 }
2024 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
2025 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2026 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2027 "ahci_claim_free_slot: HBA cannot support multiple-"
2028 "use of the command list for non-queued commands",
2029 NULL);
2030 return (AHCI_FAILURE);
2031 }
2032 free_slots = (~ahci_portp->ahciport_pending_tags) &
2033 AHCI_SLOT_MASK(ahci_ctlp);
2034 } else if (command_type == AHCI_NCQ_CMD) {
2035 /* NCQ command request */
2036 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2037 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2038 "ahci_claim_free_slot: there is still pending "
2039 "non-queued command(s) in the command list, "
2040 "so no available slot for the queued command",
2041 NULL);
2042 return (AHCI_FAILURE);
2043 }
2044
2045 /*
2046 * NCQ commands cannot be sent to different port multiplier
2047 * ports in Command-Based Switching mode
2048 */
2049 /*
2050 * NOTE: In Command-Based Switching mode, AHCI controller
2051 * usually reports a 'Handshake Error' when multiple NCQ
2052 * commands are outstanding simultaneously.
2053 */
2054 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
2055 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2056 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
2057 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
2058 AHCIPORT_NCQ_PMPORT(ahci_portp) !=
2059 addrp->aa_pmport) {
2060 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2061 "ahci_claim_free_slot: there is still "
2062 "pending queued command(s) in the "
2063 "command list for another Port Multiplier "
2064 "port, so no available slot.", NULL);
2065 return (AHCI_FAILURE);
2066 }
2067 }
2068
2069 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
2070 AHCI_NCQ_SLOT_MASK(ahci_portp);
2071 } else if (command_type == AHCI_ERR_RETRI_CMD) {
2072 /* Error retrieval command request */
2073 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2074 "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
2075 "SENSE or READ LOG EXT command", NULL);
2076 slot = 0;
2077 goto out;
2078 } else if (command_type == AHCI_RDWR_PMULT_CMD) {
2079 /*
2080 * An extra check on PxCI. Sometimes PxCI bits may not be
2081 * cleared during hot-plug or error recovery process.
2082 */
2083 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2084 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
2085
2086 if (port_cmd_issue != 0) {
2087 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2088 "ahci_claim_free_slot: there is still pending "
2089 "command(s) in command list (0x%x/0x%x, PxCI %x),"
2090 "so no available slot for R/W PMULT command.",
2091 NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
2092 NCQ_CMD_IN_PROGRESS(ahci_portp),
2093 port_cmd_issue);
2094 return (AHCI_FAILURE);
2095 }
2096
2097 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2098 "ahci_claim_free_slot: slot 0 is allocated for "
2099 "READ/WRITE PORTMULT command", NULL);
2100 slot = 0;
2101 goto out;
2102 }
2103
2104 slot = ddi_ffs(free_slots) - 1;
2105 if (slot == -1) {
2106 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2107 "ahci_claim_free_slot: no empty slots", NULL);
2108 return (AHCI_FAILURE);
2109 }
2110
2111 /*
2112 * According to the AHCI spec, to allow a simple mechanism for the
2113 * HBA to map command list slots to queue entries, software must
2114 * match the tag number it uses to the slot it is placing the command
2115 * in. For example, if a queued command is placed in slot 5, the tag
2116 * for that command must be 5.
2117 */
2118 if (command_type == AHCI_NCQ_CMD) {
2119 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2120 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2121 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2122 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2123 }
2124 }
2125
2126 ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2127
2128 out:
2129 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2130 "ahci_claim_free_slot: found slot: 0x%x", slot);
2131
2132 return (slot);
2133 }
2134
2135 /*
2136 * Builds the Command Table for the sata packet and delivers it to controller.
2137 *
2138 * Returns:
2139 * slot number if we can obtain a slot successfully
2140 * otherwise, return AHCI_FAILURE
2141 */
2142 static int
2143 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2144 ahci_addr_t *addrp, sata_pkt_t *spkt)
2145 {
2146 int cmd_slot;
2147 sata_cmd_t *scmd;
2148 ahci_fis_h2d_register_t *h2d_register_fisp;
2149 ahci_cmd_table_t *cmd_table;
2150 ahci_cmd_header_t *cmd_header;
2151 int ncookies;
2152 int i;
2153 int command_type = AHCI_NON_NCQ_CMD;
2154 int ncq_qdepth;
2155 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2156 uint8_t port, pmport;
2157 #if AHCI_DEBUG
2158 uint32_t *ptr;
2159 uint8_t *ptr2;
2160 #endif
2161
2162 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2163
2164 port = addrp->aa_port;
2165 pmport = addrp->aa_pmport;
2166
2167 spkt->satapkt_reason = SATA_PKT_BUSY;
2168
2169 scmd = &spkt->satapkt_cmd;
2170
2171 /* Check if the command is a NCQ command */
2172 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2173 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2174 command_type = AHCI_NCQ_CMD;
2175
2176 /*
2177 * When NCQ is support, system software must determine the
2178 * maximum tag allowed by the device and the HBA, and it
2179 * must use a value not beyond of the lower bound of the two.
2180 *
2181 * Sata module is going to calculate the qdepth and send
2182 * down to HBA driver via sata_cmd.
2183 */
2184 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2185
2186 /*
2187 * At the moment, the driver doesn't support the dynamic
2188 * setting of the maximum ncq depth, and the value can be
2189 * set either during the attach or after hot-plug insertion.
2190 */
2191 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2192 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2193 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2194 "ahci_deliver_satapkt: port %d the max tags for "
2195 "NCQ command is %d", port, ncq_qdepth);
2196 } else {
2197 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2198 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2199 " port %d the max tag for NCQ command is "
2200 "requested to change from %d to %d, at the"
2201 " moment the driver doesn't support the "
2202 "dynamic change so it's going to "
2203 "still use the previous tag value",
2204 instance, port,
2205 ahci_portp->ahciport_max_ncq_tags,
2206 ncq_qdepth);
2207 }
2208 }
2209 }
2210
2211 /* Check if the command is an error retrieval command */
2212 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2213 command_type = AHCI_ERR_RETRI_CMD;
2214
2215 /* Check if the command is an read/write pmult command */
2216 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2217 command_type = AHCI_RDWR_PMULT_CMD;
2218
2219 /* Check if there is an empty command slot */
2220 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2221 addrp, command_type);
2222 if (cmd_slot == AHCI_FAILURE) {
2223 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2224 return (AHCI_FAILURE);
2225 }
2226
2227 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2228 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2229 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2230 cmd_slot, port, (void *)spkt);
2231
2232 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2233 bzero((void *)cmd_table, ahci_cmd_table_size);
2234
2235 /* For data transfer operations, it is the H2D Register FIS */
2236 h2d_register_fisp =
2237 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2238
2239 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2240
2241 /*
2242 * PMP field only make sense when target is a port multiplier or a
2243 * device behind a port multiplier. Otherwise should set it to 0.
2244 */
2245 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2246 SET_FIS_PMP(h2d_register_fisp, pmport);
2247
2248 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2249 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2250 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2251 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2252
2253 switch (scmd->satacmd_addr_type) {
2254
2255 case 0:
2256 /*
2257 * satacmd_addr_type will be 0 for the commands below:
2258 * ATAPI command
2259 * SATAC_IDLE_IM
2260 * SATAC_STANDBY_IM
2261 * SATAC_DOWNLOAD_MICROCODE
2262 * SATAC_FLUSH_CACHE
2263 * SATAC_SET_FEATURES
2264 * SATAC_SMART
2265 * SATAC_ID_PACKET_DEVICE
2266 * SATAC_ID_DEVICE
2267 * SATAC_READ_PORTMULT
2268 * SATAC_WRITE_PORTMULT
2269 */
2270 /* FALLTHRU */
2271
2272 case ATA_ADDR_LBA:
2273 /* FALLTHRU */
2274
2275 case ATA_ADDR_LBA28:
2276 /* LBA[7:0] */
2277 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2278
2279 /* LBA[15:8] */
2280 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2281
2282 /* LBA[23:16] */
2283 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2284
2285 /* LBA [27:24] (also called dev_head) */
2286 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2287
2288 break;
2289
2290 case ATA_ADDR_LBA48:
2291 /* LBA[7:0] */
2292 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2293
2294 /* LBA[15:8] */
2295 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2296
2297 /* LBA[23:16] */
2298 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2299
2300 /* LBA [31:24] */
2301 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2302 scmd->satacmd_lba_low_msb);
2303
2304 /* LBA [39:32] */
2305 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2306 scmd->satacmd_lba_mid_msb);
2307
2308 /* LBA [47:40] */
2309 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2310 scmd->satacmd_lba_high_msb);
2311
2312 /* Set dev_head */
2313 SET_FIS_DEV_HEAD(h2d_register_fisp,
2314 scmd->satacmd_device_reg);
2315
2316 /* Set the extended sector count and features */
2317 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2318 scmd->satacmd_sec_count_msb);
2319 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2320 scmd->satacmd_features_reg_ext);
2321 break;
2322 }
2323
2324 /*
2325 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2326 * filled into features field, and sector count 8:15 is filled into
2327 * features (exp) field. The hba driver doesn't need to anything
2328 * special with regard to this, since sata framework has already
2329 * done so.
2330 *
2331 * However the driver needs to make sure TAG is filled into sector
2332 * field.
2333 */
2334 if (command_type == AHCI_NCQ_CMD) {
2335 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2336 (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2337 }
2338
2339 ncookies = scmd->satacmd_num_dma_cookies;
2340 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2341 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2342 ncookies, ahci_dma_prdt_number);
2343
2344 ASSERT(ncookies <= ahci_dma_prdt_number);
2345 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2346
2347 /* *** now fill the scatter gather list ******* */
2348 for (i = 0; i < ncookies; i++) {
2349 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2350 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2351 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2352 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2353 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2354 scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2355 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2356 scmd->satacmd_dma_cookie_list[i].dmac_size;
2357 }
2358
2359 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2360 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2361 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2362
2363 /* The ACMD field is filled in for ATAPI command */
2364 if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2365 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2366 SATA_ATAPI_MAX_CDB_LEN);
2367 }
2368
2369 /* Set Command Header in Command List */
2370 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2371 BZERO_DESCR_INFO(cmd_header);
2372 BZERO_PRD_BYTE_COUNT(cmd_header);
2373
2374 /* Set the number of entries in the PRD table */
2375 SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2376
2377 /* Set the length of the command in the CFIS area */
2378 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2379
2380 /*
2381 * PMP field only make sense when target is a port multiplier or a
2382 * device behind a port multiplier. Otherwise should set it to 0.
2383 */
2384 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2385 SET_PORT_MULTI_PORT(cmd_header, pmport);
2386
2387 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2388 "sata_data_direction = 0x%x",
2389 scmd->satacmd_flags.sata_data_direction);
2390
2391 /* Set A bit if it is an ATAPI command */
2392 if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2393 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2394
2395 /* Set W bit if data is going to the device */
2396 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2397 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2398
2399 /*
2400 * Set the prefetchable bit - this bit is only valid if the PRDTL
2401 * field is non-zero or the ATAPI 'A' bit is set in the command
2402 * header. This bit cannot be set when using native command
2403 * queuing commands or when using FIS-based switching with a Port
2404 * multiplier.
2405 */
2406 if (command_type != AHCI_NCQ_CMD)
2407 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2408
2409 /*
2410 * Now remember the sata packet in ahciport_slot_pkts[].
2411 * Error retrieval command and r/w port multiplier command will
2412 * be stored specifically for each port.
2413 */
2414 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2415 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2416 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2417
2418 /*
2419 * Keep the timeout value
2420 */
2421 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2422
2423 /*
2424 * If the intial timout is less than 1 tick, then make it longer by
2425 * 1 tick to avoid immediate timeout
2426 */
2427 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2428 ahci_watchdog_timeout)
2429 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2430 ahci_watchdog_timeout;
2431
2432 #if AHCI_DEBUG
2433 if (ahci_debug_flags & AHCIDBG_ATACMD &&
2434 scmd->satacmd_cmd_reg != SATAC_PACKET ||
2435 ahci_debug_flags & AHCIDBG_ATAPICMD &&
2436 scmd->satacmd_cmd_reg == SATAC_PACKET) {
2437
2438 /* Dump the command header and table */
2439 ahci_log(ahci_ctlp, CE_WARN, "\n");
2440 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2441 "0x%p cmd_reg 0x%x port %d", spkt,
2442 scmd->satacmd_cmd_reg, port);
2443 ptr = (uint32_t *)cmd_header;
2444 ahci_log(ahci_ctlp, CE_WARN,
2445 " Command Header:%8x %8x %8x %8x",
2446 ptr[0], ptr[1], ptr[2], ptr[3]);
2447
2448 /* Dump the H2D register FIS */
2449 ptr = (uint32_t *)h2d_register_fisp;
2450 ahci_log(ahci_ctlp, CE_WARN,
2451 " Command FIS: %8x %8x %8x %8x",
2452 ptr[0], ptr[1], ptr[2], ptr[3]);
2453
2454 /* Dump the ACMD register FIS */
2455 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2456 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2457 if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2458 ahci_log(ahci_ctlp, CE_WARN,
2459 " ATAPI command: %2x %2x %2x %2x "
2460 "%2x %2x %2x %2x",
2461 ptr2[8 * i], ptr2[8 * i + 1],
2462 ptr2[8 * i + 2], ptr2[8 * i + 3],
2463 ptr2[8 * i + 4], ptr2[8 * i + 5],
2464 ptr2[8 * i + 6], ptr2[8 * i + 7]);
2465
2466 /* Dump the PRDT */
2467 for (i = 0; i < ncookies; i++) {
2468 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2469 ahci_log(ahci_ctlp, CE_WARN,
2470 " Cookie %d: %8x %8x %8x %8x",
2471 i, ptr[0], ptr[1], ptr[2], ptr[3]);
2472 }
2473 }
2474 #endif
2475
2476 (void) ddi_dma_sync(
2477 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2478 0,
2479 ahci_cmd_table_size,
2480 DDI_DMA_SYNC_FORDEV);
2481
2482 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2483 cmd_slot * sizeof (ahci_cmd_header_t),
2484 sizeof (ahci_cmd_header_t),
2485 DDI_DMA_SYNC_FORDEV);
2486
2487 if ((ahci_check_dma_handle(ahci_portp->
2488 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2489 ahci_check_dma_handle(ahci_portp->
2490 ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2491 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2492 DDI_SERVICE_UNAFFECTED);
2493 return (AHCI_FAILURE);
2494 }
2495
2496 /* Set the corresponding bit in the PxSACT.DS for queued command */
2497 if (command_type == AHCI_NCQ_CMD) {
2498 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2499 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2500 (0x1 << cmd_slot));
2501 }
2502
2503 /* Indicate to the HBA that a command is active. */
2504 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2505 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2506 (0x1 << cmd_slot));
2507
2508 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2509 "exit: port %d", port);
2510
2511 /* Make sure the command is started by the PxSACT/PxCI */
2512 if (ahci_check_acc_handle(ahci_ctlp->
2513 ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2514 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2515 DDI_SERVICE_UNAFFECTED);
2516 return (AHCI_FAILURE);
2517 }
2518
2519 return (cmd_slot);
2520 }
2521
2522 /*
2523 * Called by the sata framework to abort the previously sent packet(s).
2524 *
2525 * Reset device to abort commands.
2526 */
2527 static int
2528 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2529 {
2530 ahci_ctl_t *ahci_ctlp;
2531 ahci_port_t *ahci_portp;
2532 uint32_t slot_status = 0;
2533 uint32_t aborted_tags = 0;
2534 uint32_t finished_tags = 0;
2535 uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2536 uint8_t port;
2537 int tmp_slot;
2538 int instance = ddi_get_instance(dip);
2539
2540 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2541 port = ahci_ctlp->ahcictl_cport_to_port[cport];
2542
2543 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2544 "ahci_tran_abort enter: port %d", port);
2545
2546 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2547 mutex_enter(&ahci_portp->ahciport_mutex);
2548
2549 /*
2550 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2551 * commands are being mopped, therefore there is nothing else to do
2552 */
2553 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2554 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2555 "ahci_tran_abort: port %d is in "
2556 "mopping process, so just return directly ", port);
2557 mutex_exit(&ahci_portp->ahciport_mutex);
2558 return (SATA_SUCCESS);
2559 }
2560
2561 /*
2562 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2563 * command is being executed so no other commands is outstanding,
2564 * nothing to do.
2565 */
2566 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2567 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2568 "ahci_tran_abort: port %d is reading/writing "
2569 "port multiplier, so just return directly ", port);
2570 mutex_exit(&ahci_portp->ahciport_mutex);
2571 return (SATA_SUCCESS);
2572 }
2573
2574 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2575 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2576 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2577 /*
2578 * In case the targer driver would send the request before
2579 * sata framework can have the opportunity to process those
2580 * event reports.
2581 */
2582 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2583 spkt->satapkt_device.satadev_state =
2584 ahci_portp->ahciport_port_state;
2585 ahci_update_sata_registers(ahci_ctlp, port,
2586 &spkt->satapkt_device);
2587 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2588 "ahci_tran_abort returning SATA_FAILURE while "
2589 "port in FAILED/SHUTDOWN/PWROFF state: "
2590 "port: %d", port);
2591 mutex_exit(&ahci_portp->ahciport_mutex);
2592 return (SATA_FAILURE);
2593 }
2594
2595 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2596 /*
2597 * ahci_intr_phyrdy_change() may have rendered it to
2598 * AHCI_PORT_TYPE_NODEV.
2599 */
2600 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2601 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2602 spkt->satapkt_device.satadev_state =
2603 ahci_portp->ahciport_port_state;
2604 ahci_update_sata_registers(ahci_ctlp, port,
2605 &spkt->satapkt_device);
2606 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2607 "ahci_tran_abort returning SATA_FAILURE while "
2608 "no device attached: port: %d", port);
2609 mutex_exit(&ahci_portp->ahciport_mutex);
2610 return (SATA_FAILURE);
2611 }
2612
2613 if (flag == SATA_ABORT_ALL_PACKETS) {
2614 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2615 aborted_tags = ahci_portp->ahciport_pending_tags;
2616 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2617 aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2618
2619 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2620 instance, port);
2621 } else {
2622 aborted_tags = 0xffffffff;
2623 /*
2624 * Aborting one specific packet, first search the
2625 * ahciport_slot_pkts[] list for matching spkt.
2626 */
2627 for (tmp_slot = 0;
2628 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2629 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2630 aborted_tags = (0x1 << tmp_slot);
2631 break;
2632 }
2633 }
2634
2635 if (aborted_tags == 0xffffffff) {
2636 /* request packet is not on the pending list */
2637 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2638 "Cannot find the aborting pkt 0x%p on the "
2639 "pending list", (void *)spkt);
2640 ahci_update_sata_registers(ahci_ctlp, port,
2641 &spkt->satapkt_device);
2642 mutex_exit(&ahci_portp->ahciport_mutex);
2643 return (SATA_FAILURE);
2644 }
2645 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2646 instance, port, (void *)spkt);
2647 }
2648
2649 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2650 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2651 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2652 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2653 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2654 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2655
2656 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2657 ahci_portp->ahciport_mop_in_progress++;
2658
2659 /*
2660 * To abort the packet(s), first we are trying to clear PxCMD.ST
2661 * to stop the port, and if the port can be stopped
2662 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2663 * then we just send back the aborted packet(s) with ABORTED flag
2664 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2665 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2666 * perform a COMRESET.
2667 */
2668 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2669 ahci_portp, port, 0, NULL);
2670
2671 /*
2672 * Compute which have finished and which need to be retried.
2673 *
2674 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2675 * minus the slot_status. The aborted_tags has to be deducted by
2676 * finished_tags since we can't possibly abort a tag which had finished
2677 * already.
2678 */
2679 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2680 finished_tags = ahci_portp->ahciport_pending_tags &
2681 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2682 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2683 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2684 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2685
2686 aborted_tags &= ~finished_tags;
2687
2688 ahci_mop_commands(ahci_ctlp,
2689 ahci_portp,
2690 slot_status,
2691 0, /* failed tags */
2692 0, /* timeout tags */
2693 aborted_tags,
2694 0); /* reset tags */
2695
2696 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2697 mutex_exit(&ahci_portp->ahciport_mutex);
2698
2699 return (SATA_SUCCESS);
2700 }
2701
2702 /*
2703 * Used to do device reset and reject all the pending packets on a device
2704 * during the reset operation.
2705 *
2706 * NOTE: ONLY called by ahci_tran_reset_dport
2707 */
2708 static int
2709 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2710 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2711 {
2712 uint32_t slot_status = 0;
2713 uint32_t reset_tags = 0;
2714 uint32_t finished_tags = 0;
2715 uint8_t port = addrp->aa_port;
2716 sata_device_t sdevice;
2717 int ret;
2718
2719 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2720
2721 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2722 "ahci_reset_device_reject_pkts on port: %d", port);
2723
2724 /*
2725 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2726 * commands are being mopped, therefore there is nothing else to do
2727 */
2728 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2729 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2730 "ahci_reset_device_reject_pkts: port %d is in "
2731 "mopping process, so return directly ", port);
2732 return (SATA_SUCCESS);
2733 }
2734
2735 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2736 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2737 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2738 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2739 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2740 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2741 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2742 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2743 }
2744
2745 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2746 != AHCI_SUCCESS) {
2747 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2748 "Try to do a port reset after software "
2749 "reset failed", port);
2750 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2751 if (ret != AHCI_SUCCESS) {
2752 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2753 "ahci_reset_device_reject_pkts: port %d "
2754 "failed", port);
2755 return (SATA_FAILURE);
2756 }
2757 }
2758 /* Set the reset in progress flag */
2759 ahci_portp->ahciport_reset_in_progress = 1;
2760
2761 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2762 ahci_portp->ahciport_mop_in_progress++;
2763
2764 /* Indicate to the framework that a reset has happened */
2765 bzero((void *)&sdevice, sizeof (sata_device_t));
2766 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2767 sdevice.satadev_addr.pmport = 0;
2768 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2769 sdevice.satadev_state = SATA_DSTATE_RESET |
2770 SATA_DSTATE_PWR_ACTIVE;
2771 mutex_exit(&ahci_portp->ahciport_mutex);
2772 sata_hba_event_notify(
2773 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2774 &sdevice,
2775 SATA_EVNT_DEVICE_RESET);
2776 mutex_enter(&ahci_portp->ahciport_mutex);
2777
2778 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2779 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2780
2781 /* Next try to mop the pending commands */
2782 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2783 finished_tags = ahci_portp->ahciport_pending_tags &
2784 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2785 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2786 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2787 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2788
2789 reset_tags &= ~finished_tags;
2790
2791 ahci_mop_commands(ahci_ctlp,
2792 ahci_portp,
2793 slot_status,
2794 0, /* failed tags */
2795 0, /* timeout tags */
2796 0, /* aborted tags */
2797 reset_tags); /* reset tags */
2798
2799 return (SATA_SUCCESS);
2800 }
2801
2802 /*
2803 * Used to do device reset and reject all the pending packets on a device
2804 * during the reset operation.
2805 *
2806 * NOTE: ONLY called by ahci_tran_reset_dport
2807 */
2808 static int
2809 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2810 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2811 {
2812 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2813 uint8_t port = addrp->aa_port;
2814 uint8_t pmport = addrp->aa_pmport;
2815 sata_device_t sdevice;
2816
2817 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2818
2819 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2820 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2821
2822 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2823 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2824 "ahci_reset_pmdevice_reject_pkts: port %d is in "
2825 "mopping process, so return directly ", port);
2826 return (SATA_SUCCESS);
2827 }
2828
2829 /* Checking for outstanding commands */
2830 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2831 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2832 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2833 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2834 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2835 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2836 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2837 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2838 }
2839
2840 /* Issue SOFTWARE reset command. */
2841 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2842 != AHCI_SUCCESS) {
2843 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2844 "Try to do a port reset after software "
2845 "reset failed", port);
2846 return (SATA_FAILURE);
2847 }
2848
2849 /* Set the reset in progress flag */
2850 ahci_portp->ahciport_reset_in_progress = 1;
2851
2852 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2853 ahci_portp->ahciport_mop_in_progress++;
2854
2855 /* Indicate to the framework that a reset has happened */
2856 bzero((void *)&sdevice, sizeof (sata_device_t));
2857 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2858 sdevice.satadev_addr.pmport = pmport;
2859 if (AHCI_ADDR_IS_PMULT(addrp))
2860 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2861 else
2862 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2863 sdevice.satadev_state = SATA_DSTATE_RESET |
2864 SATA_DSTATE_PWR_ACTIVE;
2865 mutex_exit(&ahci_portp->ahciport_mutex);
2866 sata_hba_event_notify(
2867 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2868 &sdevice,
2869 SATA_EVNT_DEVICE_RESET);
2870 mutex_enter(&ahci_portp->ahciport_mutex);
2871
2872 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2873 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2874 port, pmport);
2875
2876 /* Next try to mop the pending commands */
2877 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2878 finished_tags = ahci_portp->ahciport_pending_tags &
2879 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2880 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2881 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2882 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2883 reset_tags &= ~finished_tags;
2884
2885 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2886 "reset_tags = %x, finished_tags = %x, slot_status = %x",
2887 reset_tags, finished_tags, slot_status);
2888
2889 /*
2890 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2891 * try to reset a single device behind a port multiplier will
2892 * terminate all the commands on that HBA port. We need mop these
2893 * commands as well.
2894 */
2895 ahci_mop_commands(ahci_ctlp,
2896 ahci_portp,
2897 slot_status,
2898 0, /* failed tags */
2899 0, /* timeout tags */
2900 0, /* aborted tags */
2901 reset_tags); /* reset tags */
2902
2903 return (SATA_SUCCESS);
2904 }
2905
2906 /*
2907 * Used to do port reset and reject all the pending packets on a port during
2908 * the reset operation.
2909 */
2910 static int
2911 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2912 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2913 {
2914 uint32_t slot_status = 0;
2915 uint32_t reset_tags = 0;
2916 uint32_t finished_tags = 0;
2917 uint8_t port = addrp->aa_port;
2918
2919 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2920
2921 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2922 "ahci_reset_port_reject_pkts at port: %d", port);
2923
2924 /*
2925 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2926 * commands are being mopped, therefore there is nothing else to do
2927 */
2928 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2929 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2930 "ahci_reset_port_reject_pkts: port %d is in "
2931 "mopping process, so return directly ", port);
2932 return (SATA_SUCCESS);
2933 }
2934
2935 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2936 ahci_portp->ahciport_mop_in_progress++;
2937
2938 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2939 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2940 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2941 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2942 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2943 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2944 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2945 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2946 }
2947
2948 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2949 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2950 NULL) != AHCI_SUCCESS) {
2951
2952 /* Clear mop flag */
2953 ahci_portp->ahciport_mop_in_progress--;
2954 if (ahci_portp->ahciport_mop_in_progress == 0)
2955 ahci_portp->ahciport_flags &=
2956 ~AHCI_PORT_FLAG_MOPPING;
2957 return (SATA_FAILURE);
2958 }
2959
2960 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2961 finished_tags = ahci_portp->ahciport_pending_tags &
2962 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2963 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2964 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2965 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2966
2967 reset_tags &= ~finished_tags;
2968
2969 ahci_mop_commands(ahci_ctlp,
2970 ahci_portp,
2971 slot_status,
2972 0, /* failed tags */
2973 0, /* timeout tags */
2974 0, /* aborted tags */
2975 reset_tags); /* reset tags */
2976
2977 return (SATA_SUCCESS);
2978 }
2979
2980 /*
2981 * Used to do hba reset and reject all the pending packets on all ports
2982 * during the reset operation.
2983 */
2984 static int
2985 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2986 {
2987 ahci_port_t *ahci_portp;
2988 uint32_t slot_status[AHCI_MAX_PORTS];
2989 uint32_t reset_tags[AHCI_MAX_PORTS];
2990 uint32_t finished_tags[AHCI_MAX_PORTS];
2991 int port;
2992 int ret = SATA_SUCCESS;
2993
2994 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2995 "ahci_reset_hba_reject_pkts enter", NULL);
2996
2997 bzero(slot_status, sizeof (slot_status));
2998 bzero(reset_tags, sizeof (reset_tags));
2999 bzero(finished_tags, sizeof (finished_tags));
3000
3001 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3002 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3003 continue;
3004 }
3005
3006 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3007
3008 mutex_enter(&ahci_portp->ahciport_mutex);
3009 ahci_portp->ahciport_reset_in_progress = 1;
3010 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3011 slot_status[port] = ddi_get32(
3012 ahci_ctlp->ahcictl_ahci_acc_handle,
3013 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3014 reset_tags[port] = slot_status[port] &
3015 AHCI_SLOT_MASK(ahci_ctlp);
3016 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3017 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3018 port, reset_tags[port],
3019 ahci_portp->ahciport_pending_tags);
3020 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3021 slot_status[port] = ddi_get32(
3022 ahci_ctlp->ahcictl_ahci_acc_handle,
3023 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3024 reset_tags[port] = slot_status[port] &
3025 AHCI_NCQ_SLOT_MASK(ahci_portp);
3026 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3027 "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3028 port, reset_tags[port],
3029 ahci_portp->ahciport_pending_tags);
3030 }
3031 mutex_exit(&ahci_portp->ahciport_mutex);
3032 }
3033
3034 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
3035 ret = SATA_FAILURE;
3036 }
3037
3038 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3039 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3040 continue;
3041 }
3042
3043 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3044
3045 mutex_enter(&ahci_portp->ahciport_mutex);
3046 /*
3047 * To prevent recursive enter to ahci_mop_commands, we need
3048 * check AHCI_PORT_FLAG_MOPPING flag.
3049 */
3050 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3051 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3052 "ahci_reset_hba_reject_pkts: port %d is in "
3053 "mopping process, so return directly ", port);
3054 mutex_exit(&ahci_portp->ahciport_mutex);
3055 continue;
3056 }
3057
3058 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3059 ahci_portp->ahciport_mop_in_progress++;
3060
3061 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
3062 finished_tags[port] =
3063 ahci_portp->ahciport_pending_tags &
3064 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
3065 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
3066 finished_tags[port] =
3067 ahci_portp->ahciport_pending_ncq_tags &
3068 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
3069
3070 reset_tags[port] &= ~finished_tags[port];
3071
3072 ahci_mop_commands(ahci_ctlp,
3073 ahci_portp,
3074 slot_status[port],
3075 0, /* failed tags */
3076 0, /* timeout tags */
3077 0, /* aborted tags */
3078 reset_tags[port]); /* reset tags */
3079 mutex_exit(&ahci_portp->ahciport_mutex);
3080 }
3081 out:
3082 return (ret);
3083 }
3084
3085 /*
3086 * Called by sata framework to reset a port(s) or device.
3087 */
3088 static int
3089 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
3090 {
3091 ahci_ctl_t *ahci_ctlp;
3092 ahci_port_t *ahci_portp;
3093 ahci_addr_t addr;
3094 uint8_t cport = sd->satadev_addr.cport;
3095 uint8_t pmport = sd->satadev_addr.pmport;
3096 uint8_t port;
3097 int ret = SATA_SUCCESS;
3098 int instance = ddi_get_instance(dip);
3099
3100 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3101 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3102 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3103
3104 ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3105
3106 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3107 "ahci_tran_reset_dport enter: cport %d", cport);
3108
3109 switch (sd->satadev_addr.qual) {
3110 case SATA_ADDR_PMPORT:
3111 /*
3112 * If we want to issue a COMRESET on a pmport, we need to
3113 * reject the outstanding commands on that pmport. According
3114 * to AHCI spec, PxCI register could only be cleared by
3115 * clearing PxCMD.ST, which will halt the controller port - as
3116 * well as other pmports.
3117 *
3118 * Therefore we directly reset the controller port for
3119 * simplicity. ahci_tran_probe_port() will handle reset stuff
3120 * like initializing the given pmport.
3121 */
3122 /* FALLTHRU */
3123 case SATA_ADDR_CPORT:
3124 /* Port reset */
3125 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3126 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3127 "port %d reset port", instance, port);
3128
3129 mutex_enter(&ahci_portp->ahciport_mutex);
3130 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3131 mutex_exit(&ahci_portp->ahciport_mutex);
3132
3133 break;
3134
3135 case SATA_ADDR_DPMPORT:
3136 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3137 "port %d:%d reset device", instance, port, pmport);
3138 /* FALLTHRU */
3139 case SATA_ADDR_DCPORT:
3140 /* Device reset */
3141 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3142 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3143 "port %d reset device", instance, port);
3144
3145 mutex_enter(&ahci_portp->ahciport_mutex);
3146 /*
3147 * software reset request must be sent to SATA_PMULT_HOSTPORT
3148 * if target is a port multiplier:
3149 */
3150 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3151 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3152 AHCI_ADDR_SET_PMULT(&addr, port);
3153
3154 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3155 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3156 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3157 /*
3158 * In case the targer driver would send the request
3159 * before sata framework can have the opportunity to
3160 * process those event reports.
3161 */
3162 sd->satadev_state = ahci_portp->ahciport_port_state;
3163 ahci_update_sata_registers(ahci_ctlp, port, sd);
3164 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3165 "ahci_tran_reset_dport returning SATA_FAILURE "
3166 "while port in FAILED/SHUTDOWN/PWROFF state: "
3167 "port: %d", port);
3168 mutex_exit(&ahci_portp->ahciport_mutex);
3169 ret = SATA_FAILURE;
3170 break;
3171 }
3172
3173 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3174 SATA_DTYPE_NONE) {
3175 /*
3176 * ahci_intr_phyrdy_change() may have rendered it to
3177 * AHCI_PORT_TYPE_NODEV.
3178 */
3179 sd->satadev_type = SATA_DTYPE_NONE;
3180 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3181 &addr);
3182 ahci_update_sata_registers(ahci_ctlp, port, sd);
3183 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3184 "ahci_tran_reset_dport returning SATA_FAILURE "
3185 "while no device attached: port: %d", port);
3186 mutex_exit(&ahci_portp->ahciport_mutex);
3187 ret = SATA_FAILURE;
3188 break;
3189 }
3190
3191 if (AHCI_ADDR_IS_PORT(&addr)) {
3192 ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3193 ahci_portp, &addr);
3194 } else {
3195 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3196 ahci_portp, &addr);
3197 }
3198
3199 mutex_exit(&ahci_portp->ahciport_mutex);
3200 break;
3201
3202 case SATA_ADDR_CNTRL:
3203 /* Reset the whole controller */
3204 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3205 "reset the whole hba", instance);
3206 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3207 break;
3208
3209 default:
3210 ret = SATA_FAILURE;
3211 }
3212
3213 return (ret);
3214 }
3215
3216 /*
3217 * Called by sata framework to activate a port as part of hotplug.
3218 * (cfgadm -c connect satax/y)
3219 * Support port multiplier.
3220 */
3221 static int
3222 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3223 {
3224 ahci_ctl_t *ahci_ctlp;
3225 ahci_port_t *ahci_portp;
3226 ahci_addr_t addr;
3227 uint8_t cport = satadev->satadev_addr.cport;
3228 uint8_t pmport = satadev->satadev_addr.pmport;
3229 uint8_t port;
3230 int instance = ddi_get_instance(dip);
3231
3232 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3233 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3234
3235 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3236 "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3237
3238 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3239
3240 mutex_enter(&ahci_portp->ahciport_mutex);
3241 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3242 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3243
3244 if (AHCI_ADDR_IS_PORT(&addr)) {
3245 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3246 instance, port);
3247
3248 /* Enable the interrupts on the port */
3249 ahci_enable_port_intrs(ahci_ctlp, port);
3250
3251 /*
3252 * Reset the port so that the PHY communication would be
3253 * re-established. But this reset is an internal operation
3254 * and the sata module doesn't need to know about it.
3255 * Moreover, the port with a device attached will be started
3256 * too.
3257 */
3258 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3259 ahci_portp, port,
3260 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3261 NULL);
3262
3263 /*
3264 * Need to check the link status and device status of the port
3265 * and consider raising power if the port was in D3 state
3266 */
3267 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3268 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3269 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3270 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3271 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3272 instance, port, pmport);
3273 /* AHCI_ADDR_PMPORT */
3274 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3275 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3276 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3277 }
3278
3279 satadev->satadev_state = ahci_portp->ahciport_port_state;
3280
3281 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3282
3283 mutex_exit(&ahci_portp->ahciport_mutex);
3284 return (SATA_SUCCESS);
3285 }
3286
3287 /*
3288 * Called by sata framework to deactivate a port as part of hotplug.
3289 * (cfgadm -c disconnect satax/y)
3290 * Support port multiplier.
3291 */
3292 static int
3293 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3294 {
3295 ahci_ctl_t *ahci_ctlp;
3296 ahci_port_t *ahci_portp;
3297 ahci_addr_t addr;
3298 uint8_t cport = satadev->satadev_addr.cport;
3299 uint8_t pmport = satadev->satadev_addr.pmport;
3300 uint8_t port;
3301 uint32_t port_scontrol;
3302 int instance = ddi_get_instance(dip);
3303
3304 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3305 port = ahci_ctlp->ahcictl_cport_to_port[cport];
3306
3307 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3308 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3309
3310 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3311 mutex_enter(&ahci_portp->ahciport_mutex);
3312 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3313 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3314
3315 if (AHCI_ADDR_IS_PORT(&addr)) {
3316 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3317 instance, port);
3318
3319 /* Disable the interrupts on the port */
3320 ahci_disable_port_intrs(ahci_ctlp, port);
3321
3322 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3323
3324 /* First to abort all the pending commands */
3325 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3326
3327 /* Then stop the port */
3328 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3329 ahci_portp, port);
3330 }
3331
3332 /* Next put the PHY offline */
3333 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3334 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3335 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3336 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3337 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3338 } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3339 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3340 instance, port, pmport);
3341
3342 ahci_disable_port_intrs(ahci_ctlp, port);
3343 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3344 != SATA_DTYPE_NONE)
3345 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3346
3347 /* Re-enable the interrupts for the other pmports */
3348 ahci_enable_port_intrs(ahci_ctlp, port);
3349 }
3350
3351 /* Update port state */
3352 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3353 satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3354
3355 ahci_update_sata_registers(ahci_ctlp, port, satadev);
3356
3357 mutex_exit(&ahci_portp->ahciport_mutex);
3358 return (SATA_SUCCESS);
3359 }
3360
3361 /*
3362 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3363 * when a device is unplugged or a port is deactivated.
3364 */
3365 static void
3366 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3367 ahci_port_t *ahci_portp, uint8_t port)
3368 {
3369 uint32_t slot_status = 0;
3370 uint32_t abort_tags = 0;
3371
3372 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3373
3374 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3375 "ahci_reject_all_abort_pkts at port: %d", port);
3376
3377 /* Read/write port multiplier command takes highest priority */
3378 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3379 slot_status = 0x1;
3380 abort_tags = 0x1;
3381 goto out;
3382 }
3383
3384 /*
3385 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3386 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3387 * to get the error data, if yes when the device is removed, the
3388 * command needs to be aborted too.
3389 */
3390 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3391 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3392 slot_status = 0x1;
3393 abort_tags = 0x1;
3394 goto out;
3395 } else {
3396 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3397 "ahci_reject_all_abort_pkts return directly "
3398 "port %d no needs to reject any outstanding "
3399 "commands", port);
3400 return;
3401 }
3402 }
3403
3404 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3405 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3406 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3407 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3408 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3409 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3410 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3411 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3412 }
3413
3414 out:
3415 /* No need to do mop when there is no outstanding commands */
3416 if (slot_status != 0) {
3417 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3418 ahci_portp->ahciport_mop_in_progress++;
3419
3420 ahci_mop_commands(ahci_ctlp,
3421 ahci_portp,
3422 slot_status,
3423 0, /* failed tags */
3424 0, /* timeout tags */
3425 abort_tags, /* aborting tags */
3426 0); /* reset tags */
3427 }
3428 }
3429
3430 #if defined(__lock_lint)
3431 static int
3432 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3433 {
3434 return (SATA_SUCCESS);
3435 }
3436 #endif
3437
3438 /*
3439 * Initialize fma capabilities and register with IO fault services.
3440 */
3441 static void
3442 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3443 {
3444 /*
3445 * Need to change iblock to priority for new MSI intr
3446 */
3447 ddi_iblock_cookie_t fm_ibc;
3448
3449 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3450 ahci_ctlp->ahcictl_dip,
3451 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3452 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3453 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3454
3455 /* Only register with IO Fault Services if we have some capability */
3456 if (ahci_ctlp->ahcictl_fm_cap) {
3457 /* Adjust access and dma attributes for FMA */
3458 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3459 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3460 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3461 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3462 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3463
3464 /*
3465 * Register capabilities with IO Fault Services.
3466 * ahcictl_fm_cap will be updated to indicate
3467 * capabilities actually supported (not requested.)
3468 */
3469 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3470 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3471
3472 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3473 cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3474 ddi_get_instance(ahci_ctlp->ahcictl_dip));
3475 return;
3476 }
3477 /*
3478 * Initialize pci ereport capabilities if ereport
3479 * capable (should always be.)
3480 */
3481 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3482 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3483 pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3484 }
3485
3486 /*
3487 * Register error callback if error callback capable.
3488 */
3489 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3490 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3491 ahci_fm_error_cb, (void *) ahci_ctlp);
3492 }
3493
3494 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3495 "ahci_fm_fini: fma enabled.", NULL);
3496 }
3497 }
3498
3499 /*
3500 * Releases fma capabilities and un-registers with IO fault services.
3501 */
3502 static void
3503 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3504 {
3505 /* Only unregister FMA capabilities if registered */
3506 if (ahci_ctlp->ahcictl_fm_cap) {
3507 /*
3508 * Un-register error callback if error callback capable.
3509 */
3510 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3511 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3512 }
3513
3514 /*
3515 * Release any resources allocated by pci_ereport_setup()
3516 */
3517 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3518 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3519 pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3520 }
3521
3522 /* Unregister from IO Fault Services */
3523 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3524
3525 /* Adjust access and dma attributes for FMA */
3526 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3527 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3528 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3529 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3530 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3531
3532 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3533 "ahci_fm_fini: fma disabled.", NULL);
3534 }
3535 }
3536
3537 /*ARGSUSED*/
3538 static int
3539 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3540 {
3541 /*
3542 * as the driver can always deal with an error in any dma or
3543 * access handle, we can just return the fme_status value.
3544 */
3545 pci_ereport_post(dip, err, NULL);
3546 return (err->fme_status);
3547 }
3548
3549 int
3550 ahci_check_acc_handle(ddi_acc_handle_t handle)
3551 {
3552 ddi_fm_error_t de;
3553
3554 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3555 return (de.fme_status);
3556 }
3557
3558 int
3559 ahci_check_dma_handle(ddi_dma_handle_t handle)
3560 {
3561 ddi_fm_error_t de;
3562
3563 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3564 return (de.fme_status);
3565 }
3566
3567 /*
3568 * Generate an ereport
3569 */
3570 void
3571 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3572 {
3573 uint64_t ena;
3574 char buf[FM_MAX_CLASS];
3575
3576 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3577 ena = fm_ena_generate(0, FM_ENA_FMT1);
3578 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3579 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3580 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3581 FM_EREPORT_VERSION, NULL);
3582 }
3583 }
3584
3585 /*
3586 * Check if all handles are correctly allocated.
3587 */
3588 static int
3589 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3590 {
3591 int port;
3592
3593 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3594 return (DDI_FAILURE);
3595 }
3596
3597 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3598 ahci_port_t *ahci_portp;
3599
3600 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3601 continue;
3602
3603 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3604
3605 mutex_enter(&ahci_portp->ahciport_mutex);
3606 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3607 mutex_exit(&ahci_portp->ahciport_mutex);
3608 return (DDI_FAILURE);
3609 }
3610 mutex_exit(&ahci_portp->ahciport_mutex);
3611 }
3612
3613 return (DDI_SUCCESS);
3614 }
3615
3616 /*
3617 * Check the access handles for the controller. Note that
3618 * ahcictl_pci_conf_handle is only used in attach process.
3619 */
3620 static int
3621 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3622 {
3623 if ((ahci_check_acc_handle(ahci_ctlp->
3624 ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3625 (ahci_check_acc_handle(ahci_ctlp->
3626 ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3627 return (DDI_FAILURE);
3628 }
3629 return (DDI_SUCCESS);
3630 }
3631
3632 /*
3633 * Check the DMA handles and the access handles of a controller port.
3634 */
3635 static int
3636 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3637 {
3638 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3639 int slot;
3640
3641 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3642
3643 if ((ahci_check_dma_handle(ahci_portp->
3644 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3645 (ahci_check_dma_handle(ahci_portp->
3646 ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3647 (ahci_check_acc_handle(ahci_portp->
3648 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3649 (ahci_check_acc_handle(ahci_portp->
3650 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3651 return (DDI_FAILURE);
3652 }
3653 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3654 if (ahci_check_slot_handle(ahci_portp, slot)
3655 != DDI_SUCCESS) {
3656 return (DDI_FAILURE);
3657 }
3658 }
3659 return (DDI_SUCCESS);
3660 }
3661
3662 /*
3663 * Check the DMA handles and the access handles of a cmd table slot.
3664 */
3665 static int
3666 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3667 {
3668 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3669
3670 if ((ahci_check_acc_handle(ahci_portp->
3671 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3672 (ahci_check_dma_handle(ahci_portp->
3673 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3674 return (DDI_FAILURE);
3675 }
3676 return (DDI_SUCCESS);
3677 }
3678
3679 /*
3680 * Allocate the ports structure, only called by ahci_attach
3681 */
3682 static int
3683 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3684 {
3685 int port, cport = 0;
3686
3687 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3688 "ahci_alloc_ports_state enter", NULL);
3689
3690 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3691
3692 /* Allocate structures only for the implemented ports */
3693 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3694 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3695 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3696 "hba port %d not implemented", port);
3697 continue;
3698 }
3699
3700 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3701 ahci_ctlp->ahcictl_port_to_cport[port] =
3702 (uint8_t)cport++;
3703
3704 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3705 goto err_out;
3706 }
3707 }
3708
3709 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3710 return (AHCI_SUCCESS);
3711
3712 err_out:
3713 for (port--; port >= 0; port--) {
3714 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3715 ahci_dealloc_port_state(ahci_ctlp, port);
3716 }
3717 }
3718
3719 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3720 return (AHCI_FAILURE);
3721 }
3722
3723 /*
3724 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3725 */
3726 static void
3727 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3728 {
3729 int port;
3730
3731 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3732 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3733 /* if this port is implemented by the HBA */
3734 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3735 ahci_dealloc_port_state(ahci_ctlp, port);
3736 }
3737 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3738 }
3739
3740 /*
3741 * Drain the taskq.
3742 */
3743 static void
3744 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3745 {
3746 ahci_port_t *ahci_portp;
3747 int port;
3748
3749 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3750 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3751 continue;
3752 }
3753
3754 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3755
3756 mutex_enter(&ahci_portp->ahciport_mutex);
3757 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3758 mutex_exit(&ahci_portp->ahciport_mutex);
3759 }
3760 }
3761
3762 /*
3763 * Initialize the controller and all ports. And then try to start the ports
3764 * if there are devices attached.
3765 *
3766 * This routine can be called from three seperate cases: DDI_ATTACH,
3767 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3768 * other two cases; device signature probing are attempted only during
3769 * DDI_ATTACH case.
3770 */
3771 static int
3772 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3773 {
3774 ahci_port_t *ahci_portp;
3775 ahci_addr_t addr;
3776 int port;
3777
3778 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3779 "ahci_initialize_controller enter", NULL);
3780
3781 /* Disable the whole controller interrupts */
3782 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3783 ahci_disable_all_intrs(ahci_ctlp);
3784 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3785
3786 /* Initialize the implemented ports and structures */
3787 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3788 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3789 continue;
3790 }
3791
3792 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3793 mutex_enter(&ahci_portp->ahciport_mutex);
3794
3795 /*
3796 * Ensure that the controller is not in the running state
3797 * by checking every implemented port's PxCMD register
3798 */
3799 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3800
3801 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3802 != AHCI_SUCCESS) {
3803 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3804 "ahci_initialize_controller: failed to "
3805 "initialize port %d", port);
3806 /*
3807 * Set the port state to SATA_PSTATE_FAILED if
3808 * failed to initialize it.
3809 */
3810 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3811 }
3812
3813 mutex_exit(&ahci_portp->ahciport_mutex);
3814 }
3815
3816 /* Enable the whole controller interrupts */
3817 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3818 ahci_enable_all_intrs(ahci_ctlp);
3819 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3820
3821 return (AHCI_SUCCESS);
3822 }
3823
3824 /*
3825 * Reverse of ahci_initialize_controller()
3826 *
3827 * We only need to stop the ports and disable the interrupt.
3828 */
3829 static void
3830 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3831 {
3832 ahci_port_t *ahci_portp;
3833 int port;
3834
3835 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3836 "ahci_uninitialize_controller enter", NULL);
3837
3838 /* disable all the interrupts. */
3839 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3840 ahci_disable_all_intrs(ahci_ctlp);
3841 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3842
3843 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3844 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3845 continue;
3846 }
3847
3848 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3849
3850 /* Stop the port by clearing PxCMD.ST */
3851 mutex_enter(&ahci_portp->ahciport_mutex);
3852
3853 /*
3854 * Here we must disable the port interrupt because
3855 * ahci_disable_all_intrs only clear GHC.IE, and IS
3856 * register will be still set if PxIE is enabled.
3857 * When ahci shares one IRQ with other drivers, the
3858 * intr handler may claim the intr mistakenly.
3859 */
3860 ahci_disable_port_intrs(ahci_ctlp, port);
3861 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3862 ahci_portp, port);
3863 mutex_exit(&ahci_portp->ahciport_mutex);
3864 }
3865 }
3866
3867 /*
3868 * ahci_alloc_pmult()
3869 * 1. Setting HBA port registers which are necessary for a port multiplier.
3870 * (Set PxCMD.PMA while PxCMD.ST is '0')
3871 * 2. Allocate ahci_pmult_info structure.
3872 *
3873 * NOTE: Must stop port before the function is called.
3874 */
3875 static void
3876 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3877 {
3878 uint32_t port_cmd_status;
3879 uint8_t port = ahci_portp->ahciport_port_num;
3880
3881 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3882
3883 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3884 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3885
3886 /* The port must have been stopped before. */
3887 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3888
3889 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3890 /* set PMA bit */
3891 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3892 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3893 port_cmd_status|AHCI_CMD_STATUS_PMA);
3894
3895 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3896 "ahci_alloc_pmult: "
3897 "PxCMD.PMA bit set at port %d.", port);
3898 }
3899
3900 /* Allocate port multiplier information structure */
3901 if (ahci_portp->ahciport_pmult_info == NULL) {
3902 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3903 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3904 }
3905
3906 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3907 }
3908
3909 /*
3910 * ahci_dealloc_pmult()
3911 * 1. Clearing related registers when a port multiplier is detached.
3912 * (Clear PxCMD.PMA while PxCMD.ST is '0')
3913 * 2. Deallocate ahci_pmult_info structure.
3914 *
3915 * NOTE: Must stop port before the function is called.
3916 */
3917 static void
3918 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3919 {
3920 uint32_t port_cmd_status;
3921 uint8_t port = ahci_portp->ahciport_port_num;
3922
3923 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3924
3925 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3926 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3927
3928 if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3929 /* Clear PMA bit */
3930 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3931 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3932 (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3933
3934 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3935 "ahci_dealloc_pmult: "
3936 "PxCMD.PMA bit cleared at port %d.", port);
3937 }
3938
3939 /* Release port multiplier information structure */
3940 if (ahci_portp->ahciport_pmult_info != NULL) {
3941 kmem_free(ahci_portp->ahciport_pmult_info,
3942 sizeof (ahci_pmult_info_t));
3943 ahci_portp->ahciport_pmult_info = NULL;
3944 }
3945 }
3946
3947 /*
3948 * Staggered Spin-up.
3949 */
3950 static void
3951 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3952 {
3953 uint32_t cap_status;
3954 uint32_t port_cmd_status;
3955
3956 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3957
3958 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3959 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3960
3961 /* Check for staggered spin-up support */
3962 if (!(cap_status & AHCI_HBA_CAP_SSS))
3963 return;
3964
3965 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3966 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3967
3968 /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3969 if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3970 return;
3971
3972 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3973
3974 /* Set PxCMD.SUD */
3975 port_cmd_status |= AHCI_CMD_STATUS_SUD;
3976 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3977 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3978 port_cmd_status);
3979 }
3980
3981 /*
3982 * The routine is to initialize a port. First put the port in NotRunning
3983 * state, then enable port interrupt and clear Serror register. And under
3984 * AHCI_ATTACH case, find device signature and then try to start the port.
3985 *
3986 * Called by
3987 * 1. ahci_initialize_controller
3988 * 2. ahci_intr_phyrdy_change (hotplug)
3989 */
3990 static int
3991 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3992 ahci_addr_t *addrp)
3993 {
3994 uint32_t port_sstatus, port_task_file, port_cmd_status;
3995 uint8_t port = addrp->aa_port;
3996 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */
3997 int ret;
3998
3999 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4000
4001 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */
4002 ASSERT(AHCI_ADDR_IS_PORT(addrp));
4003
4004 /*
4005 * At the time being, only probe ports/devices and get the types of
4006 * attached devices during DDI_ATTACH. In fact, the device can be
4007 * changed during power state changes, but at the time being, we
4008 * don't support the situation.
4009 */
4010 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4011 resuming = B_FALSE;
4012 } else {
4013 /* check for DDI_RESUME case */
4014 mutex_exit(&ahci_portp->ahciport_mutex);
4015 mutex_enter(&ahci_ctlp->ahcictl_mutex);
4016 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
4017 resuming = B_FALSE;
4018 mutex_exit(&ahci_ctlp->ahcictl_mutex);
4019 mutex_enter(&ahci_portp->ahciport_mutex);
4020 }
4021
4022 if (resuming) {
4023 /*
4024 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
4025 * and PxFBU registers in case these registers were cleared
4026 * during the suspend.
4027 */
4028 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4029 "ahci_initialize_port: port %d "
4030 "set PxCLB, PxCLBU, PxFB and PxFBU "
4031 "during resume", port);
4032
4033 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
4034 AHCI_SUCCESS)
4035 return (AHCI_FAILURE);
4036 }
4037
4038 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4039 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4040
4041 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4042 "ahci_initialize_port: port %d ", port);
4043
4044 /*
4045 * Check whether the port is in NotRunning state, if not,
4046 * put the port in NotRunning state
4047 */
4048 if (port_cmd_status &
4049 (AHCI_CMD_STATUS_ST |
4050 AHCI_CMD_STATUS_CR |
4051 AHCI_CMD_STATUS_FRE |
4052 AHCI_CMD_STATUS_FR)) {
4053 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
4054 ahci_portp, port);
4055 }
4056
4057 /* Make sure the drive is spun-up */
4058 ahci_staggered_spin_up(ahci_ctlp, port);
4059
4060 /* Disable interrupt */
4061 ahci_disable_port_intrs(ahci_ctlp, port);
4062
4063 /* Device is unknown at first */
4064 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4065
4066 /* Disable the interface power management */
4067 ahci_disable_interface_pm(ahci_ctlp, port);
4068
4069 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4070 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
4071 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4072 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4073
4074 /* Check physcial link status */
4075 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
4076 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
4077
4078 /* Check interface status */
4079 port_task_file & AHCI_TFD_STS_BSY ||
4080 port_task_file & AHCI_TFD_STS_DRQ ||
4081
4082 /* Check whether port reset must be executed */
4083 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
4084
4085 /* Always reset port on RESUME */
4086 resuming != B_FALSE) {
4087
4088 /* Something went wrong, we need do some reset things */
4089 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
4090
4091 /* Does port reset succeed on HBA port? */
4092 if (ret != AHCI_SUCCESS) {
4093 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4094 "ahci_initialize_port:"
4095 "port reset failed at port %d", port);
4096 return (AHCI_FAILURE);
4097 }
4098
4099 /* Is port failed? */
4100 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4101 SATA_PSTATE_FAILED) {
4102 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4103 "ahci_initialize_port: port %d state 0x%x",
4104 port, ahci_portp->ahciport_port_state);
4105 return (AHCI_FAILURE);
4106 }
4107 }
4108
4109 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4110 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4111
4112 /*
4113 * Try to get the device signature if the port is not empty.
4114 */
4115 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4116 SATA_DTYPE_NONE)
4117 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4118
4119 /* Return directly if no device connected */
4120 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4121 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4122 "No device connected to port %d", port);
4123 goto out;
4124 }
4125
4126 /* If this is a port multiplier, we need do some initialization */
4127 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4128 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4129 "Port multiplier found at port %d", port);
4130 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4131 }
4132
4133 /* Try to start the port */
4134 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4135 != AHCI_SUCCESS) {
4136 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4137 "failed to start port %d", port);
4138 return (AHCI_FAILURE);
4139 }
4140 out:
4141 /* Enable port interrupts */
4142 ahci_enable_port_intrs(ahci_ctlp, port);
4143
4144 return (AHCI_SUCCESS);
4145 }
4146
4147 /*
4148 * Handle hardware defect, and check the capabilities. For example,
4149 * power management capabilty and MSI capability.
4150 */
4151 static int
4152 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4153 {
4154 ushort_t caps_ptr, cap_count, cap;
4155 #if AHCI_DEBUG
4156 ushort_t pmcap, pmcsr;
4157 ushort_t msimc;
4158 #endif
4159 uint8_t revision;
4160
4161 ahci_ctlp->ahcictl_venid =
4162 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4163 PCI_CONF_VENID);
4164
4165 ahci_ctlp->ahcictl_devid =
4166 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4167 PCI_CONF_DEVID);
4168
4169 /*
4170 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4171 * controllers with 0x00 revision id work on 4-byte aligned buffer,
4172 * which is a bug and was fixed after 0x00 revision id controllers.
4173 *
4174 * Moreover, VT8251 cannot use multiple command slots in the command
4175 * list for non-queued commands because the previous register content
4176 * of PxCI can be re-written in the register write, so a flag will be
4177 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4178 *
4179 * For VT8251, software reset also has the same defect as the below
4180 * AMD/ATI chipset. That is, software reset will get failed if 0xf
4181 * is filled in pmport field. Therefore, another software reset need
4182 * to be done with 0 filled in pmport field.
4183 */
4184 if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4185 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4186 PCI_CONF_REVID);
4187 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4188 "revision id = 0x%x", revision);
4189 if (revision == 0x00) {
4190 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4191 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4192 "change ddi_attr_align to 0x4", NULL);
4193 }
4194
4195 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4196 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4197 "VT8251 cannot use multiple command lists for "
4198 "non-queued commands", NULL);
4199
4200 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4201 }
4202
4203 /*
4204 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4205 * DMA addressing for communication memory descriptors though S64A bit
4206 * of CAP register declares it supports. Even though 64-bit DMA for
4207 * data buffer works on ASUS M2A-VM with newer BIOS, three other
4208 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4209 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4210 *
4211 * Due to certain hardware issue, the chipset must do port reset during
4212 * initialization, otherwise, when retrieving device signature,
4213 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4214 * need to set.
4215 *
4216 * For this chipset software reset will get failure if the pmport of
4217 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4218 * multiplier is connected to the port. In order to fix the issue,
4219 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4220 * reset got failure, the driver will try to do another software reset
4221 * with pmport 0.
4222 */
4223 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4224 ahci_ctlp->ahcictl_devid == 0x4380) {
4225 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4226 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4227 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4228 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4229
4230 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4231 "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4232 "communication memory descriptors though CAP indicates "
4233 "support, so force it to use 32-bit DMA", NULL);
4234 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4235 "ATI SB600 need to do a port reset during initialization",
4236 NULL);
4237 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4238 "ATI SB600 will get software reset failure if pmport "
4239 "is set 0xf and no port multiplier is attached", NULL);
4240 }
4241
4242 /*
4243 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4244 * vendor ID and device ID (0x1002,0x4391).
4245 *
4246 * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4247 * DMA addressing for communication memory descriptors though S64A bit
4248 * of CAP register declares the support. However, it does support
4249 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4250 * set for this controller.
4251 *
4252 * SB710 has the same initialization issue as SB600, so it also need
4253 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4254 *
4255 * SB700 also has the same issue about software reset, and thus
4256 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4257 */
4258 if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4259 ahci_ctlp->ahcictl_devid == 0x4391) {
4260 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4261 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4262 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4263
4264 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4265 "ATI SB700/750 cannot do 64-bit DMA for communication "
4266 "memory descriptors though CAP indicates support, "
4267 "so force it to use 32-bit DMA", NULL);
4268 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4269 "ATI SB710 need to do a port reset during initialization",
4270 NULL);
4271 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4272 "ATI SB700 will get software reset failure if pmport "
4273 "is set 0xf and no port multiplier is attached", NULL);
4274 }
4275
4276 /*
4277 * Check if capabilities list is supported and if so,
4278 * get initial capabilities pointer and clear bits 0,1.
4279 */
4280 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4281 PCI_CONF_STAT) & PCI_STAT_CAP) {
4282 caps_ptr = P2ALIGN(pci_config_get8(
4283 ahci_ctlp->ahcictl_pci_conf_handle,
4284 PCI_CONF_CAP_PTR), 4);
4285 } else {
4286 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4287 }
4288
4289 /*
4290 * Walk capabilities if supported.
4291 */
4292 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4293
4294 /*
4295 * Check that we haven't exceeded the maximum number of
4296 * capabilities and that the pointer is in a valid range.
4297 */
4298 if (++cap_count > PCI_CAP_MAX_PTR) {
4299 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4300 "too many device capabilities", NULL);
4301 return (AHCI_FAILURE);
4302 }
4303 if (caps_ptr < PCI_CAP_PTR_OFF) {
4304 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4305 "capabilities pointer 0x%x out of range",
4306 caps_ptr);
4307 return (AHCI_FAILURE);
4308 }
4309
4310 /*
4311 * Get next capability and check that it is valid.
4312 * For now, we only support power management.
4313 */
4314 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4315 caps_ptr);
4316 switch (cap) {
4317 case PCI_CAP_ID_PM:
4318
4319 /* power management supported */
4320 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4321
4322 /* Save PMCSR offset */
4323 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4324
4325 #if AHCI_DEBUG
4326 pmcap = pci_config_get16(
4327 ahci_ctlp->ahcictl_pci_conf_handle,
4328 caps_ptr + PCI_PMCAP);
4329 pmcsr = pci_config_get16(
4330 ahci_ctlp->ahcictl_pci_conf_handle,
4331 ahci_ctlp->ahcictl_pmcsr_offset);
4332 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4333 "Power Management capability found PCI_PMCAP "
4334 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4335 if ((pmcap & 0x3) == 0x3)
4336 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4337 "PCI Power Management Interface "
4338 "spec 1.2 compliant", NULL);
4339 #endif
4340 break;
4341
4342 case PCI_CAP_ID_MSI:
4343 #if AHCI_DEBUG
4344 msimc = pci_config_get16(
4345 ahci_ctlp->ahcictl_pci_conf_handle,
4346 caps_ptr + PCI_MSI_CTRL);
4347 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4348 "Message Signaled Interrupt capability found "
4349 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4350 #endif
4351 AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4352 "MSI capability found", NULL);
4353 break;
4354
4355 case PCI_CAP_ID_PCIX:
4356 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4357 "PCI-X capability found", NULL);
4358 break;
4359
4360 case PCI_CAP_ID_PCI_E:
4361 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4362 "PCI Express capability found", NULL);
4363 break;
4364
4365 case PCI_CAP_ID_MSI_X:
4366 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4367 "MSI-X capability found", NULL);
4368 break;
4369
4370 case PCI_CAP_ID_SATA:
4371 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4372 "SATA capability found", NULL);
4373 break;
4374
4375 case PCI_CAP_ID_VS:
4376 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4377 "Vendor Specific capability found", NULL);
4378 break;
4379
4380 default:
4381 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4382 "unrecognized capability 0x%x", cap);
4383 break;
4384 }
4385
4386 /*
4387 * Get next capabilities pointer and clear bits 0,1.
4388 */
4389 caps_ptr = P2ALIGN(pci_config_get8(
4390 ahci_ctlp->ahcictl_pci_conf_handle,
4391 (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4392 }
4393
4394 return (AHCI_SUCCESS);
4395 }
4396
4397 /*
4398 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4399 * PORTMULT command. SYNC & POLLING mode is used.
4400 */
4401 static int
4402 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4403 uint8_t regn, uint32_t *pregv, uint8_t type)
4404 {
4405 ahci_port_t *ahci_portp;
4406 ahci_addr_t pmult_addr;
4407 sata_pkt_t *spkt;
4408 sata_cmd_t *scmd;
4409 sata_device_t sata_device;
4410 uint8_t port = addrp->aa_port;
4411 uint8_t pmport = addrp->aa_pmport;
4412 uint8_t cport;
4413 uint32_t intr_mask;
4414 int rval;
4415 char portstr[10];
4416
4417 SET_PORTSTR(portstr, addrp);
4418 cport = ahci_ctlp->ahcictl_port_to_cport[port];
4419 ahci_portp = ahci_ctlp->ahcictl_ports[port];
4420
4421 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4422 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4423
4424 /* Check the existence of the port multiplier */
4425 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4426 return (AHCI_FAILURE);
4427
4428 /* Request a READ/WRITE PORTMULT sata packet. */
4429 bzero(&sata_device, sizeof (sata_device_t));
4430 sata_device.satadev_addr.cport = cport;
4431 sata_device.satadev_addr.pmport = pmport;
4432 sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4433 sata_device.satadev_rev = SATA_DEVICE_REV;
4434
4435 /*
4436 * Make sure no command is outstanding here. All R/W PMULT requests
4437 * come from
4438 *
4439 * 1. ahci_attach()
4440 * The port should be empty.
4441 *
4442 * 2. ahci_tran_probe_port()
4443 * Any request from SATA framework (via ahci_tran_start) should be
4444 * rejected if R/W PMULT command is outstanding.
4445 *
4446 * If we are doing mopping, do not check those flags because no
4447 * command will be actually outstanding.
4448 *
4449 * If the port has been occupied by any other commands, the probe
4450 * function will return a SATA_RETRY. SATA framework will retry
4451 * later.
4452 */
4453 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4454 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4455 "R/W PMULT failed: R/W PMULT in progress at port %d.",
4456 port, ahci_portp->ahciport_flags);
4457 return (AHCI_FAILURE);
4458 }
4459
4460 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4461 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4462 NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4463 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4464 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4465 "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4466 port, ahci_portp->ahciport_flags);
4467 return (AHCI_FAILURE);
4468 }
4469
4470 /*
4471 * The port multiplier is gone. This may happen when
4472 * 1. Cutting off the power of an enclosure. The device lose the power
4473 * before port multiplier.
4474 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4475 *
4476 * The issued command should be aborted and the following command
4477 * should not be continued.
4478 */
4479 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4480 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4481 "READ/WRITE PMULT failed: "
4482 "port-mult is removed from port %d", port);
4483 return (AHCI_FAILURE);
4484 }
4485
4486 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4487
4488 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4489 &sata_device, regn, *pregv, type);
4490
4491 /*
4492 * READ/WRITE PORTMULT command is intended to sent to the control port
4493 * of the port multiplier.
4494 */
4495 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4496
4497 ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4498
4499 /* No interrupt here. Store the interrupt enable mask. */
4500 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4501 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4502 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4503 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4504
4505 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4506
4507 if (rval == AHCI_SUCCESS &&
4508 spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4509 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4510 scmd = &spkt->satapkt_cmd;
4511 *pregv = scmd->satacmd_lba_high_lsb << 24 |
4512 scmd->satacmd_lba_mid_lsb << 16 |
4513 scmd->satacmd_lba_low_lsb << 8 |
4514 scmd->satacmd_sec_count_lsb;
4515 }
4516 } else {
4517 /* Failed or not completed. */
4518 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4519 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4520 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4521 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4522 rval = AHCI_FAILURE;
4523 }
4524 out:
4525 /* Restore the interrupt mask */
4526 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4527 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4528
4529 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4530 ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4531 sata_free_rdwr_pmult_pkt(spkt);
4532 return (rval);
4533 }
4534
4535 static int
4536 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4537 uint8_t regn, uint32_t *pregv)
4538 {
4539 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4540 SATA_RDWR_PMULT_PKT_TYPE_READ);
4541 }
4542
4543 static int
4544 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4545 uint8_t regn, uint32_t regv)
4546 {
4547 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v,
4548 SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4549 }
4550
4551 #define READ_PMULT(addrp, r, pv, out) \
4552 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \
4553 goto out;
4554
4555 #define WRITE_PMULT(addrp, r, v, out) \
4556 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \
4557 goto out;
4558
4559 /*
4560 * Update sata registers on port multiplier, including GSCR/PSCR registers.
4561 * ahci_update_pmult_gscr()
4562 * ahci_update_pmult_pscr()
4563 */
4564 static int
4565 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4566 sata_pmult_gscr_t *sg)
4567 {
4568 ASSERT(MUTEX_HELD(
4569 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4570
4571 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4572 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4573 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4574 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4575
4576 return (AHCI_SUCCESS);
4577
4578 err: /* R/W PMULT error */
4579 return (AHCI_FAILURE);
4580 }
4581
4582 static int
4583 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4584 sata_device_t *sd)
4585 {
4586 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4587 ASSERT(MUTEX_HELD(
4588 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4589
4590 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4591 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4592 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4593 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4594
4595 return (AHCI_SUCCESS);
4596
4597 err: /* R/W PMULT error */
4598 return (AHCI_FAILURE);
4599 }
4600
4601 /*
4602 * ahci_initialize_pmult()
4603 *
4604 * Initialize a port multiplier, including
4605 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4606 * 2. Redefine MASK register. (SATA Chap 16.?)
4607 */
4608 static int
4609 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4610 ahci_addr_t *addrp, sata_device_t *sd)
4611 {
4612 sata_pmult_gscr_t sg;
4613 uint32_t gscr64;
4614 uint8_t port = addrp->aa_port;
4615
4616 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4617
4618 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4619 "[Initialize] Port-multiplier at port %d.", port);
4620
4621 /*
4622 * Enable features of port multiplier. Currently only
4623 * Asynchronous Notification is enabled.
4624 */
4625 /* Check gscr64 for supported features. */
4626 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4627
4628 if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4629 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4630 "port %d: Port Multiplier supports "
4631 "Asynchronous Notification.", port);
4632
4633 /* Write to gscr96 to enabled features */
4634 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4635 SATA_PMULT_CAP_SNOTIF, err);
4636
4637 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4638 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4639 AHCI_SNOTIF_CLEAR_ALL);
4640 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4641 "port %d: PMult PxSNTF cleared.", port);
4642
4643 }
4644
4645 /*
4646 * Now we need to update gscr33 register to enable hot-plug interrupt
4647 * for sub devices behind port multiplier.
4648 */
4649 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4650 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4651 "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4652
4653 /*
4654 * Fetch the number of device ports of the port multiplier
4655 */
4656 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4657 return (AHCI_FAILURE);
4658
4659 /* Register the port multiplier to SATA Framework. */
4660 mutex_exit(&ahci_portp->ahciport_mutex);
4661 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4662 mutex_enter(&ahci_portp->ahciport_mutex);
4663
4664 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4665 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4666
4667 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4668 "port %d: pmult sub-port number updated to %x.", port,
4669 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4670
4671 /* Till now port-mult is successfully initialized */
4672 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4673 return (AHCI_SUCCESS);
4674
4675 err: /* R/W PMULT error */
4676 return (AHCI_FAILURE);
4677 }
4678
4679 /*
4680 * Initialize a port multiplier port. According to spec, firstly we need
4681 * issue a COMRESET, then a software reset to get its signature.
4682 *
4683 * NOTE: This function should only be called in ahci_probe_pmport()
4684 */
4685 static int
4686 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4687 ahci_addr_t *addrp)
4688 {
4689 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4690 uint8_t port = addrp->aa_port;
4691 uint8_t pmport = addrp->aa_pmport;
4692 int ret = AHCI_FAILURE;
4693
4694 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4695 ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4696
4697 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4698 "ahci_initialize_pmport: port %d:%d", port, pmport);
4699
4700 /* Check HBA port state */
4701 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4702 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4703 "ahci_initialize_pmport:"
4704 "port %d:%d Port Multiplier is failed.",
4705 port, pmport);
4706 return (AHCI_FAILURE);
4707 }
4708
4709 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4710 return (AHCI_FAILURE);
4711 }
4712 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4713
4714 /* Checking for outstanding commands */
4715 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4716 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4717 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4718 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4719 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4720 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4721 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4722 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4723 }
4724
4725 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4726 ahci_portp->ahciport_mop_in_progress++;
4727
4728 /* Clear status */
4729 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4730
4731 /* Firstly assume an unknown device */
4732 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4733
4734 ahci_disable_port_intrs(ahci_ctlp, port);
4735
4736 /* port reset is necessary for port multiplier port */
4737 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4738 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4739 "ahci_initialize_pmport:"
4740 "port reset failed at port %d:%d",
4741 port, pmport);
4742 goto out;
4743 }
4744
4745 /* Is port failed? */
4746 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4747 SATA_PSTATE_FAILED) {
4748 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4749 "ahci_initialize_pmport: port %d:%d failed. "
4750 "state = 0x%x", port, pmport,
4751 ahci_portp->ahciport_port_state);
4752 goto out;
4753 }
4754
4755 /* Is there any device attached? */
4756 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4757 == SATA_DTYPE_NONE) {
4758 /* Do not waste time on an empty port */
4759 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4760 "ahci_initialize_pmport: No device is found "
4761 "at port %d:%d", port, pmport);
4762 ret = AHCI_SUCCESS;
4763 goto out;
4764 }
4765
4766 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4767 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4768 "port %d:%d is ready now.", port, pmport);
4769
4770 /*
4771 * Till now we can assure a device attached to that HBA port and work
4772 * correctly. Now try to get the device signature. This is an optional
4773 * step. If failed, unknown device is assumed, then SATA module will
4774 * continue to use IDENTIFY DEVICE to get the information of the
4775 * device.
4776 */
4777 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4778
4779 ret = AHCI_SUCCESS;
4780
4781 out:
4782 /* Next try to mop the pending commands */
4783 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4784 finished_tags = ahci_portp->ahciport_pending_tags &
4785 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4786 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4787 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4788 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4789 reset_tags &= ~finished_tags;
4790
4791 ahci_mop_commands(ahci_ctlp,
4792 ahci_portp,
4793 slot_status,
4794 0, /* failed tags */
4795 0, /* timeout tags */
4796 0, /* aborted tags */
4797 reset_tags); /* reset tags */
4798
4799 /* Clear PxSNTF register if supported. */
4800 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4801 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4802 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4803 AHCI_SNOTIF_CLEAR_ALL);
4804 }
4805
4806 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4807 ahci_enable_port_intrs(ahci_ctlp, port);
4808 return (ret);
4809 }
4810
4811 /*
4812 * ahci_probe_pmult()
4813 *
4814 * This function will be called to probe a port multiplier, which will
4815 * handle hotplug events on port multiplier ports.
4816 *
4817 * NOTE: Only called from ahci_tran_probe_port()
4818 */
4819 static int
4820 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4821 ahci_addr_t *addrp)
4822 {
4823 sata_device_t sdevice;
4824 ahci_addr_t pmport_addr;
4825 uint32_t gscr32, port_hotplug_tags;
4826 uint32_t pmport_sstatus;
4827 int dev_exists_now = 0, dev_existed_previously = 0;
4828 uint8_t port = addrp->aa_port;
4829 int npmport;
4830
4831 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4832
4833 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4834 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4835 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4836
4837 do {
4838 npmport = ddi_ffs(port_hotplug_tags) - 1;
4839 if (npmport == -1)
4840 /* no pending hot plug events. */
4841 return (AHCI_SUCCESS);
4842
4843 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4844 "hot-plug event at port %d:%d", port, npmport);
4845
4846 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4847
4848 /* Check previous device at that port */
4849 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4850 != SATA_DTYPE_NONE)
4851 dev_existed_previously = 1;
4852
4853 /* PxSStatus tells the presence of device. */
4854 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4855 &pmport_sstatus, err);
4856
4857 if (SSTATUS_GET_DET(pmport_sstatus) ==
4858 SSTATUS_DET_DEVPRE_PHYCOM)
4859 dev_exists_now = 1;
4860
4861 /*
4862 * Clear PxSERR is critical. The transition from 0 to 1 will
4863 * emit a FIS which generates an asynchronous notification
4864 * event at controller. If we fail to clear the PxSERR, the
4865 * Async Notif events will no longer be activated on this
4866 * pmport.
4867 */
4868 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4869 AHCI_SERROR_CLEAR_ALL, err);
4870
4871 bzero((void *)&sdevice, sizeof (sata_device_t));
4872 sdevice.satadev_addr.cport = ahci_ctlp->
4873 ahcictl_port_to_cport[port];
4874 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4875 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4876 sdevice.satadev_state = SATA_PSTATE_PWRON;
4877
4878 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4879 "[Existence] %d -> %d", dev_existed_previously,
4880 dev_exists_now);
4881
4882 if (dev_exists_now) {
4883 if (dev_existed_previously) {
4884 /* Link (may) not change: Exist -> Exist * */
4885 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4886 "ahci_probe_pmult: port %d:%d "
4887 "device link lost/established",
4888 port, npmport);
4889
4890 mutex_exit(&ahci_portp->ahciport_mutex);
4891 sata_hba_event_notify(
4892 ahci_ctlp->ahcictl_sata_hba_tran->
4893 sata_tran_hba_dip,
4894 &sdevice,
4895 SATA_EVNT_LINK_LOST|
4896 SATA_EVNT_LINK_ESTABLISHED);
4897 mutex_enter(&ahci_portp->ahciport_mutex);
4898 } else {
4899 /* Link change: None -> Exist */
4900 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4901 "ahci_probe_pmult: port %d:%d "
4902 "device link established", port, npmport);
4903
4904 /* Clear port state */
4905 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4906 SATA_STATE_UNKNOWN);
4907 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4908 "ahci_probe_pmult: port %d "
4909 "ahciport_port_state [Cleared].", port);
4910
4911 mutex_exit(&ahci_portp->ahciport_mutex);
4912 sata_hba_event_notify(
4913 ahci_ctlp->ahcictl_sata_hba_tran->
4914 sata_tran_hba_dip,
4915 &sdevice,
4916 SATA_EVNT_LINK_ESTABLISHED);
4917 mutex_enter(&ahci_portp->ahciport_mutex);
4918 }
4919 } else { /* No device exists now */
4920 if (dev_existed_previously) {
4921
4922 /* Link change: Exist -> None */
4923 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4924 "ahci_probe_pmult: port %d:%d "
4925 "device link lost", port, npmport);
4926
4927 /* An existing device is lost. */
4928 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4929 SATA_STATE_UNKNOWN);
4930 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4931 SATA_DTYPE_NONE);
4932
4933 mutex_exit(&ahci_portp->ahciport_mutex);
4934 sata_hba_event_notify(
4935 ahci_ctlp->ahcictl_sata_hba_tran->
4936 sata_tran_hba_dip,
4937 &sdevice,
4938 SATA_EVNT_LINK_LOST);
4939 mutex_enter(&ahci_portp->ahciport_mutex);
4940 }
4941 }
4942
4943 CLEAR_BIT(port_hotplug_tags, npmport);
4944 } while (port_hotplug_tags != 0);
4945
4946 return (AHCI_SUCCESS);
4947
4948 err: /* R/W PMULT error */
4949 return (AHCI_FAILURE);
4950 }
4951
4952 /*
4953 * Probe and initialize a port multiplier port.
4954 * A port multiplier port could only be initilaizer here.
4955 */
4956 static int
4957 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4958 ahci_addr_t *addrp, sata_device_t *sd)
4959 {
4960 uint32_t port_state;
4961 uint8_t port = addrp->aa_port;
4962 ahci_addr_t addr_pmult;
4963
4964 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4965
4966 /*
4967 * Check the parent - port multiplier first.
4968 */
4969
4970 /*
4971 * Parent port multiplier might have been removed. This event will be
4972 * ignored and failure.
4973 */
4974 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4975 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4976 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4977 "ahci_tran_probe_port: "
4978 "parent device removed, ignore event.", NULL);
4979
4980 return (AHCI_FAILURE);
4981 }
4982
4983 /* The port is ready? */
4984 port_state = ahci_portp->ahciport_port_state;
4985 if (!(port_state & SATA_STATE_READY)) {
4986 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4987 "ahci_tran_probe_port: "
4988 "parent port-mult is NOT ready.", NULL);
4989
4990 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4991 ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4992 AHCI_SUCCESS) {
4993 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4994 "ahci_tran_probe_port: "
4995 "restart port-mult failed.", NULL);
4996 return (AHCI_FAILURE);
4997 }
4998 }
4999
5000 /*
5001 * If port-mult is restarted due to some reason, we need
5002 * re-initialized the PMult.
5003 */
5004 if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
5005 /* Initialize registers on a port multiplier */
5006 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
5007 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
5008 &addr_pmult, sd) != AHCI_SUCCESS)
5009 return (AHCI_FAILURE);
5010 }
5011
5012 /*
5013 * Then we check the port-mult port
5014 */
5015 /* Is this pmport initialized? */
5016 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5017 if (!(port_state & SATA_STATE_READY)) {
5018
5019 /* ahci_initialize_pmport() will set READY state */
5020 if (ahci_initialize_pmport(ahci_ctlp,
5021 ahci_portp, addrp) != AHCI_SUCCESS)
5022 return (AHCI_FAILURE);
5023 }
5024
5025 return (AHCI_SUCCESS);
5026 }
5027
5028 /*
5029 * AHCI device reset ...; a single device on one of the ports is reset,
5030 * but the HBA and physical communication remain intact. This is the
5031 * least intrusive.
5032 *
5033 * When issuing a software reset sequence, there should not be other
5034 * commands in the command list, so we will first clear and then re-set
5035 * PxCMD.ST to clear PxCI. And before issuing the software reset,
5036 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
5037 * cleared unless command list override (PxCMD.CLO) is supported.
5038 */
5039 static int
5040 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5041 ahci_addr_t *addrp)
5042 {
5043 ahci_fis_h2d_register_t *h2d_register_fisp;
5044 ahci_cmd_table_t *cmd_table;
5045 ahci_cmd_header_t *cmd_header;
5046 uint32_t port_cmd_status, port_cmd_issue, port_task_file;
5047 int slot, loop_count;
5048 uint8_t port = addrp->aa_port;
5049 uint8_t pmport = addrp->aa_pmport;
5050 int rval = AHCI_FAILURE;
5051
5052 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5053
5054 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5055 "port %d:%d device software resetting (FIS)", port, pmport);
5056
5057 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
5058 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5059 port) != AHCI_SUCCESS) {
5060 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5061 "ahci_software_reset: cannot stop HBA port %d.", port);
5062 goto out;
5063 }
5064
5065 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
5066 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5067 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5068
5069 if (port_task_file & AHCI_TFD_STS_BSY ||
5070 port_task_file & AHCI_TFD_STS_DRQ) {
5071 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
5072 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5073 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
5074 "cannot issue a software reset.", port_task_file);
5075 goto out;
5076 }
5077
5078 /*
5079 * If HBA Support CLO, as Command List Override (CAP.SCLO is
5080 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
5081 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
5082 */
5083 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5084 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
5085
5086 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5087 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5088 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5089 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5090 port_cmd_status|AHCI_CMD_STATUS_CLO);
5091
5092 /* Waiting till PxCMD.SCLO bit is cleared */
5093 loop_count = 0;
5094 do {
5095 /* Wait for 10 millisec */
5096 drv_usecwait(AHCI_10MS_USECS);
5097
5098 /* We are effectively timing out after 1 sec. */
5099 if (loop_count++ > 100) {
5100 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5101 "SCLO time out. port %d is busy.", port);
5102 goto out;
5103 }
5104
5105 port_cmd_status =
5106 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5107 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5108 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5109
5110 /* Re-check */
5111 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5112 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5113 if (port_task_file & AHCI_TFD_STS_BSY ||
5114 port_task_file & AHCI_TFD_STS_DRQ) {
5115 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5116 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5117 port_task_file);
5118 goto out;
5119 }
5120 }
5121
5122 /* Then start port */
5123 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5124 != AHCI_SUCCESS) {
5125 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5126 "ahci_software_reset: cannot start AHCI port %d.", port);
5127 goto out;
5128 }
5129
5130 /*
5131 * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5132 * ahci_port.ahciport_pending_ncq_tags may fail
5133 * ahci_claim_free_slot(). Actually according to spec, by clearing
5134 * PxCMD.ST there is no command outstanding while executing software
5135 * reseting. Hence we directly use slot 0 instead of
5136 * ahci_claim_free_slot().
5137 */
5138 slot = 0;
5139
5140 /* Now send the first H2D Register FIS with SRST set to 1 */
5141 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5142 bzero((void *)cmd_table, ahci_cmd_table_size);
5143
5144 h2d_register_fisp =
5145 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5146
5147 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5148 SET_FIS_PMP(h2d_register_fisp, pmport);
5149 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5150
5151 /* Set Command Header in Command List */
5152 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5153 BZERO_DESCR_INFO(cmd_header);
5154 BZERO_PRD_BYTE_COUNT(cmd_header);
5155 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5156 SET_PORT_MULTI_PORT(cmd_header, pmport);
5157
5158 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5159 SET_RESET(cmd_header, 1);
5160 SET_WRITE(cmd_header, 1);
5161
5162 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5163 0,
5164 ahci_cmd_table_size,
5165 DDI_DMA_SYNC_FORDEV);
5166
5167 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5168 slot * sizeof (ahci_cmd_header_t),
5169 sizeof (ahci_cmd_header_t),
5170 DDI_DMA_SYNC_FORDEV);
5171
5172 /* Indicate to the HBA that a command is active. */
5173 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5174 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5175 (0x1 << slot));
5176
5177 loop_count = 0;
5178
5179 /* Loop till the first command is finished */
5180 do {
5181 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5182 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5183
5184 /* We are effectively timing out after 1 sec. */
5185 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5186 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5187 "the first SRST FIS is timed out, "
5188 "loop_count = %d", loop_count);
5189 goto out;
5190 }
5191 /* Wait for 10 millisec */
5192 drv_usecwait(AHCI_10MS_USECS);
5193 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5194
5195 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5196 "ahci_software_reset: 1st loop count: %d, "
5197 "port_cmd_issue = 0x%x, slot = 0x%x",
5198 loop_count, port_cmd_issue, slot);
5199
5200 /* According to ATA spec, we need wait at least 5 microsecs here. */
5201 drv_usecwait(AHCI_1MS_USECS);
5202
5203 /* Now send the second H2D Register FIS with SRST cleard to zero */
5204 cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5205 bzero((void *)cmd_table, ahci_cmd_table_size);
5206
5207 h2d_register_fisp =
5208 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5209
5210 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5211 SET_FIS_PMP(h2d_register_fisp, pmport);
5212
5213 /* Set Command Header in Command List */
5214 cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5215 BZERO_DESCR_INFO(cmd_header);
5216 BZERO_PRD_BYTE_COUNT(cmd_header);
5217 SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5218 SET_PORT_MULTI_PORT(cmd_header, pmport);
5219
5220 SET_WRITE(cmd_header, 1);
5221
5222 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5223 0,
5224 ahci_cmd_table_size,
5225 DDI_DMA_SYNC_FORDEV);
5226
5227 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5228 slot * sizeof (ahci_cmd_header_t),
5229 sizeof (ahci_cmd_header_t),
5230 DDI_DMA_SYNC_FORDEV);
5231
5232 /* Indicate to the HBA that a command is active. */
5233 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5234 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5235 (0x1 << slot));
5236
5237 loop_count = 0;
5238
5239 /* Loop till the second command is finished */
5240 do {
5241 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5242 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5243
5244 /* We are effectively timing out after 1 sec. */
5245 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5246 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5247 "the second SRST FIS is timed out, "
5248 "loop_count = %d", loop_count);
5249 goto out;
5250 }
5251
5252 /* Wait for 10 millisec */
5253 drv_usecwait(AHCI_10MS_USECS);
5254 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5255
5256 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5257 "ahci_software_reset: 2nd loop count: %d, "
5258 "port_cmd_issue = 0x%x, slot = 0x%x",
5259 loop_count, port_cmd_issue, slot);
5260
5261 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5262 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5263 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5264 DDI_SERVICE_UNAFFECTED);
5265 goto out;
5266 }
5267
5268 rval = AHCI_SUCCESS;
5269 out:
5270 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5271 "ahci_software_reset: %s at port %d:%d",
5272 rval == AHCI_SUCCESS ? "succeed" : "failed",
5273 port, pmport);
5274
5275 return (rval);
5276 }
5277
5278 /*
5279 * AHCI port reset ...; the physical communication between the HBA and device
5280 * on a port are disabled. This is more intrusive.
5281 *
5282 * When an HBA or port reset occurs, Phy communication is going to
5283 * be re-established with the device through a COMRESET followed by the
5284 * normal out-of-band communication sequence defined in Serial ATA. At
5285 * the end of reset, the device, if working properly, will send a D2H
5286 * Register FIS, which contains the device signature. When the HBA receives
5287 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5288 * the PxSIG register with the signature.
5289 *
5290 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5291 * function is called. If not, it is assumed the interface is in hung
5292 * condition.
5293 */
5294 static int
5295 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5296 ahci_addr_t *addrp)
5297 {
5298 ahci_addr_t pmult_addr;
5299 uint32_t port_cmd_status;
5300 uint32_t port_scontrol, port_sstatus;
5301 uint32_t port_task_file;
5302 uint32_t port_state;
5303 uint8_t port = addrp->aa_port;
5304
5305 int loop_count;
5306 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5307
5308 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5309
5310 /* Target is a port multiplier port? */
5311 if (AHCI_ADDR_IS_PMPORT(addrp))
5312 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5313
5314 /* Otherwise it must be an HBA port. */
5315 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5316
5317 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5318 "Port %d port resetting...", port);
5319 ahci_portp->ahciport_port_state = 0;
5320
5321 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5322 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5323
5324 /*
5325 * According to the spec, SUD bit should be set here,
5326 * but JMicron JMB363 doesn't follow it, so print
5327 * a debug message.
5328 */
5329 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5330 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5331 "ahci_port_reset: port %d SUD bit not set", port);
5332
5333 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5334 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5335 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5336
5337 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5338 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5339 port_scontrol);
5340
5341 /* Enable PxCMD.FRE to read device */
5342 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5343 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5344 port_cmd_status|AHCI_CMD_STATUS_FRE);
5345
5346 /*
5347 * The port enters P:StartComm state, and the HBA tells the link layer
5348 * to start communication, which involves sending COMRESET to the
5349 * device. And the HBA resets PxTFD.STS to 7Fh.
5350 *
5351 * Give time for COMRESET to percolate, according to the AHCI
5352 * spec, software shall wait at least 1 millisecond before
5353 * clearing PxSCTL.DET
5354 */
5355 drv_usecwait(AHCI_1MS_USECS * 2);
5356
5357 /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5358 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5359 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5360 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5361 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5362 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5363 port_scontrol);
5364
5365 /*
5366 * When a COMINIT is received from the device, then the port enters
5367 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5368 * PxSSTS.DET to 1h to indicate a device is detected but communication
5369 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5370 * a COMINIT has been received.
5371 */
5372 /*
5373 * The DET field is valid only if IPM field indicates
5374 * that the interface is in active state.
5375 */
5376 loop_count = 0;
5377 for (;;) {
5378 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5379 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5380
5381 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5382 /*
5383 * If the interface is not active, the DET field
5384 * is considered not accurate. So we want to
5385 * continue looping.
5386 */
5387 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5388 }
5389
5390 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5391 break;
5392
5393 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5394 /*
5395 * We are effectively timing out after 0.1 sec.
5396 */
5397 break;
5398 }
5399
5400 /* Wait for 10 millisec */
5401 drv_usecwait(AHCI_10MS_USECS);
5402 }
5403
5404 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5405 "ahci_port_reset: 1st loop count: %d, "
5406 "port_sstatus = 0x%x port %d",
5407 loop_count, port_sstatus, port);
5408
5409 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5410 /*
5411 * Either the port is not active or there
5412 * is no device present.
5413 */
5414 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5415 return (AHCI_SUCCESS);
5416 }
5417
5418 /* Clear port serror register for the port */
5419 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5420 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5421 AHCI_SERROR_CLEAR_ALL);
5422
5423 /*
5424 * Devices should return a FIS contains its signature to HBA after
5425 * COMINIT signal. Check whether a D2H Register FIS is received by
5426 * polling PxTFD.STS.
5427 */
5428 loop_count = 0;
5429 for (;;) {
5430 port_task_file =
5431 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5432 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5433
5434 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5435 AHCI_TFD_STS_ERR)) == 0)
5436 break;
5437
5438 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5439 /*
5440 * We are effectively timing out after 11 sec.
5441 */
5442 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5443 "the device hardware has been initialized and "
5444 "the power-up diagnostics failed",
5445 instance, port);
5446
5447 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5448 "port %d: some or all of BSY, DRQ and ERR in "
5449 "PxTFD.STS are not clear. We need another "
5450 "software reset.", port);
5451
5452 /* Clear port serror register for the port */
5453 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5454 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5455 AHCI_SERROR_CLEAR_ALL);
5456
5457 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5458
5459 /* Try another software reset. */
5460 if (ahci_software_reset(ahci_ctlp, ahci_portp,
5461 &pmult_addr) != AHCI_SUCCESS) {
5462 AHCIPORT_SET_STATE(ahci_portp, addrp,
5463 SATA_PSTATE_FAILED);
5464 return (AHCI_FAILURE);
5465 }
5466 break;
5467 }
5468
5469 /* Wait for 10 millisec */
5470 drv_usecwait(AHCI_10MS_USECS);
5471 }
5472
5473 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5474 "ahci_port_reset: 2nd loop count: %d, "
5475 "port_task_file = 0x%x port %d",
5476 loop_count, port_task_file, port);
5477
5478 /* Clear port serror register for the port */
5479 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5480 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5481 AHCI_SERROR_CLEAR_ALL);
5482
5483 /* Set port as ready */
5484 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5485 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5486
5487 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5488 "ahci_port_reset: succeed at port %d.", port);
5489 return (AHCI_SUCCESS);
5490 }
5491
5492 /*
5493 * COMRESET on a port multiplier port.
5494 *
5495 * NOTE: Only called in ahci_port_reset()
5496 */
5497 static int
5498 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5499 ahci_addr_t *addrp)
5500 {
5501 uint32_t port_scontrol, port_sstatus, port_serror;
5502 uint32_t port_cmd_status, port_intr_status;
5503 uint32_t port_state;
5504 uint8_t port = addrp->aa_port;
5505 uint8_t pmport = addrp->aa_pmport;
5506 int loop_count;
5507 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5508
5509 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5510 "port %d:%d: pmport resetting", port, pmport);
5511
5512 /* Initialize pmport state */
5513 AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5514
5515 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5516 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5517 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5518
5519 /* PxCMD.FRE should be set before. */
5520 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5521 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5522 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5523 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5524 return (AHCI_FAILURE);
5525
5526 /*
5527 * Give time for COMRESET to percolate, according to the AHCI
5528 * spec, software shall wait at least 1 millisecond before
5529 * clearing PxSCTL.DET
5530 */
5531 drv_usecwait(AHCI_1MS_USECS*2);
5532
5533 /*
5534 * Fetch the SCONTROL again and rewrite the DET part with 0
5535 * This will generate an Asychronous Notification events.
5536 */
5537 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5538 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5539 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5540
5541 /*
5542 * The port enters P:StartComm state, and HBA tells link layer to
5543 * start communication, which involves sending COMRESET to device.
5544 * And the HBA resets PxTFD.STS to 7Fh.
5545 *
5546 * When a COMINIT is received from the device, then the port enters
5547 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5548 * PxSSTS.DET to 1h to indicate a device is detected but communication
5549 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5550 * a COMINIT has been received.
5551 */
5552 /*
5553 * The DET field is valid only if IPM field indicates
5554 * that the interface is in active state.
5555 */
5556 loop_count = 0;
5557 do {
5558 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5559
5560 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5561 /*
5562 * If the interface is not active, the DET field
5563 * is considered not accurate. So we want to
5564 * continue looping.
5565 */
5566 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5567 }
5568
5569 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5570 /*
5571 * We are effectively timing out after 0.1 sec.
5572 */
5573 break;
5574 }
5575
5576 /* Wait for 10 millisec */
5577 drv_usecwait(AHCI_10MS_USECS);
5578
5579 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5580
5581 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5582 "ahci_pmport_reset: 1st loop count: %d, "
5583 "port_sstatus = 0x%x port %d:%d",
5584 loop_count, port_sstatus, port, pmport);
5585
5586 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5587 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5588 /*
5589 * Either the port is not active or there
5590 * is no device present.
5591 */
5592 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5593 "ahci_pmport_reset: "
5594 "no device attached to port %d:%d",
5595 port, pmport);
5596 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5597 return (AHCI_SUCCESS);
5598 }
5599
5600 /* Now we can make sure there is a device connected to the port */
5601 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5602 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5603
5604 if (!(port_serror & (1 << 26))) {
5605 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5606 "COMINIT signal from the device not received port %d:%d",
5607 instance, port, pmport);
5608
5609 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5610 return (AHCI_FAILURE);
5611 }
5612
5613 /*
5614 * After clear PxSERR register, we will receive a D2H FIS.
5615 * Normally this FIS will cause a IPMS error according to AHCI spec
5616 * v1.2 because there is no command outstanding for it. So we need
5617 * to ignore this error.
5618 */
5619 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5620 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5621
5622 /* Now we need to check the D2H FIS by checking IPMS error. */
5623 loop_count = 0;
5624 do {
5625 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5626 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5627
5628 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5629 /*
5630 * No D2H FIS received. This is possible according
5631 * to SATA 2.6 spec.
5632 */
5633 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5634 "PxIS.IPMS is not set, we need another "
5635 "software reset.", port, pmport);
5636
5637 break;
5638 }
5639
5640 /* Wait for 10 millisec */
5641 mutex_exit(&ahci_portp->ahciport_mutex);
5642 delay(AHCI_10MS_TICKS);
5643 mutex_enter(&ahci_portp->ahciport_mutex);
5644
5645 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5646
5647 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5648 "ahci_pmport_reset: 2st loop count: %d, "
5649 "port_sstatus = 0x%x port %d:%d",
5650 loop_count, port_sstatus, port, pmport);
5651
5652 /* Clear IPMS */
5653 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5654 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5655 AHCI_INTR_STATUS_IPMS);
5656 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5657
5658 /* This pmport is now ready for ahci_tran_start() */
5659 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5660 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5661
5662 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5663 "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5664 return (AHCI_SUCCESS);
5665
5666 err: /* R/W PMULT error */
5667 /* IPMS flags might be set before. */
5668 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5669 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5670 "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5671
5672 return (AHCI_FAILURE);
5673 }
5674
5675 /*
5676 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5677 * This is the most intrusive.
5678 *
5679 * When an HBA reset occurs, Phy communication will be re-established with
5680 * the device through a COMRESET followed by the normal out-of-band
5681 * communication sequence defined in Serial ATA. At the end of reset, the
5682 * device, if working properly, will send a D2H Register FIS, which contains
5683 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5684 * and PxTFD.ERR register fields, and updates the PxSIG register with the
5685 * signature.
5686 *
5687 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5688 */
5689 static int
5690 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5691 {
5692 ahci_port_t *ahci_portp;
5693 uint32_t ghc_control;
5694 uint8_t port;
5695 int loop_count;
5696 int rval = AHCI_SUCCESS;
5697
5698 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5699 NULL);
5700
5701 mutex_enter(&ahci_ctlp->ahcictl_mutex);
5702
5703 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5704 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5705
5706 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5707 ghc_control |= AHCI_HBA_GHC_HR;
5708 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5709 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5710
5711 /*
5712 * Wait until HBA Reset complete or timeout
5713 */
5714 loop_count = 0;
5715 do {
5716 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5717 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5718
5719 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5720 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5721 "ahci hba reset is timing out, "
5722 "ghc_control = 0x%x", ghc_control);
5723 /* We are effectively timing out after 1 sec. */
5724 break;
5725 }
5726
5727 /* Wait for 10 millisec */
5728 drv_usecwait(AHCI_10MS_USECS);
5729 } while (ghc_control & AHCI_HBA_GHC_HR);
5730
5731 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5732 "ahci_hba_reset: 1st loop count: %d, "
5733 "ghc_control = 0x%x", loop_count, ghc_control);
5734
5735 if (ghc_control & AHCI_HBA_GHC_HR) {
5736 /* The hba is not reset for some reasons */
5737 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5738 "hba reset failed: HBA in a hung or locked state", NULL);
5739 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5740 return (AHCI_FAILURE);
5741 }
5742
5743 /*
5744 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5745 */
5746 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5747 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5748 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5749 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5750 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5751
5752 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5753
5754 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5755 /* Only check implemented ports */
5756 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5757 continue;
5758 }
5759
5760 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5761 mutex_enter(&ahci_portp->ahciport_mutex);
5762
5763 /* Make sure the drive is spun-up */
5764 ahci_staggered_spin_up(ahci_ctlp, port);
5765
5766 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5767 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5768 AHCI_SUCCESS) {
5769 rval = AHCI_FAILURE;
5770 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5771 "ahci_hba_reset: port %d failed", port);
5772 /*
5773 * Set the port state to SATA_PSTATE_FAILED if
5774 * failed to initialize it.
5775 */
5776 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5777 }
5778
5779 mutex_exit(&ahci_portp->ahciport_mutex);
5780 }
5781
5782 return (rval);
5783 }
5784
5785 /*
5786 * This routine is only called from AHCI_ATTACH or phyrdy change
5787 * case. It first calls software reset, then stop the port and try to
5788 * read PxSIG register to find the type of device attached to the port.
5789 *
5790 * The caller should make sure a valid device exists on specified port and
5791 * physical communication has been established so that the signature could
5792 * be retrieved by software reset.
5793 *
5794 * NOTE: The port interrupts should be disabled before the function is called.
5795 */
5796 static void
5797 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5798 ahci_addr_t *addrp)
5799 {
5800 ahci_addr_t dev_addr;
5801 uint32_t signature;
5802 uint8_t port = addrp->aa_port;
5803 uint8_t pmport = addrp->aa_pmport;
5804 int rval;
5805
5806 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5807 ASSERT(AHCI_ADDR_IS_VALID(addrp));
5808
5809 /*
5810 * If the HBA doesn't support port multiplier, then the driver
5811 * doesn't need to bother to check port multiplier device.
5812 *
5813 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5814 * connected to Silicon Image 4723, to which the two sata drives
5815 * attached can be set with RAID1, RAID0 or Spanning mode.
5816 *
5817 * We found software reset will get failure if port multiplier address
5818 * 0xf is used by software reset, so just ignore the check since
5819 * ICH7 doesn't support port multiplier device at all.
5820 */
5821 if (AHCI_ADDR_IS_PORT(addrp) &&
5822 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5823 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5824 "ahci_find_dev_signature enter: port %d", port);
5825
5826 /*
5827 * NOTE: when the ahci address is a HBA port, we do not know
5828 * it is a device or a port multiplier that attached. we need
5829 * try a software reset at port multiplier address (0xf
5830 * pmport)
5831 */
5832 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5833 } else {
5834 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5835 "ahci_find_dev_signature enter: port %d:%d",
5836 port, pmport);
5837 dev_addr = *addrp;
5838 }
5839
5840 /* Assume it is unknown. */
5841 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5842
5843 /* Issue a software reset to get the signature */
5844 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5845 if (rval != AHCI_SUCCESS) {
5846
5847 /*
5848 * Try to do software reset again with pmport set with 0 if
5849 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5850 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5851 */
5852 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5853 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5854 dev_addr.aa_pmport = 0;
5855 rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5856 &dev_addr);
5857 }
5858
5859 if (rval != AHCI_SUCCESS) {
5860 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5861 "ahci_find_dev_signature: software reset failed "
5862 "at port %d:%d, cannot get signature.",
5863 port, pmport);
5864
5865 AHCIPORT_SET_STATE(ahci_portp, addrp,
5866 SATA_PSTATE_FAILED);
5867 return;
5868 }
5869 }
5870
5871 /*
5872 * ahci_software_reset has started the port, so we need manually stop
5873 * the port again.
5874 */
5875 if (AHCI_ADDR_IS_PORT(addrp)) {
5876 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5877 ahci_portp, port) != AHCI_SUCCESS) {
5878 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5879 "ahci_find_dev_signature: cannot stop port %d.",
5880 port);
5881 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5882 return;
5883 }
5884 }
5885
5886 /* Now we can make sure that a valid signature is received. */
5887 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5888 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5889
5890 if (AHCI_ADDR_IS_PMPORT(addrp)) {
5891 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5892 "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5893 signature, port, pmport);
5894 } else {
5895 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5896 "ahci_find_dev_signature: signature = 0x%x at port %d",
5897 signature, port);
5898 }
5899
5900 /* NOTE: Only support ATAPI device at controller port. */
5901 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5902 signature = SATA_DTYPE_UNKNOWN;
5903
5904 switch (signature) {
5905
5906 case AHCI_SIGNATURE_DISK:
5907 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5908 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5909 "Disk is found at port: %d", port);
5910 break;
5911
5912 case AHCI_SIGNATURE_ATAPI:
5913 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5914 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5915 "ATAPI device is found at port: %d", port);
5916 break;
5917
5918 case AHCI_SIGNATURE_PORT_MULTIPLIER:
5919 /* Port Multiplier cannot recursively attached. */
5920 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5921 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5922 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5923 "Port Multiplier is found at port: %d", port);
5924 break;
5925
5926 default:
5927 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5928 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5929 "Unknown device is found at port: %d", port);
5930 }
5931 }
5932
5933 /*
5934 * According to the spec, to reliably detect hot plug removals, software
5935 * must disable interface power management. Software should perform the
5936 * following initialization on a port after a device is attached:
5937 * Set PxSCTL.IPM to 3h to disable interface state transitions
5938 * Set PxCMD.ALPE to '0' to disable aggressive power management
5939 * Disable device initiated interface power management by SET FEATURE
5940 *
5941 * We can ignore the last item because by default the feature is disabled
5942 */
5943 static void
5944 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5945 {
5946 uint32_t port_scontrol, port_cmd_status;
5947
5948 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5949 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5950 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5951 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5952 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5953
5954 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5955 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5956 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5957 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5958 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5959 }
5960
5961 /*
5962 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5963 * to 1, then set it firstly.
5964 *
5965 * Each port contains two major DMA engines. One DMA engine walks through
5966 * the command list, and is controlled by PxCMD.ST. The second DMA engine
5967 * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5968 *
5969 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5970 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5971 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5972 *
5973 * Software shall not set PxCMD.ST to '1' unless a functional device is
5974 * present on the port(as determined by PxTFD.STS.BSY = '0',
5975 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5976 */
5977 static int
5978 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5979 {
5980 uint32_t port_cmd_status;
5981
5982 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5983
5984 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5985
5986 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5987 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5988 "the state for port %d is 0x%x",
5989 port, ahci_portp->ahciport_port_state);
5990 return (AHCI_FAILURE);
5991 }
5992
5993 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5994 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5995 "no device is attached at port %d", port);
5996 return (AHCI_FAILURE);
5997 }
5998
5999 /* First to set PxCMD.FRE before setting PxCMD.ST. */
6000 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6001 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6002
6003 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
6004 port_cmd_status |= AHCI_CMD_STATUS_FRE;
6005 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6006 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6007 port_cmd_status);
6008 }
6009
6010 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6011 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6012
6013 port_cmd_status |= AHCI_CMD_STATUS_ST;
6014
6015 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6016 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6017 port_cmd_status);
6018
6019 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
6020
6021 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
6022 "PxCMD.ST set to '1' at port %d", port);
6023
6024 return (AHCI_SUCCESS);
6025 }
6026
6027 /*
6028 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
6029 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
6030 * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
6031 */
6032 static int
6033 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6034 {
6035 uint8_t port = ahci_portp->ahciport_port_num;
6036 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6037 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6038
6039 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6040
6041 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
6042 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
6043 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
6044 port) != AHCI_SUCCESS)
6045 return (AHCI_FAILURE);
6046
6047 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6048 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6049 }
6050
6051 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
6052 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
6053 int loop_count = 0;
6054
6055 /* Clear PxCMD.FRE */
6056 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
6057 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6058 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6059 port_cmd_status);
6060
6061 /* Wait until PxCMD.FR is cleared */
6062 for (;;) {
6063 port_cmd_status =
6064 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6065 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6066
6067 if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
6068 break;
6069
6070 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
6071 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
6072 "ahci_setup_port_base_addresses: cannot "
6073 "clear PxCMD.FR for port %d.", port);
6074
6075 /*
6076 * We are effectively timing out after 0.5 sec.
6077 * This value is specified in AHCI spec.
6078 */
6079 return (AHCI_FAILURE);
6080 }
6081
6082 /* Wait for 1 millisec */
6083 drv_usecwait(AHCI_1MS_USECS);
6084 }
6085 }
6086
6087 /* Step 3: Config Port Command List Base Address */
6088 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6089 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
6090 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6091
6092 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6093 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
6094 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
6095
6096 /* Step 4: Config Port Received FIS Base Address */
6097 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6098 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
6099 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6100
6101 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6102 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6103 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6104
6105 return (AHCI_SUCCESS);
6106 }
6107
6108 /*
6109 * Allocate the ahci_port_t including Received FIS and Command List.
6110 * The argument - port is the physical port number, and not logical
6111 * port number seen by the SATA framework.
6112 */
6113 static int
6114 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6115 {
6116 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6117 ahci_port_t *ahci_portp;
6118 char taskq_name[64] = "event_handle_taskq";
6119
6120 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6121
6122 ahci_portp =
6123 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6124
6125 ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6126 ahci_portp->ahciport_port_num = port;
6127
6128 /* Initialize the port condition variable */
6129 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6130
6131 /* Initialize the port mutex */
6132 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6133 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6134
6135 mutex_enter(&ahci_portp->ahciport_mutex);
6136
6137 /*
6138 * Allocate memory for received FIS structure and
6139 * command list for this port
6140 */
6141 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6142 goto err_case1;
6143 }
6144
6145 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6146 goto err_case2;
6147 }
6148
6149 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6150 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6151 AHCI_SUCCESS) {
6152 goto err_case3;
6153 }
6154
6155 (void) snprintf(taskq_name + strlen(taskq_name),
6156 sizeof (taskq_name) - strlen(taskq_name),
6157 "_port%d", port);
6158
6159 /* Create the taskq for the port */
6160 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6161 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6162 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6163 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6164 goto err_case3;
6165 }
6166
6167 /* Allocate the argument for the taskq */
6168 ahci_portp->ahciport_event_args =
6169 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6170
6171 ahci_portp->ahciport_event_args->ahciea_addrp =
6172 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6173
6174 /* Initialize the done queue */
6175 ahci_portp->ahciport_doneq = NULL;
6176 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6177 ahci_portp->ahciport_doneq_len = 0;
6178
6179 mutex_exit(&ahci_portp->ahciport_mutex);
6180
6181 return (AHCI_SUCCESS);
6182
6183 err_case3:
6184 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6185
6186 err_case2:
6187 ahci_dealloc_rcvd_fis(ahci_portp);
6188
6189 err_case1:
6190 mutex_exit(&ahci_portp->ahciport_mutex);
6191 mutex_destroy(&ahci_portp->ahciport_mutex);
6192 cv_destroy(&ahci_portp->ahciport_cv);
6193
6194 kmem_free(ahci_portp, sizeof (ahci_port_t));
6195
6196 return (AHCI_FAILURE);
6197 }
6198
6199 /*
6200 * Reverse of ahci_alloc_port_state().
6201 */
6202 static void
6203 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6204 {
6205 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6206
6207 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6208 ASSERT(ahci_portp != NULL);
6209
6210 mutex_enter(&ahci_portp->ahciport_mutex);
6211 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6212 sizeof (ahci_addr_t));
6213 ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6214 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6215 ahci_portp->ahciport_event_args = NULL;
6216 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6217 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6218 ahci_dealloc_rcvd_fis(ahci_portp);
6219 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6220 mutex_exit(&ahci_portp->ahciport_mutex);
6221
6222 mutex_destroy(&ahci_portp->ahciport_mutex);
6223 cv_destroy(&ahci_portp->ahciport_cv);
6224
6225 kmem_free(ahci_portp, sizeof (ahci_port_t));
6226
6227 ahci_ctlp->ahcictl_ports[port] = NULL;
6228 }
6229
6230 /*
6231 * Allocates memory for the Received FIS Structure
6232 */
6233 static int
6234 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6235 {
6236 size_t rcvd_fis_size;
6237 size_t ret_len;
6238 uint_t cookie_count;
6239
6240 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6241
6242 rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6243
6244 /* allocate rcvd FIS dma handle. */
6245 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6246 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6247 DDI_DMA_SLEEP,
6248 NULL,
6249 &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6250 DDI_SUCCESS) {
6251 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6252 "rcvd FIS dma handle alloc failed", NULL);
6253
6254 return (AHCI_FAILURE);
6255 }
6256
6257 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6258 rcvd_fis_size,
6259 &accattr,
6260 DDI_DMA_CONSISTENT,
6261 DDI_DMA_SLEEP,
6262 NULL,
6263 (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6264 &ret_len,
6265 &ahci_portp->ahciport_rcvd_fis_acc_handle) != 0) {
6266
6267 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6268 "rcvd FIS dma mem alloc fail", NULL);
6269 /* error.. free the dma handle. */
6270 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6271 return (AHCI_FAILURE);
6272 }
6273
6274 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6275 NULL,
6276 (caddr_t)ahci_portp->ahciport_rcvd_fis,
6277 rcvd_fis_size,
6278 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6279 DDI_DMA_SLEEP,
6280 NULL,
6281 &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6282 &cookie_count) != DDI_DMA_MAPPED) {
6283
6284 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6285 "rcvd FIS dma handle bind fail", NULL);
6286 /* error.. free the dma handle & free the memory. */
6287 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6288 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6289 return (AHCI_FAILURE);
6290 }
6291
6292 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6293
6294 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6295 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6296 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6297 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6298
6299 return (AHCI_SUCCESS);
6300 }
6301
6302 /*
6303 * Deallocates the Received FIS Structure
6304 */
6305 static void
6306 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6307 {
6308 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6309
6310 /* Unbind the cmd list dma handle first. */
6311 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6312
6313 /* Then free the underlying memory. */
6314 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6315
6316 /* Now free the handle itself. */
6317 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6318 }
6319
6320 /*
6321 * Allocates memory for the Command List, which contains up to 32 entries.
6322 * Each entry contains a command header, which is a 32-byte structure that
6323 * includes the pointer to the command table.
6324 */
6325 static int
6326 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6327 {
6328 size_t cmd_list_size;
6329 size_t ret_len;
6330 uint_t cookie_count;
6331
6332 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6333
6334 cmd_list_size =
6335 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6336
6337 /* allocate cmd list dma handle. */
6338 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6339 &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6340 DDI_DMA_SLEEP,
6341 NULL,
6342 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6343
6344 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6345 "cmd list dma handle alloc failed", NULL);
6346 return (AHCI_FAILURE);
6347 }
6348
6349 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6350 cmd_list_size,
6351 &accattr,
6352 DDI_DMA_CONSISTENT,
6353 DDI_DMA_SLEEP,
6354 NULL,
6355 (caddr_t *)&ahci_portp->ahciport_cmd_list,
6356 &ret_len,
6357 &ahci_portp->ahciport_cmd_list_acc_handle) != 0) {
6358
6359 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6360 "cmd list dma mem alloc fail", NULL);
6361 /* error.. free the dma handle. */
6362 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6363 return (AHCI_FAILURE);
6364 }
6365
6366 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6367 NULL,
6368 (caddr_t)ahci_portp->ahciport_cmd_list,
6369 cmd_list_size,
6370 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6371 DDI_DMA_SLEEP,
6372 NULL,
6373 &ahci_portp->ahciport_cmd_list_dma_cookie,
6374 &cookie_count) != DDI_DMA_MAPPED) {
6375
6376 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6377 "cmd list dma handle bind fail", NULL);
6378 /* error.. free the dma handle & free the memory. */
6379 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6380 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6381 return (AHCI_FAILURE);
6382 }
6383
6384 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6385
6386 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6387 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6388
6389 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6390 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6391
6392 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6393 goto err_out;
6394 }
6395
6396 return (AHCI_SUCCESS);
6397
6398 err_out:
6399 /* Unbind the cmd list dma handle first. */
6400 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6401
6402 /* Then free the underlying memory. */
6403 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6404
6405 /* Now free the handle itself. */
6406 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6407
6408 return (AHCI_FAILURE);
6409 }
6410
6411 /*
6412 * Deallocates the Command List
6413 */
6414 static void
6415 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6416 {
6417 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6418
6419 /* First dealloc command table */
6420 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6421
6422 /* Unbind the cmd list dma handle first. */
6423 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6424
6425 /* Then free the underlying memory. */
6426 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6427
6428 /* Now free the handle itself. */
6429 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6430 }
6431
6432 /*
6433 * Allocates memory for all Command Tables, which contains Command FIS,
6434 * ATAPI Command and Physical Region Descriptor Table.
6435 */
6436 static int
6437 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6438 {
6439 size_t ret_len;
6440 ddi_dma_cookie_t cmd_table_dma_cookie;
6441 uint_t cookie_count;
6442 int slot;
6443
6444 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6445
6446 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6447 "ahci_alloc_cmd_tables: port %d enter",
6448 ahci_portp->ahciport_port_num);
6449
6450 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6451 /* Allocate cmd table dma handle. */
6452 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6453 &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6454 DDI_DMA_SLEEP,
6455 NULL,
6456 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6457 DDI_SUCCESS) {
6458
6459 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6460 "cmd table dma handle alloc failed", NULL);
6461
6462 goto err_out;
6463 }
6464
6465 if (ddi_dma_mem_alloc(
6466 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6467 ahci_cmd_table_size,
6468 &accattr,
6469 DDI_DMA_CONSISTENT,
6470 DDI_DMA_SLEEP,
6471 NULL,
6472 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6473 &ret_len,
6474 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 0) {
6475
6476 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6477 "cmd table dma mem alloc fail", NULL);
6478
6479 /* error.. free the dma handle. */
6480 ddi_dma_free_handle(
6481 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6482 goto err_out;
6483 }
6484
6485 if (ddi_dma_addr_bind_handle(
6486 ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6487 NULL,
6488 (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6489 ahci_cmd_table_size,
6490 DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6491 DDI_DMA_SLEEP,
6492 NULL,
6493 &cmd_table_dma_cookie,
6494 &cookie_count) != DDI_DMA_MAPPED) {
6495
6496 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6497 "cmd table dma handle bind fail", NULL);
6498 /* error.. free the dma handle & free the memory. */
6499 ddi_dma_mem_free(
6500 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6501 ddi_dma_free_handle(
6502 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6503 goto err_out;
6504 }
6505
6506 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6507 ahci_cmd_table_size);
6508
6509 /* Config Port Command Table Base Address */
6510 SET_COMMAND_TABLE_BASE_ADDR(
6511 (&ahci_portp->ahciport_cmd_list[slot]),
6512 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6513
6514 #ifndef __lock_lint
6515 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6516 (&ahci_portp->ahciport_cmd_list[slot]),
6517 cmd_table_dma_cookie.dmac_laddress >> 32);
6518 #endif
6519 }
6520
6521 return (AHCI_SUCCESS);
6522 err_out:
6523
6524 for (slot--; slot >= 0; slot--) {
6525 /* Unbind the cmd table dma handle first */
6526 (void) ddi_dma_unbind_handle(
6527 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6528
6529 /* Then free the underlying memory */
6530 ddi_dma_mem_free(
6531 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6532
6533 /* Now free the handle itself */
6534 ddi_dma_free_handle(
6535 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6536 }
6537
6538 return (AHCI_FAILURE);
6539 }
6540
6541 /*
6542 * Deallocates memory for all Command Tables.
6543 */
6544 static void
6545 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6546 {
6547 int slot;
6548
6549 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6550
6551 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6552 "ahci_dealloc_cmd_tables: %d enter",
6553 ahci_portp->ahciport_port_num);
6554
6555 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6556 /* Unbind the cmd table dma handle first. */
6557 (void) ddi_dma_unbind_handle(
6558 ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6559
6560 /* Then free the underlying memory. */
6561 ddi_dma_mem_free(
6562 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6563
6564 /* Now free the handle itself. */
6565 ddi_dma_free_handle(
6566 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6567 }
6568 }
6569
6570 /*
6571 * Update SATA registers at controller ports
6572 */
6573 static void
6574 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6575 sata_device_t *sd)
6576 {
6577 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6578
6579 sd->satadev_scr.sstatus =
6580 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6581 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6582 sd->satadev_scr.serror =
6583 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6584 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6585 sd->satadev_scr.scontrol =
6586 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6587 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6588 sd->satadev_scr.sactive =
6589 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6590 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6591 }
6592
6593 /*
6594 * For poll mode, ahci_port_intr will be called to emulate the interrupt
6595 */
6596 static void
6597 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6598 {
6599 uint32_t port_intr_status;
6600 uint32_t port_intr_enable;
6601
6602 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6603 "ahci_port_intr enter: port %d", port);
6604
6605 mutex_enter(&ahci_portp->ahciport_mutex);
6606 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6607 /* For SATA_OPMODE_POLLING commands */
6608 port_intr_enable =
6609 (AHCI_INTR_STATUS_DHRS |
6610 AHCI_INTR_STATUS_PSS |
6611 AHCI_INTR_STATUS_SDBS |
6612 AHCI_INTR_STATUS_UFS |
6613 AHCI_INTR_STATUS_PCS |
6614 AHCI_INTR_STATUS_PRCS |
6615 AHCI_INTR_STATUS_OFS |
6616 AHCI_INTR_STATUS_INFS |
6617 AHCI_INTR_STATUS_IFS |
6618 AHCI_INTR_STATUS_HBDS |
6619 AHCI_INTR_STATUS_HBFS |
6620 AHCI_INTR_STATUS_TFES);
6621 } else {
6622 /*
6623 * port_intr_enable indicates that the corresponding interrrupt
6624 * reporting is enabled.
6625 */
6626 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6627 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6628 }
6629
6630 /* IPMS error in port reset should be ignored according AHCI spec. */
6631 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6632 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6633 mutex_exit(&ahci_portp->ahciport_mutex);
6634
6635 /*
6636 * port_intr_stats indicates that the corresponding interrupt
6637 * condition is active.
6638 */
6639 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6640 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6641
6642 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6643 "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6644 "port_intr_enable = 0x%x",
6645 port, port_intr_status, port_intr_enable);
6646
6647 port_intr_status &= port_intr_enable;
6648
6649 /*
6650 * Pending interrupt events are indicated by the PxIS register.
6651 * Make sure we don't miss any event.
6652 */
6653 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6654 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6655 DDI_SERVICE_UNAFFECTED);
6656 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6657 DDI_FME_VERSION);
6658 return;
6659 }
6660
6661 /* First clear the port interrupts status */
6662 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6663 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6664 port_intr_status);
6665
6666 /* Check the completed non-queued commands */
6667 if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6668 AHCI_INTR_STATUS_PSS)) {
6669 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6670 ahci_portp, port);
6671 }
6672
6673 /* Check the completed queued commands */
6674 if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6675 (void) ahci_intr_set_device_bits(ahci_ctlp,
6676 ahci_portp, port);
6677 }
6678
6679 /* Check the port connect change status interrupt bit */
6680 if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6681 (void) ahci_intr_port_connect_change(ahci_ctlp,
6682 ahci_portp, port);
6683 }
6684
6685 /* Check the device mechanical presence status interrupt bit */
6686 if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6687 (void) ahci_intr_device_mechanical_presence_status(
6688 ahci_ctlp, ahci_portp, port);
6689 }
6690
6691 /* Check the PhyRdy change status interrupt bit */
6692 if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6693 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6694 port);
6695 }
6696
6697 /*
6698 * Check the non-fatal error interrupt bits, there are four
6699 * kinds of non-fatal errors at the time being:
6700 *
6701 * PxIS.UFS - Unknown FIS Error
6702 * PxIS.OFS - Overflow Error
6703 * PxIS.INFS - Interface Non-Fatal Error
6704 * PxIS.IPMS - Incorrect Port Multiplier Status Error
6705 *
6706 * For these non-fatal errors, the HBA can continue to operate,
6707 * so the driver just log the error messages.
6708 */
6709 if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6710 AHCI_INTR_STATUS_OFS |
6711 AHCI_INTR_STATUS_IPMS |
6712 AHCI_INTR_STATUS_INFS)) {
6713 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6714 port, port_intr_status);
6715 }
6716
6717 /*
6718 * Check the fatal error interrupt bits, there are four kinds
6719 * of fatal errors for AHCI controllers:
6720 *
6721 * PxIS.HBFS - Host Bus Fatal Error
6722 * PxIS.HBDS - Host Bus Data Error
6723 * PxIS.IFS - Interface Fatal Error
6724 * PxIS.TFES - Task File Error
6725 *
6726 * The fatal error means the HBA can not recover from it by
6727 * itself, and it will try to abort the transfer, and the software
6728 * must intervene to restart the port.
6729 */
6730 if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6731 AHCI_INTR_STATUS_HBDS |
6732 AHCI_INTR_STATUS_HBFS |
6733 AHCI_INTR_STATUS_TFES))
6734 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6735 port, port_intr_status);
6736
6737 /* Check the cold port detect interrupt bit */
6738 if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6739 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6740 }
6741
6742 /* Second clear the corresponding bit in IS.IPS */
6743 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6744 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6745
6746 /* Try to recover at the end of the interrupt handler. */
6747 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6748 DDI_FM_OK) {
6749 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6750 DDI_SERVICE_UNAFFECTED);
6751 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6752 DDI_FME_VERSION);
6753 }
6754 }
6755
6756 /*
6757 * Interrupt service handler
6758 */
6759 static uint_t
6760 ahci_intr(caddr_t arg1, caddr_t arg2)
6761 {
6762 #ifndef __lock_lint
6763 _NOTE(ARGUNUSED(arg2))
6764 #endif
6765 /* LINTED */
6766 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6767 ahci_port_t *ahci_portp;
6768 int32_t global_intr_status;
6769 uint8_t port;
6770
6771 /*
6772 * global_intr_status indicates that the corresponding port has
6773 * an interrupt pending.
6774 */
6775 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6776 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6777
6778 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6779 /* The interrupt is not ours */
6780 return (DDI_INTR_UNCLAIMED);
6781 }
6782
6783 /*
6784 * Check the handle after reading global_intr_status - we don't want
6785 * to miss any port with pending interrupts.
6786 */
6787 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6788 DDI_FM_OK) {
6789 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6790 DDI_SERVICE_UNAFFECTED);
6791 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6792 DDI_FME_VERSION);
6793 return (DDI_INTR_UNCLAIMED);
6794 }
6795
6796 /* Loop for all the ports */
6797 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6798 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6799 continue;
6800 }
6801 if (!((0x1 << port) & global_intr_status)) {
6802 continue;
6803 }
6804
6805 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6806
6807 /* Call ahci_port_intr */
6808 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6809 }
6810
6811 return (DDI_INTR_CLAIMED);
6812 }
6813
6814 /*
6815 * For non-queued commands, when the corresponding bit in the PxCI register
6816 * is cleared, it means the command is completed successfully. And according
6817 * to the HBA state machine, there are three conditions which possibly will
6818 * try to clear the PxCI register bit.
6819 * 1. Receive one D2H Register FIS which is with 'I' bit set
6820 * 2. Update PIO Setup FIS
6821 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6822 *
6823 * Process completed non-queued commands when the interrupt status bit -
6824 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6825 *
6826 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6827 * with the 'I' bit set. And the following commands will send thus
6828 * FIS with 'I' bit set upon the successful completion:
6829 * 1. Non-data commands
6830 * 2. DMA data-in command
6831 * 3. DMA data-out command
6832 * 4. PIO data-out command
6833 * 5. PACKET non-data commands
6834 * 6. PACKET PIO data-in command
6835 * 7. PACKET PIO data-out command
6836 * 8. PACKET DMA data-in command
6837 * 9. PACKET DMA data-out command
6838 *
6839 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6840 * with the 'I' bit set. And the following commands will send this
6841 * FIS upon the successful completion:
6842 * 1. PIO data-in command
6843 */
6844 static int
6845 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6846 ahci_port_t *ahci_portp, uint8_t port)
6847 {
6848 uint32_t port_cmd_issue = 0;
6849 uint32_t finished_tags;
6850 int finished_slot;
6851 sata_pkt_t *satapkt;
6852 ahci_fis_d2h_register_t *rcvd_fisp;
6853 #if AHCI_DEBUG
6854 ahci_cmd_header_t *cmd_header;
6855 uint32_t cmd_dmacount;
6856 #endif
6857
6858 mutex_enter(&ahci_portp->ahciport_mutex);
6859
6860 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6861 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6862 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6863 /*
6864 * Spurious interrupt. Nothing to be done.
6865 */
6866 mutex_exit(&ahci_portp->ahciport_mutex);
6867 return (AHCI_SUCCESS);
6868 }
6869
6870 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6871 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6872
6873 /* If the PxCI corrupts, don't complete the commmands. */
6874 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6875 != DDI_FM_OK) {
6876 mutex_exit(&ahci_portp->ahciport_mutex);
6877 return (AHCI_FAILURE);
6878 }
6879
6880 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6881 /* Slot 0 is always used during error recovery */
6882 finished_tags = 0x1 & ~port_cmd_issue;
6883 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6884 "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6885 "retrieval is finished, and finished_tags = 0x%x",
6886 port, finished_tags);
6887 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6888 finished_tags = 0x1 & ~port_cmd_issue;
6889 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6890 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6891 "port multiplier is finished, and finished_tags = 0x%x",
6892 port, finished_tags);
6893
6894 } else {
6895
6896 finished_tags = ahci_portp->ahciport_pending_tags &
6897 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6898 }
6899
6900 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6901 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6902 "port_cmd_issue = 0x%x finished_tags = 0x%x",
6903 ahci_portp->ahciport_pending_tags, port_cmd_issue,
6904 finished_tags);
6905
6906 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6907 (finished_tags == 0x1)) {
6908 satapkt = ahci_portp->ahciport_err_retri_pkt;
6909 ASSERT(satapkt != NULL);
6910
6911 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6912 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6913 "with SATA_PKT_COMPLETED", (void *)satapkt);
6914
6915 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6916 goto out;
6917 }
6918
6919 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6920 (finished_tags == 0x1)) {
6921 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6922 ASSERT(satapkt != NULL);
6923
6924 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6925 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6926 "with SATA_PKT_COMPLETED", (void *)satapkt);
6927
6928 /* READ PORTMULT need copy out FIS content. */
6929 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6930 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6931 ahcirf_d2h_register_fis);
6932 satapkt->satapkt_cmd.satacmd_status_reg =
6933 GET_RFIS_STATUS(rcvd_fisp);
6934 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6935 }
6936
6937 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6938 goto out;
6939 }
6940
6941 while (finished_tags) {
6942 finished_slot = ddi_ffs(finished_tags) - 1;
6943 if (finished_slot == -1) {
6944 goto out;
6945 }
6946
6947 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6948 ASSERT(satapkt != NULL);
6949 #if AHCI_DEBUG
6950 /*
6951 * For non-native queued commands, the PRD byte count field
6952 * shall contain an accurate count of the number of bytes
6953 * transferred for the command before the PxCI bit is cleared
6954 * to '0' for the command.
6955 *
6956 * The purpose of this field is to let software know how many
6957 * bytes transferred for a given operation in order to
6958 * determine if underflow occurred. When issuing native command
6959 * queuing commands, this field should not be used and is not
6960 * required to be valid since in this case underflow is always
6961 * illegal.
6962 *
6963 * For data reads, the HBA will update its PRD byte count with
6964 * the total number of bytes received from the last FIS, and
6965 * may be able to continue normally. For data writes, the
6966 * device will detect an error, and HBA most likely will get
6967 * a fatal error.
6968 *
6969 * Therefore, here just put code to debug part. And please
6970 * refer to the comment above ahci_intr_fatal_error for the
6971 * definition of underflow error.
6972 */
6973 cmd_dmacount =
6974 ahci_portp->ahciport_prd_bytecounts[finished_slot];
6975 if (cmd_dmacount) {
6976 cmd_header =
6977 &ahci_portp->ahciport_cmd_list[finished_slot];
6978 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6979 "ahci_intr_cmd_cmplt: port %d, "
6980 "PRD Byte Count = 0x%x, "
6981 "ahciport_prd_bytecounts = 0x%x", port,
6982 cmd_header->ahcich_prd_byte_count,
6983 cmd_dmacount);
6984
6985 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6986 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6987 "ahci_intr_cmd_cmplt: port %d, "
6988 "an underflow occurred", port);
6989 }
6990 }
6991 #endif
6992
6993 /*
6994 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6995 * feature, sata_special_regs flag will be set, and the
6996 * driver should copy the status and the other corresponding
6997 * register values in the D2H Register FIS received (It's
6998 * working on Non-data protocol) from the device back to
6999 * the sata_cmd.
7000 *
7001 * For every AHCI port, there is only one Received FIS
7002 * structure, which contains the FISes received from the
7003 * device, So we're trying to copy the content of D2H
7004 * Register FIS in the Received FIS structure back to
7005 * the sata_cmd.
7006 */
7007 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
7008 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
7009 ahcirf_d2h_register_fis);
7010 satapkt->satapkt_cmd.satacmd_status_reg =
7011 GET_RFIS_STATUS(rcvd_fisp);
7012 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
7013 }
7014
7015 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7016 "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
7017 "with SATA_PKT_COMPLETED", (void *)satapkt);
7018
7019 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
7020 CLEAR_BIT(finished_tags, finished_slot);
7021 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7022
7023 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7024 }
7025 out:
7026 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
7027 "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
7028 ahci_portp->ahciport_pending_tags);
7029
7030 ahci_flush_doneq(ahci_portp);
7031
7032 mutex_exit(&ahci_portp->ahciport_mutex);
7033
7034 return (AHCI_SUCCESS);
7035 }
7036
7037 /*
7038 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
7039 * with the 'I' bit set and has been copied into system memory. It will
7040 * be sent under the following situations:
7041 *
7042 * 1. NCQ command is completed
7043 *
7044 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
7045 * via the Set Device Bits FIS. When such event is generated, the software
7046 * needs to read PxSACT register and compares the current value to the
7047 * list of commands previously issue by software. ahciport_pending_ncq_tags
7048 * keeps the tags of previously issued commands.
7049 *
7050 * 2. Asynchronous Notification
7051 *
7052 * Asynchronous Notification is a feature in SATA spec 2.6.
7053 *
7054 * 1) ATAPI device will send a signal to the host when media is inserted or
7055 * removed and avoids polling the device for media changes. The signal
7056 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
7057 * set to '1'. At the moment, it's not supported yet.
7058 *
7059 * 2) Port multiplier will send a signal to the host when a hot plug event
7060 * has occured on a port multiplier port. It is used when command based
7061 * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
7062 */
7063 static int
7064 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
7065 ahci_port_t *ahci_portp, uint8_t port)
7066 {
7067 ahci_addr_t addr;
7068
7069 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7070 "ahci_intr_set_device_bits enter: port %d", port);
7071
7072 /* Initialize HBA port address */
7073 AHCI_ADDR_SET_PORT(&addr, port);
7074
7075 /* NCQ plug handler */
7076 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
7077
7078 /* Check port multiplier's asynchronous notification events */
7079 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
7080 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
7081 ahci_portp, port);
7082 }
7083
7084 /* ATAPI events is not supported yet */
7085
7086 return (AHCI_SUCCESS);
7087 }
7088 /*
7089 * NCQ interrupt handler. Called upon a NCQ command is completed.
7090 * Only be called from ahci_intr_set_device_bits().
7091 */
7092 static int
7093 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7094 ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7095 {
7096 uint32_t port_sactive;
7097 uint32_t port_cmd_issue;
7098 uint32_t issued_tags;
7099 int issued_slot;
7100 uint32_t finished_tags;
7101 int finished_slot;
7102 uint8_t port = addrp->aa_port;
7103 sata_pkt_t *satapkt;
7104
7105 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7106 "ahci_intr_set_device_bits enter: port %d", port);
7107
7108 mutex_enter(&ahci_portp->ahciport_mutex);
7109 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7110 mutex_exit(&ahci_portp->ahciport_mutex);
7111 return (AHCI_SUCCESS);
7112 }
7113
7114 /*
7115 * First the handler got which commands are finished by checking
7116 * PxSACT register
7117 */
7118 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7119 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7120
7121 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7122 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7123
7124 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7125 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7126 "port_sactive = 0x%x", port,
7127 ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7128
7129 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7130 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7131
7132 /*
7133 * For NCQ commands, the software can determine which command has
7134 * already been transmitted to the device by checking PxCI register.
7135 */
7136 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7137 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7138
7139 issued_tags = ahci_portp->ahciport_pending_tags &
7140 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7141
7142 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7143 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7144 != DDI_FM_OK) {
7145 mutex_exit(&ahci_portp->ahciport_mutex);
7146 return (AHCI_FAILURE);
7147 }
7148
7149 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7150 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7151 "port_cmd_issue = 0x%x", port,
7152 ahci_portp->ahciport_pending_tags, port_cmd_issue);
7153
7154 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7155 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7156
7157 /*
7158 * Clear ahciport_pending_tags bit when the corresponding command
7159 * is already sent down to the device.
7160 */
7161 while (issued_tags) {
7162 issued_slot = ddi_ffs(issued_tags) - 1;
7163 if (issued_slot == -1) {
7164 goto next;
7165 }
7166 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7167 CLEAR_BIT(issued_tags, issued_slot);
7168 }
7169
7170 next:
7171 while (finished_tags) {
7172 finished_slot = ddi_ffs(finished_tags) - 1;
7173 if (finished_slot == -1) {
7174 goto out;
7175 }
7176
7177 /* The command is certainly transmitted to the device */
7178 ASSERT(!(ahci_portp->ahciport_pending_tags &
7179 (0x1 << finished_slot)));
7180
7181 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7182 ASSERT(satapkt != NULL);
7183
7184 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7185 "ahci_intr_set_device_bits: sending up pkt 0x%p "
7186 "with SATA_PKT_COMPLETED", (void *)satapkt);
7187
7188 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7189 CLEAR_BIT(finished_tags, finished_slot);
7190 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7191
7192 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7193 }
7194 out:
7195 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7196 "ahci_intr_set_device_bits: port %d "
7197 "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7198 port, ahci_portp->ahciport_pending_ncq_tags,
7199 ahci_portp->ahciport_pending_tags);
7200
7201 ahci_flush_doneq(ahci_portp);
7202
7203 mutex_exit(&ahci_portp->ahciport_mutex);
7204
7205 return (AHCI_SUCCESS);
7206 }
7207
7208 /*
7209 * Port multiplier asynchronous notification event handler. Called upon a
7210 * device is hot plugged/pulled.
7211 *
7212 * The async-notification event will only be recorded by ahcipmi_snotif_tags
7213 * here and will be handled by ahci_probe_pmult().
7214 *
7215 * NOTE: called only from ahci_port_intr().
7216 */
7217 static int
7218 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7219 uint8_t port)
7220 {
7221 sata_device_t sdevice;
7222
7223 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7224 "ahci_intr_pmult_sntf_events enter: port %d ", port);
7225
7226 /* no hot-plug while attaching process */
7227 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7228 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7229 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7230 return (AHCI_SUCCESS);
7231 }
7232 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7233
7234 mutex_enter(&ahci_portp->ahciport_mutex);
7235 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7236 mutex_exit(&ahci_portp->ahciport_mutex);
7237 return (AHCI_SUCCESS);
7238 }
7239
7240 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7241
7242 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7243 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7244 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7245 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7246 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7247 AHCI_SNOTIF_CLEAR_ALL);
7248
7249 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7250 mutex_exit(&ahci_portp->ahciport_mutex);
7251 return (AHCI_SUCCESS);
7252 }
7253
7254 /* Port Multiplier sub-device hot-plug handler */
7255 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7256 mutex_exit(&ahci_portp->ahciport_mutex);
7257 return (AHCI_SUCCESS);
7258 }
7259
7260 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7261 /* Not allowed to re-enter. */
7262 mutex_exit(&ahci_portp->ahciport_mutex);
7263 return (AHCI_SUCCESS);
7264 }
7265
7266 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7267
7268 /*
7269 * NOTE:
7270 * Even if Asynchronous Notification is supported (and enabled) by
7271 * both controller and the port multiplier, the content of PxSNTF
7272 * register is always set to 0x8000 by async notification event. We
7273 * need to check GSCR[32] on the port multiplier to find out the
7274 * owner of this event.
7275 * This is not accord with SATA spec 2.6 and needs further
7276 * clarification.
7277 */
7278 /* hot-plug will not reported while reseting. */
7279 if (ahci_portp->ahciport_reset_in_progress == 1) {
7280 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7281 "port %d snotif event ignored", port);
7282 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7283 mutex_exit(&ahci_portp->ahciport_mutex);
7284 return (AHCI_SUCCESS);
7285 }
7286
7287 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7288 "PxSNTF is set to 0x%x by port multiplier",
7289 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7290
7291 /*
7292 * Now we need do some necessary operation and inform SATA framework
7293 * that link/device events has happened.
7294 */
7295 bzero((void *)&sdevice, sizeof (sata_device_t));
7296 sdevice.satadev_addr.cport = ahci_ctlp->
7297 ahcictl_port_to_cport[port];
7298 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7299 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7300 sdevice.satadev_state = SATA_PSTATE_PWRON;
7301
7302 /* Just reject packets, do not stop that port. */
7303 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7304
7305 mutex_exit(&ahci_portp->ahciport_mutex);
7306 sata_hba_event_notify(
7307 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7308 &sdevice,
7309 SATA_EVNT_PMULT_LINK_CHANGED);
7310 mutex_enter(&ahci_portp->ahciport_mutex);
7311
7312 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7313 mutex_exit(&ahci_portp->ahciport_mutex);
7314
7315 return (AHCI_SUCCESS);
7316 }
7317
7318 /*
7319 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7320 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7321 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7322 * indicates a COMINIT signal was received.
7323 *
7324 * Hot plug insertion is detected by reception of a COMINIT signal from the
7325 * device. On reception of unsolicited COMINIT, the HBA shall generate a
7326 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7327 * begin the normal communication negotiation sequence as outlined in the
7328 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7329 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7330 * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7331 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7332 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7333 * to handle hot plug insertion. In this interrupt handler, just do nothing
7334 * but print some log message and clear the bit.
7335 */
7336 static int
7337 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7338 ahci_port_t *ahci_portp, uint8_t port)
7339 {
7340 #if AHCI_DEBUG
7341 uint32_t port_serror;
7342 #endif
7343
7344 mutex_enter(&ahci_portp->ahciport_mutex);
7345
7346 #if AHCI_DEBUG
7347 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7348 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7349
7350 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7351 "ahci_intr_port_connect_change: port %d, "
7352 "port_serror = 0x%x", port, port_serror);
7353 #endif
7354
7355 /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7356 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7357 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7358 SERROR_EXCHANGED_ERR);
7359
7360 mutex_exit(&ahci_portp->ahciport_mutex);
7361
7362 return (AHCI_SUCCESS);
7363 }
7364
7365 /*
7366 * Hot Plug Operation for platforms that support Mechanical Presence
7367 * Switches.
7368 *
7369 * When set, it indicates that a mechanical presence switch attached to this
7370 * port has been opened or closed, which may lead to a change in the connection
7371 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7372 * are set to '1'.
7373 *
7374 * At the moment, this interrupt is not needed and disabled and we just log
7375 * the debug message.
7376 */
7377 static int
7378 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7379 ahci_port_t *ahci_portp, uint8_t port)
7380 {
7381 uint32_t cap_status, port_cmd_status;
7382
7383 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7384 "ahci_intr_device_mechanical_presence_status enter, "
7385 "port %d", port);
7386
7387 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7388 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7389
7390 mutex_enter(&ahci_portp->ahciport_mutex);
7391 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7392 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7393
7394 if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7395 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7396 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7397 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7398 "the interrupt: cap_status = 0x%x, "
7399 "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7400 mutex_exit(&ahci_portp->ahciport_mutex);
7401
7402 return (AHCI_SUCCESS);
7403 }
7404
7405 #if AHCI_DEBUG
7406 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7407 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7408 "The mechanical presence switch is open: "
7409 "port %d, port_cmd_status = 0x%x",
7410 port, port_cmd_status);
7411 } else {
7412 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7413 "The mechanical presence switch is close: "
7414 "port %d, port_cmd_status = 0x%x",
7415 port, port_cmd_status);
7416 }
7417 #endif
7418
7419 mutex_exit(&ahci_portp->ahciport_mutex);
7420
7421 return (AHCI_SUCCESS);
7422 }
7423
7424 /*
7425 * Native Hot Plug Support.
7426 *
7427 * When set, it indicates that the internal PHYRDY signal changed state.
7428 * This bit reflects the state of PxSERR.DIAG.N.
7429 *
7430 * There are three kinds of conditions to generate this interrupt event:
7431 * 1. a device is inserted
7432 * 2. a device is disconnected
7433 * 3. when the link enters/exits a Partial or Slumber interface power
7434 * management state
7435 *
7436 * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7437 * bit may be set due to the link entering the Partial or Slumber power
7438 * management state, rather than due to a hot plug insertion or removal
7439 * event. So far, the interface power management is disabled, so the
7440 * driver can reliably get removal detection notification via the
7441 * PxSERR.DIAG.N bit.
7442 */
7443 static int
7444 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7445 ahci_port_t *ahci_portp, uint8_t port)
7446 {
7447 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7448 sata_device_t sdevice;
7449 int dev_exists_now = 0;
7450 int dev_existed_previously = 0;
7451 ahci_addr_t port_addr;
7452
7453 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7454 "ahci_intr_phyrdy_change enter, port %d", port);
7455
7456 /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7457 mutex_enter(&ahci_portp->ahciport_mutex);
7458 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7459 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7460 SERROR_PHY_RDY_CHG);
7461 mutex_exit(&ahci_portp->ahciport_mutex);
7462
7463 mutex_enter(&ahci_ctlp->ahcictl_mutex);
7464 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7465 (ahci_portp == NULL)) {
7466 /* The whole controller setup is not yet done. */
7467 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7468 return (AHCI_SUCCESS);
7469 }
7470 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7471
7472 mutex_enter(&ahci_portp->ahciport_mutex);
7473
7474 /* SStatus tells the presence of device. */
7475 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7476 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7477
7478 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7479 dev_exists_now = 1;
7480 }
7481
7482 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7483 dev_existed_previously = 1;
7484 }
7485
7486 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7487 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7488 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7489 "ahci_intr_phyrdy_change: port %d "
7490 "AHCI_PORT_FLAG_NODEV is cleared", port);
7491 if (dev_exists_now == 0)
7492 dev_existed_previously = 1;
7493 }
7494
7495 bzero((void *)&sdevice, sizeof (sata_device_t));
7496 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7497 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7498 sdevice.satadev_addr.pmport = 0;
7499 sdevice.satadev_state = SATA_PSTATE_PWRON;
7500 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7501
7502 AHCI_ADDR_SET_PORT(&port_addr, port);
7503
7504 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7505 if (dev_exists_now) {
7506 if (dev_existed_previously) { /* 1 -> 1 */
7507 /* Things are fine now. The loss was temporary. */
7508 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7509 "ahci_intr_phyrdy_change port %d "
7510 "device link lost/established", port);
7511
7512 mutex_exit(&ahci_portp->ahciport_mutex);
7513 sata_hba_event_notify(
7514 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7515 &sdevice,
7516 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7517 mutex_enter(&ahci_portp->ahciport_mutex);
7518
7519 } else { /* 0 -> 1 */
7520 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7521 "ahci_intr_phyrdy_change: port %d "
7522 "device link established", port);
7523
7524 /*
7525 * A new device has been detected. The new device
7526 * might be a port multiplier instead of a drive, so
7527 * we cannot update the signature directly.
7528 */
7529 (void) ahci_initialize_port(ahci_ctlp,
7530 ahci_portp, &port_addr);
7531
7532 /* Try to start the port */
7533 if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7534 != AHCI_SUCCESS) {
7535 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7536 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7537 "ahci_intr_phyrdy_change: port %d failed "
7538 "at start port", port);
7539 }
7540
7541 /* Clear the max queue depth for inserted device */
7542 ahci_portp->ahciport_max_ncq_tags = 0;
7543
7544 mutex_exit(&ahci_portp->ahciport_mutex);
7545 sata_hba_event_notify(
7546 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7547 &sdevice,
7548 SATA_EVNT_LINK_ESTABLISHED);
7549 mutex_enter(&ahci_portp->ahciport_mutex);
7550
7551 }
7552 } else { /* No device exists now */
7553
7554 if (dev_existed_previously) { /* 1 -> 0 */
7555 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7556 "ahci_intr_phyrdy_change: port %d "
7557 "device link lost", port);
7558
7559 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7560 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7561 ahci_portp, port);
7562
7563 if (ahci_portp->ahciport_device_type ==
7564 SATA_DTYPE_PMULT) {
7565 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7566 }
7567
7568 /* An existing device is lost. */
7569 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7570 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7571
7572 mutex_exit(&ahci_portp->ahciport_mutex);
7573 sata_hba_event_notify(
7574 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7575 &sdevice,
7576 SATA_EVNT_LINK_LOST);
7577 mutex_enter(&ahci_portp->ahciport_mutex);
7578 }
7579 }
7580 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7581
7582 mutex_exit(&ahci_portp->ahciport_mutex);
7583
7584 return (AHCI_SUCCESS);
7585 }
7586
7587 /*
7588 * PxIS.UFS - Unknown FIS Error
7589 *
7590 * This interrupt event means an unknown FIS was received and has been
7591 * copied into system memory. An unknown FIS is not considered an illegal
7592 * FIS, unless the length received is more than 64 bytes. If an unknown
7593 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7594 * normal operation. If the unknown FIS is more than 64 bytes, then it
7595 * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7596 * a fatal error.
7597 *
7598 * PxIS.IPMS - Incorrect Port Multiplier Status
7599 *
7600 * IPMS Indicates that the HBA received a FIS from a device that did not
7601 * have a command outstanding. The IPMS bit may be set during enumeration
7602 * of devices on a Port Multiplier due to the normal Port Multiplier
7603 * enumeration process. It is recommended that IPMS only be used after
7604 * enumeration is complete on the Port Multiplier (copied from spec).
7605 *
7606 * PxIS.OFS - Overflow Error
7607 *
7608 * Command list overflow is defined as software building a command table
7609 * that has fewer total bytes than the transaction given to the device.
7610 * On device writes, the HBA will run out of data, and on reads, there
7611 * will be no room to put the data.
7612 *
7613 * For an overflow on data read, either PIO or DMA, the HBA will set
7614 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7615 * non-fatal error when the HBA can continues. Sometimes, it will cause
7616 * a fatal error and need the software to do something.
7617 *
7618 * For an overflow on data write, setting PxIS.OFS is optional for both
7619 * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7620 * software to clean up from this serious error.
7621 *
7622 * PxIS.INFS - Interface Non-Fatal Error
7623 *
7624 * This interrupt event indicates that the HBA encountered an error on
7625 * the Serial ATA interface but was able to continue operation. The kind
7626 * of error usually occurred during a non-Data FIS, and under this condition
7627 * the FIS will be re-transmitted by HBA automatically.
7628 *
7629 * When the FMA is implemented, there should be a stat structure to
7630 * record how many every kind of error happens.
7631 */
7632 static int
7633 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7634 uint8_t port, uint32_t intr_status)
7635 {
7636 uint32_t port_serror;
7637 #if AHCI_DEBUG
7638 uint32_t port_cmd_status;
7639 uint32_t port_cmd_issue;
7640 uint32_t port_sactive;
7641 int current_slot;
7642 uint32_t current_tags;
7643 sata_pkt_t *satapkt;
7644 ahci_cmd_header_t *cmd_header;
7645 uint32_t cmd_dmacount;
7646 #endif
7647
7648 mutex_enter(&ahci_portp->ahciport_mutex);
7649
7650 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7651 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7652
7653 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7654 "ahci_intr_non_fatal_error: port %d, "
7655 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7656
7657 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7658
7659 if (intr_status & AHCI_INTR_STATUS_UFS) {
7660 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7661 "ahci port %d has unknown FIS error", port);
7662
7663 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7664 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7665 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7666 SERROR_FIS_TYPE);
7667 }
7668
7669 #if AHCI_DEBUG
7670 if (intr_status & AHCI_INTR_STATUS_IPMS) {
7671 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7672 "has Incorrect Port Multiplier Status error", port);
7673 }
7674
7675 if (intr_status & AHCI_INTR_STATUS_OFS) {
7676 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7677 "ahci port %d has overflow error", port);
7678 }
7679
7680 if (intr_status & AHCI_INTR_STATUS_INFS) {
7681 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7682 "ahci port %d has interface non fatal error", port);
7683 }
7684
7685 /*
7686 * Record the error occurred command's slot.
7687 */
7688 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7689 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7690 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7691 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7692
7693 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7694 AHCI_CMD_STATUS_CCS_SHIFT;
7695
7696 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7697 satapkt = ahci_portp->ahciport_err_retri_pkt;
7698 ASSERT(satapkt != NULL);
7699 ASSERT(current_slot == 0);
7700 } else {
7701 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7702 }
7703
7704 if (satapkt != NULL) {
7705 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7706 "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7707 "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7708 satapkt->satapkt_cmd.satacmd_cmd_reg);
7709
7710 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7711 "ahci_intr_non_fatal_error: port %d, "
7712 "satapkt 0x%p is being processed when error occurs",
7713 port, (void *)satapkt);
7714
7715 /*
7716 * PRD Byte Count field of command header is not
7717 * required to reflect the total number of bytes
7718 * transferred when an overflow occurs, so here
7719 * just log the value.
7720 */
7721 cmd_dmacount =
7722 ahci_portp->ahciport_prd_bytecounts[current_slot];
7723 if (cmd_dmacount) {
7724 cmd_header = &ahci_portp->
7725 ahciport_cmd_list[current_slot];
7726 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7727 "ahci_intr_non_fatal_error: port %d, "
7728 "PRD Byte Count = 0x%x, "
7729 "ahciport_prd_bytecounts = 0x%x", port,
7730 cmd_header->ahcich_prd_byte_count,
7731 cmd_dmacount);
7732 }
7733 }
7734 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7735 /*
7736 * For queued command, list those command which have already
7737 * been transmitted to the device and still not completed.
7738 */
7739 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7740 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7741
7742 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7743 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7744
7745 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7746 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7747 "port_sactive = 0x%x port_cmd_issue = 0x%x",
7748 ahci_portp->ahciport_pending_ncq_tags,
7749 port_sactive, port_cmd_issue);
7750
7751 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7752 port_sactive & ~port_cmd_issue &
7753 AHCI_NCQ_SLOT_MASK(ahci_portp);
7754
7755 while (current_tags) {
7756 current_slot = ddi_ffs(current_tags) - 1;
7757 if (current_slot == -1) {
7758 goto out;
7759 }
7760
7761 satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7762 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7763 ahci_ctlp, "ahci_intr_non_fatal_error: "
7764 "port %d, satapkt 0x%p is outstanding when "
7765 "error occurs", port, (void *)satapkt);
7766
7767 CLEAR_BIT(current_tags, current_slot);
7768 }
7769 }
7770 out:
7771 #endif
7772 mutex_exit(&ahci_portp->ahciport_mutex);
7773
7774 return (AHCI_SUCCESS);
7775 }
7776
7777 /*
7778 * According to the AHCI spec, the error types include system memory
7779 * errors, interface errors, port multiplier errors, device errors,
7780 * command list overflow, command list underflow, native command
7781 * queuing tag errors and pio data transfer errors.
7782 *
7783 * System memory errors such as target abort, master abort, and parity
7784 * may cause the host to stop, and they are serious errors and needed
7785 * to be recovered with software intervention. When system software
7786 * has given a pointer to the HBA that doesn't exist in physical memory,
7787 * a master/target abort error occurs, and PxIS.HBFS will be set. A
7788 * data error such as CRC or parity occurs, the HBA aborts the transfer
7789 * (if necessary) and PxIS.HBDS will be set.
7790 *
7791 * Interface errors are errors that occur due to electrical issues on
7792 * the interface, or protocol miscommunication between the device and
7793 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7794 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7795 * causes PxIS.IFS/PxIS.INFS to be set are
7796 * 1. in PxSERR.ERR, P bit is set to '1'
7797 * 2. in PxSERR.DIAG, C or H bit is set to '1'
7798 * 3. PhyRdy drop unexpectly, N bit is set to '1'
7799 * If the error occurred during a non-data FIS, the FIS must be
7800 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7801 * the error occurred during a data FIS, the transfer will stop, so
7802 * the error is fatal and PxIS.IFS is set.
7803 *
7804 * When a FIS arrives that updates the taskfile, the HBA checks to see
7805 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7806 * stops processing any more commands.
7807 *
7808 * Command list overflow is defined as software building a command table
7809 * that has fewer total bytes than the transaction given to the device.
7810 * On device writes, the HBA will run out of data, and on reads, there
7811 * will be no room to put the data. For an overflow on data read, either
7812 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7813 * For an overflow on data write, setting PxIS.OFS is optional for both
7814 * DMA and PIO, and a COMRESET is required by software to clean up from
7815 * this serious error.
7816 *
7817 * Command list underflow is defined as software building a command
7818 * table that has more total bytes than the transaction given to the
7819 * device. For data writes, both PIO and DMA, the device will detect
7820 * an error and end the transfer. And these errors are most likely going
7821 * to be fatal errors that will cause the port to be restarted. For
7822 * data reads, the HBA updates its PRD byte count, and may be
7823 * able to continue normally, but is not required to. And The HBA is
7824 * not required to detect underflow conditions for native command
7825 * queuing command.
7826 *
7827 * The HBA does not actively check incoming DMA Setup FISes to ensure
7828 * that the PxSACT register bit for that slot is set. Existing error
7829 * mechanisms, such as host bus failure, or bad protocol, are used to
7830 * recover from this case.
7831 *
7832 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7833 * DATA FIS must be an integral number of Dwords. If the HBA receives
7834 * a request which is not an integral number of Dwords, the HBA
7835 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7836 * software restarts the port. And the HBA ensures that the size
7837 * of the DATA FIS received during a PIO command matches the size in
7838 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7839 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7840 * stop running until software restarts the port.
7841 */
7842 /*
7843 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7844 *
7845 * PxIS.IFS indicates that the hba encountered an error on the serial ata
7846 * interface which caused the transfer to stop.
7847 *
7848 * PxIS.HBDS indicates that the hba encountered a data error
7849 * (uncorrectable ecc/parity) when reading from or writing to system memory.
7850 *
7851 * PxIS.HBFS indicates that the hba encountered a host bus error that it
7852 * cannot recover from, such as a bad software pointer.
7853 *
7854 * PxIS.TFES is set whenever the status register is updated by the device
7855 * and the error bit (bit 0) is set.
7856 */
7857 static int
7858 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7859 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7860 {
7861 uint32_t port_cmd_status;
7862 uint32_t port_serror;
7863 uint32_t task_file_status;
7864 int failed_slot;
7865 sata_pkt_t *spkt = NULL;
7866 uint8_t err_byte;
7867 ahci_event_arg_t *args;
7868 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7869 uint32_t failed_tags = 0;
7870 int task_fail_flag = 0, task_abort_flag = 0;
7871 uint32_t slot_status;
7872
7873 mutex_enter(&ahci_portp->ahciport_mutex);
7874
7875 /*
7876 * ahci_intr_phyrdy_change() may have rendered it to
7877 * SATA_DTYPE_NONE.
7878 */
7879 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7880 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7881 "ahci_intr_fatal_error: port %d no device attached, "
7882 "and just return without doing anything", port);
7883 goto out0;
7884 }
7885
7886 if (intr_status & AHCI_INTR_STATUS_TFES) {
7887 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7888 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7889 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7890 "ahci_intr_fatal_error: port %d "
7891 "task_file_status = 0x%x", port, task_file_status);
7892 task_fail_flag = 1;
7893
7894 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7895 >> AHCI_TFD_ERR_SHIFT;
7896 if (err_byte == SATA_ERROR_ABORT)
7897 task_abort_flag = 1;
7898 }
7899
7900 /*
7901 * Here we just log the fatal error info in interrupt context.
7902 * Misc recovery processing will be handled in task queue.
7903 */
7904 if (task_fail_flag == 1) {
7905 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7906 /*
7907 * Read PxCMD.CCS to determine the slot that the HBA
7908 * was processing when the error occurred.
7909 */
7910 port_cmd_status = ddi_get32(
7911 ahci_ctlp->ahcictl_ahci_acc_handle,
7912 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7913 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7914 AHCI_CMD_STATUS_CCS_SHIFT;
7915 failed_tags = 0x1 << failed_slot;
7916
7917 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7918 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7919 "ahci_intr_fatal_error: spkt 0x%p is being "
7920 "processed when fatal error occurred for port %d",
7921 spkt, port);
7922
7923 /*
7924 * Won't emit the error message if it is an IDENTIFY
7925 * DEVICE command sent to an ATAPI device.
7926 */
7927 if ((spkt != NULL) &&
7928 (spkt->satapkt_cmd.satacmd_cmd_reg ==
7929 SATAC_ID_DEVICE) &&
7930 (task_abort_flag == 1))
7931 goto out1;
7932
7933 /*
7934 * Won't emit the error message if it is an ATAPI PACKET
7935 * command
7936 */
7937 if ((spkt != NULL) &&
7938 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7939 goto out1;
7940
7941 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7942 slot_status = ddi_get32(
7943 ahci_ctlp->ahcictl_ahci_acc_handle,
7944 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7945 failed_tags = slot_status &
7946 AHCI_NCQ_SLOT_MASK(ahci_portp);
7947 }
7948 }
7949
7950 /* print the fatal error type */
7951 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7952 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7953
7954 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7955 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7956
7957 /* print PxSERR related error message */
7958 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7959
7960 /* print task file register value */
7961 if (task_fail_flag == 1) {
7962 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7963 "= 0x%x", instance, port, task_file_status);
7964 if (task_abort_flag == 1) {
7965 cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7966 "port %d are aborted", instance, port);
7967 ahci_dump_commands(ahci_ctlp, port, failed_tags);
7968 }
7969 }
7970
7971 out1:
7972 /* Prepare the argument for the taskq */
7973 args = ahci_portp->ahciport_event_args;
7974 args->ahciea_ctlp = (void *)ahci_ctlp;
7975 args->ahciea_portp = (void *)ahci_portp;
7976 args->ahciea_event = intr_status;
7977 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7978
7979 /* Start the taskq to handle error recovery */
7980 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7981 ahci_events_handler,
7982 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7983 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7984 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7985 "port %d failed", instance, port);
7986 }
7987 out0:
7988 mutex_exit(&ahci_portp->ahciport_mutex);
7989
7990 return (AHCI_SUCCESS);
7991 }
7992
7993 /*
7994 * Hot Plug Operation for platforms that support Cold Presence Detect.
7995 *
7996 * When set, a device status has changed as detected by the cold presence
7997 * detect logic. This bit can either be set due to a non-connected port
7998 * receiving a device, or a connected port having its device removed.
7999 * This bit is only valid if the port supports cold presence detect as
8000 * indicated by PxCMD.CPD set to '1'.
8001 *
8002 * At the moment, this interrupt is not needed and disabled and we just
8003 * log the debug message.
8004 */
8005 static int
8006 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
8007 ahci_port_t *ahci_portp, uint8_t port)
8008 {
8009 uint32_t port_cmd_status;
8010 sata_device_t sdevice;
8011
8012 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8013 "ahci_intr_cold_port_detect enter, port %d", port);
8014
8015 mutex_enter(&ahci_portp->ahciport_mutex);
8016
8017 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8018 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8019 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
8020 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8021 "port %d does not support cold presence detect, so "
8022 "we just ignore this interrupt", port);
8023 mutex_exit(&ahci_portp->ahciport_mutex);
8024 return (AHCI_SUCCESS);
8025 }
8026
8027 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8028 "port %d device status has changed", port);
8029
8030 bzero((void *)&sdevice, sizeof (sata_device_t));
8031 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
8032 sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
8033 sdevice.satadev_addr.pmport = 0;
8034 sdevice.satadev_state = SATA_PSTATE_PWRON;
8035
8036 if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
8037 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8038 "port %d: a device is hot plugged", port);
8039 mutex_exit(&ahci_portp->ahciport_mutex);
8040 sata_hba_event_notify(
8041 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8042 &sdevice,
8043 SATA_EVNT_DEVICE_ATTACHED);
8044 mutex_enter(&ahci_portp->ahciport_mutex);
8045
8046 } else {
8047 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8048 "port %d: a device is hot unplugged", port);
8049 mutex_exit(&ahci_portp->ahciport_mutex);
8050 sata_hba_event_notify(
8051 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8052 &sdevice,
8053 SATA_EVNT_DEVICE_DETACHED);
8054 mutex_enter(&ahci_portp->ahciport_mutex);
8055 }
8056
8057 mutex_exit(&ahci_portp->ahciport_mutex);
8058
8059 return (AHCI_SUCCESS);
8060 }
8061
8062 /*
8063 * Enable the interrupts for a particular port.
8064 */
8065 static void
8066 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8067 {
8068 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8069
8070 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8071 "ahci_enable_port_intrs enter, port %d", port);
8072
8073 /*
8074 * Clear port interrupt status before enabling interrupt
8075 */
8076 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8077 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
8078 AHCI_PORT_INTR_MASK);
8079
8080 /*
8081 * Clear the pending bit from IS.IPS
8082 */
8083 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8084 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
8085
8086 /*
8087 * Enable the following interrupts:
8088 * Device to Host Register FIS Interrupt (DHRS)
8089 * PIO Setup FIS Interrupt (PSS)
8090 * Set Device Bits Interrupt (SDBS)
8091 * Unknown FIS Interrupt (UFS)
8092 * Port Connect Change Status (PCS)
8093 * PhyRdy Change Status (PRCS)
8094 * Overflow Status (OFS)
8095 * Interface Non-fatal Error Status (INFS)
8096 * Interface Fatal Error Status (IFS)
8097 * Host Bus Data Error Status (HBDS)
8098 * Host Bus Fatal Error Status (HBFS)
8099 * Task File Error Status (TFES)
8100 */
8101 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8102 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8103 (AHCI_INTR_STATUS_DHRS |
8104 AHCI_INTR_STATUS_PSS |
8105 AHCI_INTR_STATUS_SDBS |
8106 AHCI_INTR_STATUS_UFS |
8107 AHCI_INTR_STATUS_DPS |
8108 AHCI_INTR_STATUS_PCS |
8109 AHCI_INTR_STATUS_PRCS |
8110 AHCI_INTR_STATUS_OFS |
8111 AHCI_INTR_STATUS_INFS |
8112 AHCI_INTR_STATUS_IFS |
8113 AHCI_INTR_STATUS_HBDS |
8114 AHCI_INTR_STATUS_HBFS |
8115 AHCI_INTR_STATUS_TFES));
8116 }
8117
8118 /*
8119 * Enable interrupts for all the ports.
8120 */
8121 static void
8122 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8123 {
8124 uint32_t ghc_control;
8125
8126 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8127
8128 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8129
8130 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8131 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8132
8133 ghc_control |= AHCI_HBA_GHC_IE;
8134
8135 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8136 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8137 }
8138
8139 /*
8140 * Disable interrupts for a particular port.
8141 */
8142 static void
8143 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8144 {
8145 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8146 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8147
8148 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8149 "ahci_disable_port_intrs enter, port %d", port);
8150
8151 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8152 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8153 }
8154
8155 /*
8156 * Disable interrupts for the whole HBA.
8157 *
8158 * The global bit is cleared, then all interrupt sources from all
8159 * ports are disabled.
8160 */
8161 static void
8162 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8163 {
8164 uint32_t ghc_control;
8165
8166 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8167 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8168
8169 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8170 NULL);
8171
8172 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8173 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8174
8175 ghc_control &= ~AHCI_HBA_GHC_IE;
8176
8177 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8178 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8179 }
8180
8181 /*
8182 * Handle FIXED or MSI interrupts.
8183 */
8184 /*
8185 * According to AHCI spec, the HBA may support several interrupt modes:
8186 * * pin based interrupts (FIXED)
8187 * * single MSI message interrupts
8188 * * multiple MSI based message interrupts
8189 *
8190 * For pin based interrupts, the software interrupt handler need to check IS
8191 * register to find out which port has pending interrupts. And then check
8192 * PxIS register to find out which interrupt events happened on that port.
8193 *
8194 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8195 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8196 * in that software interrupt handler need to check IS register to determine
8197 * which port triggered the interrupts since it uses a single message for all
8198 * port interrupts.
8199 *
8200 * HBA may optionally support multiple MSI message for better performance. In
8201 * this mode, each port may have its own interrupt message, and thus generation
8202 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8203 * represents a power-of-2 wrapper on the number of implemented ports, and
8204 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8205 * maximum number of assigned interrupts. When the number of MSI messages
8206 * allocated is less than the number requested, then hardware may have two
8207 * implementation behaviors:
8208 * * assign each ports its own interrupt and then force all additional
8209 * ports to share the last interrupt message, and this condition is
8210 * indicated by clearing GHC.MRSM to '0'
8211 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8212 * When multiple-message MSI is enabled, hardware will still set IS register
8213 * as single message case. And this IS register may be used by software when
8214 * fewer than the requested number of messages is granted in order to determine
8215 * which port had the interrupt.
8216 *
8217 * Note: The current ahci driver only supports the first two interrupt modes:
8218 * pin based interrupts and single MSI message interrupts, and the reason
8219 * is indicated in below code.
8220 */
8221 static int
8222 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8223 {
8224 dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8225 int count, avail, actual;
8226 int i, rc;
8227
8228 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8229 "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8230
8231 /* get number of interrupts. */
8232 rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8233 if ((rc != DDI_SUCCESS) || (count == 0)) {
8234 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8235 "ddi_intr_get_nintrs() failed, "
8236 "rc %d count %d\n", rc, count);
8237 return (DDI_FAILURE);
8238 }
8239
8240 /* get number of available interrupts. */
8241 rc = ddi_intr_get_navail(dip, intr_type, &avail);
8242 if ((rc != DDI_SUCCESS) || (avail == 0)) {
8243 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8244 "ddi_intr_get_navail() failed, "
8245 "rc %d avail %d\n", rc, avail);
8246 return (DDI_FAILURE);
8247 }
8248
8249 #if AHCI_DEBUG
8250 if (avail < count) {
8251 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8252 "ddi_intr_get_nintrs returned %d, navail() returned %d",
8253 count, avail);
8254 }
8255 #endif
8256
8257 /*
8258 * Note: So far Solaris restricts the maximum number of messages for
8259 * x86 to 2, that is avail is 2, so here we set the count with 1 to
8260 * force the driver to use single MSI message interrupt. In future if
8261 * Solaris remove the restriction, then we need to delete the below
8262 * code and try to use multiple interrupt routine to gain better
8263 * performance.
8264 */
8265 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8266 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8267 "force to use one interrupt routine though the "
8268 "HBA supports %d interrupt", count);
8269 count = 1;
8270 }
8271
8272 /* Allocate an array of interrupt handles. */
8273 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8274 ahci_ctlp->ahcictl_intr_htable =
8275 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8276
8277 /* call ddi_intr_alloc(). */
8278 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8279 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8280
8281 if ((rc != DDI_SUCCESS) || (actual == 0)) {
8282 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8283 "ddi_intr_alloc() failed, rc %d count %d actual %d "
8284 "avail %d\n", rc, count, actual, avail);
8285 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8286 ahci_ctlp->ahcictl_intr_size);
8287 return (DDI_FAILURE);
8288 }
8289
8290 /* use interrupt count returned */
8291 #if AHCI_DEBUG
8292 if (actual < count) {
8293 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8294 "Requested: %d, Received: %d", count, actual);
8295 }
8296 #endif
8297
8298 ahci_ctlp->ahcictl_intr_cnt = actual;
8299
8300 /*
8301 * Get priority for first, assume remaining are all the same.
8302 */
8303 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8304 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8305 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8306 "ddi_intr_get_pri() failed", NULL);
8307
8308 /* Free already allocated intr. */
8309 for (i = 0; i < actual; i++) {
8310 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8311 }
8312
8313 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8314 ahci_ctlp->ahcictl_intr_size);
8315 return (DDI_FAILURE);
8316 }
8317
8318 /* Test for high level interrupt. */
8319 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8320 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8321 "ahci_add_intrs: Hi level intr not supported", NULL);
8322
8323 /* Free already allocated intr. */
8324 for (i = 0; i < actual; i++) {
8325 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8326 }
8327
8328 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8329 sizeof (ddi_intr_handle_t));
8330
8331 return (DDI_FAILURE);
8332 }
8333
8334 /* Call ddi_intr_add_handler(). */
8335 for (i = 0; i < actual; i++) {
8336 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8337 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8338 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8339 "ddi_intr_add_handler() failed", NULL);
8340
8341 /* Free already allocated intr. */
8342 for (i = 0; i < actual; i++) {
8343 (void) ddi_intr_free(
8344 ahci_ctlp->ahcictl_intr_htable[i]);
8345 }
8346
8347 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8348 ahci_ctlp->ahcictl_intr_size);
8349 return (DDI_FAILURE);
8350 }
8351 }
8352
8353 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8354 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8355 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8356 "ddi_intr_get_cap() failed", NULL);
8357
8358 /* Free already allocated intr. */
8359 for (i = 0; i < actual; i++) {
8360 (void) ddi_intr_free(
8361 ahci_ctlp->ahcictl_intr_htable[i]);
8362 }
8363
8364 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8365 ahci_ctlp->ahcictl_intr_size);
8366 return (DDI_FAILURE);
8367 }
8368
8369 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8370 /* Call ddi_intr_block_enable() for MSI. */
8371 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8372 ahci_ctlp->ahcictl_intr_cnt);
8373 } else {
8374 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8375 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8376 (void) ddi_intr_enable(
8377 ahci_ctlp->ahcictl_intr_htable[i]);
8378 }
8379 }
8380
8381 return (DDI_SUCCESS);
8382 }
8383
8384 /*
8385 * Removes the registered interrupts irrespective of whether they
8386 * were legacy or MSI.
8387 *
8388 * NOTE: The controller interrupts must be disabled before calling
8389 * this routine.
8390 */
8391 static void
8392 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8393 {
8394 int x;
8395
8396 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8397
8398 /* Disable all interrupts. */
8399 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8400 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8401 /* Call ddi_intr_block_disable(). */
8402 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8403 ahci_ctlp->ahcictl_intr_cnt);
8404 } else {
8405 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8406 (void) ddi_intr_disable(
8407 ahci_ctlp->ahcictl_intr_htable[x]);
8408 }
8409 }
8410
8411 /* Call ddi_intr_remove_handler(). */
8412 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8413 (void) ddi_intr_remove_handler(
8414 ahci_ctlp->ahcictl_intr_htable[x]);
8415 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8416 }
8417
8418 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8419 }
8420
8421 /*
8422 * This routine tries to put port into P:NotRunning state by clearing
8423 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8424 * and PxCMD.CR to '0'.
8425 */
8426 static int
8427 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8428 ahci_port_t *ahci_portp, uint8_t port)
8429 {
8430 uint32_t port_cmd_status;
8431 int loop_count;
8432
8433 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8434 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8435
8436 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8437 "ahci_put_port_into_notrunning_state enter: port %d", port);
8438
8439 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8440 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8441
8442 port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8443 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8444 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8445
8446 /* Wait until PxCMD.CR is cleared */
8447 loop_count = 0;
8448 do {
8449 port_cmd_status =
8450 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8451 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8452
8453 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8454 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8455 "clearing port %d CMD.CR timeout, "
8456 "port_cmd_status = 0x%x", port,
8457 port_cmd_status);
8458 /*
8459 * We are effectively timing out after 0.5 sec.
8460 * This value is specified in AHCI spec.
8461 */
8462 break;
8463 }
8464
8465 /* Wait for 10 millisec */
8466 drv_usecwait(AHCI_10MS_USECS);
8467 } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8468
8469 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8470
8471 if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8472 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8473 "ahci_put_port_into_notrunning_state: failed to clear "
8474 "PxCMD.CR to '0' after loop count: %d, and "
8475 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8476 return (AHCI_FAILURE);
8477 } else {
8478 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8479 "ahci_put_port_into_notrunning_state: succeeded to clear "
8480 "PxCMD.CR to '0' after loop count: %d, and "
8481 "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8482 return (AHCI_SUCCESS);
8483 }
8484 }
8485
8486 /*
8487 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8488 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8489 * stable state, then set PxCMD.ST to '1' to start the port directly.
8490 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8491 * COMRESET to the device to put it in an idle state.
8492 *
8493 * The fifth argument returns whether the port reset is involved during
8494 * the process.
8495 *
8496 * The routine will be called under following scenarios:
8497 * + To reset the HBA
8498 * + To abort the packet(s)
8499 * + To reset the port
8500 * + To activate the port
8501 * + Fatal error recovery
8502 * + To abort the timeout packet(s)
8503 *
8504 * NOTES!!! During this procedure, PxSERR register will be cleared, and
8505 * according to the spec, the clearance of three bits will also clear
8506 * three interrupt status bits.
8507 * 1. PxSERR.DIAG.F will clear PxIS.UFS
8508 * 2. PxSERR.DIAG.X will clear PxIS.PCS
8509 * 3. PxSERR.DIAG.N will clear PxIS.PRCS
8510 *
8511 * Among these three interrupt events, the driver needs to take care of
8512 * PxIS.PRCS, which is the hot plug event. When the driver found out
8513 * a device was unplugged, it will call the interrupt handler.
8514 */
8515 static int
8516 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8517 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8518 {
8519 uint32_t port_sstatus;
8520 uint32_t task_file_status;
8521 sata_device_t sdevice;
8522 int rval;
8523 ahci_addr_t addr_port;
8524 ahci_pmult_info_t *pminfo = NULL;
8525 int dev_exists_begin = 0;
8526 int dev_exists_end = 0;
8527 uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8528 int npmport = 0;
8529 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8530
8531 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8532
8533 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8534 "ahci_restart_port_wait_till_ready: port %d enter", port);
8535
8536 AHCI_ADDR_SET_PORT(&addr_port, port);
8537
8538 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8539 dev_exists_begin = 1;
8540
8541 /* First clear PxCMD.ST */
8542 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8543 port);
8544 if (rval != AHCI_SUCCESS)
8545 /*
8546 * If PxCMD.CR does not clear within a reasonable time, it
8547 * may assume the interface is in a hung condition and may
8548 * continue with issuing the port reset.
8549 */
8550 goto reset;
8551
8552 /* Then clear PxSERR */
8553 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8554 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8555 AHCI_SERROR_CLEAR_ALL);
8556
8557 /* Then get PxTFD */
8558 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8559 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8560
8561 /*
8562 * Check whether the device is in a stable status, if yes,
8563 * then start the port directly. However for ahci_tran_reset_dport,
8564 * we may have to perform a port reset.
8565 */
8566 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8567 !(flag & AHCI_PORT_RESET))
8568 goto out;
8569
8570 reset:
8571 /*
8572 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8573 * a COMRESET to the device
8574 */
8575 ahci_disable_port_intrs(ahci_ctlp, port);
8576 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8577 ahci_enable_port_intrs(ahci_ctlp, port);
8578
8579 #ifdef AHCI_DEBUG
8580 if (rval != AHCI_SUCCESS)
8581 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8582 "ahci_restart_port_wait_till_ready: port %d failed",
8583 port);
8584 #endif
8585
8586 if (reset_flag != NULL)
8587 *reset_flag = 1;
8588
8589 /* Indicate to the framework that a reset has happened. */
8590 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8591 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8592 !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8593 /* Set the reset in progress flag */
8594 ahci_portp->ahciport_reset_in_progress = 1;
8595
8596 bzero((void *)&sdevice, sizeof (sata_device_t));
8597 sdevice.satadev_addr.cport =
8598 ahci_ctlp->ahcictl_port_to_cport[port];
8599 sdevice.satadev_addr.pmport = 0;
8600 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8601
8602 sdevice.satadev_state = SATA_DSTATE_RESET |
8603 SATA_DSTATE_PWR_ACTIVE;
8604 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8605 mutex_exit(&ahci_portp->ahciport_mutex);
8606 sata_hba_event_notify(
8607 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8608 &sdevice,
8609 SATA_EVNT_DEVICE_RESET);
8610 mutex_enter(&ahci_portp->ahciport_mutex);
8611 }
8612
8613 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8614 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8615 } else {
8616 ahci_portp->ahciport_reset_in_progress = 0;
8617 }
8618
8619 out:
8620 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8621
8622 /* SStatus tells the presence of device. */
8623 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8624 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8625
8626 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8627 dev_exists_end = 1;
8628 }
8629
8630 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8631 return (rval);
8632
8633 /* Check whether a hot plug event happened */
8634 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8635 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8636 "ahci_restart_port_wait_till_ready: port %d "
8637 "device is removed", port);
8638 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8639 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8640 "ahci_restart_port_wait_till_ready: port %d "
8641 "AHCI_PORT_FLAG_NODEV flag is set", port);
8642 mutex_exit(&ahci_portp->ahciport_mutex);
8643 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8644 mutex_enter(&ahci_portp->ahciport_mutex);
8645
8646 return (rval);
8647 }
8648
8649
8650 /* 0/1 -> 1 : device may change */
8651 /*
8652 * May be called by ahci_fatal_error_recovery_handler, so
8653 * don't issue software if the previous device is ATAPI.
8654 */
8655 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8656 return (rval);
8657
8658 /*
8659 * The COMRESET will make port multiplier enter legacy mode.
8660 * Issue a software reset to make it work again.
8661 */
8662 ahci_disable_port_intrs(ahci_ctlp, port);
8663 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8664 ahci_enable_port_intrs(ahci_ctlp, port);
8665
8666 /*
8667 * Following codes are specific for the port multiplier
8668 */
8669 if (previous_dev_type != SATA_DTYPE_PMULT &&
8670 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8671 /* in case previous_dev_type is corrupt */
8672 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8673 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8674 return (rval);
8675 }
8676
8677 /* Device change: PMult -> Non-PMult */
8678 if (previous_dev_type == SATA_DTYPE_PMULT &&
8679 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8680 /*
8681 * This might happen because
8682 * 1. Software reset failed. Port multiplier is not correctly
8683 * enumerated.
8684 * 2. Another non-port-multiplier device is attached. Perhaps
8685 * the port multiplier was replaced by another device by
8686 * whatever reason, but AHCI driver missed hot-plug event.
8687 *
8688 * Now that the port has been initialized, we just need to
8689 * update the port structure according new device, then report
8690 * and wait SATA framework to probe new device.
8691 */
8692
8693 /* Force to release pmult resource */
8694 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8695 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8696
8697 bzero((void *)&sdevice, sizeof (sata_device_t));
8698 sdevice.satadev_addr.cport =
8699 ahci_ctlp->ahcictl_port_to_cport[port];
8700 sdevice.satadev_addr.pmport = 0;
8701 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8702
8703 sdevice.satadev_state = SATA_DSTATE_RESET |
8704 SATA_DSTATE_PWR_ACTIVE;
8705
8706 mutex_exit(&ahci_portp->ahciport_mutex);
8707 sata_hba_event_notify(
8708 ahci_ctlp->ahcictl_dip,
8709 &sdevice,
8710 SATA_EVNT_DEVICE_RESET);
8711 mutex_enter(&ahci_portp->ahciport_mutex);
8712
8713 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8714 "Port multiplier is [Gone] at port %d ", port);
8715 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8716 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8717
8718 return (AHCI_SUCCESS);
8719 }
8720
8721 /* Device change: Non-PMult -> PMult */
8722 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8723
8724 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8725 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8726
8727 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8728 }
8729 pminfo = ahci_portp->ahciport_pmult_info;
8730 ASSERT(pminfo != NULL);
8731
8732 /* Device (may) change: PMult -> PMult */
8733 /*
8734 * First initialize port multiplier. Set state to READY and wait for
8735 * probe entry point to initialize it
8736 */
8737 ahci_portp->ahciport_port_state = SATA_STATE_READY;
8738
8739 /*
8740 * It's a little complicated while target is a port multiplier. we
8741 * need to COMRESET all pmports behind that PMult otherwise those
8742 * sub-links between the PMult and the sub-devices will be in an
8743 * inactive state (indicated by PSCR0/PxSSTS) and the following access
8744 * to those sub-devices will be rejected by Link-Fatal-Error.
8745 */
8746 /*
8747 * The PxSNTF will be set soon after the pmult is plugged. While the
8748 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8749 * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8750 * will initialized it.
8751 */
8752 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8753 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8754
8755 /* Report reset event. */
8756 ahci_portp->ahciport_reset_in_progress = 1;
8757
8758 bzero((void *)&sdevice, sizeof (sata_device_t));
8759 sdevice.satadev_addr.cport = cport;
8760 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8761 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8762 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8763 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8764 SATA_EVNT_DEVICE_RESET);
8765
8766 return (rval);
8767 }
8768
8769 /*
8770 * This routine may be called under four scenarios:
8771 * a) do the recovery from fatal error
8772 * b) or we need to timeout some commands
8773 * c) or we need to abort some commands
8774 * d) or we need reset device/port/controller
8775 *
8776 * In all these scenarios, we need to send any pending unfinished
8777 * commands up to sata framework.
8778 */
8779 static void
8780 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8781 ahci_port_t *ahci_portp,
8782 uint32_t slot_status,
8783 uint32_t failed_tags,
8784 uint32_t timeout_tags,
8785 uint32_t aborted_tags,
8786 uint32_t reset_tags)
8787 {
8788 uint32_t finished_tags = 0;
8789 uint32_t unfinished_tags = 0;
8790 int tmp_slot;
8791 sata_pkt_t *satapkt;
8792 int ncq_cmd_in_progress = 0;
8793 int err_retri_cmd_in_progress = 0;
8794 int rdwr_pmult_cmd_in_progress = 0;
8795
8796 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8797
8798 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8799 "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8800 ahci_portp->ahciport_port_num, slot_status);
8801
8802 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8803 "ahci_mop_commands: failed_tags: 0x%x, "
8804 "timeout_tags: 0x%x aborted_tags: 0x%x, "
8805 "reset_tags: 0x%x", failed_tags,
8806 timeout_tags, aborted_tags, reset_tags);
8807
8808 #ifdef AHCI_DEBUG
8809 if (ahci_debug_flags & AHCIDBG_ERRS) {
8810 int i;
8811 char msg_buf[200] = {0, };
8812 for (i = 0x1f; i >= 0; i--) {
8813 if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8814 msg_buf[i] = 'X';
8815 else
8816 msg_buf[i] = '.';
8817 }
8818 msg_buf[0x20] = '\0';
8819 cmn_err(CE_NOTE, "port[%d] slots: %s",
8820 ahci_portp->ahciport_port_num, msg_buf);
8821 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8822 (void *)ahci_portp->ahciport_err_retri_pkt,
8823 (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8824 }
8825 #endif
8826
8827 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8828 finished_tags = ahci_portp->ahciport_pending_tags &
8829 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8830
8831 unfinished_tags = slot_status &
8832 AHCI_SLOT_MASK(ahci_ctlp) &
8833 ~failed_tags &
8834 ~aborted_tags &
8835 ~reset_tags &
8836 ~timeout_tags;
8837 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8838 ncq_cmd_in_progress = 1;
8839 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8840 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8841
8842 unfinished_tags = slot_status &
8843 AHCI_NCQ_SLOT_MASK(ahci_portp) &
8844 ~failed_tags &
8845 ~aborted_tags &
8846 ~reset_tags &
8847 ~timeout_tags;
8848 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8849
8850 /*
8851 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8852 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8853 * complete successfully due to one of the following three
8854 * conditions:
8855 *
8856 * 1. Fatal error - failed_tags includes its slot
8857 * 2. Timed out - timeout_tags includes its slot
8858 * 3. Aborted when hot unplug - aborted_tags includes its
8859 * slot
8860 *
8861 * Please note that the command is always sent down in Slot 0
8862 */
8863 err_retri_cmd_in_progress = 1;
8864 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8865 "ahci_mop_commands is called for port %d while "
8866 "REQUEST SENSE or READ LOG EXT for error retrieval "
8867 "is being executed slot_status = 0x%x",
8868 ahci_portp->ahciport_port_num, slot_status);
8869 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8870 ASSERT(slot_status == 0x1);
8871 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8872 rdwr_pmult_cmd_in_progress = 1;
8873 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8874 "ahci_mop_commands is called for port %d while "
8875 "READ/WRITE PORTMULT command is being executed",
8876 ahci_portp->ahciport_port_num);
8877
8878 ASSERT(slot_status == 0x1);
8879 }
8880
8881 #ifdef AHCI_DEBUG
8882 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8883 "ahci_mop_commands: finished_tags: 0x%x, "
8884 "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8885 #endif
8886
8887 /* Send up finished packets with SATA_PKT_COMPLETED */
8888 while (finished_tags) {
8889 tmp_slot = ddi_ffs(finished_tags) - 1;
8890 if (tmp_slot == -1) {
8891 break;
8892 }
8893
8894 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8895 ASSERT(satapkt != NULL);
8896
8897 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8898 "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8899 (void *)satapkt);
8900
8901 /*
8902 * Cannot fetch the return register content since the port
8903 * was restarted, so the corresponding tag will be set to
8904 * aborted tags.
8905 */
8906 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8907 CLEAR_BIT(finished_tags, tmp_slot);
8908 aborted_tags |= tmp_slot;
8909 continue;
8910 }
8911
8912 if (ncq_cmd_in_progress)
8913 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8914 tmp_slot);
8915 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8916 CLEAR_BIT(finished_tags, tmp_slot);
8917 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8918
8919 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8920 }
8921
8922 /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8923 while (failed_tags) {
8924 if (err_retri_cmd_in_progress) {
8925 satapkt = ahci_portp->ahciport_err_retri_pkt;
8926 ASSERT(satapkt != NULL);
8927 ASSERT(failed_tags == 0x1);
8928
8929 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8930 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8931 (void *)satapkt);
8932 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8933 break;
8934 }
8935 if (rdwr_pmult_cmd_in_progress) {
8936 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8937 ASSERT(satapkt != NULL);
8938 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8939 "ahci_mop_commands: sending up "
8940 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8941 (void *)satapkt);
8942 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8943 break;
8944 }
8945
8946 tmp_slot = ddi_ffs(failed_tags) - 1;
8947 if (tmp_slot == -1) {
8948 break;
8949 }
8950
8951 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8952 ASSERT(satapkt != NULL);
8953
8954 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8955 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8956 (void *)satapkt);
8957
8958 if (ncq_cmd_in_progress)
8959 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8960 tmp_slot);
8961 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8962 CLEAR_BIT(failed_tags, tmp_slot);
8963 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8964
8965 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8966 }
8967
8968 /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8969 while (timeout_tags) {
8970 if (err_retri_cmd_in_progress) {
8971 satapkt = ahci_portp->ahciport_err_retri_pkt;
8972 ASSERT(satapkt != NULL);
8973 ASSERT(timeout_tags == 0x1);
8974
8975 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8976 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8977 (void *)satapkt);
8978 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8979 break;
8980 }
8981 if (rdwr_pmult_cmd_in_progress) {
8982 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8983 ASSERT(satapkt != NULL);
8984 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8985 "ahci_mop_commands: sending up "
8986 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8987 (void *)satapkt);
8988 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8989 break;
8990 }
8991
8992 tmp_slot = ddi_ffs(timeout_tags) - 1;
8993 if (tmp_slot == -1) {
8994 break;
8995 }
8996
8997 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8998 ASSERT(satapkt != NULL);
8999
9000 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9001 "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
9002 (void *)satapkt);
9003
9004 if (ncq_cmd_in_progress)
9005 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9006 tmp_slot);
9007 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9008 CLEAR_BIT(timeout_tags, tmp_slot);
9009 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9010
9011 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
9012 }
9013
9014 /* Send up aborted packets with SATA_PKT_ABORTED */
9015 while (aborted_tags) {
9016 if (err_retri_cmd_in_progress) {
9017 satapkt = ahci_portp->ahciport_err_retri_pkt;
9018 ASSERT(satapkt != NULL);
9019 ASSERT(aborted_tags == 0x1);
9020
9021 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9022 "sending up pkt 0x%p with SATA_PKT_ABORTED",
9023 (void *)satapkt);
9024 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9025 break;
9026 }
9027 if (rdwr_pmult_cmd_in_progress) {
9028 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9029 ASSERT(satapkt != NULL);
9030 ASSERT(aborted_tags == 0x1);
9031 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9032 "ahci_mop_commands: sending up "
9033 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
9034 (void *)satapkt);
9035 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9036 break;
9037 }
9038
9039 tmp_slot = ddi_ffs(aborted_tags) - 1;
9040 if (tmp_slot == -1) {
9041 break;
9042 }
9043
9044 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9045 ASSERT(satapkt != NULL);
9046
9047 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9048 "sending up pkt 0x%p with SATA_PKT_ABORTED",
9049 (void *)satapkt);
9050
9051 if (ncq_cmd_in_progress)
9052 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9053 tmp_slot);
9054 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9055 CLEAR_BIT(aborted_tags, tmp_slot);
9056 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9057
9058 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9059 }
9060
9061 /* Send up reset packets with SATA_PKT_RESET. */
9062 while (reset_tags) {
9063 if (rdwr_pmult_cmd_in_progress) {
9064 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9065 ASSERT(satapkt != NULL);
9066 ASSERT(aborted_tags == 0x1);
9067 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9068 "ahci_mop_commands: sending up "
9069 "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
9070 (void *)satapkt);
9071 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9072 break;
9073 }
9074
9075 tmp_slot = ddi_ffs(reset_tags) - 1;
9076 if (tmp_slot == -1) {
9077 break;
9078 }
9079
9080 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9081 ASSERT(satapkt != NULL);
9082
9083 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9084 "sending up pkt 0x%p with SATA_PKT_RESET",
9085 (void *)satapkt);
9086
9087 if (ncq_cmd_in_progress)
9088 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9089 tmp_slot);
9090 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9091 CLEAR_BIT(reset_tags, tmp_slot);
9092 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9093
9094 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9095 }
9096
9097 /* Send up unfinished packets with SATA_PKT_RESET */
9098 while (unfinished_tags) {
9099 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9100 if (tmp_slot == -1) {
9101 break;
9102 }
9103
9104 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9105 ASSERT(satapkt != NULL);
9106
9107 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9108 "sending up pkt 0x%p with SATA_PKT_RESET",
9109 (void *)satapkt);
9110
9111 if (ncq_cmd_in_progress)
9112 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9113 tmp_slot);
9114 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9115 CLEAR_BIT(unfinished_tags, tmp_slot);
9116 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9117
9118 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9119 }
9120
9121 ahci_portp->ahciport_mop_in_progress--;
9122 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9123
9124 if (ahci_portp->ahciport_mop_in_progress == 0)
9125 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9126
9127 ahci_flush_doneq(ahci_portp);
9128 }
9129
9130 /*
9131 * This routine is going to first request a READ LOG EXT sata pkt from sata
9132 * module, and then deliver it to the HBA to get the ncq failure context.
9133 * The return value is the exactly failed tags.
9134 */
9135 static uint32_t
9136 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9137 uint8_t port)
9138 {
9139 sata_device_t sdevice;
9140 sata_pkt_t *rdlog_spkt, *spkt;
9141 ddi_dma_handle_t buf_dma_handle;
9142 ahci_addr_t addr;
9143 int loop_count;
9144 int rval;
9145 int failed_slot;
9146 uint32_t failed_tags = 0;
9147 struct sata_ncq_error_recovery_page *ncq_err_page;
9148
9149 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9150 "ahci_get_rdlogext_data enter: port %d", port);
9151
9152 /* Prepare the sdevice data */
9153 bzero((void *)&sdevice, sizeof (sata_device_t));
9154 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9155
9156 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9157 sdevice.satadev_addr.pmport = 0;
9158
9159 /* Translate sata_device.satadev_addr -> ahci_addr */
9160 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9161
9162 /*
9163 * Call the sata hba interface to get a rdlog spkt
9164 */
9165 loop_count = 0;
9166 loop:
9167 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9168 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9169 if (rdlog_spkt == NULL) {
9170 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9171 /* Sleep for a while */
9172 drv_usecwait(AHCI_10MS_USECS);
9173 goto loop;
9174 }
9175 /* Timed out after 1s */
9176 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9177 "failed to get rdlog spkt for port %d", port);
9178 return (failed_tags);
9179 }
9180
9181 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9182
9183 /*
9184 * This flag is used to handle the specific error recovery when the
9185 * READ LOG EXT command gets a failure (fatal error or time-out).
9186 */
9187 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9188
9189 /*
9190 * This start is not supposed to fail because after port is restarted,
9191 * the whole command list is empty.
9192 */
9193 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9194 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9195 ahci_portp->ahciport_err_retri_pkt = NULL;
9196
9197 /* Remove the flag after READ LOG EXT command is completed */
9198 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9199
9200 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9201 /* Update the request log data */
9202 buf_dma_handle = *(ddi_dma_handle_t *)
9203 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9204 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9205 DDI_DMA_SYNC_FORKERNEL);
9206 if (rval == DDI_SUCCESS) {
9207 ncq_err_page =
9208 (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9209 satapkt_cmd.satacmd_bp->b_un.b_addr;
9210
9211 /* Get the failed tag */
9212 failed_slot = ncq_err_page->ncq_tag;
9213 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9214 "ahci_get_rdlogext_data: port %d "
9215 "failed slot %d", port, failed_slot);
9216 if (failed_slot & NQ) {
9217 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9218 "the failed slot is not a valid tag", NULL);
9219 goto out;
9220 }
9221
9222 failed_slot &= NCQ_TAG_MASK;
9223 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9224 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9225 "ahci_get_rdlogext_data: failed spkt 0x%p",
9226 (void *)spkt);
9227 if (spkt == NULL) {
9228 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9229 "the failed slot spkt is NULL", NULL);
9230 goto out;
9231 }
9232
9233 failed_tags = 0x1 << failed_slot;
9234
9235 /* Fill out the error context */
9236 ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9237 ncq_err_page);
9238 ahci_update_sata_registers(ahci_ctlp, port,
9239 &spkt->satapkt_device);
9240 }
9241 }
9242 out:
9243 sata_free_error_retrieval_pkt(rdlog_spkt);
9244
9245 return (failed_tags);
9246 }
9247
9248 /*
9249 * This routine is going to first request a REQUEST SENSE sata pkt from sata
9250 * module, and then deliver it to the HBA to get the sense data and copy
9251 * the sense data back to the orignal failed sata pkt, and free the REQUEST
9252 * SENSE sata pkt later.
9253 */
9254 static void
9255 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9256 uint8_t port, sata_pkt_t *spkt)
9257 {
9258 sata_device_t sdevice;
9259 sata_pkt_t *rs_spkt;
9260 sata_cmd_t *sata_cmd;
9261 ddi_dma_handle_t buf_dma_handle;
9262 ahci_addr_t addr;
9263 int loop_count;
9264 #if AHCI_DEBUG
9265 struct scsi_extended_sense *rqsense;
9266 #endif
9267
9268 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9269 "ahci_get_rqsense_data enter: port %d", port);
9270
9271 /* Prepare the sdevice data */
9272 bzero((void *)&sdevice, sizeof (sata_device_t));
9273 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9274
9275 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9276 sdevice.satadev_addr.pmport = 0;
9277
9278 /* Translate sata_device.satadev_addr -> ahci_addr */
9279 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9280
9281 sata_cmd = &spkt->satapkt_cmd;
9282
9283 /*
9284 * Call the sata hba interface to get a rs spkt
9285 */
9286 loop_count = 0;
9287 loop:
9288 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9289 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9290 if (rs_spkt == NULL) {
9291 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9292 /* Sleep for a while */
9293 drv_usecwait(AHCI_10MS_USECS);
9294 goto loop;
9295
9296 }
9297 /* Timed out after 1s */
9298 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9299 "failed to get rs spkt for port %d", port);
9300 return;
9301 }
9302
9303 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9304
9305 /*
9306 * This flag is used to handle the specific error recovery when the
9307 * REQUEST SENSE command gets a faiure (fatal error or time-out).
9308 */
9309 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9310
9311 /*
9312 * This start is not supposed to fail because after port is restarted,
9313 * the whole command list is empty.
9314 */
9315 ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9316 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9317 ahci_portp->ahciport_err_retri_pkt = NULL;
9318
9319 /* Remove the flag after REQUEST SENSE command is completed */
9320 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9321
9322 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9323 /* Update the request sense data */
9324 buf_dma_handle = *(ddi_dma_handle_t *)
9325 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9326 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9327 DDI_DMA_SYNC_FORKERNEL);
9328 /* Copy the request sense data */
9329 bcopy(rs_spkt->
9330 satapkt_cmd.satacmd_bp->b_un.b_addr,
9331 &sata_cmd->satacmd_rqsense,
9332 SATA_ATAPI_MIN_RQSENSE_LEN);
9333 #if AHCI_DEBUG
9334 rqsense = (struct scsi_extended_sense *)
9335 sata_cmd->satacmd_rqsense;
9336
9337 /* Dump the sense data */
9338 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9339 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9340 "Sense data for satapkt %p ATAPI cmd 0x%x",
9341 spkt, sata_cmd->satacmd_acdb[0]);
9342 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9343 " es_code 0x%x es_class 0x%x "
9344 "es_key 0x%x es_add_code 0x%x "
9345 "es_qual_code 0x%x",
9346 rqsense->es_code, rqsense->es_class,
9347 rqsense->es_key, rqsense->es_add_code,
9348 rqsense->es_qual_code);
9349 #endif
9350 }
9351
9352 sata_free_error_retrieval_pkt(rs_spkt);
9353 }
9354
9355 /*
9356 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9357 * the port must be restarted. When the HBA detects thus error, it may try
9358 * to abort a transfer. And if the transfer was aborted, the device is
9359 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9360 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9361 * that the device is in a stable status and transfers may be restarted without
9362 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9363 * then the software will send the COMRESET to do the port reset.
9364 *
9365 * Software should perform the appropriate error recovery actions based on
9366 * whether non-queued commands were being issued or natived command queuing
9367 * commands were being issued.
9368 *
9369 * And software will complete the command that had the error with error mark
9370 * to higher level software.
9371 *
9372 * Fatal errors include the following:
9373 * PxIS.IFS - Interface Fatal Error Status
9374 * PxIS.HBDS - Host Bus Data Error Status
9375 * PxIS.HBFS - Host Bus Fatal Error Status
9376 * PxIS.TFES - Task File Error Status
9377 */
9378 static void
9379 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9380 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9381 {
9382 uint32_t port_cmd_status;
9383 uint32_t slot_status = 0;
9384 uint32_t failed_tags = 0;
9385 int failed_slot;
9386 int reset_flag = 0, flag = 0;
9387 ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9388 sata_cmd_t *sata_cmd = NULL;
9389 sata_pkt_t *spkt = NULL;
9390 #if AHCI_DEBUG
9391 ahci_cmd_header_t *cmd_header;
9392 #endif
9393 uint8_t port = addrp->aa_port;
9394 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9395 int rval;
9396
9397 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9398
9399 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9400 "ahci_fatal_error_recovery_handler enter: port %d", port);
9401
9402 /* Port multiplier error */
9403 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9404 /* FBS code is neither completed nor tested. */
9405 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9406 port, intr_status);
9407
9408 /* Force a port reset */
9409 flag = AHCI_PORT_RESET;
9410 }
9411
9412 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9413 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9414
9415 /* Read PxCI to see which commands are still outstanding */
9416 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9417 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9418
9419 /*
9420 * Read PxCMD.CCS to determine the slot that the HBA
9421 * was processing when the error occurred.
9422 */
9423 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9424 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9425 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9426 AHCI_CMD_STATUS_CCS_SHIFT;
9427
9428 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9429 spkt = ahci_portp->ahciport_err_retri_pkt;
9430 ASSERT(spkt != NULL);
9431 } else {
9432 spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9433 if (spkt == NULL) {
9434 /* May happen when interface errors occur? */
9435 goto next;
9436 }
9437 }
9438
9439 #if AHCI_DEBUG
9440 /*
9441 * Debugging purpose...
9442 */
9443 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9444 cmd_header =
9445 &ahci_portp->ahciport_cmd_list[failed_slot];
9446 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9447 "ahci_fatal_error_recovery_handler: port %d, "
9448 "PRD Byte Count = 0x%x, "
9449 "ahciport_prd_bytecounts = 0x%x", port,
9450 cmd_header->ahcich_prd_byte_count,
9451 ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9452 }
9453 #endif
9454
9455 sata_cmd = &spkt->satapkt_cmd;
9456
9457 /* Fill out the status and error registers for PxIS.TFES */
9458 if (intr_status & AHCI_INTR_STATUS_TFES) {
9459 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9460 ahcirf_d2h_register_fis);
9461
9462 /* Copy the error context back to the sata_cmd */
9463 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9464 }
9465
9466 /* The failed command must be one of the outstanding commands */
9467 failed_tags = 0x1 << failed_slot;
9468 ASSERT(failed_tags & slot_status);
9469
9470 /* Update the sata registers, especially PxSERR register */
9471 ahci_update_sata_registers(ahci_ctlp, port,
9472 &spkt->satapkt_device);
9473
9474 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9475 /* Read PxSACT to see which commands are still outstanding */
9476 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9477 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9478 }
9479 next:
9480
9481 #if AHCI_DEBUG
9482 /*
9483 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9484 * set, it means a fatal error happened after REQUEST SENSE command
9485 * or READ LOG EXT command is delivered to the HBA during the error
9486 * recovery process. At this time, the only outstanding command is
9487 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9488 */
9489 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9490 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9491 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9492 "command or READ LOG EXT command for error data retrieval "
9493 "failed", port);
9494 ASSERT(slot_status == 0x1);
9495 ASSERT(failed_slot == 0);
9496 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9497 SCMD_REQUEST_SENSE ||
9498 spkt->satapkt_cmd.satacmd_cmd_reg ==
9499 SATAC_READ_LOG_EXT);
9500 }
9501 #endif
9502
9503 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9504 ahci_portp->ahciport_mop_in_progress++;
9505
9506 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9507 port, flag, &reset_flag);
9508
9509 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9510 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9511 if (rval == AHCI_SUCCESS)
9512 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9513 "succeed", instance, port);
9514 else
9515 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9516 "failed", instance, port);
9517 }
9518
9519 /*
9520 * Won't retrieve error information:
9521 * 1. Port reset was involved to recover
9522 * 2. Device is gone
9523 * 3. IDENTIFY DEVICE command sent to ATAPI device
9524 * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9525 */
9526 if (reset_flag ||
9527 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9528 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9529 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9530 goto out;
9531
9532 /*
9533 * Deliver READ LOG EXT to gather information about the error when
9534 * a COMRESET has not been performed as part of the error recovery
9535 * during NCQ command processing.
9536 */
9537 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9538 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9539 ahci_portp, port);
9540 goto out;
9541 }
9542
9543 /*
9544 * Deliver REQUEST SENSE for ATAPI command to gather information about
9545 * the error when a COMRESET has not been performed as part of the
9546 * error recovery.
9547 */
9548 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9549 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9550 out:
9551 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9552 "ahci_fatal_error_recovery_handler: port %d fatal error "
9553 "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9554 "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9555 port, slot_status, ahci_portp->ahciport_pending_tags,
9556 ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9557
9558 ahci_mop_commands(ahci_ctlp,
9559 ahci_portp,
9560 slot_status,
9561 failed_tags, /* failed tags */
9562 0, /* timeout tags */
9563 0, /* aborted tags */
9564 0); /* reset tags */
9565 }
9566
9567 /*
9568 * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9569 * 1. device specific.PxFBS.SDE=1
9570 * 2. Non Device specific.
9571 * Nothing will be done when Command-based switching is employed.
9572 *
9573 * Currently code is neither completed nor tested.
9574 */
9575 static void
9576 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9577 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9578 {
9579 #ifndef __lock_lint
9580 _NOTE(ARGUNUSED(intr_status))
9581 #endif
9582 uint32_t port_fbs_ctrl;
9583 int loop_count = 0;
9584 ahci_addr_t addr;
9585
9586 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9587
9588 /* Nothing will be done under Command-based switching. */
9589 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9590 return;
9591
9592 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9593 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9594
9595 if (!(port_fbs_ctrl & AHCI_FBS_EN))
9596 /* FBS is not enabled. */
9597 return;
9598
9599 /* Problem's getting complicated now. */
9600 /*
9601 * If FIS-based switching is used, we need to check
9602 * the PxFBS to see the error type.
9603 */
9604 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9605 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9606
9607 /* Refer to spec(v1.2) 9.3.6.1 */
9608 if (port_fbs_ctrl & AHCI_FBS_SDE) {
9609 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9610 "A Device Sepcific Error: port %d", port);
9611 /*
9612 * Controller has paused commands for all other
9613 * sub-devices until PxFBS.DEC is set.
9614 */
9615 ahci_reject_all_abort_pkts(ahci_ctlp,
9616 ahci_portp, 0);
9617
9618 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9619 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9620 port_fbs_ctrl | AHCI_FBS_DEC);
9621
9622 /*
9623 * Wait controller clear PxFBS.DEC,
9624 * then we can continue.
9625 */
9626 loop_count = 0;
9627 do {
9628 port_fbs_ctrl = ddi_get32(ahci_ctlp->
9629 ahcictl_ahci_acc_handle, (uint32_t *)
9630 AHCI_PORT_PxFBS(ahci_ctlp, port));
9631
9632 if (loop_count++ > 1000)
9633 /*
9634 * Esclate the error. Follow
9635 * non-device specific error
9636 * procedure.
9637 */
9638 return;
9639
9640 drv_usecwait(AHCI_100US_USECS);
9641 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9642
9643 /*
9644 * Issue a software reset to ensure drive is in
9645 * a known state.
9646 */
9647 (void) ahci_software_reset(ahci_ctlp,
9648 ahci_portp, &addr);
9649
9650 } else {
9651
9652 /* Process Non-Device Specific Error. */
9653 /* This will be handled later on. */
9654 cmn_err(CE_NOTE, "!FBS is not supported now.");
9655 }
9656 }
9657 /*
9658 * Handle events - fatal error recovery
9659 */
9660 static void
9661 ahci_events_handler(void *args)
9662 {
9663 ahci_event_arg_t *ahci_event_arg;
9664 ahci_ctl_t *ahci_ctlp;
9665 ahci_port_t *ahci_portp;
9666 ahci_addr_t *addrp;
9667 uint32_t event;
9668 int instance;
9669
9670 ahci_event_arg = (ahci_event_arg_t *)args;
9671
9672 ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9673 ahci_portp = ahci_event_arg->ahciea_portp;
9674 addrp = ahci_event_arg->ahciea_addrp;
9675 event = ahci_event_arg->ahciea_event;
9676 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9677
9678 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9679 "ahci_events_handler enter: port %d intr_status = 0x%x",
9680 ahci_portp->ahciport_port_num, event);
9681
9682 mutex_enter(&ahci_portp->ahciport_mutex);
9683
9684 /*
9685 * ahci_intr_phyrdy_change() may have rendered it to
9686 * SATA_DTYPE_NONE.
9687 */
9688 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9689 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9690 "ahci_events_handler: port %d no device attached, "
9691 "and just return without doing anything",
9692 ahci_portp->ahciport_port_num);
9693
9694 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9695 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9696 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9697 "succeed", instance, ahci_portp->ahciport_port_num);
9698 }
9699
9700 goto out;
9701 }
9702
9703 if (event & (AHCI_INTR_STATUS_IFS |
9704 AHCI_INTR_STATUS_HBDS |
9705 AHCI_INTR_STATUS_HBFS |
9706 AHCI_INTR_STATUS_TFES))
9707 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9708 addrp, event);
9709
9710 out:
9711 mutex_exit(&ahci_portp->ahciport_mutex);
9712 }
9713
9714 /*
9715 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9716 * detect there are some commands which are timed out.
9717 */
9718 static void
9719 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9720 uint8_t port, uint32_t tmp_timeout_tags)
9721 {
9722 uint32_t slot_status = 0;
9723 uint32_t finished_tags = 0;
9724 uint32_t timeout_tags = 0;
9725
9726 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9727 "ahci_timeout_pkts enter: port %d", port);
9728
9729 mutex_enter(&ahci_portp->ahciport_mutex);
9730
9731 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9732 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9733 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9734 /* Read PxCI to see which commands are still outstanding */
9735 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9736 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9737 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9738 /* Read PxSACT to see which commands are still outstanding */
9739 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9740 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9741 }
9742
9743 #if AHCI_DEBUG
9744 /*
9745 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9746 * set, it means a fatal error happened after REQUEST SENSE command
9747 * or READ LOG EXT command is delivered to the HBA during the error
9748 * recovery process. At this time, the only outstanding command is
9749 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9750 */
9751 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9752 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9753 "ahci_timeout_pkts called while REQUEST SENSE "
9754 "command or READ LOG EXT command for error recovery "
9755 "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9756 "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9757 tmp_timeout_tags, slot_status,
9758 ahci_portp->ahciport_pending_tags,
9759 ahci_portp->ahciport_pending_ncq_tags);
9760 ASSERT(slot_status == 0x1);
9761 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9762 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9763 "ahci_timeout_pkts called while executing R/W PMULT "
9764 "command timeout_tags = 0x%x, slot_status = 0x%x",
9765 tmp_timeout_tags, slot_status);
9766 ASSERT(slot_status == 0x1);
9767 }
9768 #endif
9769
9770 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9771 ahci_portp->ahciport_mop_in_progress++;
9772
9773 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9774 port, AHCI_PORT_RESET, NULL);
9775
9776 /*
9777 * Re-identify timeout tags because some previously checked commands
9778 * could already complete.
9779 */
9780 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9781 finished_tags = ahci_portp->ahciport_pending_tags &
9782 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9783 timeout_tags = tmp_timeout_tags & ~finished_tags;
9784
9785 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9786 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9787 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9788 "pending_tags = 0x%x ",
9789 port, finished_tags, timeout_tags,
9790 slot_status, ahci_portp->ahciport_pending_tags);
9791 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9792 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9793 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9794 timeout_tags = tmp_timeout_tags & ~finished_tags;
9795
9796 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9797 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9798 "timeout_tags = 0x%x, port_sactive = 0x%x, "
9799 "pending_ncq_tags = 0x%x ",
9800 port, finished_tags, timeout_tags,
9801 slot_status, ahci_portp->ahciport_pending_ncq_tags);
9802 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9803 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9804 timeout_tags = tmp_timeout_tags;
9805 }
9806
9807 ahci_mop_commands(ahci_ctlp,
9808 ahci_portp,
9809 slot_status,
9810 0, /* failed tags */
9811 timeout_tags, /* timeout tags */
9812 0, /* aborted tags */
9813 0); /* reset tags */
9814
9815 mutex_exit(&ahci_portp->ahciport_mutex);
9816 }
9817
9818 /*
9819 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9820 * for long time.
9821 */
9822 static void
9823 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9824 {
9825 ahci_port_t *ahci_portp;
9826 sata_pkt_t *spkt;
9827 uint32_t pending_tags;
9828 uint32_t timeout_tags;
9829 uint32_t port_cmd_status;
9830 uint32_t port_sactive;
9831 uint8_t port;
9832 int tmp_slot;
9833 int current_slot;
9834 uint32_t current_tags;
9835 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9836
9837 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9838
9839 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9840 "ahci_watchdog_handler entered", NULL);
9841
9842 current_slot = 0;
9843 current_tags = 0;
9844 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9845 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9846 continue;
9847 }
9848
9849 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9850
9851 mutex_enter(&ahci_portp->ahciport_mutex);
9852 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9853 mutex_exit(&ahci_portp->ahciport_mutex);
9854 continue;
9855 }
9856
9857 /* Skip the check for those ports in error recovery */
9858 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9859 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9860 mutex_exit(&ahci_portp->ahciport_mutex);
9861 continue;
9862 }
9863
9864 pending_tags = 0;
9865 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9866 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9867
9868 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9869 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9870 current_slot = 0;
9871 pending_tags = 0x1;
9872 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9873 current_slot =
9874 (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9875 AHCI_CMD_STATUS_CCS_SHIFT;
9876 pending_tags = ahci_portp->ahciport_pending_tags;
9877 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9878 port_sactive = ddi_get32(
9879 ahci_ctlp->ahcictl_ahci_acc_handle,
9880 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9881 current_tags = port_sactive &
9882 ~port_cmd_status &
9883 AHCI_NCQ_SLOT_MASK(ahci_portp);
9884 pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9885 }
9886
9887 timeout_tags = 0;
9888 while (pending_tags) {
9889 tmp_slot = ddi_ffs(pending_tags) - 1;
9890 if (tmp_slot == -1) {
9891 break;
9892 }
9893
9894 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9895 spkt = ahci_portp->ahciport_err_retri_pkt;
9896 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9897 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9898 else
9899 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9900
9901 if ((spkt != NULL) && spkt->satapkt_time &&
9902 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9903 /*
9904 * If a packet has survived for more than it's
9905 * max life cycles, it is a candidate for time
9906 * out.
9907 */
9908 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9909 ahci_watchdog_timeout;
9910
9911 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9912 > 0)
9913 goto next;
9914
9915 #if AHCI_DEBUG
9916 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9917 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9918 ahci_ctlp, "watchdog: the current "
9919 "tags is 0x%x", current_tags);
9920 } else {
9921 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9922 ahci_ctlp, "watchdog: the current "
9923 "slot is %d", current_slot);
9924 }
9925 #endif
9926
9927 /*
9928 * We need to check whether the HBA has
9929 * begun to execute the command, if not,
9930 * then re-set the timer of the command.
9931 */
9932 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9933 (tmp_slot != current_slot) ||
9934 NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9935 ((0x1 << tmp_slot) & current_tags)) {
9936 ahci_portp->ahciport_slot_timeout \
9937 [tmp_slot] = spkt->satapkt_time;
9938 } else {
9939 timeout_tags |= (0x1 << tmp_slot);
9940 cmn_err(CE_WARN, "!ahci%d: watchdog "
9941 "port %d satapkt 0x%p timed out\n",
9942 instance, port, (void *)spkt);
9943 }
9944 }
9945 next:
9946 CLEAR_BIT(pending_tags, tmp_slot);
9947 }
9948
9949 if (timeout_tags) {
9950 mutex_exit(&ahci_portp->ahciport_mutex);
9951 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9952 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9953 port, timeout_tags);
9954 mutex_enter(&ahci_ctlp->ahcictl_mutex);
9955 mutex_enter(&ahci_portp->ahciport_mutex);
9956 }
9957
9958 mutex_exit(&ahci_portp->ahciport_mutex);
9959 }
9960
9961 /* Re-install the watchdog timeout handler */
9962 if (ahci_ctlp->ahcictl_timeout_id != 0) {
9963 ahci_ctlp->ahcictl_timeout_id =
9964 timeout((void (*)(void *))ahci_watchdog_handler,
9965 (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9966 }
9967
9968 mutex_exit(&ahci_ctlp->ahcictl_mutex);
9969 }
9970
9971 /*
9972 * Fill the error context into sata_cmd for non-queued command error.
9973 */
9974 static void
9975 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9976 {
9977 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9978 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9979 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9980 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9981 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9982 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9983 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9984
9985 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9986 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9987 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9988 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9989 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9990 }
9991 }
9992
9993 /*
9994 * Fill the ncq error page into sata_cmd for queued command error.
9995 */
9996 static void
9997 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9998 struct sata_ncq_error_recovery_page *ncq_err_page)
9999 {
10000 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
10001 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
10002 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
10003 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
10004 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
10005 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
10006 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
10007 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
10008 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
10009 scmd->satacmd_status_reg = ncq_err_page->ncq_status;
10010 scmd->satacmd_error_reg = ncq_err_page->ncq_error;
10011 }
10012
10013 /*
10014 * Put the respective register value to sata_cmd_t for satacmd_flags.
10015 */
10016 static void
10017 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
10018 {
10019 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
10020 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
10021 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
10022 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
10023 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
10024 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
10025 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
10026 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
10027 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
10028 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
10029 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
10030 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
10031 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
10032 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
10033 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
10034 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
10035 if (scmd->satacmd_flags.sata_copy_out_device_reg)
10036 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
10037 if (scmd->satacmd_flags.sata_copy_out_error_reg)
10038 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
10039 }
10040
10041 static void
10042 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10043 uint32_t intr_status)
10044 {
10045 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
10046
10047 if (intr_status & AHCI_INTR_STATUS_IFS)
10048 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
10049 "error", instance, port);
10050
10051 if (intr_status & AHCI_INTR_STATUS_HBDS)
10052 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
10053 instance, port);
10054
10055 if (intr_status & AHCI_INTR_STATUS_HBFS)
10056 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
10057 instance, port);
10058
10059 if (intr_status & AHCI_INTR_STATUS_TFES)
10060 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
10061 instance, port);
10062
10063 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
10064 "recovery", instance, port);
10065 }
10066
10067 static void
10068 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
10069 uint32_t slot_tags)
10070 {
10071 ahci_port_t *ahci_portp;
10072 int tmp_slot;
10073 sata_pkt_t *spkt;
10074 sata_cmd_t cmd;
10075
10076 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10077 ASSERT(ahci_portp != NULL);
10078
10079 while (slot_tags) {
10080 tmp_slot = ddi_ffs(slot_tags) - 1;
10081 if (tmp_slot == -1) {
10082 break;
10083 }
10084
10085 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
10086 if (spkt != NULL) {
10087 cmd = spkt->satapkt_cmd;
10088
10089 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
10090 "features_reg = 0x%x sec_count_msb = 0x%x "
10091 "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
10092 "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
10093 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
10094 "lba_high_lsb = 0x%x device_reg = 0x%x "
10095 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10096 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10097 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10098 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10099 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10100 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10101 cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10102 *((uint32_t *)&(cmd.satacmd_flags)));
10103 }
10104
10105 CLEAR_BIT(slot_tags, tmp_slot);
10106 }
10107 }
10108
10109 /*
10110 * Dump the serror message to the log.
10111 */
10112 static void
10113 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10114 uint32_t port_serror, int debug_only)
10115 {
10116 static char err_buf[512];
10117 static char err_msg_header[16];
10118 char *err_msg = err_buf;
10119
10120 *err_buf = '\0';
10121 *err_msg_header = '\0';
10122
10123 if (port_serror & SERROR_DATA_ERR_FIXED) {
10124 err_msg = strcat(err_msg,
10125 "\tRecovered Data Integrity Error (I)\n");
10126 }
10127
10128 if (port_serror & SERROR_COMM_ERR_FIXED) {
10129 err_msg = strcat(err_msg,
10130 "\tRecovered Communication Error (M)\n");
10131 }
10132
10133 if (port_serror & SERROR_DATA_ERR) {
10134 err_msg = strcat(err_msg,
10135 "\tTransient Data Integrity Error (T)\n");
10136 }
10137
10138 if (port_serror & SERROR_PERSISTENT_ERR) {
10139 err_msg = strcat(err_msg,
10140 "\tPersistent Communication or Data Integrity Error (C)\n");
10141 }
10142
10143 if (port_serror & SERROR_PROTOCOL_ERR) {
10144 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10145 }
10146
10147 if (port_serror & SERROR_INT_ERR) {
10148 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10149 }
10150
10151 if (port_serror & SERROR_PHY_RDY_CHG) {
10152 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10153 }
10154
10155 if (port_serror & SERROR_PHY_INT_ERR) {
10156 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10157 }
10158
10159 if (port_serror & SERROR_COMM_WAKE) {
10160 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10161 }
10162
10163 if (port_serror & SERROR_10B_TO_8B_ERR) {
10164 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10165 }
10166
10167 if (port_serror & SERROR_DISPARITY_ERR) {
10168 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10169 }
10170
10171 if (port_serror & SERROR_CRC_ERR) {
10172 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10173 }
10174
10175 if (port_serror & SERROR_HANDSHAKE_ERR) {
10176 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10177 }
10178
10179 if (port_serror & SERROR_LINK_SEQ_ERR) {
10180 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10181 }
10182
10183 if (port_serror & SERROR_TRANS_ERR) {
10184 err_msg = strcat(err_msg,
10185 "\tTransport state transition error (T)\n");
10186 }
10187
10188 if (port_serror & SERROR_FIS_TYPE) {
10189 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10190 }
10191
10192 if (port_serror & SERROR_EXCHANGED_ERR) {
10193 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10194 }
10195
10196 if (*err_msg == '\0')
10197 return;
10198
10199 if (debug_only) {
10200 (void) sprintf(err_msg_header, "port %d", port);
10201 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10202 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10203 } else if (ahci_ctlp) {
10204 cmn_err(CE_WARN, "!ahci%d: %s %s",
10205 ddi_get_instance(ahci_ctlp->ahcictl_dip),
10206 err_msg_header, err_msg);
10207
10208 /* sata trace debug */
10209 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10210 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10211 err_msg_header, err_msg);
10212 } else {
10213 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10214
10215 /* sata trace debug */
10216 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10217 }
10218 }
10219
10220 /*
10221 * Translate the sata_address_t type into the ahci_addr_t type.
10222 * sata_device.satadev_addr structure is used as source.
10223 */
10224 static void
10225 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10226 ahci_addr_t *ahci_addrp)
10227 {
10228 sata_address_t *sata_addrp = &sd->satadev_addr;
10229 ahci_addrp->aa_port =
10230 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10231 ahci_addrp->aa_pmport = sata_addrp->pmport;
10232
10233 switch (sata_addrp->qual) {
10234 case SATA_ADDR_DCPORT:
10235 case SATA_ADDR_CPORT:
10236 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10237 break;
10238 case SATA_ADDR_PMULT:
10239 case SATA_ADDR_PMULT_SPEC:
10240 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10241 break;
10242 case SATA_ADDR_DPMPORT:
10243 case SATA_ADDR_PMPORT:
10244 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10245 break;
10246 case SATA_ADDR_NULL:
10247 default:
10248 /* something went wrong */
10249 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10250 break;
10251 }
10252 }
10253
10254 /*
10255 * This routine is to calculate the total number of ports implemented
10256 * by the HBA.
10257 */
10258 static int
10259 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10260 {
10261 uint8_t i;
10262 int num = 0;
10263
10264 for (i = 0; i < AHCI_MAX_PORTS; i++) {
10265 if (((uint32_t)0x1 << i) & ports_implemented)
10266 num++;
10267 }
10268
10269 return (num);
10270 }
10271
10272 #if AHCI_DEBUG
10273 static void
10274 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10275 {
10276 static char name[16];
10277 va_list ap;
10278
10279 mutex_enter(&ahci_log_mutex);
10280
10281 va_start(ap, fmt);
10282 if (ahci_ctlp) {
10283 (void) sprintf(name, "ahci%d: ",
10284 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10285 } else {
10286 (void) sprintf(name, "ahci: ");
10287 }
10288
10289 (void) vsprintf(ahci_log_buf, fmt, ap);
10290 va_end(ap);
10291
10292 cmn_err(level, "%s%s", name, ahci_log_buf);
10293
10294 mutex_exit(&ahci_log_mutex);
10295 }
10296 #endif
10297
10298 /*
10299 * quiesce(9E) entry point.
10300 *
10301 * This function is called when the system is single-threaded at high
10302 * PIL with preemption disabled. Therefore, this function must not be
10303 * blocked. Because no taskqs are running, there is no need for us to
10304 * take any action for enclosure services which are running in the
10305 * taskq context, especially as no interrupts are generated by it nor
10306 * are any messages expected to come in.
10307 *
10308 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10309 * DDI_FAILURE indicates an error condition and should almost never happen.
10310 */
10311 static int
10312 ahci_quiesce(dev_info_t *dip)
10313 {
10314 ahci_ctl_t *ahci_ctlp;
10315 ahci_port_t *ahci_portp;
10316 int instance, port;
10317
10318 instance = ddi_get_instance(dip);
10319 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10320
10321 if (ahci_ctlp == NULL)
10322 return (DDI_FAILURE);
10323
10324 #if AHCI_DEBUG
10325 ahci_debug_flags = 0;
10326 #endif
10327
10328 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10329
10330 /* disable all the interrupts. */
10331 ahci_disable_all_intrs(ahci_ctlp);
10332
10333 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10334 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10335 continue;
10336 }
10337
10338 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10339
10340 /*
10341 * Stop the port by clearing PxCMD.ST
10342 *
10343 * Here we must disable the port interrupt because
10344 * ahci_disable_all_intrs only clear GHC.IE, and IS
10345 * register will be still set if PxIE is enabled.
10346 * When ahci shares one IRQ with other drivers, the
10347 * intr handler may claim the intr mistakenly.
10348 */
10349 ahci_disable_port_intrs(ahci_ctlp, port);
10350 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10351 ahci_portp, port);
10352 }
10353
10354 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10355
10356 return (DDI_SUCCESS);
10357 }
10358
10359 /*
10360 * The function will add a sata packet to the done queue.
10361 */
10362 static void
10363 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10364 {
10365 ASSERT(satapkt != NULL);
10366 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10367
10368 /* set the reason for all packets */
10369 satapkt->satapkt_reason = reason;
10370 satapkt->satapkt_hba_driver_private = NULL;
10371
10372 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10373 satapkt->satapkt_comp) {
10374 /*
10375 * only add to queue when mode is not synch and there is
10376 * completion callback
10377 */
10378 *ahci_portp->ahciport_doneqtail = satapkt;
10379 ahci_portp->ahciport_doneqtail =
10380 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10381 ahci_portp->ahciport_doneq_len++;
10382
10383 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10384 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10385 /*
10386 * for sync/non-poll mode, just call cv_broadcast
10387 */
10388 cv_broadcast(&ahci_portp->ahciport_cv);
10389 }
10390
10391 /*
10392 * The function will call completion callback of sata packet on the
10393 * completed queue
10394 */
10395 static void
10396 ahci_flush_doneq(ahci_port_t *ahci_portp)
10397 {
10398 sata_pkt_t *satapkt, *next;
10399
10400 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10401
10402 if (ahci_portp->ahciport_doneq) {
10403 satapkt = ahci_portp->ahciport_doneq;
10404
10405 ahci_portp->ahciport_doneq = NULL;
10406 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10407 ahci_portp->ahciport_doneq_len = 0;
10408
10409 mutex_exit(&ahci_portp->ahciport_mutex);
10410
10411 while (satapkt != NULL) {
10412 next = satapkt->satapkt_hba_driver_private;
10413 satapkt->satapkt_hba_driver_private = NULL;
10414
10415 /* Call the callback */
10416 (*satapkt->satapkt_comp)(satapkt);
10417
10418 satapkt = next;
10419 }
10420
10421 mutex_enter(&ahci_portp->ahciport_mutex);
10422 }
10423 }
10424
10425 /*
10426 * Sets the state for the specified port on the controller to desired state.
10427 * This must be run in the context of the enclosure taskq which ensures that
10428 * only one event is outstanding at any time.
10429 */
10430 static boolean_t
10431 ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire)
10432 {
10433 ahci_em_led_msg_t msg;
10434 ahci_em_msg_hdr_t hdr;
10435 uint32_t msgval, hdrval;
10436 uint_t i, max_delay = ahci_em_tx_delay_count;
10437
10438 msg.alm_hba = port;
10439 msg.alm_pminfo = 0;
10440 msg.alm_value = 0;
10441
10442 if (desire & AHCI_EM_LED_IDENT_ENABLE) {
10443 msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF;
10444 }
10445
10446 if (desire & AHCI_EM_LED_FAULT_ENABLE) {
10447 msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF;
10448 }
10449
10450 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 &&
10451 (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) {
10452 msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF;
10453 }
10454
10455 hdr.aemh_rsvd = 0;
10456 hdr.aemh_mlen = sizeof (ahci_em_led_msg_t);
10457 hdr.aemh_dlen = 0;
10458 hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED;
10459
10460 bcopy(&msg, &msgval, sizeof (msgval));
10461 bcopy(&hdr, &hdrval, sizeof (hdrval));
10462
10463 /*
10464 * First, make sure we can transmit. We should not have been placed in a
10465 * situation where an outstanding transmission is going on.
10466 */
10467 for (i = 0; i < max_delay; i++) {
10468 uint32_t val;
10469
10470 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10471 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10472 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10473 break;
10474
10475 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10476 }
10477
10478 if (i == max_delay)
10479 return (B_FALSE);
10480
10481 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10482 (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval);
10483 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10484 (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval);
10485 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10486 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM);
10487
10488 for (i = 0; i < max_delay; i++) {
10489 uint32_t val;
10490
10491 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10492 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10493 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10494 break;
10495
10496 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10497 }
10498
10499 if (i == max_delay)
10500 return (B_FALSE);
10501
10502 return (B_TRUE);
10503 }
10504
10505 typedef struct ahci_em_led_task_arg {
10506 ahci_ctl_t *aelta_ctl;
10507 uint8_t aelta_port;
10508 uint_t aelta_op;
10509 ahci_em_led_state_t aelta_state;
10510 uint_t aelta_ret;
10511 kcondvar_t aelta_cv;
10512 uint_t aelta_ref;
10513 } ahci_em_led_task_arg_t;
10514
10515 static void
10516 ahci_em_led_task_free(ahci_em_led_task_arg_t *task)
10517 {
10518 ASSERT3U(task->aelta_ref, ==, 0);
10519 cv_destroy(&task->aelta_cv);
10520 kmem_free(task, sizeof (*task));
10521 }
10522
10523 static void
10524 ahci_em_led_task(void *arg)
10525 {
10526 boolean_t ret, cleanup = B_FALSE;
10527 ahci_em_led_task_arg_t *led = arg;
10528 ahci_em_led_state_t state;
10529
10530 mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10531 if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) {
10532 led->aelta_ret = EIO;
10533 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10534 return;
10535 }
10536
10537 state = led->aelta_ctl->ahcictl_em_state[led->aelta_port];
10538 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10539
10540 switch (led->aelta_op) {
10541 case AHCI_EM_IOC_SET_OP_ADD:
10542 state |= led->aelta_state;
10543 break;
10544 case AHCI_EM_IOC_SET_OP_REM:
10545 state &= ~led->aelta_state;
10546 break;
10547 case AHCI_EM_IOC_SET_OP_SET:
10548 state = led->aelta_state;
10549 break;
10550 default:
10551 led->aelta_ret = ENOTSUP;
10552 return;
10553 }
10554
10555 ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state);
10556
10557 mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10558 if (ret) {
10559 led->aelta_ctl->ahcictl_em_state[led->aelta_port] = state;
10560 led->aelta_ret = 0;
10561 } else {
10562 led->aelta_ret = EIO;
10563 led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10564 }
10565 led->aelta_ref--;
10566 if (led->aelta_ref > 0) {
10567 cv_signal(&led->aelta_cv);
10568 } else {
10569 cleanup = B_TRUE;
10570 }
10571 mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10572
10573 if (cleanup) {
10574 ahci_em_led_task_free(led);
10575 }
10576 }
10577
10578 static void
10579 ahci_em_reset(void *arg)
10580 {
10581 uint_t i, max_delay = ahci_em_reset_delay_count;
10582 ahci_ctl_t *ahci_ctlp = arg;
10583
10584 /*
10585 * We've been asked to reset the device. The caller should have set the
10586 * resetting flag. Make sure that we don't have a request to quiesce.
10587 */
10588 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10589 ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING);
10590 if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) {
10591 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10592 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10593 return;
10594 }
10595 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10596
10597 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10598 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST);
10599 for (i = 0; i < max_delay; i++) {
10600 uint32_t val;
10601
10602 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10603 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10604 if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0)
10605 break;
10606
10607 delay(drv_usectohz(ahci_em_reset_delay_ms * 1000));
10608 }
10609
10610 if (i == max_delay) {
10611 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10612 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10613 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10614 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10615 cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting",
10616 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10617 return;
10618 }
10619
10620 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10621
10622 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i))
10623 continue;
10624
10625 /*
10626 * Try to flush all the LEDs as part of reset. If it fails,
10627 * drive on.
10628 */
10629 if (!ahci_em_set_led(ahci_ctlp, i,
10630 ahci_ctlp->ahcictl_em_state[i])) {
10631 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10632 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10633 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10634 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10635 cmn_err(CE_WARN, "!ahci%d: enclosure timed out "
10636 "setting port %u",
10637 ddi_get_instance(ahci_ctlp->ahcictl_dip), i);
10638 return;
10639 }
10640 }
10641
10642 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10643 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10644 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY;
10645 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10646 }
10647
10648 static boolean_t
10649 ahci_em_init(ahci_ctl_t *ahci_ctlp)
10650 {
10651 char name[128];
10652
10653 /*
10654 * First make sure we actually have enclosure services and if so, that
10655 * we have the hardware support that we care about for this.
10656 */
10657 if (ahci_ctlp->ahcictl_em_loc == 0 ||
10658 (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0)
10659 return (B_TRUE);
10660
10661 /*
10662 * Next, make sure that the buffer is large enough for us. We need two
10663 * dwords or 8 bytes. The location register is stored in dwords.
10664 */
10665 if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) <
10666 AHCI_EM_BUFFER_MIN) {
10667 return (B_TRUE);
10668 }
10669
10670 ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc &
10671 AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4;
10672 ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr;
10673
10674 bzero(ahci_ctlp->ahcictl_em_state,
10675 sizeof (ahci_ctlp->ahcictl_em_state));
10676
10677 (void) snprintf(name, sizeof (name), "ahcti_em_taskq%d",
10678 ddi_get_instance(ahci_ctlp->ahcictl_dip));
10679 if ((ahci_ctlp->ahcictl_em_taskq =
10680 ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1,
10681 TASKQ_DEFAULTPRI, 0)) == NULL) {
10682 cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em "
10683 "services", ddi_get_instance(ahci_ctlp->ahcictl_dip));
10684 return (B_FALSE);
10685 }
10686
10687 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10688 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT | AHCI_EM_RESETTING;
10689 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10690 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10691 ahci_ctlp, DDI_SLEEP);
10692
10693 return (B_TRUE);
10694 }
10695
10696 static int
10697 ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10698 {
10699 int i;
10700 ahci_ioc_em_get_t get;
10701
10702 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10703 return (ENOTSUP);
10704 }
10705
10706 bzero(&get, sizeof (get));
10707 get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented;
10708 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) {
10709 get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY;
10710 }
10711
10712 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10713 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10714 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) {
10715 continue;
10716 }
10717 get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i];
10718 }
10719 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10720
10721 if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0)
10722 return (EFAULT);
10723
10724 return (0);
10725 }
10726
10727 static int
10728 ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10729 {
10730 int ret;
10731 ahci_ioc_em_set_t set;
10732 ahci_em_led_task_arg_t *task;
10733 boolean_t signal, cleanup;
10734
10735 if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0)
10736 return (EFAULT);
10737
10738 if (set.aiems_port > ahci_ctlp->ahcictl_num_ports)
10739 return (EINVAL);
10740
10741 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) {
10742 return (EINVAL);
10743 }
10744
10745 if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE |
10746 AHCI_EM_LED_FAULT_ENABLE |
10747 AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) {
10748 return (EINVAL);
10749 }
10750
10751 switch (set.aiems_op) {
10752 case AHCI_EM_IOC_SET_OP_ADD:
10753 case AHCI_EM_IOC_SET_OP_REM:
10754 case AHCI_EM_IOC_SET_OP_SET:
10755 break;
10756 default:
10757 return (EINVAL);
10758 }
10759
10760 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10761 return (ENOTSUP);
10762 }
10763
10764 if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 &&
10765 ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) {
10766 return (ENOTSUP);
10767 }
10768
10769 task = kmem_alloc(sizeof (*task), KM_NOSLEEP_LAZY);
10770 if (task == NULL) {
10771 return (ENOMEM);
10772 }
10773
10774 task->aelta_ctl = ahci_ctlp;
10775 task->aelta_port = (uint8_t)set.aiems_port;
10776 task->aelta_op = set.aiems_op;
10777 task->aelta_state = set.aiems_leds;
10778
10779 cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL);
10780
10781 /*
10782 * Initialize the reference count to two. One for us and one for the
10783 * taskq. This will be used in case we get canceled.
10784 */
10785 task->aelta_ref = 2;
10786
10787 /*
10788 * Once dispatched, the task state is protected by our global mutex.
10789 */
10790 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq,
10791 ahci_em_led_task, task, DDI_SLEEP);
10792
10793 signal = B_FALSE;
10794 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10795 while (task->aelta_ref > 1) {
10796 if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) ==
10797 0) {
10798 signal = B_TRUE;
10799 break;
10800 }
10801 }
10802
10803 /*
10804 * Remove our reference count. If we were woken up because of a signal
10805 * then the taskq may still be dispatched. In which case we shouldn't
10806 * free this memory until it is done. In that case, the taskq will take
10807 * care of it.
10808 */
10809 task->aelta_ref--;
10810 cleanup = (task->aelta_ref == 0);
10811 if (signal) {
10812 ret = EINTR;
10813 } else {
10814 ret = task->aelta_ret;
10815 }
10816 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10817
10818 if (cleanup) {
10819 ahci_em_led_task_free(task);
10820 }
10821
10822 return (ret);
10823 }
10824
10825 static int
10826 ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg)
10827 {
10828 int inst;
10829 ahci_ctl_t *ahci_ctlp;
10830
10831 inst = ddi_get_instance(dip);
10832 if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) {
10833 return (ENXIO);
10834 }
10835
10836 switch (cmd) {
10837 case AHCI_EM_IOC_GET:
10838 return (ahci_em_ioctl_get(ahci_ctlp, arg));
10839 case AHCI_EM_IOC_SET:
10840 return (ahci_em_ioctl_set(ahci_ctlp, arg));
10841 default:
10842 return (ENOTTY);
10843 }
10844
10845 }
10846
10847 static void
10848 ahci_em_quiesce(ahci_ctl_t *ahci_ctlp)
10849 {
10850 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10851 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10852 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10853 return;
10854 }
10855 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE;
10856 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10857
10858 ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq);
10859 }
10860
10861 static void
10862 ahci_em_suspend(ahci_ctl_t *ahci_ctlp)
10863 {
10864 ahci_em_quiesce(ahci_ctlp);
10865
10866 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10867 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY;
10868 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10869 }
10870
10871 static void
10872 ahci_em_resume(ahci_ctl_t *ahci_ctlp)
10873 {
10874 mutex_enter(&ahci_ctlp->ahcictl_mutex);
10875 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10876 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10877 return;
10878 }
10879 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
10880 mutex_exit(&ahci_ctlp->ahcictl_mutex);
10881
10882 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10883 ahci_ctlp, DDI_SLEEP);
10884 }
10885
10886 static void
10887 ahci_em_fini(ahci_ctl_t *ahci_ctlp)
10888 {
10889 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10890 return;
10891 }
10892
10893 ahci_em_quiesce(ahci_ctlp);
10894 ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq);
10895 ahci_ctlp->ahcictl_em_taskq = NULL;
10896 }