1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2018 Nexenta Systems, Inc.
14 * Copyright 2016 The MathWorks, Inc. All rights reserved.
15 * Copyright 2017 Joyent, Inc.
16 */
17
18 #ifndef _NVME_VAR_H
19 #define _NVME_VAR_H
20
21 #include <sys/ddi.h>
22 #include <sys/sunddi.h>
23 #include <sys/blkdev.h>
24 #include <sys/taskq_impl.h>
25 #include <sys/list.h>
26
27 /*
28 * NVMe driver state
29 */
30
31 #ifdef __cplusplus
32 extern "C" {
33 #endif
34
35 #define NVME_FMA_INIT 0x1
36 #define NVME_REGS_MAPPED 0x2
37 #define NVME_ADMIN_QUEUE 0x4
38 #define NVME_CTRL_LIMITS 0x8
39 #define NVME_INTERRUPTS 0x10
40
41 #define NVME_MIN_ADMIN_QUEUE_LEN 16
42 #define NVME_MIN_IO_QUEUE_LEN 16
43 #define NVME_DEFAULT_ADMIN_QUEUE_LEN 256
44 #define NVME_DEFAULT_IO_QUEUE_LEN 1024
45 #define NVME_DEFAULT_ASYNC_EVENT_LIMIT 10
46 #define NVME_MIN_ASYNC_EVENT_LIMIT 1
47 #define NVME_DEFAULT_MIN_BLOCK_SIZE 512
48
49
50 typedef struct nvme nvme_t;
51 typedef struct nvme_namespace nvme_namespace_t;
52 typedef struct nvme_minor_state nvme_minor_state_t;
53 typedef struct nvme_dma nvme_dma_t;
54 typedef struct nvme_cmd nvme_cmd_t;
55 typedef struct nvme_qpair nvme_qpair_t;
56 typedef struct nvme_task_arg nvme_task_arg_t;
57
58 struct nvme_minor_state {
59 kmutex_t nm_mutex;
60 boolean_t nm_oexcl;
61 uint_t nm_ocnt;
62 };
63
64 struct nvme_dma {
65 ddi_dma_handle_t nd_dmah;
66 ddi_acc_handle_t nd_acch;
67 ddi_dma_cookie_t nd_cookie;
68 uint_t nd_ncookie;
69 caddr_t nd_memp;
70 size_t nd_len;
71 boolean_t nd_cached;
72 };
73
74 struct nvme_cmd {
75 struct list_node nc_list;
76
77 nvme_sqe_t nc_sqe;
78 nvme_cqe_t nc_cqe;
79
80 void (*nc_callback)(void *);
81 bd_xfer_t *nc_xfer;
82 boolean_t nc_completed;
83 boolean_t nc_dontpanic;
84 uint16_t nc_sqid;
85
86 nvme_dma_t *nc_dma;
87
88 kmutex_t nc_mutex;
89 kcondvar_t nc_cv;
90
91 taskq_ent_t nc_tqent;
92 nvme_t *nc_nvme;
93 };
94
95 struct nvme_qpair {
96 size_t nq_nentry;
97
98 nvme_dma_t *nq_sqdma;
99 nvme_sqe_t *nq_sq;
100 uint_t nq_sqhead;
101 uint_t nq_sqtail;
102 uintptr_t nq_sqtdbl;
103
104 nvme_dma_t *nq_cqdma;
105 nvme_cqe_t *nq_cq;
106 uint_t nq_cqhead;
107 uint_t nq_cqtail;
108 uintptr_t nq_cqhdbl;
109
110 nvme_cmd_t **nq_cmd;
111 uint16_t nq_next_cmd;
112 uint_t nq_active_cmds;
113 int nq_phase;
114
115 kmutex_t nq_mutex;
116 ksema_t nq_sema;
117 };
118
119 struct nvme {
120 dev_info_t *n_dip;
121 int n_progress;
122
123 caddr_t n_regs;
124 ddi_acc_handle_t n_regh;
125
126 kmem_cache_t *n_cmd_cache;
127 kmem_cache_t *n_prp_cache;
128
129 size_t n_inth_sz;
130 ddi_intr_handle_t *n_inth;
131 int n_intr_cnt;
132 uint_t n_intr_pri;
133 int n_intr_cap;
134 int n_intr_type;
135 int n_intr_types;
136
137 char *n_product;
138 char *n_vendor;
139
140 nvme_version_t n_version;
141 boolean_t n_dead;
142 boolean_t n_strict_version;
143 boolean_t n_ignore_unknown_vendor_status;
144 uint32_t n_admin_queue_len;
145 uint32_t n_io_queue_len;
146 uint16_t n_async_event_limit;
147 uint_t n_min_block_size;
148 uint16_t n_abort_command_limit;
149 uint64_t n_max_data_transfer_size;
150 boolean_t n_write_cache_present;
151 boolean_t n_write_cache_enabled;
152 int n_error_log_len;
153 boolean_t n_lba_range_supported;
154 boolean_t n_auto_pst_supported;
155 boolean_t n_async_event_supported;
156 boolean_t n_progress_supported;
157
158 int n_nssr_supported;
159 int n_doorbell_stride;
160 int n_timeout;
161 int n_arbitration_mechanisms;
162 int n_cont_queues_reqd;
163 int n_max_queue_entries;
164 int n_pageshift;
165 int n_pagesize;
166
167 int n_namespace_count;
168 uint16_t n_ioq_count;
169
170 nvme_identify_ctrl_t *n_idctl;
171
172 nvme_qpair_t *n_adminq;
173 nvme_qpair_t **n_ioq;
174
175 nvme_namespace_t *n_ns;
176
177 ddi_dma_attr_t n_queue_dma_attr;
178 ddi_dma_attr_t n_prp_dma_attr;
179 ddi_dma_attr_t n_sgl_dma_attr;
180 ddi_device_acc_attr_t n_reg_acc_attr;
181 ddi_iblock_cookie_t n_fm_ibc;
182 int n_fm_cap;
183
184 ksema_t n_abort_sema;
185
186 ddi_taskq_t *n_cmd_taskq;
187
188 /* state for devctl minor node */
189 nvme_minor_state_t n_minor;
190
191 /* errors detected by driver */
192 uint32_t n_dma_bind_err;
193 uint32_t n_abort_failed;
194 uint32_t n_cmd_timeout;
195 uint32_t n_cmd_aborted;
196 uint32_t n_wrong_logpage;
197 uint32_t n_unknown_logpage;
198 uint32_t n_too_many_cookies;
199
200 /* errors detected by hardware */
201 uint32_t n_data_xfr_err;
202 uint32_t n_internal_err;
203 uint32_t n_abort_rq_err;
204 uint32_t n_abort_sq_del;
205 uint32_t n_nvm_cap_exc;
206 uint32_t n_nvm_ns_notrdy;
207 uint32_t n_inv_cq_err;
208 uint32_t n_inv_qid_err;
209 uint32_t n_max_qsz_exc;
210 uint32_t n_inv_int_vect;
211 uint32_t n_inv_log_page;
212 uint32_t n_inv_format;
213 uint32_t n_inv_q_del;
214 uint32_t n_cnfl_attr;
215 uint32_t n_inv_prot;
216 uint32_t n_readonly;
217
218 /* errors reported by asynchronous events */
219 uint32_t n_diagfail_event;
220 uint32_t n_persistent_event;
221 uint32_t n_transient_event;
222 uint32_t n_fw_load_event;
223 uint32_t n_reliability_event;
224 uint32_t n_temperature_event;
225 uint32_t n_spare_event;
226 uint32_t n_vendor_event;
227 uint32_t n_unknown_event;
228
229 };
230
231 struct nvme_namespace {
232 nvme_t *ns_nvme;
233 uint8_t ns_eui64[8];
234 char ns_name[17];
235
236 bd_handle_t ns_bd_hdl;
237
238 uint32_t ns_id;
239 size_t ns_block_count;
240 size_t ns_block_size;
241 size_t ns_best_block_size;
242
243 boolean_t ns_ignore;
244
245 nvme_identify_nsid_t *ns_idns;
246
247 /* state for attachment point minor node */
248 nvme_minor_state_t ns_minor;
249
250 /*
251 * If a namespace has no EUI64, we create a devid in
252 * nvme_prepare_devid().
253 */
254 char *ns_devid;
255 };
256
257 struct nvme_task_arg {
258 nvme_t *nt_nvme;
259 nvme_cmd_t *nt_cmd;
260 };
261
262
263 #ifdef __cplusplus
264 }
265 #endif
266
267 #endif /* _NVME_VAR_H */