1 | /* $NetBSD: nvmevar.h,v 1.11 2016/11/01 14:39:38 jdolecek Exp $ */ |
2 | /* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ |
3 | |
4 | /* |
5 | * Copyright (c) 2014 David Gwynne <dlg@openbsd.org> |
6 | * |
7 | * Permission to use, copy, modify, and distribute this software for any |
8 | * purpose with or without fee is hereby granted, provided that the above |
9 | * copyright notice and this permission notice appear in all copies. |
10 | * |
11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
14 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
16 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
17 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
18 | */ |
19 | |
20 | #include <sys/bus.h> |
21 | #include <sys/cpu.h> |
22 | #include <sys/device.h> |
23 | #include <sys/mutex.h> |
24 | #include <sys/pool.h> |
25 | #include <sys/queue.h> |
26 | |
27 | struct nvme_dmamem { |
28 | bus_dmamap_t ndm_map; |
29 | bus_dma_segment_t ndm_seg; |
30 | size_t ndm_size; |
31 | void *ndm_kva; |
32 | }; |
33 | #define NVME_DMA_MAP(_ndm) ((_ndm)->ndm_map) |
34 | #define NVME_DMA_LEN(_ndm) ((_ndm)->ndm_map->dm_segs[0].ds_len) |
35 | #define NVME_DMA_DVA(_ndm) ((uint64_t)(_ndm)->ndm_map->dm_segs[0].ds_addr) |
36 | #define NVME_DMA_KVA(_ndm) ((void *)(_ndm)->ndm_kva) |
37 | |
38 | struct nvme_softc; |
39 | struct nvme_queue; |
40 | |
41 | typedef void (*nvme_nnc_done)(void *, struct buf *, uint16_t); |
42 | |
43 | struct nvme_ccb { |
44 | SIMPLEQ_ENTRY(nvme_ccb) ccb_entry; |
45 | |
46 | /* DMA handles */ |
47 | bus_dmamap_t ccb_dmamap; |
48 | |
49 | bus_addr_t ccb_prpl_off; |
50 | uint64_t ccb_prpl_dva; |
51 | uint64_t *ccb_prpl; |
52 | |
53 | /* command context */ |
54 | uint16_t ccb_id; |
55 | void *ccb_cookie; |
56 | #define NVME_CCB_FREE 0xbeefdeed |
57 | void (*ccb_done)(struct nvme_queue *, |
58 | struct nvme_ccb *, struct nvme_cqe *); |
59 | |
60 | /* namespace context */ |
61 | void *nnc_cookie; |
62 | nvme_nnc_done nnc_done; |
63 | uint16_t nnc_nsid; |
64 | uint16_t nnc_flags; |
65 | #define NVME_NS_CTX_F_READ __BIT(0) |
66 | #define NVME_NS_CTX_F_POLL __BIT(1) |
67 | |
68 | struct buf *nnc_buf; |
69 | daddr_t nnc_blkno; |
70 | size_t nnc_datasize; |
71 | int nnc_secsize; |
72 | }; |
73 | |
74 | struct nvme_queue { |
75 | struct nvme_softc *q_sc; |
76 | kmutex_t q_sq_mtx; |
77 | kmutex_t q_cq_mtx; |
78 | struct nvme_dmamem *q_sq_dmamem; |
79 | struct nvme_dmamem *q_cq_dmamem; |
80 | bus_size_t q_sqtdbl; /* submission queue tail doorbell */ |
81 | bus_size_t q_cqhdbl; /* completion queue head doorbell */ |
82 | uint16_t q_id; |
83 | uint32_t q_entries; |
84 | uint32_t q_sq_tail; |
85 | uint32_t q_cq_head; |
86 | uint16_t q_cq_phase; |
87 | |
88 | kmutex_t q_ccb_mtx; |
89 | uint16_t q_nccbs; /* total number of ccbs */ |
90 | uint16_t q_nccbs_avail; /* available ccbs */ |
91 | struct nvme_ccb *q_ccbs; |
92 | SIMPLEQ_HEAD(, nvme_ccb) q_ccb_list; |
93 | struct nvme_dmamem *q_ccb_prpls; |
94 | }; |
95 | |
96 | struct nvme_namespace { |
97 | struct nvm_identify_namespace *ident; |
98 | device_t dev; |
99 | uint32_t flags; |
100 | #define NVME_NS_F_OPEN __BIT(0) |
101 | }; |
102 | |
103 | struct nvme_softc { |
104 | device_t sc_dev; |
105 | |
106 | bus_space_tag_t sc_iot; |
107 | bus_space_handle_t sc_ioh; |
108 | bus_size_t sc_ios; |
109 | bus_dma_tag_t sc_dmat; |
110 | |
111 | int (*sc_intr_establish)(struct nvme_softc *, |
112 | uint16_t qid, struct nvme_queue *); |
113 | int (*sc_intr_disestablish)(struct nvme_softc *, |
114 | uint16_t qid); |
115 | void **sc_ih; /* interrupt handlers */ |
116 | void **sc_softih; /* softintr handlers */ |
117 | |
118 | u_int sc_rdy_to; /* RDY timeout */ |
119 | size_t sc_mps; /* memory page size */ |
120 | size_t sc_mdts; /* max data trasfer size */ |
121 | u_int sc_max_sgl; /* max S/G segments */ |
122 | |
123 | struct nvm_identify_controller |
124 | sc_identify; |
125 | |
126 | u_int sc_nn; /* namespace count */ |
127 | struct nvme_namespace *sc_namespaces; |
128 | |
129 | bool sc_use_mq; |
130 | u_int sc_nq; /* # of io queue (sc_q) */ |
131 | struct nvme_queue *sc_admin_q; |
132 | struct nvme_queue **sc_q; |
133 | |
134 | uint32_t sc_flags; |
135 | #define NVME_F_ATTACHED __BIT(0) |
136 | #define NVME_F_OPEN __BIT(1) |
137 | }; |
138 | |
139 | #define lemtoh16(p) le16toh(*((uint16_t *)(p))) |
140 | #define lemtoh32(p) le32toh(*((uint32_t *)(p))) |
141 | #define lemtoh64(p) le64toh(*((uint64_t *)(p))) |
142 | #define htolem16(p, x) (*((uint16_t *)(p)) = htole16(x)) |
143 | #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) |
144 | #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) |
145 | |
146 | struct nvme_attach_args { |
147 | uint16_t naa_nsid; |
148 | uint32_t naa_qentries; /* total number of queue slots */ |
149 | uint32_t naa_maxphys; /* maximum device transfer size */ |
150 | }; |
151 | |
152 | int nvme_attach(struct nvme_softc *); |
153 | int nvme_detach(struct nvme_softc *, int flags); |
154 | int nvme_rescan(device_t, const char *, const int *); |
155 | void nvme_childdet(device_t, device_t); |
156 | int nvme_intr(void *); |
157 | void nvme_softintr_intx(void *); |
158 | int nvme_intr_msi(void *); |
159 | void nvme_softintr_msi(void *); |
160 | |
161 | static inline struct nvme_queue * |
162 | nvme_get_q(struct nvme_softc *sc) |
163 | { |
164 | return sc->sc_q[cpu_index(curcpu()) % sc->sc_nq]; |
165 | } |
166 | |
167 | /* |
168 | * namespace |
169 | */ |
170 | static inline struct nvme_namespace * |
171 | nvme_ns_get(struct nvme_softc *sc, uint16_t nsid) |
172 | { |
173 | if (nsid == 0 || nsid - 1 >= sc->sc_nn) |
174 | return NULL; |
175 | return &sc->sc_namespaces[nsid - 1]; |
176 | } |
177 | |
178 | int nvme_ns_identify(struct nvme_softc *, uint16_t); |
179 | void nvme_ns_free(struct nvme_softc *, uint16_t); |
180 | int nvme_ns_dobio(struct nvme_softc *, uint16_t, void *, |
181 | struct buf *, void *, size_t, int, daddr_t, int, nvme_nnc_done); |
182 | int nvme_ns_sync(struct nvme_softc *, uint16_t, void *, int, nvme_nnc_done); |
183 | |