1/* $NetBSD: nvme.c,v 1.22 2016/11/01 14:46:31 jdolecek Exp $ */
2/* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
3
4/*
5 * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
6 *
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 */
19
20#include <sys/cdefs.h>
21__KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.22 2016/11/01 14:46:31 jdolecek Exp $");
22
23#include <sys/param.h>
24#include <sys/systm.h>
25#include <sys/kernel.h>
26#include <sys/atomic.h>
27#include <sys/bus.h>
28#include <sys/buf.h>
29#include <sys/conf.h>
30#include <sys/device.h>
31#include <sys/kmem.h>
32#include <sys/once.h>
33#include <sys/proc.h>
34#include <sys/queue.h>
35#include <sys/mutex.h>
36
37#include <uvm/uvm_extern.h>
38
39#include <dev/ic/nvmereg.h>
40#include <dev/ic/nvmevar.h>
41#include <dev/ic/nvmeio.h>
42
43int nvme_adminq_size = 32;
44int nvme_ioq_size = 1024;
45
46static int nvme_print(void *, const char *);
47
48static int nvme_ready(struct nvme_softc *, uint32_t);
49static int nvme_enable(struct nvme_softc *, u_int);
50static int nvme_disable(struct nvme_softc *);
51static int nvme_shutdown(struct nvme_softc *);
52
53static void nvme_version(struct nvme_softc *, uint32_t);
54#ifdef NVME_DEBUG
55static void nvme_dumpregs(struct nvme_softc *);
56#endif
57static int nvme_identify(struct nvme_softc *, u_int);
58static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *,
59 void *);
60
61static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t);
62static void nvme_ccbs_free(struct nvme_queue *);
63
64static struct nvme_ccb *
65 nvme_ccb_get(struct nvme_queue *);
66static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *);
67
68static int nvme_poll(struct nvme_softc *, struct nvme_queue *,
69 struct nvme_ccb *, void (*)(struct nvme_queue *,
70 struct nvme_ccb *, void *), int);
71static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *);
72static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *,
73 struct nvme_cqe *);
74static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *);
75static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *,
76 struct nvme_cqe *);
77
78static struct nvme_queue *
79 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int);
80static int nvme_q_create(struct nvme_softc *, struct nvme_queue *);
81static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *);
82static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *,
83 struct nvme_ccb *, void (*)(struct nvme_queue *,
84 struct nvme_ccb *, void *));
85static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q);
86static void nvme_q_free(struct nvme_softc *, struct nvme_queue *);
87
88static struct nvme_dmamem *
89 nvme_dmamem_alloc(struct nvme_softc *, size_t);
90static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *);
91static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *,
92 int);
93
94static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *,
95 void *);
96static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *,
97 struct nvme_cqe *);
98static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *,
99 void *);
100static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *,
101 struct nvme_cqe *);
102
103static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *,
104 void *);
105static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *,
106 struct nvme_cqe *);
107static int nvme_command_passthrough(struct nvme_softc *,
108 struct nvme_pt_command *, uint16_t, struct lwp *, bool);
109
110#define NVME_TIMO_QOP 5 /* queue create and delete timeout */
111#define NVME_TIMO_IDENT 10 /* probe identify timeout */
112#define NVME_TIMO_PT -1 /* passthrough cmd timeout */
113#define NVME_TIMO_SY 60 /* sync cache timeout */
114
115#define nvme_read4(_s, _r) \
116 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r))
117#define nvme_write4(_s, _r, _v) \
118 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
119#ifdef __LP64__
120#define nvme_read8(_s, _r) \
121 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r))
122#define nvme_write8(_s, _r, _v) \
123 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
124#else /* __LP64__ */
125static inline uint64_t
126nvme_read8(struct nvme_softc *sc, bus_size_t r)
127{
128 uint64_t v;
129 uint32_t *a = (uint32_t *)&v;
130
131#if _BYTE_ORDER == _LITTLE_ENDIAN
132 a[0] = nvme_read4(sc, r);
133 a[1] = nvme_read4(sc, r + 4);
134#else /* _BYTE_ORDER == _LITTLE_ENDIAN */
135 a[1] = nvme_read4(sc, r);
136 a[0] = nvme_read4(sc, r + 4);
137#endif
138
139 return v;
140}
141
142static inline void
143nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v)
144{
145 uint32_t *a = (uint32_t *)&v;
146
147#if _BYTE_ORDER == _LITTLE_ENDIAN
148 nvme_write4(sc, r, a[0]);
149 nvme_write4(sc, r + 4, a[1]);
150#else /* _BYTE_ORDER == _LITTLE_ENDIAN */
151 nvme_write4(sc, r, a[1]);
152 nvme_write4(sc, r + 4, a[0]);
153#endif
154}
155#endif /* __LP64__ */
156#define nvme_barrier(_s, _r, _l, _f) \
157 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f))
158
159static void
160nvme_version(struct nvme_softc *sc, uint32_t ver)
161{
162 const char *v = NULL;
163
164 switch (ver) {
165 case NVME_VS_1_0:
166 v = "1.0";
167 break;
168 case NVME_VS_1_1:
169 v = "1.1";
170 break;
171 case NVME_VS_1_2:
172 v = "1.2";
173 break;
174 default:
175 aprint_error_dev(sc->sc_dev, "unknown version 0x%08x\n", ver);
176 return;
177 }
178
179 aprint_normal_dev(sc->sc_dev, "NVMe %s\n", v);
180}
181
182#ifdef NVME_DEBUG
183static __used void
184nvme_dumpregs(struct nvme_softc *sc)
185{
186 uint64_t r8;
187 uint32_t r4;
188
189#define DEVNAME(_sc) device_xname((_sc)->sc_dev)
190 r8 = nvme_read8(sc, NVME_CAP);
191 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP));
192 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc),
193 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8)));
194 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc),
195 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8)));
196 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8));
197 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8));
198 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8));
199 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8));
200 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8));
201 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8));
202 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8));
203
204 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS));
205
206 r4 = nvme_read4(sc, NVME_CC);
207 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4);
208 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4),
209 (1 << NVME_CC_IOCQES_R(r4)));
210 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4),
211 (1 << NVME_CC_IOSQES_R(r4)));
212 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4));
213 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4));
214 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4),
215 (1 << NVME_CC_MPS_R(r4)));
216 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4));
217 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0);
218
219 r4 = nvme_read4(sc, NVME_CSTS);
220 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4);
221 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY);
222 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS);
223 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK);
224
225 r4 = nvme_read4(sc, NVME_AQA);
226 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4);
227 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4));
228 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4));
229
230 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ));
231 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ));
232#undef DEVNAME
233}
234#endif /* NVME_DEBUG */
235
236static int
237nvme_ready(struct nvme_softc *sc, uint32_t rdy)
238{
239 u_int i = 0;
240 uint32_t cc;
241
242 cc = nvme_read4(sc, NVME_CC);
243 if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) {
244 aprint_error_dev(sc->sc_dev,
245 "controller enabled status expected %d, found to be %d\n",
246 (rdy != 0), ((cc & NVME_CC_EN) != 0));
247 return ENXIO;
248 }
249
250 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) {
251 if (i++ > sc->sc_rdy_to)
252 return ENXIO;
253
254 delay(1000);
255 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ);
256 }
257
258 return 0;
259}
260
261static int
262nvme_enable(struct nvme_softc *sc, u_int mps)
263{
264 uint32_t cc, csts;
265
266 cc = nvme_read4(sc, NVME_CC);
267 csts = nvme_read4(sc, NVME_CSTS);
268
269 if (ISSET(cc, NVME_CC_EN)) {
270 aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n");
271
272 if (ISSET(csts, NVME_CSTS_RDY))
273 return 1;
274
275 goto waitready;
276 }
277
278 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem));
279 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
280 delay(5000);
281 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem));
282 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
283 delay(5000);
284
285 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) |
286 NVME_AQA_ASQS(sc->sc_admin_q->q_entries));
287 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
288 delay(5000);
289
290 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK |
291 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK);
292 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1));
293 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE));
294 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM));
295 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR));
296 SET(cc, NVME_CC_MPS(mps));
297 SET(cc, NVME_CC_EN);
298
299 nvme_write4(sc, NVME_CC, cc);
300 nvme_barrier(sc, 0, sc->sc_ios,
301 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
302 delay(5000);
303
304 waitready:
305 return nvme_ready(sc, NVME_CSTS_RDY);
306}
307
308static int
309nvme_disable(struct nvme_softc *sc)
310{
311 uint32_t cc, csts;
312
313 cc = nvme_read4(sc, NVME_CC);
314 csts = nvme_read4(sc, NVME_CSTS);
315
316 if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY))
317 nvme_ready(sc, NVME_CSTS_RDY);
318
319 CLR(cc, NVME_CC_EN);
320
321 nvme_write4(sc, NVME_CC, cc);
322 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ);
323
324 delay(5000);
325
326 return nvme_ready(sc, 0);
327}
328
329int
330nvme_attach(struct nvme_softc *sc)
331{
332 uint64_t cap;
333 uint32_t reg;
334 u_int dstrd;
335 u_int mps = PAGE_SHIFT;
336 uint16_t adminq_entries = nvme_adminq_size;
337 uint16_t ioq_entries = nvme_ioq_size;
338 int i;
339
340 reg = nvme_read4(sc, NVME_VS);
341 if (reg == 0xffffffff) {
342 aprint_error_dev(sc->sc_dev, "invalid mapping\n");
343 return 1;
344 }
345
346 nvme_version(sc, reg);
347
348 cap = nvme_read8(sc, NVME_CAP);
349 dstrd = NVME_CAP_DSTRD(cap);
350 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) {
351 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u "
352 "is greater than CPU page size %u\n",
353 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT);
354 return 1;
355 }
356 if (NVME_CAP_MPSMAX(cap) < mps)
357 mps = NVME_CAP_MPSMAX(cap);
358 if (ioq_entries > NVME_CAP_MQES(cap))
359 ioq_entries = NVME_CAP_MQES(cap);
360
361 /* set initial values to be used for admin queue during probe */
362 sc->sc_rdy_to = NVME_CAP_TO(cap);
363 sc->sc_mps = 1 << mps;
364 sc->sc_mdts = MAXPHYS;
365 sc->sc_max_sgl = 2;
366
367 if (nvme_disable(sc) != 0) {
368 aprint_error_dev(sc->sc_dev, "unable to disable controller\n");
369 return 1;
370 }
371
372 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd);
373 if (sc->sc_admin_q == NULL) {
374 aprint_error_dev(sc->sc_dev,
375 "unable to allocate admin queue\n");
376 return 1;
377 }
378 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q))
379 goto free_admin_q;
380
381 if (nvme_enable(sc, mps) != 0) {
382 aprint_error_dev(sc->sc_dev, "unable to enable controller\n");
383 goto disestablish_admin_q;
384 }
385
386 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) {
387 aprint_error_dev(sc->sc_dev, "unable to identify controller\n");
388 goto disable;
389 }
390
391 /* we know how big things are now */
392 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps;
393
394 /* reallocate ccbs of admin queue with new max sgl. */
395 nvme_ccbs_free(sc->sc_admin_q);
396 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries);
397
398 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP);
399 if (sc->sc_q == NULL) {
400 aprint_error_dev(sc->sc_dev, "unable to allocate io queue\n");
401 goto disable;
402 }
403 for (i = 0; i < sc->sc_nq; i++) {
404 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd);
405 if (sc->sc_q[i] == NULL) {
406 aprint_error_dev(sc->sc_dev,
407 "unable to allocate io queue\n");
408 goto free_q;
409 }
410 if (nvme_q_create(sc, sc->sc_q[i]) != 0) {
411 aprint_error_dev(sc->sc_dev,
412 "unable to create io queue\n");
413 nvme_q_free(sc, sc->sc_q[i]);
414 goto free_q;
415 }
416 }
417
418 if (!sc->sc_use_mq)
419 nvme_write4(sc, NVME_INTMC, 1);
420
421 /* probe subdevices */
422 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn,
423 KM_SLEEP);
424 if (sc->sc_namespaces == NULL)
425 goto free_q;
426 nvme_rescan(sc->sc_dev, "nvme", &i);
427
428 return 0;
429
430free_q:
431 while (--i >= 0) {
432 nvme_q_delete(sc, sc->sc_q[i]);
433 nvme_q_free(sc, sc->sc_q[i]);
434 }
435disable:
436 nvme_disable(sc);
437disestablish_admin_q:
438 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
439free_admin_q:
440 nvme_q_free(sc, sc->sc_admin_q);
441
442 return 1;
443}
444
445int
446nvme_rescan(device_t self, const char *attr, const int *flags)
447{
448 struct nvme_softc *sc = device_private(self);
449 struct nvme_attach_args naa;
450 uint64_t cap;
451 int ioq_entries = nvme_ioq_size;
452 int i;
453
454 cap = nvme_read8(sc, NVME_CAP);
455 if (ioq_entries > NVME_CAP_MQES(cap))
456 ioq_entries = NVME_CAP_MQES(cap);
457
458 for (i = 0; i < sc->sc_nn; i++) {
459 if (sc->sc_namespaces[i].dev)
460 continue;
461 memset(&naa, 0, sizeof(naa));
462 naa.naa_nsid = i + 1;
463 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq;
464 naa.naa_maxphys = sc->sc_mdts;
465 sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa,
466 nvme_print);
467 }
468 return 0;
469}
470
471static int
472nvme_print(void *aux, const char *pnp)
473{
474 struct nvme_attach_args *naa = aux;
475
476 if (pnp)
477 aprint_normal("at %s", pnp);
478
479 if (naa->naa_nsid > 0)
480 aprint_normal(" nsid %d", naa->naa_nsid);
481
482 return UNCONF;
483}
484
485int
486nvme_detach(struct nvme_softc *sc, int flags)
487{
488 int i, error;
489
490 error = config_detach_children(sc->sc_dev, flags);
491 if (error)
492 return error;
493
494 error = nvme_shutdown(sc);
495 if (error)
496 return error;
497
498 /* from now on we are committed to detach, following will never fail */
499 for (i = 0; i < sc->sc_nq; i++)
500 nvme_q_free(sc, sc->sc_q[i]);
501 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq);
502 nvme_q_free(sc, sc->sc_admin_q);
503
504 return 0;
505}
506
507static int
508nvme_shutdown(struct nvme_softc *sc)
509{
510 uint32_t cc, csts;
511 bool disabled = false;
512 int i;
513
514 if (!sc->sc_use_mq)
515 nvme_write4(sc, NVME_INTMS, 1);
516
517 for (i = 0; i < sc->sc_nq; i++) {
518 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) {
519 aprint_error_dev(sc->sc_dev,
520 "unable to delete io queue %d, disabling\n", i + 1);
521 disabled = true;
522 }
523 }
524 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
525 if (disabled)
526 goto disable;
527
528 cc = nvme_read4(sc, NVME_CC);
529 CLR(cc, NVME_CC_SHN_MASK);
530 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL));
531 nvme_write4(sc, NVME_CC, cc);
532
533 for (i = 0; i < 4000; i++) {
534 nvme_barrier(sc, 0, sc->sc_ios,
535 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
536 csts = nvme_read4(sc, NVME_CSTS);
537 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE)
538 return 0;
539
540 delay(1000);
541 }
542
543 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n");
544
545disable:
546 nvme_disable(sc);
547 return 0;
548}
549
550void
551nvme_childdet(device_t self, device_t child)
552{
553 struct nvme_softc *sc = device_private(self);
554 int i;
555
556 for (i = 0; i < sc->sc_nn; i++) {
557 if (sc->sc_namespaces[i].dev == child) {
558 /* Already freed ns->ident. */
559 sc->sc_namespaces[i].dev = NULL;
560 break;
561 }
562 }
563}
564
565int
566nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid)
567{
568 struct nvme_sqe sqe;
569 struct nvm_identify_namespace *identify;
570 struct nvme_dmamem *mem;
571 struct nvme_ccb *ccb;
572 struct nvme_namespace *ns;
573 int rv;
574
575 KASSERT(nsid > 0);
576
577 ccb = nvme_ccb_get(sc->sc_admin_q);
578 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
579
580 mem = nvme_dmamem_alloc(sc, sizeof(*identify));
581 if (mem == NULL)
582 return ENOMEM;
583
584 memset(&sqe, 0, sizeof(sqe));
585 sqe.opcode = NVM_ADMIN_IDENTIFY;
586 htolem32(&sqe.nsid, nsid);
587 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem));
588 htolem32(&sqe.cdw10, 0);
589
590 ccb->ccb_done = nvme_empty_done;
591 ccb->ccb_cookie = &sqe;
592
593 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
594 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT);
595 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
596
597 nvme_ccb_put(sc->sc_admin_q, ccb);
598
599 if (rv != 0) {
600 rv = EIO;
601 goto done;
602 }
603
604 /* commit */
605
606 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP);
607 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem));
608 //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify));
609
610 ns = nvme_ns_get(sc, nsid);
611 KASSERT(ns);
612 ns->ident = identify;
613
614done:
615 nvme_dmamem_free(sc, mem);
616
617 return rv;
618}
619
620int
621nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie,
622 struct buf *bp, void *data, size_t datasize,
623 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done)
624{
625 struct nvme_queue *q = nvme_get_q(sc);
626 struct nvme_ccb *ccb;
627 bus_dmamap_t dmap;
628 int i, error;
629
630 ccb = nvme_ccb_get(q);
631 if (ccb == NULL)
632 return EAGAIN;
633
634 ccb->ccb_done = nvme_ns_io_done;
635 ccb->ccb_cookie = cookie;
636
637 /* namespace context */
638 ccb->nnc_nsid = nsid;
639 ccb->nnc_flags = flags;
640 ccb->nnc_buf = bp;
641 ccb->nnc_datasize = datasize;
642 ccb->nnc_secsize = secsize;
643 ccb->nnc_blkno = blkno;
644 ccb->nnc_done = nnc_done;
645
646 dmap = ccb->ccb_dmamap;
647 error = bus_dmamap_load(sc->sc_dmat, dmap, data,
648 datasize, NULL,
649 (ISSET(flags, NVME_NS_CTX_F_POLL) ?
650 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) |
651 (ISSET(flags, NVME_NS_CTX_F_READ) ?
652 BUS_DMA_READ : BUS_DMA_WRITE));
653 if (error) {
654 nvme_ccb_put(q, ccb);
655 return error;
656 }
657
658 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
659 ISSET(flags, NVME_NS_CTX_F_READ) ?
660 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
661
662 if (dmap->dm_nsegs > 2) {
663 for (i = 1; i < dmap->dm_nsegs; i++) {
664 htolem64(&ccb->ccb_prpl[i - 1],
665 dmap->dm_segs[i].ds_addr);
666 }
667 bus_dmamap_sync(sc->sc_dmat,
668 NVME_DMA_MAP(q->q_ccb_prpls),
669 ccb->ccb_prpl_off,
670 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
671 BUS_DMASYNC_PREWRITE);
672 }
673
674 if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
675 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0)
676 return EIO;
677 return 0;
678 }
679
680 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill);
681 return 0;
682}
683
684static void
685nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
686{
687 struct nvme_sqe_io *sqe = slot;
688 bus_dmamap_t dmap = ccb->ccb_dmamap;
689
690 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
691 NVM_CMD_READ : NVM_CMD_WRITE;
692 htolem32(&sqe->nsid, ccb->nnc_nsid);
693
694 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
695 switch (dmap->dm_nsegs) {
696 case 1:
697 break;
698 case 2:
699 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
700 break;
701 default:
702 /* the prp list is already set up and synced */
703 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
704 break;
705 }
706
707 htolem64(&sqe->slba, ccb->nnc_blkno);
708
709 /* guaranteed by upper layers, but check just in case */
710 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0);
711 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1);
712}
713
714static void
715nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb,
716 struct nvme_cqe *cqe)
717{
718 struct nvme_softc *sc = q->q_sc;
719 bus_dmamap_t dmap = ccb->ccb_dmamap;
720 void *nnc_cookie = ccb->ccb_cookie;
721 nvme_nnc_done nnc_done = ccb->nnc_done;
722 struct buf *bp = ccb->nnc_buf;
723
724 if (dmap->dm_nsegs > 2) {
725 bus_dmamap_sync(sc->sc_dmat,
726 NVME_DMA_MAP(q->q_ccb_prpls),
727 ccb->ccb_prpl_off,
728 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
729 BUS_DMASYNC_POSTWRITE);
730 }
731
732 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
733 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
734 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
735
736 bus_dmamap_unload(sc->sc_dmat, dmap);
737 nvme_ccb_put(q, ccb);
738
739 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags));
740}
741
742int
743nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie,
744 int flags, nvme_nnc_done nnc_done)
745{
746 struct nvme_queue *q = nvme_get_q(sc);
747 struct nvme_ccb *ccb;
748
749 ccb = nvme_ccb_get(q);
750 if (ccb == NULL)
751 return EAGAIN;
752
753 ccb->ccb_done = nvme_ns_sync_done;
754 ccb->ccb_cookie = cookie;
755
756 /* namespace context */
757 ccb->nnc_nsid = nsid;
758 ccb->nnc_flags = flags;
759 ccb->nnc_done = nnc_done;
760
761 if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
762 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0)
763 return EIO;
764 return 0;
765 }
766
767 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill);
768 return 0;
769}
770
771static void
772nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
773{
774 struct nvme_sqe *sqe = slot;
775
776 sqe->opcode = NVM_CMD_FLUSH;
777 htolem32(&sqe->nsid, ccb->nnc_nsid);
778}
779
780static void
781nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb,
782 struct nvme_cqe *cqe)
783{
784 void *cookie = ccb->ccb_cookie;
785 nvme_nnc_done nnc_done = ccb->nnc_done;
786
787 nvme_ccb_put(q, ccb);
788
789 nnc_done(cookie, NULL, lemtoh16(&cqe->flags));
790}
791
792void
793nvme_ns_free(struct nvme_softc *sc, uint16_t nsid)
794{
795 struct nvme_namespace *ns;
796 struct nvm_identify_namespace *identify;
797
798 ns = nvme_ns_get(sc, nsid);
799 KASSERT(ns);
800
801 identify = ns->ident;
802 ns->ident = NULL;
803 if (identify != NULL)
804 kmem_free(identify, sizeof(*identify));
805}
806
807static void
808nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
809{
810 struct nvme_softc *sc = q->q_sc;
811 struct nvme_sqe *sqe = slot;
812 struct nvme_pt_command *pt = ccb->ccb_cookie;
813 bus_dmamap_t dmap = ccb->ccb_dmamap;
814 int i;
815
816 sqe->opcode = pt->cmd.opcode;
817 htolem32(&sqe->nsid, pt->cmd.nsid);
818
819 if (pt->buf != NULL && pt->len > 0) {
820 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
821 switch (dmap->dm_nsegs) {
822 case 1:
823 break;
824 case 2:
825 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
826 break;
827 default:
828 for (i = 1; i < dmap->dm_nsegs; i++) {
829 htolem64(&ccb->ccb_prpl[i - 1],
830 dmap->dm_segs[i].ds_addr);
831 }
832 bus_dmamap_sync(sc->sc_dmat,
833 NVME_DMA_MAP(q->q_ccb_prpls),
834 ccb->ccb_prpl_off,
835 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
836 BUS_DMASYNC_PREWRITE);
837 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
838 break;
839 }
840 }
841
842 htolem32(&sqe->cdw10, pt->cmd.cdw10);
843 htolem32(&sqe->cdw11, pt->cmd.cdw11);
844 htolem32(&sqe->cdw12, pt->cmd.cdw12);
845 htolem32(&sqe->cdw13, pt->cmd.cdw13);
846 htolem32(&sqe->cdw14, pt->cmd.cdw14);
847 htolem32(&sqe->cdw15, pt->cmd.cdw15);
848}
849
850static void
851nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe)
852{
853 struct nvme_softc *sc = q->q_sc;
854 struct nvme_pt_command *pt = ccb->ccb_cookie;
855 bus_dmamap_t dmap = ccb->ccb_dmamap;
856
857 if (pt->buf != NULL && pt->len > 0) {
858 if (dmap->dm_nsegs > 2) {
859 bus_dmamap_sync(sc->sc_dmat,
860 NVME_DMA_MAP(q->q_ccb_prpls),
861 ccb->ccb_prpl_off,
862 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
863 BUS_DMASYNC_POSTWRITE);
864 }
865
866 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
867 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
868 bus_dmamap_unload(sc->sc_dmat, dmap);
869 }
870
871 pt->cpl.cdw0 = cqe->cdw0;
872 pt->cpl.flags = cqe->flags & ~NVME_CQE_PHASE;
873}
874
875static int
876nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt,
877 uint16_t nsid, struct lwp *l, bool is_adminq)
878{
879 struct nvme_queue *q;
880 struct nvme_ccb *ccb;
881 void *buf = NULL;
882 int error;
883
884 /* limit command size to maximum data transfer size */
885 if ((pt->buf == NULL && pt->len > 0) ||
886 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts)))
887 return EINVAL;
888
889 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc);
890 ccb = nvme_ccb_get(q);
891 if (ccb == NULL)
892 return EBUSY;
893
894 if (pt->buf != NULL) {
895 KASSERT(pt->len > 0);
896 buf = kmem_alloc(pt->len, KM_SLEEP);
897 if (buf == NULL) {
898 error = ENOMEM;
899 goto ccb_put;
900 }
901 if (!pt->is_read) {
902 error = copyin(pt->buf, buf, pt->len);
903 if (error)
904 goto kmem_free;
905 }
906 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf,
907 pt->len, NULL,
908 BUS_DMA_WAITOK |
909 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE));
910 if (error)
911 goto kmem_free;
912 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap,
913 0, ccb->ccb_dmamap->dm_mapsize,
914 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
915 }
916
917 ccb->ccb_done = nvme_pt_done;
918 ccb->ccb_cookie = pt;
919
920 pt->cmd.nsid = nsid;
921 if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) {
922 error = EIO;
923 goto out;
924 }
925
926 error = 0;
927out:
928 if (buf != NULL) {
929 if (error == 0 && pt->is_read)
930 error = copyout(buf, pt->buf, pt->len);
931kmem_free:
932 kmem_free(buf, pt->len);
933 }
934ccb_put:
935 nvme_ccb_put(q, ccb);
936 return error;
937}
938
939static void
940nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
941 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *))
942{
943 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem);
944 uint32_t tail;
945
946 mutex_enter(&q->q_sq_mtx);
947 tail = q->q_sq_tail;
948 if (++q->q_sq_tail >= q->q_entries)
949 q->q_sq_tail = 0;
950
951 sqe += tail;
952
953 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
954 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE);
955 memset(sqe, 0, sizeof(*sqe));
956 (*fill)(q, ccb, sqe);
957 sqe->cid = ccb->ccb_id;
958 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
959 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE);
960
961 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail);
962 mutex_exit(&q->q_sq_mtx);
963}
964
965struct nvme_poll_state {
966 struct nvme_sqe s;
967 struct nvme_cqe c;
968};
969
970static int
971nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
972 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec)
973{
974 struct nvme_poll_state state;
975 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *);
976 void *cookie;
977 uint16_t flags;
978 int step = 10;
979 int maxloop = timo_sec * 1000000 / step;
980 int error = 0;
981
982 memset(&state, 0, sizeof(state));
983 (*fill)(q, ccb, &state.s);
984
985 done = ccb->ccb_done;
986 cookie = ccb->ccb_cookie;
987
988 ccb->ccb_done = nvme_poll_done;
989 ccb->ccb_cookie = &state;
990
991 nvme_q_submit(sc, q, ccb, nvme_poll_fill);
992 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) {
993 if (nvme_q_complete(sc, q) == 0)
994 delay(step);
995
996 if (timo_sec >= 0 && --maxloop <= 0) {
997 error = ETIMEDOUT;
998 break;
999 }
1000 }
1001
1002 ccb->ccb_cookie = cookie;
1003 done(q, ccb, &state.c);
1004
1005 if (error == 0) {
1006 flags = lemtoh16(&state.c.flags);
1007 return flags & ~NVME_CQE_PHASE;
1008 } else {
1009 return 1;
1010 }
1011}
1012
1013static void
1014nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1015{
1016 struct nvme_sqe *sqe = slot;
1017 struct nvme_poll_state *state = ccb->ccb_cookie;
1018
1019 *sqe = state->s;
1020}
1021
1022static void
1023nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1024 struct nvme_cqe *cqe)
1025{
1026 struct nvme_poll_state *state = ccb->ccb_cookie;
1027
1028 SET(cqe->flags, htole16(NVME_CQE_PHASE));
1029 state->c = *cqe;
1030}
1031
1032static void
1033nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1034{
1035 struct nvme_sqe *src = ccb->ccb_cookie;
1036 struct nvme_sqe *dst = slot;
1037
1038 *dst = *src;
1039}
1040
1041static void
1042nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1043 struct nvme_cqe *cqe)
1044{
1045}
1046
1047static int
1048nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q)
1049{
1050 struct nvme_ccb *ccb;
1051 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe;
1052 uint16_t flags;
1053 int rv = 0;
1054
1055 mutex_enter(&q->q_cq_mtx);
1056
1057 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
1058 for (;;) {
1059 cqe = &ring[q->q_cq_head];
1060 flags = lemtoh16(&cqe->flags);
1061 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase)
1062 break;
1063
1064 ccb = &q->q_ccbs[cqe->cid];
1065
1066 if (++q->q_cq_head >= q->q_entries) {
1067 q->q_cq_head = 0;
1068 q->q_cq_phase ^= NVME_CQE_PHASE;
1069 }
1070
1071#ifdef DEBUG
1072 /*
1073 * If we get spurious completion notification, something
1074 * is seriously hosed up. Very likely DMA to some random
1075 * memory place happened, so just bail out.
1076 */
1077 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) {
1078 panic("%s: invalid ccb detected",
1079 device_xname(sc->sc_dev));
1080 /* NOTREACHED */
1081 }
1082#endif
1083
1084 rv++;
1085
1086 /*
1087 * Unlock the mutex before calling the ccb_done callback
1088 * and re-lock afterwards. The callback triggers lddone()
1089 * which schedules another i/o, and also calls nvme_ccb_put().
1090 * Unlock/relock avoids possibility of deadlock.
1091 */
1092 mutex_exit(&q->q_cq_mtx);
1093 ccb->ccb_done(q, ccb, cqe);
1094 mutex_enter(&q->q_cq_mtx);
1095 }
1096 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
1097
1098 if (rv)
1099 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head);
1100
1101 mutex_exit(&q->q_cq_mtx);
1102
1103 if (rv) {
1104 mutex_enter(&q->q_ccb_mtx);
1105 q->q_nccbs_avail += rv;
1106 mutex_exit(&q->q_ccb_mtx);
1107 }
1108
1109 return rv;
1110}
1111
1112static int
1113nvme_identify(struct nvme_softc *sc, u_int mps)
1114{
1115 char sn[41], mn[81], fr[17];
1116 struct nvm_identify_controller *identify;
1117 struct nvme_dmamem *mem;
1118 struct nvme_ccb *ccb;
1119 u_int mdts;
1120 int rv = 1;
1121
1122 ccb = nvme_ccb_get(sc->sc_admin_q);
1123 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
1124
1125 mem = nvme_dmamem_alloc(sc, sizeof(*identify));
1126 if (mem == NULL)
1127 return 1;
1128
1129 ccb->ccb_done = nvme_empty_done;
1130 ccb->ccb_cookie = mem;
1131
1132 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
1133 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify,
1134 NVME_TIMO_IDENT);
1135 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
1136
1137 nvme_ccb_put(sc->sc_admin_q, ccb);
1138
1139 if (rv != 0)
1140 goto done;
1141
1142 identify = NVME_DMA_KVA(mem);
1143
1144 strnvisx(sn, sizeof(sn), (const char *)identify->sn,
1145 sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1146 strnvisx(mn, sizeof(mn), (const char *)identify->mn,
1147 sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1148 strnvisx(fr, sizeof(fr), (const char *)identify->fr,
1149 sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1150 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr,
1151 sn);
1152
1153 if (identify->mdts > 0) {
1154 mdts = (1 << identify->mdts) * (1 << mps);
1155 if (mdts < sc->sc_mdts)
1156 sc->sc_mdts = mdts;
1157 }
1158
1159 sc->sc_nn = lemtoh32(&identify->nn);
1160
1161 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify));
1162
1163done:
1164 nvme_dmamem_free(sc, mem);
1165
1166 return rv;
1167}
1168
1169static int
1170nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q)
1171{
1172 struct nvme_sqe_q sqe;
1173 struct nvme_ccb *ccb;
1174 int rv;
1175
1176 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0)
1177 return 1;
1178
1179 ccb = nvme_ccb_get(sc->sc_admin_q);
1180 KASSERT(ccb != NULL);
1181
1182 ccb->ccb_done = nvme_empty_done;
1183 ccb->ccb_cookie = &sqe;
1184
1185 memset(&sqe, 0, sizeof(sqe));
1186 sqe.opcode = NVM_ADMIN_ADD_IOCQ;
1187 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem));
1188 htolem16(&sqe.qsize, q->q_entries - 1);
1189 htolem16(&sqe.qid, q->q_id);
1190 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC;
1191 if (sc->sc_use_mq)
1192 htolem16(&sqe.cqid, q->q_id); /* qid == vector */
1193
1194 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1195 if (rv != 0)
1196 goto fail;
1197
1198 ccb->ccb_done = nvme_empty_done;
1199 ccb->ccb_cookie = &sqe;
1200
1201 memset(&sqe, 0, sizeof(sqe));
1202 sqe.opcode = NVM_ADMIN_ADD_IOSQ;
1203 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem));
1204 htolem16(&sqe.qsize, q->q_entries - 1);
1205 htolem16(&sqe.qid, q->q_id);
1206 htolem16(&sqe.cqid, q->q_id);
1207 sqe.qflags = NVM_SQE_Q_PC;
1208
1209 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1210 if (rv != 0)
1211 goto fail;
1212
1213fail:
1214 nvme_ccb_put(sc->sc_admin_q, ccb);
1215 return rv;
1216}
1217
1218static int
1219nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q)
1220{
1221 struct nvme_sqe_q sqe;
1222 struct nvme_ccb *ccb;
1223 int rv;
1224
1225 ccb = nvme_ccb_get(sc->sc_admin_q);
1226 KASSERT(ccb != NULL);
1227
1228 ccb->ccb_done = nvme_empty_done;
1229 ccb->ccb_cookie = &sqe;
1230
1231 memset(&sqe, 0, sizeof(sqe));
1232 sqe.opcode = NVM_ADMIN_DEL_IOSQ;
1233 htolem16(&sqe.qid, q->q_id);
1234
1235 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1236 if (rv != 0)
1237 goto fail;
1238
1239 ccb->ccb_done = nvme_empty_done;
1240 ccb->ccb_cookie = &sqe;
1241
1242 memset(&sqe, 0, sizeof(sqe));
1243 sqe.opcode = NVM_ADMIN_DEL_IOCQ;
1244 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem));
1245 htolem16(&sqe.qid, q->q_id);
1246
1247 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1248 if (rv != 0)
1249 goto fail;
1250
1251fail:
1252 nvme_ccb_put(sc->sc_admin_q, ccb);
1253
1254 if (rv == 0 && sc->sc_use_mq) {
1255 if (sc->sc_intr_disestablish(sc, q->q_id))
1256 rv = 1;
1257 }
1258
1259 return rv;
1260}
1261
1262static void
1263nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1264{
1265 struct nvme_sqe *sqe = slot;
1266 struct nvme_dmamem *mem = ccb->ccb_cookie;
1267
1268 sqe->opcode = NVM_ADMIN_IDENTIFY;
1269 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem));
1270 htolem32(&sqe->cdw10, 1);
1271}
1272
1273static int
1274nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs)
1275{
1276 struct nvme_softc *sc = q->q_sc;
1277 struct nvme_ccb *ccb;
1278 bus_addr_t off;
1279 uint64_t *prpl;
1280 u_int i;
1281
1282 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO);
1283 SIMPLEQ_INIT(&q->q_ccb_list);
1284
1285 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP);
1286 if (q->q_ccbs == NULL)
1287 return 1;
1288
1289 q->q_nccbs = nccbs;
1290 q->q_nccbs_avail = nccbs;
1291 q->q_ccb_prpls = nvme_dmamem_alloc(sc,
1292 sizeof(*prpl) * sc->sc_max_sgl * nccbs);
1293
1294 prpl = NVME_DMA_KVA(q->q_ccb_prpls);
1295 off = 0;
1296
1297 for (i = 0; i < nccbs; i++) {
1298 ccb = &q->q_ccbs[i];
1299
1300 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts,
1301 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */,
1302 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
1303 &ccb->ccb_dmamap) != 0)
1304 goto free_maps;
1305
1306 ccb->ccb_id = i;
1307 ccb->ccb_prpl = prpl;
1308 ccb->ccb_prpl_off = off;
1309 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off;
1310
1311 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry);
1312
1313 prpl += sc->sc_max_sgl;
1314 off += sizeof(*prpl) * sc->sc_max_sgl;
1315 }
1316
1317 return 0;
1318
1319free_maps:
1320 nvme_ccbs_free(q);
1321 return 1;
1322}
1323
1324static struct nvme_ccb *
1325nvme_ccb_get(struct nvme_queue *q)
1326{
1327 struct nvme_ccb *ccb = NULL;
1328
1329 mutex_enter(&q->q_ccb_mtx);
1330 if (q->q_nccbs_avail > 0) {
1331 ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
1332 KASSERT(ccb != NULL);
1333 q->q_nccbs_avail--;
1334
1335 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1336#ifdef DEBUG
1337 ccb->ccb_cookie = NULL;
1338#endif
1339 }
1340 mutex_exit(&q->q_ccb_mtx);
1341
1342 return ccb;
1343}
1344
1345static void
1346nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb)
1347{
1348
1349 mutex_enter(&q->q_ccb_mtx);
1350#ifdef DEBUG
1351 ccb->ccb_cookie = (void *)NVME_CCB_FREE;
1352#endif
1353 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry);
1354 mutex_exit(&q->q_ccb_mtx);
1355}
1356
1357static void
1358nvme_ccbs_free(struct nvme_queue *q)
1359{
1360 struct nvme_softc *sc = q->q_sc;
1361 struct nvme_ccb *ccb;
1362
1363 mutex_enter(&q->q_ccb_mtx);
1364 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) {
1365 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1366 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1367 }
1368 mutex_exit(&q->q_ccb_mtx);
1369
1370 nvme_dmamem_free(sc, q->q_ccb_prpls);
1371 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs);
1372 q->q_ccbs = NULL;
1373 mutex_destroy(&q->q_ccb_mtx);
1374}
1375
1376static struct nvme_queue *
1377nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd)
1378{
1379 struct nvme_queue *q;
1380
1381 q = kmem_alloc(sizeof(*q), KM_SLEEP);
1382 if (q == NULL)
1383 return NULL;
1384
1385 q->q_sc = sc;
1386 q->q_sq_dmamem = nvme_dmamem_alloc(sc,
1387 sizeof(struct nvme_sqe) * entries);
1388 if (q->q_sq_dmamem == NULL)
1389 goto free;
1390
1391 q->q_cq_dmamem = nvme_dmamem_alloc(sc,
1392 sizeof(struct nvme_cqe) * entries);
1393 if (q->q_cq_dmamem == NULL)
1394 goto free_sq;
1395
1396 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem));
1397 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem));
1398
1399 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO);
1400 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO);
1401 q->q_sqtdbl = NVME_SQTDBL(id, dstrd);
1402 q->q_cqhdbl = NVME_CQHDBL(id, dstrd);
1403 q->q_id = id;
1404 q->q_entries = entries;
1405 q->q_sq_tail = 0;
1406 q->q_cq_head = 0;
1407 q->q_cq_phase = NVME_CQE_PHASE;
1408
1409 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE);
1410 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
1411
1412 /*
1413 * Due to definition of full and empty queue (queue is empty
1414 * when head == tail, full when tail is one less then head),
1415 * we can actually only have (entries - 1) in-flight commands.
1416 */
1417 if (nvme_ccbs_alloc(q, entries - 1) != 0) {
1418 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n");
1419 goto free_cq;
1420 }
1421
1422 return q;
1423
1424free_cq:
1425 nvme_dmamem_free(sc, q->q_cq_dmamem);
1426free_sq:
1427 nvme_dmamem_free(sc, q->q_sq_dmamem);
1428free:
1429 kmem_free(q, sizeof(*q));
1430
1431 return NULL;
1432}
1433
1434static void
1435nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q)
1436{
1437 nvme_ccbs_free(q);
1438 mutex_destroy(&q->q_sq_mtx);
1439 mutex_destroy(&q->q_cq_mtx);
1440 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
1441 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE);
1442 nvme_dmamem_free(sc, q->q_cq_dmamem);
1443 nvme_dmamem_free(sc, q->q_sq_dmamem);
1444 kmem_free(q, sizeof(*q));
1445}
1446
1447int
1448nvme_intr(void *xsc)
1449{
1450 struct nvme_softc *sc = xsc;
1451
1452 /*
1453 * INTx is level triggered, controller deasserts the interrupt only
1454 * when we advance command queue head via write to the doorbell.
1455 * Tell the controller to block the interrupts while we process
1456 * the queue(s).
1457 */
1458 nvme_write4(sc, NVME_INTMS, 1);
1459
1460 softint_schedule(sc->sc_softih[0]);
1461
1462 /* don't know, might not have been for us */
1463 return 1;
1464}
1465
1466void
1467nvme_softintr_intx(void *xq)
1468{
1469 struct nvme_queue *q = xq;
1470 struct nvme_softc *sc = q->q_sc;
1471
1472 nvme_q_complete(sc, sc->sc_admin_q);
1473 if (sc->sc_q != NULL)
1474 nvme_q_complete(sc, sc->sc_q[0]);
1475
1476 /*
1477 * Processing done, tell controller to issue interrupts again. There
1478 * is no race, as NVMe spec requires the controller to maintain state,
1479 * and assert the interrupt whenever there are unacknowledged
1480 * completion queue entries.
1481 */
1482 nvme_write4(sc, NVME_INTMC, 1);
1483}
1484
1485int
1486nvme_intr_msi(void *xq)
1487{
1488 struct nvme_queue *q = xq;
1489
1490 KASSERT(q && q->q_sc && q->q_sc->sc_softih
1491 && q->q_sc->sc_softih[q->q_id]);
1492
1493 /*
1494 * MSI/MSI-X are edge triggered, so can handover processing to softint
1495 * without masking the interrupt.
1496 */
1497 softint_schedule(q->q_sc->sc_softih[q->q_id]);
1498
1499 return 1;
1500}
1501
1502void
1503nvme_softintr_msi(void *xq)
1504{
1505 struct nvme_queue *q = xq;
1506 struct nvme_softc *sc = q->q_sc;
1507
1508 nvme_q_complete(sc, q);
1509}
1510
1511static struct nvme_dmamem *
1512nvme_dmamem_alloc(struct nvme_softc *sc, size_t size)
1513{
1514 struct nvme_dmamem *ndm;
1515 int nsegs;
1516
1517 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP);
1518 if (ndm == NULL)
1519 return NULL;
1520
1521 ndm->ndm_size = size;
1522
1523 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
1524 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0)
1525 goto ndmfree;
1526
1527 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg,
1528 1, &nsegs, BUS_DMA_WAITOK) != 0)
1529 goto destroy;
1530
1531 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size,
1532 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0)
1533 goto free;
1534 memset(ndm->ndm_kva, 0, size);
1535
1536 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size,
1537 NULL, BUS_DMA_WAITOK) != 0)
1538 goto unmap;
1539
1540 return ndm;
1541
1542unmap:
1543 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size);
1544free:
1545 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
1546destroy:
1547 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
1548ndmfree:
1549 kmem_free(ndm, sizeof(*ndm));
1550 return NULL;
1551}
1552
1553static void
1554nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops)
1555{
1556 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem),
1557 0, NVME_DMA_LEN(mem), ops);
1558}
1559
1560void
1561nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm)
1562{
1563 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map);
1564 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size);
1565 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
1566 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
1567 kmem_free(ndm, sizeof(*ndm));
1568}
1569
1570/*
1571 * ioctl
1572 */
1573
1574dev_type_open(nvmeopen);
1575dev_type_close(nvmeclose);
1576dev_type_ioctl(nvmeioctl);
1577
1578const struct cdevsw nvme_cdevsw = {
1579 .d_open = nvmeopen,
1580 .d_close = nvmeclose,
1581 .d_read = noread,
1582 .d_write = nowrite,
1583 .d_ioctl = nvmeioctl,
1584 .d_stop = nostop,
1585 .d_tty = notty,
1586 .d_poll = nopoll,
1587 .d_mmap = nommap,
1588 .d_kqfilter = nokqfilter,
1589 .d_discard = nodiscard,
1590 .d_flag = D_OTHER,
1591};
1592
1593extern struct cfdriver nvme_cd;
1594
1595/*
1596 * Accept an open operation on the control device.
1597 */
1598int
1599nvmeopen(dev_t dev, int flag, int mode, struct lwp *l)
1600{
1601 struct nvme_softc *sc;
1602 int unit = minor(dev) / 0x10000;
1603 int nsid = minor(dev) & 0xffff;
1604 int nsidx;
1605
1606 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL)
1607 return ENXIO;
1608 if ((sc->sc_flags & NVME_F_ATTACHED) == 0)
1609 return ENXIO;
1610
1611 if (nsid == 0) {
1612 /* controller */
1613 if (ISSET(sc->sc_flags, NVME_F_OPEN))
1614 return EBUSY;
1615 SET(sc->sc_flags, NVME_F_OPEN);
1616 } else {
1617 /* namespace */
1618 nsidx = nsid - 1;
1619 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL)
1620 return ENXIO;
1621 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN))
1622 return EBUSY;
1623 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
1624 }
1625 return 0;
1626}
1627
1628/*
1629 * Accept the last close on the control device.
1630 */
1631int
1632nvmeclose(dev_t dev, int flag, int mode, struct lwp *l)
1633{
1634 struct nvme_softc *sc;
1635 int unit = minor(dev) / 0x10000;
1636 int nsid = minor(dev) & 0xffff;
1637 int nsidx;
1638
1639 sc = device_lookup_private(&nvme_cd, unit);
1640 if (sc == NULL)
1641 return ENXIO;
1642
1643 if (nsid == 0) {
1644 /* controller */
1645 CLR(sc->sc_flags, NVME_F_OPEN);
1646 } else {
1647 /* namespace */
1648 nsidx = nsid - 1;
1649 if (nsidx >= sc->sc_nn)
1650 return ENXIO;
1651 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
1652 }
1653
1654 return 0;
1655}
1656
1657/*
1658 * Handle control operations.
1659 */
1660int
1661nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
1662{
1663 struct nvme_softc *sc;
1664 int unit = minor(dev) / 0x10000;
1665 int nsid = minor(dev) & 0xffff;
1666 struct nvme_pt_command *pt;
1667
1668 sc = device_lookup_private(&nvme_cd, unit);
1669 if (sc == NULL)
1670 return ENXIO;
1671
1672 switch (cmd) {
1673 case NVME_PASSTHROUGH_CMD:
1674 pt = data;
1675 return nvme_command_passthrough(sc, data,
1676 nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0);
1677 }
1678
1679 return ENOTTY;
1680}
1681