1/* $NetBSD: xhci.c,v 1.67 2016/09/03 12:07:41 skrll Exp $ */
2
3/*
4 * Copyright (c) 2013 Jonathan A. Kollasch
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * USB rev 2.0 and rev 3.1 specification
31 * http://www.usb.org/developers/docs/
32 * xHCI rev 1.1 specification
33 * http://www.intel.com/technology/usb/spec.htm
34 */
35
36#include <sys/cdefs.h>
37__KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.67 2016/09/03 12:07:41 skrll Exp $");
38
39#ifdef _KERNEL_OPT
40#include "opt_usb.h"
41#endif
42
43#include <sys/param.h>
44#include <sys/systm.h>
45#include <sys/kernel.h>
46#include <sys/kmem.h>
47#include <sys/device.h>
48#include <sys/select.h>
49#include <sys/proc.h>
50#include <sys/queue.h>
51#include <sys/mutex.h>
52#include <sys/condvar.h>
53#include <sys/bus.h>
54#include <sys/cpu.h>
55#include <sys/sysctl.h>
56
57#include <machine/endian.h>
58
59#include <dev/usb/usb.h>
60#include <dev/usb/usbdi.h>
61#include <dev/usb/usbdivar.h>
62#include <dev/usb/usbdi_util.h>
63#include <dev/usb/usbhist.h>
64#include <dev/usb/usb_mem.h>
65#include <dev/usb/usb_quirks.h>
66
67#include <dev/usb/xhcireg.h>
68#include <dev/usb/xhcivar.h>
69#include <dev/usb/usbroothub.h>
70
71
72#ifdef USB_DEBUG
73#ifndef XHCI_DEBUG
74#define xhcidebug 0
75#else /* !XHCI_DEBUG */
76static int xhcidebug = 0;
77
78SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup")
79{
80 int err;
81 const struct sysctlnode *rnode;
82 const struct sysctlnode *cnode;
83
84 err = sysctl_createv(clog, 0, NULL, &rnode,
85 CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci",
86 SYSCTL_DESCR("xhci global controls"),
87 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
88
89 if (err)
90 goto fail;
91
92 /* control debugging printfs */
93 err = sysctl_createv(clog, 0, &rnode, &cnode,
94 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT,
95 "debug", SYSCTL_DESCR("Enable debugging output"),
96 NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL);
97 if (err)
98 goto fail;
99
100 return;
101fail:
102 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err);
103}
104
105#endif /* !XHCI_DEBUG */
106#endif /* USB_DEBUG */
107
108#define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D)
109#define XHCIHIST_FUNC() USBHIST_FUNC()
110#define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug)
111
112#define XHCI_DCI_SLOT 0
113#define XHCI_DCI_EP_CONTROL 1
114
115#define XHCI_ICI_INPUT_CONTROL 0
116
117struct xhci_pipe {
118 struct usbd_pipe xp_pipe;
119 struct usb_task xp_async_task;
120};
121
122#define XHCI_COMMAND_RING_TRBS 256
123#define XHCI_EVENT_RING_TRBS 256
124#define XHCI_EVENT_RING_SEGMENTS 1
125#define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT
126
127static usbd_status xhci_open(struct usbd_pipe *);
128static void xhci_close_pipe(struct usbd_pipe *);
129static int xhci_intr1(struct xhci_softc * const);
130static void xhci_softintr(void *);
131static void xhci_poll(struct usbd_bus *);
132static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int);
133static void xhci_freex(struct usbd_bus *, struct usbd_xfer *);
134static void xhci_get_lock(struct usbd_bus *, kmutex_t **);
135static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int,
136 struct usbd_port *);
137static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *,
138 void *, int);
139
140static usbd_status xhci_configure_endpoint(struct usbd_pipe *);
141//static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *);
142static usbd_status xhci_reset_endpoint(struct usbd_pipe *);
143static usbd_status xhci_stop_endpoint(struct usbd_pipe *);
144
145static void xhci_host_dequeue(struct xhci_ring * const);
146static usbd_status xhci_set_dequeue(struct usbd_pipe *);
147
148static usbd_status xhci_do_command(struct xhci_softc * const,
149 struct xhci_trb * const, int);
150static usbd_status xhci_do_command_locked(struct xhci_softc * const,
151 struct xhci_trb * const, int);
152static usbd_status xhci_init_slot(struct usbd_device *, uint32_t);
153static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *, int, int);
154static usbd_status xhci_set_address(struct usbd_device *, uint32_t, bool);
155static usbd_status xhci_enable_slot(struct xhci_softc * const,
156 uint8_t * const);
157static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t);
158static usbd_status xhci_address_device(struct xhci_softc * const,
159 uint64_t, uint8_t, bool);
160static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int);
161static usbd_status xhci_update_ep0_mps(struct xhci_softc * const,
162 struct xhci_slot * const, u_int);
163static usbd_status xhci_ring_init(struct xhci_softc * const,
164 struct xhci_ring * const, size_t, size_t);
165static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const);
166
167static void xhci_setup_ctx(struct usbd_pipe *);
168static void xhci_setup_route(struct usbd_pipe *, uint32_t *);
169static void xhci_setup_tthub(struct usbd_pipe *, uint32_t *);
170static void xhci_setup_maxburst(struct usbd_pipe *, uint32_t *);
171static uint32_t xhci_bival2ival(uint32_t, uint32_t);
172
173static void xhci_noop(struct usbd_pipe *);
174
175static usbd_status xhci_root_intr_transfer(struct usbd_xfer *);
176static usbd_status xhci_root_intr_start(struct usbd_xfer *);
177static void xhci_root_intr_abort(struct usbd_xfer *);
178static void xhci_root_intr_close(struct usbd_pipe *);
179static void xhci_root_intr_done(struct usbd_xfer *);
180
181static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *);
182static usbd_status xhci_device_ctrl_start(struct usbd_xfer *);
183static void xhci_device_ctrl_abort(struct usbd_xfer *);
184static void xhci_device_ctrl_close(struct usbd_pipe *);
185static void xhci_device_ctrl_done(struct usbd_xfer *);
186
187static usbd_status xhci_device_intr_transfer(struct usbd_xfer *);
188static usbd_status xhci_device_intr_start(struct usbd_xfer *);
189static void xhci_device_intr_abort(struct usbd_xfer *);
190static void xhci_device_intr_close(struct usbd_pipe *);
191static void xhci_device_intr_done(struct usbd_xfer *);
192
193static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *);
194static usbd_status xhci_device_bulk_start(struct usbd_xfer *);
195static void xhci_device_bulk_abort(struct usbd_xfer *);
196static void xhci_device_bulk_close(struct usbd_pipe *);
197static void xhci_device_bulk_done(struct usbd_xfer *);
198
199static void xhci_timeout(void *);
200static void xhci_timeout_task(void *);
201
202static const struct usbd_bus_methods xhci_bus_methods = {
203 .ubm_open = xhci_open,
204 .ubm_softint = xhci_softintr,
205 .ubm_dopoll = xhci_poll,
206 .ubm_allocx = xhci_allocx,
207 .ubm_freex = xhci_freex,
208 .ubm_getlock = xhci_get_lock,
209 .ubm_newdev = xhci_new_device,
210 .ubm_rhctrl = xhci_roothub_ctrl,
211};
212
213static const struct usbd_pipe_methods xhci_root_intr_methods = {
214 .upm_transfer = xhci_root_intr_transfer,
215 .upm_start = xhci_root_intr_start,
216 .upm_abort = xhci_root_intr_abort,
217 .upm_close = xhci_root_intr_close,
218 .upm_cleartoggle = xhci_noop,
219 .upm_done = xhci_root_intr_done,
220};
221
222
223static const struct usbd_pipe_methods xhci_device_ctrl_methods = {
224 .upm_transfer = xhci_device_ctrl_transfer,
225 .upm_start = xhci_device_ctrl_start,
226 .upm_abort = xhci_device_ctrl_abort,
227 .upm_close = xhci_device_ctrl_close,
228 .upm_cleartoggle = xhci_noop,
229 .upm_done = xhci_device_ctrl_done,
230};
231
232static const struct usbd_pipe_methods xhci_device_isoc_methods = {
233 .upm_cleartoggle = xhci_noop,
234};
235
236static const struct usbd_pipe_methods xhci_device_bulk_methods = {
237 .upm_transfer = xhci_device_bulk_transfer,
238 .upm_start = xhci_device_bulk_start,
239 .upm_abort = xhci_device_bulk_abort,
240 .upm_close = xhci_device_bulk_close,
241 .upm_cleartoggle = xhci_noop,
242 .upm_done = xhci_device_bulk_done,
243};
244
245static const struct usbd_pipe_methods xhci_device_intr_methods = {
246 .upm_transfer = xhci_device_intr_transfer,
247 .upm_start = xhci_device_intr_start,
248 .upm_abort = xhci_device_intr_abort,
249 .upm_close = xhci_device_intr_close,
250 .upm_cleartoggle = xhci_noop,
251 .upm_done = xhci_device_intr_done,
252};
253
254static inline uint32_t
255xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset)
256{
257 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset);
258}
259
260static inline uint32_t
261xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset)
262{
263 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset);
264}
265
266static inline void
267xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset,
268 uint32_t value)
269{
270 bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value);
271}
272
273#if 0 /* unused */
274static inline void
275xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset,
276 uint32_t value)
277{
278 bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value);
279}
280#endif /* unused */
281
282static inline uint32_t
283xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset)
284{
285 return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset);
286}
287
288static inline uint32_t
289xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset)
290{
291 return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset);
292}
293
294static inline void
295xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset,
296 uint32_t value)
297{
298 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value);
299}
300
301static inline uint64_t
302xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset)
303{
304 uint64_t value;
305
306 if (sc->sc_ac64) {
307#ifdef XHCI_USE_BUS_SPACE_8
308 value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset);
309#else
310 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset);
311 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh,
312 offset + 4) << 32;
313#endif
314 } else {
315 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset);
316 }
317
318 return value;
319}
320
321static inline void
322xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset,
323 uint64_t value)
324{
325 if (sc->sc_ac64) {
326#ifdef XHCI_USE_BUS_SPACE_8
327 bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value);
328#else
329 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0,
330 (value >> 0) & 0xffffffff);
331 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4,
332 (value >> 32) & 0xffffffff);
333#endif
334 } else {
335 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value);
336 }
337}
338
339static inline uint32_t
340xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset)
341{
342 return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset);
343}
344
345static inline void
346xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset,
347 uint32_t value)
348{
349 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value);
350}
351
352#if 0 /* unused */
353static inline uint64_t
354xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset)
355{
356 uint64_t value;
357
358 if (sc->sc_ac64) {
359#ifdef XHCI_USE_BUS_SPACE_8
360 value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset);
361#else
362 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset);
363 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh,
364 offset + 4) << 32;
365#endif
366 } else {
367 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset);
368 }
369
370 return value;
371}
372#endif /* unused */
373
374static inline void
375xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset,
376 uint64_t value)
377{
378 if (sc->sc_ac64) {
379#ifdef XHCI_USE_BUS_SPACE_8
380 bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value);
381#else
382 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0,
383 (value >> 0) & 0xffffffff);
384 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4,
385 (value >> 32) & 0xffffffff);
386#endif
387 } else {
388 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value);
389 }
390}
391
392#if 0 /* unused */
393static inline uint32_t
394xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset)
395{
396 return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset);
397}
398#endif /* unused */
399
400static inline void
401xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset,
402 uint32_t value)
403{
404 bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value);
405}
406
407/* --- */
408
409static inline uint8_t
410xhci_ep_get_type(usb_endpoint_descriptor_t * const ed)
411{
412 u_int eptype = 0;
413
414 switch (UE_GET_XFERTYPE(ed->bmAttributes)) {
415 case UE_CONTROL:
416 eptype = 0x0;
417 break;
418 case UE_ISOCHRONOUS:
419 eptype = 0x1;
420 break;
421 case UE_BULK:
422 eptype = 0x2;
423 break;
424 case UE_INTERRUPT:
425 eptype = 0x3;
426 break;
427 }
428
429 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) ||
430 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN))
431 return eptype | 0x4;
432 else
433 return eptype;
434}
435
436static u_int
437xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed)
438{
439 /* xHCI 1.0 section 4.5.1 */
440 u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress);
441 u_int in = 0;
442
443 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) ||
444 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN))
445 in = 1;
446
447 return epaddr * 2 + in;
448}
449
450static inline u_int
451xhci_dci_to_ici(const u_int i)
452{
453 return i + 1;
454}
455
456static inline void *
457xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs,
458 const u_int dci)
459{
460 return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci);
461}
462
463#if 0 /* unused */
464static inline bus_addr_t
465xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs,
466 const u_int dci)
467{
468 return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci);
469}
470#endif /* unused */
471
472static inline void *
473xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs,
474 const u_int ici)
475{
476 return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici);
477}
478
479static inline bus_addr_t
480xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs,
481 const u_int ici)
482{
483 return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici);
484}
485
486static inline struct xhci_trb *
487xhci_ring_trbv(struct xhci_ring * const xr, u_int idx)
488{
489 return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx);
490}
491
492static inline bus_addr_t
493xhci_ring_trbp(struct xhci_ring * const xr, u_int idx)
494{
495 return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx);
496}
497
498static inline void
499xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status,
500 uint32_t control)
501{
502 trb->trb_0 = htole64(parameter);
503 trb->trb_2 = htole32(status);
504 trb->trb_3 = htole32(control);
505}
506
507static int
508xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx)
509{
510 /* base address of TRBs */
511 bus_addr_t trbp = xhci_ring_trbp(xr, 0);
512
513 /* trb_0 range sanity check */
514 if (trb_0 == 0 || trb_0 < trbp ||
515 (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 ||
516 (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) {
517 return 1;
518 }
519 *idx = (trb_0 - trbp) / sizeof(struct xhci_trb);
520 return 0;
521}
522
523static unsigned int
524xhci_get_epstate(struct xhci_softc * const sc, struct xhci_slot * const xs,
525 u_int dci)
526{
527 uint32_t *cp;
528
529 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
530 cp = xhci_slot_get_dcv(sc, xs, dci);
531 return XHCI_EPCTX_0_EPSTATE_GET(le32toh(cp[0]));
532}
533
534/* --- */
535
536void
537xhci_childdet(device_t self, device_t child)
538{
539 struct xhci_softc * const sc = device_private(self);
540
541 KASSERT(sc->sc_child == child);
542 if (child == sc->sc_child)
543 sc->sc_child = NULL;
544}
545
546int
547xhci_detach(struct xhci_softc *sc, int flags)
548{
549 int rv = 0;
550
551 if (sc->sc_child != NULL)
552 rv = config_detach(sc->sc_child, flags);
553
554 if (rv != 0)
555 return rv;
556
557 /* XXX unconfigure/free slots */
558
559 /* verify: */
560 xhci_rt_write_4(sc, XHCI_IMAN(0), 0);
561 xhci_op_write_4(sc, XHCI_USBCMD, 0);
562 /* do we need to wait for stop? */
563
564 xhci_op_write_8(sc, XHCI_CRCR, 0);
565 xhci_ring_free(sc, &sc->sc_cr);
566 cv_destroy(&sc->sc_command_cv);
567
568 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0);
569 xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0);
570 xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY);
571 xhci_ring_free(sc, &sc->sc_er);
572
573 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma);
574
575 xhci_op_write_8(sc, XHCI_DCBAAP, 0);
576 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma);
577
578 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots);
579
580 mutex_destroy(&sc->sc_lock);
581 mutex_destroy(&sc->sc_intr_lock);
582
583 pool_cache_destroy(sc->sc_xferpool);
584
585 return rv;
586}
587
588int
589xhci_activate(device_t self, enum devact act)
590{
591 struct xhci_softc * const sc = device_private(self);
592
593 switch (act) {
594 case DVACT_DEACTIVATE:
595 sc->sc_dying = true;
596 return 0;
597 default:
598 return EOPNOTSUPP;
599 }
600}
601
602bool
603xhci_suspend(device_t dv, const pmf_qual_t *qual)
604{
605 return false;
606}
607
608bool
609xhci_resume(device_t dv, const pmf_qual_t *qual)
610{
611 return false;
612}
613
614bool
615xhci_shutdown(device_t self, int flags)
616{
617 return false;
618}
619
620static int
621xhci_hc_reset(struct xhci_softc * const sc)
622{
623 uint32_t usbcmd, usbsts;
624 int i;
625
626 /* Check controller not ready */
627 for (i = 0; i < XHCI_WAIT_CNR; i++) {
628 usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
629 if ((usbsts & XHCI_STS_CNR) == 0)
630 break;
631 usb_delay_ms(&sc->sc_bus, 1);
632 }
633 if (i >= XHCI_WAIT_CNR) {
634 aprint_error_dev(sc->sc_dev, "controller not ready timeout\n");
635 return EIO;
636 }
637
638 /* Halt controller */
639 usbcmd = 0;
640 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd);
641 usb_delay_ms(&sc->sc_bus, 1);
642
643 /* Reset controller */
644 usbcmd = XHCI_CMD_HCRST;
645 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd);
646 for (i = 0; i < XHCI_WAIT_HCRST; i++) {
647 usbcmd = xhci_op_read_4(sc, XHCI_USBCMD);
648 if ((usbcmd & XHCI_CMD_HCRST) == 0)
649 break;
650 usb_delay_ms(&sc->sc_bus, 1);
651 }
652 if (i >= XHCI_WAIT_HCRST) {
653 aprint_error_dev(sc->sc_dev, "host controller reset timeout\n");
654 return EIO;
655 }
656
657 /* Check controller not ready */
658 for (i = 0; i < XHCI_WAIT_CNR; i++) {
659 usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
660 if ((usbsts & XHCI_STS_CNR) == 0)
661 break;
662 usb_delay_ms(&sc->sc_bus, 1);
663 }
664 if (i >= XHCI_WAIT_CNR) {
665 aprint_error_dev(sc->sc_dev,
666 "controller not ready timeout after reset\n");
667 return EIO;
668 }
669
670 return 0;
671}
672
673
674static void
675hexdump(const char *msg, const void *base, size_t len)
676{
677#if 0
678 size_t cnt;
679 const uint32_t *p;
680 extern paddr_t vtophys(vaddr_t);
681
682 p = base;
683 cnt = 0;
684
685 printf("*** %s (%zu bytes @ %p %p)\n", msg, len, base,
686 (void *)vtophys((vaddr_t)base));
687
688 while (cnt < len) {
689 if (cnt % 16 == 0)
690 printf("%p: ", p);
691 else if (cnt % 8 == 0)
692 printf(" |");
693 printf(" %08x", *p++);
694 cnt += 4;
695 if (cnt % 16 == 0)
696 printf("\n");
697 }
698 if (cnt % 16 != 0)
699 printf("\n");
700#endif
701}
702
703/* Process extended capabilities */
704static void
705xhci_ecp(struct xhci_softc *sc, uint32_t hcc)
706{
707 uint32_t ecp, ecr;
708
709 XHCIHIST_FUNC(); XHCIHIST_CALLED();
710
711 ecp = XHCI_HCC_XECP(hcc) * 4;
712 while (ecp != 0) {
713 ecr = xhci_read_4(sc, ecp);
714 aprint_debug_dev(sc->sc_dev, "ECR %x: %08x\n", ecp, ecr);
715 switch (XHCI_XECP_ID(ecr)) {
716 case XHCI_ID_PROTOCOLS: {
717 uint32_t w4, w8, wc;
718 uint16_t w2;
719 w2 = (ecr >> 16) & 0xffff;
720 w4 = xhci_read_4(sc, ecp + 4);
721 w8 = xhci_read_4(sc, ecp + 8);
722 wc = xhci_read_4(sc, ecp + 0xc);
723 aprint_debug_dev(sc->sc_dev,
724 " SP: %08x %08x %08x %08x\n", ecr, w4, w8, wc);
725 /* unused */
726 if (w4 == 0x20425355 && (w2 & 0xff00) == 0x0300) {
727 sc->sc_ss_port_start = (w8 >> 0) & 0xff;;
728 sc->sc_ss_port_count = (w8 >> 8) & 0xff;;
729 }
730 if (w4 == 0x20425355 && (w2 & 0xff00) == 0x0200) {
731 sc->sc_hs_port_start = (w8 >> 0) & 0xff;
732 sc->sc_hs_port_count = (w8 >> 8) & 0xff;
733 }
734 break;
735 }
736 case XHCI_ID_USB_LEGACY: {
737 uint8_t bios_sem;
738
739 /* Take host controller ownership from BIOS */
740 bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM);
741 if (bios_sem) {
742 /* sets xHCI to be owned by OS */
743 xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1);
744 aprint_debug_dev(sc->sc_dev,
745 "waiting for BIOS to give up control\n");
746 for (int i = 0; i < 5000; i++) {
747 bios_sem = xhci_read_1(sc, ecp +
748 XHCI_XECP_BIOS_SEM);
749 if (bios_sem == 0)
750 break;
751 DELAY(1000);
752 }
753 if (bios_sem) {
754 aprint_error_dev(sc->sc_dev,
755 "timed out waiting for BIOS\n");
756 }
757 }
758 break;
759 }
760 default:
761 break;
762 }
763 ecr = xhci_read_4(sc, ecp);
764 if (XHCI_XECP_NEXT(ecr) == 0) {
765 ecp = 0;
766 } else {
767 ecp += XHCI_XECP_NEXT(ecr) * 4;
768 }
769 }
770}
771
772#define XHCI_HCCPREV1_BITS \
773 "\177\020" /* New bitmask */ \
774 "f\020\020XECP\0" \
775 "f\014\4MAXPSA\0" \
776 "b\013CFC\0" \
777 "b\012SEC\0" \
778 "b\011SBD\0" \
779 "b\010FSE\0" \
780 "b\7NSS\0" \
781 "b\6LTC\0" \
782 "b\5LHRC\0" \
783 "b\4PIND\0" \
784 "b\3PPC\0" \
785 "b\2CZC\0" \
786 "b\1BNC\0" \
787 "b\0AC64\0" \
788 "\0"
789#define XHCI_HCCV1_x_BITS \
790 "\177\020" /* New bitmask */ \
791 "f\020\020XECP\0" \
792 "f\014\4MAXPSA\0" \
793 "b\013CFC\0" \
794 "b\012SEC\0" \
795 "b\011SPC\0" \
796 "b\010PAE\0" \
797 "b\7NSS\0" \
798 "b\6LTC\0" \
799 "b\5LHRC\0" \
800 "b\4PIND\0" \
801 "b\3PPC\0" \
802 "b\2CSZ\0" \
803 "b\1BNC\0" \
804 "b\0AC64\0" \
805 "\0"
806
807int
808xhci_init(struct xhci_softc *sc)
809{
810 bus_size_t bsz;
811 uint32_t cap, hcs1, hcs2, hcs3, hcc, dboff, rtsoff;
812 uint32_t pagesize, config;
813 int i = 0;
814 uint16_t hciversion;
815 uint8_t caplength;
816
817 XHCIHIST_FUNC(); XHCIHIST_CALLED();
818
819 sc->sc_bus.ub_revision = USBREV_3_0;
820 sc->sc_bus.ub_usedma = true;
821
822 cap = xhci_read_4(sc, XHCI_CAPLENGTH);
823 caplength = XHCI_CAP_CAPLENGTH(cap);
824 hciversion = XHCI_CAP_HCIVERSION(cap);
825
826 if (hciversion < XHCI_HCIVERSION_0_96 ||
827 hciversion > XHCI_HCIVERSION_1_0) {
828 aprint_normal_dev(sc->sc_dev,
829 "xHCI version %x.%x not known to be supported\n",
830 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff);
831 } else {
832 aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n",
833 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff);
834 }
835
836 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength,
837 &sc->sc_cbh) != 0) {
838 aprint_error_dev(sc->sc_dev, "capability subregion failure\n");
839 return ENOMEM;
840 }
841
842 hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1);
843 sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1);
844 sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1);
845 sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1);
846 hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2);
847 hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3);
848 aprint_debug_dev(sc->sc_dev,
849 "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3);
850
851 hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS);
852 sc->sc_ac64 = XHCI_HCC_AC64(hcc);
853 sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32;
854
855 char sbuf[128];
856 if (hciversion < XHCI_HCIVERSION_1_0)
857 snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, hcc);
858 else
859 snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, hcc);
860 aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf);
861 aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4);
862
863 /* print PSI and take ownership from BIOS */
864 xhci_ecp(sc, hcc);
865
866 bsz = XHCI_PORTSC(sc->sc_maxports + 1);
867 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz,
868 &sc->sc_obh) != 0) {
869 aprint_error_dev(sc->sc_dev, "operational subregion failure\n");
870 return ENOMEM;
871 }
872
873 dboff = xhci_cap_read_4(sc, XHCI_DBOFF);
874 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff,
875 sc->sc_maxslots * 4, &sc->sc_dbh) != 0) {
876 aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n");
877 return ENOMEM;
878 }
879
880 rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF);
881 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff,
882 sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) {
883 aprint_error_dev(sc->sc_dev, "runtime subregion failure\n");
884 return ENOMEM;
885 }
886
887 int rv;
888 rv = xhci_hc_reset(sc);
889 if (rv != 0) {
890 return rv;
891 }
892
893 if (sc->sc_vendor_init)
894 sc->sc_vendor_init(sc);
895
896 pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE);
897 aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize);
898 pagesize = ffs(pagesize);
899 if (pagesize == 0) {
900 aprint_error_dev(sc->sc_dev, "pagesize is 0\n");
901 return EIO;
902 }
903 sc->sc_pgsz = 1 << (12 + (pagesize - 1));
904 aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz);
905 aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n",
906 (uint32_t)sc->sc_maxslots);
907 aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports);
908
909 usbd_status err;
910
911 sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2);
912 aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf);
913 if (sc->sc_maxspbuf != 0) {
914 err = usb_allocmem(&sc->sc_bus,
915 sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t),
916 &sc->sc_spbufarray_dma);
917 if (err) {
918 aprint_error_dev(sc->sc_dev,
919 "spbufarray init fail, err %d\n", err);
920 return ENOMEM;
921 }
922
923 sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) *
924 sc->sc_maxspbuf, KM_SLEEP);
925 uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0);
926 for (i = 0; i < sc->sc_maxspbuf; i++) {
927 usb_dma_t * const dma = &sc->sc_spbuf_dma[i];
928 /* allocate contexts */
929 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz,
930 sc->sc_pgsz, dma);
931 if (err) {
932 aprint_error_dev(sc->sc_dev,
933 "spbufarray_dma init fail, err %d\n", err);
934 rv = ENOMEM;
935 goto bad1;
936 }
937 spbufarray[i] = htole64(DMAADDR(dma, 0));
938 usb_syncmem(dma, 0, sc->sc_pgsz,
939 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
940 }
941
942 usb_syncmem(&sc->sc_spbufarray_dma, 0,
943 sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE);
944 }
945
946 config = xhci_op_read_4(sc, XHCI_CONFIG);
947 config &= ~0xFF;
948 config |= sc->sc_maxslots & 0xFF;
949 xhci_op_write_4(sc, XHCI_CONFIG, config);
950
951 err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS,
952 XHCI_COMMAND_RING_SEGMENTS_ALIGN);
953 if (err) {
954 aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n",
955 err);
956 rv = ENOMEM;
957 goto bad1;
958 }
959
960 err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS,
961 XHCI_EVENT_RING_SEGMENTS_ALIGN);
962 if (err) {
963 aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n",
964 err);
965 rv = ENOMEM;
966 goto bad2;
967 }
968
969 usb_dma_t *dma;
970 size_t size;
971 size_t align;
972
973 dma = &sc->sc_eventst_dma;
974 size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE,
975 XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN);
976 KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size);
977 align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN;
978 err = usb_allocmem(&sc->sc_bus, size, align, dma);
979 if (err) {
980 aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n",
981 err);
982 rv = ENOMEM;
983 goto bad3;
984 }
985
986 memset(KERNADDR(dma, 0), 0, size);
987 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE);
988 aprint_debug_dev(sc->sc_dev, "eventst: %016jx %p %zx\n",
989 (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0),
990 KERNADDR(&sc->sc_eventst_dma, 0),
991 sc->sc_eventst_dma.udma_block->size);
992
993 dma = &sc->sc_dcbaa_dma;
994 size = (1 + sc->sc_maxslots) * sizeof(uint64_t);
995 KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size);
996 align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN;
997 err = usb_allocmem(&sc->sc_bus, size, align, dma);
998 if (err) {
999 aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err);
1000 rv = ENOMEM;
1001 goto bad4;
1002 }
1003 aprint_debug_dev(sc->sc_dev, "dcbaa: %016jx %p %zx\n",
1004 (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0),
1005 KERNADDR(&sc->sc_dcbaa_dma, 0),
1006 sc->sc_dcbaa_dma.udma_block->size);
1007
1008 memset(KERNADDR(dma, 0), 0, size);
1009 if (sc->sc_maxspbuf != 0) {
1010 /*
1011 * DCBA entry 0 hold the scratchbuf array pointer.
1012 */
1013 *(uint64_t *)KERNADDR(dma, 0) =
1014 htole64(DMAADDR(&sc->sc_spbufarray_dma, 0));
1015 }
1016 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE);
1017
1018 sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots,
1019 KM_SLEEP);
1020 if (sc->sc_slots == NULL) {
1021 aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err);
1022 rv = ENOMEM;
1023 goto bad;
1024 }
1025
1026 sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0,
1027 "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL);
1028 if (sc->sc_xferpool == NULL) {
1029 aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n",
1030 err);
1031 rv = ENOMEM;
1032 goto bad;
1033 }
1034
1035 cv_init(&sc->sc_command_cv, "xhcicmd");
1036 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
1037 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB);
1038
1039 /* Set up the bus struct. */
1040 sc->sc_bus.ub_methods = &xhci_bus_methods;
1041 sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe);
1042
1043 struct xhci_erste *erst;
1044 erst = KERNADDR(&sc->sc_eventst_dma, 0);
1045 erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0));
1046 erst[0].erste_2 = htole32(sc->sc_er.xr_ntrb);
1047 erst[0].erste_3 = htole32(0);
1048 usb_syncmem(&sc->sc_eventst_dma, 0,
1049 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE);
1050
1051 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS);
1052 xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0));
1053 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) |
1054 XHCI_ERDP_LO_BUSY);
1055 xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0));
1056 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) |
1057 sc->sc_cr.xr_cs);
1058
1059#if 0
1060 hexdump("eventst", KERNADDR(&sc->sc_eventst_dma, 0),
1061 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS);
1062#endif
1063
1064 xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA);
1065 if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0)
1066 /* Intel xhci needs interrupt rate moderated. */
1067 xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP);
1068 else
1069 xhci_rt_write_4(sc, XHCI_IMOD(0), 0);
1070 aprint_debug_dev(sc->sc_dev, "current IMOD %u\n",
1071 xhci_rt_read_4(sc, XHCI_IMOD(0)));
1072
1073 xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); /* Go! */
1074 aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n",
1075 xhci_op_read_4(sc, XHCI_USBCMD));
1076
1077 return 0;
1078
1079 bad:
1080 if (sc->sc_xferpool) {
1081 pool_cache_destroy(sc->sc_xferpool);
1082 sc->sc_xferpool = NULL;
1083 }
1084
1085 if (sc->sc_slots) {
1086 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) *
1087 sc->sc_maxslots);
1088 sc->sc_slots = NULL;
1089 }
1090
1091 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma);
1092 bad4:
1093 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma);
1094 bad3:
1095 xhci_ring_free(sc, &sc->sc_er);
1096 bad2:
1097 xhci_ring_free(sc, &sc->sc_cr);
1098 i = sc->sc_maxspbuf;
1099 bad1:
1100 for (int j = 0; j < i; j++)
1101 usb_freemem(&sc->sc_bus, &sc->sc_spbuf_dma[j]);
1102 usb_freemem(&sc->sc_bus, &sc->sc_spbufarray_dma);
1103
1104 return rv;
1105}
1106
1107int
1108xhci_intr(void *v)
1109{
1110 struct xhci_softc * const sc = v;
1111 int ret = 0;
1112
1113 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1114
1115 if (sc == NULL)
1116 return 0;
1117
1118 mutex_spin_enter(&sc->sc_intr_lock);
1119
1120 if (sc->sc_dying || !device_has_power(sc->sc_dev))
1121 goto done;
1122
1123 /* If we get an interrupt while polling, then just ignore it. */
1124 if (sc->sc_bus.ub_usepolling) {
1125#ifdef DIAGNOSTIC
1126 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0);
1127#endif
1128 goto done;
1129 }
1130
1131 ret = xhci_intr1(sc);
1132done:
1133 mutex_spin_exit(&sc->sc_intr_lock);
1134 return ret;
1135}
1136
1137int
1138xhci_intr1(struct xhci_softc * const sc)
1139{
1140 uint32_t usbsts;
1141 uint32_t iman;
1142
1143 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1144
1145 usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
1146 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0);
1147#if 0
1148 if ((usbsts & (XHCI_STS_EINT|XHCI_STS_PCD)) == 0) {
1149 return 0;
1150 }
1151#endif
1152 xhci_op_write_4(sc, XHCI_USBSTS,
1153 usbsts & (2|XHCI_STS_EINT|XHCI_STS_PCD)); /* XXX */
1154 usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
1155 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0);
1156
1157 iman = xhci_rt_read_4(sc, XHCI_IMAN(0));
1158 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0);
1159 iman |= XHCI_IMAN_INTR_PEND;
1160 xhci_rt_write_4(sc, XHCI_IMAN(0), iman);
1161 iman = xhci_rt_read_4(sc, XHCI_IMAN(0));
1162 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0);
1163 usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
1164 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0);
1165
1166 usb_schedsoftintr(&sc->sc_bus);
1167
1168 return 1;
1169}
1170
1171/*
1172 * 3 port speed types used in USB stack
1173 *
1174 * usbdi speed
1175 * definition: USB_SPEED_* in usb.h
1176 * They are used in struct usbd_device in USB stack.
1177 * ioctl interface uses these values too.
1178 * port_status speed
1179 * definition: UPS_*_SPEED in usb.h
1180 * They are used in usb_port_status_t and valid only for USB 2.0.
1181 * Speed value is always 0 for Super Speed or more, and dwExtPortStatus
1182 * of usb_port_status_ext_t indicates port speed.
1183 * Note that some 3.0 values overlap with 2.0 values.
1184 * (e.g. 0x200 means UPS_POER_POWER_SS in SS and
1185 * means UPS_LOW_SPEED in HS.)
1186 * port status returned from hub also uses these values.
1187 * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed
1188 * or more.
1189 * xspeed:
1190 * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1)
1191 * They are used in only slot context and PORTSC reg of xhci.
1192 * The difference between usbdi speed and xspeed is
1193 * that FS and LS values are swapped.
1194 */
1195
1196/* convert usbdi speed to xspeed */
1197static int
1198xhci_speed2xspeed(int speed)
1199{
1200 switch (speed) {
1201 case USB_SPEED_LOW: return 2;
1202 case USB_SPEED_FULL: return 1;
1203 default: return speed;
1204 }
1205}
1206
1207#if 0
1208/* convert xspeed to usbdi speed */
1209static int
1210xhci_xspeed2speed(int xspeed)
1211{
1212 switch (xspeed) {
1213 case 1: return USB_SPEED_FULL;
1214 case 2: return USB_SPEED_LOW;
1215 default: return xspeed;
1216 }
1217}
1218#endif
1219
1220/* convert xspeed to port status speed */
1221static int
1222xhci_xspeed2psspeed(int xspeed)
1223{
1224 switch (xspeed) {
1225 case 0: return 0;
1226 case 1: return UPS_FULL_SPEED;
1227 case 2: return UPS_LOW_SPEED;
1228 case 3: return UPS_HIGH_SPEED;
1229 default: return UPS_OTHER_SPEED;
1230 }
1231}
1232
1233/*
1234 * Construct input contexts and issue TRB to open pipe.
1235 */
1236static usbd_status
1237xhci_configure_endpoint(struct usbd_pipe *pipe)
1238{
1239 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1240 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1241 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1242 struct xhci_trb trb;
1243 usbd_status err;
1244
1245 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1246 DPRINTFN(4, "slot %u dci %u epaddr 0x%02x attr 0x%02x",
1247 xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress,
1248 pipe->up_endpoint->ue_edesc->bmAttributes);
1249
1250 /* XXX ensure input context is available? */
1251
1252 memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz);
1253
1254 /* set up context */
1255 xhci_setup_ctx(pipe);
1256
1257 hexdump("input control context", xhci_slot_get_icv(sc, xs, 0),
1258 sc->sc_ctxsz * 1);
1259 hexdump("input endpoint context", xhci_slot_get_icv(sc, xs,
1260 xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1);
1261
1262 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0);
1263 trb.trb_2 = 0;
1264 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1265 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP);
1266
1267 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
1268
1269 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
1270 hexdump("output context", xhci_slot_get_dcv(sc, xs, dci),
1271 sc->sc_ctxsz * 1);
1272
1273 return err;
1274}
1275
1276#if 0
1277static usbd_status
1278xhci_unconfigure_endpoint(struct usbd_pipe *pipe)
1279{
1280#ifdef USB_DEBUG
1281 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1282#endif
1283
1284 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1285 DPRINTFN(4, "slot %u", xs->xs_idx, 0, 0, 0);
1286
1287 return USBD_NORMAL_COMPLETION;
1288}
1289#endif
1290
1291/* 4.6.8, 6.4.3.7 */
1292static usbd_status
1293xhci_reset_endpoint_locked(struct usbd_pipe *pipe)
1294{
1295 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1296 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1297 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1298 struct xhci_trb trb;
1299 usbd_status err;
1300
1301 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1302 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0);
1303
1304 KASSERT(mutex_owned(&sc->sc_lock));
1305
1306 trb.trb_0 = 0;
1307 trb.trb_2 = 0;
1308 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1309 XHCI_TRB_3_EP_SET(dci) |
1310 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP);
1311
1312 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1313
1314 return err;
1315}
1316
1317static usbd_status
1318xhci_reset_endpoint(struct usbd_pipe *pipe)
1319{
1320 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1321
1322 mutex_enter(&sc->sc_lock);
1323 usbd_status ret = xhci_reset_endpoint_locked(pipe);
1324 mutex_exit(&sc->sc_lock);
1325
1326 return ret;
1327}
1328
1329/*
1330 * 4.6.9, 6.4.3.8
1331 * Stop execution of TDs on xfer ring.
1332 * Should be called with sc_lock held.
1333 */
1334static usbd_status
1335xhci_stop_endpoint(struct usbd_pipe *pipe)
1336{
1337 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1338 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1339 struct xhci_trb trb;
1340 usbd_status err;
1341 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1342
1343 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1344 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0);
1345
1346 KASSERT(mutex_owned(&sc->sc_lock));
1347
1348 trb.trb_0 = 0;
1349 trb.trb_2 = 0;
1350 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1351 XHCI_TRB_3_EP_SET(dci) |
1352 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP);
1353
1354 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1355
1356 return err;
1357}
1358
1359/*
1360 * Set TR Dequeue Pointer.
1361 * xHCI 1.1 4.6.10 6.4.3.9
1362 * Purge all of the TRBs on ring and reinitialize ring.
1363 * Set TR dequeue Pointr to 0 and Cycle State to 1.
1364 * EPSTATE of endpoint must be ERROR or STOPPED, otherwise CONTEXT_STATE
1365 * error will be generated.
1366 */
1367static usbd_status
1368xhci_set_dequeue_locked(struct usbd_pipe *pipe)
1369{
1370 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1371 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1372 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1373 struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr;
1374 struct xhci_trb trb;
1375 usbd_status err;
1376
1377 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1378 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0);
1379
1380 KASSERT(mutex_owned(&sc->sc_lock));
1381
1382 xhci_host_dequeue(xr);
1383
1384 /* set DCS */
1385 trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */
1386 trb.trb_2 = 0;
1387 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1388 XHCI_TRB_3_EP_SET(dci) |
1389 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE);
1390
1391 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1392
1393 return err;
1394}
1395
1396static usbd_status
1397xhci_set_dequeue(struct usbd_pipe *pipe)
1398{
1399 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1400
1401 mutex_enter(&sc->sc_lock);
1402 usbd_status ret = xhci_set_dequeue_locked(pipe);
1403 mutex_exit(&sc->sc_lock);
1404
1405 return ret;
1406}
1407
1408/*
1409 * Open new pipe: called from usbd_setup_pipe_flags.
1410 * Fills methods of pipe.
1411 * If pipe is not for ep0, calls configure_endpoint.
1412 */
1413static usbd_status
1414xhci_open(struct usbd_pipe *pipe)
1415{
1416 struct usbd_device * const dev = pipe->up_dev;
1417 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus);
1418 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
1419 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
1420
1421 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1422 DPRINTFN(1, "addr %d depth %d port %d speed %d", dev->ud_addr,
1423 dev->ud_depth, dev->ud_powersrc->up_portno, dev->ud_speed);
1424 DPRINTFN(1, " dci %u type 0x%02x epaddr 0x%02x attr 0x%02x",
1425 xhci_ep_get_dci(ed), ed->bDescriptorType, ed->bEndpointAddress,
1426 ed->bmAttributes);
1427 DPRINTFN(1, " mps %u ival %u", UGETW(ed->wMaxPacketSize), ed->bInterval,
1428 0, 0);
1429
1430 if (sc->sc_dying)
1431 return USBD_IOERROR;
1432
1433 /* Root Hub */
1434 if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) {
1435 switch (ed->bEndpointAddress) {
1436 case USB_CONTROL_ENDPOINT:
1437 pipe->up_methods = &roothub_ctrl_methods;
1438 break;
1439 case UE_DIR_IN | USBROOTHUB_INTR_ENDPT:
1440 pipe->up_methods = &xhci_root_intr_methods;
1441 break;
1442 default:
1443 pipe->up_methods = NULL;
1444 DPRINTFN(0, "bad bEndpointAddress 0x%02x",
1445 ed->bEndpointAddress, 0, 0, 0);
1446 return USBD_INVAL;
1447 }
1448 return USBD_NORMAL_COMPLETION;
1449 }
1450
1451 switch (xfertype) {
1452 case UE_CONTROL:
1453 pipe->up_methods = &xhci_device_ctrl_methods;
1454 break;
1455 case UE_ISOCHRONOUS:
1456 pipe->up_methods = &xhci_device_isoc_methods;
1457 return USBD_INVAL;
1458 break;
1459 case UE_BULK:
1460 pipe->up_methods = &xhci_device_bulk_methods;
1461 break;
1462 case UE_INTERRUPT:
1463 pipe->up_methods = &xhci_device_intr_methods;
1464 break;
1465 default:
1466 return USBD_IOERROR;
1467 break;
1468 }
1469
1470 if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT)
1471 return xhci_configure_endpoint(pipe);
1472
1473 return USBD_NORMAL_COMPLETION;
1474}
1475
1476/*
1477 * Closes pipe, called from usbd_kill_pipe via close methods.
1478 * If the endpoint to be closed is ep0, disable_slot.
1479 * Should be called with sc_lock held.
1480 */
1481static void
1482xhci_close_pipe(struct usbd_pipe *pipe)
1483{
1484 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1485 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1486 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
1487 const u_int dci = xhci_ep_get_dci(ed);
1488 struct xhci_trb trb;
1489 uint32_t *cp;
1490
1491 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1492
1493 if (sc->sc_dying)
1494 return;
1495
1496 /* xs is uninitialized before xhci_init_slot */
1497 if (xs == NULL || xs->xs_idx == 0)
1498 return;
1499
1500 DPRINTFN(4, "pipe %p slot %u dci %u", pipe, xs->xs_idx, dci, 0);
1501
1502 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx");
1503 KASSERT(mutex_owned(&sc->sc_lock));
1504
1505 if (pipe->up_dev->ud_depth == 0)
1506 return;
1507
1508 if (dci == XHCI_DCI_EP_CONTROL) {
1509 DPRINTFN(4, "closing ep0", 0, 0, 0, 0);
1510 xhci_disable_slot(sc, xs->xs_idx);
1511 return;
1512 }
1513
1514 if (xhci_get_epstate(sc, xs, dci) != XHCI_EPSTATE_STOPPED)
1515 (void)xhci_stop_endpoint(pipe);
1516
1517 /*
1518 * set appropriate bit to be dropped.
1519 * don't set DC bit to 1, otherwise all endpoints
1520 * would be deconfigured.
1521 */
1522 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL);
1523 cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci));
1524 cp[1] = htole32(0);
1525
1526 /* XXX should be most significant one, not dci? */
1527 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT));
1528 cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci));
1529
1530 /* configure ep context performs an implicit dequeue */
1531 xhci_host_dequeue(&xs->xs_ep[dci].xe_tr);
1532
1533 /* sync input contexts before they are read from memory */
1534 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE);
1535
1536 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0);
1537 trb.trb_2 = 0;
1538 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1539 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP);
1540
1541 (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1542 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
1543}
1544
1545/*
1546 * Abort transfer.
1547 * Should be called with sc_lock held.
1548 */
1549static void
1550xhci_abort_xfer(struct usbd_xfer *xfer, usbd_status status)
1551{
1552 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
1553 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
1554 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
1555
1556 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1557 DPRINTFN(4, "xfer %p pipe %p status %d",
1558 xfer, xfer->ux_pipe, status, 0);
1559
1560 KASSERT(mutex_owned(&sc->sc_lock));
1561
1562 if (sc->sc_dying) {
1563 /* If we're dying, just do the software part. */
1564 DPRINTFN(4, "xfer %p dying %u", xfer, xfer->ux_status, 0, 0);
1565 xfer->ux_status = status;
1566 callout_stop(&xfer->ux_callout);
1567 usb_transfer_complete(xfer);
1568 return;
1569 }
1570
1571 /*
1572 * If an abort is already in progress then just wait for it to
1573 * complete and return.
1574 */
1575 if (xfer->ux_hcflags & UXFER_ABORTING) {
1576 DPRINTFN(4, "already aborting", 0, 0, 0, 0);
1577#ifdef DIAGNOSTIC
1578 if (status == USBD_TIMEOUT)
1579 DPRINTFN(4, "TIMEOUT while aborting", 0, 0, 0, 0);
1580#endif
1581 /* Override the status which might be USBD_TIMEOUT. */
1582 xfer->ux_status = status;
1583 DPRINTFN(4, "xfer %p waiting for abort to finish", xfer, 0, 0,
1584 0);
1585 xfer->ux_hcflags |= UXFER_ABORTWAIT;
1586 while (xfer->ux_hcflags & UXFER_ABORTING)
1587 cv_wait(&xfer->ux_hccv, &sc->sc_lock);
1588 return;
1589 }
1590 xfer->ux_hcflags |= UXFER_ABORTING;
1591
1592 /*
1593 * Step 1: Stop xfer timeout timer.
1594 */
1595 xfer->ux_status = status;
1596 callout_stop(&xfer->ux_callout);
1597
1598 /*
1599 * Step 2: Stop execution of TD on the ring.
1600 */
1601 switch (xhci_get_epstate(sc, xs, dci)) {
1602 case XHCI_EPSTATE_HALTED:
1603 (void)xhci_reset_endpoint_locked(xfer->ux_pipe);
1604 break;
1605 case XHCI_EPSTATE_STOPPED:
1606 break;
1607 default:
1608 (void)xhci_stop_endpoint(xfer->ux_pipe);
1609 break;
1610 }
1611#ifdef DIAGNOSTIC
1612 uint32_t epst = xhci_get_epstate(sc, xs, dci);
1613 if (epst != XHCI_EPSTATE_STOPPED)
1614 DPRINTFN(4, "dci %u not stopped %u", dci, epst, 0, 0);
1615#endif
1616
1617 /*
1618 * Step 3: Remove any vestiges of the xfer from the ring.
1619 */
1620 xhci_set_dequeue_locked(xfer->ux_pipe);
1621
1622 /*
1623 * Step 4: Notify completion to waiting xfers.
1624 */
1625 int wake = xfer->ux_hcflags & UXFER_ABORTWAIT;
1626 xfer->ux_hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
1627 usb_transfer_complete(xfer);
1628 if (wake) {
1629 cv_broadcast(&xfer->ux_hccv);
1630 }
1631 DPRINTFN(14, "end", 0, 0, 0, 0);
1632
1633 KASSERT(mutex_owned(&sc->sc_lock));
1634}
1635
1636static void
1637xhci_host_dequeue(struct xhci_ring * const xr)
1638{
1639 /* When dequeueing the controller, update our struct copy too */
1640 memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE);
1641 usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE,
1642 BUS_DMASYNC_PREWRITE);
1643 memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies));
1644
1645 xr->xr_ep = 0;
1646 xr->xr_cs = 1;
1647}
1648
1649/*
1650 * Recover STALLed endpoint.
1651 * xHCI 1.1 sect 4.10.2.1
1652 * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove
1653 * all transfers on transfer ring.
1654 * These are done in thread context asynchronously.
1655 */
1656static void
1657xhci_clear_endpoint_stall_async_task(void *cookie)
1658{
1659 struct usbd_xfer * const xfer = cookie;
1660 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
1661 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
1662 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
1663 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr;
1664
1665 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1666 DPRINTFN(4, "xfer %p slot %u dci %u", xfer, xs->xs_idx, dci, 0);
1667
1668 xhci_reset_endpoint(xfer->ux_pipe);
1669 xhci_set_dequeue(xfer->ux_pipe);
1670
1671 mutex_enter(&sc->sc_lock);
1672 tr->is_halted = false;
1673 usb_transfer_complete(xfer);
1674 mutex_exit(&sc->sc_lock);
1675 DPRINTFN(4, "ends", 0, 0, 0, 0);
1676}
1677
1678static usbd_status
1679xhci_clear_endpoint_stall_async(struct usbd_xfer *xfer)
1680{
1681 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
1682 struct xhci_pipe * const xp = (struct xhci_pipe *)xfer->ux_pipe;
1683
1684 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1685 DPRINTFN(4, "xfer %p", xfer, 0, 0, 0);
1686
1687 if (sc->sc_dying) {
1688 return USBD_IOERROR;
1689 }
1690
1691 usb_init_task(&xp->xp_async_task,
1692 xhci_clear_endpoint_stall_async_task, xfer, USB_TASKQ_MPSAFE);
1693 usb_add_task(xfer->ux_pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC);
1694 DPRINTFN(4, "ends", 0, 0, 0, 0);
1695
1696 return USBD_NORMAL_COMPLETION;
1697}
1698
1699/* Process roothub port status/change events and notify to uhub_intr. */
1700static void
1701xhci_rhpsc(struct xhci_softc * const sc, u_int port)
1702{
1703 struct usbd_xfer * const xfer = sc->sc_intrxfer;
1704 uint8_t *p;
1705
1706 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1707 DPRINTFN(4, "xhci%d: port %u status change", device_unit(sc->sc_dev),
1708 port, 0, 0);
1709
1710 if (xfer == NULL)
1711 return;
1712
1713 if (port > sc->sc_maxports)
1714 return;
1715
1716 p = xfer->ux_buf;
1717 memset(p, 0, xfer->ux_length);
1718 p[port/NBBY] |= 1 << (port%NBBY);
1719 xfer->ux_actlen = xfer->ux_length;
1720 xfer->ux_status = USBD_NORMAL_COMPLETION;
1721 usb_transfer_complete(xfer);
1722}
1723
1724/* Process Transfer Events */
1725static void
1726xhci_event_transfer(struct xhci_softc * const sc,
1727 const struct xhci_trb * const trb)
1728{
1729 uint64_t trb_0;
1730 uint32_t trb_2, trb_3;
1731 uint8_t trbcode;
1732 u_int slot, dci;
1733 struct xhci_slot *xs;
1734 struct xhci_ring *xr;
1735 struct xhci_xfer *xx;
1736 struct usbd_xfer *xfer;
1737 usbd_status err;
1738
1739 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1740
1741 trb_0 = le64toh(trb->trb_0);
1742 trb_2 = le32toh(trb->trb_2);
1743 trb_3 = le32toh(trb->trb_3);
1744 trbcode = XHCI_TRB_2_ERROR_GET(trb_2);
1745 slot = XHCI_TRB_3_SLOT_GET(trb_3);
1746 dci = XHCI_TRB_3_EP_GET(trb_3);
1747 xs = &sc->sc_slots[slot];
1748 xr = &xs->xs_ep[dci].xe_tr;
1749
1750 /* sanity check */
1751 KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots,
1752 "invalid xs_idx %u slot %u", xs->xs_idx, slot);
1753
1754 int idx = 0;
1755 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) {
1756 if (xhci_trb_get_idx(xr, trb_0, &idx)) {
1757 DPRINTFN(0, "invalid trb_0 0x%"PRIx64, trb_0, 0, 0, 0);
1758 return;
1759 }
1760 xx = xr->xr_cookies[idx];
1761
1762 /* clear cookie of consumed TRB */
1763 xr->xr_cookies[idx] = NULL;
1764
1765 /*
1766 * xx is NULL if pipe is opened but xfer is not started.
1767 * It happens when stopping idle pipe.
1768 */
1769 if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) {
1770 DPRINTFN(1, "Ignore #%u: cookie %p cc %u dci %u",
1771 idx, xx, trbcode, dci);
1772 DPRINTFN(1, " orig TRB %"PRIx64" type %u", trb_0,
1773 XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)),
1774 0, 0);
1775 return;
1776 }
1777 } else {
1778 /* When ED != 0, trb_0 is virtual addr of struct xhci_xfer. */
1779 xx = (void *)(uintptr_t)(trb_0 & ~0x3);
1780 }
1781 /* XXX this may not happen */
1782 if (xx == NULL) {
1783 DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0);
1784 return;
1785 }
1786 xfer = &xx->xx_xfer;
1787 /* XXX this may happen when detaching */
1788 if (xfer == NULL) {
1789 DPRINTFN(1, "xx(%p)->xx_xfer is NULL trb_0 %#"PRIx64,
1790 xx, trb_0, 0, 0);
1791 return;
1792 }
1793 DPRINTFN(14, "xfer %p", xfer, 0, 0, 0);
1794 /* XXX I dunno why this happens */
1795 KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer);
1796
1797 if (!xfer->ux_pipe->up_repeat &&
1798 SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) {
1799 DPRINTFN(1, "xfer(%p)->pipe not queued", xfer, 0, 0, 0);
1800 return;
1801 }
1802
1803 /* 4.11.5.2 Event Data TRB */
1804 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) {
1805 DPRINTFN(14, "transfer Event Data: 0x%016"PRIx64" 0x%08"PRIx32
1806 " %02x", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0);
1807 if ((trb_0 & 0x3) == 0x3) {
1808 xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2);
1809 }
1810 }
1811
1812 switch (trbcode) {
1813 case XHCI_TRB_ERROR_SHORT_PKT:
1814 case XHCI_TRB_ERROR_SUCCESS:
1815 /*
1816 * A ctrl transfer can generate two events if it has a Data
1817 * stage. A short data stage can be OK and should not
1818 * complete the transfer as the status stage needs to be
1819 * performed.
1820 *
1821 * Note: Data and Status stage events point at same xfer.
1822 * ux_actlen and ux_dmabuf will be passed to
1823 * usb_transfer_complete after the Status stage event.
1824 *
1825 * It can be distingished which stage generates the event:
1826 * + by checking least 3 bits of trb_0 if ED==1.
1827 * (see xhci_device_ctrl_start).
1828 * + by checking the type of original TRB if ED==0.
1829 *
1830 * In addition, intr, bulk, and isoc transfer currently
1831 * consists of single TD, so the "skip" is not needed.
1832 * ctrl xfer uses EVENT_DATA, and others do not.
1833 * Thus driver can switch the flow by checking ED bit.
1834 */
1835 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) {
1836 if (xfer->ux_actlen == 0)
1837 xfer->ux_actlen = xfer->ux_length -
1838 XHCI_TRB_2_REM_GET(trb_2);
1839 if (XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3))
1840 == XHCI_TRB_TYPE_DATA_STAGE) {
1841 return;
1842 }
1843 } else if ((trb_0 & 0x3) == 0x3) {
1844 return;
1845 }
1846 err = USBD_NORMAL_COMPLETION;
1847 break;
1848 case XHCI_TRB_ERROR_STOPPED:
1849 case XHCI_TRB_ERROR_LENGTH:
1850 case XHCI_TRB_ERROR_STOPPED_SHORT:
1851 /*
1852 * don't complete the transfer being aborted
1853 * as abort_xfer does instead.
1854 */
1855 if (xfer->ux_hcflags & UXFER_ABORTING) {
1856 DPRINTFN(14, "ignore aborting xfer %p", xfer, 0, 0, 0);
1857 return;
1858 }
1859 err = USBD_CANCELLED;
1860 break;
1861 case XHCI_TRB_ERROR_STALL:
1862 case XHCI_TRB_ERROR_BABBLE:
1863 DPRINTFN(1, "ERR %u slot %u dci %u", trbcode, slot, dci, 0);
1864 xr->is_halted = true;
1865 err = USBD_STALLED;
1866 /*
1867 * Stalled endpoints can be recoverd by issuing
1868 * command TRB TYPE_RESET_EP on xHCI instead of
1869 * issuing request CLEAR_FEATURE UF_ENDPOINT_HALT
1870 * on the endpoint. However, this function may be
1871 * called from softint context (e.g. from umass),
1872 * in that case driver gets KASSERT in cv_timedwait
1873 * in xhci_do_command.
1874 * To avoid this, this runs reset_endpoint and
1875 * usb_transfer_complete in usb task thread
1876 * asynchronously (and then umass issues clear
1877 * UF_ENDPOINT_HALT).
1878 */
1879 xfer->ux_status = err;
1880 callout_stop(&xfer->ux_callout);
1881 xhci_clear_endpoint_stall_async(xfer);
1882 return;
1883 default:
1884 DPRINTFN(1, "ERR %u slot %u dci %u", trbcode, slot, dci, 0);
1885 err = USBD_IOERROR;
1886 break;
1887 }
1888 xfer->ux_status = err;
1889
1890 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) {
1891 if ((trb_0 & 0x3) == 0x0) {
1892 callout_stop(&xfer->ux_callout);
1893 usb_transfer_complete(xfer);
1894 }
1895 } else {
1896 callout_stop(&xfer->ux_callout);
1897 usb_transfer_complete(xfer);
1898 }
1899}
1900
1901/* Process Command complete events */
1902static void
1903xhci_event_cmd(struct xhci_softc * const sc, const struct xhci_trb * const trb)
1904{
1905 uint64_t trb_0;
1906 uint32_t trb_2, trb_3;
1907
1908 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1909
1910 trb_0 = le64toh(trb->trb_0);
1911 trb_2 = le32toh(trb->trb_2);
1912 trb_3 = le32toh(trb->trb_3);
1913
1914 if (trb_0 == sc->sc_command_addr) {
1915 sc->sc_result_trb.trb_0 = trb_0;
1916 sc->sc_result_trb.trb_2 = trb_2;
1917 sc->sc_result_trb.trb_3 = trb_3;
1918 if (XHCI_TRB_2_ERROR_GET(trb_2) !=
1919 XHCI_TRB_ERROR_SUCCESS) {
1920 DPRINTFN(1, "command completion "
1921 "failure: 0x%016"PRIx64" 0x%08"PRIx32" "
1922 "0x%08"PRIx32, trb_0, trb_2, trb_3, 0);
1923 }
1924 cv_signal(&sc->sc_command_cv);
1925 } else {
1926 DPRINTFN(1, "spurious event: %p 0x%016"PRIx64" "
1927 "0x%08"PRIx32" 0x%08"PRIx32, trb, trb_0,
1928 trb_2, trb_3);
1929 }
1930}
1931
1932/*
1933 * Process events.
1934 * called from xhci_softintr
1935 */
1936static void
1937xhci_handle_event(struct xhci_softc * const sc,
1938 const struct xhci_trb * const trb)
1939{
1940 uint64_t trb_0;
1941 uint32_t trb_2, trb_3;
1942
1943 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1944
1945 trb_0 = le64toh(trb->trb_0);
1946 trb_2 = le32toh(trb->trb_2);
1947 trb_3 = le32toh(trb->trb_3);
1948
1949 DPRINTFN(14, "event: %p 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32,
1950 trb, trb_0, trb_2, trb_3);
1951
1952 /*
1953 * 4.11.3.1, 6.4.2.1
1954 * TRB Pointer is invalid for these completion codes.
1955 */
1956 switch (XHCI_TRB_2_ERROR_GET(trb_2)) {
1957 case XHCI_TRB_ERROR_RING_UNDERRUN:
1958 case XHCI_TRB_ERROR_RING_OVERRUN:
1959 case XHCI_TRB_ERROR_VF_RING_FULL:
1960 return;
1961 default:
1962 if (trb_0 == 0) {
1963 return;
1964 }
1965 break;
1966 }
1967
1968 switch (XHCI_TRB_3_TYPE_GET(trb_3)) {
1969 case XHCI_TRB_EVENT_TRANSFER:
1970 xhci_event_transfer(sc, trb);
1971 break;
1972 case XHCI_TRB_EVENT_CMD_COMPLETE:
1973 xhci_event_cmd(sc, trb);
1974 break;
1975 case XHCI_TRB_EVENT_PORT_STS_CHANGE:
1976 xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff));
1977 break;
1978 default:
1979 break;
1980 }
1981}
1982
1983static void
1984xhci_softintr(void *v)
1985{
1986 struct usbd_bus * const bus = v;
1987 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
1988 struct xhci_ring * const er = &sc->sc_er;
1989 struct xhci_trb *trb;
1990 int i, j, k;
1991
1992 XHCIHIST_FUNC(); XHCIHIST_CALLED();
1993
1994 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
1995
1996 i = er->xr_ep;
1997 j = er->xr_cs;
1998
1999 DPRINTFN(16, "er: xr_ep %d xr_cs %d", i, j, 0, 0);
2000
2001 while (1) {
2002 usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE,
2003 BUS_DMASYNC_POSTREAD);
2004 trb = &er->xr_trb[i];
2005 k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
2006
2007 if (j != k)
2008 break;
2009
2010 xhci_handle_event(sc, trb);
2011
2012 i++;
2013 if (i == er->xr_ntrb) {
2014 i = 0;
2015 j ^= 1;
2016 }
2017 }
2018
2019 er->xr_ep = i;
2020 er->xr_cs = j;
2021
2022 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) |
2023 XHCI_ERDP_LO_BUSY);
2024
2025 DPRINTFN(16, "ends", 0, 0, 0, 0);
2026
2027 return;
2028}
2029
2030static void
2031xhci_poll(struct usbd_bus *bus)
2032{
2033 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2034
2035 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2036
2037 mutex_spin_enter(&sc->sc_intr_lock);
2038 xhci_intr1(sc);
2039 mutex_spin_exit(&sc->sc_intr_lock);
2040
2041 return;
2042}
2043
2044static struct usbd_xfer *
2045xhci_allocx(struct usbd_bus *bus, unsigned int nframes)
2046{
2047 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2048 struct usbd_xfer *xfer;
2049
2050 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2051
2052 xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT);
2053 if (xfer != NULL) {
2054 memset(xfer, 0, sizeof(struct xhci_xfer));
2055#ifdef DIAGNOSTIC
2056 xfer->ux_state = XFER_BUSY;
2057#endif
2058 }
2059
2060 return xfer;
2061}
2062
2063static void
2064xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
2065{
2066 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2067
2068 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2069
2070#ifdef DIAGNOSTIC
2071 if (xfer->ux_state != XFER_BUSY) {
2072 DPRINTFN(0, "xfer=%p not busy, 0x%08x",
2073 xfer, xfer->ux_state, 0, 0);
2074 }
2075 xfer->ux_state = XFER_FREE;
2076#endif
2077 pool_cache_put(sc->sc_xferpool, xfer);
2078}
2079
2080static void
2081xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock)
2082{
2083 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2084
2085 *lock = &sc->sc_lock;
2086}
2087
2088extern uint32_t usb_cookie_no;
2089
2090/*
2091 * xHCI 4.3
2092 * Called when uhub_explore finds a new device (via usbd_new_device).
2093 * Port initialization and speed detection (4.3.1) are already done in uhub.c.
2094 * This function does:
2095 * Allocate and construct dev structure of default endpoint (ep0).
2096 * Allocate and open pipe of ep0.
2097 * Enable slot and initialize slot context.
2098 * Set Address.
2099 * Read initial device descriptor.
2100 * Determine initial MaxPacketSize (mps) by speed.
2101 * Read full device descriptor.
2102 * Register this device.
2103 * Finally state of device transitions ADDRESSED.
2104 */
2105static usbd_status
2106xhci_new_device(device_t parent, struct usbd_bus *bus, int depth,
2107 int speed, int port, struct usbd_port *up)
2108{
2109 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2110 struct usbd_device *dev;
2111 usbd_status err;
2112 usb_device_descriptor_t *dd;
2113 struct xhci_slot *xs;
2114 uint32_t *cp;
2115
2116 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2117 DPRINTFN(4, "port %u depth %u speed %u up %p", port, depth, speed, up);
2118
2119 dev = kmem_zalloc(sizeof(*dev), KM_SLEEP);
2120 if (dev == NULL)
2121 return USBD_NOMEM;
2122
2123 dev->ud_bus = bus;
2124 dev->ud_quirks = &usbd_no_quirk;
2125 dev->ud_addr = 0;
2126 dev->ud_ddesc.bMaxPacketSize = 0;
2127 dev->ud_depth = depth;
2128 dev->ud_powersrc = up;
2129 dev->ud_myhub = up->up_parent;
2130 dev->ud_speed = speed;
2131 dev->ud_langid = USBD_NOLANG;
2132 dev->ud_cookie.cookie = ++usb_cookie_no;
2133
2134 /* Set up default endpoint handle. */
2135 dev->ud_ep0.ue_edesc = &dev->ud_ep0desc;
2136 /* doesn't matter, just don't let it uninitialized */
2137 dev->ud_ep0.ue_toggle = 0;
2138
2139 /* Set up default endpoint descriptor. */
2140 dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE;
2141 dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT;
2142 dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT;
2143 dev->ud_ep0desc.bmAttributes = UE_CONTROL;
2144 dev->ud_ep0desc.bInterval = 0;
2145
2146 /* 4.3, 4.8.2.1 */
2147 switch (speed) {
2148 case USB_SPEED_SUPER:
2149 case USB_SPEED_SUPER_PLUS:
2150 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET);
2151 break;
2152 case USB_SPEED_FULL:
2153 /* XXX using 64 as initial mps of ep0 in FS */
2154 case USB_SPEED_HIGH:
2155 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET);
2156 break;
2157 case USB_SPEED_LOW:
2158 default:
2159 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET);
2160 break;
2161 }
2162
2163 up->up_dev = dev;
2164
2165 /* Establish the default pipe. */
2166 err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL,
2167 &dev->ud_pipe0);
2168 if (err) {
2169 goto bad;
2170 }
2171
2172 dd = &dev->ud_ddesc;
2173
2174 if ((depth == 0) && (port == 0)) {
2175 KASSERT(bus->ub_devices[dev->ud_addr] == NULL);
2176 bus->ub_devices[dev->ud_addr] = dev;
2177 err = usbd_get_initial_ddesc(dev, dd);
2178 if (err) {
2179 DPRINTFN(1, "get_initial_ddesc %u", err, 0, 0, 0);
2180 goto bad;
2181 }
2182
2183 err = usbd_reload_device_desc(dev);
2184 if (err) {
2185 DPRINTFN(1, "reload desc %u", err, 0, 0, 0);
2186 goto bad;
2187 }
2188 } else {
2189 uint8_t slot = 0;
2190
2191 /* 4.3.2 */
2192 err = xhci_enable_slot(sc, &slot);
2193 if (err) {
2194 DPRINTFN(1, "enable slot %u", err, 0, 0, 0);
2195 goto bad;
2196 }
2197
2198 xs = &sc->sc_slots[slot];
2199 dev->ud_hcpriv = xs;
2200
2201 /* 4.3.3 initialize slot structure */
2202 err = xhci_init_slot(dev, slot);
2203 if (err) {
2204 DPRINTFN(1, "init slot %u", err, 0, 0, 0);
2205 dev->ud_hcpriv = NULL;
2206 /*
2207 * We have to disable_slot here because
2208 * xs->xs_idx == 0 when xhci_init_slot fails,
2209 * in that case usbd_remove_dev won't work.
2210 */
2211 mutex_enter(&sc->sc_lock);
2212 xhci_disable_slot(sc, slot);
2213 mutex_exit(&sc->sc_lock);
2214 goto bad;
2215 }
2216
2217 /* 4.3.4 Address Assignment */
2218 err = xhci_set_address(dev, slot, false);
2219 if (err) {
2220 DPRINTFN(1, "set address w/o bsr %u", err, 0, 0, 0);
2221 goto bad;
2222 }
2223
2224 /* Allow device time to set new address */
2225 usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE);
2226
2227 cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT);
2228 //hexdump("slot context", cp, sc->sc_ctxsz);
2229 uint8_t addr = XHCI_SCTX_3_DEV_ADDR_GET(le32toh(cp[3]));
2230 DPRINTFN(4, "device address %u", addr, 0, 0, 0);
2231 /* XXX ensure we know when the hardware does something
2232 we can't yet cope with */
2233 KASSERTMSG(addr >= 1 && addr <= 127, "addr %d", addr);
2234 dev->ud_addr = addr;
2235 /* XXX dev->ud_addr not necessarily unique on bus */
2236 KASSERT(bus->ub_devices[dev->ud_addr] == NULL);
2237 bus->ub_devices[dev->ud_addr] = dev;
2238
2239 err = usbd_get_initial_ddesc(dev, dd);
2240 if (err) {
2241 DPRINTFN(1, "get_initial_ddesc %u", err, 0, 0, 0);
2242 goto bad;
2243 }
2244
2245 /* 4.8.2.1 */
2246 if (USB_IS_SS(speed)) {
2247 if (dd->bMaxPacketSize != 9) {
2248 printf("%s: invalid mps 2^%u for SS ep0,"
2249 " using 512\n",
2250 device_xname(sc->sc_dev),
2251 dd->bMaxPacketSize);
2252 dd->bMaxPacketSize = 9;
2253 }
2254 USETW(dev->ud_ep0desc.wMaxPacketSize,
2255 (1 << dd->bMaxPacketSize));
2256 } else
2257 USETW(dev->ud_ep0desc.wMaxPacketSize,
2258 dd->bMaxPacketSize);
2259 DPRINTFN(4, "bMaxPacketSize %u", dd->bMaxPacketSize, 0, 0, 0);
2260 err = xhci_update_ep0_mps(sc, xs,
2261 UGETW(dev->ud_ep0desc.wMaxPacketSize));
2262 if (err) {
2263 DPRINTFN(1, "update mps of ep0 %u", err, 0, 0, 0);
2264 goto bad;
2265 }
2266
2267 err = usbd_reload_device_desc(dev);
2268 if (err) {
2269 DPRINTFN(1, "reload desc %u", err, 0, 0, 0);
2270 goto bad;
2271 }
2272 }
2273
2274 DPRINTFN(1, "adding unit addr=%d, rev=%02x,",
2275 dev->ud_addr, UGETW(dd->bcdUSB), 0, 0);
2276 DPRINTFN(1, " class=%d, subclass=%d, protocol=%d,",
2277 dd->bDeviceClass, dd->bDeviceSubClass,
2278 dd->bDeviceProtocol, 0);
2279 DPRINTFN(1, " mps=%d, len=%d, noconf=%d, speed=%d",
2280 dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations,
2281 dev->ud_speed);
2282
2283 usbd_get_device_strings(dev);
2284
2285 usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev);
2286
2287 if ((depth == 0) && (port == 0)) {
2288 usbd_attach_roothub(parent, dev);
2289 DPRINTFN(1, "root_hub %p", bus->ub_roothub, 0, 0, 0);
2290 return USBD_NORMAL_COMPLETION;
2291 }
2292
2293
2294 err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr);
2295 bad:
2296 if (err != USBD_NORMAL_COMPLETION) {
2297 usbd_remove_device(dev, up);
2298 }
2299
2300 return err;
2301}
2302
2303static usbd_status
2304xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr,
2305 size_t ntrb, size_t align)
2306{
2307 usbd_status err;
2308 size_t size = ntrb * XHCI_TRB_SIZE;
2309
2310 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2311
2312 err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma);
2313 if (err)
2314 return err;
2315 mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
2316 xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP);
2317 xr->xr_trb = xhci_ring_trbv(xr, 0);
2318 xr->xr_ntrb = ntrb;
2319 xr->is_halted = false;
2320 xhci_host_dequeue(xr);
2321
2322 return USBD_NORMAL_COMPLETION;
2323}
2324
2325static void
2326xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr)
2327{
2328 usb_freemem(&sc->sc_bus, &xr->xr_dma);
2329 mutex_destroy(&xr->xr_lock);
2330 kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb);
2331}
2332
2333static void
2334xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr,
2335 void *cookie, struct xhci_trb * const trbs, size_t ntrbs)
2336{
2337 size_t i;
2338 u_int ri;
2339 u_int cs;
2340 uint64_t parameter;
2341 uint32_t status;
2342 uint32_t control;
2343
2344 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2345
2346 KASSERTMSG(ntrbs <= XHCI_XFER_NTRB, "ntrbs %zu", ntrbs);
2347 for (i = 0; i < ntrbs; i++) {
2348 DPRINTFN(12, "xr %p trbs %p num %zu", xr, trbs, i, 0);
2349 DPRINTFN(12, " %016"PRIx64" %08"PRIx32" %08"PRIx32,
2350 trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0);
2351 KASSERTMSG(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) !=
2352 XHCI_TRB_TYPE_LINK, "trbs[%zu].trb3 %#x", i, trbs[i].trb_3);
2353 }
2354
2355 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0);
2356
2357 ri = xr->xr_ep;
2358 cs = xr->xr_cs;
2359
2360 /*
2361 * Although the xhci hardware can do scatter/gather dma from
2362 * arbitrary sized buffers, there is a non-obvious restriction
2363 * that a LINK trb is only allowed at the end of a burst of
2364 * transfers - which might be 16kB.
2365 * Arbitrary aligned LINK trb definitely fail on Ivy bridge.
2366 * The simple solution is not to allow a LINK trb in the middle
2367 * of anything - as here.
2368 * XXX: (dsl) There are xhci controllers out there (eg some made by
2369 * ASMedia) that seem to lock up if they process a LINK trb but
2370 * cannot process the linked-to trb yet.
2371 * The code should write the 'cycle' bit on the link trb AFTER
2372 * adding the other trb.
2373 */
2374 u_int firstep = xr->xr_ep;
2375 u_int firstcs = xr->xr_cs;
2376
2377 for (i = 0; i < ntrbs; ) {
2378 u_int oldri = ri;
2379 u_int oldcs = cs;
2380
2381 if (ri >= (xr->xr_ntrb - 1)) {
2382 /* Put Link TD at the end of ring */
2383 parameter = xhci_ring_trbp(xr, 0);
2384 status = 0;
2385 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
2386 XHCI_TRB_3_TC_BIT;
2387 xr->xr_cookies[ri] = NULL;
2388 xr->xr_ep = 0;
2389 xr->xr_cs ^= 1;
2390 ri = xr->xr_ep;
2391 cs = xr->xr_cs;
2392 } else {
2393 parameter = trbs[i].trb_0;
2394 status = trbs[i].trb_2;
2395 control = trbs[i].trb_3;
2396
2397 xr->xr_cookies[ri] = cookie;
2398 ri++;
2399 i++;
2400 }
2401 /*
2402 * If this is a first TRB, mark it invalid to prevent
2403 * xHC from running it immediately.
2404 */
2405 if (oldri == firstep) {
2406 if (oldcs) {
2407 control &= ~XHCI_TRB_3_CYCLE_BIT;
2408 } else {
2409 control |= XHCI_TRB_3_CYCLE_BIT;
2410 }
2411 } else {
2412 if (oldcs) {
2413 control |= XHCI_TRB_3_CYCLE_BIT;
2414 } else {
2415 control &= ~XHCI_TRB_3_CYCLE_BIT;
2416 }
2417 }
2418 xhci_trb_put(&xr->xr_trb[oldri], parameter, status, control);
2419 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * oldri,
2420 XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE);
2421 }
2422
2423 /* Now invert cycle bit of first TRB */
2424 if (firstcs) {
2425 xr->xr_trb[firstep].trb_3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
2426 } else {
2427 xr->xr_trb[firstep].trb_3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
2428 }
2429 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * firstep,
2430 XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE);
2431
2432 xr->xr_ep = ri;
2433 xr->xr_cs = cs;
2434
2435 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0);
2436}
2437
2438/*
2439 * Stop execution commands, purge all commands on command ring, and
2440 * rewind dequeue pointer.
2441 */
2442static void
2443xhci_abort_command(struct xhci_softc *sc)
2444{
2445 struct xhci_ring * const cr = &sc->sc_cr;
2446 uint64_t crcr;
2447 int i;
2448
2449 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2450 DPRINTFN(14, "command %#"PRIx64" timeout, aborting",
2451 sc->sc_command_addr, 0, 0, 0);
2452
2453 mutex_enter(&cr->xr_lock);
2454
2455 /* 4.6.1.2 Aborting a Command */
2456 crcr = xhci_op_read_8(sc, XHCI_CRCR);
2457 xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA);
2458
2459 for (i = 0; i < 500; i++) {
2460 crcr = xhci_op_read_8(sc, XHCI_CRCR);
2461 if ((crcr & XHCI_CRCR_LO_CRR) == 0)
2462 break;
2463 usb_delay_ms(&sc->sc_bus, 1);
2464 }
2465 if ((crcr & XHCI_CRCR_LO_CRR) != 0) {
2466 DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0);
2467 /* reset HC here? */
2468 }
2469
2470 /* reset command ring dequeue pointer */
2471 cr->xr_ep = 0;
2472 cr->xr_cs = 1;
2473 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs);
2474
2475 mutex_exit(&cr->xr_lock);
2476}
2477
2478/*
2479 * Put a command on command ring, ring bell, set timer, and cv_timedwait.
2480 * Command completion is notified by cv_signal from xhci_event_cmd()
2481 * (called from xhci_softint), or timed-out.
2482 * The completion code is copied to sc->sc_result_trb in xhci_event_cmd(),
2483 * then do_command examines it.
2484 */
2485static usbd_status
2486xhci_do_command_locked(struct xhci_softc * const sc,
2487 struct xhci_trb * const trb, int timeout)
2488{
2489 struct xhci_ring * const cr = &sc->sc_cr;
2490 usbd_status err;
2491
2492 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2493 DPRINTFN(12, "input: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32,
2494 trb->trb_0, trb->trb_2, trb->trb_3, 0);
2495
2496 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx");
2497 KASSERT(mutex_owned(&sc->sc_lock));
2498
2499 /* XXX KASSERT may fire when cv_timedwait unlocks sc_lock */
2500 KASSERT(sc->sc_command_addr == 0);
2501 /*
2502 * If enqueue pointer points at last of ring, it's Link TRB,
2503 * command TRB will be stored in 0th TRB.
2504 */
2505 if (cr->xr_ep == cr->xr_ntrb - 1)
2506 sc->sc_command_addr = xhci_ring_trbp(cr, 0);
2507 else
2508 sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep);
2509
2510 mutex_enter(&cr->xr_lock);
2511 xhci_ring_put(sc, cr, NULL, trb, 1);
2512 mutex_exit(&cr->xr_lock);
2513
2514 xhci_db_write_4(sc, XHCI_DOORBELL(0), 0);
2515
2516 if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock,
2517 MAX(1, mstohz(timeout))) == EWOULDBLOCK) {
2518 xhci_abort_command(sc);
2519 err = USBD_TIMEOUT;
2520 goto timedout;
2521 }
2522
2523 trb->trb_0 = sc->sc_result_trb.trb_0;
2524 trb->trb_2 = sc->sc_result_trb.trb_2;
2525 trb->trb_3 = sc->sc_result_trb.trb_3;
2526
2527 DPRINTFN(12, "output: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"",
2528 trb->trb_0, trb->trb_2, trb->trb_3, 0);
2529
2530 switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) {
2531 case XHCI_TRB_ERROR_SUCCESS:
2532 err = USBD_NORMAL_COMPLETION;
2533 break;
2534 default:
2535 case 192 ... 223:
2536 err = USBD_IOERROR;
2537 break;
2538 case 224 ... 255:
2539 err = USBD_NORMAL_COMPLETION;
2540 break;
2541 }
2542
2543timedout:
2544 sc->sc_command_addr = 0;
2545 return err;
2546}
2547
2548static usbd_status
2549xhci_do_command(struct xhci_softc * const sc, struct xhci_trb * const trb,
2550 int timeout)
2551{
2552
2553 mutex_enter(&sc->sc_lock);
2554 usbd_status ret = xhci_do_command_locked(sc, trb, timeout);
2555 mutex_exit(&sc->sc_lock);
2556
2557 return ret;
2558}
2559
2560static usbd_status
2561xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp)
2562{
2563 struct xhci_trb trb;
2564 usbd_status err;
2565
2566 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2567
2568 trb.trb_0 = 0;
2569 trb.trb_2 = 0;
2570 trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT);
2571
2572 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
2573 if (err != USBD_NORMAL_COMPLETION) {
2574 return err;
2575 }
2576
2577 *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3);
2578
2579 return err;
2580}
2581
2582/*
2583 * xHCI 4.6.4
2584 * Deallocate ring and device/input context DMA buffers, and disable_slot.
2585 * All endpoints in the slot should be stopped.
2586 * Should be called with sc_lock held.
2587 */
2588static usbd_status
2589xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot)
2590{
2591 struct xhci_trb trb;
2592 struct xhci_slot *xs;
2593 usbd_status err;
2594
2595 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2596
2597 if (sc->sc_dying)
2598 return USBD_IOERROR;
2599
2600 trb.trb_0 = 0;
2601 trb.trb_2 = 0;
2602 trb.trb_3 = htole32(
2603 XHCI_TRB_3_SLOT_SET(slot) |
2604 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT));
2605
2606 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
2607
2608 if (!err) {
2609 xs = &sc->sc_slots[slot];
2610 if (xs->xs_idx != 0) {
2611 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, 32);
2612 xhci_set_dcba(sc, 0, slot);
2613 memset(xs, 0, sizeof(*xs));
2614 }
2615 }
2616
2617 return err;
2618}
2619
2620/*
2621 * Set address of device and transition slot state from ENABLED to ADDRESSED
2622 * if Block Setaddress Request (BSR) is false.
2623 * If BSR==true, transition slot state from ENABLED to DEFAULT.
2624 * see xHCI 1.1 4.5.3, 3.3.4
2625 * Should be called without sc_lock held.
2626 */
2627static usbd_status
2628xhci_address_device(struct xhci_softc * const sc,
2629 uint64_t icp, uint8_t slot_id, bool bsr)
2630{
2631 struct xhci_trb trb;
2632 usbd_status err;
2633
2634 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2635
2636 trb.trb_0 = icp;
2637 trb.trb_2 = 0;
2638 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) |
2639 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) |
2640 (bsr ? XHCI_TRB_3_BSR_BIT : 0);
2641
2642 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
2643
2644 if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS)
2645 err = USBD_NO_ADDR;
2646
2647 return err;
2648}
2649
2650static usbd_status
2651xhci_update_ep0_mps(struct xhci_softc * const sc,
2652 struct xhci_slot * const xs, u_int mps)
2653{
2654 struct xhci_trb trb;
2655 usbd_status err;
2656 uint32_t * cp;
2657
2658 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2659 DPRINTFN(4, "slot %u mps %u", xs->xs_idx, mps, 0, 0);
2660
2661 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL);
2662 cp[0] = htole32(0);
2663 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL));
2664
2665 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL));
2666 cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps));
2667
2668 /* sync input contexts before they are read from memory */
2669 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE);
2670 hexdump("input context", xhci_slot_get_icv(sc, xs, 0),
2671 sc->sc_ctxsz * 4);
2672
2673 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0);
2674 trb.trb_2 = 0;
2675 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
2676 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX);
2677
2678 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
2679 return err;
2680}
2681
2682static void
2683xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si)
2684{
2685 uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0);
2686
2687 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2688 DPRINTFN(4, "dcbaa %p dc %016"PRIx64" slot %d",
2689 &dcbaa[si], dcba, si, 0);
2690
2691 dcbaa[si] = htole64(dcba);
2692 usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t),
2693 BUS_DMASYNC_PREWRITE);
2694}
2695
2696/*
2697 * Allocate device and input context DMA buffer, and
2698 * TRB DMA buffer for each endpoint.
2699 */
2700static usbd_status
2701xhci_init_slot(struct usbd_device *dev, uint32_t slot)
2702{
2703 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus);
2704 struct xhci_slot *xs;
2705 usbd_status err;
2706 u_int dci;
2707
2708 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2709 DPRINTFN(4, "slot %u", slot, 0, 0, 0);
2710
2711 xs = &sc->sc_slots[slot];
2712
2713 /* allocate contexts */
2714 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz,
2715 &xs->xs_dc_dma);
2716 if (err)
2717 return err;
2718 memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz);
2719
2720 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz,
2721 &xs->xs_ic_dma);
2722 if (err)
2723 goto bad1;
2724 memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz);
2725
2726 for (dci = 0; dci < 32; dci++) {
2727 //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint));
2728 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci]));
2729 if (dci == XHCI_DCI_SLOT)
2730 continue;
2731 err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr,
2732 XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN);
2733 if (err) {
2734 DPRINTFN(0, "ring init failure", 0, 0, 0, 0);
2735 goto bad2;
2736 }
2737 }
2738
2739 bad2:
2740 if (err == USBD_NORMAL_COMPLETION) {
2741 xs->xs_idx = slot;
2742 } else {
2743 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, dci);
2744 }
2745
2746 return err;
2747
2748 bad1:
2749 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma);
2750 xs->xs_idx = 0;
2751 return err;
2752}
2753
2754static void
2755xhci_free_slot(struct xhci_softc *sc, struct xhci_slot *xs, int start_dci,
2756 int end_dci)
2757{
2758 u_int dci;
2759
2760 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2761 DPRINTFN(4, "slot %u start %u end %u", xs->xs_idx, start_dci, end_dci,
2762 0);
2763
2764 for (dci = start_dci; dci < end_dci; dci++) {
2765 xhci_ring_free(sc, &xs->xs_ep[dci].xe_tr);
2766 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci]));
2767 }
2768 usb_freemem(&sc->sc_bus, &xs->xs_ic_dma);
2769 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma);
2770 xs->xs_idx = 0;
2771}
2772
2773/*
2774 * Setup slot context, set Device Context Base Address, and issue
2775 * Set Address Device command.
2776 */
2777static usbd_status
2778xhci_set_address(struct usbd_device *dev, uint32_t slot, bool bsr)
2779{
2780 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus);
2781 struct xhci_slot *xs;
2782 usbd_status err;
2783
2784 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2785 DPRINTFN(4, "slot %u bsr %u", slot, bsr, 0, 0);
2786
2787 xs = &sc->sc_slots[slot];
2788
2789 xhci_setup_ctx(dev->ud_pipe0);
2790
2791 hexdump("input context", xhci_slot_get_icv(sc, xs, 0),
2792 sc->sc_ctxsz * 3);
2793
2794 xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot);
2795
2796 err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, bsr);
2797
2798 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
2799 hexdump("output context", xhci_slot_get_dcv(sc, xs, 0),
2800 sc->sc_ctxsz * 2);
2801
2802 return err;
2803}
2804
2805/*
2806 * 4.8.2, 6.2.3.2
2807 * construct slot/endpoint context parameters and do syncmem
2808 */
2809static void
2810xhci_setup_ctx(struct usbd_pipe *pipe)
2811{
2812 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
2813 struct usbd_device *dev = pipe->up_dev;
2814 struct xhci_slot * const xs = dev->ud_hcpriv;
2815 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
2816 const u_int dci = xhci_ep_get_dci(ed);
2817 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
2818 uint32_t *cp;
2819 uint16_t mps = UGETW(ed->wMaxPacketSize);
2820 uint8_t speed = dev->ud_speed;
2821 uint8_t ival = ed->bInterval;
2822
2823 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2824 DPRINTFN(4, "pipe %p: slot %u dci %u speed %u", pipe, xs->xs_idx, dci,
2825 speed);
2826
2827 /* set up initial input control context */
2828 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL);
2829 cp[0] = htole32(0);
2830 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci));
2831 if (dci == XHCI_DCI_EP_CONTROL)
2832 cp[1] |= htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT));
2833 cp[7] = htole32(0);
2834
2835 /* set up input slot context */
2836 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT));
2837 cp[0] =
2838 XHCI_SCTX_0_CTX_NUM_SET(dci) |
2839 XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed));
2840 cp[1] = 0;
2841 cp[2] = XHCI_SCTX_2_IRQ_TARGET_SET(0);
2842 cp[3] = 0;
2843 xhci_setup_route(pipe, cp);
2844 xhci_setup_tthub(pipe, cp);
2845
2846 cp[0] = htole32(cp[0]);
2847 cp[1] = htole32(cp[1]);
2848 cp[2] = htole32(cp[2]);
2849 cp[3] = htole32(cp[3]);
2850
2851 /* set up input endpoint context */
2852 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci));
2853 cp[0] =
2854 XHCI_EPCTX_0_EPSTATE_SET(0) |
2855 XHCI_EPCTX_0_MULT_SET(0) |
2856 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
2857 XHCI_EPCTX_0_LSA_SET(0) |
2858 XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0);
2859 cp[1] =
2860 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) |
2861 XHCI_EPCTX_1_HID_SET(0) |
2862 XHCI_EPCTX_1_MAXB_SET(0);
2863
2864 if (xfertype != UE_ISOCHRONOUS)
2865 cp[1] |= XHCI_EPCTX_1_CERR_SET(3);
2866
2867 if (xfertype == UE_CONTROL)
2868 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); /* 6.2.3 */
2869 else if (USB_IS_SS(speed))
2870 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(mps);
2871 else
2872 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(UE_GET_SIZE(mps));
2873
2874 xhci_setup_maxburst(pipe, cp);
2875
2876 switch (xfertype) {
2877 case UE_CONTROL:
2878 break;
2879 case UE_BULK:
2880 /* XXX Set MaxPStreams, HID, and LSA if streams enabled */
2881 break;
2882 case UE_INTERRUPT:
2883 if (pipe->up_interval != USBD_DEFAULT_INTERVAL)
2884 ival = pipe->up_interval;
2885
2886 ival = xhci_bival2ival(ival, speed);
2887 cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival);
2888 break;
2889 case UE_ISOCHRONOUS:
2890 if (pipe->up_interval != USBD_DEFAULT_INTERVAL)
2891 ival = pipe->up_interval;
2892
2893 /* xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 */
2894 if (speed == USB_SPEED_FULL)
2895 ival += 3; /* 1ms -> 125us */
2896 ival--;
2897 cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival);
2898 break;
2899 default:
2900 break;
2901 }
2902 DPRINTFN(4, "setting ival %u MaxBurst %#x",
2903 XHCI_EPCTX_0_IVAL_GET(cp[0]), XHCI_EPCTX_1_MAXB_GET(cp[1]), 0, 0);
2904
2905 /* rewind TR dequeue pointer in xHC */
2906 /* can't use xhci_ep_get_dci() yet? */
2907 *(uint64_t *)(&cp[2]) = htole64(
2908 xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) |
2909 XHCI_EPCTX_2_DCS_SET(1));
2910
2911 cp[0] = htole32(cp[0]);
2912 cp[1] = htole32(cp[1]);
2913 cp[4] = htole32(cp[4]);
2914
2915 /* rewind TR dequeue pointer in driver */
2916 struct xhci_ring *xr = &xs->xs_ep[dci].xe_tr;
2917 mutex_enter(&xr->xr_lock);
2918 xhci_host_dequeue(xr);
2919 mutex_exit(&xr->xr_lock);
2920
2921 /* sync input contexts before they are read from memory */
2922 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE);
2923}
2924
2925/*
2926 * Setup route string and roothub port of given device for slot context
2927 */
2928static void
2929xhci_setup_route(struct usbd_pipe *pipe, uint32_t *cp)
2930{
2931 struct usbd_device *dev = pipe->up_dev;
2932 struct usbd_port *up = dev->ud_powersrc;
2933 struct usbd_device *hub;
2934 struct usbd_device *adev;
2935 uint8_t rhport = 0;
2936 uint32_t route = 0;
2937
2938 XHCIHIST_FUNC(); XHCIHIST_CALLED();
2939
2940 /* Locate root hub port and Determine route string */
2941 /* 4.3.3 route string does not include roothub port */
2942 for (hub = dev; hub != NULL; hub = hub->ud_myhub) {
2943 uint32_t dep;
2944
2945 DPRINTFN(4, "hub %p depth %d upport %p upportno %d",
2946 hub, hub->ud_depth, hub->ud_powersrc,
2947 hub->ud_powersrc ? hub->ud_powersrc->up_portno : -1);
2948
2949 if (hub->ud_powersrc == NULL)
2950 break;
2951 dep = hub->ud_depth;
2952 if (dep == 0)
2953 break;
2954 rhport = hub->ud_powersrc->up_portno;
2955 if (dep > USB_HUB_MAX_DEPTH)
2956 continue;
2957
2958 route |=
2959 (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport)
2960 << ((dep - 1) * 4);
2961 }
2962 route = route >> 4;
2963 DPRINTFN(4, "rhport %u Route %05x hub %p", rhport, route, hub, 0);
2964
2965 /* Locate port on upstream high speed hub */
2966 for (adev = dev, hub = up->up_parent;
2967 hub != NULL && hub->ud_speed != USB_SPEED_HIGH;
2968 adev = hub, hub = hub->ud_myhub)
2969 ;
2970 if (hub) {
2971 int p;
2972 for (p = 0; p < hub->ud_hub->uh_hubdesc.bNbrPorts; p++) {
2973 if (hub->ud_hub->uh_ports[p].up_dev == adev) {
2974 dev->ud_myhsport = &hub->ud_hub->uh_ports[p];
2975 goto found;
2976 }
2977 }
2978 panic("xhci_setup_route: cannot find HS port");
2979 found:
2980 DPRINTFN(4, "high speed port %d", p, 0, 0, 0);
2981 } else {
2982 dev->ud_myhsport = NULL;
2983 }
2984
2985 cp[0] |= XHCI_SCTX_0_ROUTE_SET(route);
2986 cp[1] |= XHCI_SCTX_1_RH_PORT_SET(rhport);
2987}
2988
2989/*
2990 * Setup whether device is hub, whether device uses MTT, and
2991 * TT informations if it uses MTT.
2992 */
2993static void
2994xhci_setup_tthub(struct usbd_pipe *pipe, uint32_t *cp)
2995{
2996 struct usbd_device *dev = pipe->up_dev;
2997 usb_device_descriptor_t * const dd = &dev->ud_ddesc;
2998 uint32_t speed = dev->ud_speed;
2999 uint8_t tthubslot, ttportnum;
3000 bool ishub;
3001 bool usemtt;
3002
3003 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3004
3005 /*
3006 * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2
3007 * tthubslot:
3008 * This is the slot ID of parent HS hub
3009 * if LS/FS device is connected && connected through HS hub.
3010 * This is 0 if device is not LS/FS device ||
3011 * parent hub is not HS hub ||
3012 * attached to root hub.
3013 * ttportnum:
3014 * This is the downstream facing port of parent HS hub
3015 * if LS/FS device is connected.
3016 * This is 0 if device is not LS/FS device ||
3017 * parent hub is not HS hub ||
3018 * attached to root hub.
3019 */
3020 if (dev->ud_myhsport != NULL &&
3021 dev->ud_myhub != NULL && dev->ud_myhub->ud_depth != 0 &&
3022 (dev->ud_myhub != NULL &&
3023 dev->ud_myhub->ud_speed == USB_SPEED_HIGH) &&
3024 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) {
3025 ttportnum = dev->ud_myhsport->up_portno;
3026 tthubslot = dev->ud_myhsport->up_parent->ud_addr;
3027 } else {
3028 ttportnum = 0;
3029 tthubslot = 0;
3030 }
3031 DPRINTFN(4, "myhsport %p ttportnum=%d tthubslot=%d",
3032 dev->ud_myhsport, ttportnum, tthubslot, 0);
3033
3034 /* ishub is valid after reading UDESC_DEVICE */
3035 ishub = (dd->bDeviceClass == UDCLASS_HUB);
3036
3037 /* dev->ud_hub is valid after reading UDESC_HUB */
3038 if (ishub && dev->ud_hub) {
3039 usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc;
3040 uint8_t ttt =
3041 __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK);
3042
3043 cp[1] |= XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts);
3044 cp[2] |= XHCI_SCTX_2_TT_THINK_TIME_SET(ttt);
3045 DPRINTFN(4, "nports=%d ttt=%d", hd->bNbrPorts, ttt, 0, 0);
3046 }
3047
3048#define IS_TTHUB(dd) \
3049 ((dd)->bDeviceProtocol == UDPROTO_HSHUBSTT || \
3050 (dd)->bDeviceProtocol == UDPROTO_HSHUBMTT)
3051
3052 /*
3053 * MTT flag is set if
3054 * 1. this is HS hub && MTT is enabled
3055 * or
3056 * 2. this is not hub && this is LS or FS device &&
3057 * MTT of parent HS hub (and its parent, too) is enabled
3058 */
3059 if (ishub && speed == USB_SPEED_HIGH && IS_TTHUB(dd))
3060 usemtt = true;
3061 else if (!ishub &&
3062 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) &&
3063 dev->ud_myhub != NULL && dev->ud_myhub->ud_depth != 0 &&
3064 (dev->ud_myhub != NULL &&
3065 dev->ud_myhub->ud_speed == USB_SPEED_HIGH) &&
3066 dev->ud_myhsport != NULL &&
3067 IS_TTHUB(&dev->ud_myhsport->up_parent->ud_ddesc))
3068 usemtt = true;
3069 else
3070 usemtt = false;
3071 DPRINTFN(4, "class %u proto %u ishub %d usemtt %d",
3072 dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt);
3073
3074#undef IS_TTHUB
3075
3076 cp[0] |=
3077 XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) |
3078 XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0);
3079 cp[2] |=
3080 XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) |
3081 XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum);
3082}
3083
3084/* set up params for periodic endpoint */
3085static void
3086xhci_setup_maxburst(struct usbd_pipe *pipe, uint32_t *cp)
3087{
3088 struct usbd_device *dev = pipe->up_dev;
3089 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
3090 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
3091 usbd_desc_iter_t iter;
3092 const usb_cdc_descriptor_t *cdcd;
3093 uint32_t maxb = 0;
3094 uint16_t mps = UGETW(ed->wMaxPacketSize);
3095 uint8_t speed = dev->ud_speed;
3096 uint8_t ep;
3097
3098 /* config desc is NULL when opening ep0 */
3099 if (dev == NULL || dev->ud_cdesc == NULL)
3100 goto no_cdcd;
3101 cdcd = (const usb_cdc_descriptor_t *)usb_find_desc(dev,
3102 UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY);
3103 if (cdcd == NULL)
3104 goto no_cdcd;
3105 usb_desc_iter_init(dev, &iter);
3106 iter.cur = (const void *)cdcd;
3107
3108 /* find endpoint_ss_comp desc for ep of this pipe */
3109 for (ep = 0;;) {
3110 cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter);
3111 if (cdcd == NULL)
3112 break;
3113 if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) {
3114 ep = ((const usb_endpoint_descriptor_t *)cdcd)->
3115 bEndpointAddress;
3116 if (UE_GET_ADDR(ep) ==
3117 UE_GET_ADDR(ed->bEndpointAddress)) {
3118 cdcd = (const usb_cdc_descriptor_t *)
3119 usb_desc_iter_next(&iter);
3120 break;
3121 }
3122 ep = 0;
3123 }
3124 }
3125 if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) {
3126 const usb_endpoint_ss_comp_descriptor_t * esscd =
3127 (const usb_endpoint_ss_comp_descriptor_t *)cdcd;
3128 maxb = esscd->bMaxBurst;
3129 }
3130
3131 no_cdcd:
3132 /* 6.2.3.4, 4.8.2.4 */
3133 if (USB_IS_SS(speed)) {
3134 /* USB 3.1 9.6.6 */
3135 cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(mps);
3136 /* USB 3.1 9.6.7 */
3137 cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb);
3138#ifdef notyet
3139 if (xfertype == UE_ISOCHRONOUS) {
3140 }
3141 if (XHCI_HCC2_LEC(sc->sc_hcc2) != 0) {
3142 /* use ESIT */
3143 cp[4] |= XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x);
3144 cp[0] |= XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(x);
3145
3146 /* XXX if LEC = 1, set ESIT instead */
3147 cp[0] |= XHCI_EPCTX_0_MULT_SET(0);
3148 } else {
3149 /* use ival */
3150 }
3151#endif
3152 } else {
3153 /* USB 2.0 9.6.6 */
3154 cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(UE_GET_SIZE(mps));
3155
3156 /* 6.2.3.4 */
3157 if (speed == USB_SPEED_HIGH &&
3158 (xfertype == UE_ISOCHRONOUS || xfertype == UE_INTERRUPT)) {
3159 maxb = UE_GET_TRANS(mps);
3160 } else {
3161 /* LS/FS or HS CTRL or HS BULK */
3162 maxb = 0;
3163 }
3164 cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb);
3165 }
3166}
3167
3168/*
3169 * Convert endpoint bInterval value to endpoint context interval value
3170 * for Interrupt pipe.
3171 * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6
3172 */
3173static uint32_t
3174xhci_bival2ival(uint32_t ival, uint32_t speed)
3175{
3176 if (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) {
3177 int i;
3178
3179 /*
3180 * round ival down to "the nearest base 2 multiple of
3181 * bInterval * 8".
3182 * bInterval is at most 255 as its type is uByte.
3183 * 255(ms) = 2040(x 125us) < 2^11, so start with 10.
3184 */
3185 for (i = 10; i > 0; i--) {
3186 if ((ival * 8) >= (1 << i))
3187 break;
3188 }
3189 ival = i;
3190 } else {
3191 /* Interval = bInterval-1 for SS/HS */
3192 ival--;
3193 }
3194
3195 return ival;
3196}
3197
3198/* ----- */
3199
3200static void
3201xhci_noop(struct usbd_pipe *pipe)
3202{
3203 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3204}
3205
3206/*
3207 * Process root hub request.
3208 */
3209static int
3210xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req,
3211 void *buf, int buflen)
3212{
3213 struct xhci_softc * const sc = XHCI_BUS2SC(bus);
3214 usb_port_status_t ps;
3215 int l, totlen = 0;
3216 uint16_t len, value, index;
3217 int port, i;
3218 uint32_t v;
3219
3220 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3221
3222 if (sc->sc_dying)
3223 return -1;
3224
3225 len = UGETW(req->wLength);
3226 value = UGETW(req->wValue);
3227 index = UGETW(req->wIndex);
3228
3229 DPRINTFN(12, "rhreq: %04x %04x %04x %04x",
3230 req->bmRequestType | (req->bRequest << 8), value, index, len);
3231
3232#define C(x,y) ((x) | ((y) << 8))
3233 switch (C(req->bRequest, req->bmRequestType)) {
3234 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3235 DPRINTFN(8, "getdesc: wValue=0x%04x", value, 0, 0, 0);
3236 if (len == 0)
3237 break;
3238 switch (value) {
3239 case C(0, UDESC_DEVICE): {
3240 usb_device_descriptor_t devd;
3241 totlen = min(buflen, sizeof(devd));
3242 memcpy(&devd, buf, totlen);
3243 USETW(devd.idVendor, sc->sc_id_vendor);
3244 memcpy(buf, &devd, totlen);
3245 break;
3246 }
3247#define sd ((usb_string_descriptor_t *)buf)
3248 case C(1, UDESC_STRING):
3249 /* Vendor */
3250 totlen = usb_makestrdesc(sd, len, sc->sc_vendor);
3251 break;
3252 case C(2, UDESC_STRING):
3253 /* Product */
3254 totlen = usb_makestrdesc(sd, len, "xHCI Root Hub");
3255 break;
3256#undef sd
3257 default:
3258 /* default from usbroothub */
3259 return buflen;
3260 }
3261 break;
3262
3263 /* Hub requests */
3264 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3265 break;
3266 /* Clear Port Feature request */
3267 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3268 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE port=%d feature=%d",
3269 index, value, 0, 0);
3270 if (index < 1 || index > sc->sc_maxports) {
3271 return -1;
3272 }
3273 port = XHCI_PORTSC(index);
3274 v = xhci_op_read_4(sc, port);
3275 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0);
3276 v &= ~XHCI_PS_CLEAR;
3277 switch (value) {
3278 case UHF_PORT_ENABLE:
3279 xhci_op_write_4(sc, port, v & ~XHCI_PS_PED);
3280 break;
3281 case UHF_PORT_SUSPEND:
3282 return -1;
3283 case UHF_PORT_POWER:
3284 break;
3285 case UHF_PORT_TEST:
3286 case UHF_PORT_INDICATOR:
3287 return -1;
3288 case UHF_C_PORT_CONNECTION:
3289 xhci_op_write_4(sc, port, v | XHCI_PS_CSC);
3290 break;
3291 case UHF_C_PORT_ENABLE:
3292 case UHF_C_PORT_SUSPEND:
3293 case UHF_C_PORT_OVER_CURRENT:
3294 return -1;
3295 case UHF_C_BH_PORT_RESET:
3296 xhci_op_write_4(sc, port, v | XHCI_PS_WRC);
3297 break;
3298 case UHF_C_PORT_RESET:
3299 xhci_op_write_4(sc, port, v | XHCI_PS_PRC);
3300 break;
3301 case UHF_C_PORT_LINK_STATE:
3302 xhci_op_write_4(sc, port, v | XHCI_PS_PLC);
3303 break;
3304 case UHF_C_PORT_CONFIG_ERROR:
3305 xhci_op_write_4(sc, port, v | XHCI_PS_CEC);
3306 break;
3307 default:
3308 return -1;
3309 }
3310 break;
3311 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3312 if (len == 0)
3313 break;
3314 if ((value & 0xff) != 0) {
3315 return -1;
3316 }
3317 usb_hub_descriptor_t hubd;
3318
3319 totlen = min(buflen, sizeof(hubd));
3320 memcpy(&hubd, buf, totlen);
3321 hubd.bNbrPorts = sc->sc_maxports;
3322 USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH);
3323 hubd.bPwrOn2PwrGood = 200;
3324 for (i = 0, l = sc->sc_maxports; l > 0; i++, l -= 8)
3325 hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
3326 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
3327 totlen = min(totlen, hubd.bDescLength);
3328 memcpy(buf, &hubd, totlen);
3329 break;
3330 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3331 if (len != 4) {
3332 return -1;
3333 }
3334 memset(buf, 0, len); /* ? XXX */
3335 totlen = len;
3336 break;
3337 /* Get Port Status request */
3338 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3339 DPRINTFN(8, "get port status i=%d", index, 0, 0, 0);
3340 if (index < 1 || index > sc->sc_maxports) {
3341 return -1;
3342 }
3343 if (len != 4) {
3344 return -1;
3345 }
3346 v = xhci_op_read_4(sc, XHCI_PORTSC(index));
3347 DPRINTFN(4, "getrhportsc %d %08x", index, v, 0, 0);
3348 i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v));
3349 if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS;
3350 if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED;
3351 if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR;
3352 //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND;
3353 if (v & XHCI_PS_PR) i |= UPS_RESET;
3354 if (v & XHCI_PS_PP) {
3355 if (i & UPS_OTHER_SPEED)
3356 i |= UPS_PORT_POWER_SS;
3357 else
3358 i |= UPS_PORT_POWER;
3359 }
3360 if (i & UPS_OTHER_SPEED)
3361 i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v));
3362 if (sc->sc_vendor_port_status)
3363 i = sc->sc_vendor_port_status(sc, v, i);
3364 USETW(ps.wPortStatus, i);
3365 i = 0;
3366 if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS;
3367 if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED;
3368 if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR;
3369 if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET;
3370 if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET;
3371 if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE;
3372 if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR;
3373 USETW(ps.wPortChange, i);
3374 totlen = min(len, sizeof(ps));
3375 memcpy(buf, &ps, totlen);
3376 break;
3377 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3378 return -1;
3379 case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE):
3380 break;
3381 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3382 break;
3383 /* Set Port Feature request */
3384 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): {
3385 int optval = (index >> 8) & 0xff;
3386 index &= 0xff;
3387 if (index < 1 || index > sc->sc_maxports) {
3388 return -1;
3389 }
3390 port = XHCI_PORTSC(index);
3391 v = xhci_op_read_4(sc, port);
3392 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0);
3393 v &= ~XHCI_PS_CLEAR;
3394 switch (value) {
3395 case UHF_PORT_ENABLE:
3396 xhci_op_write_4(sc, port, v | XHCI_PS_PED);
3397 break;
3398 case UHF_PORT_SUSPEND:
3399 /* XXX suspend */
3400 break;
3401 case UHF_PORT_RESET:
3402 v &= ~(XHCI_PS_PED | XHCI_PS_PR);
3403 xhci_op_write_4(sc, port, v | XHCI_PS_PR);
3404 /* Wait for reset to complete. */
3405 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
3406 if (sc->sc_dying) {
3407 return -1;
3408 }
3409 v = xhci_op_read_4(sc, port);
3410 if (v & XHCI_PS_PR) {
3411 xhci_op_write_4(sc, port, v & ~XHCI_PS_PR);
3412 usb_delay_ms(&sc->sc_bus, 10);
3413 /* XXX */
3414 }
3415 break;
3416 case UHF_PORT_POWER:
3417 /* XXX power control */
3418 break;
3419 /* XXX more */
3420 case UHF_C_PORT_RESET:
3421 xhci_op_write_4(sc, port, v | XHCI_PS_PRC);
3422 break;
3423 case UHF_PORT_U1_TIMEOUT:
3424 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) {
3425 return -1;
3426 }
3427 port = XHCI_PORTPMSC(index);
3428 v = xhci_op_read_4(sc, port);
3429 v &= ~XHCI_PM3_U1TO_SET(0xff);
3430 v |= XHCI_PM3_U1TO_SET(optval);
3431 xhci_op_write_4(sc, port, v);
3432 break;
3433 case UHF_PORT_U2_TIMEOUT:
3434 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) {
3435 return -1;
3436 }
3437 port = XHCI_PORTPMSC(index);
3438 v = xhci_op_read_4(sc, port);
3439 v &= ~XHCI_PM3_U2TO_SET(0xff);
3440 v |= XHCI_PM3_U2TO_SET(optval);
3441 xhci_op_write_4(sc, port, v);
3442 break;
3443 default:
3444 return -1;
3445 }
3446 }
3447 break;
3448 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3449 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3450 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3451 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3452 break;
3453 default:
3454 /* default from usbroothub */
3455 return buflen;
3456 }
3457
3458 return totlen;
3459}
3460
3461/* root hub interrupt */
3462
3463static usbd_status
3464xhci_root_intr_transfer(struct usbd_xfer *xfer)
3465{
3466 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3467 usbd_status err;
3468
3469 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3470
3471 /* Insert last in queue. */
3472 mutex_enter(&sc->sc_lock);
3473 err = usb_insert_transfer(xfer);
3474 mutex_exit(&sc->sc_lock);
3475 if (err)
3476 return err;
3477
3478 /* Pipe isn't running, start first */
3479 return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3480}
3481
3482/* Wait for roothub port status/change */
3483static usbd_status
3484xhci_root_intr_start(struct usbd_xfer *xfer)
3485{
3486 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3487
3488 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3489
3490 if (sc->sc_dying)
3491 return USBD_IOERROR;
3492
3493 mutex_enter(&sc->sc_lock);
3494 sc->sc_intrxfer = xfer;
3495 mutex_exit(&sc->sc_lock);
3496
3497 return USBD_IN_PROGRESS;
3498}
3499
3500static void
3501xhci_root_intr_abort(struct usbd_xfer *xfer)
3502{
3503 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3504
3505 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3506
3507 KASSERT(mutex_owned(&sc->sc_lock));
3508 KASSERT(xfer->ux_pipe->up_intrxfer == xfer);
3509
3510 sc->sc_intrxfer = NULL;
3511
3512 xfer->ux_status = USBD_CANCELLED;
3513 usb_transfer_complete(xfer);
3514}
3515
3516static void
3517xhci_root_intr_close(struct usbd_pipe *pipe)
3518{
3519 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
3520
3521 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3522
3523 KASSERT(mutex_owned(&sc->sc_lock));
3524
3525 sc->sc_intrxfer = NULL;
3526}
3527
3528static void
3529xhci_root_intr_done(struct usbd_xfer *xfer)
3530{
3531 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3532
3533}
3534
3535/* -------------- */
3536/* device control */
3537
3538static usbd_status
3539xhci_device_ctrl_transfer(struct usbd_xfer *xfer)
3540{
3541 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3542 usbd_status err;
3543
3544 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3545
3546 /* Insert last in queue. */
3547 mutex_enter(&sc->sc_lock);
3548 err = usb_insert_transfer(xfer);
3549 mutex_exit(&sc->sc_lock);
3550 if (err)
3551 return err;
3552
3553 /* Pipe isn't running, start first */
3554 return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3555}
3556
3557static usbd_status
3558xhci_device_ctrl_start(struct usbd_xfer *xfer)
3559{
3560 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3561 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
3562 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
3563 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr;
3564 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
3565 usb_device_request_t * const req = &xfer->ux_request;
3566 const int isread = usbd_xfer_isread(xfer);
3567 const uint32_t len = UGETW(req->wLength);
3568 usb_dma_t * const dma = &xfer->ux_dmabuf;
3569 uint64_t parameter;
3570 uint32_t status;
3571 uint32_t control;
3572 u_int i;
3573
3574 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3575 DPRINTFN(12, "req: %04x %04x %04x %04x",
3576 req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue),
3577 UGETW(req->wIndex), UGETW(req->wLength));
3578
3579 /* we rely on the bottom bits for extra info */
3580 KASSERTMSG(((uintptr_t)xfer & 0x3) == 0x0, "xfer %zx",
3581 (uintptr_t) xfer);
3582
3583 KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0);
3584
3585 i = 0;
3586
3587 /* setup phase */
3588 memcpy(&parameter, req, sizeof(parameter));
3589 status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req));
3590 control = ((len == 0) ? XHCI_TRB_3_TRT_NONE :
3591 (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) |
3592 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) |
3593 XHCI_TRB_3_IDT_BIT;
3594 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control);
3595
3596 if (len != 0) {
3597 /* data phase */
3598 parameter = DMAADDR(dma, 0);
3599 KASSERTMSG(len <= 0x10000, "len %d", len);
3600 status = XHCI_TRB_2_IRQ_SET(0) |
3601 XHCI_TRB_2_TDSZ_SET(1) |
3602 XHCI_TRB_2_BYTES_SET(len);
3603 control = (isread ? XHCI_TRB_3_DIR_IN : 0) |
3604 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) |
3605 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) |
3606 XHCI_TRB_3_IOC_BIT;
3607 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control);
3608 }
3609
3610 parameter = 0;
3611 status = XHCI_TRB_2_IRQ_SET(0);
3612 /* the status stage has inverted direction */
3613 control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) |
3614 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) |
3615 XHCI_TRB_3_IOC_BIT;
3616 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control);
3617
3618 mutex_enter(&tr->xr_lock);
3619 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i);
3620 mutex_exit(&tr->xr_lock);
3621
3622 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
3623
3624 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) {
3625 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout),
3626 xhci_timeout, xfer);
3627 }
3628
3629 return USBD_IN_PROGRESS;
3630}
3631
3632static void
3633xhci_device_ctrl_done(struct usbd_xfer *xfer)
3634{
3635 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3636 usb_device_request_t *req = &xfer->ux_request;
3637 int len = UGETW(req->wLength);
3638 int rd = req->bmRequestType & UT_READ;
3639
3640 if (len)
3641 usb_syncmem(&xfer->ux_dmabuf, 0, len,
3642 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3643}
3644
3645static void
3646xhci_device_ctrl_abort(struct usbd_xfer *xfer)
3647{
3648 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3649
3650 xhci_abort_xfer(xfer, USBD_CANCELLED);
3651}
3652
3653static void
3654xhci_device_ctrl_close(struct usbd_pipe *pipe)
3655{
3656 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3657
3658 xhci_close_pipe(pipe);
3659}
3660
3661/* ------------------ */
3662/* device isochronous */
3663
3664/* ----------- */
3665/* device bulk */
3666
3667static usbd_status
3668xhci_device_bulk_transfer(struct usbd_xfer *xfer)
3669{
3670 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3671 usbd_status err;
3672
3673 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3674
3675 /* Insert last in queue. */
3676 mutex_enter(&sc->sc_lock);
3677 err = usb_insert_transfer(xfer);
3678 mutex_exit(&sc->sc_lock);
3679 if (err)
3680 return err;
3681
3682 /*
3683 * Pipe isn't running (otherwise err would be USBD_INPROG),
3684 * so start it first.
3685 */
3686 return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3687}
3688
3689static usbd_status
3690xhci_device_bulk_start(struct usbd_xfer *xfer)
3691{
3692 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3693 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
3694 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
3695 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr;
3696 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
3697 const uint32_t len = xfer->ux_length;
3698 usb_dma_t * const dma = &xfer->ux_dmabuf;
3699 uint64_t parameter;
3700 uint32_t status;
3701 uint32_t control;
3702 u_int i = 0;
3703
3704 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3705
3706 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0);
3707
3708 if (sc->sc_dying)
3709 return USBD_IOERROR;
3710
3711 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0);
3712
3713 parameter = DMAADDR(dma, 0);
3714 /*
3715 * XXX: (dsl) The physical buffer must not cross a 64k boundary.
3716 * If the user supplied buffer crosses such a boundary then 2
3717 * (or more) TRB should be used.
3718 * If multiple TRB are used the td_size field must be set correctly.
3719 * For v1.0 devices (like ivy bridge) this is the number of usb data
3720 * blocks needed to complete the transfer.
3721 * Setting it to 1 in the last TRB causes an extra zero-length
3722 * data block be sent.
3723 * The earlier documentation differs, I don't know how it behaves.
3724 */
3725 KASSERTMSG(len <= 0x10000, "len %d", len);
3726 status = XHCI_TRB_2_IRQ_SET(0) |
3727 XHCI_TRB_2_TDSZ_SET(1) |
3728 XHCI_TRB_2_BYTES_SET(len);
3729 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) |
3730 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) |
3731 XHCI_TRB_3_IOC_BIT;
3732 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control);
3733
3734 mutex_enter(&tr->xr_lock);
3735 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i);
3736 mutex_exit(&tr->xr_lock);
3737
3738 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
3739
3740 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) {
3741 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout),
3742 xhci_timeout, xfer);
3743 }
3744
3745 return USBD_IN_PROGRESS;
3746}
3747
3748static void
3749xhci_device_bulk_done(struct usbd_xfer *xfer)
3750{
3751#ifdef USB_DEBUG
3752 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
3753 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
3754#endif
3755 const int isread = usbd_xfer_isread(xfer);
3756
3757 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3758
3759 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0);
3760
3761 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
3762 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3763}
3764
3765static void
3766xhci_device_bulk_abort(struct usbd_xfer *xfer)
3767{
3768 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3769
3770 xhci_abort_xfer(xfer, USBD_CANCELLED);
3771}
3772
3773static void
3774xhci_device_bulk_close(struct usbd_pipe *pipe)
3775{
3776 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3777
3778 xhci_close_pipe(pipe);
3779}
3780
3781/* ---------------- */
3782/* device interrupt */
3783
3784static usbd_status
3785xhci_device_intr_transfer(struct usbd_xfer *xfer)
3786{
3787 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3788 usbd_status err;
3789
3790 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3791
3792 /* Insert last in queue. */
3793 mutex_enter(&sc->sc_lock);
3794 err = usb_insert_transfer(xfer);
3795 mutex_exit(&sc->sc_lock);
3796 if (err)
3797 return err;
3798
3799 /*
3800 * Pipe isn't running (otherwise err would be USBD_INPROG),
3801 * so start it first.
3802 */
3803 return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3804}
3805
3806static usbd_status
3807xhci_device_intr_start(struct usbd_xfer *xfer)
3808{
3809 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3810 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
3811 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
3812 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr;
3813 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
3814 const uint32_t len = xfer->ux_length;
3815 usb_dma_t * const dma = &xfer->ux_dmabuf;
3816 uint64_t parameter;
3817 uint32_t status;
3818 uint32_t control;
3819 u_int i = 0;
3820
3821 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3822
3823 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0);
3824
3825 if (sc->sc_dying)
3826 return USBD_IOERROR;
3827
3828 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0);
3829
3830 parameter = DMAADDR(dma, 0);
3831 KASSERTMSG(len <= 0x10000, "len %d", len);
3832 status = XHCI_TRB_2_IRQ_SET(0) |
3833 XHCI_TRB_2_TDSZ_SET(1) |
3834 XHCI_TRB_2_BYTES_SET(len);
3835 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) |
3836 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) |
3837 XHCI_TRB_3_IOC_BIT;
3838 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control);
3839
3840 mutex_enter(&tr->xr_lock);
3841 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i);
3842 mutex_exit(&tr->xr_lock);
3843
3844 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
3845
3846 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) {
3847 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout),
3848 xhci_timeout, xfer);
3849 }
3850
3851 return USBD_IN_PROGRESS;
3852}
3853
3854static void
3855xhci_device_intr_done(struct usbd_xfer *xfer)
3856{
3857 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer);
3858#ifdef USB_DEBUG
3859 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
3860 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
3861#endif
3862 const int isread = usbd_xfer_isread(xfer);
3863
3864 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3865
3866 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0);
3867
3868 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
3869
3870 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
3871 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3872}
3873
3874static void
3875xhci_device_intr_abort(struct usbd_xfer *xfer)
3876{
3877 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer);
3878
3879 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3880
3881 KASSERT(mutex_owned(&sc->sc_lock));
3882 DPRINTFN(15, "%p", xfer, 0, 0, 0);
3883 KASSERT(xfer->ux_pipe->up_intrxfer == xfer);
3884 xhci_abort_xfer(xfer, USBD_CANCELLED);
3885}
3886
3887static void
3888xhci_device_intr_close(struct usbd_pipe *pipe)
3889{
3890 //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
3891
3892 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3893 DPRINTFN(15, "%p", pipe, 0, 0, 0);
3894
3895 xhci_close_pipe(pipe);
3896}
3897
3898/* ------------ */
3899
3900static void
3901xhci_timeout(void *addr)
3902{
3903 struct xhci_xfer * const xx = addr;
3904 struct usbd_xfer * const xfer = &xx->xx_xfer;
3905 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3906
3907 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3908
3909 if (sc->sc_dying) {
3910 return;
3911 }
3912
3913 usb_init_task(&xx->xx_abort_task, xhci_timeout_task, addr,
3914 USB_TASKQ_MPSAFE);
3915 usb_add_task(xx->xx_xfer.ux_pipe->up_dev, &xx->xx_abort_task,
3916 USB_TASKQ_HC);
3917}
3918
3919static void
3920xhci_timeout_task(void *addr)
3921{
3922 struct usbd_xfer * const xfer = addr;
3923 struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3924
3925 XHCIHIST_FUNC(); XHCIHIST_CALLED();
3926
3927 mutex_enter(&sc->sc_lock);
3928 xhci_abort_xfer(xfer, USBD_TIMEOUT);
3929 mutex_exit(&sc->sc_lock);
3930}
3931