1/* $NetBSD: ehci.c,v 1.254 2016/10/03 00:32:37 dholland Exp $ */
2
3/*
4 * Copyright (c) 2004-2012 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Lennart Augustsson (lennart@augustsson.net), Charles M. Hannum,
9 * Jeremy Morse (jeremy.morse@gmail.com), Jared D. McNeill
10 * (jmcneill@invisible.ca) and Matthew R. Green (mrg@eterna.com.au).
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34/*
35 * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
36 *
37 * The EHCI 1.0 spec can be found at
38 * http://www.intel.com/technology/usb/spec.htm
39 * and the USB 2.0 spec at
40 * http://www.usb.org/developers/docs/
41 *
42 */
43
44/*
45 * TODO:
46 * 1) hold off explorations by companion controllers until ehci has started.
47 *
48 * 2) The hub driver needs to handle and schedule the transaction translator,
49 * to assign place in frame where different devices get to go. See chapter
50 * on hubs in USB 2.0 for details.
51 *
52 * 3) Command failures are not recovered correctly.
53 */
54
55#include <sys/cdefs.h>
56__KERNEL_RCSID(0, "$NetBSD: ehci.c,v 1.254 2016/10/03 00:32:37 dholland Exp $");
57
58#include "ohci.h"
59#include "uhci.h"
60
61#ifdef _KERNEL_OPT
62#include "opt_usb.h"
63#endif
64
65#include <sys/param.h>
66
67#include <sys/bus.h>
68#include <sys/cpu.h>
69#include <sys/device.h>
70#include <sys/kernel.h>
71#include <sys/kmem.h>
72#include <sys/mutex.h>
73#include <sys/proc.h>
74#include <sys/queue.h>
75#include <sys/select.h>
76#include <sys/sysctl.h>
77#include <sys/systm.h>
78
79#include <machine/endian.h>
80
81#include <dev/usb/usb.h>
82#include <dev/usb/usbdi.h>
83#include <dev/usb/usbdivar.h>
84#include <dev/usb/usbhist.h>
85#include <dev/usb/usb_mem.h>
86#include <dev/usb/usb_quirks.h>
87
88#include <dev/usb/ehcireg.h>
89#include <dev/usb/ehcivar.h>
90#include <dev/usb/usbroothub.h>
91
92
93#ifdef USB_DEBUG
94#ifndef EHCI_DEBUG
95#define ehcidebug 0
96#else
97static int ehcidebug = 0;
98
99SYSCTL_SETUP(sysctl_hw_ehci_setup, "sysctl hw.ehci setup")
100{
101 int err;
102 const struct sysctlnode *rnode;
103 const struct sysctlnode *cnode;
104
105 err = sysctl_createv(clog, 0, NULL, &rnode,
106 CTLFLAG_PERMANENT, CTLTYPE_NODE, "ehci",
107 SYSCTL_DESCR("ehci global controls"),
108 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
109
110 if (err)
111 goto fail;
112
113 /* control debugging printfs */
114 err = sysctl_createv(clog, 0, &rnode, &cnode,
115 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT,
116 "debug", SYSCTL_DESCR("Enable debugging output"),
117 NULL, 0, &ehcidebug, sizeof(ehcidebug), CTL_CREATE, CTL_EOL);
118 if (err)
119 goto fail;
120
121 return;
122fail:
123 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err);
124}
125
126#endif /* EHCI_DEBUG */
127#endif /* USB_DEBUG */
128
129#define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(ehcidebug,FMT,A,B,C,D)
130#define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(ehcidebug,N,FMT,A,B,C,D)
131#define EHCIHIST_FUNC() USBHIST_FUNC()
132#define EHCIHIST_CALLED() USBHIST_CALLED(ehcidebug)
133
134struct ehci_pipe {
135 struct usbd_pipe pipe;
136 int nexttoggle;
137
138 ehci_soft_qh_t *sqh;
139 union {
140 /* Control pipe */
141 struct {
142 usb_dma_t reqdma;
143 } ctrl;
144 /* Interrupt pipe */
145 struct {
146 u_int length;
147 } intr;
148 /* Iso pipe */
149 struct {
150 u_int next_frame;
151 u_int cur_xfers;
152 } isoc;
153 };
154};
155
156typedef TAILQ_HEAD(ex_completeq, ehci_xfer) ex_completeq_t;
157
158Static usbd_status ehci_open(struct usbd_pipe *);
159Static void ehci_poll(struct usbd_bus *);
160Static void ehci_softintr(void *);
161Static int ehci_intr1(ehci_softc_t *);
162Static void ehci_check_qh_intr(ehci_softc_t *, struct ehci_xfer *,
163 ex_completeq_t *);
164Static void ehci_check_itd_intr(ehci_softc_t *, struct ehci_xfer *,
165 ex_completeq_t *);
166Static void ehci_check_sitd_intr(ehci_softc_t *, struct ehci_xfer *,
167 ex_completeq_t *);
168Static void ehci_idone(struct ehci_xfer *, ex_completeq_t *);
169Static void ehci_timeout(void *);
170Static void ehci_timeout_task(void *);
171Static void ehci_intrlist_timeout(void *);
172Static void ehci_doorbell(void *);
173Static void ehci_pcd(void *);
174
175Static struct usbd_xfer *
176 ehci_allocx(struct usbd_bus *, unsigned int);
177Static void ehci_freex(struct usbd_bus *, struct usbd_xfer *);
178
179Static void ehci_get_lock(struct usbd_bus *, kmutex_t **);
180Static int ehci_roothub_ctrl(struct usbd_bus *,
181 usb_device_request_t *, void *, int);
182
183Static usbd_status ehci_root_intr_transfer(struct usbd_xfer *);
184Static usbd_status ehci_root_intr_start(struct usbd_xfer *);
185Static void ehci_root_intr_abort(struct usbd_xfer *);
186Static void ehci_root_intr_close(struct usbd_pipe *);
187Static void ehci_root_intr_done(struct usbd_xfer *);
188
189Static int ehci_device_ctrl_init(struct usbd_xfer *);
190Static void ehci_device_ctrl_fini(struct usbd_xfer *);
191Static usbd_status ehci_device_ctrl_transfer(struct usbd_xfer *);
192Static usbd_status ehci_device_ctrl_start(struct usbd_xfer *);
193Static void ehci_device_ctrl_abort(struct usbd_xfer *);
194Static void ehci_device_ctrl_close(struct usbd_pipe *);
195Static void ehci_device_ctrl_done(struct usbd_xfer *);
196
197Static int ehci_device_bulk_init(struct usbd_xfer *);
198Static void ehci_device_bulk_fini(struct usbd_xfer *);
199Static usbd_status ehci_device_bulk_transfer(struct usbd_xfer *);
200Static usbd_status ehci_device_bulk_start(struct usbd_xfer *);
201Static void ehci_device_bulk_abort(struct usbd_xfer *);
202Static void ehci_device_bulk_close(struct usbd_pipe *);
203Static void ehci_device_bulk_done(struct usbd_xfer *);
204
205Static int ehci_device_intr_init(struct usbd_xfer *);
206Static void ehci_device_intr_fini(struct usbd_xfer *);
207Static usbd_status ehci_device_intr_transfer(struct usbd_xfer *);
208Static usbd_status ehci_device_intr_start(struct usbd_xfer *);
209Static void ehci_device_intr_abort(struct usbd_xfer *);
210Static void ehci_device_intr_close(struct usbd_pipe *);
211Static void ehci_device_intr_done(struct usbd_xfer *);
212
213Static int ehci_device_isoc_init(struct usbd_xfer *);
214Static void ehci_device_isoc_fini(struct usbd_xfer *);
215Static usbd_status ehci_device_isoc_transfer(struct usbd_xfer *);
216Static void ehci_device_isoc_abort(struct usbd_xfer *);
217Static void ehci_device_isoc_close(struct usbd_pipe *);
218Static void ehci_device_isoc_done(struct usbd_xfer *);
219
220Static int ehci_device_fs_isoc_init(struct usbd_xfer *);
221Static void ehci_device_fs_isoc_fini(struct usbd_xfer *);
222Static usbd_status ehci_device_fs_isoc_transfer(struct usbd_xfer *);
223Static void ehci_device_fs_isoc_abort(struct usbd_xfer *);
224Static void ehci_device_fs_isoc_close(struct usbd_pipe *);
225Static void ehci_device_fs_isoc_done(struct usbd_xfer *);
226
227Static void ehci_device_clear_toggle(struct usbd_pipe *);
228Static void ehci_noop(struct usbd_pipe *);
229
230Static void ehci_disown(ehci_softc_t *, int, int);
231
232Static ehci_soft_qh_t * ehci_alloc_sqh(ehci_softc_t *);
233Static void ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
234
235Static ehci_soft_qtd_t *ehci_alloc_sqtd(ehci_softc_t *);
236Static void ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
237Static int ehci_alloc_sqtd_chain(ehci_softc_t *,
238 struct usbd_xfer *, int, int, ehci_soft_qtd_t **);
239Static void ehci_free_sqtds(ehci_softc_t *, struct ehci_xfer *);
240
241Static void ehci_reset_sqtd_chain(ehci_softc_t *, struct usbd_xfer *,
242 int, int, int *, ehci_soft_qtd_t **);
243Static void ehci_append_sqtd(ehci_soft_qtd_t *, ehci_soft_qtd_t *);
244
245Static ehci_soft_itd_t *ehci_alloc_itd(ehci_softc_t *);
246Static ehci_soft_sitd_t *
247 ehci_alloc_sitd(ehci_softc_t *);
248
249Static void ehci_remove_itd_chain(ehci_softc_t *, ehci_soft_itd_t *);
250Static void ehci_remove_sitd_chain(ehci_softc_t *, ehci_soft_sitd_t *);
251Static void ehci_free_itd_chain(ehci_softc_t *, ehci_soft_itd_t *);
252Static void ehci_free_sitd_chain(ehci_softc_t *, ehci_soft_sitd_t *);
253
254static inline void
255ehci_free_itd_locked(ehci_softc_t *sc, ehci_soft_itd_t *itd)
256{
257
258 LIST_INSERT_HEAD(&sc->sc_freeitds, itd, free_list);
259}
260
261static inline void
262ehci_free_sitd_locked(ehci_softc_t *sc, ehci_soft_sitd_t *sitd)
263{
264
265 LIST_INSERT_HEAD(&sc->sc_freesitds, sitd, free_list);
266}
267
268Static void ehci_abort_isoc_xfer(struct usbd_xfer *, usbd_status);
269
270Static usbd_status ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
271 int);
272
273Static void ehci_add_qh(ehci_softc_t *, ehci_soft_qh_t *,
274 ehci_soft_qh_t *);
275Static void ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
276 ehci_soft_qh_t *);
277Static void ehci_set_qh_qtd(ehci_soft_qh_t *, ehci_soft_qtd_t *);
278Static void ehci_sync_hc(ehci_softc_t *);
279
280Static void ehci_close_pipe(struct usbd_pipe *, ehci_soft_qh_t *);
281Static void ehci_abort_xfer(struct usbd_xfer *, usbd_status);
282
283#ifdef EHCI_DEBUG
284Static ehci_softc_t *theehci;
285void ehci_dump(void);
286#endif
287
288#ifdef EHCI_DEBUG
289Static void ehci_dump_regs(ehci_softc_t *);
290Static void ehci_dump_sqtds(ehci_soft_qtd_t *);
291Static void ehci_dump_sqtd(ehci_soft_qtd_t *);
292Static void ehci_dump_qtd(ehci_qtd_t *);
293Static void ehci_dump_sqh(ehci_soft_qh_t *);
294Static void ehci_dump_sitd(struct ehci_soft_itd *);
295Static void ehci_dump_itds(ehci_soft_itd_t *);
296Static void ehci_dump_itd(struct ehci_soft_itd *);
297Static void ehci_dump_exfer(struct ehci_xfer *);
298#endif
299
300#define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
301
302static inline void
303ehci_add_intr_list(ehci_softc_t *sc, struct ehci_xfer *ex)
304{
305
306 TAILQ_INSERT_TAIL(&sc->sc_intrhead, ex, ex_next);
307}
308
309static inline void
310ehci_del_intr_list(ehci_softc_t *sc, struct ehci_xfer *ex)
311{
312
313 TAILQ_REMOVE(&sc->sc_intrhead, ex, ex_next);
314}
315
316Static const struct usbd_bus_methods ehci_bus_methods = {
317 .ubm_open = ehci_open,
318 .ubm_softint = ehci_softintr,
319 .ubm_dopoll = ehci_poll,
320 .ubm_allocx = ehci_allocx,
321 .ubm_freex = ehci_freex,
322 .ubm_getlock = ehci_get_lock,
323 .ubm_rhctrl = ehci_roothub_ctrl,
324};
325
326Static const struct usbd_pipe_methods ehci_root_intr_methods = {
327 .upm_transfer = ehci_root_intr_transfer,
328 .upm_start = ehci_root_intr_start,
329 .upm_abort = ehci_root_intr_abort,
330 .upm_close = ehci_root_intr_close,
331 .upm_cleartoggle = ehci_noop,
332 .upm_done = ehci_root_intr_done,
333};
334
335Static const struct usbd_pipe_methods ehci_device_ctrl_methods = {
336 .upm_init = ehci_device_ctrl_init,
337 .upm_fini = ehci_device_ctrl_fini,
338 .upm_transfer = ehci_device_ctrl_transfer,
339 .upm_start = ehci_device_ctrl_start,
340 .upm_abort = ehci_device_ctrl_abort,
341 .upm_close = ehci_device_ctrl_close,
342 .upm_cleartoggle = ehci_noop,
343 .upm_done = ehci_device_ctrl_done,
344};
345
346Static const struct usbd_pipe_methods ehci_device_intr_methods = {
347 .upm_init = ehci_device_intr_init,
348 .upm_fini = ehci_device_intr_fini,
349 .upm_transfer = ehci_device_intr_transfer,
350 .upm_start = ehci_device_intr_start,
351 .upm_abort = ehci_device_intr_abort,
352 .upm_close = ehci_device_intr_close,
353 .upm_cleartoggle = ehci_device_clear_toggle,
354 .upm_done = ehci_device_intr_done,
355};
356
357Static const struct usbd_pipe_methods ehci_device_bulk_methods = {
358 .upm_init = ehci_device_bulk_init,
359 .upm_fini = ehci_device_bulk_fini,
360 .upm_transfer = ehci_device_bulk_transfer,
361 .upm_start = ehci_device_bulk_start,
362 .upm_abort = ehci_device_bulk_abort,
363 .upm_close = ehci_device_bulk_close,
364 .upm_cleartoggle = ehci_device_clear_toggle,
365 .upm_done = ehci_device_bulk_done,
366};
367
368Static const struct usbd_pipe_methods ehci_device_isoc_methods = {
369 .upm_init = ehci_device_isoc_init,
370 .upm_fini = ehci_device_isoc_fini,
371 .upm_transfer = ehci_device_isoc_transfer,
372 .upm_abort = ehci_device_isoc_abort,
373 .upm_close = ehci_device_isoc_close,
374 .upm_cleartoggle = ehci_noop,
375 .upm_done = ehci_device_isoc_done,
376};
377
378Static const struct usbd_pipe_methods ehci_device_fs_isoc_methods = {
379 .upm_init = ehci_device_fs_isoc_init,
380 .upm_fini = ehci_device_fs_isoc_fini,
381 .upm_transfer = ehci_device_fs_isoc_transfer,
382 .upm_abort = ehci_device_fs_isoc_abort,
383 .upm_close = ehci_device_fs_isoc_close,
384 .upm_cleartoggle = ehci_noop,
385 .upm_done = ehci_device_fs_isoc_done,
386};
387
388static const uint8_t revbits[EHCI_MAX_POLLRATE] = {
3890x00,0x40,0x20,0x60,0x10,0x50,0x30,0x70,0x08,0x48,0x28,0x68,0x18,0x58,0x38,0x78,
3900x04,0x44,0x24,0x64,0x14,0x54,0x34,0x74,0x0c,0x4c,0x2c,0x6c,0x1c,0x5c,0x3c,0x7c,
3910x02,0x42,0x22,0x62,0x12,0x52,0x32,0x72,0x0a,0x4a,0x2a,0x6a,0x1a,0x5a,0x3a,0x7a,
3920x06,0x46,0x26,0x66,0x16,0x56,0x36,0x76,0x0e,0x4e,0x2e,0x6e,0x1e,0x5e,0x3e,0x7e,
3930x01,0x41,0x21,0x61,0x11,0x51,0x31,0x71,0x09,0x49,0x29,0x69,0x19,0x59,0x39,0x79,
3940x05,0x45,0x25,0x65,0x15,0x55,0x35,0x75,0x0d,0x4d,0x2d,0x6d,0x1d,0x5d,0x3d,0x7d,
3950x03,0x43,0x23,0x63,0x13,0x53,0x33,0x73,0x0b,0x4b,0x2b,0x6b,0x1b,0x5b,0x3b,0x7b,
3960x07,0x47,0x27,0x67,0x17,0x57,0x37,0x77,0x0f,0x4f,0x2f,0x6f,0x1f,0x5f,0x3f,0x7f,
397};
398
399int
400ehci_init(ehci_softc_t *sc)
401{
402 uint32_t vers, sparams, cparams, hcr;
403 u_int i;
404 usbd_status err;
405 ehci_soft_qh_t *sqh;
406 u_int ncomp;
407
408 EHCIHIST_FUNC(); EHCIHIST_CALLED();
409#ifdef EHCI_DEBUG
410 theehci = sc;
411#endif
412
413 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
414 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB);
415 cv_init(&sc->sc_softwake_cv, "ehciab");
416 cv_init(&sc->sc_doorbell, "ehcidi");
417
418 sc->sc_xferpool = pool_cache_init(sizeof(struct ehci_xfer), 0, 0, 0,
419 "ehcixfer", NULL, IPL_USB, NULL, NULL, NULL);
420
421 sc->sc_doorbell_si = softint_establish(SOFTINT_USB | SOFTINT_MPSAFE,
422 ehci_doorbell, sc);
423 KASSERT(sc->sc_doorbell_si != NULL);
424 sc->sc_pcd_si = softint_establish(SOFTINT_USB | SOFTINT_MPSAFE,
425 ehci_pcd, sc);
426 KASSERT(sc->sc_pcd_si != NULL);
427
428 sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
429
430 vers = EREAD2(sc, EHCI_HCIVERSION);
431 aprint_verbose("%s: EHCI version %x.%x\n", device_xname(sc->sc_dev),
432 vers >> 8, vers & 0xff);
433
434 sparams = EREAD4(sc, EHCI_HCSPARAMS);
435 DPRINTF("sparams=%#x", sparams, 0, 0, 0);
436 sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
437 ncomp = EHCI_HCS_N_CC(sparams);
438 if (ncomp != sc->sc_ncomp) {
439 aprint_verbose("%s: wrong number of companions (%d != %d)\n",
440 device_xname(sc->sc_dev), ncomp, sc->sc_ncomp);
441#if NOHCI == 0 || NUHCI == 0
442 aprint_error("%s: ohci or uhci probably not configured\n",
443 device_xname(sc->sc_dev));
444#endif
445 if (ncomp < sc->sc_ncomp)
446 sc->sc_ncomp = ncomp;
447 }
448 if (sc->sc_ncomp > 0) {
449 KASSERT(!(sc->sc_flags & EHCIF_ETTF));
450 aprint_normal("%s: companion controller%s, %d port%s each:",
451 device_xname(sc->sc_dev), sc->sc_ncomp!=1 ? "s" : "",
452 EHCI_HCS_N_PCC(sparams),
453 EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
454 for (i = 0; i < sc->sc_ncomp; i++)
455 aprint_normal(" %s", device_xname(sc->sc_comps[i]));
456 aprint_normal("\n");
457 }
458 sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
459 sc->sc_hasppc = EHCI_HCS_PPC(sparams);
460
461 cparams = EREAD4(sc, EHCI_HCCPARAMS);
462 DPRINTF("cparams=%#x", cparams, 0, 0, 0);
463
464 if (EHCI_HCC_64BIT(cparams)) {
465 /* MUST clear segment register if 64 bit capable. */
466 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
467 }
468
469 if (cparams & EHCI_HCC_IST_FULLFRAME) {
470 sc->sc_istthreshold = 0;
471 } else {
472 sc->sc_istthreshold = EHCI_HCC_GET_IST_THRESHOLD(cparams);
473 }
474
475 sc->sc_bus.ub_revision = USBREV_2_0;
476 sc->sc_bus.ub_usedma = true;
477 sc->sc_bus.ub_dmaflags = USBMALLOC_MULTISEG;
478
479 /* Reset the controller */
480 DPRINTF("resetting", 0, 0, 0, 0);
481 EOWRITE4(sc, EHCI_USBCMD, 0); /* Halt controller */
482 usb_delay_ms(&sc->sc_bus, 1);
483 EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
484 for (i = 0; i < 100; i++) {
485 usb_delay_ms(&sc->sc_bus, 1);
486 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
487 if (!hcr)
488 break;
489 }
490 if (hcr) {
491 aprint_error("%s: reset timeout\n", device_xname(sc->sc_dev));
492 return EIO;
493 }
494 if (sc->sc_vendor_init)
495 sc->sc_vendor_init(sc);
496
497 /* XXX need proper intr scheduling */
498 sc->sc_rand = 96;
499
500 /* frame list size at default, read back what we got and use that */
501 switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
502 case 0: sc->sc_flsize = 1024; break;
503 case 1: sc->sc_flsize = 512; break;
504 case 2: sc->sc_flsize = 256; break;
505 case 3: return EIO;
506 }
507 err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
508 EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
509 if (err)
510 return err;
511 DPRINTF("flsize=%d", sc->sc_flsize, 0, 0, 0);
512 sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
513
514 for (i = 0; i < sc->sc_flsize; i++) {
515 sc->sc_flist[i] = EHCI_NULL;
516 }
517
518 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
519
520 sc->sc_softitds = kmem_zalloc(sc->sc_flsize * sizeof(ehci_soft_itd_t *),
521 KM_SLEEP);
522 if (sc->sc_softitds == NULL)
523 return ENOMEM;
524 LIST_INIT(&sc->sc_freeitds);
525 LIST_INIT(&sc->sc_freesitds);
526 TAILQ_INIT(&sc->sc_intrhead);
527
528 /* Set up the bus struct. */
529 sc->sc_bus.ub_methods = &ehci_bus_methods;
530 sc->sc_bus.ub_pipesize= sizeof(struct ehci_pipe);
531
532 sc->sc_eintrs = EHCI_NORMAL_INTRS;
533
534 /*
535 * Allocate the interrupt dummy QHs. These are arranged to give poll
536 * intervals that are powers of 2 times 1ms.
537 */
538 for (i = 0; i < EHCI_INTRQHS; i++) {
539 sqh = ehci_alloc_sqh(sc);
540 if (sqh == NULL) {
541 err = ENOMEM;
542 goto bad1;
543 }
544 sc->sc_islots[i].sqh = sqh;
545 }
546 for (i = 0; i < EHCI_INTRQHS; i++) {
547 sqh = sc->sc_islots[i].sqh;
548 if (i == 0) {
549 /* The last (1ms) QH terminates. */
550 sqh->qh.qh_link = EHCI_NULL;
551 sqh->next = NULL;
552 } else {
553 /* Otherwise the next QH has half the poll interval */
554 sqh->next = sc->sc_islots[(i + 1) / 2 - 1].sqh;
555 sqh->qh.qh_link = htole32(sqh->next->physaddr |
556 EHCI_LINK_QH);
557 }
558 sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
559 sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
560 sqh->qh.qh_curqtd = EHCI_NULL;
561 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
562 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
563 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
564 sqh->sqtd = NULL;
565 usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
566 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
567 }
568 /* Point the frame list at the last level (128ms). */
569 for (i = 0; i < sc->sc_flsize; i++) {
570 int j;
571
572 j = (i & ~(EHCI_MAX_POLLRATE-1)) |
573 revbits[i & (EHCI_MAX_POLLRATE-1)];
574 sc->sc_flist[j] = htole32(EHCI_LINK_QH |
575 sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
576 i)].sqh->physaddr);
577 }
578 usb_syncmem(&sc->sc_fldma, 0, sc->sc_flsize * sizeof(ehci_link_t),
579 BUS_DMASYNC_PREWRITE);
580
581 /* Allocate dummy QH that starts the async list. */
582 sqh = ehci_alloc_sqh(sc);
583 if (sqh == NULL) {
584 err = ENOMEM;
585 goto bad1;
586 }
587 /* Fill the QH */
588 sqh->qh.qh_endp =
589 htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
590 sqh->qh.qh_link =
591 htole32(sqh->physaddr | EHCI_LINK_QH);
592 sqh->qh.qh_curqtd = EHCI_NULL;
593 sqh->next = NULL;
594 /* Fill the overlay qTD */
595 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
596 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
597 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
598 sqh->sqtd = NULL;
599 usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
600 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
601#ifdef EHCI_DEBUG
602 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
603 ehci_dump_sqh(sqh);
604 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
605#endif
606
607 /* Point to async list */
608 sc->sc_async_head = sqh;
609 EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
610
611 callout_init(&sc->sc_tmo_intrlist, CALLOUT_MPSAFE);
612
613 /* Turn on controller */
614 EOWRITE4(sc, EHCI_USBCMD,
615 EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
616 (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
617 EHCI_CMD_ASE |
618 EHCI_CMD_PSE |
619 EHCI_CMD_RS);
620
621 /* Take over port ownership */
622 EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
623
624 for (i = 0; i < 100; i++) {
625 usb_delay_ms(&sc->sc_bus, 1);
626 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
627 if (!hcr)
628 break;
629 }
630 if (hcr) {
631 aprint_error("%s: run timeout\n", device_xname(sc->sc_dev));
632 return EIO;
633 }
634
635 /* Enable interrupts */
636 DPRINTF("enabling interupts", 0, 0, 0, 0);
637 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
638
639 return 0;
640
641#if 0
642 bad2:
643 ehci_free_sqh(sc, sc->sc_async_head);
644#endif
645 bad1:
646 usb_freemem(&sc->sc_bus, &sc->sc_fldma);
647 return err;
648}
649
650int
651ehci_intr(void *v)
652{
653 ehci_softc_t *sc = v;
654 int ret = 0;
655
656 EHCIHIST_FUNC(); EHCIHIST_CALLED();
657
658 if (sc == NULL)
659 return 0;
660
661 mutex_spin_enter(&sc->sc_intr_lock);
662
663 if (sc->sc_dying || !device_has_power(sc->sc_dev))
664 goto done;
665
666 /* If we get an interrupt while polling, then just ignore it. */
667 if (sc->sc_bus.ub_usepolling) {
668 uint32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
669
670 if (intrs)
671 EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
672 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0);
673 goto done;
674 }
675
676 ret = ehci_intr1(sc);
677
678done:
679 mutex_spin_exit(&sc->sc_intr_lock);
680 return ret;
681}
682
683Static int
684ehci_intr1(ehci_softc_t *sc)
685{
686 uint32_t intrs, eintrs;
687
688 EHCIHIST_FUNC(); EHCIHIST_CALLED();
689
690 /* In case the interrupt occurs before initialization has completed. */
691 if (sc == NULL) {
692#ifdef DIAGNOSTIC
693 printf("ehci_intr1: sc == NULL\n");
694#endif
695 return 0;
696 }
697
698 KASSERT(mutex_owned(&sc->sc_intr_lock));
699
700 intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
701 if (!intrs)
702 return 0;
703
704 eintrs = intrs & sc->sc_eintrs;
705 DPRINTF("sc=%p intrs=%#x(%#x) eintrs=%#x", sc, intrs,
706 EOREAD4(sc, EHCI_USBSTS), eintrs);
707 if (!eintrs)
708 return 0;
709
710 EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
711 if (eintrs & EHCI_STS_IAA) {
712 DPRINTF("door bell", 0, 0, 0, 0);
713 kpreempt_disable();
714 KASSERT(sc->sc_doorbell_si != NULL);
715 softint_schedule(sc->sc_doorbell_si);
716 kpreempt_enable();
717 eintrs &= ~EHCI_STS_IAA;
718 }
719 if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
720 DPRINTF("INT=%d ERRINT=%d",
721 eintrs & EHCI_STS_INT ? 1 : 0,
722 eintrs & EHCI_STS_ERRINT ? 1 : 0, 0, 0);
723 usb_schedsoftintr(&sc->sc_bus);
724 eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
725 }
726 if (eintrs & EHCI_STS_HSE) {
727 printf("%s: unrecoverable error, controller halted\n",
728 device_xname(sc->sc_dev));
729 /* XXX what else */
730 }
731 if (eintrs & EHCI_STS_PCD) {
732 kpreempt_disable();
733 KASSERT(sc->sc_pcd_si != NULL);
734 softint_schedule(sc->sc_pcd_si);
735 kpreempt_enable();
736 eintrs &= ~EHCI_STS_PCD;
737 }
738
739 if (eintrs != 0) {
740 /* Block unprocessed interrupts. */
741 sc->sc_eintrs &= ~eintrs;
742 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
743 printf("%s: blocking intrs 0x%x\n",
744 device_xname(sc->sc_dev), eintrs);
745 }
746
747 return 1;
748}
749
750Static void
751ehci_doorbell(void *addr)
752{
753 ehci_softc_t *sc = addr;
754
755 mutex_enter(&sc->sc_lock);
756 cv_broadcast(&sc->sc_doorbell);
757 mutex_exit(&sc->sc_lock);
758}
759
760Static void
761ehci_pcd(void *addr)
762{
763 ehci_softc_t *sc = addr;
764 struct usbd_xfer *xfer;
765 u_char *p;
766 int i, m;
767
768 EHCIHIST_FUNC(); EHCIHIST_CALLED();
769
770 mutex_enter(&sc->sc_lock);
771 xfer = sc->sc_intrxfer;
772
773 if (xfer == NULL) {
774 /* Just ignore the change. */
775 goto done;
776 }
777
778 p = xfer->ux_buf;
779 m = min(sc->sc_noport, xfer->ux_length * 8 - 1);
780 memset(p, 0, xfer->ux_length);
781 for (i = 1; i <= m; i++) {
782 /* Pick out CHANGE bits from the status reg. */
783 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
784 p[i/8] |= 1 << (i%8);
785 if (i % 8 == 7)
786 DPRINTF("change(%d)=0x%02x", i / 8, p[i/8], 0, 0);
787 }
788 xfer->ux_actlen = xfer->ux_length;
789 xfer->ux_status = USBD_NORMAL_COMPLETION;
790
791 usb_transfer_complete(xfer);
792
793done:
794 mutex_exit(&sc->sc_lock);
795}
796
797Static void
798ehci_softintr(void *v)
799{
800 struct usbd_bus *bus = v;
801 ehci_softc_t *sc = EHCI_BUS2SC(bus);
802 struct ehci_xfer *ex, *nextex;
803
804 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
805
806 EHCIHIST_FUNC(); EHCIHIST_CALLED();
807
808 ex_completeq_t cq;
809 TAILQ_INIT(&cq);
810
811 /*
812 * The only explanation I can think of for why EHCI is as brain dead
813 * as UHCI interrupt-wise is that Intel was involved in both.
814 * An interrupt just tells us that something is done, we have no
815 * clue what, so we need to scan through all active transfers. :-(
816 */
817
818 /*
819 * ehci_idone will remove transfer from sc->sc_intrhead if it's
820 * complete and add to our cq list
821 *
822 */
823 TAILQ_FOREACH_SAFE(ex, &sc->sc_intrhead, ex_next, nextex) {
824 switch (ex->ex_type) {
825 case EX_CTRL:
826 case EX_BULK:
827 case EX_INTR:
828 ehci_check_qh_intr(sc, ex, &cq);
829 break;
830 case EX_ISOC:
831 ehci_check_itd_intr(sc, ex, &cq);
832 break;
833 case EX_FS_ISOC:
834 ehci_check_sitd_intr(sc, ex, &cq);
835 break;
836 default:
837 KASSERT(false);
838 }
839
840 }
841
842 /*
843 * We abuse ex_next for the interrupt and complete lists and
844 * interrupt transfers will get re-added here so use
845 * the _SAFE version of TAILQ_FOREACH.
846 */
847 TAILQ_FOREACH_SAFE(ex, &cq, ex_next, nextex) {
848 usb_transfer_complete(&ex->ex_xfer);
849 }
850
851 /* Schedule a callout to catch any dropped transactions. */
852 if ((sc->sc_flags & EHCIF_DROPPED_INTR_WORKAROUND) &&
853 !TAILQ_EMPTY(&sc->sc_intrhead))
854 callout_reset(&sc->sc_tmo_intrlist,
855 hz, ehci_intrlist_timeout, sc);
856
857 if (sc->sc_softwake) {
858 sc->sc_softwake = 0;
859 cv_broadcast(&sc->sc_softwake_cv);
860 }
861}
862
863Static void
864ehci_check_qh_intr(ehci_softc_t *sc, struct ehci_xfer *ex, ex_completeq_t *cq)
865{
866 ehci_soft_qtd_t *sqtd, *fsqtd, *lsqtd;
867 uint32_t status;
868
869 EHCIHIST_FUNC(); EHCIHIST_CALLED();
870
871 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
872
873 if (ex->ex_type == EX_CTRL) {
874 fsqtd = ex->ex_setup;
875 lsqtd = ex->ex_status;
876 } else {
877 fsqtd = ex->ex_sqtdstart;
878 lsqtd = ex->ex_sqtdend;
879 }
880 KASSERTMSG(fsqtd != NULL && lsqtd != NULL,
881 "xfer %p xt %d fsqtd %p lsqtd %p", ex, ex->ex_type, fsqtd, lsqtd);
882
883 /*
884 * If the last TD is still active we need to check whether there
885 * is an error somewhere in the middle, or whether there was a
886 * short packet (SPD and not ACTIVE).
887 */
888 usb_syncmem(&lsqtd->dma,
889 lsqtd->offs + offsetof(ehci_qtd_t, qtd_status),
890 sizeof(lsqtd->qtd.qtd_status),
891 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
892 status = le32toh(lsqtd->qtd.qtd_status);
893 usb_syncmem(&lsqtd->dma,
894 lsqtd->offs + offsetof(ehci_qtd_t, qtd_status),
895 sizeof(lsqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
896 if (status & EHCI_QTD_ACTIVE) {
897 DPRINTFN(10, "active ex=%p", ex, 0, 0, 0);
898
899 /* last qTD has already been checked */
900 for (sqtd = fsqtd; sqtd != lsqtd; sqtd = sqtd->nextqtd) {
901 usb_syncmem(&sqtd->dma,
902 sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
903 sizeof(sqtd->qtd.qtd_status),
904 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
905 status = le32toh(sqtd->qtd.qtd_status);
906 usb_syncmem(&sqtd->dma,
907 sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
908 sizeof(sqtd->qtd.qtd_status), BUS_DMASYNC_PREREAD);
909 /* If there's an active QTD the xfer isn't done. */
910 if (status & EHCI_QTD_ACTIVE)
911 break;
912 /* Any kind of error makes the xfer done. */
913 if (status & EHCI_QTD_HALTED)
914 goto done;
915 /* Handle short packets */
916 if (EHCI_QTD_GET_BYTES(status) != 0) {
917 /*
918 * If we get here for a control transfer then
919 * we need to let the hardware complete the
920 * status phase. That is, we're not done
921 * quite yet.
922 *
923 * Otherwise, we're done.
924 */
925 if (ex->ex_type == EX_CTRL) {
926 break;
927 }
928 goto done;
929 }
930 }
931 DPRINTFN(10, "ex=%p std=%p still active", ex, ex->ex_sqtdstart,
932 0, 0);
933#ifdef EHCI_DEBUG
934 DPRINTFN(5, "--- still active start ---", 0, 0, 0, 0);
935 ehci_dump_sqtds(ex->ex_sqtdstart);
936 DPRINTFN(5, "--- still active end ---", 0, 0, 0, 0);
937#endif
938 return;
939 }
940 done:
941 DPRINTFN(10, "ex=%p done", ex, 0, 0, 0);
942 callout_stop(&ex->ex_xfer.ux_callout);
943 ehci_idone(ex, cq);
944}
945
946Static void
947ehci_check_itd_intr(ehci_softc_t *sc, struct ehci_xfer *ex, ex_completeq_t *cq)
948{
949 ehci_soft_itd_t *itd;
950 int i;
951
952 EHCIHIST_FUNC(); EHCIHIST_CALLED();
953
954 KASSERT(mutex_owned(&sc->sc_lock));
955
956 if (&ex->ex_xfer != SIMPLEQ_FIRST(&ex->ex_xfer.ux_pipe->up_queue))
957 return;
958
959 KASSERTMSG(ex->ex_itdstart != NULL && ex->ex_itdend != NULL,
960 "xfer %p fitd %p litd %p", ex, ex->ex_itdstart, ex->ex_itdend);
961
962 itd = ex->ex_itdend;
963
964 /*
965 * check no active transfers in last itd, meaning we're finished
966 */
967
968 usb_syncmem(&itd->dma, itd->offs + offsetof(ehci_itd_t, itd_ctl),
969 sizeof(itd->itd.itd_ctl),
970 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
971
972 for (i = 0; i < EHCI_ITD_NUFRAMES; i++) {
973 if (le32toh(itd->itd.itd_ctl[i]) & EHCI_ITD_ACTIVE)
974 break;
975 }
976
977 if (i == EHCI_ITD_NUFRAMES) {
978 goto done; /* All 8 descriptors inactive, it's done */
979 }
980
981 usb_syncmem(&itd->dma, itd->offs + offsetof(ehci_itd_t, itd_ctl),
982 sizeof(itd->itd.itd_ctl), BUS_DMASYNC_PREREAD);
983
984 DPRINTFN(10, "ex %p itd %p still active", ex, ex->ex_itdstart, 0, 0);
985 return;
986done:
987 DPRINTF("ex %p done", ex, 0, 0, 0);
988 callout_stop(&ex->ex_xfer.ux_callout);
989 ehci_idone(ex, cq);
990}
991
992void
993ehci_check_sitd_intr(ehci_softc_t *sc, struct ehci_xfer *ex, ex_completeq_t *cq)
994{
995 ehci_soft_sitd_t *sitd;
996
997 EHCIHIST_FUNC(); EHCIHIST_CALLED();
998
999 KASSERT(mutex_owned(&sc->sc_lock));
1000
1001 if (&ex->ex_xfer != SIMPLEQ_FIRST(&ex->ex_xfer.ux_pipe->up_queue))
1002 return;
1003
1004 KASSERTMSG(ex->ex_sitdstart != NULL && ex->ex_sitdend != NULL,
1005 "xfer %p fsitd %p lsitd %p", ex, ex->ex_sitdstart, ex->ex_sitdend);
1006
1007 sitd = ex->ex_sitdend;
1008
1009 /*
1010 * check no active transfers in last sitd, meaning we're finished
1011 */
1012
1013 usb_syncmem(&sitd->dma, sitd->offs + offsetof(ehci_sitd_t, sitd_trans),
1014 sizeof(sitd->sitd.sitd_trans),
1015 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1016
1017 bool active = ((le32toh(sitd->sitd.sitd_trans) & EHCI_SITD_ACTIVE) != 0);
1018
1019 usb_syncmem(&sitd->dma, sitd->offs + offsetof(ehci_sitd_t, sitd_trans),
1020 sizeof(sitd->sitd.sitd_trans), BUS_DMASYNC_PREREAD);
1021
1022 if (active)
1023 return;
1024
1025 DPRINTFN(10, "ex=%p done", ex, 0, 0, 0);
1026 callout_stop(&(ex->ex_xfer.ux_callout));
1027 ehci_idone(ex, cq);
1028}
1029
1030
1031Static void
1032ehci_idone(struct ehci_xfer *ex, ex_completeq_t *cq)
1033{
1034 struct usbd_xfer *xfer = &ex->ex_xfer;
1035 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
1036 struct ehci_softc *sc = EHCI_XFER2SC(xfer);
1037 ehci_soft_qtd_t *sqtd, *fsqtd, *lsqtd;
1038 uint32_t status = 0, nstatus = 0;
1039 int actlen = 0;
1040
1041 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1042
1043 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
1044
1045 DPRINTF("ex=%p", ex, 0, 0, 0);
1046
1047 if (xfer->ux_status == USBD_CANCELLED ||
1048 xfer->ux_status == USBD_TIMEOUT) {
1049 DPRINTF("aborted xfer=%p", xfer, 0, 0, 0);
1050 return;
1051 }
1052
1053#ifdef DIAGNOSTIC
1054#ifdef EHCI_DEBUG
1055 if (ex->ex_isdone) {
1056 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
1057 ehci_dump_exfer(ex);
1058 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
1059 }
1060#endif
1061 KASSERTMSG(!ex->ex_isdone, "xfer %p type %d status %d", xfer,
1062 ex->ex_type, xfer->ux_status);
1063 ex->ex_isdone = true;
1064#endif
1065
1066 DPRINTF("xfer=%p, pipe=%p ready", xfer, epipe, 0, 0);
1067
1068 /* The transfer is done, compute actual length and status. */
1069 if (ex->ex_type == EX_ISOC) {
1070 /* HS isoc transfer */
1071
1072 struct ehci_soft_itd *itd;
1073 int i, nframes, len, uframes;
1074
1075 nframes = 0;
1076
1077#ifdef EHCI_DEBUG
1078 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
1079 ehci_dump_itds(ex->ex_itdstart);
1080 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
1081#endif
1082
1083 i = xfer->ux_pipe->up_endpoint->ue_edesc->bInterval;
1084 uframes = min(1 << (i - 1), USB_UFRAMES_PER_FRAME);
1085
1086 for (itd = ex->ex_itdstart; itd != NULL; itd = itd->xfer_next) {
1087 usb_syncmem(&itd->dma,
1088 itd->offs + offsetof(ehci_itd_t,itd_ctl),
1089 sizeof(itd->itd.itd_ctl),
1090 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1091
1092 for (i = 0; i < EHCI_ITD_NUFRAMES; i += uframes) {
1093 /*
1094 * XXX - driver didn't fill in the frame full
1095 * of uframes. This leads to scheduling
1096 * inefficiencies, but working around
1097 * this doubles complexity of tracking
1098 * an xfer.
1099 */
1100 if (nframes >= xfer->ux_nframes)
1101 break;
1102
1103 status = le32toh(itd->itd.itd_ctl[i]);
1104 len = EHCI_ITD_GET_LEN(status);
1105 if (EHCI_ITD_GET_STATUS(status) != 0)
1106 len = 0; /*No valid data on error*/
1107
1108 xfer->ux_frlengths[nframes++] = len;
1109 actlen += len;
1110 }
1111 usb_syncmem(&itd->dma,
1112 itd->offs + offsetof(ehci_itd_t,itd_ctl),
1113 sizeof(itd->itd.itd_ctl), BUS_DMASYNC_PREREAD);
1114
1115 if (nframes >= xfer->ux_nframes)
1116 break;
1117 }
1118
1119 xfer->ux_actlen = actlen;
1120 xfer->ux_status = USBD_NORMAL_COMPLETION;
1121 goto end;
1122 } else if (ex->ex_type == EX_FS_ISOC) {
1123 /* FS isoc transfer */
1124 struct ehci_soft_sitd *sitd;
1125 int nframes, len;
1126
1127 nframes = 0;
1128
1129 for (sitd = ex->ex_sitdstart; sitd != NULL;
1130 sitd = sitd->xfer_next) {
1131 usb_syncmem(&sitd->dma,
1132 sitd->offs + offsetof(ehci_sitd_t, sitd_trans),
1133 sizeof(sitd->sitd.sitd_trans),
1134 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1135
1136 /*
1137 * XXX - driver didn't fill in the frame full
1138 * of uframes. This leads to scheduling
1139 * inefficiencies, but working around
1140 * this doubles complexity of tracking
1141 * an xfer.
1142 */
1143 if (nframes >= xfer->ux_nframes)
1144 break;
1145
1146 status = le32toh(sitd->sitd.sitd_trans);
1147 usb_syncmem(&sitd->dma,
1148 sitd->offs + offsetof(ehci_sitd_t, sitd_trans),
1149 sizeof(sitd->sitd.sitd_trans), BUS_DMASYNC_PREREAD);
1150
1151 len = EHCI_SITD_GET_LEN(status);
1152 if (status & (EHCI_SITD_ERR|EHCI_SITD_BUFERR|
1153 EHCI_SITD_BABBLE|EHCI_SITD_XACTERR|EHCI_SITD_MISS)) {
1154 /* No valid data on error */
1155 len = xfer->ux_frlengths[nframes];
1156 }
1157
1158 /*
1159 * frlengths[i]: # of bytes to send
1160 * len: # of bytes host didn't send
1161 */
1162 xfer->ux_frlengths[nframes] -= len;
1163 /* frlengths[i]: # of bytes host sent */
1164 actlen += xfer->ux_frlengths[nframes++];
1165
1166 if (nframes >= xfer->ux_nframes)
1167 break;
1168 }
1169
1170 xfer->ux_actlen = actlen;
1171 xfer->ux_status = USBD_NORMAL_COMPLETION;
1172 goto end;
1173 }
1174 KASSERT(ex->ex_type == EX_CTRL || ex->ex_type == EX_INTR ||
1175 ex->ex_type == EX_BULK);
1176
1177 /* Continue processing xfers using queue heads */
1178 if (ex->ex_type == EX_CTRL) {
1179 fsqtd = ex->ex_setup;
1180 lsqtd = ex->ex_status;
1181 } else {
1182 fsqtd = ex->ex_sqtdstart;
1183 lsqtd = ex->ex_sqtdend;
1184 }
1185#ifdef EHCI_DEBUG
1186 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
1187 ehci_dump_sqtds(fsqtd);
1188 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
1189#endif
1190
1191 for (sqtd = fsqtd; sqtd != lsqtd->nextqtd; sqtd = sqtd->nextqtd) {
1192 usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
1193 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1194 nstatus = le32toh(sqtd->qtd.qtd_status);
1195 usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
1196 BUS_DMASYNC_PREREAD);
1197 if (nstatus & EHCI_QTD_ACTIVE)
1198 break;
1199
1200 status = nstatus;
1201 if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
1202 actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
1203 }
1204
1205 /*
1206 * If there are left over TDs we need to update the toggle.
1207 * The default pipe doesn't need it since control transfers
1208 * start the toggle at 0 every time.
1209 * For a short transfer we need to update the toggle for the missing
1210 * packets within the qTD.
1211 */
1212 if ((sqtd != lsqtd->nextqtd || EHCI_QTD_GET_BYTES(status)) &&
1213 xfer->ux_pipe->up_dev->ud_pipe0 != xfer->ux_pipe) {
1214 DPRINTF("toggle update status=0x%08x nstatus=0x%08x",
1215 status, nstatus, 0, 0);
1216#if 0
1217 ehci_dump_sqh(epipe->sqh);
1218 ehci_dump_sqtds(ex->ex_sqtdstart);
1219#endif
1220 epipe->nexttoggle = EHCI_QTD_GET_TOGGLE(nstatus);
1221 }
1222
1223 DPRINTF("len=%d actlen=%d status=0x%08x", xfer->ux_length, actlen,
1224 status, 0);
1225 xfer->ux_actlen = actlen;
1226 if (status & EHCI_QTD_HALTED) {
1227#ifdef EHCI_DEBUG
1228 DPRINTF("halted addr=%d endpt=0x%02x",
1229 xfer->ux_pipe->up_dev->ud_addr,
1230 xfer->ux_pipe->up_endpoint->ue_edesc->bEndpointAddress,
1231 0, 0);
1232 DPRINTF("cerr=%d pid=%d",
1233 EHCI_QTD_GET_CERR(status), EHCI_QTD_GET_PID(status),
1234 0, 0);
1235 DPRINTF("active =%d halted=%d buferr=%d babble=%d",
1236 status & EHCI_QTD_ACTIVE ? 1 : 0,
1237 status & EHCI_QTD_HALTED ? 1 : 0,
1238 status & EHCI_QTD_BUFERR ? 1 : 0,
1239 status & EHCI_QTD_BABBLE ? 1 : 0);
1240
1241 DPRINTF("xacterr=%d missed=%d split =%d ping =%d",
1242 status & EHCI_QTD_XACTERR ? 1 : 0,
1243 status & EHCI_QTD_MISSEDMICRO ? 1 : 0,
1244 status & EHCI_QTD_SPLITXSTATE ? 1 : 0,
1245 status & EHCI_QTD_PINGSTATE ? 1 : 0);
1246
1247 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
1248 ehci_dump_sqh(epipe->sqh);
1249 ehci_dump_sqtds(ex->ex_sqtdstart);
1250 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
1251#endif
1252 /* low&full speed has an extra error flag */
1253 if (EHCI_QH_GET_EPS(epipe->sqh->qh.qh_endp) !=
1254 EHCI_QH_SPEED_HIGH)
1255 status &= EHCI_QTD_STATERRS | EHCI_QTD_PINGSTATE;
1256 else
1257 status &= EHCI_QTD_STATERRS;
1258 if (status == 0) /* no other errors means a stall */ {
1259 xfer->ux_status = USBD_STALLED;
1260 } else {
1261 xfer->ux_status = USBD_IOERROR; /* more info XXX */
1262 }
1263 /* XXX need to reset TT on missed microframe */
1264 if (status & EHCI_QTD_MISSEDMICRO) {
1265 printf("%s: missed microframe, TT reset not "
1266 "implemented, hub might be inoperational\n",
1267 device_xname(sc->sc_dev));
1268 }
1269 } else {
1270 xfer->ux_status = USBD_NORMAL_COMPLETION;
1271 }
1272
1273 end:
1274
1275 ehci_del_intr_list(sc, ex);
1276 TAILQ_INSERT_TAIL(cq, ex, ex_next);
1277
1278 DPRINTF("ex=%p done", ex, 0, 0, 0);
1279}
1280
1281Static void
1282ehci_poll(struct usbd_bus *bus)
1283{
1284 ehci_softc_t *sc = EHCI_BUS2SC(bus);
1285
1286 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1287
1288#ifdef EHCI_DEBUG
1289 static int last;
1290 int new;
1291 new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1292 if (new != last) {
1293 DPRINTF("intrs=0x%04x", new, 0, 0, 0);
1294 last = new;
1295 }
1296#endif
1297
1298 if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs) {
1299 mutex_spin_enter(&sc->sc_intr_lock);
1300 ehci_intr1(sc);
1301 mutex_spin_exit(&sc->sc_intr_lock);
1302 }
1303}
1304
1305void
1306ehci_childdet(device_t self, device_t child)
1307{
1308 struct ehci_softc *sc = device_private(self);
1309
1310 KASSERT(sc->sc_child == child);
1311 sc->sc_child = NULL;
1312}
1313
1314int
1315ehci_detach(struct ehci_softc *sc, int flags)
1316{
1317 int rv = 0;
1318
1319 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1320
1321 if (sc->sc_child != NULL)
1322 rv = config_detach(sc->sc_child, flags);
1323
1324 if (rv != 0)
1325 return rv;
1326
1327 callout_halt(&sc->sc_tmo_intrlist, NULL);
1328 callout_destroy(&sc->sc_tmo_intrlist);
1329
1330 /* XXX free other data structures XXX */
1331 if (sc->sc_softitds)
1332 kmem_free(sc->sc_softitds,
1333 sc->sc_flsize * sizeof(ehci_soft_itd_t *));
1334 cv_destroy(&sc->sc_doorbell);
1335 cv_destroy(&sc->sc_softwake_cv);
1336
1337#if 0
1338 /* XXX destroyed in ehci_pci.c as it controls ehci_intr access */
1339
1340 softint_disestablish(sc->sc_doorbell_si);
1341 softint_disestablish(sc->sc_pcd_si);
1342
1343 mutex_destroy(&sc->sc_lock);
1344 mutex_destroy(&sc->sc_intr_lock);
1345#endif
1346
1347 pool_cache_destroy(sc->sc_xferpool);
1348
1349 EOWRITE4(sc, EHCI_CONFIGFLAG, 0);
1350
1351 return rv;
1352}
1353
1354
1355int
1356ehci_activate(device_t self, enum devact act)
1357{
1358 struct ehci_softc *sc = device_private(self);
1359
1360 switch (act) {
1361 case DVACT_DEACTIVATE:
1362 sc->sc_dying = 1;
1363 return 0;
1364 default:
1365 return EOPNOTSUPP;
1366 }
1367}
1368
1369/*
1370 * Handle suspend/resume.
1371 *
1372 * We need to switch to polling mode here, because this routine is
1373 * called from an interrupt context. This is all right since we
1374 * are almost suspended anyway.
1375 *
1376 * Note that this power handler isn't to be registered directly; the
1377 * bus glue needs to call out to it.
1378 */
1379bool
1380ehci_suspend(device_t dv, const pmf_qual_t *qual)
1381{
1382 ehci_softc_t *sc = device_private(dv);
1383 int i;
1384 uint32_t cmd, hcr;
1385
1386 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1387
1388 mutex_spin_enter(&sc->sc_intr_lock);
1389 sc->sc_bus.ub_usepolling++;
1390 mutex_spin_exit(&sc->sc_intr_lock);
1391
1392 for (i = 1; i <= sc->sc_noport; i++) {
1393 cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1394 if ((cmd & EHCI_PS_PO) == 0 && (cmd & EHCI_PS_PE) == EHCI_PS_PE)
1395 EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_SUSP);
1396 }
1397
1398 sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
1399
1400 cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
1401 EOWRITE4(sc, EHCI_USBCMD, cmd);
1402
1403 for (i = 0; i < 100; i++) {
1404 hcr = EOREAD4(sc, EHCI_USBSTS) & (EHCI_STS_ASS | EHCI_STS_PSS);
1405 if (hcr == 0)
1406 break;
1407
1408 usb_delay_ms(&sc->sc_bus, 1);
1409 }
1410 if (hcr != 0)
1411 printf("%s: reset timeout\n", device_xname(dv));
1412
1413 cmd &= ~EHCI_CMD_RS;
1414 EOWRITE4(sc, EHCI_USBCMD, cmd);
1415
1416 for (i = 0; i < 100; i++) {
1417 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1418 if (hcr == EHCI_STS_HCH)
1419 break;
1420
1421 usb_delay_ms(&sc->sc_bus, 1);
1422 }
1423 if (hcr != EHCI_STS_HCH)
1424 printf("%s: config timeout\n", device_xname(dv));
1425
1426 mutex_spin_enter(&sc->sc_intr_lock);
1427 sc->sc_bus.ub_usepolling--;
1428 mutex_spin_exit(&sc->sc_intr_lock);
1429
1430 return true;
1431}
1432
1433bool
1434ehci_resume(device_t dv, const pmf_qual_t *qual)
1435{
1436 ehci_softc_t *sc = device_private(dv);
1437 int i;
1438 uint32_t cmd, hcr;
1439
1440 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1441
1442 /* restore things in case the bios sucks */
1443 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1444 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1445 EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1446 sc->sc_async_head->physaddr | EHCI_LINK_QH);
1447
1448 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs & ~EHCI_INTR_PCIE);
1449
1450 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1451
1452 hcr = 0;
1453 for (i = 1; i <= sc->sc_noport; i++) {
1454 cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1455 if ((cmd & EHCI_PS_PO) == 0 &&
1456 (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
1457 EOWRITE4(sc, EHCI_PORTSC(i), cmd | EHCI_PS_FPR);
1458 hcr = 1;
1459 }
1460 }
1461
1462 if (hcr) {
1463 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1464
1465 for (i = 1; i <= sc->sc_noport; i++) {
1466 cmd = EOREAD4(sc, EHCI_PORTSC(i)) & ~EHCI_PS_CLEAR;
1467 if ((cmd & EHCI_PS_PO) == 0 &&
1468 (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
1469 EOWRITE4(sc, EHCI_PORTSC(i),
1470 cmd & ~EHCI_PS_FPR);
1471 }
1472 }
1473
1474 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1475 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1476
1477 for (i = 0; i < 100; i++) {
1478 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1479 if (hcr != EHCI_STS_HCH)
1480 break;
1481
1482 usb_delay_ms(&sc->sc_bus, 1);
1483 }
1484 if (hcr == EHCI_STS_HCH)
1485 printf("%s: config timeout\n", device_xname(dv));
1486
1487 return true;
1488}
1489
1490/*
1491 * Shut down the controller when the system is going down.
1492 */
1493bool
1494ehci_shutdown(device_t self, int flags)
1495{
1496 ehci_softc_t *sc = device_private(self);
1497
1498 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1499
1500 EOWRITE4(sc, EHCI_USBCMD, 0); /* Halt controller */
1501 EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1502 return true;
1503}
1504
1505Static struct usbd_xfer *
1506ehci_allocx(struct usbd_bus *bus, unsigned int nframes)
1507{
1508 struct ehci_softc *sc = EHCI_BUS2SC(bus);
1509 struct usbd_xfer *xfer;
1510
1511 xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT);
1512 if (xfer != NULL) {
1513 memset(xfer, 0, sizeof(struct ehci_xfer));
1514#ifdef DIAGNOSTIC
1515 struct ehci_xfer *ex = EHCI_XFER2EXFER(xfer);
1516 ex->ex_isdone = true;
1517 xfer->ux_state = XFER_BUSY;
1518#endif
1519 }
1520 return xfer;
1521}
1522
1523Static void
1524ehci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
1525{
1526 struct ehci_softc *sc = EHCI_BUS2SC(bus);
1527 struct ehci_xfer *ex __diagused = EHCI_XFER2EXFER(xfer);
1528
1529 KASSERTMSG(xfer->ux_state == XFER_BUSY, "xfer %p state %d\n", xfer,
1530 xfer->ux_state);
1531 KASSERT(ex->ex_isdone);
1532
1533#ifdef DIAGNOSTIC
1534 xfer->ux_state = XFER_FREE;
1535#endif
1536
1537 pool_cache_put(sc->sc_xferpool, xfer);
1538}
1539
1540Static void
1541ehci_get_lock(struct usbd_bus *bus, kmutex_t **lock)
1542{
1543 struct ehci_softc *sc = EHCI_BUS2SC(bus);
1544
1545 *lock = &sc->sc_lock;
1546}
1547
1548Static void
1549ehci_device_clear_toggle(struct usbd_pipe *pipe)
1550{
1551 struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(pipe);
1552
1553 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1554
1555 DPRINTF("epipe=%p status=0x%08x", epipe,
1556 epipe->sqh->qh.qh_qtd.qtd_status, 0, 0);
1557#ifdef EHCI_DEBUG
1558 if (ehcidebug)
1559 usbd_dump_pipe(pipe);
1560#endif
1561 epipe->nexttoggle = 0;
1562}
1563
1564Static void
1565ehci_noop(struct usbd_pipe *pipe)
1566{
1567}
1568
1569#ifdef EHCI_DEBUG
1570/*
1571 * Unused function - this is meant to be called from a kernel
1572 * debugger.
1573 */
1574void
1575ehci_dump(void)
1576{
1577 ehci_softc_t *sc = theehci;
1578 int i;
1579 printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1580 EOREAD4(sc, EHCI_USBCMD),
1581 EOREAD4(sc, EHCI_USBSTS),
1582 EOREAD4(sc, EHCI_USBINTR));
1583 printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1584 EOREAD4(sc, EHCI_FRINDEX),
1585 EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1586 EOREAD4(sc, EHCI_PERIODICLISTBASE),
1587 EOREAD4(sc, EHCI_ASYNCLISTADDR));
1588 for (i = 1; i <= sc->sc_noport; i++)
1589 printf("port %d status=0x%08x\n", i,
1590 EOREAD4(sc, EHCI_PORTSC(i)));
1591}
1592
1593Static void
1594ehci_dump_regs(ehci_softc_t *sc)
1595{
1596 int i;
1597
1598 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1599
1600 DPRINTF("cmd = 0x%08x sts = 0x%08x ien = 0x%08x",
1601 EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS),
1602 EOREAD4(sc, EHCI_USBINTR), 0);
1603 DPRINTF("frindex = 0x%08x ctrdsegm = 0x%08x periodic = 0x%08x "
1604 "async = 0x%08x",
1605 EOREAD4(sc, EHCI_FRINDEX), EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1606 EOREAD4(sc, EHCI_PERIODICLISTBASE),
1607 EOREAD4(sc, EHCI_ASYNCLISTADDR));
1608 for (i = 1; i <= sc->sc_noport; i += 2) {
1609 if (i == sc->sc_noport) {
1610 DPRINTF("port %d status = 0x%08x", i,
1611 EOREAD4(sc, EHCI_PORTSC(i)), 0, 0);
1612 } else {
1613 DPRINTF(
1614 "port %d status = 0x%08x port %d status = 0x%08x",
1615 i, EOREAD4(sc, EHCI_PORTSC(i)),
1616 i+1, EOREAD4(sc, EHCI_PORTSC(i+1)));
1617 }
1618 }
1619}
1620
1621#define ehci_dump_link(link, type) do { \
1622 DPRINTF(" link 0x%08x (T = %d):", \
1623 link, \
1624 link & EHCI_LINK_TERMINATE ? 1 : 0, 0, 0); \
1625 if (type) { \
1626 DPRINTF( \
1627 " ITD = %d QH = %d SITD = %d FSTN = %d",\
1628 EHCI_LINK_TYPE(link) == EHCI_LINK_ITD ? 1 : 0, \
1629 EHCI_LINK_TYPE(link) == EHCI_LINK_QH ? 1 : 0, \
1630 EHCI_LINK_TYPE(link) == EHCI_LINK_SITD ? 1 : 0, \
1631 EHCI_LINK_TYPE(link) == EHCI_LINK_FSTN ? 1 : 0); \
1632 } \
1633} while(0)
1634
1635Static void
1636ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1637{
1638 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1639 int i;
1640 uint32_t stop = 0;
1641
1642 for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1643 ehci_dump_sqtd(sqtd);
1644 usb_syncmem(&sqtd->dma,
1645 sqtd->offs + offsetof(ehci_qtd_t, qtd_next),
1646 sizeof(sqtd->qtd),
1647 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1648 stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1649 usb_syncmem(&sqtd->dma,
1650 sqtd->offs + offsetof(ehci_qtd_t, qtd_next),
1651 sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1652 }
1653 if (!stop)
1654 DPRINTF("dump aborted, too many TDs", 0, 0, 0, 0);
1655}
1656
1657Static void
1658ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1659{
1660 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1661
1662 usb_syncmem(&sqtd->dma, sqtd->offs,
1663 sizeof(sqtd->qtd), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1664
1665 DPRINTFN(10, "QTD(%p) at 0x%08x:", sqtd, sqtd->physaddr, 0, 0);
1666 ehci_dump_qtd(&sqtd->qtd);
1667
1668 usb_syncmem(&sqtd->dma, sqtd->offs,
1669 sizeof(sqtd->qtd), BUS_DMASYNC_PREREAD);
1670}
1671
1672Static void
1673ehci_dump_qtd(ehci_qtd_t *qtd)
1674{
1675 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1676 uint32_t s = le32toh(qtd->qtd_status);
1677
1678 DPRINTFN(10,
1679 " next = 0x%08x altnext = 0x%08x status = 0x%08x",
1680 qtd->qtd_next, qtd->qtd_altnext, s, 0);
1681 DPRINTFN(10,
1682 " toggle = %d ioc = %d bytes = %#x "
1683 "c_page = %#x", EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_IOC(s),
1684 EHCI_QTD_GET_BYTES(s), EHCI_QTD_GET_C_PAGE(s));
1685 DPRINTFN(10,
1686 " cerr = %d pid = %d stat = %x",
1687 EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s), EHCI_QTD_GET_STATUS(s),
1688 0);
1689 DPRINTFN(10,
1690 "active =%d halted=%d buferr=%d babble=%d",
1691 s & EHCI_QTD_ACTIVE ? 1 : 0,
1692 s & EHCI_QTD_HALTED ? 1 : 0,
1693 s & EHCI_QTD_BUFERR ? 1 : 0,
1694 s & EHCI_QTD_BABBLE ? 1 : 0);
1695 DPRINTFN(10,
1696 "xacterr=%d missed=%d split =%d ping =%d",
1697 s & EHCI_QTD_XACTERR ? 1 : 0,
1698 s & EHCI_QTD_MISSEDMICRO ? 1 : 0,
1699 s & EHCI_QTD_SPLITXSTATE ? 1 : 0,
1700 s & EHCI_QTD_PINGSTATE ? 1 : 0);
1701 DPRINTFN(10,
1702 "buffer[0] = %#x buffer[1] = %#x "
1703 "buffer[2] = %#x buffer[3] = %#x",
1704 le32toh(qtd->qtd_buffer[0]), le32toh(qtd->qtd_buffer[1]),
1705 le32toh(qtd->qtd_buffer[2]), le32toh(qtd->qtd_buffer[3]));
1706 DPRINTFN(10,
1707 "buffer[4] = %#x", le32toh(qtd->qtd_buffer[4]), 0, 0, 0);
1708}
1709
1710Static void
1711ehci_dump_sqh(ehci_soft_qh_t *sqh)
1712{
1713 ehci_qh_t *qh = &sqh->qh;
1714 ehci_link_t link;
1715 uint32_t endp, endphub;
1716 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1717
1718 usb_syncmem(&sqh->dma, sqh->offs,
1719 sizeof(sqh->qh), BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1720
1721 DPRINTFN(10, "QH(%p) at %#x:", sqh, sqh->physaddr, 0, 0);
1722 link = le32toh(qh->qh_link);
1723 ehci_dump_link(link, true);
1724
1725 endp = le32toh(qh->qh_endp);
1726 DPRINTFN(10, " endp = %#x", endp, 0, 0, 0);
1727 DPRINTFN(10, " addr = 0x%02x inact = %d endpt = %d eps = %d",
1728 EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1729 EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp));
1730 DPRINTFN(10, " dtc = %d hrecl = %d",
1731 EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp), 0, 0);
1732 DPRINTFN(10, " ctl = %d nrl = %d mpl = %#x(%d)",
1733 EHCI_QH_GET_CTL(endp),EHCI_QH_GET_NRL(endp),
1734 EHCI_QH_GET_MPL(endp), EHCI_QH_GET_MPL(endp));
1735
1736 endphub = le32toh(qh->qh_endphub);
1737 DPRINTFN(10, " endphub = %#x", endphub, 0, 0, 0);
1738 DPRINTFN(10, " smask = 0x%02x cmask = 0x%02x",
1739 EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub), 1, 0);
1740 DPRINTFN(10, " huba = 0x%02x port = %d mult = %d",
1741 EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1742 EHCI_QH_GET_MULT(endphub), 0);
1743
1744 link = le32toh(qh->qh_curqtd);
1745 ehci_dump_link(link, false);
1746 DPRINTFN(10, "Overlay qTD:", 0, 0, 0, 0);
1747 ehci_dump_qtd(&qh->qh_qtd);
1748
1749 usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1750 BUS_DMASYNC_PREREAD);
1751}
1752
1753Static void
1754ehci_dump_itds(ehci_soft_itd_t *itd)
1755{
1756 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1757 int i;
1758 uint32_t stop = 0;
1759
1760 for (i = 0; itd && i < 20 && !stop; itd = itd->xfer_next, i++) {
1761 ehci_dump_itd(itd);
1762 usb_syncmem(&itd->dma,
1763 itd->offs + offsetof(ehci_itd_t, itd_next),
1764 sizeof(itd->itd),
1765 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1766 stop = itd->itd.itd_next & htole32(EHCI_LINK_TERMINATE);
1767 usb_syncmem(&itd->dma,
1768 itd->offs + offsetof(ehci_itd_t, itd_next),
1769 sizeof(itd->itd), BUS_DMASYNC_PREREAD);
1770 }
1771 if (!stop)
1772 DPRINTF("dump aborted, too many TDs", 0, 0, 0, 0);
1773}
1774
1775Static void
1776ehci_dump_itd(struct ehci_soft_itd *itd)
1777{
1778 ehci_isoc_trans_t t;
1779 ehci_isoc_bufr_ptr_t b, b2, b3;
1780 int i;
1781
1782 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1783
1784 DPRINTF("ITD: next phys = %#x", itd->itd.itd_next, 0, 0, 0);
1785
1786 for (i = 0; i < EHCI_ITD_NUFRAMES; i++) {
1787 t = le32toh(itd->itd.itd_ctl[i]);
1788 DPRINTF("ITDctl %d: stat = %x len = %x",
1789 i, EHCI_ITD_GET_STATUS(t), EHCI_ITD_GET_LEN(t), 0);
1790 DPRINTF(" ioc = %x pg = %x offs = %x",
1791 EHCI_ITD_GET_IOC(t), EHCI_ITD_GET_PG(t),
1792 EHCI_ITD_GET_OFFS(t), 0);
1793 }
1794 DPRINTF("ITDbufr: ", 0, 0, 0, 0);
1795 for (i = 0; i < EHCI_ITD_NBUFFERS; i++)
1796 DPRINTF(" %x",
1797 EHCI_ITD_GET_BPTR(le32toh(itd->itd.itd_bufr[i])), 0, 0, 0);
1798
1799 b = le32toh(itd->itd.itd_bufr[0]);
1800 b2 = le32toh(itd->itd.itd_bufr[1]);
1801 b3 = le32toh(itd->itd.itd_bufr[2]);
1802 DPRINTF(" ep = %x daddr = %x dir = %d",
1803 EHCI_ITD_GET_EP(b), EHCI_ITD_GET_DADDR(b), EHCI_ITD_GET_DIR(b2), 0);
1804 DPRINTF(" maxpkt = %x multi = %x",
1805 EHCI_ITD_GET_MAXPKT(b2), EHCI_ITD_GET_MULTI(b3), 0, 0);
1806}
1807
1808Static void
1809ehci_dump_sitd(struct ehci_soft_itd *itd)
1810{
1811 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1812
1813 DPRINTF("SITD %p next = %p prev = %p",
1814 itd, itd->frame_list.next, itd->frame_list.prev, 0);
1815 DPRINTF(" xfernext=%p physaddr=%X slot=%d",
1816 itd->xfer_next, itd->physaddr, itd->slot, 0);
1817}
1818
1819Static void
1820ehci_dump_exfer(struct ehci_xfer *ex)
1821{
1822 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1823
1824 DPRINTF("ex = %p type %d isdone", ex, ex->ex_type,
1825 ex->ex_isdone, 0);
1826
1827 switch (ex->ex_type) {
1828 case EX_CTRL:
1829 DPRINTF(" setup = %p data = %p status = %p",
1830 ex->ex_setup, ex->ex_data, ex->ex_status, 0);
1831 break;
1832 case EX_BULK:
1833 case EX_INTR:
1834 DPRINTF(" qtdstart = %p qtdend = %p",
1835 ex->ex_sqtdstart, ex->ex_sqtdend, 0, 0);
1836 break;
1837 case EX_ISOC:
1838 DPRINTF(" itdstart = %p itdend = %p",
1839 ex->ex_itdstart, ex->ex_itdend, 0, 0);
1840 break;
1841 case EX_FS_ISOC:
1842 DPRINTF(" sitdstart = %p sitdend = %p",
1843 ex->ex_sitdstart, ex->ex_sitdend, 0, 0);
1844 break;
1845 default:
1846 DPRINTF(" unknown type", 0, 0, 0, 0);
1847 }
1848}
1849#endif
1850
1851Static usbd_status
1852ehci_open(struct usbd_pipe *pipe)
1853{
1854 struct usbd_device *dev = pipe->up_dev;
1855 ehci_softc_t *sc = EHCI_PIPE2SC(pipe);
1856 usb_endpoint_descriptor_t *ed = pipe->up_endpoint->ue_edesc;
1857 uint8_t rhaddr = dev->ud_bus->ub_rhaddr;
1858 uint8_t addr = dev->ud_addr;
1859 uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
1860 struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(pipe);
1861 ehci_soft_qh_t *sqh;
1862 usbd_status err;
1863 int ival, speed, naks;
1864 int hshubaddr, hshubport;
1865
1866 EHCIHIST_FUNC(); EHCIHIST_CALLED();
1867
1868 DPRINTF("pipe=%p, addr=%d, endpt=%d (%d)", pipe, addr,
1869 ed->bEndpointAddress, rhaddr);
1870
1871 if (dev->ud_myhsport) {
1872 /*
1873 * When directly attached FS/LS device while doing embedded
1874 * transaction translations and we are the hub, set the hub
1875 * address to 0 (us).
1876 */
1877 if (!(sc->sc_flags & EHCIF_ETTF)
1878 || (dev->ud_myhsport->up_parent->ud_addr != rhaddr)) {
1879 hshubaddr = dev->ud_myhsport->up_parent->ud_addr;
1880 } else {
1881 hshubaddr = 0;
1882 }
1883 hshubport = dev->ud_myhsport->up_portno;
1884 } else {
1885 hshubaddr = 0;
1886 hshubport = 0;
1887 }
1888
1889 if (sc->sc_dying)
1890 return USBD_IOERROR;
1891
1892 /* toggle state needed for bulk endpoints */
1893 epipe->nexttoggle = pipe->up_endpoint->ue_toggle;
1894
1895 if (addr == rhaddr) {
1896 switch (ed->bEndpointAddress) {
1897 case USB_CONTROL_ENDPOINT:
1898 pipe->up_methods = &roothub_ctrl_methods;
1899 break;
1900 case UE_DIR_IN | USBROOTHUB_INTR_ENDPT:
1901 pipe->up_methods = &ehci_root_intr_methods;
1902 break;
1903 default:
1904 DPRINTF("bad bEndpointAddress 0x%02x",
1905 ed->bEndpointAddress, 0, 0, 0);
1906 return USBD_INVAL;
1907 }
1908 return USBD_NORMAL_COMPLETION;
1909 }
1910
1911 /* XXX All this stuff is only valid for async. */
1912 switch (dev->ud_speed) {
1913 case USB_SPEED_LOW: speed = EHCI_QH_SPEED_LOW; break;
1914 case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1915 case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1916 default: panic("ehci_open: bad device speed %d", dev->ud_speed);
1917 }
1918 if (speed == EHCI_QH_SPEED_LOW && xfertype == UE_ISOCHRONOUS) {
1919 DPRINTF("hshubaddr=%d hshubport=%d", hshubaddr, hshubport, 0,
1920 0);
1921 return USBD_INVAL;
1922 }
1923
1924 /*
1925 * For interrupt transfer, nak throttling must be disabled, but for
1926 * the other transfer type, nak throttling should be enabled from the
1927 * viewpoint that avoids the memory thrashing.
1928 */
1929 naks = (xfertype == UE_INTERRUPT) ? 0
1930 : ((speed == EHCI_QH_SPEED_HIGH) ? 4 : 0);
1931
1932 /* Allocate sqh for everything, save isoc xfers */
1933 if (xfertype != UE_ISOCHRONOUS) {
1934 sqh = ehci_alloc_sqh(sc);
1935 if (sqh == NULL)
1936 return USBD_NOMEM;
1937 /* qh_link filled when the QH is added */
1938 sqh->qh.qh_endp = htole32(
1939 EHCI_QH_SET_ADDR(addr) |
1940 EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1941 EHCI_QH_SET_EPS(speed) |
1942 EHCI_QH_DTC |
1943 EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1944 (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1945 EHCI_QH_CTL : 0) |
1946 EHCI_QH_SET_NRL(naks)
1947 );
1948 sqh->qh.qh_endphub = htole32(
1949 EHCI_QH_SET_MULT(1) |
1950 EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x02 : 0)
1951 );
1952 if (speed != EHCI_QH_SPEED_HIGH)
1953 sqh->qh.qh_endphub |= htole32(
1954 EHCI_QH_SET_PORT(hshubport) |
1955 EHCI_QH_SET_HUBA(hshubaddr) |
1956 (xfertype == UE_INTERRUPT ?
1957 EHCI_QH_SET_CMASK(0x08) : 0)
1958 );
1959 sqh->qh.qh_curqtd = EHCI_NULL;
1960 /* Fill the overlay qTD */
1961 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
1962 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1963 sqh->qh.qh_qtd.qtd_status = htole32(0);
1964
1965 usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
1966 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1967 epipe->sqh = sqh;
1968 } else {
1969 sqh = NULL;
1970 } /*xfertype == UE_ISOC*/
1971
1972 switch (xfertype) {
1973 case UE_CONTROL:
1974 err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1975 0, &epipe->ctrl.reqdma);
1976#ifdef EHCI_DEBUG
1977 if (err)
1978 printf("ehci_open: usb_allocmem()=%d\n", err);
1979#endif
1980 if (err)
1981 goto bad;
1982 pipe->up_methods = &ehci_device_ctrl_methods;
1983 mutex_enter(&sc->sc_lock);
1984 ehci_add_qh(sc, sqh, sc->sc_async_head);
1985 mutex_exit(&sc->sc_lock);
1986 break;
1987 case UE_BULK:
1988 pipe->up_methods = &ehci_device_bulk_methods;
1989 mutex_enter(&sc->sc_lock);
1990 ehci_add_qh(sc, sqh, sc->sc_async_head);
1991 mutex_exit(&sc->sc_lock);
1992 break;
1993 case UE_INTERRUPT:
1994 pipe->up_methods = &ehci_device_intr_methods;
1995 ival = pipe->up_interval;
1996 if (ival == USBD_DEFAULT_INTERVAL) {
1997 if (speed == EHCI_QH_SPEED_HIGH) {
1998 if (ed->bInterval > 16) {
1999 /*
2000 * illegal with high-speed, but there
2001 * were documentation bugs in the spec,
2002 * so be generous
2003 */
2004 ival = 256;
2005 } else
2006 ival = (1 << (ed->bInterval - 1)) / 8;
2007 } else
2008 ival = ed->bInterval;
2009 }
2010 err = ehci_device_setintr(sc, sqh, ival);
2011 if (err)
2012 goto bad;
2013 break;
2014 case UE_ISOCHRONOUS:
2015 pipe->up_serialise = false;
2016 if (speed == EHCI_QH_SPEED_HIGH)
2017 pipe->up_methods = &ehci_device_isoc_methods;
2018 else
2019 pipe->up_methods = &ehci_device_fs_isoc_methods;
2020 if (ed->bInterval == 0 || ed->bInterval > 16) {
2021 printf("ehci: opening pipe with invalid bInterval\n");
2022 err = USBD_INVAL;
2023 goto bad;
2024 }
2025 if (UGETW(ed->wMaxPacketSize) == 0) {
2026 printf("ehci: zero length endpoint open request\n");
2027 err = USBD_INVAL;
2028 goto bad;
2029 }
2030 epipe->isoc.next_frame = 0;
2031 epipe->isoc.cur_xfers = 0;
2032 break;
2033 default:
2034 DPRINTF("bad xfer type %d", xfertype, 0, 0, 0);
2035 err = USBD_INVAL;
2036 goto bad;
2037 }
2038 return USBD_NORMAL_COMPLETION;
2039
2040 bad:
2041 if (sqh != NULL) {
2042 mutex_enter(&sc->sc_lock);
2043 ehci_free_sqh(sc, sqh);
2044 mutex_exit(&sc->sc_lock);
2045 }
2046 return err;
2047}
2048
2049/*
2050 * Add an ED to the schedule. Called with USB lock held.
2051 */
2052Static void
2053ehci_add_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
2054{
2055
2056 KASSERT(mutex_owned(&sc->sc_lock));
2057
2058 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2059
2060 usb_syncmem(&head->dma, head->offs + offsetof(ehci_qh_t, qh_link),
2061 sizeof(head->qh.qh_link), BUS_DMASYNC_POSTWRITE);
2062
2063 sqh->next = head->next;
2064 sqh->qh.qh_link = head->qh.qh_link;
2065
2066 usb_syncmem(&sqh->dma, sqh->offs + offsetof(ehci_qh_t, qh_link),
2067 sizeof(sqh->qh.qh_link), BUS_DMASYNC_PREWRITE);
2068
2069 head->next = sqh;
2070 head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
2071
2072 usb_syncmem(&head->dma, head->offs + offsetof(ehci_qh_t, qh_link),
2073 sizeof(head->qh.qh_link), BUS_DMASYNC_PREWRITE);
2074
2075#ifdef EHCI_DEBUG
2076 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
2077 ehci_dump_sqh(sqh);
2078 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
2079#endif
2080}
2081
2082/*
2083 * Remove an ED from the schedule. Called with USB lock held.
2084 */
2085Static void
2086ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
2087{
2088 ehci_soft_qh_t *p;
2089
2090 KASSERT(mutex_owned(&sc->sc_lock));
2091
2092 /* XXX */
2093 for (p = head; p != NULL && p->next != sqh; p = p->next)
2094 ;
2095 if (p == NULL)
2096 panic("ehci_rem_qh: ED not found");
2097 usb_syncmem(&sqh->dma, sqh->offs + offsetof(ehci_qh_t, qh_link),
2098 sizeof(sqh->qh.qh_link), BUS_DMASYNC_POSTWRITE);
2099 p->next = sqh->next;
2100 p->qh.qh_link = sqh->qh.qh_link;
2101 usb_syncmem(&p->dma, p->offs + offsetof(ehci_qh_t, qh_link),
2102 sizeof(p->qh.qh_link), BUS_DMASYNC_PREWRITE);
2103
2104 ehci_sync_hc(sc);
2105}
2106
2107Static void
2108ehci_set_qh_qtd(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
2109{
2110 int i;
2111 uint32_t status;
2112
2113 /* Save toggle bit and ping status. */
2114 usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
2115 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2116 status = sqh->qh.qh_qtd.qtd_status &
2117 htole32(EHCI_QTD_TOGGLE_MASK |
2118 EHCI_QTD_SET_STATUS(EHCI_QTD_PINGSTATE));
2119 /* Set HALTED to make hw leave it alone. */
2120 sqh->qh.qh_qtd.qtd_status =
2121 htole32(EHCI_QTD_SET_STATUS(EHCI_QTD_HALTED));
2122 usb_syncmem(&sqh->dma,
2123 sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
2124 sizeof(sqh->qh.qh_qtd.qtd_status),
2125 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2126 sqh->qh.qh_curqtd = 0;
2127 sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
2128 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2129 for (i = 0; i < EHCI_QTD_NBUFFERS; i++)
2130 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2131 sqh->sqtd = sqtd;
2132 usb_syncmem(&sqh->dma, sqh->offs, sizeof(sqh->qh),
2133 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2134 /* Set !HALTED && !ACTIVE to start execution, preserve some fields */
2135 sqh->qh.qh_qtd.qtd_status = status;
2136 usb_syncmem(&sqh->dma,
2137 sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
2138 sizeof(sqh->qh.qh_qtd.qtd_status),
2139 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2140}
2141
2142/*
2143 * Ensure that the HC has released all references to the QH. We do this
2144 * by asking for a Async Advance Doorbell interrupt and then we wait for
2145 * the interrupt.
2146 * To make this easier we first obtain exclusive use of the doorbell.
2147 */
2148Static void
2149ehci_sync_hc(ehci_softc_t *sc)
2150{
2151 int error __diagused;
2152
2153 KASSERT(mutex_owned(&sc->sc_lock));
2154
2155 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2156
2157 if (sc->sc_dying) {
2158 DPRINTF("dying", 0, 0, 0, 0);
2159 return;
2160 }
2161 /* ask for doorbell */
2162 EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
2163 DPRINTF("cmd = 0x%08x sts = 0x%08x",
2164 EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS), 0, 0);
2165
2166 error = cv_timedwait(&sc->sc_doorbell, &sc->sc_lock, hz); /* bell wait */
2167
2168 DPRINTF("cmd = 0x%08x sts = 0x%08x ... done",
2169 EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS), 0, 0);
2170#ifdef DIAGNOSTIC
2171 if (error == EWOULDBLOCK) {
2172 printf("ehci_sync_hc: timed out\n");
2173 } else if (error) {
2174 printf("ehci_sync_hc: cv_timedwait: error %d\n", error);
2175 }
2176#endif
2177}
2178
2179Static void
2180ehci_remove_itd_chain(ehci_softc_t *sc, struct ehci_soft_itd *itd)
2181{
2182
2183 KASSERT(mutex_owned(&sc->sc_lock));
2184
2185 for (; itd != NULL; itd = itd->xfer_next) {
2186 struct ehci_soft_itd *prev = itd->frame_list.prev;
2187
2188 /* Unlink itd from hardware chain, or frame array */
2189 if (prev == NULL) { /* We're at the table head */
2190 sc->sc_softitds[itd->slot] = itd->frame_list.next;
2191 sc->sc_flist[itd->slot] = itd->itd.itd_next;
2192 usb_syncmem(&sc->sc_fldma,
2193 sizeof(ehci_link_t) * itd->slot,
2194 sizeof(ehci_link_t),
2195 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2196
2197 if (itd->frame_list.next != NULL)
2198 itd->frame_list.next->frame_list.prev = NULL;
2199 } else {
2200 /* XXX this part is untested... */
2201 prev->itd.itd_next = itd->itd.itd_next;
2202 usb_syncmem(&itd->dma,
2203 itd->offs + offsetof(ehci_itd_t, itd_next),
2204 sizeof(itd->itd.itd_next), BUS_DMASYNC_PREWRITE);
2205
2206 prev->frame_list.next = itd->frame_list.next;
2207 if (itd->frame_list.next != NULL)
2208 itd->frame_list.next->frame_list.prev = prev;
2209 }
2210 }
2211}
2212
2213Static void
2214ehci_free_itd_chain(ehci_softc_t *sc, struct ehci_soft_itd *itd)
2215{
2216 struct ehci_soft_itd *next;
2217
2218 mutex_enter(&sc->sc_lock);
2219 next = NULL;
2220 for (; itd != NULL; itd = next) {
2221 next = itd->xfer_next;
2222 ehci_free_itd_locked(sc, itd);
2223 }
2224 mutex_exit(&sc->sc_lock);
2225}
2226
2227Static void
2228ehci_remove_sitd_chain(ehci_softc_t *sc, struct ehci_soft_sitd *sitd)
2229{
2230
2231 KASSERT(mutex_owned(&sc->sc_lock));
2232
2233 for (; sitd != NULL; sitd = sitd->xfer_next) {
2234 struct ehci_soft_sitd *prev = sitd->frame_list.prev;
2235
2236 /* Unlink sitd from hardware chain, or frame array */
2237 if (prev == NULL) { /* We're at the table head */
2238 sc->sc_softsitds[sitd->slot] = sitd->frame_list.next;
2239 sc->sc_flist[sitd->slot] = sitd->sitd.sitd_next;
2240 usb_syncmem(&sc->sc_fldma,
2241 sizeof(ehci_link_t) * sitd->slot,
2242 sizeof(ehci_link_t),
2243 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2244
2245 if (sitd->frame_list.next != NULL)
2246 sitd->frame_list.next->frame_list.prev = NULL;
2247 } else {
2248 /* XXX this part is untested... */
2249 prev->sitd.sitd_next = sitd->sitd.sitd_next;
2250 usb_syncmem(&sitd->dma,
2251 sitd->offs + offsetof(ehci_sitd_t, sitd_next),
2252 sizeof(sitd->sitd.sitd_next), BUS_DMASYNC_PREWRITE);
2253
2254 prev->frame_list.next = sitd->frame_list.next;
2255 if (sitd->frame_list.next != NULL)
2256 sitd->frame_list.next->frame_list.prev = prev;
2257 }
2258 }
2259}
2260
2261Static void
2262ehci_free_sitd_chain(ehci_softc_t *sc, struct ehci_soft_sitd *sitd)
2263{
2264
2265 mutex_enter(&sc->sc_lock);
2266 struct ehci_soft_sitd *next = NULL;
2267 for (; sitd != NULL; sitd = next) {
2268 next = sitd->xfer_next;
2269 ehci_free_sitd_locked(sc, sitd);
2270 }
2271 mutex_exit(&sc->sc_lock);
2272}
2273
2274/***********/
2275
2276Static int
2277ehci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req,
2278 void *buf, int buflen)
2279{
2280 ehci_softc_t *sc = EHCI_BUS2SC(bus);
2281 usb_hub_descriptor_t hubd;
2282 usb_port_status_t ps;
2283 uint16_t len, value, index;
2284 int l, totlen = 0;
2285 int port, i;
2286 uint32_t v;
2287
2288 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2289
2290 if (sc->sc_dying)
2291 return -1;
2292
2293 DPRINTF("type=0x%02x request=%02x", req->bmRequestType, req->bRequest,
2294 0, 0);
2295
2296 len = UGETW(req->wLength);
2297 value = UGETW(req->wValue);
2298 index = UGETW(req->wIndex);
2299
2300#define C(x,y) ((x) | ((y) << 8))
2301 switch (C(req->bRequest, req->bmRequestType)) {
2302 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2303 if (len == 0)
2304 break;
2305 switch (value) {
2306 case C(0, UDESC_DEVICE): {
2307 usb_device_descriptor_t devd;
2308 totlen = min(buflen, sizeof(devd));
2309 memcpy(&devd, buf, totlen);
2310 USETW(devd.idVendor, sc->sc_id_vendor);
2311 memcpy(buf, &devd, totlen);
2312 break;
2313
2314 }
2315#define sd ((usb_string_descriptor_t *)buf)
2316 case C(1, UDESC_STRING):
2317 /* Vendor */
2318 totlen = usb_makestrdesc(sd, len, sc->sc_vendor);
2319 break;
2320 case C(2, UDESC_STRING):
2321 /* Product */
2322 totlen = usb_makestrdesc(sd, len, "EHCI root hub");
2323 break;
2324#undef sd
2325 default:
2326 /* default from usbroothub */
2327 return buflen;
2328 }
2329 break;
2330
2331 /* Hub requests */
2332 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2333 break;
2334 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2335 DPRINTF("UR_CLEAR_PORT_FEATURE port=%d feature=%d", index,
2336 value, 0, 0);
2337 if (index < 1 || index > sc->sc_noport) {
2338 return -1;
2339 }
2340 port = EHCI_PORTSC(index);
2341 v = EOREAD4(sc, port);
2342 DPRINTF("portsc=0x%08x", v, 0, 0, 0);
2343 v &= ~EHCI_PS_CLEAR;
2344 switch (value) {
2345 case UHF_PORT_ENABLE:
2346 EOWRITE4(sc, port, v &~ EHCI_PS_PE);
2347 break;
2348 case UHF_PORT_SUSPEND:
2349 if (!(v & EHCI_PS_SUSP)) /* not suspended */
2350 break;
2351 v &= ~EHCI_PS_SUSP;
2352 EOWRITE4(sc, port, v | EHCI_PS_FPR);
2353 /* see USB2 spec ch. 7.1.7.7 */
2354 usb_delay_ms(&sc->sc_bus, 20);
2355 EOWRITE4(sc, port, v);
2356 usb_delay_ms(&sc->sc_bus, 2);
2357#ifdef DEBUG
2358 v = EOREAD4(sc, port);
2359 if (v & (EHCI_PS_FPR | EHCI_PS_SUSP))
2360 printf("ehci: resume failed: %x\n", v);
2361#endif
2362 break;
2363 case UHF_PORT_POWER:
2364 if (sc->sc_hasppc)
2365 EOWRITE4(sc, port, v &~ EHCI_PS_PP);
2366 break;
2367 case UHF_PORT_TEST:
2368 DPRINTF("clear port test %d", index, 0, 0, 0);
2369 break;
2370 case UHF_PORT_INDICATOR:
2371 DPRINTF("clear port ind %d", index, 0, 0, 0);
2372 EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
2373 break;
2374 case UHF_C_PORT_CONNECTION:
2375 EOWRITE4(sc, port, v | EHCI_PS_CSC);
2376 break;
2377 case UHF_C_PORT_ENABLE:
2378 EOWRITE4(sc, port, v | EHCI_PS_PEC);
2379 break;
2380 case UHF_C_PORT_SUSPEND:
2381 /* how? */
2382 break;
2383 case UHF_C_PORT_OVER_CURRENT:
2384 EOWRITE4(sc, port, v | EHCI_PS_OCC);
2385 break;
2386 case UHF_C_PORT_RESET:
2387 sc->sc_isreset[index] = 0;
2388 break;
2389 default:
2390 return -1;
2391 }
2392#if 0
2393 switch(value) {
2394 case UHF_C_PORT_CONNECTION:
2395 case UHF_C_PORT_ENABLE:
2396 case UHF_C_PORT_SUSPEND:
2397 case UHF_C_PORT_OVER_CURRENT:
2398 case UHF_C_PORT_RESET:
2399 default:
2400 break;
2401 }
2402#endif
2403 break;
2404 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2405 if (len == 0)
2406 break;
2407 if ((value & 0xff) != 0) {
2408 return -1;
2409 }
2410 totlen = min(buflen, sizeof(hubd));
2411 memcpy(&hubd, buf, totlen);
2412 hubd.bNbrPorts = sc->sc_noport;
2413 v = EOREAD4(sc, EHCI_HCSPARAMS);
2414 USETW(hubd.wHubCharacteristics,
2415 EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
2416 EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
2417 ? UHD_PORT_IND : 0);
2418 hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
2419 for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
2420 hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
2421 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
2422 totlen = min(totlen, hubd.bDescLength);
2423 memcpy(buf, &hubd, totlen);
2424 break;
2425 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2426 if (len != 4) {
2427 return -1;
2428 }
2429 memset(buf, 0, len); /* ? XXX */
2430 totlen = len;
2431 break;
2432 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2433 DPRINTF("get port status i=%d", index, 0, 0, 0);
2434 if (index < 1 || index > sc->sc_noport) {
2435 return -1;
2436 }
2437 if (len != 4) {
2438 return -1;
2439 }
2440 v = EOREAD4(sc, EHCI_PORTSC(index));
2441 DPRINTF("port status=0x%04x", v, 0, 0, 0);
2442
2443 i = UPS_HIGH_SPEED;
2444 if (sc->sc_flags & EHCIF_ETTF) {
2445 /*
2446 * If we are doing embedded transaction translation,
2447 * then directly attached LS/FS devices are reset by
2448 * the EHCI controller itself. PSPD is encoded
2449 * the same way as in USBSTATUS.
2450 */
2451 i = __SHIFTOUT(v, EHCI_PS_PSPD) * UPS_LOW_SPEED;
2452 }
2453 if (v & EHCI_PS_CS) i |= UPS_CURRENT_CONNECT_STATUS;
2454 if (v & EHCI_PS_PE) i |= UPS_PORT_ENABLED;
2455 if (v & EHCI_PS_SUSP) i |= UPS_SUSPEND;
2456 if (v & EHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR;
2457 if (v & EHCI_PS_PR) i |= UPS_RESET;
2458 if (v & EHCI_PS_PP) i |= UPS_PORT_POWER;
2459 if (sc->sc_vendor_port_status)
2460 i = sc->sc_vendor_port_status(sc, v, i);
2461 USETW(ps.wPortStatus, i);
2462 i = 0;
2463 if (v & EHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS;
2464 if (v & EHCI_PS_PEC) i |= UPS_C_PORT_ENABLED;
2465 if (v & EHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR;
2466 if (sc->sc_isreset[index]) i |= UPS_C_PORT_RESET;
2467 USETW(ps.wPortChange, i);
2468 totlen = min(len, sizeof(ps));
2469 memcpy(buf, &ps, totlen);
2470 break;
2471 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2472 return -1;
2473 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2474 break;
2475 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2476 if (index < 1 || index > sc->sc_noport) {
2477 return -1;
2478 }
2479 port = EHCI_PORTSC(index);
2480 v = EOREAD4(sc, port);
2481 DPRINTF("portsc=0x%08x", v, 0, 0, 0);
2482 v &= ~EHCI_PS_CLEAR;
2483 switch(value) {
2484 case UHF_PORT_ENABLE:
2485 EOWRITE4(sc, port, v | EHCI_PS_PE);
2486 break;
2487 case UHF_PORT_SUSPEND:
2488 EOWRITE4(sc, port, v | EHCI_PS_SUSP);
2489 break;
2490 case UHF_PORT_RESET:
2491 DPRINTF("reset port %d", index, 0, 0, 0);
2492 if (EHCI_PS_IS_LOWSPEED(v)
2493 && sc->sc_ncomp > 0
2494 && !(sc->sc_flags & EHCIF_ETTF)) {
2495 /*
2496 * Low speed device on non-ETTF controller or
2497 * unaccompanied controller, give up ownership.
2498 */
2499 ehci_disown(sc, index, 1);
2500 break;
2501 }
2502 /* Start reset sequence. */
2503 v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
2504 EOWRITE4(sc, port, v | EHCI_PS_PR);
2505 /* Wait for reset to complete. */
2506 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
2507 if (sc->sc_dying) {
2508 return -1;
2509 }
2510 /*
2511 * An embedded transaction translator will automatically
2512 * terminate the reset sequence so there's no need to
2513 * it.
2514 */
2515 v = EOREAD4(sc, port);
2516 if (v & EHCI_PS_PR) {
2517 /* Terminate reset sequence. */
2518 EOWRITE4(sc, port, v & ~EHCI_PS_PR);
2519 /* Wait for HC to complete reset. */
2520 usb_delay_ms(&sc->sc_bus,
2521 EHCI_PORT_RESET_COMPLETE);
2522 if (sc->sc_dying) {
2523 return -1;
2524 }
2525 }
2526
2527 v = EOREAD4(sc, port);
2528 DPRINTF("ehci after reset, status=0x%08x", v, 0, 0, 0);
2529 if (v & EHCI_PS_PR) {
2530 printf("%s: port reset timeout\n",
2531 device_xname(sc->sc_dev));
2532 return USBD_TIMEOUT;
2533 }
2534 if (!(v & EHCI_PS_PE)) {
2535 /* Not a high speed device, give up ownership.*/
2536 ehci_disown(sc, index, 0);
2537 break;
2538 }
2539 sc->sc_isreset[index] = 1;
2540 DPRINTF("ehci port %d reset, status = 0x%08x", index,
2541 v, 0, 0);
2542 break;
2543 case UHF_PORT_POWER:
2544 DPRINTF("set port power %d (has PPC = %d)", index,
2545 sc->sc_hasppc, 0, 0);
2546 if (sc->sc_hasppc)
2547 EOWRITE4(sc, port, v | EHCI_PS_PP);
2548 break;
2549 case UHF_PORT_TEST:
2550 DPRINTF("set port test %d", index, 0, 0, 0);
2551 break;
2552 case UHF_PORT_INDICATOR:
2553 DPRINTF("set port ind %d", index, 0, 0, 0);
2554 EOWRITE4(sc, port, v | EHCI_PS_PIC);
2555 break;
2556 default:
2557 return -1;
2558 }
2559 break;
2560 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2561 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2562 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2563 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2564 break;
2565 default:
2566 /* default from usbroothub */
2567 DPRINTF("returning %d (usbroothub default)", buflen, 0, 0, 0);
2568
2569 return buflen;
2570 }
2571
2572 DPRINTF("returning %d", totlen, 0, 0, 0);
2573
2574 return totlen;
2575}
2576
2577Static void
2578ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2579{
2580 int port;
2581 uint32_t v;
2582
2583 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2584
2585 DPRINTF("index=%d lowspeed=%d", index, lowspeed, 0, 0);
2586#ifdef DIAGNOSTIC
2587 if (sc->sc_npcomp != 0) {
2588 int i = (index-1) / sc->sc_npcomp;
2589 if (i >= sc->sc_ncomp)
2590 printf("%s: strange port\n",
2591 device_xname(sc->sc_dev));
2592 else
2593 printf("%s: handing over %s speed device on "
2594 "port %d to %s\n",
2595 device_xname(sc->sc_dev),
2596 lowspeed ? "low" : "full",
2597 index, device_xname(sc->sc_comps[i]));
2598 } else {
2599 printf("%s: npcomp == 0\n", device_xname(sc->sc_dev));
2600 }
2601#endif
2602 port = EHCI_PORTSC(index);
2603 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2604 EOWRITE4(sc, port, v | EHCI_PS_PO);
2605}
2606
2607Static usbd_status
2608ehci_root_intr_transfer(struct usbd_xfer *xfer)
2609{
2610 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
2611 usbd_status err;
2612
2613 /* Insert last in queue. */
2614 mutex_enter(&sc->sc_lock);
2615 err = usb_insert_transfer(xfer);
2616 mutex_exit(&sc->sc_lock);
2617 if (err)
2618 return err;
2619
2620 /* Pipe isn't running, start first */
2621 return ehci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
2622}
2623
2624Static usbd_status
2625ehci_root_intr_start(struct usbd_xfer *xfer)
2626{
2627 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
2628
2629 if (sc->sc_dying)
2630 return USBD_IOERROR;
2631
2632 mutex_enter(&sc->sc_lock);
2633 sc->sc_intrxfer = xfer;
2634 mutex_exit(&sc->sc_lock);
2635
2636 return USBD_IN_PROGRESS;
2637}
2638
2639/* Abort a root interrupt request. */
2640Static void
2641ehci_root_intr_abort(struct usbd_xfer *xfer)
2642{
2643 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
2644
2645 KASSERT(mutex_owned(&sc->sc_lock));
2646 KASSERT(xfer->ux_pipe->up_intrxfer == xfer);
2647
2648 sc->sc_intrxfer = NULL;
2649
2650 xfer->ux_status = USBD_CANCELLED;
2651 usb_transfer_complete(xfer);
2652}
2653
2654/* Close the root pipe. */
2655Static void
2656ehci_root_intr_close(struct usbd_pipe *pipe)
2657{
2658 ehci_softc_t *sc = EHCI_PIPE2SC(pipe);
2659
2660 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2661
2662 KASSERT(mutex_owned(&sc->sc_lock));
2663
2664 sc->sc_intrxfer = NULL;
2665}
2666
2667Static void
2668ehci_root_intr_done(struct usbd_xfer *xfer)
2669{
2670}
2671
2672/************************/
2673
2674Static ehci_soft_qh_t *
2675ehci_alloc_sqh(ehci_softc_t *sc)
2676{
2677 ehci_soft_qh_t *sqh;
2678 usbd_status err;
2679 int i, offs;
2680 usb_dma_t dma;
2681
2682 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2683
2684 mutex_enter(&sc->sc_lock);
2685 if (sc->sc_freeqhs == NULL) {
2686 DPRINTF("allocating chunk", 0, 0, 0, 0);
2687 mutex_exit(&sc->sc_lock);
2688
2689 err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2690 EHCI_PAGE_SIZE, &dma);
2691#ifdef EHCI_DEBUG
2692 if (err)
2693 printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2694#endif
2695 if (err)
2696 return NULL;
2697
2698 mutex_enter(&sc->sc_lock);
2699 for (i = 0; i < EHCI_SQH_CHUNK; i++) {
2700 offs = i * EHCI_SQH_SIZE;
2701 sqh = KERNADDR(&dma, offs);
2702 sqh->physaddr = DMAADDR(&dma, offs);
2703 sqh->dma = dma;
2704 sqh->offs = offs;
2705 sqh->next = sc->sc_freeqhs;
2706 sc->sc_freeqhs = sqh;
2707 }
2708 }
2709 sqh = sc->sc_freeqhs;
2710 sc->sc_freeqhs = sqh->next;
2711 mutex_exit(&sc->sc_lock);
2712
2713 memset(&sqh->qh, 0, sizeof(ehci_qh_t));
2714 sqh->next = NULL;
2715 return sqh;
2716}
2717
2718Static void
2719ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2720{
2721 KASSERT(mutex_owned(&sc->sc_lock));
2722
2723 sqh->next = sc->sc_freeqhs;
2724 sc->sc_freeqhs = sqh;
2725}
2726
2727Static ehci_soft_qtd_t *
2728ehci_alloc_sqtd(ehci_softc_t *sc)
2729{
2730 ehci_soft_qtd_t *sqtd = NULL;
2731 usbd_status err;
2732 int i, offs;
2733 usb_dma_t dma;
2734
2735 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2736
2737 mutex_enter(&sc->sc_lock);
2738 if (sc->sc_freeqtds == NULL) {
2739 DPRINTF("allocating chunk", 0, 0, 0, 0);
2740 mutex_exit(&sc->sc_lock);
2741
2742 err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2743 EHCI_PAGE_SIZE, &dma);
2744#ifdef EHCI_DEBUG
2745 if (err)
2746 printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2747#endif
2748 if (err)
2749 goto done;
2750
2751 mutex_enter(&sc->sc_lock);
2752 for (i = 0; i < EHCI_SQTD_CHUNK; i++) {
2753 offs = i * EHCI_SQTD_SIZE;
2754 sqtd = KERNADDR(&dma, offs);
2755 sqtd->physaddr = DMAADDR(&dma, offs);
2756 sqtd->dma = dma;
2757 sqtd->offs = offs;
2758
2759 sqtd->nextqtd = sc->sc_freeqtds;
2760 sc->sc_freeqtds = sqtd;
2761 }
2762 }
2763
2764 sqtd = sc->sc_freeqtds;
2765 sc->sc_freeqtds = sqtd->nextqtd;
2766 mutex_exit(&sc->sc_lock);
2767
2768 memset(&sqtd->qtd, 0, sizeof(ehci_qtd_t));
2769 sqtd->nextqtd = NULL;
2770 sqtd->xfer = NULL;
2771
2772done:
2773 return sqtd;
2774}
2775
2776Static void
2777ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2778{
2779
2780 mutex_enter(&sc->sc_lock);
2781 sqtd->nextqtd = sc->sc_freeqtds;
2782 sc->sc_freeqtds = sqtd;
2783 mutex_exit(&sc->sc_lock);
2784}
2785
2786Static int
2787ehci_alloc_sqtd_chain(ehci_softc_t *sc, struct usbd_xfer *xfer,
2788 int alen, int rd, ehci_soft_qtd_t **sp)
2789{
2790 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
2791 uint16_t flags = xfer->ux_flags;
2792
2793 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2794
2795 ASSERT_SLEEPABLE();
2796 KASSERT(sp);
2797 KASSERT(alen != 0 || (!rd && (flags & USBD_FORCE_SHORT_XFER)));
2798
2799 size_t nsqtd = (!rd && (flags & USBD_FORCE_SHORT_XFER)) ? 1 : 0;
2800 nsqtd += ((alen + EHCI_PAGE_SIZE - 1) / EHCI_PAGE_SIZE);
2801 exfer->ex_sqtds = kmem_zalloc(sizeof(ehci_soft_qtd_t *) * nsqtd,
2802 KM_SLEEP);
2803 exfer->ex_nsqtd = nsqtd;
2804
2805 DPRINTF("xfer %p len %d nsqtd %d flags %x", xfer, alen, nsqtd, flags);
2806
2807 for (size_t j = 0; j < exfer->ex_nsqtd;) {
2808 ehci_soft_qtd_t *cur = ehci_alloc_sqtd(sc);
2809 if (cur == NULL)
2810 goto nomem;
2811 exfer->ex_sqtds[j++] = cur;
2812
2813 cur->xfer = xfer;
2814 cur->len = 0;
2815
2816 }
2817
2818 *sp = exfer->ex_sqtds[0];
2819 DPRINTF("return sqtd=%p", *sp, 0, 0, 0);
2820
2821 return 0;
2822
2823 nomem:
2824 ehci_free_sqtds(sc, exfer);
2825 kmem_free(exfer->ex_sqtds, sizeof(ehci_soft_qtd_t *) * nsqtd);
2826 DPRINTF("no memory", 0, 0, 0, 0);
2827 return ENOMEM;
2828}
2829
2830Static void
2831ehci_free_sqtds(ehci_softc_t *sc, struct ehci_xfer *exfer)
2832{
2833 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2834 DPRINTF("exfer=%p", exfer, 0, 0, 0);
2835
2836 mutex_enter(&sc->sc_lock);
2837 for (size_t i = 0; i < exfer->ex_nsqtd; i++) {
2838 ehci_soft_qtd_t *sqtd = exfer->ex_sqtds[i];
2839
2840 if (sqtd == NULL)
2841 break;
2842
2843 sqtd->nextqtd = sc->sc_freeqtds;
2844 sc->sc_freeqtds = sqtd;
2845 }
2846 mutex_exit(&sc->sc_lock);
2847}
2848
2849Static void
2850ehci_append_sqtd(ehci_soft_qtd_t *sqtd, ehci_soft_qtd_t *prev)
2851{
2852 if (prev) {
2853 prev->nextqtd = sqtd;
2854 prev->qtd.qtd_next = htole32(sqtd->physaddr);
2855 prev->qtd.qtd_altnext = prev->qtd.qtd_next;
2856 usb_syncmem(&prev->dma, prev->offs, sizeof(prev->qtd),
2857 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2858 }
2859}
2860
2861Static void
2862ehci_reset_sqtd_chain(ehci_softc_t *sc, struct usbd_xfer *xfer,
2863 int length, int isread, int *toggle, ehci_soft_qtd_t **lsqtd)
2864{
2865 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
2866 usb_dma_t *dma = &xfer->ux_dmabuf;
2867 uint16_t flags = xfer->ux_flags;
2868 ehci_soft_qtd_t *sqtd, *prev;
2869 int tog = *toggle;
2870 int mps = UGETW(xfer->ux_pipe->up_endpoint->ue_edesc->wMaxPacketSize);
2871 int len = length;
2872
2873 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2874 DPRINTF("xfer=%p len %d isread %d toggle %d", xfer, len, isread, tog);
2875 DPRINTF(" VA %p", KERNADDR(&xfer->ux_dmabuf, 0), 0, 0, 0);
2876
2877 KASSERT(length != 0 || (!isread && (flags & USBD_FORCE_SHORT_XFER)));
2878
2879 const uint32_t qtdstatus = EHCI_QTD_ACTIVE |
2880 EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2881 EHCI_QTD_SET_CERR(3)
2882 ;
2883
2884 sqtd = prev = NULL;
2885 size_t curoffs = 0;
2886 size_t j = 0;
2887 for (; len != 0 && j < exfer->ex_nsqtd; prev = sqtd) {
2888 sqtd = exfer->ex_sqtds[j++];
2889 DPRINTF("sqtd[%d]=%p prev %p", j, sqtd, prev, 0);
2890
2891 /*
2892 * The EHCI hardware can handle at most 5 pages and they do
2893 * not have to be contiguous
2894 */
2895 vaddr_t va = (vaddr_t)KERNADDR(dma, curoffs);
2896 vaddr_t va_offs = EHCI_PAGE_OFFSET(va);
2897 size_t curlen = len;
2898 if (curlen >= EHCI_QTD_MAXTRANSFER - va_offs) {
2899 /* must use multiple TDs, fill as much as possible. */
2900 curlen = EHCI_QTD_MAXTRANSFER - va_offs;
2901
2902 /* the length must be a multiple of the max size */
2903 curlen -= curlen % mps;
2904 }
2905 KASSERT(curlen != 0);
2906 DPRINTF(" len=%d curlen=%d curoffs=%zu", len, curlen,
2907 curoffs, 0);
2908
2909 /* Fill the qTD */
2910 sqtd->qtd.qtd_next = sqtd->qtd.qtd_altnext = EHCI_NULL;
2911 sqtd->qtd.qtd_status = htole32(
2912 qtdstatus |
2913 EHCI_QTD_SET_BYTES(curlen) |
2914 EHCI_QTD_SET_TOGGLE(tog));
2915
2916 /* Find number of pages we'll be using, insert dma addresses */
2917 size_t pages = EHCI_NPAGES(curlen);
2918 KASSERT(pages <= EHCI_QTD_NBUFFERS);
2919 size_t pageoffs = EHCI_PAGE(curoffs);
2920 for (size_t i = 0; i < pages; i++) {
2921 paddr_t a = DMAADDR(dma,
2922 pageoffs + i * EHCI_PAGE_SIZE);
2923 sqtd->qtd.qtd_buffer[i] = htole32(EHCI_PAGE(a));
2924 /* Cast up to avoid compiler warnings */
2925 sqtd->qtd.qtd_buffer_hi[i] = htole32((uint64_t)a >> 32);
2926 DPRINTF(" buffer[%d/%d] 0x%08x 0x%08x", i, pages,
2927 le32toh(sqtd->qtd.qtd_buffer_hi[i]),
2928 le32toh(sqtd->qtd.qtd_buffer[i]));
2929 }
2930 /* First buffer pointer requires a page offset to start at */
2931 sqtd->qtd.qtd_buffer[0] |= htole32(va_offs);
2932
2933 usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
2934 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2935
2936 sqtd->len = curlen;
2937
2938 DPRINTF(" va %p pa %p len %d", va,
2939 DMAADDR(&xfer->ux_dmabuf, curoffs), curlen, 0);
2940
2941 ehci_append_sqtd(sqtd, prev);
2942
2943 if (((curlen + mps - 1) / mps) & 1) {
2944 tog ^= 1;
2945 }
2946
2947 curoffs += curlen;
2948 len -= curlen;
2949 }
2950 KASSERTMSG(len == 0, "xfer %p olen %d len %d mps %d ex_nsqtd %zu j %zu",
2951 xfer, length, len, mps, exfer->ex_nsqtd, j);
2952
2953 if (!isread &&
2954 (flags & USBD_FORCE_SHORT_XFER) &&
2955 length % mps == 0) {
2956 /* Force a 0 length transfer at the end. */
2957
2958 KASSERTMSG(j < exfer->ex_nsqtd, "j=%zu nsqtd=%zu", j,
2959 exfer->ex_nsqtd);
2960 prev = sqtd;
2961 sqtd = exfer->ex_sqtds[j++];
2962 memset(&sqtd->qtd, 0, sizeof(sqtd->qtd));
2963 sqtd->qtd.qtd_next = sqtd->qtd.qtd_altnext = EHCI_NULL;
2964 sqtd->qtd.qtd_status = htole32(
2965 qtdstatus |
2966 EHCI_QTD_SET_BYTES(0) |
2967 EHCI_QTD_SET_TOGGLE(tog));
2968
2969 usb_syncmem(&sqtd->dma, sqtd->offs, sizeof(sqtd->qtd),
2970 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2971
2972 ehci_append_sqtd(sqtd, prev);
2973 tog ^= 1;
2974 }
2975
2976 *lsqtd = sqtd;
2977 *toggle = tog;
2978}
2979
2980Static ehci_soft_itd_t *
2981ehci_alloc_itd(ehci_softc_t *sc)
2982{
2983 struct ehci_soft_itd *itd, *freeitd;
2984 usbd_status err;
2985 usb_dma_t dma;
2986
2987 EHCIHIST_FUNC(); EHCIHIST_CALLED();
2988
2989 mutex_enter(&sc->sc_lock);
2990
2991 freeitd = LIST_FIRST(&sc->sc_freeitds);
2992 if (freeitd == NULL) {
2993 DPRINTF("allocating chunk", 0, 0, 0, 0);
2994 mutex_exit(&sc->sc_lock);
2995 err = usb_allocmem(&sc->sc_bus, EHCI_ITD_SIZE * EHCI_ITD_CHUNK,
2996 EHCI_PAGE_SIZE, &dma);
2997
2998 if (err) {
2999 DPRINTF("alloc returned %d", err, 0, 0, 0);
3000 return NULL;
3001 }
3002 mutex_enter(&sc->sc_lock);
3003
3004 for (int i = 0; i < EHCI_ITD_CHUNK; i++) {
3005 int offs = i * EHCI_ITD_SIZE;
3006 itd = KERNADDR(&dma, offs);
3007 itd->physaddr = DMAADDR(&dma, offs);
3008 itd->dma = dma;
3009 itd->offs = offs;
3010 LIST_INSERT_HEAD(&sc->sc_freeitds, itd, free_list);
3011 }
3012 freeitd = LIST_FIRST(&sc->sc_freeitds);
3013 }
3014
3015 itd = freeitd;
3016 LIST_REMOVE(itd, free_list);
3017 mutex_exit(&sc->sc_lock);
3018 memset(&itd->itd, 0, sizeof(ehci_itd_t));
3019
3020 itd->frame_list.next = NULL;
3021 itd->frame_list.prev = NULL;
3022 itd->xfer_next = NULL;
3023 itd->slot = 0;
3024
3025 return itd;
3026}
3027
3028Static ehci_soft_sitd_t *
3029ehci_alloc_sitd(ehci_softc_t *sc)
3030{
3031 struct ehci_soft_sitd *sitd, *freesitd;
3032 usbd_status err;
3033 int i, offs;
3034 usb_dma_t dma;
3035
3036 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3037
3038 mutex_enter(&sc->sc_lock);
3039 freesitd = LIST_FIRST(&sc->sc_freesitds);
3040 if (freesitd == NULL) {
3041 DPRINTF("allocating chunk", 0, 0, 0, 0);
3042 mutex_exit(&sc->sc_lock);
3043 err = usb_allocmem(&sc->sc_bus, EHCI_SITD_SIZE * EHCI_SITD_CHUNK,
3044 EHCI_PAGE_SIZE, &dma);
3045
3046 if (err) {
3047 DPRINTF("alloc returned %d", err, 0, 0,
3048 0);
3049 return NULL;
3050 }
3051
3052 mutex_enter(&sc->sc_lock);
3053 for (i = 0; i < EHCI_SITD_CHUNK; i++) {
3054 offs = i * EHCI_SITD_SIZE;
3055 sitd = KERNADDR(&dma, offs);
3056 sitd->physaddr = DMAADDR(&dma, offs);
3057 sitd->dma = dma;
3058 sitd->offs = offs;
3059 LIST_INSERT_HEAD(&sc->sc_freesitds, sitd, free_list);
3060 }
3061 freesitd = LIST_FIRST(&sc->sc_freesitds);
3062 }
3063
3064 sitd = freesitd;
3065 LIST_REMOVE(sitd, free_list);
3066 mutex_exit(&sc->sc_lock);
3067
3068 memset(&sitd->sitd, 0, sizeof(ehci_sitd_t));
3069
3070 sitd->frame_list.next = NULL;
3071 sitd->frame_list.prev = NULL;
3072 sitd->xfer_next = NULL;
3073 sitd->slot = 0;
3074
3075 return sitd;
3076}
3077
3078/****************/
3079
3080/*
3081 * Close a reqular pipe.
3082 * Assumes that there are no pending transactions.
3083 */
3084Static void
3085ehci_close_pipe(struct usbd_pipe *pipe, ehci_soft_qh_t *head)
3086{
3087 struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(pipe);
3088 ehci_softc_t *sc = EHCI_PIPE2SC(pipe);
3089 ehci_soft_qh_t *sqh = epipe->sqh;
3090
3091 KASSERT(mutex_owned(&sc->sc_lock));
3092
3093 ehci_rem_qh(sc, sqh, head);
3094 ehci_free_sqh(sc, epipe->sqh);
3095}
3096
3097/*
3098 * Abort a device request.
3099 * If this routine is called at splusb() it guarantees that the request
3100 * will be removed from the hardware scheduling and that the callback
3101 * for it will be called with USBD_CANCELLED status.
3102 * It's impossible to guarantee that the requested transfer will not
3103 * have happened since the hardware runs concurrently.
3104 * If the transaction has already happened we rely on the ordinary
3105 * interrupt processing to process it.
3106 * XXX This is most probably wrong.
3107 * XXXMRG this doesn't make sense anymore.
3108 */
3109Static void
3110ehci_abort_xfer(struct usbd_xfer *xfer, usbd_status status)
3111{
3112 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
3113 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3114 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3115 ehci_soft_qh_t *sqh = epipe->sqh;
3116 ehci_soft_qtd_t *sqtd, *fsqtd, *lsqtd;
3117 ehci_physaddr_t cur;
3118 uint32_t qhstatus;
3119 int hit;
3120 int wake;
3121
3122 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3123
3124 DPRINTF("xfer=%p pipe=%p", xfer, epipe, 0, 0);
3125
3126 KASSERT(mutex_owned(&sc->sc_lock));
3127 ASSERT_SLEEPABLE();
3128
3129 if (sc->sc_dying) {
3130 /* If we're dying, just do the software part. */
3131 xfer->ux_status = status; /* make software ignore it */
3132 callout_stop(&xfer->ux_callout);
3133 usb_transfer_complete(xfer);
3134 return;
3135 }
3136
3137 /*
3138 * If an abort is already in progress then just wait for it to
3139 * complete and return.
3140 */
3141 if (xfer->ux_hcflags & UXFER_ABORTING) {
3142 DPRINTF("already aborting", 0, 0, 0, 0);
3143#ifdef DIAGNOSTIC
3144 if (status == USBD_TIMEOUT)
3145 printf("ehci_abort_xfer: TIMEOUT while aborting\n");
3146#endif
3147 /* Override the status which might be USBD_TIMEOUT. */
3148 xfer->ux_status = status;
3149 DPRINTF("waiting for abort to finish", 0, 0, 0, 0);
3150 xfer->ux_hcflags |= UXFER_ABORTWAIT;
3151 while (xfer->ux_hcflags & UXFER_ABORTING)
3152 cv_wait(&xfer->ux_hccv, &sc->sc_lock);
3153 return;
3154 }
3155 xfer->ux_hcflags |= UXFER_ABORTING;
3156
3157 /*
3158 * Step 1: Make interrupt routine and hardware ignore xfer.
3159 */
3160 xfer->ux_status = status; /* make software ignore it */
3161 callout_stop(&xfer->ux_callout);
3162 ehci_del_intr_list(sc, exfer);
3163
3164 usb_syncmem(&sqh->dma,
3165 sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3166 sizeof(sqh->qh.qh_qtd.qtd_status),
3167 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3168 qhstatus = sqh->qh.qh_qtd.qtd_status;
3169 sqh->qh.qh_qtd.qtd_status = qhstatus | htole32(EHCI_QTD_HALTED);
3170 usb_syncmem(&sqh->dma,
3171 sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3172 sizeof(sqh->qh.qh_qtd.qtd_status),
3173 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3174
3175 if (exfer->ex_type == EX_CTRL) {
3176 fsqtd = exfer->ex_setup;
3177 lsqtd = exfer->ex_status;
3178 } else {
3179 fsqtd = exfer->ex_sqtdstart;
3180 lsqtd = exfer->ex_sqtdend;
3181 }
3182 for (sqtd = fsqtd; ; sqtd = sqtd->nextqtd) {
3183 usb_syncmem(&sqtd->dma,
3184 sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
3185 sizeof(sqtd->qtd.qtd_status),
3186 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3187 sqtd->qtd.qtd_status |= htole32(EHCI_QTD_HALTED);
3188 usb_syncmem(&sqtd->dma,
3189 sqtd->offs + offsetof(ehci_qtd_t, qtd_status),
3190 sizeof(sqtd->qtd.qtd_status),
3191 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3192 if (sqtd == lsqtd)
3193 break;
3194 }
3195
3196 /*
3197 * Step 2: Wait until we know hardware has finished any possible
3198 * use of the xfer. Also make sure the soft interrupt routine
3199 * has run.
3200 */
3201 ehci_sync_hc(sc);
3202 sc->sc_softwake = 1;
3203 usb_schedsoftintr(&sc->sc_bus);
3204 cv_wait(&sc->sc_softwake_cv, &sc->sc_lock);
3205
3206 /*
3207 * Step 3: Remove any vestiges of the xfer from the hardware.
3208 * The complication here is that the hardware may have executed
3209 * beyond the xfer we're trying to abort. So as we're scanning
3210 * the TDs of this xfer we check if the hardware points to
3211 * any of them.
3212 */
3213
3214 usb_syncmem(&sqh->dma,
3215 sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3216 sizeof(sqh->qh.qh_curqtd),
3217 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3218 cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
3219 hit = 0;
3220 for (sqtd = fsqtd; ; sqtd = sqtd->nextqtd) {
3221 hit |= cur == sqtd->physaddr;
3222 if (sqtd == lsqtd)
3223 break;
3224 }
3225 sqtd = sqtd->nextqtd;
3226 /* Zap curqtd register if hardware pointed inside the xfer. */
3227 if (hit && sqtd != NULL) {
3228 DPRINTF("cur=0x%08x", sqtd->physaddr, 0, 0, 0);
3229 sqh->qh.qh_curqtd = htole32(sqtd->physaddr); /* unlink qTDs */
3230 usb_syncmem(&sqh->dma,
3231 sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3232 sizeof(sqh->qh.qh_curqtd),
3233 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3234 sqh->qh.qh_qtd.qtd_status = qhstatus;
3235 usb_syncmem(&sqh->dma,
3236 sqh->offs + offsetof(ehci_qh_t, qh_qtd.qtd_status),
3237 sizeof(sqh->qh.qh_qtd.qtd_status),
3238 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3239 } else {
3240 DPRINTF("no hit", 0, 0, 0, 0);
3241 usb_syncmem(&sqh->dma,
3242 sqh->offs + offsetof(ehci_qh_t, qh_curqtd),
3243 sizeof(sqh->qh.qh_curqtd),
3244 BUS_DMASYNC_PREREAD);
3245 }
3246
3247 /*
3248 * Step 4: Execute callback.
3249 */
3250#ifdef DIAGNOSTIC
3251 exfer->ex_isdone = true;
3252#endif
3253 wake = xfer->ux_hcflags & UXFER_ABORTWAIT;
3254 xfer->ux_hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
3255 usb_transfer_complete(xfer);
3256 if (wake) {
3257 cv_broadcast(&xfer->ux_hccv);
3258 }
3259
3260 KASSERT(mutex_owned(&sc->sc_lock));
3261}
3262
3263Static void
3264ehci_abort_isoc_xfer(struct usbd_xfer *xfer, usbd_status status)
3265{
3266 ehci_isoc_trans_t trans_status;
3267 struct ehci_xfer *exfer;
3268 ehci_softc_t *sc;
3269 struct ehci_soft_itd *itd;
3270 struct ehci_soft_sitd *sitd;
3271 int i, wake;
3272
3273 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3274
3275 exfer = EHCI_XFER2EXFER(xfer);
3276 sc = EHCI_XFER2SC(xfer);
3277
3278 DPRINTF("xfer %p pipe %p", xfer, xfer->ux_pipe, 0, 0);
3279
3280 KASSERT(mutex_owned(&sc->sc_lock));
3281
3282 if (sc->sc_dying) {
3283 xfer->ux_status = status;
3284 callout_stop(&xfer->ux_callout);
3285 usb_transfer_complete(xfer);
3286 return;
3287 }
3288
3289 if (xfer->ux_hcflags & UXFER_ABORTING) {
3290 DPRINTF("already aborting", 0, 0, 0, 0);
3291
3292#ifdef DIAGNOSTIC
3293 if (status == USBD_TIMEOUT)
3294 printf("ehci_abort_isoc_xfer: TIMEOUT while aborting\n");
3295#endif
3296
3297 xfer->ux_status = status;
3298 DPRINTF("waiting for abort to finish", 0, 0, 0, 0);
3299 xfer->ux_hcflags |= UXFER_ABORTWAIT;
3300 while (xfer->ux_hcflags & UXFER_ABORTING)
3301 cv_wait(&xfer->ux_hccv, &sc->sc_lock);
3302 goto done;
3303 }
3304 xfer->ux_hcflags |= UXFER_ABORTING;
3305
3306 xfer->ux_status = status;
3307 callout_stop(&xfer->ux_callout);
3308 ehci_del_intr_list(sc, exfer);
3309
3310 if (xfer->ux_pipe->up_dev->ud_speed == USB_SPEED_HIGH) {
3311 for (itd = exfer->ex_itdstart; itd != NULL;
3312 itd = itd->xfer_next) {
3313 usb_syncmem(&itd->dma,
3314 itd->offs + offsetof(ehci_itd_t, itd_ctl),
3315 sizeof(itd->itd.itd_ctl),
3316 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3317
3318 for (i = 0; i < 8; i++) {
3319 trans_status = le32toh(itd->itd.itd_ctl[i]);
3320 trans_status &= ~EHCI_ITD_ACTIVE;
3321 itd->itd.itd_ctl[i] = htole32(trans_status);
3322 }
3323
3324 usb_syncmem(&itd->dma,
3325 itd->offs + offsetof(ehci_itd_t, itd_ctl),
3326 sizeof(itd->itd.itd_ctl),
3327 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3328 }
3329 } else {
3330 for (sitd = exfer->ex_sitdstart; sitd != NULL;
3331 sitd = sitd->xfer_next) {
3332 usb_syncmem(&sitd->dma,
3333 sitd->offs + offsetof(ehci_sitd_t, sitd_buffer),
3334 sizeof(sitd->sitd.sitd_buffer),
3335 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
3336
3337 trans_status = le32toh(sitd->sitd.sitd_trans);
3338 trans_status &= ~EHCI_SITD_ACTIVE;
3339 sitd->sitd.sitd_trans = htole32(trans_status);
3340
3341 usb_syncmem(&sitd->dma,
3342 sitd->offs + offsetof(ehci_sitd_t, sitd_buffer),
3343 sizeof(sitd->sitd.sitd_buffer),
3344 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3345 }
3346 }
3347
3348 sc->sc_softwake = 1;
3349 usb_schedsoftintr(&sc->sc_bus);
3350 cv_wait(&sc->sc_softwake_cv, &sc->sc_lock);
3351
3352#ifdef DIAGNOSTIC
3353 exfer->ex_isdone = true;
3354#endif
3355 wake = xfer->ux_hcflags & UXFER_ABORTWAIT;
3356 xfer->ux_hcflags &= ~(UXFER_ABORTING | UXFER_ABORTWAIT);
3357 usb_transfer_complete(xfer);
3358 if (wake) {
3359 cv_broadcast(&xfer->ux_hccv);
3360 }
3361
3362done:
3363 KASSERT(mutex_owned(&sc->sc_lock));
3364 return;
3365}
3366
3367Static void
3368ehci_timeout(void *addr)
3369{
3370 struct usbd_xfer *xfer = addr;
3371 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3372 struct usbd_pipe *pipe = xfer->ux_pipe;
3373 struct usbd_device *dev = pipe->up_dev;
3374 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3375
3376 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3377
3378 DPRINTF("exfer %p", exfer, 0, 0, 0);
3379#ifdef EHCI_DEBUG
3380 if (ehcidebug >= 2)
3381 usbd_dump_pipe(pipe);
3382#endif
3383
3384 if (sc->sc_dying) {
3385 mutex_enter(&sc->sc_lock);
3386 ehci_abort_xfer(xfer, USBD_TIMEOUT);
3387 mutex_exit(&sc->sc_lock);
3388 return;
3389 }
3390
3391 /* Execute the abort in a process context. */
3392 usb_init_task(&exfer->ex_aborttask, ehci_timeout_task, xfer,
3393 USB_TASKQ_MPSAFE);
3394 usb_add_task(dev, &exfer->ex_aborttask, USB_TASKQ_HC);
3395}
3396
3397Static void
3398ehci_timeout_task(void *addr)
3399{
3400 struct usbd_xfer *xfer = addr;
3401 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3402
3403 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3404
3405 DPRINTF("xfer=%p", xfer, 0, 0, 0);
3406
3407 mutex_enter(&sc->sc_lock);
3408 ehci_abort_xfer(xfer, USBD_TIMEOUT);
3409 mutex_exit(&sc->sc_lock);
3410}
3411
3412/************************/
3413
3414Static int
3415ehci_device_ctrl_init(struct usbd_xfer *xfer)
3416{
3417 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3418 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
3419 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3420 usb_device_request_t *req = &xfer->ux_request;
3421 ehci_soft_qtd_t *setup, *status, *next;
3422 int isread = req->bmRequestType & UT_READ;
3423 int len = xfer->ux_bufsize;
3424 int err;
3425
3426 exfer->ex_type = EX_CTRL;
3427 exfer->ex_status = NULL;
3428 exfer->ex_data = NULL;
3429 exfer->ex_setup = ehci_alloc_sqtd(sc);
3430 if (exfer->ex_setup == NULL) {
3431 err = ENOMEM;
3432 goto bad1;
3433 }
3434 exfer->ex_status = ehci_alloc_sqtd(sc);
3435 if (exfer->ex_status == NULL) {
3436 err = ENOMEM;
3437 goto bad2;
3438 }
3439 setup = exfer->ex_setup;
3440 status = exfer->ex_status;
3441 exfer->ex_nsqtd = 0;
3442 next = status;
3443 /* Set up data transaction */
3444 if (len != 0) {
3445 err = ehci_alloc_sqtd_chain(sc, xfer, len, isread,
3446 &exfer->ex_data);
3447 if (err)
3448 goto bad3;
3449 next = exfer->ex_data;
3450 }
3451
3452 /* Clear toggle */
3453 setup->qtd.qtd_status = htole32(
3454 EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
3455 EHCI_QTD_SET_TOGGLE(0) |
3456 EHCI_QTD_SET_BYTES(sizeof(*req))
3457 );
3458 setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->ctrl.reqdma, 0));
3459 setup->qtd.qtd_buffer_hi[0] = 0;
3460 setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
3461 setup->nextqtd = next;
3462 setup->xfer = xfer;
3463 setup->len = sizeof(*req);
3464
3465 status->qtd.qtd_status = htole32(
3466 EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
3467 EHCI_QTD_SET_TOGGLE(1) |
3468 EHCI_QTD_IOC
3469 );
3470 status->qtd.qtd_buffer[0] = 0;
3471 status->qtd.qtd_buffer_hi[0] = 0;
3472 status->qtd.qtd_next = status->qtd.qtd_altnext = EHCI_NULL;
3473 status->nextqtd = NULL;
3474 status->xfer = xfer;
3475 status->len = 0;
3476
3477 return 0;
3478bad3:
3479 ehci_free_sqtd(sc, exfer->ex_status);
3480bad2:
3481 ehci_free_sqtd(sc, exfer->ex_setup);
3482bad1:
3483 return err;
3484}
3485
3486Static void
3487ehci_device_ctrl_fini(struct usbd_xfer *xfer)
3488{
3489 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3490 struct ehci_xfer *ex = EHCI_XFER2EXFER(xfer);
3491
3492 KASSERT(ex->ex_type == EX_CTRL);
3493
3494 ehci_free_sqtd(sc, ex->ex_setup);
3495 ehci_free_sqtd(sc, ex->ex_status);
3496 ehci_free_sqtds(sc, ex);
3497 if (ex->ex_nsqtd)
3498 kmem_free(ex->ex_sqtds,
3499 sizeof(ehci_soft_qtd_t *) * ex->ex_nsqtd);
3500}
3501
3502Static usbd_status
3503ehci_device_ctrl_transfer(struct usbd_xfer *xfer)
3504{
3505 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3506 usbd_status err;
3507
3508 /* Insert last in queue. */
3509 mutex_enter(&sc->sc_lock);
3510 err = usb_insert_transfer(xfer);
3511 mutex_exit(&sc->sc_lock);
3512 if (err)
3513 return err;
3514
3515 /* Pipe isn't running, start first */
3516 return ehci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3517}
3518
3519Static usbd_status
3520ehci_device_ctrl_start(struct usbd_xfer *xfer)
3521{
3522 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
3523 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3524 usb_device_request_t *req = &xfer->ux_request;
3525 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3526 ehci_soft_qtd_t *setup, *status, *next;
3527 ehci_soft_qh_t *sqh;
3528
3529 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3530
3531 KASSERT(xfer->ux_rqflags & URQ_REQUEST);
3532
3533 if (sc->sc_dying)
3534 return USBD_IOERROR;
3535
3536 const int isread = req->bmRequestType & UT_READ;
3537 const int len = UGETW(req->wLength);
3538
3539 DPRINTF("type=0x%02x, request=0x%02x, wValue=0x%04x, wIndex=0x%04x",
3540 req->bmRequestType, req->bRequest, UGETW(req->wValue),
3541 UGETW(req->wIndex));
3542 DPRINTF("len=%d, addr=%d, endpt=%d", len, epipe->pipe.up_dev->ud_addr,
3543 epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress, 0);
3544
3545 sqh = epipe->sqh;
3546
3547 KASSERTMSG(EHCI_QH_GET_ADDR(le32toh(sqh->qh.qh_endp)) == epipe->pipe.up_dev->ud_addr,
3548 "address QH %" __PRIuBIT " pipe %d\n",
3549 EHCI_QH_GET_ADDR(le32toh(sqh->qh.qh_endp)),
3550 epipe->pipe.up_dev->ud_addr);
3551 KASSERTMSG(EHCI_QH_GET_MPL(le32toh(sqh->qh.qh_endp)) ==
3552 UGETW(epipe->pipe.up_endpoint->ue_edesc->wMaxPacketSize),
3553 "MPS QH %" __PRIuBIT " pipe %d\n",
3554 EHCI_QH_GET_MPL(le32toh(sqh->qh.qh_endp)),
3555 UGETW(epipe->pipe.up_endpoint->ue_edesc->wMaxPacketSize));
3556
3557 setup = exfer->ex_setup;
3558 status = exfer->ex_status;
3559
3560 DPRINTF("setup %p status %p data %p", setup, status, exfer->ex_data, 0);
3561 KASSERTMSG(setup != NULL && status != NULL,
3562 "Failed memory allocation, setup %p status %p",
3563 setup, status);
3564
3565 memcpy(KERNADDR(&epipe->ctrl.reqdma, 0), req, sizeof(*req));
3566 usb_syncmem(&epipe->ctrl.reqdma, 0, sizeof(*req), BUS_DMASYNC_PREWRITE);
3567
3568 /* Clear toggle */
3569 setup->qtd.qtd_status &= ~htole32(
3570 EHCI_QTD_STATUS_MASK |
3571 EHCI_QTD_BYTES_MASK |
3572 EHCI_QTD_TOGGLE_MASK |
3573 EHCI_QTD_CERR_MASK
3574 );
3575 setup->qtd.qtd_status |= htole32(
3576 EHCI_QTD_ACTIVE |
3577 EHCI_QTD_SET_CERR(3) |
3578 EHCI_QTD_SET_TOGGLE(0) |
3579 EHCI_QTD_SET_BYTES(sizeof(*req))
3580 );
3581 setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->ctrl.reqdma, 0));
3582 setup->qtd.qtd_buffer_hi[0] = 0;
3583
3584 next = status;
3585 status->qtd.qtd_status &= ~htole32(
3586 EHCI_QTD_STATUS_MASK |
3587 EHCI_QTD_PID_MASK |
3588 EHCI_QTD_BYTES_MASK |
3589 EHCI_QTD_TOGGLE_MASK |
3590 EHCI_QTD_CERR_MASK
3591 );
3592 status->qtd.qtd_status |= htole32(
3593 EHCI_QTD_ACTIVE |
3594 EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
3595 EHCI_QTD_SET_CERR(3) |
3596 EHCI_QTD_SET_TOGGLE(1) |
3597 EHCI_QTD_SET_BYTES(0) |
3598 EHCI_QTD_IOC
3599 );
3600 KASSERT(status->qtd.qtd_status & htole32(EHCI_QTD_TOGGLE_MASK));
3601
3602 KASSERT(exfer->ex_isdone);
3603#ifdef DIAGNOSTIC
3604 exfer->ex_isdone = false;
3605#endif
3606
3607 /* Set up data transaction */
3608 if (len != 0) {
3609 ehci_soft_qtd_t *end;
3610
3611 /* Start toggle at 1. */
3612 int toggle = 1;
3613 next = exfer->ex_data;
3614 KASSERTMSG(next != NULL, "Failed memory allocation");
3615 ehci_reset_sqtd_chain(sc, xfer, len, isread, &toggle, &end);
3616 end->nextqtd = status;
3617 end->qtd.qtd_next = end->qtd.qtd_altnext =
3618 htole32(status->physaddr);
3619
3620 usb_syncmem(&end->dma, end->offs, sizeof(end->qtd),
3621 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3622
3623 usb_syncmem(&xfer->ux_dmabuf, 0, len,
3624 isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
3625 }
3626
3627 setup->nextqtd = next;
3628 setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
3629
3630 usb_syncmem(&setup->dma, setup->offs, sizeof(setup->qtd),
3631 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3632
3633 usb_syncmem(&status->dma, status->offs, sizeof(status->qtd),
3634 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3635
3636 KASSERT(status->qtd.qtd_status & htole32(EHCI_QTD_TOGGLE_MASK));
3637
3638#ifdef EHCI_DEBUG
3639 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
3640 ehci_dump_sqh(sqh);
3641 ehci_dump_sqtds(setup);
3642 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
3643#endif
3644
3645 mutex_enter(&sc->sc_lock);
3646
3647 /* Insert qTD in QH list - also does usb_syncmem(sqh) */
3648 ehci_set_qh_qtd(sqh, setup);
3649 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) {
3650 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout),
3651 ehci_timeout, xfer);
3652 }
3653 ehci_add_intr_list(sc, exfer);
3654 xfer->ux_status = USBD_IN_PROGRESS;
3655 mutex_exit(&sc->sc_lock);
3656
3657#if 0
3658#ifdef EHCI_DEBUG
3659 DPRINTFN(10, "status=%x, dump:", EOREAD4(sc, EHCI_USBSTS), 0, 0, 0);
3660// delay(10000);
3661 ehci_dump_regs(sc);
3662 ehci_dump_sqh(sc->sc_async_head);
3663 ehci_dump_sqh(sqh);
3664 ehci_dump_sqtds(setup);
3665#endif
3666#endif
3667
3668 return USBD_IN_PROGRESS;
3669}
3670
3671Static void
3672ehci_device_ctrl_done(struct usbd_xfer *xfer)
3673{
3674 ehci_softc_t *sc __diagused = EHCI_XFER2SC(xfer);
3675 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
3676 usb_device_request_t *req = &xfer->ux_request;
3677 int len = UGETW(req->wLength);
3678 int rd = req->bmRequestType & UT_READ;
3679
3680 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3681 DPRINTF("xfer=%p", xfer, 0, 0, 0);
3682
3683 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
3684 KASSERT(xfer->ux_rqflags & URQ_REQUEST);
3685
3686 usb_syncmem(&epipe->ctrl.reqdma, 0, sizeof(*req),
3687 BUS_DMASYNC_POSTWRITE);
3688 if (len)
3689 usb_syncmem(&xfer->ux_dmabuf, 0, len,
3690 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3691
3692 DPRINTF("length=%d", xfer->ux_actlen, 0, 0, 0);
3693}
3694
3695/* Abort a device control request. */
3696Static void
3697ehci_device_ctrl_abort(struct usbd_xfer *xfer)
3698{
3699 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3700
3701 DPRINTF("xfer=%p", xfer, 0, 0, 0);
3702 ehci_abort_xfer(xfer, USBD_CANCELLED);
3703}
3704
3705/* Close a device control pipe. */
3706Static void
3707ehci_device_ctrl_close(struct usbd_pipe *pipe)
3708{
3709 ehci_softc_t *sc = EHCI_PIPE2SC(pipe);
3710 /*struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(pipe);*/
3711
3712 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3713
3714 KASSERT(mutex_owned(&sc->sc_lock));
3715
3716 DPRINTF("pipe=%p", pipe, 0, 0, 0);
3717
3718 ehci_close_pipe(pipe, sc->sc_async_head);
3719}
3720
3721/*
3722 * Some EHCI chips from VIA seem to trigger interrupts before writing back the
3723 * qTD status, or miss signalling occasionally under heavy load. If the host
3724 * machine is too fast, we we can miss transaction completion - when we scan
3725 * the active list the transaction still seems to be active. This generally
3726 * exhibits itself as a umass stall that never recovers.
3727 *
3728 * We work around this behaviour by setting up this callback after any softintr
3729 * that completes with transactions still pending, giving us another chance to
3730 * check for completion after the writeback has taken place.
3731 */
3732Static void
3733ehci_intrlist_timeout(void *arg)
3734{
3735 ehci_softc_t *sc = arg;
3736
3737 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3738
3739 usb_schedsoftintr(&sc->sc_bus);
3740}
3741
3742/************************/
3743
3744Static int
3745ehci_device_bulk_init(struct usbd_xfer *xfer)
3746{
3747 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3748 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3749 usb_endpoint_descriptor_t *ed = xfer->ux_pipe->up_endpoint->ue_edesc;
3750 int endpt = ed->bEndpointAddress;
3751 int isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3752 int len = xfer->ux_bufsize;
3753 int err = 0;
3754
3755 exfer->ex_type = EX_BULK;
3756 exfer->ex_nsqtd = 0;
3757 err = ehci_alloc_sqtd_chain(sc, xfer, len, isread,
3758 &exfer->ex_sqtdstart);
3759
3760 return err;
3761}
3762
3763Static void
3764ehci_device_bulk_fini(struct usbd_xfer *xfer)
3765{
3766 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3767 struct ehci_xfer *ex = EHCI_XFER2EXFER(xfer);
3768
3769 KASSERT(ex->ex_type == EX_BULK);
3770
3771 ehci_free_sqtds(sc, ex);
3772 if (ex->ex_nsqtd)
3773 kmem_free(ex->ex_sqtds, sizeof(ehci_soft_qtd_t *) * ex->ex_nsqtd);
3774}
3775
3776Static usbd_status
3777ehci_device_bulk_transfer(struct usbd_xfer *xfer)
3778{
3779 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3780 usbd_status err;
3781
3782 /* Insert last in queue. */
3783 mutex_enter(&sc->sc_lock);
3784 err = usb_insert_transfer(xfer);
3785 mutex_exit(&sc->sc_lock);
3786 if (err)
3787 return err;
3788
3789 /* Pipe isn't running, start first */
3790 return ehci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3791}
3792
3793Static usbd_status
3794ehci_device_bulk_start(struct usbd_xfer *xfer)
3795{
3796 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
3797 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3798 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3799 ehci_soft_qh_t *sqh;
3800 ehci_soft_qtd_t *end;
3801 int len, isread, endpt;
3802
3803 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3804
3805 DPRINTF("xfer=%p len=%d flags=%d", xfer, xfer->ux_length,
3806 xfer->ux_flags, 0);
3807
3808 if (sc->sc_dying)
3809 return USBD_IOERROR;
3810
3811 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
3812 KASSERT(xfer->ux_length <= xfer->ux_bufsize);
3813
3814 len = xfer->ux_length;
3815 endpt = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
3816 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3817 sqh = epipe->sqh;
3818
3819 KASSERT(exfer->ex_isdone);
3820#ifdef DIAGNOSTIC
3821 exfer->ex_isdone = false;
3822#endif
3823
3824 /* Take lock here to protect nexttoggle */
3825 mutex_enter(&sc->sc_lock);
3826
3827 ehci_reset_sqtd_chain(sc, xfer, len, isread, &epipe->nexttoggle, &end);
3828
3829 exfer->ex_sqtdend = end;
3830 end->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
3831 usb_syncmem(&end->dma, end->offs, sizeof(end->qtd),
3832 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
3833
3834#ifdef EHCI_DEBUG
3835 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
3836 ehci_dump_sqh(sqh);
3837 ehci_dump_sqtds(exfer->ex_sqtdstart);
3838 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
3839#endif
3840
3841 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
3842 isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
3843
3844 /* also does usb_syncmem(sqh) */
3845 ehci_set_qh_qtd(sqh, exfer->ex_sqtdstart);
3846 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) {
3847 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout),
3848 ehci_timeout, xfer);
3849 }
3850 ehci_add_intr_list(sc, exfer);
3851 xfer->ux_status = USBD_IN_PROGRESS;
3852 mutex_exit(&sc->sc_lock);
3853
3854#if 0
3855#ifdef EHCI_DEBUG
3856 DPRINTFN(5, "data(2)", 0, 0, 0, 0);
3857// delay(10000);
3858 DPRINTFN(5, "data(3)", 0, 0, 0, 0);
3859 ehci_dump_regs(sc);
3860#if 0
3861 printf("async_head:\n");
3862 ehci_dump_sqh(sc->sc_async_head);
3863#endif
3864 DPRINTF("sqh:", 0, 0, 0, 0);
3865 ehci_dump_sqh(sqh);
3866 ehci_dump_sqtds(exfer->ex_sqtdstart);
3867#endif
3868#endif
3869
3870 return USBD_IN_PROGRESS;
3871}
3872
3873Static void
3874ehci_device_bulk_abort(struct usbd_xfer *xfer)
3875{
3876 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3877
3878 DPRINTF("xfer %p", xfer, 0, 0, 0);
3879 ehci_abort_xfer(xfer, USBD_CANCELLED);
3880}
3881
3882/*
3883 * Close a device bulk pipe.
3884 */
3885Static void
3886ehci_device_bulk_close(struct usbd_pipe *pipe)
3887{
3888 ehci_softc_t *sc = EHCI_PIPE2SC(pipe);
3889 struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(pipe);
3890
3891 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3892
3893 KASSERT(mutex_owned(&sc->sc_lock));
3894
3895 DPRINTF("pipe=%p", pipe, 0, 0, 0);
3896 pipe->up_endpoint->ue_toggle = epipe->nexttoggle;
3897 ehci_close_pipe(pipe, sc->sc_async_head);
3898}
3899
3900Static void
3901ehci_device_bulk_done(struct usbd_xfer *xfer)
3902{
3903 ehci_softc_t *sc __diagused = EHCI_XFER2SC(xfer);
3904 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
3905 int endpt = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
3906 int rd = UE_GET_DIR(endpt) == UE_DIR_IN;
3907
3908 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3909
3910 DPRINTF("xfer=%p, actlen=%d", xfer, xfer->ux_actlen, 0, 0);
3911
3912 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
3913
3914 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
3915 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3916
3917 DPRINTF("length=%d", xfer->ux_actlen, 0, 0, 0);
3918}
3919
3920/************************/
3921
3922Static usbd_status
3923ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
3924{
3925 struct ehci_soft_islot *isp;
3926 int islot, lev;
3927
3928 /* Find a poll rate that is large enough. */
3929 for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3930 if (EHCI_ILEV_IVAL(lev) <= ival)
3931 break;
3932
3933 /* Pick an interrupt slot at the right level. */
3934 /* XXX could do better than picking at random */
3935 sc->sc_rand = (sc->sc_rand + 191) % sc->sc_flsize;
3936 islot = EHCI_IQHIDX(lev, sc->sc_rand);
3937
3938 sqh->islot = islot;
3939 isp = &sc->sc_islots[islot];
3940 mutex_enter(&sc->sc_lock);
3941 ehci_add_qh(sc, sqh, isp->sqh);
3942 mutex_exit(&sc->sc_lock);
3943
3944 return USBD_NORMAL_COMPLETION;
3945}
3946
3947
3948Static int
3949ehci_device_intr_init(struct usbd_xfer *xfer)
3950{
3951 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
3952 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3953 usb_endpoint_descriptor_t *ed = xfer->ux_pipe->up_endpoint->ue_edesc;
3954 int endpt = ed->bEndpointAddress;
3955 int isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3956 int len = xfer->ux_bufsize;
3957 int err;
3958
3959 EHCIHIST_FUNC(); EHCIHIST_CALLED();
3960
3961 DPRINTF("xfer=%p len=%d flags=%d", xfer, xfer->ux_length,
3962 xfer->ux_flags, 0);
3963
3964 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
3965 KASSERT(len != 0);
3966
3967 exfer->ex_type = EX_INTR;
3968 exfer->ex_nsqtd = 0;
3969 err = ehci_alloc_sqtd_chain(sc, xfer, len, isread,
3970 &exfer->ex_sqtdstart);
3971
3972 return err;
3973}
3974
3975Static void
3976ehci_device_intr_fini(struct usbd_xfer *xfer)
3977{
3978 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3979 struct ehci_xfer *ex = EHCI_XFER2EXFER(xfer);
3980
3981 KASSERT(ex->ex_type == EX_INTR);
3982
3983 ehci_free_sqtds(sc, ex);
3984 if (ex->ex_nsqtd)
3985 kmem_free(ex->ex_sqtds, sizeof(ehci_soft_qtd_t *) * ex->ex_nsqtd);
3986}
3987
3988Static usbd_status
3989ehci_device_intr_transfer(struct usbd_xfer *xfer)
3990{
3991 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
3992 usbd_status err;
3993
3994 /* Insert last in queue. */
3995 mutex_enter(&sc->sc_lock);
3996 err = usb_insert_transfer(xfer);
3997 mutex_exit(&sc->sc_lock);
3998 if (err)
3999 return err;
4000
4001 /*
4002 * Pipe isn't running (otherwise err would be USBD_INPROG),
4003 * so start it first.
4004 */
4005 return ehci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
4006}
4007
4008Static usbd_status
4009ehci_device_intr_start(struct usbd_xfer *xfer)
4010{
4011 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
4012 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4013 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4014 ehci_soft_qtd_t *end;
4015 ehci_soft_qh_t *sqh;
4016 int len, isread, endpt;
4017
4018 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4019
4020 DPRINTF("xfer=%p len=%d flags=%d", xfer, xfer->ux_length,
4021 xfer->ux_flags, 0);
4022
4023 if (sc->sc_dying)
4024 return USBD_IOERROR;
4025
4026 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
4027 KASSERT(xfer->ux_length <= xfer->ux_bufsize);
4028
4029 len = xfer->ux_length;
4030 endpt = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
4031 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
4032 sqh = epipe->sqh;
4033
4034 KASSERT(exfer->ex_isdone);
4035#ifdef DIAGNOSTIC
4036 exfer->ex_isdone = false;
4037#endif
4038
4039 /* Take lock to protect nexttoggle */
4040 mutex_enter(&sc->sc_lock);
4041
4042 ehci_reset_sqtd_chain(sc, xfer, len, isread, &epipe->nexttoggle, &end);
4043
4044 end->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
4045 usb_syncmem(&end->dma, end->offs, sizeof(end->qtd),
4046 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4047 exfer->ex_sqtdend = end;
4048
4049#ifdef EHCI_DEBUG
4050 DPRINTFN(5, "--- dump start ---", 0, 0, 0, 0);
4051 ehci_dump_sqh(sqh);
4052 ehci_dump_sqtds(exfer->ex_sqtdstart);
4053 DPRINTFN(5, "--- dump end ---", 0, 0, 0, 0);
4054#endif
4055
4056 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4057 isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
4058
4059 /* also does usb_syncmem(sqh) */
4060 ehci_set_qh_qtd(sqh, exfer->ex_sqtdstart);
4061 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) {
4062 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout),
4063 ehci_timeout, xfer);
4064 }
4065 ehci_add_intr_list(sc, exfer);
4066 xfer->ux_status = USBD_IN_PROGRESS;
4067 mutex_exit(&sc->sc_lock);
4068
4069#if 0
4070#ifdef EHCI_DEBUG
4071 DPRINTFN(5, "data(2)", 0, 0, 0, 0);
4072// delay(10000);
4073 DPRINTFN(5, "data(3)", 0, 0, 0, 0);
4074 ehci_dump_regs(sc);
4075 DPRINTFN(5, "sqh:", 0, 0, 0, 0);
4076 ehci_dump_sqh(sqh);
4077 ehci_dump_sqtds(exfer->ex_sqtdstart);
4078#endif
4079#endif
4080
4081 return USBD_IN_PROGRESS;
4082}
4083
4084Static void
4085ehci_device_intr_abort(struct usbd_xfer *xfer)
4086{
4087 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4088
4089 DPRINTF("xfer=%p", xfer, 0, 0, 0);
4090 KASSERT(xfer->ux_pipe->up_intrxfer == xfer);
4091
4092 /*
4093 * XXX - abort_xfer uses ehci_sync_hc, which syncs via the advance
4094 * async doorbell. That's dependent on the async list, wheras
4095 * intr xfers are periodic, should not use this?
4096 */
4097 ehci_abort_xfer(xfer, USBD_CANCELLED);
4098}
4099
4100Static void
4101ehci_device_intr_close(struct usbd_pipe *pipe)
4102{
4103 ehci_softc_t *sc = EHCI_PIPE2SC(pipe);
4104 struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(pipe);
4105 struct ehci_soft_islot *isp;
4106
4107 KASSERT(mutex_owned(&sc->sc_lock));
4108
4109 isp = &sc->sc_islots[epipe->sqh->islot];
4110 ehci_close_pipe(pipe, isp->sqh);
4111}
4112
4113Static void
4114ehci_device_intr_done(struct usbd_xfer *xfer)
4115{
4116 ehci_softc_t *sc __diagused = EHCI_XFER2SC(xfer);
4117 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
4118 int isread, endpt;
4119
4120 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4121
4122 DPRINTF("xfer=%p, actlen=%d", xfer, xfer->ux_actlen,
4123 0, 0);
4124
4125 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock));
4126
4127 endpt = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
4128 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
4129 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4130 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
4131}
4132
4133/************************/
4134Static int
4135ehci_device_fs_isoc_init(struct usbd_xfer *xfer)
4136{
4137 struct ehci_pipe *epipe = EHCI_PIPE2EPIPE(xfer->ux_pipe);
4138 struct usbd_device *dev = xfer->ux_pipe->up_dev;
4139 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4140 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4141 ehci_soft_sitd_t *sitd, *prev, *start, *stop;
4142 int i, k, frames;
4143 u_int huba, dir;
4144 int err;
4145
4146 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4147
4148 start = NULL;
4149 sitd = NULL;
4150
4151 DPRINTF("xfer %p len %d flags %d", xfer, xfer->ux_length,
4152 xfer->ux_flags, 0);
4153
4154 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
4155 KASSERT(xfer->ux_nframes != 0);
4156 KASSERT(exfer->ex_isdone);
4157
4158 exfer->ex_type = EX_FS_ISOC;
4159 /*
4160 * Step 1: Allocate and initialize sitds.
4161 */
4162 i = epipe->pipe.up_endpoint->ue_edesc->bInterval;
4163 if (i > 16 || i == 0) {
4164 /* Spec page 271 says intervals > 16 are invalid */
4165 DPRINTF("bInterval %d invalid", i, 0, 0, 0);
4166
4167 return EINVAL;
4168 }
4169
4170 frames = xfer->ux_nframes;
4171 for (i = 0, prev = NULL; i < frames; i++, prev = sitd) {
4172 sitd = ehci_alloc_sitd(sc);
4173 if (sitd == NULL) {
4174 err = ENOMEM;
4175 goto fail;
4176 }
4177
4178 if (prev)
4179 prev->xfer_next = sitd;
4180 else
4181 start = sitd;
4182
4183 huba = dev->ud_myhsport->up_parent->ud_addr;
4184
4185#if 0
4186 if (sc->sc_flags & EHCIF_FREESCALE) {
4187 // Set hub address to 0 if embedded TT is used.
4188 if (huba == sc->sc_addr)
4189 huba = 0;
4190 }
4191#endif
4192
4193 k = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
4194 dir = UE_GET_DIR(k) ? 1 : 0;
4195 sitd->sitd.sitd_endp =
4196 htole32(EHCI_SITD_SET_ENDPT(UE_GET_ADDR(k)) |
4197 EHCI_SITD_SET_DADDR(dev->ud_addr) |
4198 EHCI_SITD_SET_PORT(dev->ud_myhsport->up_portno) |
4199 EHCI_SITD_SET_HUBA(huba) |
4200 EHCI_SITD_SET_DIR(dir));
4201
4202 sitd->sitd.sitd_back = htole32(EHCI_LINK_TERMINATE);
4203 } /* End of frame */
4204
4205 sitd->sitd.sitd_trans |= htole32(EHCI_SITD_IOC);
4206
4207 stop = sitd;
4208 stop->xfer_next = NULL;
4209 exfer->ex_sitdstart = start;
4210 exfer->ex_sitdend = stop;
4211
4212 return 0;
4213
4214fail:
4215 mutex_enter(&sc->sc_lock);
4216 ehci_soft_sitd_t *next;
4217 for (sitd = start; sitd; sitd = next) {
4218 next = sitd->xfer_next;
4219 ehci_free_sitd_locked(sc, sitd);
4220 }
4221 mutex_exit(&sc->sc_lock);
4222
4223 return err;
4224}
4225
4226Static void
4227ehci_device_fs_isoc_fini(struct usbd_xfer *xfer)
4228{
4229 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4230 struct ehci_xfer *ex = EHCI_XFER2EXFER(xfer);
4231
4232 KASSERT(ex->ex_type == EX_FS_ISOC);
4233
4234 ehci_free_sitd_chain(sc, ex->ex_sitdstart);
4235}
4236
4237Static usbd_status
4238ehci_device_fs_isoc_transfer(struct usbd_xfer *xfer)
4239{
4240 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4241 usbd_status __diagused err;
4242
4243 mutex_enter(&sc->sc_lock);
4244 err = usb_insert_transfer(xfer);
4245 mutex_exit(&sc->sc_lock);
4246
4247 KASSERT(err == USBD_NORMAL_COMPLETION);
4248
4249 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);;
4250 struct usbd_device *dev = xfer->ux_pipe->up_dev;;
4251 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4252 ehci_soft_sitd_t *sitd;
4253 usb_dma_t *dma_buf;
4254 int i, j, k, frames;
4255 int offs, total_length;
4256 int frindex;
4257 u_int dir;
4258
4259 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4260
4261 sitd = NULL;
4262 total_length = 0;
4263
4264
4265 DPRINTF("xfer %p len %d flags %d", xfer, xfer->ux_length,
4266 xfer->ux_flags, 0);
4267
4268 if (sc->sc_dying)
4269 return USBD_IOERROR;
4270
4271 /*
4272 * To avoid complication, don't allow a request right now that'll span
4273 * the entire frame table. To within 4 frames, to allow some leeway
4274 * on either side of where the hc currently is.
4275 */
4276 if (epipe->pipe.up_endpoint->ue_edesc->bInterval *
4277 xfer->ux_nframes >= sc->sc_flsize - 4) {
4278 printf("ehci: isoc descriptor requested that spans the entire"
4279 "frametable, too many frames\n");
4280 return USBD_INVAL;
4281 }
4282
4283 KASSERT(xfer->ux_nframes != 0 && xfer->ux_frlengths);
4284 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
4285 KASSERT(exfer->ex_isdone);
4286#ifdef DIAGNOSTIC
4287 exfer->ex_isdone = false;
4288#endif
4289
4290 /*
4291 * Step 1: Initialize sitds.
4292 */
4293
4294 frames = xfer->ux_nframes;
4295 dma_buf = &xfer->ux_dmabuf;
4296 offs = 0;
4297
4298 for (sitd = exfer->ex_sitdstart, i = 0; i < frames;
4299 i++, sitd = sitd->xfer_next) {
4300 KASSERT(sitd != NULL);
4301 KASSERT(xfer->ux_frlengths[i] <= 0x3ff);
4302
4303 sitd->sitd.sitd_trans = htole32(EHCI_SITD_ACTIVE |
4304 EHCI_SITD_SET_LEN(xfer->ux_frlengths[i]));
4305
4306 /* Set page0 index and offset - TP and T-offset are set below */
4307 sitd->sitd.sitd_buffer[0] = htole32(DMAADDR(dma_buf, offs));
4308
4309 total_length += xfer->ux_frlengths[i];
4310 offs += xfer->ux_frlengths[i];
4311
4312 sitd->sitd.sitd_buffer[1] =
4313 htole32(EHCI_SITD_SET_BPTR(DMAADDR(dma_buf, offs - 1)));
4314
4315 u_int huba __diagused = dev->ud_myhsport->up_parent->ud_addr;
4316
4317#if 0
4318 if (sc->sc_flags & EHCIF_FREESCALE) {
4319 // Set hub address to 0 if embedded TT is used.
4320 if (huba == sc->sc_addr)
4321 huba = 0;
4322 }
4323#endif
4324
4325 k = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
4326 dir = UE_GET_DIR(k) ? 1 : 0;
4327 KASSERT(sitd->sitd.sitd_endp == htole32(
4328 EHCI_SITD_SET_ENDPT(UE_GET_ADDR(k)) |
4329 EHCI_SITD_SET_DADDR(dev->ud_addr) |
4330 EHCI_SITD_SET_PORT(dev->ud_myhsport->up_portno) |
4331 EHCI_SITD_SET_HUBA(huba) |
4332 EHCI_SITD_SET_DIR(dir)));
4333 KASSERT(sitd->sitd.sitd_back == htole32(EHCI_LINK_TERMINATE));
4334
4335 uint8_t sa = 0;
4336 uint8_t sb = 0;
4337 u_int temp, tlen;
4338
4339 if (dir == 0) { /* OUT */
4340 temp = 0;
4341 tlen = xfer->ux_frlengths[i];
4342 if (tlen <= 188) {
4343 temp |= 1; /* T-count = 1, TP = ALL */
4344 tlen = 1;
4345 } else {
4346 tlen += 187;
4347 tlen /= 188;
4348 temp |= tlen; /* T-count = [1..6] */
4349 temp |= 8; /* TP = Begin */
4350 }
4351 sitd->sitd.sitd_buffer[1] |= htole32(temp);
4352
4353 tlen += sa;
4354
4355 if (tlen >= 8) {
4356 sb = 0;
4357 } else {
4358 sb = (1 << tlen);
4359 }
4360
4361 sa = (1 << sa);
4362 sa = (sb - sa) & 0x3F;
4363 sb = 0;
4364 } else {
4365 sb = (-(4 << sa)) & 0xFE;
4366 sa = (1 << sa) & 0x3F;
4367 sa = 0x01;
4368 sb = 0xfc;
4369 }
4370
4371 sitd->sitd.sitd_sched = htole32(
4372 EHCI_SITD_SET_SMASK(sa) |
4373 EHCI_SITD_SET_CMASK(sb)
4374 );
4375
4376 usb_syncmem(&sitd->dma, sitd->offs, sizeof(ehci_sitd_t),
4377 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4378 } /* End of frame */
4379
4380 sitd = exfer->ex_sitdend;
4381 sitd->sitd.sitd_trans |= htole32(EHCI_SITD_IOC);
4382
4383 usb_syncmem(&sitd->dma, sitd->offs + offsetof(ehci_sitd_t, sitd_trans),
4384 sizeof(sitd->sitd.sitd_trans),
4385 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4386
4387 usb_syncmem(&exfer->ex_xfer.ux_dmabuf, 0, total_length,
4388 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4389
4390 /*
4391 * Part 2: Transfer descriptors have now been set up, now they must
4392 * be scheduled into the periodic frame list. Erk. Not wanting to
4393 * complicate matters, transfer is denied if the transfer spans
4394 * more than the period frame list.
4395 */
4396
4397 mutex_enter(&sc->sc_lock);
4398
4399 /* Start inserting frames */
4400 if (epipe->isoc.cur_xfers > 0) {
4401 frindex = epipe->isoc.next_frame;
4402 } else {
4403 frindex = EOREAD4(sc, EHCI_FRINDEX);
4404 frindex = frindex >> 3; /* Erase microframe index */
4405 frindex += 2;
4406 }
4407
4408 if (frindex >= sc->sc_flsize)
4409 frindex &= (sc->sc_flsize - 1);
4410
4411 /* Whats the frame interval? */
4412 i = epipe->pipe.up_endpoint->ue_edesc->bInterval;
4413
4414 for (sitd = exfer->ex_sitdstart, j = 0; j < frames;
4415 j++, sitd = sitd->xfer_next) {
4416 KASSERT(sitd);
4417
4418 usb_syncmem(&sc->sc_fldma,
4419 sizeof(ehci_link_t) * frindex,
4420 sizeof(ehci_link_t),
4421 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
4422
4423 sitd->sitd.sitd_next = sc->sc_flist[frindex];
4424 if (sitd->sitd.sitd_next == 0)
4425 /*
4426 * FIXME: frindex table gets initialized to NULL
4427 * or EHCI_NULL?
4428 */
4429 sitd->sitd.sitd_next = EHCI_NULL;
4430
4431 usb_syncmem(&sitd->dma,
4432 sitd->offs + offsetof(ehci_sitd_t, sitd_next),
4433 sizeof(ehci_sitd_t),
4434 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4435
4436 sc->sc_flist[frindex] =
4437 htole32(EHCI_LINK_SITD | sitd->physaddr);
4438
4439 usb_syncmem(&sc->sc_fldma,
4440 sizeof(ehci_link_t) * frindex,
4441 sizeof(ehci_link_t),
4442 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4443
4444 sitd->frame_list.next = sc->sc_softsitds[frindex];
4445 sc->sc_softsitds[frindex] = sitd;
4446 if (sitd->frame_list.next != NULL)
4447 sitd->frame_list.next->frame_list.prev = sitd;
4448 sitd->slot = frindex;
4449 sitd->frame_list.prev = NULL;
4450
4451 frindex += i;
4452 if (frindex >= sc->sc_flsize)
4453 frindex -= sc->sc_flsize;
4454 }
4455
4456 epipe->isoc.cur_xfers++;
4457 epipe->isoc.next_frame = frindex;
4458
4459 ehci_add_intr_list(sc, exfer);
4460 xfer->ux_status = USBD_IN_PROGRESS;
4461
4462 mutex_exit(&sc->sc_lock);
4463
4464 return USBD_IN_PROGRESS;
4465}
4466
4467Static void
4468ehci_device_fs_isoc_abort(struct usbd_xfer *xfer)
4469{
4470 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4471
4472 DPRINTF("xfer = %p", xfer, 0, 0, 0);
4473 ehci_abort_isoc_xfer(xfer, USBD_CANCELLED);
4474}
4475
4476Static void
4477ehci_device_fs_isoc_close(struct usbd_pipe *pipe)
4478{
4479 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4480
4481 DPRINTF("nothing in the pipe to free?", 0, 0, 0, 0);
4482}
4483
4484Static void
4485ehci_device_fs_isoc_done(struct usbd_xfer *xfer)
4486{
4487 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4488 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4489 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
4490
4491 KASSERT(mutex_owned(&sc->sc_lock));
4492
4493 epipe->isoc.cur_xfers--;
4494 ehci_remove_sitd_chain(sc, exfer->ex_itdstart);
4495
4496 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4497 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
4498}
4499
4500
4501/************************/
4502
4503
4504Static int
4505ehci_device_isoc_init(struct usbd_xfer *xfer)
4506{
4507 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4508 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
4509 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4510 ehci_soft_itd_t *itd, *prev, *start, *stop;
4511 int i, j, k;
4512 int frames, ufrperframe;
4513 int err;
4514
4515 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4516
4517 start = NULL;
4518 prev = NULL;
4519 itd = NULL;
4520
4521 KASSERT(xfer->ux_nframes != 0);
4522 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
4523 KASSERT(exfer->ex_isdone);
4524
4525 exfer->ex_type = EX_ISOC;
4526
4527 /*
4528 * Step 1: Allocate and initialize itds, how many do we need?
4529 * One per transfer if interval >= 8 microframes, less if we use
4530 * multiple microframes per frame.
4531 */
4532 i = epipe->pipe.up_endpoint->ue_edesc->bInterval;
4533 if (i > 16 || i == 0) {
4534 /* Spec page 271 says intervals > 16 are invalid */
4535 DPRINTF("bInterval %d invalid", i, 0, 0, 0);
4536 return USBD_INVAL;
4537 }
4538
4539 ufrperframe = max(1, USB_UFRAMES_PER_FRAME / (1 << (i - 1)));
4540 frames = (xfer->ux_nframes + (ufrperframe - 1)) / ufrperframe;
4541
4542 for (i = 0, prev = NULL; i < frames; i++, prev = itd) {
4543 itd = ehci_alloc_itd(sc);
4544 if (itd == NULL) {
4545 err = ENOMEM;
4546 goto fail;
4547 }
4548
4549 if (prev != NULL) {
4550 /* Maybe not as it's updated by the scheduling? */
4551 prev->itd.itd_next =
4552 htole32(itd->physaddr | EHCI_LINK_ITD);
4553
4554 prev->xfer_next = itd;
4555 } else {
4556 start = itd;
4557 }
4558
4559 /*
4560 * Other special values
4561 */
4562 k = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
4563 itd->itd.itd_bufr[0] = htole32(
4564 EHCI_ITD_SET_EP(UE_GET_ADDR(k)) |
4565 EHCI_ITD_SET_DADDR(epipe->pipe.up_dev->ud_addr));
4566
4567 k = (UE_GET_DIR(epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress))
4568 ? 1 : 0;
4569 j = UGETW(epipe->pipe.up_endpoint->ue_edesc->wMaxPacketSize);
4570 itd->itd.itd_bufr[1] |= htole32(
4571 EHCI_ITD_SET_DIR(k) |
4572 EHCI_ITD_SET_MAXPKT(UE_GET_SIZE(j)));
4573
4574 /* FIXME: handle invalid trans - should be done in openpipe */
4575 itd->itd.itd_bufr[2] |=
4576 htole32(EHCI_ITD_SET_MULTI(UE_GET_TRANS(j)+1));
4577 } /* End of frame */
4578
4579 stop = itd;
4580 stop->xfer_next = NULL;
4581
4582 exfer->ex_itdstart = start;
4583 exfer->ex_itdend = stop;
4584
4585 return 0;
4586fail:
4587 mutex_enter(&sc->sc_lock);
4588 ehci_soft_itd_t *next;
4589 for (itd = start; itd; itd = next) {
4590 next = itd->xfer_next;
4591 ehci_free_itd_locked(sc, itd);
4592 }
4593 mutex_exit(&sc->sc_lock);
4594
4595 return err;
4596
4597}
4598
4599Static void
4600ehci_device_isoc_fini(struct usbd_xfer *xfer)
4601{
4602 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4603 struct ehci_xfer *ex = EHCI_XFER2EXFER(xfer);
4604
4605 KASSERT(ex->ex_type == EX_ISOC);
4606
4607 ehci_free_itd_chain(sc, ex->ex_itdstart);
4608}
4609
4610Static usbd_status
4611ehci_device_isoc_transfer(struct usbd_xfer *xfer)
4612{
4613 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4614 usbd_status __diagused err;
4615
4616 mutex_enter(&sc->sc_lock);
4617 err = usb_insert_transfer(xfer);
4618 mutex_exit(&sc->sc_lock);
4619
4620 KASSERT(err == USBD_NORMAL_COMPLETION);
4621
4622 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
4623 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4624 ehci_soft_itd_t *itd, *prev;
4625 usb_dma_t *dma_buf;
4626 int i, j;
4627 int frames, uframes, ufrperframe;
4628 int trans_count, offs, total_length;
4629 int frindex;
4630
4631 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4632
4633 prev = NULL;
4634 itd = NULL;
4635 trans_count = 0;
4636 total_length = 0;
4637
4638 DPRINTF("xfer %p flags %d", xfer, xfer->ux_flags, 0, 0);
4639
4640 if (sc->sc_dying)
4641 return USBD_IOERROR;
4642
4643 /*
4644 * To avoid complication, don't allow a request right now that'll span
4645 * the entire frame table. To within 4 frames, to allow some leeway
4646 * on either side of where the hc currently is.
4647 */
4648 if ((1 << (epipe->pipe.up_endpoint->ue_edesc->bInterval)) *
4649 xfer->ux_nframes >= (sc->sc_flsize - 4) * 8) {
4650 DPRINTF(
4651 "isoc descriptor spans entire frametable", 0, 0, 0, 0);
4652 printf("ehci: isoc descriptor requested that spans the entire frametable, too many frames\n");
4653 return USBD_INVAL;
4654 }
4655
4656 KASSERT(xfer->ux_nframes != 0 && xfer->ux_frlengths);
4657 KASSERT(!(xfer->ux_rqflags & URQ_REQUEST));
4658 KASSERT(exfer->ex_isdone);
4659#ifdef DIAGNOSTIC
4660 exfer->ex_isdone = false;
4661#endif
4662
4663 /*
4664 * Step 1: Re-Initialize itds
4665 */
4666
4667 i = epipe->pipe.up_endpoint->ue_edesc->bInterval;
4668 if (i > 16 || i == 0) {
4669 /* Spec page 271 says intervals > 16 are invalid */
4670 DPRINTF("bInterval %d invalid", i, 0, 0, 0);
4671 return USBD_INVAL;
4672 }
4673
4674 ufrperframe = max(1, USB_UFRAMES_PER_FRAME / (1 << (i - 1)));
4675 frames = (xfer->ux_nframes + (ufrperframe - 1)) / ufrperframe;
4676 uframes = USB_UFRAMES_PER_FRAME / ufrperframe;
4677
4678 if (frames == 0) {
4679 DPRINTF("frames == 0", 0, 0, 0, 0);
4680 return USBD_INVAL;
4681 }
4682
4683 dma_buf = &xfer->ux_dmabuf;
4684 offs = 0;
4685
4686 itd = exfer->ex_itdstart;
4687 for (i = 0; i < frames; i++, itd = itd->xfer_next) {
4688 int froffs = offs;
4689
4690 if (prev != NULL) {
4691 prev->itd.itd_next =
4692 htole32(itd->physaddr | EHCI_LINK_ITD);
4693 usb_syncmem(&prev->dma,
4694 prev->offs + offsetof(ehci_itd_t, itd_next),
4695 sizeof(prev->itd.itd_next), BUS_DMASYNC_POSTWRITE);
4696 prev->xfer_next = itd;
4697 }
4698
4699 /*
4700 * Step 1.5, initialize uframes
4701 */
4702 for (j = 0; j < EHCI_ITD_NUFRAMES; j += uframes) {
4703 /* Calculate which page in the list this starts in */
4704 int addr = DMAADDR(dma_buf, froffs);
4705 addr = EHCI_PAGE_OFFSET(addr);
4706 addr += (offs - froffs);
4707 addr = EHCI_PAGE(addr);
4708 addr /= EHCI_PAGE_SIZE;
4709
4710 /*
4711 * This gets the initial offset into the first page,
4712 * looks how far further along the current uframe
4713 * offset is. Works out how many pages that is.
4714 */
4715
4716 itd->itd.itd_ctl[j] = htole32 ( EHCI_ITD_ACTIVE |
4717 EHCI_ITD_SET_LEN(xfer->ux_frlengths[trans_count]) |
4718 EHCI_ITD_SET_PG(addr) |
4719 EHCI_ITD_SET_OFFS(EHCI_PAGE_OFFSET(DMAADDR(dma_buf,offs))));
4720
4721 total_length += xfer->ux_frlengths[trans_count];
4722 offs += xfer->ux_frlengths[trans_count];
4723 trans_count++;
4724
4725 if (trans_count >= xfer->ux_nframes) { /*Set IOC*/
4726 itd->itd.itd_ctl[j] |= htole32(EHCI_ITD_IOC);
4727 break;
4728 }
4729 }
4730
4731 /*
4732 * Step 1.75, set buffer pointers. To simplify matters, all
4733 * pointers are filled out for the next 7 hardware pages in
4734 * the dma block, so no need to worry what pages to cover
4735 * and what to not.
4736 */
4737
4738 for (j = 0; j < EHCI_ITD_NBUFFERS; j++) {
4739 /*
4740 * Don't try to lookup a page that's past the end
4741 * of buffer
4742 */
4743 int page_offs = EHCI_PAGE(froffs + (EHCI_PAGE_SIZE * j));
4744 if (page_offs >= dma_buf->udma_block->size)
4745 break;
4746
4747 uint64_t page = DMAADDR(dma_buf, page_offs);
4748 page = EHCI_PAGE(page);
4749 itd->itd.itd_bufr[j] = htole32(EHCI_ITD_SET_BPTR(page));
4750 itd->itd.itd_bufr_hi[j] = htole32(page >> 32);
4751 }
4752 /*
4753 * Other special values
4754 */
4755
4756 int k = epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress;
4757 itd->itd.itd_bufr[0] |= htole32(EHCI_ITD_SET_EP(UE_GET_ADDR(k)) |
4758 EHCI_ITD_SET_DADDR(epipe->pipe.up_dev->ud_addr));
4759
4760 k = (UE_GET_DIR(epipe->pipe.up_endpoint->ue_edesc->bEndpointAddress))
4761 ? 1 : 0;
4762 j = UGETW(epipe->pipe.up_endpoint->ue_edesc->wMaxPacketSize);
4763 itd->itd.itd_bufr[1] |= htole32(EHCI_ITD_SET_DIR(k) |
4764 EHCI_ITD_SET_MAXPKT(UE_GET_SIZE(j)));
4765
4766 /* FIXME: handle invalid trans */
4767 itd->itd.itd_bufr[2] |=
4768 htole32(EHCI_ITD_SET_MULTI(UE_GET_TRANS(j)+1));
4769
4770 usb_syncmem(&itd->dma, itd->offs, sizeof(ehci_itd_t),
4771 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4772
4773 prev = itd;
4774 } /* End of frame */
4775
4776 usb_syncmem(&exfer->ex_xfer.ux_dmabuf, 0, total_length,
4777 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4778
4779 /*
4780 * Part 2: Transfer descriptors have now been set up, now they must
4781 * be scheduled into the period frame list. Erk. Not wanting to
4782 * complicate matters, transfer is denied if the transfer spans
4783 * more than the period frame list.
4784 */
4785
4786 mutex_enter(&sc->sc_lock);
4787
4788 /* Start inserting frames */
4789 if (epipe->isoc.cur_xfers > 0) {
4790 frindex = epipe->isoc.next_frame;
4791 } else {
4792 frindex = EOREAD4(sc, EHCI_FRINDEX);
4793 frindex = frindex >> 3; /* Erase microframe index */
4794 frindex += 2;
4795 }
4796
4797 if (frindex >= sc->sc_flsize)
4798 frindex &= (sc->sc_flsize - 1);
4799
4800 /* What's the frame interval? */
4801 i = (1 << (epipe->pipe.up_endpoint->ue_edesc->bInterval - 1));
4802 if (i / USB_UFRAMES_PER_FRAME == 0)
4803 i = 1;
4804 else
4805 i /= USB_UFRAMES_PER_FRAME;
4806
4807 itd = exfer->ex_itdstart;
4808 for (j = 0; j < frames; j++) {
4809 KASSERTMSG(itd != NULL, "frame %d\n", j);
4810
4811 usb_syncmem(&sc->sc_fldma,
4812 sizeof(ehci_link_t) * frindex,
4813 sizeof(ehci_link_t),
4814 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
4815
4816 itd->itd.itd_next = sc->sc_flist[frindex];
4817 if (itd->itd.itd_next == 0)
4818 /*
4819 * FIXME: frindex table gets initialized to NULL
4820 * or EHCI_NULL?
4821 */
4822 itd->itd.itd_next = EHCI_NULL;
4823
4824 usb_syncmem(&itd->dma,
4825 itd->offs + offsetof(ehci_itd_t, itd_next),
4826 sizeof(itd->itd.itd_next),
4827 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4828
4829 sc->sc_flist[frindex] = htole32(EHCI_LINK_ITD | itd->physaddr);
4830
4831 usb_syncmem(&sc->sc_fldma,
4832 sizeof(ehci_link_t) * frindex,
4833 sizeof(ehci_link_t),
4834 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
4835
4836 itd->frame_list.next = sc->sc_softitds[frindex];
4837 sc->sc_softitds[frindex] = itd;
4838 if (itd->frame_list.next != NULL)
4839 itd->frame_list.next->frame_list.prev = itd;
4840 itd->slot = frindex;
4841 itd->frame_list.prev = NULL;
4842
4843 frindex += i;
4844 if (frindex >= sc->sc_flsize)
4845 frindex -= sc->sc_flsize;
4846
4847 itd = itd->xfer_next;
4848 }
4849
4850 epipe->isoc.cur_xfers++;
4851 epipe->isoc.next_frame = frindex;
4852
4853 ehci_add_intr_list(sc, exfer);
4854 xfer->ux_status = USBD_IN_PROGRESS;
4855
4856 mutex_exit(&sc->sc_lock);
4857
4858 return USBD_IN_PROGRESS;
4859}
4860
4861Static void
4862ehci_device_isoc_abort(struct usbd_xfer *xfer)
4863{
4864 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4865
4866 DPRINTF("xfer = %p", xfer, 0, 0, 0);
4867 ehci_abort_isoc_xfer(xfer, USBD_CANCELLED);
4868}
4869
4870Static void
4871ehci_device_isoc_close(struct usbd_pipe *pipe)
4872{
4873 EHCIHIST_FUNC(); EHCIHIST_CALLED();
4874
4875 DPRINTF("nothing in the pipe to free?", 0, 0, 0, 0);
4876}
4877
4878Static void
4879ehci_device_isoc_done(struct usbd_xfer *xfer)
4880{
4881 struct ehci_xfer *exfer = EHCI_XFER2EXFER(xfer);
4882 ehci_softc_t *sc = EHCI_XFER2SC(xfer);
4883 struct ehci_pipe *epipe = EHCI_XFER2EPIPE(xfer);
4884
4885 KASSERT(mutex_owned(&sc->sc_lock));
4886
4887 epipe->isoc.cur_xfers--;
4888 ehci_remove_itd_chain(sc, exfer->ex_sitdstart);
4889 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4890 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
4891}
4892