1/* $NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $ */
2
3/*
4 * Copyright (c) 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Lennart Augustsson (lennart@augustsson.net) at
9 * Carlstedt Research & Technology.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33/*
34 * USB DMA memory allocation.
35 * We need to allocate a lot of small (many 8 byte, some larger)
36 * memory blocks that can be used for DMA. Using the bus_dma
37 * routines directly would incur large overheads in space and time.
38 */
39
40#include <sys/cdefs.h>
41__KERNEL_RCSID(0, "$NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $");
42
43#ifdef _KERNEL_OPT
44#include "opt_usb.h"
45#endif
46
47#include <sys/param.h>
48#include <sys/systm.h>
49#include <sys/kernel.h>
50#include <sys/kmem.h>
51#include <sys/queue.h>
52#include <sys/device.h> /* for usbdivar.h */
53#include <sys/bus.h>
54#include <sys/cpu.h>
55#include <sys/once.h>
56
57#ifdef DIAGNOSTIC
58#include <sys/proc.h>
59#endif
60
61#include <dev/usb/usb.h>
62#include <dev/usb/usbdi.h>
63#include <dev/usb/usbdivar.h> /* just for usb_dma_t */
64#include <dev/usb/usb_mem.h>
65#include <dev/usb/usbhist.h>
66
67#define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D)
68#define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D)
69
70#define USB_MEM_SMALL roundup(64, CACHE_LINE_SIZE)
71#define USB_MEM_CHUNKS 64
72#define USB_MEM_BLOCK (USB_MEM_SMALL * USB_MEM_CHUNKS)
73
74/* This struct is overlayed on free fragments. */
75struct usb_frag_dma {
76 usb_dma_block_t *ufd_block;
77 u_int ufd_offs;
78 LIST_ENTRY(usb_frag_dma) ufd_next;
79};
80
81Static usbd_status usb_block_allocmem(bus_dma_tag_t, size_t, size_t,
82 usb_dma_block_t **, bool);
83Static void usb_block_freemem(usb_dma_block_t *);
84
85LIST_HEAD(usb_dma_block_qh, usb_dma_block);
86Static struct usb_dma_block_qh usb_blk_freelist =
87 LIST_HEAD_INITIALIZER(usb_blk_freelist);
88kmutex_t usb_blk_lock;
89
90#ifdef DEBUG
91Static struct usb_dma_block_qh usb_blk_fraglist =
92 LIST_HEAD_INITIALIZER(usb_blk_fraglist);
93Static struct usb_dma_block_qh usb_blk_fulllist =
94 LIST_HEAD_INITIALIZER(usb_blk_fulllist);
95#endif
96Static u_int usb_blk_nfree = 0;
97/* XXX should have different free list for different tags (for speed) */
98Static LIST_HEAD(, usb_frag_dma) usb_frag_freelist =
99 LIST_HEAD_INITIALIZER(usb_frag_freelist);
100
101Static int usb_mem_init(void);
102
103Static int
104usb_mem_init(void)
105{
106
107 mutex_init(&usb_blk_lock, MUTEX_DEFAULT, IPL_NONE);
108 return 0;
109}
110
111Static usbd_status
112usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
113 usb_dma_block_t **dmap, bool multiseg)
114{
115 usb_dma_block_t *b;
116 int error;
117
118 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
119 DPRINTFN(5, "size=%zu align=%zu", size, align, 0, 0);
120
121 ASSERT_SLEEPABLE();
122 KASSERT(size != 0);
123 KASSERT(mutex_owned(&usb_blk_lock));
124
125 /* First check the free list. */
126 LIST_FOREACH(b, &usb_blk_freelist, next) {
127 /* Don't allocate multiple segments to unwilling callers */
128 if (b->nsegs != 1 && !multiseg)
129 continue;
130 if (b->tag == tag && b->size >= size && b->align >= align) {
131 LIST_REMOVE(b, next);
132 usb_blk_nfree--;
133 *dmap = b;
134 DPRINTFN(6, "free list size=%zu", b->size, 0, 0, 0);
135 return USBD_NORMAL_COMPLETION;
136 }
137 }
138
139 DPRINTFN(6, "no free", 0, 0, 0, 0);
140 mutex_exit(&usb_blk_lock);
141
142 b = kmem_zalloc(sizeof(*b), KM_SLEEP);
143 if (b == NULL) {
144 goto fail;
145 }
146
147 b->tag = tag;
148 b->size = size;
149 b->align = align;
150
151 if (!multiseg)
152 /* Caller wants one segment */
153 b->nsegs = 1;
154 else
155 b->nsegs = (size + (PAGE_SIZE-1)) / PAGE_SIZE;
156
157 b->segs = kmem_alloc(b->nsegs * sizeof(*b->segs), KM_SLEEP);
158 if (b->segs == NULL) {
159 kmem_free(b, sizeof(*b));
160 goto fail;
161 }
162 b->nsegs_alloc = b->nsegs;
163
164 error = bus_dmamem_alloc(tag, b->size, align, 0,
165 b->segs, b->nsegs,
166 &b->nsegs, BUS_DMA_WAITOK);
167 if (error)
168 goto free0;
169
170 error = bus_dmamem_map(tag, b->segs, b->nsegs, b->size,
171 &b->kaddr, BUS_DMA_WAITOK|BUS_DMA_COHERENT);
172 if (error)
173 goto free1;
174
175 error = bus_dmamap_create(tag, b->size, b->nsegs, b->size,
176 0, BUS_DMA_WAITOK, &b->map);
177 if (error)
178 goto unmap;
179
180 error = bus_dmamap_load(tag, b->map, b->kaddr, b->size, NULL,
181 BUS_DMA_WAITOK);
182 if (error)
183 goto destroy;
184
185 *dmap = b;
186#ifdef USB_FRAG_DMA_WORKAROUND
187 memset(b->kaddr, 0, b->size);
188#endif
189 mutex_enter(&usb_blk_lock);
190
191 return USBD_NORMAL_COMPLETION;
192
193 destroy:
194 bus_dmamap_destroy(tag, b->map);
195 unmap:
196 bus_dmamem_unmap(tag, b->kaddr, b->size);
197 free1:
198 bus_dmamem_free(tag, b->segs, b->nsegs);
199 free0:
200 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs));
201 kmem_free(b, sizeof(*b));
202 fail:
203 mutex_enter(&usb_blk_lock);
204
205 return USBD_NOMEM;
206}
207
208#if 0
209void
210usb_block_real_freemem(usb_dma_block_t *b)
211{
212#ifdef DIAGNOSTIC
213 if (cpu_softintr_p() || cpu_intr_p()) {
214 printf("usb_block_real_freemem: in interrupt context\n");
215 return;
216 }
217#endif
218 bus_dmamap_unload(b->tag, b->map);
219 bus_dmamap_destroy(b->tag, b->map);
220 bus_dmamem_unmap(b->tag, b->kaddr, b->size);
221 bus_dmamem_free(b->tag, b->segs, b->nsegs);
222 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs));
223 kmem_free(b, sizeof(*b));
224}
225#endif
226
227#ifdef DEBUG
228static bool
229usb_valid_block_p(usb_dma_block_t *b, struct usb_dma_block_qh *qh)
230{
231 usb_dma_block_t *xb;
232 LIST_FOREACH(xb, qh, next) {
233 if (xb == b)
234 return true;
235 }
236 return false;
237}
238#endif
239
240/*
241 * Do not free the memory unconditionally since we might be called
242 * from an interrupt context and that is BAD.
243 * XXX when should we really free?
244 */
245Static void
246usb_block_freemem(usb_dma_block_t *b)
247{
248
249 KASSERT(mutex_owned(&usb_blk_lock));
250
251 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
252 DPRINTFN(6, "size=%zu", b->size, 0, 0, 0);
253#ifdef DEBUG
254 LIST_REMOVE(b, next);
255#endif
256 LIST_INSERT_HEAD(&usb_blk_freelist, b, next);
257 usb_blk_nfree++;
258}
259
260usbd_status
261usb_allocmem(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p)
262{
263
264 return usb_allocmem_flags(bus, size, align, p, 0);
265}
266
267usbd_status
268usb_allocmem_flags(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p,
269 int flags)
270{
271 bus_dma_tag_t tag = bus->ub_dmatag;
272 usbd_status err;
273 struct usb_frag_dma *f;
274 usb_dma_block_t *b;
275 int i;
276 static ONCE_DECL(init_control);
277 bool frag;
278
279 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
280
281 ASSERT_SLEEPABLE();
282
283 RUN_ONCE(&init_control, usb_mem_init);
284
285 frag = (flags & USBMALLOC_MULTISEG);
286
287 /* If the request is large then just use a full block. */
288 if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) {
289 DPRINTFN(1, "large alloc %d", size, 0, 0, 0);
290 size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1);
291 mutex_enter(&usb_blk_lock);
292 err = usb_block_allocmem(tag, size, align, &p->udma_block, frag);
293 if (!err) {
294#ifdef DEBUG
295 LIST_INSERT_HEAD(&usb_blk_fulllist, p->udma_block, next);
296#endif
297 p->udma_block->flags = USB_DMA_FULLBLOCK;
298 p->udma_offs = 0;
299 }
300 mutex_exit(&usb_blk_lock);
301 return err;
302 }
303
304 mutex_enter(&usb_blk_lock);
305 /* Check for free fragments. */
306 LIST_FOREACH(f, &usb_frag_freelist, ufd_next) {
307 KDASSERTMSG(usb_valid_block_p(f->ufd_block, &usb_blk_fraglist),
308 "%s: usb frag %p: unknown block pointer %p",
309 __func__, f, f->ufd_block);
310 if (f->ufd_block->tag == tag)
311 break;
312 }
313 if (f == NULL) {
314 DPRINTFN(1, "adding fragments", 0, 0, 0, 0);
315 err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL, &b,
316 false);
317 if (err) {
318 mutex_exit(&usb_blk_lock);
319 return err;
320 }
321#ifdef DEBUG
322 LIST_INSERT_HEAD(&usb_blk_fraglist, b, next);
323#endif
324 b->flags = 0;
325 for (i = 0; i < USB_MEM_BLOCK; i += USB_MEM_SMALL) {
326 f = (struct usb_frag_dma *)((char *)b->kaddr + i);
327 f->ufd_block = b;
328 f->ufd_offs = i;
329 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next);
330#ifdef USB_FRAG_DMA_WORKAROUND
331 i += 1 * USB_MEM_SMALL;
332#endif
333 }
334 f = LIST_FIRST(&usb_frag_freelist);
335 }
336 p->udma_block = f->ufd_block;
337 p->udma_offs = f->ufd_offs;
338#ifdef USB_FRAG_DMA_WORKAROUND
339 p->udma_offs += USB_MEM_SMALL;
340#endif
341 LIST_REMOVE(f, ufd_next);
342 mutex_exit(&usb_blk_lock);
343 DPRINTFN(5, "use frag=%p size=%d", f, size, 0, 0);
344
345 return USBD_NORMAL_COMPLETION;
346}
347
348void
349usb_freemem(struct usbd_bus *bus, usb_dma_t *p)
350{
351 struct usb_frag_dma *f;
352
353 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
354
355 mutex_enter(&usb_blk_lock);
356 if (p->udma_block->flags & USB_DMA_FULLBLOCK) {
357 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fulllist),
358 "%s: dma %p: invalid block pointer %p",
359 __func__, p, p->udma_block);
360 DPRINTFN(1, "large free", 0, 0, 0, 0);
361 usb_block_freemem(p->udma_block);
362 mutex_exit(&usb_blk_lock);
363 return;
364 }
365 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fraglist),
366 "%s: dma %p: invalid block pointer %p",
367 __func__, p, p->udma_block);
368 //usb_syncmem(p, 0, USB_MEM_SMALL, BUS_DMASYNC_POSTREAD);
369 f = KERNADDR(p, 0);
370#ifdef USB_FRAG_DMA_WORKAROUND
371 f = (void *)((uintptr_t)f - USB_MEM_SMALL);
372#endif
373 f->ufd_block = p->udma_block;
374 f->ufd_offs = p->udma_offs;
375#ifdef USB_FRAG_DMA_WORKAROUND
376 f->ufd_offs -= USB_MEM_SMALL;
377#endif
378 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next);
379 mutex_exit(&usb_blk_lock);
380 DPRINTFN(5, "frag=%p", f, 0, 0, 0);
381}
382
383bus_addr_t
384usb_dmaaddr(usb_dma_t *dma, unsigned int offset)
385{
386 unsigned int i;
387 bus_size_t seg_offs;
388
389 offset += dma->udma_offs;
390
391 KASSERTMSG(offset < dma->udma_block->size, "offset %d vs %zu", offset,
392 dma->udma_block->size);
393
394 if (dma->udma_block->nsegs == 1) {
395 KASSERT(dma->udma_block->map->dm_segs[0].ds_len > offset);
396 return dma->udma_block->map->dm_segs[0].ds_addr + offset;
397 }
398
399 /*
400 * Search for a bus_segment_t corresponding to this offset. With no
401 * record of the offset in the map to a particular dma_segment_t, we
402 * have to iterate from the start of the list each time. Could be
403 * improved
404 */
405 seg_offs = 0;
406 for (i = 0; i < dma->udma_block->nsegs; i++) {
407 if (seg_offs + dma->udma_block->map->dm_segs[i].ds_len > offset)
408 break;
409
410 seg_offs += dma->udma_block->map->dm_segs[i].ds_len;
411 }
412
413 KASSERT(i != dma->udma_block->nsegs);
414 offset -= seg_offs;
415 return dma->udma_block->map->dm_segs[i].ds_addr + offset;
416}
417
418void
419usb_syncmem(usb_dma_t *p, bus_addr_t offset, bus_size_t len, int ops)
420{
421
422 bus_dmamap_sync(p->udma_block->tag, p->udma_block->map, p->udma_offs + offset,
423 len, ops);
424}
425