1/* $NetBSD: nouveau_subdev_vm_nvc0.c,v 1.1.1.1 2014/08/06 12:36:32 riastradh Exp $ */
2
3/*
4 * Copyright 2010 Red Hat Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Ben Skeggs
25 */
26
27#include <sys/cdefs.h>
28__KERNEL_RCSID(0, "$NetBSD: nouveau_subdev_vm_nvc0.c,v 1.1.1.1 2014/08/06 12:36:32 riastradh Exp $");
29
30#include <core/device.h>
31#include <core/gpuobj.h>
32
33#include <subdev/timer.h>
34#include <subdev/fb.h>
35#include <subdev/vm.h>
36#include <subdev/ltcg.h>
37#include <subdev/bar.h>
38
39struct nvc0_vmmgr_priv {
40 struct nouveau_vmmgr base;
41};
42
43
44/* Map from compressed to corresponding uncompressed storage type.
45 * The value 0xff represents an invalid storage type.
46 */
47const u8 nvc0_pte_storage_type_map[256] =
48{
49 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0x01, /* 0x00 */
50 0x01, 0x01, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff,
51 0xff, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff, 0x11, /* 0x10 */
52 0x11, 0x11, 0x11, 0xff, 0xff, 0xff, 0xff, 0xff,
53 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0x27, /* 0x20 */
54 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
55 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30 */
56 0xff, 0xff, 0x26, 0x27, 0x28, 0x29, 0x26, 0x27,
57 0x28, 0x29, 0xff, 0xff, 0xff, 0xff, 0x46, 0xff, /* 0x40 */
58 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
59 0xff, 0x46, 0x46, 0x46, 0x46, 0xff, 0xff, 0xff, /* 0x50 */
60 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
61 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60 */
62 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
63 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70 */
64 0xff, 0xff, 0xff, 0x7b, 0xff, 0xff, 0xff, 0xff,
65 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0x7b, /* 0x80 */
66 0x7b, 0x7b, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xff,
67 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90 */
68 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
69 0xff, 0xff, 0xff, 0x8b, 0x8c, 0x8d, 0x8e, 0xa7, /* 0xa0 */
70 0xa8, 0xa9, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff,
71 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0 */
72 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7,
73 0xa8, 0xa9, 0xaa, 0xc3, 0xff, 0xff, 0xff, 0xff, /* 0xc0 */
74 0xff, 0xff, 0xff, 0xff, 0xfe, 0xfe, 0xc3, 0xc3,
75 0xc3, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0 */
76 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe,
77 0xfe, 0xff, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xff, /* 0xe0 */
78 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xfe, 0xff,
79 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0 */
80 0xfe, 0xfe, 0xfe, 0xfe, 0xff, 0xfd, 0xfe, 0xff
81};
82
83
84static void
85nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 index,
86 struct nouveau_gpuobj *pgt[2])
87{
88 u32 pde[2] = { 0, 0 };
89
90 if (pgt[0])
91 pde[1] = 0x00000001 | (pgt[0]->addr >> 8);
92 if (pgt[1])
93 pde[0] = 0x00000001 | (pgt[1]->addr >> 8);
94
95 nv_wo32(pgd, (index * 8) + 0, pde[0]);
96 nv_wo32(pgd, (index * 8) + 4, pde[1]);
97}
98
99static inline u64
100nvc0_vm_addr(struct nouveau_vma *vma, u64 phys, u32 memtype, u32 target)
101{
102 phys >>= 8;
103
104 phys |= 0x00000001; /* present */
105 if (vma->access & NV_MEM_ACCESS_SYS)
106 phys |= 0x00000002;
107
108 phys |= ((u64)target << 32);
109 phys |= ((u64)memtype << 36);
110
111 return phys;
112}
113
114static void
115nvc0_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
116 struct nouveau_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
117{
118 u64 next = 1 << (vma->node->type - 8);
119
120 phys = nvc0_vm_addr(vma, phys, mem->memtype, 0);
121 pte <<= 3;
122
123 if (mem->tag) {
124 struct nouveau_ltcg *ltcg =
125 nouveau_ltcg(vma->vm->vmm->base.base.parent);
126 u32 tag = mem->tag->offset + (delta >> 17);
127 phys |= (u64)tag << (32 + 12);
128 next |= (u64)1 << (32 + 12);
129 ltcg->tags_clear(ltcg, tag, cnt);
130 }
131
132 while (cnt--) {
133 nv_wo32(pgt, pte + 0, lower_32_bits(phys));
134 nv_wo32(pgt, pte + 4, upper_32_bits(phys));
135 phys += next;
136 pte += 8;
137 }
138}
139
140static void
141nvc0_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
142 struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
143{
144 u32 target = (vma->access & NV_MEM_ACCESS_NOSNOOP) ? 7 : 5;
145 /* compressed storage types are invalid for system memory */
146 u32 memtype = nvc0_pte_storage_type_map[mem->memtype & 0xff];
147
148 pte <<= 3;
149 while (cnt--) {
150 u64 phys = nvc0_vm_addr(vma, *list++, memtype, target);
151 nv_wo32(pgt, pte + 0, lower_32_bits(phys));
152 nv_wo32(pgt, pte + 4, upper_32_bits(phys));
153 pte += 8;
154 }
155}
156
157static void
158nvc0_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
159{
160 pte <<= 3;
161 while (cnt--) {
162 nv_wo32(pgt, pte + 0, 0x00000000);
163 nv_wo32(pgt, pte + 4, 0x00000000);
164 pte += 8;
165 }
166}
167
168static void
169nvc0_vm_flush(struct nouveau_vm *vm)
170{
171 struct nvc0_vmmgr_priv *priv = (void *)vm->vmm;
172 struct nouveau_bar *bar = nouveau_bar(priv);
173 struct nouveau_vm_pgd *vpgd;
174 u32 type;
175
176 bar->flush(bar);
177
178 type = 0x00000001; /* PAGE_ALL */
179 if (atomic_read(&vm->engref[NVDEV_SUBDEV_BAR]))
180 type |= 0x00000004; /* HUB_ONLY */
181
182 mutex_lock(&nv_subdev(priv)->mutex);
183 list_for_each_entry(vpgd, &vm->pgd_list, head) {
184 /* looks like maybe a "free flush slots" counter, the
185 * faster you write to 0x100cbc to more it decreases
186 */
187 if (!nv_wait_ne(priv, 0x100c80, 0x00ff0000, 0x00000000)) {
188 nv_error(priv, "vm timeout 0: 0x%08x %d\n",
189 nv_rd32(priv, 0x100c80), type);
190 }
191
192 nv_wr32(priv, 0x100cb8, vpgd->obj->addr >> 8);
193 nv_wr32(priv, 0x100cbc, 0x80000000 | type);
194
195 /* wait for flush to be queued? */
196 if (!nv_wait(priv, 0x100c80, 0x00008000, 0x00008000)) {
197 nv_error(priv, "vm timeout 1: 0x%08x %d\n",
198 nv_rd32(priv, 0x100c80), type);
199 }
200 }
201 mutex_unlock(&nv_subdev(priv)->mutex);
202}
203
204static int
205nvc0_vm_create(struct nouveau_vmmgr *vmm, u64 offset, u64 length,
206 u64 mm_offset, struct nouveau_vm **pvm)
207{
208 return nouveau_vm_create(vmm, offset, length, mm_offset, 4096, pvm);
209}
210
211static int
212nvc0_vmmgr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
213 struct nouveau_oclass *oclass, void *data, u32 size,
214 struct nouveau_object **pobject)
215{
216 struct nvc0_vmmgr_priv *priv;
217 int ret;
218
219 ret = nouveau_vmmgr_create(parent, engine, oclass, "VM", "vm", &priv);
220 *pobject = nv_object(priv);
221 if (ret)
222 return ret;
223
224 priv->base.limit = 1ULL << 40;
225 priv->base.dma_bits = 40;
226 priv->base.pgt_bits = 27 - 12;
227 priv->base.spg_shift = 12;
228 priv->base.lpg_shift = 17;
229 priv->base.create = nvc0_vm_create;
230 priv->base.map_pgt = nvc0_vm_map_pgt;
231 priv->base.map = nvc0_vm_map;
232 priv->base.map_sg = nvc0_vm_map_sg;
233 priv->base.unmap = nvc0_vm_unmap;
234 priv->base.flush = nvc0_vm_flush;
235 return 0;
236}
237
238struct nouveau_oclass
239nvc0_vmmgr_oclass = {
240 .handle = NV_SUBDEV(VM, 0xc0),
241 .ofuncs = &(struct nouveau_ofuncs) {
242 .ctor = nvc0_vmmgr_ctor,
243 .dtor = _nouveau_vmmgr_dtor,
244 .init = _nouveau_vmmgr_init,
245 .fini = _nouveau_vmmgr_fini,
246 },
247};
248