1#ifndef _INTEL_RINGBUFFER_H_
2#define _INTEL_RINGBUFFER_H_
3
4#include <asm/bug.h>
5
6/*
7 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
8 * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
9 * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
10 *
11 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
12 * cacheline, the Head Pointer must not be greater than the Tail
13 * Pointer."
14 */
15#define I915_RING_FREE_SPACE 64
16
17struct intel_hw_status_page {
18 u32 *page_addr;
19 unsigned int gfx_addr;
20 struct drm_i915_gem_object *obj;
21};
22
23#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
24#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
25
26#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
27#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
28
29#define I915_READ_HEAD(ring) I915_READ(RING_HEAD((ring)->mmio_base))
30#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
31
32#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
33#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
34
35#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
36#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
37
38#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
39#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
40
41enum intel_ring_hangcheck_action {
42 HANGCHECK_IDLE = 0,
43 HANGCHECK_WAIT,
44 HANGCHECK_ACTIVE,
45 HANGCHECK_KICK,
46 HANGCHECK_HUNG,
47};
48
49#define HANGCHECK_SCORE_RING_HUNG 31
50
51struct intel_ring_hangcheck {
52 u64 acthd;
53 u32 seqno;
54 int score;
55 enum intel_ring_hangcheck_action action;
56 bool deadlock;
57};
58
59#ifdef __NetBSD__
60# define __ring_iomem
61#endif
62
63struct intel_ring_buffer {
64 const char *name;
65 enum intel_ring_id {
66 RCS = 0x0,
67 VCS,
68 BCS,
69 VECS,
70 } id;
71#define I915_NUM_RINGS 4
72 u32 mmio_base;
73#ifdef __NetBSD__
74 bus_space_tag_t bst;
75 bus_space_handle_t bsh;
76#else
77 void __iomem *virtual_start;
78#endif
79 struct drm_device *dev;
80 struct drm_i915_gem_object *obj;
81
82 u32 head;
83 u32 tail;
84 int space;
85 int size;
86 int effective_size;
87 struct intel_hw_status_page status_page;
88
89 /** We track the position of the requests in the ring buffer, and
90 * when each is retired we increment last_retired_head as the GPU
91 * must have finished processing the request and so we know we
92 * can advance the ringbuffer up to that position.
93 *
94 * last_retired_head is set to -1 after the value is consumed so
95 * we can detect new retirements.
96 */
97 u32 last_retired_head;
98
99 unsigned irq_refcount; /* protected by dev_priv->irq_lock */
100 u32 irq_enable_mask; /* bitmask to enable ring interrupt */
101 u32 trace_irq_seqno;
102 u32 sync_seqno[I915_NUM_RINGS-1];
103 bool __must_check (*irq_get)(struct intel_ring_buffer *ring);
104 void (*irq_put)(struct intel_ring_buffer *ring);
105
106 int (*init)(struct intel_ring_buffer *ring);
107
108 void (*write_tail)(struct intel_ring_buffer *ring,
109 u32 value);
110 int __must_check (*flush)(struct intel_ring_buffer *ring,
111 u32 invalidate_domains,
112 u32 flush_domains);
113 int (*add_request)(struct intel_ring_buffer *ring);
114 /* Some chipsets are not quite as coherent as advertised and need
115 * an expensive kick to force a true read of the up-to-date seqno.
116 * However, the up-to-date seqno is not always required and the last
117 * seen value is good enough. Note that the seqno will always be
118 * monotonic, even if not coherent.
119 */
120 u32 (*get_seqno)(struct intel_ring_buffer *ring,
121 bool lazy_coherency);
122 void (*set_seqno)(struct intel_ring_buffer *ring,
123 u32 seqno);
124 int (*dispatch_execbuffer)(struct intel_ring_buffer *ring,
125 u32 offset, u32 length,
126 unsigned flags);
127#define I915_DISPATCH_SECURE 0x1
128#define I915_DISPATCH_PINNED 0x2
129 void (*cleanup)(struct intel_ring_buffer *ring);
130 int (*sync_to)(struct intel_ring_buffer *ring,
131 struct intel_ring_buffer *to,
132 u32 seqno);
133
134 /* our mbox written by others */
135 u32 semaphore_register[I915_NUM_RINGS];
136 /* mboxes this ring signals to */
137 u32 signal_mbox[I915_NUM_RINGS];
138
139 /**
140 * List of objects currently involved in rendering from the
141 * ringbuffer.
142 *
143 * Includes buffers having the contents of their GPU caches
144 * flushed, not necessarily primitives. last_rendering_seqno
145 * represents when the rendering involved will be completed.
146 *
147 * A reference is held on the buffer while on this list.
148 */
149 struct list_head active_list;
150
151 /**
152 * List of breadcrumbs associated with GPU requests currently
153 * outstanding.
154 */
155 struct list_head request_list;
156
157 /**
158 * Do we have some not yet emitted requests outstanding?
159 */
160 struct drm_i915_gem_request *preallocated_lazy_request;
161 u32 outstanding_lazy_seqno;
162 bool gpu_caches_dirty;
163 bool fbc_dirty;
164
165#ifdef __NetBSD__
166 drm_waitqueue_t irq_queue;
167#else
168 wait_queue_head_t irq_queue;
169#endif
170
171 /**
172 * Do an explicit TLB flush before MI_SET_CONTEXT
173 */
174 bool itlb_before_ctx_switch;
175 struct i915_hw_context *default_context;
176 struct i915_hw_context *last_context;
177
178 struct intel_ring_hangcheck hangcheck;
179
180 struct {
181 struct drm_i915_gem_object *obj;
182 u32 gtt_offset;
183 volatile u32 *cpu_page;
184 } scratch;
185
186 /*
187 * Tables of commands the command parser needs to know about
188 * for this ring.
189 */
190 const struct drm_i915_cmd_table *cmd_tables;
191 int cmd_table_count;
192
193 /*
194 * Table of registers allowed in commands that read/write registers.
195 */
196 const u32 *reg_table;
197 int reg_count;
198
199 /*
200 * Table of registers allowed in commands that read/write registers, but
201 * only from the DRM master.
202 */
203 const u32 *master_reg_table;
204 int master_reg_count;
205
206 /*
207 * Returns the bitmask for the length field of the specified command.
208 * Return 0 for an unrecognized/invalid command.
209 *
210 * If the command parser finds an entry for a command in the ring's
211 * cmd_tables, it gets the command's length based on the table entry.
212 * If not, it calls this function to determine the per-ring length field
213 * encoding for the command (i.e. certain opcode ranges use certain bits
214 * to encode the command length in the header).
215 */
216 u32 (*get_cmd_length_mask)(u32 cmd_header);
217};
218
219static inline bool
220intel_ring_initialized(struct intel_ring_buffer *ring)
221{
222 return ring->obj != NULL;
223}
224
225static inline unsigned
226intel_ring_flag(struct intel_ring_buffer *ring)
227{
228 return 1 << ring->id;
229}
230
231static inline u32
232intel_ring_sync_index(struct intel_ring_buffer *ring,
233 struct intel_ring_buffer *other)
234{
235 int idx;
236
237 /*
238 * cs -> 0 = vcs, 1 = bcs
239 * vcs -> 0 = bcs, 1 = cs,
240 * bcs -> 0 = cs, 1 = vcs.
241 */
242
243 idx = (other - ring) - 1;
244 if (idx < 0)
245 idx += I915_NUM_RINGS;
246
247 return idx;
248}
249
250static inline u32
251intel_read_status_page(struct intel_ring_buffer *ring,
252 int reg)
253{
254 /* Ensure that the compiler doesn't optimize away the load. */
255 barrier();
256 return ring->status_page.page_addr[reg];
257}
258
259static inline void
260intel_write_status_page(struct intel_ring_buffer *ring,
261 int reg, u32 value)
262{
263 ring->status_page.page_addr[reg] = value;
264}
265
266/**
267 * Reads a dword out of the status page, which is written to from the command
268 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
269 * MI_STORE_DATA_IMM.
270 *
271 * The following dwords have a reserved meaning:
272 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
273 * 0x04: ring 0 head pointer
274 * 0x05: ring 1 head pointer (915-class)
275 * 0x06: ring 2 head pointer (915-class)
276 * 0x10-0x1b: Context status DWords (GM45)
277 * 0x1f: Last written status offset. (GM45)
278 *
279 * The area from dword 0x20 to 0x3ff is available for driver usage.
280 */
281#define I915_GEM_HWS_INDEX 0x20
282#define I915_GEM_HWS_SCRATCH_INDEX 0x30
283#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
284
285void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring);
286
287int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n);
288int __must_check intel_ring_cacheline_align(struct intel_ring_buffer *ring);
289static inline void intel_ring_emit(struct intel_ring_buffer *ring,
290 u32 data)
291{
292#ifdef __NetBSD__
293 bus_space_write_4(ring->bst, ring->bsh, ring->tail, data);
294#else
295 iowrite32(data, ring->virtual_start + ring->tail);
296#endif
297 ring->tail += 4;
298}
299static inline void intel_ring_advance(struct intel_ring_buffer *ring)
300{
301 ring->tail &= ring->size - 1;
302}
303void __intel_ring_advance(struct intel_ring_buffer *ring);
304
305int __must_check intel_ring_idle(struct intel_ring_buffer *ring);
306void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno);
307int intel_ring_flush_all_caches(struct intel_ring_buffer *ring);
308int intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring);
309
310int intel_init_render_ring_buffer(struct drm_device *dev);
311int intel_init_bsd_ring_buffer(struct drm_device *dev);
312int intel_init_blt_ring_buffer(struct drm_device *dev);
313int intel_init_vebox_ring_buffer(struct drm_device *dev);
314
315u64 intel_ring_get_active_head(struct intel_ring_buffer *ring);
316void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
317
318static inline u32 intel_ring_get_tail(struct intel_ring_buffer *ring)
319{
320 return ring->tail;
321}
322
323static inline u32 intel_ring_get_seqno(struct intel_ring_buffer *ring)
324{
325 BUG_ON(ring->outstanding_lazy_seqno == 0);
326 return ring->outstanding_lazy_seqno;
327}
328
329static inline void i915_trace_irq_get(struct intel_ring_buffer *ring, u32 seqno)
330{
331 if (ring->trace_irq_seqno == 0 && ring->irq_get(ring))
332 ring->trace_irq_seqno = seqno;
333}
334
335/* DRI warts */
336int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size);
337
338#endif /* _INTEL_RINGBUFFER_H_ */
339