1/**
2 * \file drm_irq.c
3 * IRQ support
4 *
5 * \author Rickard E. (Rik) Faith <faith@valinux.com>
6 * \author Gareth Hughes <gareth@valinux.com>
7 */
8
9/*
10 * Created: Fri Mar 19 14:30:16 1999 by faith@valinux.com
11 *
12 * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
13 * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
14 * All Rights Reserved.
15 *
16 * Permission is hereby granted, free of charge, to any person obtaining a
17 * copy of this software and associated documentation files (the "Software"),
18 * to deal in the Software without restriction, including without limitation
19 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
20 * and/or sell copies of the Software, and to permit persons to whom the
21 * Software is furnished to do so, subject to the following conditions:
22 *
23 * The above copyright notice and this permission notice (including the next
24 * paragraph) shall be included in all copies or substantial portions of the
25 * Software.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
30 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
31 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
32 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
33 * OTHER DEALINGS IN THE SOFTWARE.
34 */
35
36#include <drm/drmP.h>
37#include "drm_trace.h"
38
39#include <linux/interrupt.h> /* For task queue support */
40#include <linux/slab.h>
41
42#include <linux/vgaarb.h>
43#include <linux/export.h>
44
45#include <linux/atomic.h>
46#include <linux/ktime.h>
47#include <linux/math64.h>
48#include <linux/preempt.h>
49#include <linux/sched.h>
50
51#include <asm/bug.h>
52#include <asm/param.h>
53
54#ifdef __NetBSD__ /* XXX hurk -- selnotify &c. */
55#include <sys/poll.h>
56#include <sys/select.h>
57#endif
58
59/* Access macro for slots in vblank timestamp ringbuffer. */
60#define vblanktimestamp(dev, crtc, count) \
61 ((dev)->vblank[crtc].time[(count) % DRM_VBLANKTIME_RBSIZE])
62
63/* Retry timestamp calculation up to 3 times to satisfy
64 * drm_timestamp_precision before giving up.
65 */
66#define DRM_TIMESTAMP_MAXRETRIES 3
67
68/* Threshold in nanoseconds for detection of redundant
69 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
70 */
71#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
72
73/**
74 * Get interrupt from bus id.
75 *
76 * \param inode device inode.
77 * \param file_priv DRM file private.
78 * \param cmd command.
79 * \param arg user argument, pointing to a drm_irq_busid structure.
80 * \return zero on success or a negative number on failure.
81 *
82 * Finds the PCI device with the specified bus id and gets its IRQ number.
83 * This IOCTL is deprecated, and will now return EINVAL for any busid not equal
84 * to that of the device that this DRM instance attached to.
85 */
86int drm_irq_by_busid(struct drm_device *dev, void *data,
87 struct drm_file *file_priv)
88{
89 struct drm_irq_busid *p = data;
90
91 if (!dev->driver->bus->irq_by_busid)
92 return -EINVAL;
93
94 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
95 return -EINVAL;
96
97 return dev->driver->bus->irq_by_busid(dev, p);
98}
99
100/*
101 * Clear vblank timestamp buffer for a crtc.
102 */
103static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
104{
105 memset(dev->vblank[crtc].time, 0, sizeof(dev->vblank[crtc].time));
106}
107
108/*
109 * Disable vblank irq's on crtc, make sure that last vblank count
110 * of hardware and corresponding consistent software vblank counter
111 * are preserved, even if there are any spurious vblank irq's after
112 * disable.
113 */
114static void vblank_disable_and_save(struct drm_device *dev, int crtc)
115{
116 unsigned long irqflags;
117 u32 vblcount;
118 s64 diff_ns;
119 int vblrc;
120 struct timeval tvblank;
121 int count = DRM_TIMESTAMP_MAXRETRIES;
122
123 /* Prevent vblank irq processing while disabling vblank irqs,
124 * so no updates of timestamps or count can happen after we've
125 * disabled. Needed to prevent races in case of delayed irq's.
126 */
127 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
128
129 dev->driver->disable_vblank(dev, crtc);
130 dev->vblank[crtc].enabled = false;
131
132 /* No further vblank irq's will be processed after
133 * this point. Get current hardware vblank count and
134 * vblank timestamp, repeat until they are consistent.
135 *
136 * FIXME: There is still a race condition here and in
137 * drm_update_vblank_count() which can cause off-by-one
138 * reinitialization of software vblank counter. If gpu
139 * vblank counter doesn't increment exactly at the leading
140 * edge of a vblank interval, then we can lose 1 count if
141 * we happen to execute between start of vblank and the
142 * delayed gpu counter increment.
143 */
144 do {
145 dev->vblank[crtc].last = dev->driver->get_vblank_counter(dev, crtc);
146 vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
147 } while (dev->vblank[crtc].last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
148
149 if (!count)
150 vblrc = 0;
151
152 /* Compute time difference to stored timestamp of last vblank
153 * as updated by last invocation of drm_handle_vblank() in vblank irq.
154 */
155 vblcount = atomic_read(&dev->vblank[crtc].count);
156 diff_ns = timeval_to_ns(&tvblank) -
157 timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
158
159 /* If there is at least 1 msec difference between the last stored
160 * timestamp and tvblank, then we are currently executing our
161 * disable inside a new vblank interval, the tvblank timestamp
162 * corresponds to this new vblank interval and the irq handler
163 * for this vblank didn't run yet and won't run due to our disable.
164 * Therefore we need to do the job of drm_handle_vblank() and
165 * increment the vblank counter by one to account for this vblank.
166 *
167 * Skip this step if there isn't any high precision timestamp
168 * available. In that case we can't account for this and just
169 * hope for the best.
170 */
171 if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) {
172 atomic_inc(&dev->vblank[crtc].count);
173 smp_mb__after_atomic_inc();
174 }
175
176 /* Invalidate all timestamps while vblank irq's are off. */
177 clear_vblank_timestamps(dev, crtc);
178
179 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
180}
181
182static void vblank_disable_fn(unsigned long arg)
183{
184 struct drm_device *dev = (struct drm_device *)arg;
185 unsigned long irqflags;
186 int i;
187
188 if (!dev->vblank_disable_allowed)
189 return;
190
191 for (i = 0; i < dev->num_crtcs; i++) {
192 spin_lock_irqsave(&dev->vbl_lock, irqflags);
193 if (atomic_read(&dev->vblank[i].refcount) == 0 &&
194 dev->vblank[i].enabled) {
195 DRM_DEBUG("disabling vblank on crtc %d\n", i);
196 vblank_disable_and_save(dev, i);
197 }
198 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
199 }
200}
201
202void drm_vblank_cleanup(struct drm_device *dev)
203{
204 /* Bail if the driver didn't call drm_vblank_init() */
205 if (dev->num_crtcs == 0)
206 return;
207
208 del_timer_sync(&dev->vblank_disable_timer);
209#ifdef __NetBSD__
210 teardown_timer(&dev->vblank_disable_timer);
211#endif
212
213 vblank_disable_fn((unsigned long)dev);
214
215#ifdef __NetBSD__
216 {
217 unsigned int i;
218 for (i = 0; i < dev->num_crtcs; i++)
219 DRM_DESTROY_WAITQUEUE(&dev->vblank[i].queue);
220 }
221#endif
222
223 kfree(dev->vblank);
224
225 dev->num_crtcs = 0;
226
227#ifdef __NetBSD__
228 spin_lock_destroy(&dev->vblank_time_lock);
229 spin_lock_destroy(&dev->vbl_lock);
230#endif
231}
232EXPORT_SYMBOL(drm_vblank_cleanup);
233
234int drm_vblank_init(struct drm_device *dev, int num_crtcs)
235{
236 int i, ret = -ENOMEM;
237
238 setup_timer(&dev->vblank_disable_timer, vblank_disable_fn,
239 (unsigned long)dev);
240 spin_lock_init(&dev->vbl_lock);
241 spin_lock_init(&dev->vblank_time_lock);
242
243 dev->num_crtcs = num_crtcs;
244
245 dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
246 if (!dev->vblank)
247 goto err;
248
249 for (i = 0; i < num_crtcs; i++)
250#ifdef __NetBSD__
251 DRM_INIT_WAITQUEUE(&dev->vblank[i].queue, "drmvblnq");
252#else
253 init_waitqueue_head(&dev->vblank[i].queue);
254#endif
255
256 DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
257
258 /* Driver specific high-precision vblank timestamping supported? */
259 if (dev->driver->get_vblank_timestamp)
260 DRM_INFO("Driver supports precise vblank timestamp query.\n");
261 else
262 DRM_INFO("No driver support for vblank timestamp query.\n");
263
264 dev->vblank_disable_allowed = false;
265
266 return 0;
267
268err:
269 drm_vblank_cleanup(dev);
270 return ret;
271}
272EXPORT_SYMBOL(drm_vblank_init);
273
274static void drm_irq_vgaarb_nokms(void *cookie, bool state)
275{
276 struct drm_device *dev = cookie;
277
278 if (dev->driver->vgaarb_irq) {
279 dev->driver->vgaarb_irq(dev, state);
280 return;
281 }
282
283 if (!dev->irq_enabled)
284 return;
285
286 if (state) {
287 if (dev->driver->irq_uninstall)
288 dev->driver->irq_uninstall(dev);
289 } else {
290 if (dev->driver->irq_preinstall)
291 dev->driver->irq_preinstall(dev);
292 if (dev->driver->irq_postinstall)
293 dev->driver->irq_postinstall(dev);
294 }
295}
296
297/**
298 * Install IRQ handler.
299 *
300 * \param dev DRM device.
301 *
302 * Initializes the IRQ related data. Installs the handler, calling the driver
303 * \c irq_preinstall() and \c irq_postinstall() functions
304 * before and after the installation.
305 */
306int drm_irq_install(struct drm_device *dev)
307{
308 int ret;
309 unsigned long sh_flags = 0;
310 const char *irqname;
311
312 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
313 return -EINVAL;
314
315 if (drm_dev_to_irq(dev) == 0)
316 return -EINVAL;
317
318 mutex_lock(&dev->struct_mutex);
319
320 /* Driver must have been initialized */
321 if (!dev->dev_private) {
322 mutex_unlock(&dev->struct_mutex);
323 return -EINVAL;
324 }
325
326 if (dev->irq_enabled) {
327 mutex_unlock(&dev->struct_mutex);
328 return -EBUSY;
329 }
330 dev->irq_enabled = true;
331 mutex_unlock(&dev->struct_mutex);
332
333 DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));
334
335 /* Before installing handler */
336 if (dev->driver->irq_preinstall)
337 dev->driver->irq_preinstall(dev);
338
339 /* Install handler */
340 if (drm_core_check_feature(dev, DRIVER_IRQ_SHARED))
341 sh_flags = IRQF_SHARED;
342
343 if (dev->devname)
344 irqname = dev->devname;
345 else
346 irqname = dev->driver->name;
347
348#ifdef __NetBSD__
349 ret = (*dev->driver->bus->irq_install)(dev, dev->driver->irq_handler,
350 sh_flags, irqname, dev, &dev->irq_cookie);
351#else
352 ret = request_irq(drm_dev_to_irq(dev), dev->driver->irq_handler,
353 sh_flags, irqname, dev);
354#endif
355
356 if (ret < 0) {
357 mutex_lock(&dev->struct_mutex);
358 dev->irq_enabled = false;
359 mutex_unlock(&dev->struct_mutex);
360 return ret;
361 }
362
363 if (!drm_core_check_feature(dev, DRIVER_MODESET))
364 vga_client_register(dev->pdev, (void *)dev, drm_irq_vgaarb_nokms, NULL);
365
366 /* After installing handler */
367 if (dev->driver->irq_postinstall)
368 ret = dev->driver->irq_postinstall(dev);
369
370 if (ret < 0) {
371 mutex_lock(&dev->struct_mutex);
372 dev->irq_enabled = false;
373 mutex_unlock(&dev->struct_mutex);
374 if (!drm_core_check_feature(dev, DRIVER_MODESET))
375 vga_client_register(dev->pdev, NULL, NULL, NULL);
376#ifdef __NetBSD__
377 (*dev->driver->bus->irq_uninstall)(dev, dev->irq_cookie);
378#else
379 free_irq(drm_dev_to_irq(dev), dev);
380#endif
381 }
382
383 return ret;
384}
385EXPORT_SYMBOL(drm_irq_install);
386
387/**
388 * Uninstall the IRQ handler.
389 *
390 * \param dev DRM device.
391 *
392 * Calls the driver's \c irq_uninstall() function, and stops the irq.
393 */
394int drm_irq_uninstall(struct drm_device *dev)
395{
396 unsigned long irqflags;
397 bool irq_enabled;
398 int i;
399
400 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
401 return -EINVAL;
402
403 mutex_lock(&dev->struct_mutex);
404 irq_enabled = dev->irq_enabled;
405 dev->irq_enabled = false;
406 mutex_unlock(&dev->struct_mutex);
407
408 /*
409 * Wake up any waiters so they don't hang.
410 */
411 if (dev->num_crtcs) {
412 spin_lock_irqsave(&dev->vbl_lock, irqflags);
413 for (i = 0; i < dev->num_crtcs; i++) {
414#ifdef __NetBSD__
415 DRM_SPIN_WAKEUP_ONE(&dev->vblank[i].queue,
416 &dev->vbl_lock);
417#else
418 wake_up(&dev->vblank[i].queue);
419#endif
420 dev->vblank[i].enabled = false;
421 dev->vblank[i].last =
422 dev->driver->get_vblank_counter(dev, i);
423 }
424 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
425 }
426
427 if (!irq_enabled)
428 return -EINVAL;
429
430 DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));
431
432 if (!drm_core_check_feature(dev, DRIVER_MODESET))
433 vga_client_register(dev->pdev, NULL, NULL, NULL);
434
435 if (dev->driver->irq_uninstall)
436 dev->driver->irq_uninstall(dev);
437
438#ifdef __NetBSD__
439 (*dev->driver->bus->irq_uninstall)(dev, dev->irq_cookie);
440#else
441 free_irq(drm_dev_to_irq(dev), dev);
442#endif
443
444 return 0;
445}
446EXPORT_SYMBOL(drm_irq_uninstall);
447
448/**
449 * IRQ control ioctl.
450 *
451 * \param inode device inode.
452 * \param file_priv DRM file private.
453 * \param cmd command.
454 * \param arg user argument, pointing to a drm_control structure.
455 * \return zero on success or a negative number on failure.
456 *
457 * Calls irq_install() or irq_uninstall() according to \p arg.
458 */
459int drm_control(struct drm_device *dev, void *data,
460 struct drm_file *file_priv)
461{
462 struct drm_control *ctl = data;
463
464 /* if we haven't irq we fallback for compatibility reasons -
465 * this used to be a separate function in drm_dma.h
466 */
467
468
469 switch (ctl->func) {
470 case DRM_INST_HANDLER:
471 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
472 return 0;
473 if (drm_core_check_feature(dev, DRIVER_MODESET))
474 return 0;
475 if (dev->if_version < DRM_IF_VERSION(1, 2) &&
476 ctl->irq != drm_dev_to_irq(dev))
477 return -EINVAL;
478 return drm_irq_install(dev);
479 case DRM_UNINST_HANDLER:
480 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
481 return 0;
482 if (drm_core_check_feature(dev, DRIVER_MODESET))
483 return 0;
484 return drm_irq_uninstall(dev);
485 default:
486 return -EINVAL;
487 }
488}
489
490/**
491 * drm_calc_timestamping_constants - Calculate vblank timestamp constants
492 *
493 * @crtc drm_crtc whose timestamp constants should be updated.
494 * @mode display mode containing the scanout timings
495 *
496 * Calculate and store various constants which are later
497 * needed by vblank and swap-completion timestamping, e.g,
498 * by drm_calc_vbltimestamp_from_scanoutpos(). They are
499 * derived from crtc's true scanout timing, so they take
500 * things like panel scaling or other adjustments into account.
501 */
502void drm_calc_timestamping_constants(struct drm_crtc *crtc,
503 const struct drm_display_mode *mode)
504{
505 int linedur_ns = 0, pixeldur_ns = 0, framedur_ns = 0;
506 int dotclock = mode->crtc_clock;
507
508 /* Valid dotclock? */
509 if (dotclock > 0) {
510 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
511
512 /*
513 * Convert scanline length in pixels and video
514 * dot clock to line duration, frame duration
515 * and pixel duration in nanoseconds:
516 */
517 pixeldur_ns = 1000000 / dotclock;
518 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
519 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
520
521 /*
522 * Fields of interlaced scanout modes are only half a frame duration.
523 */
524 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
525 framedur_ns /= 2;
526 } else
527 DRM_ERROR("crtc %d: Can't calculate constants, dotclock = 0!\n",
528 crtc->base.id);
529
530 crtc->pixeldur_ns = pixeldur_ns;
531 crtc->linedur_ns = linedur_ns;
532 crtc->framedur_ns = framedur_ns;
533
534 DRM_DEBUG("crtc %d: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
535 crtc->base.id, mode->crtc_htotal,
536 mode->crtc_vtotal, mode->crtc_vdisplay);
537 DRM_DEBUG("crtc %d: clock %d kHz framedur %d linedur %d, pixeldur %d\n",
538 crtc->base.id, dotclock, framedur_ns,
539 linedur_ns, pixeldur_ns);
540}
541EXPORT_SYMBOL(drm_calc_timestamping_constants);
542
543/**
544 * drm_calc_vbltimestamp_from_scanoutpos - helper routine for kms
545 * drivers. Implements calculation of exact vblank timestamps from
546 * given drm_display_mode timings and current video scanout position
547 * of a crtc. This can be called from within get_vblank_timestamp()
548 * implementation of a kms driver to implement the actual timestamping.
549 *
550 * Should return timestamps conforming to the OML_sync_control OpenML
551 * extension specification. The timestamp corresponds to the end of
552 * the vblank interval, aka start of scanout of topmost-leftmost display
553 * pixel in the following video frame.
554 *
555 * Requires support for optional dev->driver->get_scanout_position()
556 * in kms driver, plus a bit of setup code to provide a drm_display_mode
557 * that corresponds to the true scanout timing.
558 *
559 * The current implementation only handles standard video modes. It
560 * returns as no operation if a doublescan or interlaced video mode is
561 * active. Higher level code is expected to handle this.
562 *
563 * @dev: DRM device.
564 * @crtc: Which crtc's vblank timestamp to retrieve.
565 * @max_error: Desired maximum allowable error in timestamps (nanosecs).
566 * On return contains true maximum error of timestamp.
567 * @vblank_time: Pointer to struct timeval which should receive the timestamp.
568 * @flags: Flags to pass to driver:
569 * 0 = Default.
570 * DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
571 * @refcrtc: drm_crtc* of crtc which defines scanout timing.
572 * @mode: mode which defines the scanout timings
573 *
574 * Returns negative value on error, failure or if not supported in current
575 * video mode:
576 *
577 * -EINVAL - Invalid crtc.
578 * -EAGAIN - Temporary unavailable, e.g., called before initial modeset.
579 * -ENOTSUPP - Function not supported in current display mode.
580 * -EIO - Failed, e.g., due to failed scanout position query.
581 *
582 * Returns or'ed positive status flags on success:
583 *
584 * DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping.
585 * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval.
586 *
587 */
588int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
589 int *max_error,
590 struct timeval *vblank_time,
591 unsigned flags,
592 const struct drm_crtc *refcrtc,
593 const struct drm_display_mode *mode)
594{
595 ktime_t stime, etime, mono_time_offset;
596 struct timeval tv_etime;
597 int vbl_status;
598 int vpos, hpos, i;
599 int framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns;
600 bool invbl;
601
602 if (crtc < 0 || crtc >= dev->num_crtcs) {
603 DRM_ERROR("Invalid crtc %d\n", crtc);
604 return -EINVAL;
605 }
606
607 /* Scanout position query not supported? Should not happen. */
608 if (!dev->driver->get_scanout_position) {
609 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
610 return -EIO;
611 }
612
613 /* Durations of frames, lines, pixels in nanoseconds. */
614 framedur_ns = refcrtc->framedur_ns;
615 linedur_ns = refcrtc->linedur_ns;
616 pixeldur_ns = refcrtc->pixeldur_ns;
617
618 /* If mode timing undefined, just return as no-op:
619 * Happens during initial modesetting of a crtc.
620 */
621 if (framedur_ns == 0) {
622 DRM_DEBUG("crtc %d: Noop due to uninitialized mode.\n", crtc);
623 return -EAGAIN;
624 }
625
626 /* Get current scanout position with system timestamp.
627 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
628 * if single query takes longer than max_error nanoseconds.
629 *
630 * This guarantees a tight bound on maximum error if
631 * code gets preempted or delayed for some reason.
632 */
633 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
634 /*
635 * Get vertical and horizontal scanout position vpos, hpos,
636 * and bounding timestamps stime, etime, pre/post query.
637 */
638 vbl_status = dev->driver->get_scanout_position(dev, crtc, flags, &vpos,
639 &hpos, &stime, &etime);
640
641 /*
642 * Get correction for CLOCK_MONOTONIC -> CLOCK_REALTIME if
643 * CLOCK_REALTIME is requested.
644 */
645 if (!drm_timestamp_monotonic)
646 mono_time_offset = ktime_get_monotonic_offset();
647
648 /* Return as no-op if scanout query unsupported or failed. */
649 if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
650 DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n",
651 crtc, vbl_status);
652 return -EIO;
653 }
654
655 /* Compute uncertainty in timestamp of scanout position query. */
656 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
657
658 /* Accept result with < max_error nsecs timing uncertainty. */
659 if (duration_ns <= *max_error)
660 break;
661 }
662
663 /* Noisy system timing? */
664 if (i == DRM_TIMESTAMP_MAXRETRIES) {
665 DRM_DEBUG("crtc %d: Noisy timestamp %d us > %d us [%d reps].\n",
666 crtc, duration_ns/1000, *max_error/1000, i);
667 }
668
669 /* Return upper bound of timestamp precision error. */
670 *max_error = duration_ns;
671
672 /* Check if in vblank area:
673 * vpos is >=0 in video scanout area, but negative
674 * within vblank area, counting down the number of lines until
675 * start of scanout.
676 */
677 invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
678
679 /* Convert scanout position into elapsed time at raw_time query
680 * since start of scanout at first display scanline. delta_ns
681 * can be negative if start of scanout hasn't happened yet.
682 */
683 delta_ns = vpos * linedur_ns + hpos * pixeldur_ns;
684
685 if (!drm_timestamp_monotonic)
686 etime = ktime_sub(etime, mono_time_offset);
687
688 /* save this only for debugging purposes */
689 tv_etime = ktime_to_timeval(etime);
690 /* Subtract time delta from raw timestamp to get final
691 * vblank_time timestamp for end of vblank.
692 */
693 if (delta_ns < 0)
694 etime = ktime_add_ns(etime, -delta_ns);
695 else
696 etime = ktime_sub_ns(etime, delta_ns);
697 *vblank_time = ktime_to_timeval(etime);
698
699 DRM_DEBUG("crtc %d : v %d p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n",
700 crtc, (int)vbl_status, hpos, vpos,
701 (long)tv_etime.tv_sec, (long)tv_etime.tv_usec,
702 (long)vblank_time->tv_sec, (long)vblank_time->tv_usec,
703 duration_ns/1000, i);
704
705 vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
706 if (invbl)
707 vbl_status |= DRM_VBLANKTIME_INVBL;
708
709 return vbl_status;
710}
711EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
712
713static struct timeval get_drm_timestamp(void)
714{
715 ktime_t now;
716
717 now = ktime_get();
718 if (!drm_timestamp_monotonic)
719 now = ktime_sub(now, ktime_get_monotonic_offset());
720
721 return ktime_to_timeval(now);
722}
723
724/**
725 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
726 * vblank interval.
727 *
728 * @dev: DRM device
729 * @crtc: which crtc's vblank timestamp to retrieve
730 * @tvblank: Pointer to target struct timeval which should receive the timestamp
731 * @flags: Flags to pass to driver:
732 * 0 = Default.
733 * DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
734 *
735 * Fetches the system timestamp corresponding to the time of the most recent
736 * vblank interval on specified crtc. May call into kms-driver to
737 * compute the timestamp with a high-precision GPU specific method.
738 *
739 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
740 * call, i.e., it isn't very precisely locked to the true vblank.
741 *
742 * Returns non-zero if timestamp is considered to be very precise.
743 */
744u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
745 struct timeval *tvblank, unsigned flags)
746{
747 int ret;
748
749 /* Define requested maximum error on timestamps (nanoseconds). */
750 int max_error = (int) drm_timestamp_precision * 1000;
751
752 /* Query driver if possible and precision timestamping enabled. */
753 if (dev->driver->get_vblank_timestamp && (max_error > 0)) {
754 ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
755 tvblank, flags);
756 if (ret > 0)
757 return (u32) ret;
758 }
759
760 /* GPU high precision timestamp query unsupported or failed.
761 * Return current monotonic/gettimeofday timestamp as best estimate.
762 */
763 *tvblank = get_drm_timestamp();
764
765 return 0;
766}
767EXPORT_SYMBOL(drm_get_last_vbltimestamp);
768
769/**
770 * drm_vblank_count - retrieve "cooked" vblank counter value
771 * @dev: DRM device
772 * @crtc: which counter to retrieve
773 *
774 * Fetches the "cooked" vblank count value that represents the number of
775 * vblank events since the system was booted, including lost events due to
776 * modesetting activity.
777 */
778u32 drm_vblank_count(struct drm_device *dev, int crtc)
779{
780 return atomic_read(&dev->vblank[crtc].count);
781}
782EXPORT_SYMBOL(drm_vblank_count);
783
784/**
785 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value
786 * and the system timestamp corresponding to that vblank counter value.
787 *
788 * @dev: DRM device
789 * @crtc: which counter to retrieve
790 * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
791 *
792 * Fetches the "cooked" vblank count value that represents the number of
793 * vblank events since the system was booted, including lost events due to
794 * modesetting activity. Returns corresponding system timestamp of the time
795 * of the vblank interval that corresponds to the current value vblank counter
796 * value.
797 */
798u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
799 struct timeval *vblanktime)
800{
801 u32 cur_vblank;
802
803 /* Read timestamp from slot of _vblank_time ringbuffer
804 * that corresponds to current vblank count. Retry if
805 * count has incremented during readout. This works like
806 * a seqlock.
807 */
808 do {
809 cur_vblank = atomic_read(&dev->vblank[crtc].count);
810 *vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
811 smp_rmb();
812 } while (cur_vblank != atomic_read(&dev->vblank[crtc].count));
813
814 return cur_vblank;
815}
816EXPORT_SYMBOL(drm_vblank_count_and_time);
817
818static void send_vblank_event(struct drm_device *dev,
819 struct drm_pending_vblank_event *e,
820 unsigned long seq, struct timeval *now)
821{
822 WARN_ON_SMP(!spin_is_locked(&dev->event_lock));
823 e->event.sequence = seq;
824 e->event.tv_sec = now->tv_sec;
825 e->event.tv_usec = now->tv_usec;
826
827 list_add_tail(&e->base.link,
828 &e->base.file_priv->event_list);
829#ifdef __NetBSD__
830 DRM_SPIN_WAKEUP_ONE(&e->base.file_priv->event_wait, &dev->event_lock);
831 selnotify(&e->base.file_priv->event_selq, (POLLIN | POLLRDNORM),
832 NOTE_SUBMIT);
833#else
834 wake_up_interruptible(&e->base.file_priv->event_wait);
835#endif
836 trace_drm_vblank_event_delivered(e->base.pid, e->pipe,
837 e->event.sequence);
838}
839
840/**
841 * drm_send_vblank_event - helper to send vblank event after pageflip
842 * @dev: DRM device
843 * @crtc: CRTC in question
844 * @e: the event to send
845 *
846 * Updates sequence # and timestamp on event, and sends it to userspace.
847 * Caller must hold event lock.
848 */
849void drm_send_vblank_event(struct drm_device *dev, int crtc,
850 struct drm_pending_vblank_event *e)
851{
852 struct timeval now;
853 unsigned int seq;
854 if (crtc >= 0) {
855 seq = drm_vblank_count_and_time(dev, crtc, &now);
856 } else {
857 seq = 0;
858
859 now = get_drm_timestamp();
860 }
861 e->pipe = crtc;
862 send_vblank_event(dev, e, seq, &now);
863}
864EXPORT_SYMBOL(drm_send_vblank_event);
865
866/**
867 * drm_update_vblank_count - update the master vblank counter
868 * @dev: DRM device
869 * @crtc: counter to update
870 *
871 * Call back into the driver to update the appropriate vblank counter
872 * (specified by @crtc). Deal with wraparound, if it occurred, and
873 * update the last read value so we can deal with wraparound on the next
874 * call if necessary.
875 *
876 * Only necessary when going from off->on, to account for frames we
877 * didn't get an interrupt for.
878 *
879 * Note: caller must hold dev->vbl_lock since this reads & writes
880 * device vblank fields.
881 */
882static void drm_update_vblank_count(struct drm_device *dev, int crtc)
883{
884 u32 cur_vblank, diff, tslot, rc;
885 struct timeval t_vblank;
886
887 /*
888 * Interrupts were disabled prior to this call, so deal with counter
889 * wrap if needed.
890 * NOTE! It's possible we lost a full dev->max_vblank_count events
891 * here if the register is small or we had vblank interrupts off for
892 * a long time.
893 *
894 * We repeat the hardware vblank counter & timestamp query until
895 * we get consistent results. This to prevent races between gpu
896 * updating its hardware counter while we are retrieving the
897 * corresponding vblank timestamp.
898 */
899 do {
900 cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
901 rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
902 } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
903
904 /* Deal with counter wrap */
905 diff = cur_vblank - dev->vblank[crtc].last;
906 if (cur_vblank < dev->vblank[crtc].last) {
907 diff += dev->max_vblank_count;
908
909 DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
910 crtc, dev->vblank[crtc].last, cur_vblank, diff);
911 }
912
913 DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
914 crtc, diff);
915
916 /* Reinitialize corresponding vblank timestamp if high-precision query
917 * available. Skip this step if query unsupported or failed. Will
918 * reinitialize delayed at next vblank interrupt in that case.
919 */
920 if (rc) {
921 tslot = atomic_read(&dev->vblank[crtc].count) + diff;
922 vblanktimestamp(dev, crtc, tslot) = t_vblank;
923 }
924
925 smp_mb__before_atomic_inc();
926 atomic_add(diff, &dev->vblank[crtc].count);
927 smp_mb__after_atomic_inc();
928}
929
930/**
931 * drm_vblank_get - get a reference count on vblank events
932 * @dev: DRM device
933 * @crtc: which CRTC to own
934 *
935 * Acquire a reference count on vblank events to avoid having them disabled
936 * while in use.
937 *
938 * RETURNS
939 * Zero on success, nonzero on failure.
940 */
941int drm_vblank_get(struct drm_device *dev, int crtc)
942{
943 unsigned long irqflags, irqflags2;
944 int ret = 0;
945
946 spin_lock_irqsave(&dev->vbl_lock, irqflags);
947 /* Going from 0->1 means we have to enable interrupts again */
948 if (atomic_add_return(1, &dev->vblank[crtc].refcount) == 1) {
949 spin_lock_irqsave(&dev->vblank_time_lock, irqflags2);
950 if (!dev->vblank[crtc].enabled) {
951 /* Enable vblank irqs under vblank_time_lock protection.
952 * All vblank count & timestamp updates are held off
953 * until we are done reinitializing master counter and
954 * timestamps. Filtercode in drm_handle_vblank() will
955 * prevent double-accounting of same vblank interval.
956 */
957 ret = dev->driver->enable_vblank(dev, crtc);
958 DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n",
959 crtc, ret);
960 if (ret)
961 atomic_dec(&dev->vblank[crtc].refcount);
962 else {
963 dev->vblank[crtc].enabled = true;
964 drm_update_vblank_count(dev, crtc);
965 }
966 }
967 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags2);
968 } else {
969 if (!dev->vblank[crtc].enabled) {
970 atomic_dec(&dev->vblank[crtc].refcount);
971 ret = -EINVAL;
972 }
973 }
974 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
975
976 return ret;
977}
978EXPORT_SYMBOL(drm_vblank_get);
979
980/**
981 * drm_vblank_put - give up ownership of vblank events
982 * @dev: DRM device
983 * @crtc: which counter to give up
984 *
985 * Release ownership of a given vblank counter, turning off interrupts
986 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
987 */
988void drm_vblank_put(struct drm_device *dev, int crtc)
989{
990 BUG_ON(atomic_read(&dev->vblank[crtc].refcount) == 0);
991
992 /* Last user schedules interrupt disable */
993 if (atomic_dec_and_test(&dev->vblank[crtc].refcount) &&
994 (drm_vblank_offdelay > 0))
995 mod_timer(&dev->vblank_disable_timer,
996 jiffies + ((drm_vblank_offdelay * HZ)/1000));
997}
998EXPORT_SYMBOL(drm_vblank_put);
999
1000/**
1001 * drm_vblank_off - disable vblank events on a CRTC
1002 * @dev: DRM device
1003 * @crtc: CRTC in question
1004 *
1005 * Caller must hold event lock.
1006 */
1007void drm_vblank_off(struct drm_device *dev, int crtc)
1008{
1009 struct drm_pending_vblank_event *e, *t;
1010 struct timeval now;
1011 unsigned long irqflags;
1012 unsigned int seq;
1013
1014 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1015 vblank_disable_and_save(dev, crtc);
1016#ifdef __NetBSD__
1017 DRM_SPIN_WAKEUP_ONE(&dev->vblank[crtc].queue, &dev->vbl_lock);
1018#else
1019 wake_up(&dev->vblank[crtc].queue);
1020#endif
1021
1022 /* Send any queued vblank events, lest the natives grow disquiet */
1023 seq = drm_vblank_count_and_time(dev, crtc, &now);
1024
1025 spin_lock(&dev->event_lock);
1026 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1027 if (e->pipe != crtc)
1028 continue;
1029 DRM_DEBUG("Sending premature vblank event on disable: \
1030 wanted %d, current %d\n",
1031 e->event.sequence, seq);
1032 list_del(&e->base.link);
1033 drm_vblank_put(dev, e->pipe);
1034 send_vblank_event(dev, e, seq, &now);
1035 }
1036 spin_unlock(&dev->event_lock);
1037
1038 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1039}
1040EXPORT_SYMBOL(drm_vblank_off);
1041
1042/**
1043 * drm_vblank_pre_modeset - account for vblanks across mode sets
1044 * @dev: DRM device
1045 * @crtc: CRTC in question
1046 *
1047 * Account for vblank events across mode setting events, which will likely
1048 * reset the hardware frame counter.
1049 */
1050void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
1051{
1052 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1053 if (!dev->num_crtcs)
1054 return;
1055 /*
1056 * To avoid all the problems that might happen if interrupts
1057 * were enabled/disabled around or between these calls, we just
1058 * have the kernel take a reference on the CRTC (just once though
1059 * to avoid corrupting the count if multiple, mismatch calls occur),
1060 * so that interrupts remain enabled in the interim.
1061 */
1062 if (!dev->vblank[crtc].inmodeset) {
1063 dev->vblank[crtc].inmodeset = 0x1;
1064 if (drm_vblank_get(dev, crtc) == 0)
1065 dev->vblank[crtc].inmodeset |= 0x2;
1066 }
1067}
1068EXPORT_SYMBOL(drm_vblank_pre_modeset);
1069
1070void drm_vblank_post_modeset(struct drm_device *dev, int crtc)
1071{
1072 unsigned long irqflags;
1073
1074 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1075 if (!dev->num_crtcs)
1076 return;
1077
1078 if (dev->vblank[crtc].inmodeset) {
1079 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1080 dev->vblank_disable_allowed = true;
1081 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1082
1083 if (dev->vblank[crtc].inmodeset & 0x2)
1084 drm_vblank_put(dev, crtc);
1085
1086 dev->vblank[crtc].inmodeset = 0;
1087 }
1088}
1089EXPORT_SYMBOL(drm_vblank_post_modeset);
1090
1091/**
1092 * drm_modeset_ctl - handle vblank event counter changes across mode switch
1093 * @DRM_IOCTL_ARGS: standard ioctl arguments
1094 *
1095 * Applications should call the %_DRM_PRE_MODESET and %_DRM_POST_MODESET
1096 * ioctls around modesetting so that any lost vblank events are accounted for.
1097 *
1098 * Generally the counter will reset across mode sets. If interrupts are
1099 * enabled around this call, we don't have to do anything since the counter
1100 * will have already been incremented.
1101 */
1102int drm_modeset_ctl(struct drm_device *dev, void *data,
1103 struct drm_file *file_priv)
1104{
1105 struct drm_modeset_ctl *modeset = data;
1106 unsigned int crtc;
1107
1108 /* If drm_vblank_init() hasn't been called yet, just no-op */
1109 if (!dev->num_crtcs)
1110 return 0;
1111
1112 /* KMS drivers handle this internally */
1113 if (drm_core_check_feature(dev, DRIVER_MODESET))
1114 return 0;
1115
1116 crtc = modeset->crtc;
1117 if (crtc >= dev->num_crtcs)
1118 return -EINVAL;
1119
1120 switch (modeset->cmd) {
1121 case _DRM_PRE_MODESET:
1122 drm_vblank_pre_modeset(dev, crtc);
1123 break;
1124 case _DRM_POST_MODESET:
1125 drm_vblank_post_modeset(dev, crtc);
1126 break;
1127 default:
1128 return -EINVAL;
1129 }
1130
1131 return 0;
1132}
1133
1134static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
1135 union drm_wait_vblank *vblwait,
1136 struct drm_file *file_priv)
1137{
1138 struct drm_pending_vblank_event *e;
1139 struct timeval now;
1140 unsigned long flags;
1141 unsigned int seq;
1142 int ret;
1143
1144 e = kzalloc(sizeof *e, GFP_KERNEL);
1145 if (e == NULL) {
1146 ret = -ENOMEM;
1147 goto err_put;
1148 }
1149
1150 e->pipe = pipe;
1151#ifdef __NetBSD__
1152 e->base.pid = curproc->p_pid;
1153#else
1154 e->base.pid = current->pid;
1155#endif
1156 e->event.base.type = DRM_EVENT_VBLANK;
1157 e->event.base.length = sizeof e->event;
1158 e->event.user_data = vblwait->request.signal;
1159 e->base.event = &e->event.base;
1160 e->base.file_priv = file_priv;
1161 e->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
1162
1163 spin_lock_irqsave(&dev->event_lock, flags);
1164
1165 if (file_priv->event_space < sizeof e->event) {
1166 ret = -EBUSY;
1167 goto err_unlock;
1168 }
1169
1170 file_priv->event_space -= sizeof e->event;
1171 seq = drm_vblank_count_and_time(dev, pipe, &now);
1172
1173 if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
1174 (seq - vblwait->request.sequence) <= (1 << 23)) {
1175 vblwait->request.sequence = seq + 1;
1176 vblwait->reply.sequence = vblwait->request.sequence;
1177 }
1178
1179 DRM_DEBUG("event on vblank count %d, current %d, crtc %d\n",
1180 vblwait->request.sequence, seq, pipe);
1181
1182#ifdef __NetBSD__
1183 trace_drm_vblank_event_queued(curproc->p_pid, pipe,
1184 vblwait->request.sequence);
1185#else
1186 trace_drm_vblank_event_queued(current->pid, pipe,
1187 vblwait->request.sequence);
1188#endif
1189
1190 e->event.sequence = vblwait->request.sequence;
1191 if ((seq - vblwait->request.sequence) <= (1 << 23)) {
1192 drm_vblank_put(dev, pipe);
1193 send_vblank_event(dev, e, seq, &now);
1194 vblwait->reply.sequence = seq;
1195 } else {
1196 /* drm_handle_vblank_events will call drm_vblank_put */
1197 list_add_tail(&e->base.link, &dev->vblank_event_list);
1198 vblwait->reply.sequence = vblwait->request.sequence;
1199 }
1200
1201 spin_unlock_irqrestore(&dev->event_lock, flags);
1202
1203 return 0;
1204
1205err_unlock:
1206 spin_unlock_irqrestore(&dev->event_lock, flags);
1207 kfree(e);
1208err_put:
1209 drm_vblank_put(dev, pipe);
1210 return ret;
1211}
1212
1213/**
1214 * Wait for VBLANK.
1215 *
1216 * \param inode device inode.
1217 * \param file_priv DRM file private.
1218 * \param cmd command.
1219 * \param data user argument, pointing to a drm_wait_vblank structure.
1220 * \return zero on success or a negative number on failure.
1221 *
1222 * This function enables the vblank interrupt on the pipe requested, then
1223 * sleeps waiting for the requested sequence number to occur, and drops
1224 * the vblank interrupt refcount afterwards. (vblank irq disable follows that
1225 * after a timeout with no further vblank waits scheduled).
1226 */
1227int drm_wait_vblank(struct drm_device *dev, void *data,
1228 struct drm_file *file_priv)
1229{
1230 union drm_wait_vblank *vblwait = data;
1231 int ret;
1232 unsigned int flags, seq, crtc, high_crtc;
1233
1234 if (drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
1235 if ((!drm_dev_to_irq(dev)) || (!dev->irq_enabled))
1236 return -EINVAL;
1237
1238 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1239 return -EINVAL;
1240
1241 if (vblwait->request.type &
1242 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1243 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1244 DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
1245 vblwait->request.type,
1246 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1247 _DRM_VBLANK_HIGH_CRTC_MASK));
1248 return -EINVAL;
1249 }
1250
1251 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1252 high_crtc = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1253 if (high_crtc)
1254 crtc = high_crtc >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1255 else
1256 crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1257 if (crtc >= dev->num_crtcs)
1258 return -EINVAL;
1259
1260 ret = drm_vblank_get(dev, crtc);
1261 if (ret) {
1262 DRM_DEBUG("failed to acquire vblank counter, %d\n", ret);
1263 return ret;
1264 }
1265 seq = drm_vblank_count(dev, crtc);
1266
1267 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1268 case _DRM_VBLANK_RELATIVE:
1269 vblwait->request.sequence += seq;
1270 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1271 case _DRM_VBLANK_ABSOLUTE:
1272 break;
1273 default:
1274 ret = -EINVAL;
1275 goto done;
1276 }
1277
1278 if (flags & _DRM_VBLANK_EVENT) {
1279 /* must hold on to the vblank ref until the event fires
1280 * drm_vblank_put will be called asynchronously
1281 */
1282 return drm_queue_vblank_event(dev, crtc, vblwait, file_priv);
1283 }
1284
1285 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1286 (seq - vblwait->request.sequence) <= (1<<23)) {
1287 vblwait->request.sequence = seq + 1;
1288 }
1289
1290 DRM_DEBUG("waiting on vblank count %d, crtc %d\n",
1291 vblwait->request.sequence, crtc);
1292 dev->vblank[crtc].last_wait = vblwait->request.sequence;
1293#ifdef __NetBSD__
1294 {
1295 unsigned long irqflags;
1296
1297 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1298 DRM_SPIN_WAIT_ON(ret, &dev->vblank[crtc].queue, &dev->vbl_lock,
1299 3 * HZ,
1300 (((drm_vblank_count(dev, crtc) -
1301 vblwait->request.sequence) <= (1 << 23)) ||
1302 !dev->irq_enabled));
1303 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1304 }
1305#else
1306 DRM_WAIT_ON(ret, dev->vblank[crtc].queue, 3 * HZ,
1307 (((drm_vblank_count(dev, crtc) -
1308 vblwait->request.sequence) <= (1 << 23)) ||
1309 !dev->irq_enabled));
1310#endif
1311
1312 if (ret != -EINTR) {
1313 struct timeval now;
1314
1315 vblwait->reply.sequence = drm_vblank_count_and_time(dev, crtc, &now);
1316 vblwait->reply.tval_sec = now.tv_sec;
1317 vblwait->reply.tval_usec = now.tv_usec;
1318
1319 DRM_DEBUG("returning %d to client\n",
1320 vblwait->reply.sequence);
1321 } else {
1322 DRM_DEBUG("vblank wait interrupted by signal\n");
1323 }
1324
1325done:
1326 drm_vblank_put(dev, crtc);
1327 return ret;
1328}
1329
1330static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
1331{
1332 struct drm_pending_vblank_event *e, *t;
1333 struct timeval now;
1334 unsigned long flags;
1335 unsigned int seq;
1336
1337 seq = drm_vblank_count_and_time(dev, crtc, &now);
1338
1339 spin_lock_irqsave(&dev->event_lock, flags);
1340
1341 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1342 if (e->pipe != crtc)
1343 continue;
1344 if ((seq - e->event.sequence) > (1<<23))
1345 continue;
1346
1347 DRM_DEBUG("vblank event on %d, current %d\n",
1348 e->event.sequence, seq);
1349
1350 list_del(&e->base.link);
1351 drm_vblank_put(dev, e->pipe);
1352 send_vblank_event(dev, e, seq, &now);
1353 }
1354
1355 spin_unlock_irqrestore(&dev->event_lock, flags);
1356
1357 trace_drm_vblank_event(crtc, seq);
1358}
1359
1360/**
1361 * drm_handle_vblank - handle a vblank event
1362 * @dev: DRM device
1363 * @crtc: where this event occurred
1364 *
1365 * Drivers should call this routine in their vblank interrupt handlers to
1366 * update the vblank counter and send any signals that may be pending.
1367 */
1368bool drm_handle_vblank(struct drm_device *dev, int crtc)
1369{
1370 u32 vblcount;
1371 s64 diff_ns;
1372 struct timeval tvblank;
1373 unsigned long irqflags;
1374#ifdef __NetBSD__ /* XXX vblank locking */
1375 unsigned long irqflags_vbl_lock;
1376#endif
1377
1378 if (!dev->num_crtcs)
1379 return false;
1380
1381#ifdef __NetBSD__ /* XXX vblank locking */
1382 spin_lock_irqsave(&dev->vbl_lock, irqflags_vbl_lock);
1383#endif
1384
1385 /* Need timestamp lock to prevent concurrent execution with
1386 * vblank enable/disable, as this would cause inconsistent
1387 * or corrupted timestamps and vblank counts.
1388 */
1389 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
1390
1391 /* Vblank irq handling disabled. Nothing to do. */
1392 if (!dev->vblank[crtc].enabled) {
1393 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
1394#ifdef __NetBSD__ /* XXX vblank locking */
1395 spin_unlock_irqrestore(&dev->vbl_lock, irqflags_vbl_lock);
1396#endif
1397 return false;
1398 }
1399
1400 /* Fetch corresponding timestamp for this vblank interval from
1401 * driver and store it in proper slot of timestamp ringbuffer.
1402 */
1403
1404 /* Get current timestamp and count. */
1405 vblcount = atomic_read(&dev->vblank[crtc].count);
1406 drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
1407
1408 /* Compute time difference to timestamp of last vblank */
1409 diff_ns = timeval_to_ns(&tvblank) -
1410 timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
1411
1412 /* Update vblank timestamp and count if at least
1413 * DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds
1414 * difference between last stored timestamp and current
1415 * timestamp. A smaller difference means basically
1416 * identical timestamps. Happens if this vblank has
1417 * been already processed and this is a redundant call,
1418 * e.g., due to spurious vblank interrupts. We need to
1419 * ignore those for accounting.
1420 */
1421 if (abs64(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS) {
1422 /* Store new timestamp in ringbuffer. */
1423 vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
1424
1425 /* Increment cooked vblank count. This also atomically commits
1426 * the timestamp computed above.
1427 */
1428 smp_mb__before_atomic_inc();
1429 atomic_inc(&dev->vblank[crtc].count);
1430 smp_mb__after_atomic_inc();
1431 } else {
1432 DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
1433 crtc, (int) diff_ns);
1434 }
1435
1436#ifdef __NetBSD__
1437 DRM_SPIN_WAKEUP_ONE(&dev->vblank[crtc].queue, &dev->vbl_lock);
1438#else
1439 wake_up(&dev->vblank[crtc].queue);
1440#endif
1441 drm_handle_vblank_events(dev, crtc);
1442
1443 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
1444#ifdef __NetBSD__ /* XXX vblank locking */
1445 spin_unlock_irqrestore(&dev->vbl_lock, irqflags_vbl_lock);
1446#endif
1447 return true;
1448}
1449EXPORT_SYMBOL(drm_handle_vblank);
1450