1/* $NetBSD: genfs_io.c,v 1.63 2016/09/29 19:08:48 christos Exp $ */
2
3/*
4 * Copyright (c) 1982, 1986, 1989, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 */
32
33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.63 2016/09/29 19:08:48 christos Exp $");
35
36#include <sys/param.h>
37#include <sys/systm.h>
38#include <sys/proc.h>
39#include <sys/kernel.h>
40#include <sys/mount.h>
41#include <sys/vnode.h>
42#include <sys/kmem.h>
43#include <sys/kauth.h>
44#include <sys/fstrans.h>
45#include <sys/buf.h>
46
47#include <miscfs/genfs/genfs.h>
48#include <miscfs/genfs/genfs_node.h>
49#include <miscfs/specfs/specdev.h>
50
51#include <uvm/uvm.h>
52#include <uvm/uvm_pager.h>
53
54static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *,
55 off_t, enum uio_rw);
56static void genfs_dio_iodone(struct buf *);
57
58static int genfs_getpages_read(struct vnode *, struct vm_page **, int, off_t,
59 off_t, bool, bool, bool, bool);
60static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw,
61 void (*)(struct buf *));
62static void genfs_rel_pages(struct vm_page **, unsigned int);
63static void genfs_markdirty(struct vnode *);
64
65int genfs_maxdio = MAXPHYS;
66
67static void
68genfs_rel_pages(struct vm_page **pgs, unsigned int npages)
69{
70 unsigned int i;
71
72 for (i = 0; i < npages; i++) {
73 struct vm_page *pg = pgs[i];
74
75 if (pg == NULL || pg == PGO_DONTCARE)
76 continue;
77 KASSERT(uvm_page_locked_p(pg));
78 if (pg->flags & PG_FAKE) {
79 pg->flags |= PG_RELEASED;
80 }
81 }
82 mutex_enter(&uvm_pageqlock);
83 uvm_page_unbusy(pgs, npages);
84 mutex_exit(&uvm_pageqlock);
85}
86
87static void
88genfs_markdirty(struct vnode *vp)
89{
90 struct genfs_node * const gp = VTOG(vp);
91
92 KASSERT(mutex_owned(vp->v_interlock));
93 gp->g_dirtygen++;
94 if ((vp->v_iflag & VI_ONWORKLST) == 0) {
95 vn_syncer_add_to_worklist(vp, filedelay);
96 }
97 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) {
98 vp->v_iflag |= VI_WRMAPDIRTY;
99 }
100}
101
102/*
103 * generic VM getpages routine.
104 * Return PG_BUSY pages for the given range,
105 * reading from backing store if necessary.
106 */
107
108int
109genfs_getpages(void *v)
110{
111 struct vop_getpages_args /* {
112 struct vnode *a_vp;
113 voff_t a_offset;
114 struct vm_page **a_m;
115 int *a_count;
116 int a_centeridx;
117 vm_prot_t a_access_type;
118 int a_advice;
119 int a_flags;
120 } */ * const ap = v;
121
122 off_t diskeof, memeof;
123 int i, error, npages;
124 const int flags = ap->a_flags;
125 struct vnode * const vp = ap->a_vp;
126 struct uvm_object * const uobj = &vp->v_uobj;
127 const bool async = (flags & PGO_SYNCIO) == 0;
128 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
129 const bool overwrite = (flags & PGO_OVERWRITE) != 0;
130 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0;
131 const bool glocked = (flags & PGO_GLOCKHELD) != 0;
132 const bool need_wapbl = blockalloc && vp->v_mount->mnt_wapbl;
133 bool has_trans_wapbl = false;
134 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist);
135
136 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d",
137 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count);
138
139 KASSERT(vp->v_type == VREG || vp->v_type == VDIR ||
140 vp->v_type == VLNK || vp->v_type == VBLK);
141
142startover:
143 error = 0;
144 const voff_t origvsize = vp->v_size;
145 const off_t origoffset = ap->a_offset;
146 const int orignpages = *ap->a_count;
147
148 GOP_SIZE(vp, origvsize, &diskeof, 0);
149 if (flags & PGO_PASTEOF) {
150 off_t newsize;
151#if defined(DIAGNOSTIC)
152 off_t writeeof;
153#endif /* defined(DIAGNOSTIC) */
154
155 newsize = MAX(origvsize,
156 origoffset + (orignpages << PAGE_SHIFT));
157 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM);
158#if defined(DIAGNOSTIC)
159 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM);
160 if (newsize > round_page(writeeof)) {
161 panic("%s: past eof: %" PRId64 " vs. %" PRId64,
162 __func__, newsize, round_page(writeeof));
163 }
164#endif /* defined(DIAGNOSTIC) */
165 } else {
166 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM);
167 }
168 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages);
169 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0);
170 KASSERT(orignpages > 0);
171
172 /*
173 * Bounds-check the request.
174 */
175
176 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) {
177 if ((flags & PGO_LOCKED) == 0) {
178 mutex_exit(uobj->vmobjlock);
179 }
180 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x",
181 origoffset, *ap->a_count, memeof,0);
182 error = EINVAL;
183 goto out_err;
184 }
185
186 /* uobj is locked */
187
188 if ((flags & PGO_NOTIMESTAMP) == 0 &&
189 (vp->v_type != VBLK ||
190 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
191 int updflags = 0;
192
193 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) {
194 updflags = GOP_UPDATE_ACCESSED;
195 }
196 if (memwrite) {
197 updflags |= GOP_UPDATE_MODIFIED;
198 }
199 if (updflags != 0) {
200 GOP_MARKUPDATE(vp, updflags);
201 }
202 }
203
204 /*
205 * For PGO_LOCKED requests, just return whatever's in memory.
206 */
207
208 if (flags & PGO_LOCKED) {
209 int nfound;
210 struct vm_page *pg;
211
212 KASSERT(!glocked);
213 npages = *ap->a_count;
214#if defined(DEBUG)
215 for (i = 0; i < npages; i++) {
216 pg = ap->a_m[i];
217 KASSERT(pg == NULL || pg == PGO_DONTCARE);
218 }
219#endif /* defined(DEBUG) */
220 nfound = uvn_findpages(uobj, origoffset, &npages,
221 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
222 KASSERT(npages == *ap->a_count);
223 if (nfound == 0) {
224 error = EBUSY;
225 goto out_err;
226 }
227 if (!genfs_node_rdtrylock(vp)) {
228 genfs_rel_pages(ap->a_m, npages);
229
230 /*
231 * restore the array.
232 */
233
234 for (i = 0; i < npages; i++) {
235 pg = ap->a_m[i];
236
237 if (pg != NULL && pg != PGO_DONTCARE) {
238 ap->a_m[i] = NULL;
239 }
240 KASSERT(ap->a_m[i] == NULL ||
241 ap->a_m[i] == PGO_DONTCARE);
242 }
243 } else {
244 genfs_node_unlock(vp);
245 }
246 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0);
247 if (error == 0 && memwrite) {
248 genfs_markdirty(vp);
249 }
250 goto out_err;
251 }
252 mutex_exit(uobj->vmobjlock);
253
254 /*
255 * find the requested pages and make some simple checks.
256 * leave space in the page array for a whole block.
257 */
258
259 const int fs_bshift = (vp->v_type != VBLK) ?
260 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT;
261 const int fs_bsize = 1 << fs_bshift;
262#define blk_mask (fs_bsize - 1)
263#define trunc_blk(x) ((x) & ~blk_mask)
264#define round_blk(x) (((x) + blk_mask) & ~blk_mask)
265
266 const int orignmempages = MIN(orignpages,
267 round_page(memeof - origoffset) >> PAGE_SHIFT);
268 npages = orignmempages;
269 const off_t startoffset = trunc_blk(origoffset);
270 const off_t endoffset = MIN(
271 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))),
272 round_page(memeof));
273 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT;
274
275 const int pgs_size = sizeof(struct vm_page *) *
276 ((endoffset - startoffset) >> PAGE_SHIFT);
277 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES];
278
279 if (pgs_size > sizeof(pgs_onstack)) {
280 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP);
281 if (pgs == NULL) {
282 pgs = pgs_onstack;
283 error = ENOMEM;
284 goto out_err;
285 }
286 } else {
287 pgs = pgs_onstack;
288 (void)memset(pgs, 0, pgs_size);
289 }
290
291 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld",
292 ridx, npages, startoffset, endoffset);
293
294 if (!has_trans_wapbl) {
295 fstrans_start(vp->v_mount, FSTRANS_SHARED);
296 /*
297 * XXX: This assumes that we come here only via
298 * the mmio path
299 */
300 if (need_wapbl) {
301 error = WAPBL_BEGIN(vp->v_mount);
302 if (error) {
303 fstrans_done(vp->v_mount);
304 goto out_err_free;
305 }
306 }
307 has_trans_wapbl = true;
308 }
309
310 /*
311 * hold g_glock to prevent a race with truncate.
312 *
313 * check if our idea of v_size is still valid.
314 */
315
316 KASSERT(!glocked || genfs_node_wrlocked(vp));
317 if (!glocked) {
318 if (blockalloc) {
319 genfs_node_wrlock(vp);
320 } else {
321 genfs_node_rdlock(vp);
322 }
323 }
324 mutex_enter(uobj->vmobjlock);
325 if (vp->v_size < origvsize) {
326 if (!glocked) {
327 genfs_node_unlock(vp);
328 }
329 if (pgs != pgs_onstack)
330 kmem_free(pgs, pgs_size);
331 goto startover;
332 }
333
334 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx],
335 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) {
336 if (!glocked) {
337 genfs_node_unlock(vp);
338 }
339 KASSERT(async != 0);
340 genfs_rel_pages(&pgs[ridx], orignmempages);
341 mutex_exit(uobj->vmobjlock);
342 error = EBUSY;
343 goto out_err_free;
344 }
345
346 /*
347 * if the pages are already resident, just return them.
348 */
349
350 for (i = 0; i < npages; i++) {
351 struct vm_page *pg = pgs[ridx + i];
352
353 if ((pg->flags & PG_FAKE) ||
354 (blockalloc && (pg->flags & PG_RDONLY))) {
355 break;
356 }
357 }
358 if (i == npages) {
359 if (!glocked) {
360 genfs_node_unlock(vp);
361 }
362 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0);
363 npages += ridx;
364 goto out;
365 }
366
367 /*
368 * if PGO_OVERWRITE is set, don't bother reading the pages.
369 */
370
371 if (overwrite) {
372 if (!glocked) {
373 genfs_node_unlock(vp);
374 }
375 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0);
376
377 for (i = 0; i < npages; i++) {
378 struct vm_page *pg = pgs[ridx + i];
379
380 pg->flags &= ~(PG_RDONLY|PG_CLEAN);
381 }
382 npages += ridx;
383 goto out;
384 }
385
386 /*
387 * the page wasn't resident and we're not overwriting,
388 * so we're going to have to do some i/o.
389 * find any additional pages needed to cover the expanded range.
390 */
391
392 npages = (endoffset - startoffset) >> PAGE_SHIFT;
393 if (startoffset != origoffset || npages != orignmempages) {
394 int npgs;
395
396 /*
397 * we need to avoid deadlocks caused by locking
398 * additional pages at lower offsets than pages we
399 * already have locked. unlock them all and start over.
400 */
401
402 genfs_rel_pages(&pgs[ridx], orignmempages);
403 memset(pgs, 0, pgs_size);
404
405 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x",
406 startoffset, endoffset, 0,0);
407 npgs = npages;
408 if (uvn_findpages(uobj, startoffset, &npgs, pgs,
409 async ? UFP_NOWAIT : UFP_ALL) != npages) {
410 if (!glocked) {
411 genfs_node_unlock(vp);
412 }
413 KASSERT(async != 0);
414 genfs_rel_pages(pgs, npages);
415 mutex_exit(uobj->vmobjlock);
416 error = EBUSY;
417 goto out_err_free;
418 }
419 }
420
421 mutex_exit(uobj->vmobjlock);
422 error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof,
423 async, memwrite, blockalloc, glocked);
424 if (error == 0 && async)
425 goto out_err_free;
426 if (!glocked) {
427 genfs_node_unlock(vp);
428 }
429 mutex_enter(uobj->vmobjlock);
430
431 /*
432 * we're almost done! release the pages...
433 * for errors, we free the pages.
434 * otherwise we activate them and mark them as valid and clean.
435 * also, unbusy pages that were not actually requested.
436 */
437
438 if (error) {
439 genfs_rel_pages(pgs, npages);
440 mutex_exit(uobj->vmobjlock);
441 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0);
442 goto out_err_free;
443 }
444
445out:
446 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0);
447 error = 0;
448 mutex_enter(&uvm_pageqlock);
449 for (i = 0; i < npages; i++) {
450 struct vm_page *pg = pgs[i];
451 if (pg == NULL) {
452 continue;
453 }
454 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x",
455 pg, pg->flags, 0,0);
456 if (pg->flags & PG_FAKE && !overwrite) {
457 pg->flags &= ~(PG_FAKE);
458 pmap_clear_modify(pgs[i]);
459 }
460 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0);
461 if (i < ridx || i >= ridx + orignmempages || async) {
462 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x",
463 pg, pg->offset,0,0);
464 if (pg->flags & PG_WANTED) {
465 wakeup(pg);
466 }
467 if (pg->flags & PG_FAKE) {
468 KASSERT(overwrite);
469 uvm_pagezero(pg);
470 }
471 if (pg->flags & PG_RELEASED) {
472 uvm_pagefree(pg);
473 continue;
474 }
475 uvm_pageenqueue(pg);
476 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
477 UVM_PAGE_OWN(pg, NULL);
478 }
479 }
480 mutex_exit(&uvm_pageqlock);
481 if (memwrite) {
482 genfs_markdirty(vp);
483 }
484 mutex_exit(uobj->vmobjlock);
485 if (ap->a_m != NULL) {
486 memcpy(ap->a_m, &pgs[ridx],
487 orignmempages * sizeof(struct vm_page *));
488 }
489
490out_err_free:
491 if (pgs != NULL && pgs != pgs_onstack)
492 kmem_free(pgs, pgs_size);
493out_err:
494 if (has_trans_wapbl) {
495 if (need_wapbl)
496 WAPBL_END(vp->v_mount);
497 fstrans_done(vp->v_mount);
498 }
499 return error;
500}
501
502/*
503 * genfs_getpages_read: Read the pages in with VOP_BMAP/VOP_STRATEGY.
504 */
505static int
506genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages,
507 off_t startoffset, off_t diskeof,
508 bool async, bool memwrite, bool blockalloc, bool glocked)
509{
510 struct uvm_object * const uobj = &vp->v_uobj;
511 const int fs_bshift = (vp->v_type != VBLK) ?
512 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT;
513 const int dev_bshift = (vp->v_type != VBLK) ?
514 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT;
515 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */
516 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes;
517 vaddr_t kva;
518 struct buf *bp, *mbp;
519 bool sawhole = false;
520 int i;
521 int error = 0;
522
523 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
524
525 /*
526 * read the desired page(s).
527 */
528
529 totalbytes = npages << PAGE_SHIFT;
530 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0));
531 tailbytes = totalbytes - bytes;
532 skipbytes = 0;
533
534 kva = uvm_pagermapin(pgs, npages,
535 UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK));
536 if (kva == 0)
537 return EBUSY;
538
539 mbp = getiobuf(vp, true);
540 mbp->b_bufsize = totalbytes;
541 mbp->b_data = (void *)kva;
542 mbp->b_resid = mbp->b_bcount = bytes;
543 mbp->b_cflags = BC_BUSY;
544 if (async) {
545 mbp->b_flags = B_READ | B_ASYNC;
546 mbp->b_iodone = uvm_aio_biodone;
547 } else {
548 mbp->b_flags = B_READ;
549 mbp->b_iodone = NULL;
550 }
551 if (async)
552 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED);
553 else
554 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL);
555
556 /*
557 * if EOF is in the middle of the range, zero the part past EOF.
558 * skip over pages which are not PG_FAKE since in that case they have
559 * valid data that we need to preserve.
560 */
561
562 tailstart = bytes;
563 while (tailbytes > 0) {
564 const int len = PAGE_SIZE - (tailstart & PAGE_MASK);
565
566 KASSERT(len <= tailbytes);
567 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) {
568 memset((void *)(kva + tailstart), 0, len);
569 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x",
570 kva, tailstart, len, 0);
571 }
572 tailstart += len;
573 tailbytes -= len;
574 }
575
576 /*
577 * now loop over the pages, reading as needed.
578 */
579
580 bp = NULL;
581 off_t offset;
582 for (offset = startoffset;
583 bytes > 0;
584 offset += iobytes, bytes -= iobytes) {
585 int run;
586 daddr_t lbn, blkno;
587 int pidx;
588 struct vnode *devvp;
589
590 /*
591 * skip pages which don't need to be read.
592 */
593
594 pidx = (offset - startoffset) >> PAGE_SHIFT;
595 while ((pgs[pidx]->flags & PG_FAKE) == 0) {
596 size_t b;
597
598 KASSERT((offset & (PAGE_SIZE - 1)) == 0);
599 if ((pgs[pidx]->flags & PG_RDONLY)) {
600 sawhole = true;
601 }
602 b = MIN(PAGE_SIZE, bytes);
603 offset += b;
604 bytes -= b;
605 skipbytes += b;
606 pidx++;
607 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x",
608 offset, 0,0,0);
609 if (bytes == 0) {
610 goto loopdone;
611 }
612 }
613
614 /*
615 * bmap the file to find out the blkno to read from and
616 * how much we can read in one i/o. if bmap returns an error,
617 * skip the rest of the top-level i/o.
618 */
619
620 lbn = offset >> fs_bshift;
621 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run);
622 if (error) {
623 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n",
624 lbn,error,0,0);
625 skipbytes += bytes;
626 bytes = 0;
627 goto loopdone;
628 }
629
630 /*
631 * see how many pages can be read with this i/o.
632 * reduce the i/o size if necessary to avoid
633 * overwriting pages with valid data.
634 */
635
636 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset,
637 bytes);
638 if (offset + iobytes > round_page(offset)) {
639 int pcount;
640
641 pcount = 1;
642 while (pidx + pcount < npages &&
643 pgs[pidx + pcount]->flags & PG_FAKE) {
644 pcount++;
645 }
646 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) -
647 (offset - trunc_page(offset)));
648 }
649
650 /*
651 * if this block isn't allocated, zero it instead of
652 * reading it. unless we are going to allocate blocks,
653 * mark the pages we zeroed PG_RDONLY.
654 */
655
656 if (blkno == (daddr_t)-1) {
657 int holepages = (round_page(offset + iobytes) -
658 trunc_page(offset)) >> PAGE_SHIFT;
659 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0);
660
661 sawhole = true;
662 memset((char *)kva + (offset - startoffset), 0,
663 iobytes);
664 skipbytes += iobytes;
665
666 mutex_enter(uobj->vmobjlock);
667 for (i = 0; i < holepages; i++) {
668 if (memwrite) {
669 pgs[pidx + i]->flags &= ~PG_CLEAN;
670 }
671 if (!blockalloc) {
672 pgs[pidx + i]->flags |= PG_RDONLY;
673 }
674 }
675 mutex_exit(uobj->vmobjlock);
676 continue;
677 }
678
679 /*
680 * allocate a sub-buf for this piece of the i/o
681 * (or just use mbp if there's only 1 piece),
682 * and start it going.
683 */
684
685 if (offset == startoffset && iobytes == bytes) {
686 bp = mbp;
687 } else {
688 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d",
689 vp, bp, vp->v_numoutput, 0);
690 bp = getiobuf(vp, true);
691 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes);
692 }
693 bp->b_lblkno = 0;
694
695 /* adjust physical blkno for partial blocks */
696 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >>
697 dev_bshift);
698
699 UVMHIST_LOG(ubchist,
700 "bp %p offset 0x%x bcount 0x%x blkno 0x%x",
701 bp, offset, bp->b_bcount, bp->b_blkno);
702
703 VOP_STRATEGY(devvp, bp);
704 }
705
706loopdone:
707 nestiobuf_done(mbp, skipbytes, error);
708 if (async) {
709 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0);
710 if (!glocked) {
711 genfs_node_unlock(vp);
712 }
713 return 0;
714 }
715 if (bp != NULL) {
716 error = biowait(mbp);
717 }
718
719 /* Remove the mapping (make KVA available as soon as possible) */
720 uvm_pagermapout(kva, npages);
721
722 /*
723 * if this we encountered a hole then we have to do a little more work.
724 * for read faults, we marked the page PG_RDONLY so that future
725 * write accesses to the page will fault again.
726 * for write faults, we must make sure that the backing store for
727 * the page is completely allocated while the pages are locked.
728 */
729
730 if (!error && sawhole && blockalloc) {
731 error = GOP_ALLOC(vp, startoffset,
732 npages << PAGE_SHIFT, 0, cred);
733 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d",
734 startoffset, npages << PAGE_SHIFT, error,0);
735 if (!error) {
736 mutex_enter(uobj->vmobjlock);
737 for (i = 0; i < npages; i++) {
738 struct vm_page *pg = pgs[i];
739
740 if (pg == NULL) {
741 continue;
742 }
743 pg->flags &= ~(PG_CLEAN|PG_RDONLY);
744 UVMHIST_LOG(ubchist, "mark dirty pg %p",
745 pg,0,0,0);
746 }
747 mutex_exit(uobj->vmobjlock);
748 }
749 }
750
751 putiobuf(mbp);
752 return error;
753}
754
755/*
756 * generic VM putpages routine.
757 * Write the given range of pages to backing store.
758 *
759 * => "offhi == 0" means flush all pages at or after "offlo".
760 * => object should be locked by caller. we return with the
761 * object unlocked.
762 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O).
763 * thus, a caller might want to unlock higher level resources
764 * (e.g. vm_map) before calling flush.
765 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block
766 * => if PGO_ALLPAGES is set, then all pages in the object will be processed.
767 * => NOTE: we rely on the fact that the object's memq is a TAILQ and
768 * that new pages are inserted on the tail end of the list. thus,
769 * we can make a complete pass through the object in one go by starting
770 * at the head and working towards the tail (new pages are put in
771 * front of us).
772 * => NOTE: we are allowed to lock the page queues, so the caller
773 * must not be holding the page queue lock.
774 *
775 * note on "cleaning" object and PG_BUSY pages:
776 * this routine is holding the lock on the object. the only time
777 * that it can run into a PG_BUSY page that it does not own is if
778 * some other process has started I/O on the page (e.g. either
779 * a pagein, or a pageout). if the PG_BUSY page is being paged
780 * in, then it can not be dirty (!PG_CLEAN) because no one has
781 * had a chance to modify it yet. if the PG_BUSY page is being
782 * paged out then it means that someone else has already started
783 * cleaning the page for us (how nice!). in this case, if we
784 * have syncio specified, then after we make our pass through the
785 * object we need to wait for the other PG_BUSY pages to clear
786 * off (i.e. we need to do an iosync). also note that once a
787 * page is PG_BUSY it must stay in its object until it is un-busyed.
788 *
789 * note on page traversal:
790 * we can traverse the pages in an object either by going down the
791 * linked list in "uobj->memq", or we can go over the address range
792 * by page doing hash table lookups for each address. depending
793 * on how many pages are in the object it may be cheaper to do one
794 * or the other. we set "by_list" to true if we are using memq.
795 * if the cost of a hash lookup was equal to the cost of the list
796 * traversal we could compare the number of pages in the start->stop
797 * range to the total number of pages in the object. however, it
798 * seems that a hash table lookup is more expensive than the linked
799 * list traversal, so we multiply the number of pages in the
800 * range by an estimate of the relatively higher cost of the hash lookup.
801 */
802
803int
804genfs_putpages(void *v)
805{
806 struct vop_putpages_args /* {
807 struct vnode *a_vp;
808 voff_t a_offlo;
809 voff_t a_offhi;
810 int a_flags;
811 } */ * const ap = v;
812
813 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi,
814 ap->a_flags, NULL);
815}
816
817int
818genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff,
819 int origflags, struct vm_page **busypg)
820{
821 struct uvm_object * const uobj = &vp->v_uobj;
822 kmutex_t * const slock = uobj->vmobjlock;
823 off_t off;
824 int i, error, npages, nback;
825 int freeflag;
826 /*
827 * This array is larger than it should so that it's size is constant.
828 * The right size is MAXPAGES.
829 */
830 struct vm_page *pgs[MAXPHYS / MIN_PAGE_SIZE];
831#define MAXPAGES (MAXPHYS / PAGE_SIZE)
832 struct vm_page *pg, *nextpg, *tpg, curmp, endmp;
833 bool wasclean, by_list, needs_clean, yld;
834 bool async = (origflags & PGO_SYNCIO) == 0;
835 bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
836 struct lwp * const l = curlwp ? curlwp : &lwp0;
837 struct genfs_node * const gp = VTOG(vp);
838 int flags;
839 int dirtygen;
840 bool modified;
841 bool need_wapbl;
842 bool has_trans;
843 bool cleanall;
844 bool onworklst;
845
846 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist);
847
848 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE));
849 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0);
850 KASSERT(startoff < endoff || endoff == 0);
851
852 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x",
853 vp, uobj->uo_npages, startoff, endoff - startoff);
854
855 has_trans = false;
856 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl &&
857 (origflags & PGO_JOURNALLOCKED) == 0);
858
859retry:
860 modified = false;
861 flags = origflags;
862 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 ||
863 (vp->v_iflag & VI_WRMAPDIRTY) == 0);
864 if (uobj->uo_npages == 0) {
865 if (vp->v_iflag & VI_ONWORKLST) {
866 vp->v_iflag &= ~VI_WRMAPDIRTY;
867 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
868 vn_syncer_remove_from_worklist(vp);
869 }
870 if (has_trans) {
871 if (need_wapbl)
872 WAPBL_END(vp->v_mount);
873 fstrans_done(vp->v_mount);
874 }
875 mutex_exit(slock);
876 return (0);
877 }
878
879 /*
880 * the vnode has pages, set up to process the request.
881 */
882
883 if (!has_trans && (flags & PGO_CLEANIT) != 0) {
884 mutex_exit(slock);
885 if (pagedaemon) {
886 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY);
887 if (error)
888 return error;
889 } else
890 fstrans_start(vp->v_mount, FSTRANS_LAZY);
891 if (need_wapbl) {
892 error = WAPBL_BEGIN(vp->v_mount);
893 if (error) {
894 fstrans_done(vp->v_mount);
895 return error;
896 }
897 }
898 has_trans = true;
899 mutex_enter(slock);
900 goto retry;
901 }
902
903 error = 0;
904 wasclean = (vp->v_numoutput == 0);
905 off = startoff;
906 if (endoff == 0 || flags & PGO_ALLPAGES) {
907 endoff = trunc_page(LLONG_MAX);
908 }
909 by_list = (uobj->uo_npages <=
910 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY);
911
912 /*
913 * if this vnode is known not to have dirty pages,
914 * don't bother to clean it out.
915 */
916
917 if ((vp->v_iflag & VI_ONWORKLST) == 0) {
918#if !defined(DEBUG)
919 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) {
920 goto skip_scan;
921 }
922#endif /* !defined(DEBUG) */
923 flags &= ~PGO_CLEANIT;
924 }
925
926 /*
927 * start the loop. when scanning by list, hold the last page
928 * in the list before we start. pages allocated after we start
929 * will be added to the end of the list, so we can stop at the
930 * current last page.
931 */
932
933 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean &&
934 startoff == 0 && endoff == trunc_page(LLONG_MAX) &&
935 (vp->v_iflag & VI_ONWORKLST) != 0;
936 dirtygen = gp->g_dirtygen;
937 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED;
938 if (by_list) {
939 curmp.flags = PG_MARKER;
940 endmp.flags = PG_MARKER;
941 pg = TAILQ_FIRST(&uobj->memq);
942 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue);
943 } else {
944 pg = uvm_pagelookup(uobj, off);
945 }
946 nextpg = NULL;
947 while (by_list || off < endoff) {
948
949 /*
950 * if the current page is not interesting, move on to the next.
951 */
952
953 KASSERT(pg == NULL || pg->uobject == uobj ||
954 (pg->flags & PG_MARKER) != 0);
955 KASSERT(pg == NULL ||
956 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
957 (pg->flags & (PG_BUSY|PG_MARKER)) != 0);
958 if (by_list) {
959 if (pg == &endmp) {
960 break;
961 }
962 if (pg->flags & PG_MARKER) {
963 pg = TAILQ_NEXT(pg, listq.queue);
964 continue;
965 }
966 if (pg->offset < startoff || pg->offset >= endoff ||
967 pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
968 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
969 wasclean = false;
970 }
971 pg = TAILQ_NEXT(pg, listq.queue);
972 continue;
973 }
974 off = pg->offset;
975 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
976 if (pg != NULL) {
977 wasclean = false;
978 }
979 off += PAGE_SIZE;
980 if (off < endoff) {
981 pg = uvm_pagelookup(uobj, off);
982 }
983 continue;
984 }
985
986 /*
987 * if the current page needs to be cleaned and it's busy,
988 * wait for it to become unbusy.
989 */
990
991 yld = (l->l_cpu->ci_schedstate.spc_flags &
992 SPCF_SHOULDYIELD) && !pagedaemon;
993 if (pg->flags & PG_BUSY || yld) {
994 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0);
995 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) {
996 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0);
997 error = EDEADLK;
998 if (busypg != NULL)
999 *busypg = pg;
1000 break;
1001 }
1002 if (pagedaemon) {
1003 /*
1004 * someone has taken the page while we
1005 * dropped the lock for fstrans_start.
1006 */
1007 break;
1008 }
1009 if (by_list) {
1010 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue);
1011 UVMHIST_LOG(ubchist, "curmp next %p",
1012 TAILQ_NEXT(&curmp, listq.queue), 0,0,0);
1013 }
1014 if (yld) {
1015 mutex_exit(slock);
1016 preempt();
1017 mutex_enter(slock);
1018 } else {
1019 pg->flags |= PG_WANTED;
1020 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
1021 mutex_enter(slock);
1022 }
1023 if (by_list) {
1024 UVMHIST_LOG(ubchist, "after next %p",
1025 TAILQ_NEXT(&curmp, listq.queue), 0,0,0);
1026 pg = TAILQ_NEXT(&curmp, listq.queue);
1027 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
1028 } else {
1029 pg = uvm_pagelookup(uobj, off);
1030 }
1031 continue;
1032 }
1033
1034 /*
1035 * if we're freeing, remove all mappings of the page now.
1036 * if we're cleaning, check if the page is needs to be cleaned.
1037 */
1038
1039 if (flags & PGO_FREE) {
1040 pmap_page_protect(pg, VM_PROT_NONE);
1041 } else if (flags & PGO_CLEANIT) {
1042
1043 /*
1044 * if we still have some hope to pull this vnode off
1045 * from the syncer queue, write-protect the page.
1046 */
1047
1048 if (cleanall && wasclean &&
1049 gp->g_dirtygen == dirtygen) {
1050
1051 /*
1052 * uobj pages get wired only by uvm_fault
1053 * where uobj is locked.
1054 */
1055
1056 if (pg->wire_count == 0) {
1057 pmap_page_protect(pg,
1058 VM_PROT_READ|VM_PROT_EXECUTE);
1059 } else {
1060 cleanall = false;
1061 }
1062 }
1063 }
1064
1065 if (flags & PGO_CLEANIT) {
1066 needs_clean = pmap_clear_modify(pg) ||
1067 (pg->flags & PG_CLEAN) == 0;
1068 pg->flags |= PG_CLEAN;
1069 } else {
1070 needs_clean = false;
1071 }
1072
1073 /*
1074 * if we're cleaning, build a cluster.
1075 * the cluster will consist of pages which are currently dirty,
1076 * but they will be returned to us marked clean.
1077 * if not cleaning, just operate on the one page.
1078 */
1079
1080 if (needs_clean) {
1081 KDASSERT((vp->v_iflag & VI_ONWORKLST));
1082 wasclean = false;
1083 memset(pgs, 0, sizeof(pgs));
1084 pg->flags |= PG_BUSY;
1085 UVM_PAGE_OWN(pg, "genfs_putpages");
1086
1087 /*
1088 * first look backward.
1089 */
1090
1091 npages = MIN(MAXPAGES >> 1, off >> PAGE_SHIFT);
1092 nback = npages;
1093 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0],
1094 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD);
1095 if (nback) {
1096 memmove(&pgs[0], &pgs[npages - nback],
1097 nback * sizeof(pgs[0]));
1098 if (npages - nback < nback)
1099 memset(&pgs[nback], 0,
1100 (npages - nback) * sizeof(pgs[0]));
1101 else
1102 memset(&pgs[npages - nback], 0,
1103 nback * sizeof(pgs[0]));
1104 }
1105
1106 /*
1107 * then plug in our page of interest.
1108 */
1109
1110 pgs[nback] = pg;
1111
1112 /*
1113 * then look forward to fill in the remaining space in
1114 * the array of pages.
1115 */
1116
1117 npages = MAXPAGES - nback - 1;
1118 uvn_findpages(uobj, off + PAGE_SIZE, &npages,
1119 &pgs[nback + 1],
1120 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY);
1121 npages += nback + 1;
1122 } else {
1123 pgs[0] = pg;
1124 npages = 1;
1125 nback = 0;
1126 }
1127
1128 /*
1129 * apply FREE or DEACTIVATE options if requested.
1130 */
1131
1132 if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1133 mutex_enter(&uvm_pageqlock);
1134 }
1135 for (i = 0; i < npages; i++) {
1136 tpg = pgs[i];
1137 KASSERT(tpg->uobject == uobj);
1138 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue))
1139 pg = tpg;
1140 if (tpg->offset < startoff || tpg->offset >= endoff)
1141 continue;
1142 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
1143 uvm_pagedeactivate(tpg);
1144 } else if (flags & PGO_FREE) {
1145 pmap_page_protect(tpg, VM_PROT_NONE);
1146 if (tpg->flags & PG_BUSY) {
1147 tpg->flags |= freeflag;
1148 if (pagedaemon) {
1149 uvm_pageout_start(1);
1150 uvm_pagedequeue(tpg);
1151 }
1152 } else {
1153
1154 /*
1155 * ``page is not busy''
1156 * implies that npages is 1
1157 * and needs_clean is false.
1158 */
1159
1160 nextpg = TAILQ_NEXT(tpg, listq.queue);
1161 uvm_pagefree(tpg);
1162 if (pagedaemon)
1163 uvmexp.pdfreed++;
1164 }
1165 }
1166 }
1167 if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1168 mutex_exit(&uvm_pageqlock);
1169 }
1170 if (needs_clean) {
1171 modified = true;
1172
1173 /*
1174 * start the i/o. if we're traversing by list,
1175 * keep our place in the list with a marker page.
1176 */
1177
1178 if (by_list) {
1179 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp,
1180 listq.queue);
1181 }
1182 mutex_exit(slock);
1183 error = GOP_WRITE(vp, pgs, npages, flags);
1184 mutex_enter(slock);
1185 if (by_list) {
1186 pg = TAILQ_NEXT(&curmp, listq.queue);
1187 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
1188 }
1189 if (error) {
1190 break;
1191 }
1192 if (by_list) {
1193 continue;
1194 }
1195 }
1196
1197 /*
1198 * find the next page and continue if there was no error.
1199 */
1200
1201 if (by_list) {
1202 if (nextpg) {
1203 pg = nextpg;
1204 nextpg = NULL;
1205 } else {
1206 pg = TAILQ_NEXT(pg, listq.queue);
1207 }
1208 } else {
1209 off += (npages - nback) << PAGE_SHIFT;
1210 if (off < endoff) {
1211 pg = uvm_pagelookup(uobj, off);
1212 }
1213 }
1214 }
1215 if (by_list) {
1216 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue);
1217 }
1218
1219 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 &&
1220 (vp->v_type != VBLK ||
1221 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
1222 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED);
1223 }
1224
1225 /*
1226 * if we're cleaning and there was nothing to clean,
1227 * take us off the syncer list. if we started any i/o
1228 * and we're doing sync i/o, wait for all writes to finish.
1229 */
1230
1231 if (cleanall && wasclean && gp->g_dirtygen == dirtygen &&
1232 (vp->v_iflag & VI_ONWORKLST) != 0) {
1233#if defined(DEBUG)
1234 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) {
1235 if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) {
1236 continue;
1237 }
1238 if ((pg->flags & PG_CLEAN) == 0) {
1239 printf("%s: %p: !CLEAN\n", __func__, pg);
1240 }
1241 if (pmap_is_modified(pg)) {
1242 printf("%s: %p: modified\n", __func__, pg);
1243 }
1244 }
1245#endif /* defined(DEBUG) */
1246 vp->v_iflag &= ~VI_WRMAPDIRTY;
1247 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
1248 vn_syncer_remove_from_worklist(vp);
1249 }
1250
1251#if !defined(DEBUG)
1252skip_scan:
1253#endif /* !defined(DEBUG) */
1254
1255 /* Wait for output to complete. */
1256 if (!wasclean && !async && vp->v_numoutput != 0) {
1257 while (vp->v_numoutput != 0)
1258 cv_wait(&vp->v_cv, slock);
1259 }
1260 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0;
1261 mutex_exit(slock);
1262
1263 if ((flags & PGO_RECLAIM) != 0 && onworklst) {
1264 /*
1265 * in the case of PGO_RECLAIM, ensure to make the vnode clean.
1266 * retrying is not a big deal because, in many cases,
1267 * uobj->uo_npages is already 0 here.
1268 */
1269 mutex_enter(slock);
1270 goto retry;
1271 }
1272
1273 if (has_trans) {
1274 if (need_wapbl)
1275 WAPBL_END(vp->v_mount);
1276 fstrans_done(vp->v_mount);
1277 }
1278
1279 return (error);
1280}
1281
1282int
1283genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
1284{
1285 off_t off;
1286 vaddr_t kva;
1287 size_t len;
1288 int error;
1289 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1290
1291 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x",
1292 vp, pgs, npages, flags);
1293
1294 off = pgs[0]->offset;
1295 kva = uvm_pagermapin(pgs, npages,
1296 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK);
1297 len = npages << PAGE_SHIFT;
1298
1299 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE,
1300 uvm_aio_biodone);
1301
1302 return error;
1303}
1304
1305int
1306genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
1307{
1308 off_t off;
1309 vaddr_t kva;
1310 size_t len;
1311 int error;
1312 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1313
1314 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x",
1315 vp, pgs, npages, flags);
1316
1317 off = pgs[0]->offset;
1318 kva = uvm_pagermapin(pgs, npages,
1319 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1320 len = npages << PAGE_SHIFT;
1321
1322 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE,
1323 uvm_aio_biodone);
1324
1325 return error;
1326}
1327
1328/*
1329 * Backend routine for doing I/O to vnode pages. Pages are already locked
1330 * and mapped into kernel memory. Here we just look up the underlying
1331 * device block addresses and call the strategy routine.
1332 */
1333
1334static int
1335genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags,
1336 enum uio_rw rw, void (*iodone)(struct buf *))
1337{
1338 int s, error;
1339 int fs_bshift, dev_bshift;
1340 off_t eof, offset, startoffset;
1341 size_t bytes, iobytes, skipbytes;
1342 struct buf *mbp, *bp;
1343 const bool async = (flags & PGO_SYNCIO) == 0;
1344 const bool lazy = (flags & PGO_LAZY) == 0;
1345 const bool iowrite = rw == UIO_WRITE;
1346 const int brw = iowrite ? B_WRITE : B_READ;
1347 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1348
1349 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x",
1350 vp, kva, len, flags);
1351
1352 KASSERT(vp->v_size <= vp->v_writesize);
1353 GOP_SIZE(vp, vp->v_writesize, &eof, 0);
1354 if (vp->v_type != VBLK) {
1355 fs_bshift = vp->v_mount->mnt_fs_bshift;
1356 dev_bshift = vp->v_mount->mnt_dev_bshift;
1357 } else {
1358 fs_bshift = DEV_BSHIFT;
1359 dev_bshift = DEV_BSHIFT;
1360 }
1361 error = 0;
1362 startoffset = off;
1363 bytes = MIN(len, eof - startoffset);
1364 skipbytes = 0;
1365 KASSERT(bytes != 0);
1366
1367 if (iowrite) {
1368 mutex_enter(vp->v_interlock);
1369 vp->v_numoutput += 2;
1370 mutex_exit(vp->v_interlock);
1371 }
1372 mbp = getiobuf(vp, true);
1373 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x",
1374 vp, mbp, vp->v_numoutput, bytes);
1375 mbp->b_bufsize = len;
1376 mbp->b_data = (void *)kva;
1377 mbp->b_resid = mbp->b_bcount = bytes;
1378 mbp->b_cflags = BC_BUSY | BC_AGE;
1379 if (async) {
1380 mbp->b_flags = brw | B_ASYNC;
1381 mbp->b_iodone = iodone;
1382 } else {
1383 mbp->b_flags = brw;
1384 mbp->b_iodone = NULL;
1385 }
1386 if (curlwp == uvm.pagedaemon_lwp)
1387 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED);
1388 else if (async || lazy)
1389 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL);
1390 else
1391 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL);
1392
1393 bp = NULL;
1394 for (offset = startoffset;
1395 bytes > 0;
1396 offset += iobytes, bytes -= iobytes) {
1397 int run;
1398 daddr_t lbn, blkno;
1399 struct vnode *devvp;
1400
1401 /*
1402 * bmap the file to find out the blkno to read from and
1403 * how much we can read in one i/o. if bmap returns an error,
1404 * skip the rest of the top-level i/o.
1405 */
1406
1407 lbn = offset >> fs_bshift;
1408 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run);
1409 if (error) {
1410 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n",
1411 lbn,error,0,0);
1412 skipbytes += bytes;
1413 bytes = 0;
1414 goto loopdone;
1415 }
1416
1417 /*
1418 * see how many pages can be read with this i/o.
1419 * reduce the i/o size if necessary to avoid
1420 * overwriting pages with valid data.
1421 */
1422
1423 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset,
1424 bytes);
1425
1426 /*
1427 * if this block isn't allocated, zero it instead of
1428 * reading it. unless we are going to allocate blocks,
1429 * mark the pages we zeroed PG_RDONLY.
1430 */
1431
1432 if (blkno == (daddr_t)-1) {
1433 if (!iowrite) {
1434 memset((char *)kva + (offset - startoffset), 0,
1435 iobytes);
1436 }
1437 skipbytes += iobytes;
1438 continue;
1439 }
1440
1441 /*
1442 * allocate a sub-buf for this piece of the i/o
1443 * (or just use mbp if there's only 1 piece),
1444 * and start it going.
1445 */
1446
1447 if (offset == startoffset && iobytes == bytes) {
1448 bp = mbp;
1449 } else {
1450 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d",
1451 vp, bp, vp->v_numoutput, 0);
1452 bp = getiobuf(vp, true);
1453 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes);
1454 }
1455 bp->b_lblkno = 0;
1456
1457 /* adjust physical blkno for partial blocks */
1458 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >>
1459 dev_bshift);
1460
1461 UVMHIST_LOG(ubchist,
1462 "bp %p offset 0x%x bcount 0x%x blkno 0x%x",
1463 bp, offset, bp->b_bcount, bp->b_blkno);
1464
1465 VOP_STRATEGY(devvp, bp);
1466 }
1467
1468loopdone:
1469 if (skipbytes) {
1470 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0);
1471 }
1472 nestiobuf_done(mbp, skipbytes, error);
1473 if (async) {
1474 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0);
1475 return (0);
1476 }
1477 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0);
1478 error = biowait(mbp);
1479 s = splbio();
1480 (*iodone)(mbp);
1481 splx(s);
1482 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0);
1483 return (error);
1484}
1485
1486int
1487genfs_compat_getpages(void *v)
1488{
1489 struct vop_getpages_args /* {
1490 struct vnode *a_vp;
1491 voff_t a_offset;
1492 struct vm_page **a_m;
1493 int *a_count;
1494 int a_centeridx;
1495 vm_prot_t a_access_type;
1496 int a_advice;
1497 int a_flags;
1498 } */ *ap = v;
1499
1500 off_t origoffset;
1501 struct vnode *vp = ap->a_vp;
1502 struct uvm_object *uobj = &vp->v_uobj;
1503 struct vm_page *pg, **pgs;
1504 vaddr_t kva;
1505 int i, error, orignpages, npages;
1506 struct iovec iov;
1507 struct uio uio;
1508 kauth_cred_t cred = curlwp->l_cred;
1509 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
1510
1511 error = 0;
1512 origoffset = ap->a_offset;
1513 orignpages = *ap->a_count;
1514 pgs = ap->a_m;
1515
1516 if (ap->a_flags & PGO_LOCKED) {
1517 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m,
1518 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0));
1519
1520 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0;
1521 if (error == 0 && memwrite) {
1522 genfs_markdirty(vp);
1523 }
1524 return error;
1525 }
1526 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) {
1527 mutex_exit(uobj->vmobjlock);
1528 return EINVAL;
1529 }
1530 if ((ap->a_flags & PGO_SYNCIO) == 0) {
1531 mutex_exit(uobj->vmobjlock);
1532 return 0;
1533 }
1534 npages = orignpages;
1535 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL);
1536 mutex_exit(uobj->vmobjlock);
1537 kva = uvm_pagermapin(pgs, npages,
1538 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1539 for (i = 0; i < npages; i++) {
1540 pg = pgs[i];
1541 if ((pg->flags & PG_FAKE) == 0) {
1542 continue;
1543 }
1544 iov.iov_base = (char *)kva + (i << PAGE_SHIFT);
1545 iov.iov_len = PAGE_SIZE;
1546 uio.uio_iov = &iov;
1547 uio.uio_iovcnt = 1;
1548 uio.uio_offset = origoffset + (i << PAGE_SHIFT);
1549 uio.uio_rw = UIO_READ;
1550 uio.uio_resid = PAGE_SIZE;
1551 UIO_SETUP_SYSSPACE(&uio);
1552 /* XXX vn_lock */
1553 error = VOP_READ(vp, &uio, 0, cred);
1554 if (error) {
1555 break;
1556 }
1557 if (uio.uio_resid) {
1558 memset(iov.iov_base, 0, uio.uio_resid);
1559 }
1560 }
1561 uvm_pagermapout(kva, npages);
1562 mutex_enter(uobj->vmobjlock);
1563 mutex_enter(&uvm_pageqlock);
1564 for (i = 0; i < npages; i++) {
1565 pg = pgs[i];
1566 if (error && (pg->flags & PG_FAKE) != 0) {
1567 pg->flags |= PG_RELEASED;
1568 } else {
1569 pmap_clear_modify(pg);
1570 uvm_pageactivate(pg);
1571 }
1572 }
1573 if (error) {
1574 uvm_page_unbusy(pgs, npages);
1575 }
1576 mutex_exit(&uvm_pageqlock);
1577 if (error == 0 && memwrite) {
1578 genfs_markdirty(vp);
1579 }
1580 mutex_exit(uobj->vmobjlock);
1581 return error;
1582}
1583
1584int
1585genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages,
1586 int flags)
1587{
1588 off_t offset;
1589 struct iovec iov;
1590 struct uio uio;
1591 kauth_cred_t cred = curlwp->l_cred;
1592 struct buf *bp;
1593 vaddr_t kva;
1594 int error;
1595
1596 offset = pgs[0]->offset;
1597 kva = uvm_pagermapin(pgs, npages,
1598 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK);
1599
1600 iov.iov_base = (void *)kva;
1601 iov.iov_len = npages << PAGE_SHIFT;
1602 uio.uio_iov = &iov;
1603 uio.uio_iovcnt = 1;
1604 uio.uio_offset = offset;
1605 uio.uio_rw = UIO_WRITE;
1606 uio.uio_resid = npages << PAGE_SHIFT;
1607 UIO_SETUP_SYSSPACE(&uio);
1608 /* XXX vn_lock */
1609 error = VOP_WRITE(vp, &uio, 0, cred);
1610
1611 mutex_enter(vp->v_interlock);
1612 vp->v_numoutput++;
1613 mutex_exit(vp->v_interlock);
1614
1615 bp = getiobuf(vp, true);
1616 bp->b_cflags = BC_BUSY | BC_AGE;
1617 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift;
1618 bp->b_data = (char *)kva;
1619 bp->b_bcount = npages << PAGE_SHIFT;
1620 bp->b_bufsize = npages << PAGE_SHIFT;
1621 bp->b_resid = 0;
1622 bp->b_error = error;
1623 uvm_aio_aiodone(bp);
1624 return (error);
1625}
1626
1627/*
1628 * Process a uio using direct I/O. If we reach a part of the request
1629 * which cannot be processed in this fashion for some reason, just return.
1630 * The caller must handle some additional part of the request using
1631 * buffered I/O before trying direct I/O again.
1632 */
1633
1634void
1635genfs_directio(struct vnode *vp, struct uio *uio, int ioflag)
1636{
1637 struct vmspace *vs;
1638 struct iovec *iov;
1639 vaddr_t va;
1640 size_t len;
1641 const int mask = DEV_BSIZE - 1;
1642 int error;
1643 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl &&
1644 (ioflag & IO_JOURNALLOCKED) == 0);
1645
1646 /*
1647 * We only support direct I/O to user space for now.
1648 */
1649
1650 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) {
1651 return;
1652 }
1653
1654 /*
1655 * If the vnode is mapped, we would need to get the getpages lock
1656 * to stabilize the bmap, but then we would get into trouble while
1657 * locking the pages if the pages belong to this same vnode (or a
1658 * multi-vnode cascade to the same effect). Just fall back to
1659 * buffered I/O if the vnode is mapped to avoid this mess.
1660 */
1661
1662 if (vp->v_vflag & VV_MAPPED) {
1663 return;
1664 }
1665
1666 if (need_wapbl) {
1667 error = WAPBL_BEGIN(vp->v_mount);
1668 if (error)
1669 return;
1670 }
1671
1672 /*
1673 * Do as much of the uio as possible with direct I/O.
1674 */
1675
1676 vs = uio->uio_vmspace;
1677 while (uio->uio_resid) {
1678 iov = uio->uio_iov;
1679 if (iov->iov_len == 0) {
1680 uio->uio_iov++;
1681 uio->uio_iovcnt--;
1682 continue;
1683 }
1684 va = (vaddr_t)iov->iov_base;
1685 len = MIN(iov->iov_len, genfs_maxdio);
1686 len &= ~mask;
1687
1688 /*
1689 * If the next chunk is smaller than DEV_BSIZE or extends past
1690 * the current EOF, then fall back to buffered I/O.
1691 */
1692
1693 if (len == 0 || uio->uio_offset + len > vp->v_size) {
1694 break;
1695 }
1696
1697 /*
1698 * Check alignment. The file offset must be at least
1699 * sector-aligned. The exact constraint on memory alignment
1700 * is very hardware-dependent, but requiring sector-aligned
1701 * addresses there too is safe.
1702 */
1703
1704 if (uio->uio_offset & mask || va & mask) {
1705 break;
1706 }
1707 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset,
1708 uio->uio_rw);
1709 if (error) {
1710 break;
1711 }
1712 iov->iov_base = (char *)iov->iov_base + len;
1713 iov->iov_len -= len;
1714 uio->uio_offset += len;
1715 uio->uio_resid -= len;
1716 }
1717
1718 if (need_wapbl)
1719 WAPBL_END(vp->v_mount);
1720}
1721
1722/*
1723 * Iodone routine for direct I/O. We don't do much here since the request is
1724 * always synchronous, so the caller will do most of the work after biowait().
1725 */
1726
1727static void
1728genfs_dio_iodone(struct buf *bp)
1729{
1730
1731 KASSERT((bp->b_flags & B_ASYNC) == 0);
1732 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) {
1733 mutex_enter(bp->b_objlock);
1734 vwakeup(bp);
1735 mutex_exit(bp->b_objlock);
1736 }
1737 putiobuf(bp);
1738}
1739
1740/*
1741 * Process one chunk of a direct I/O request.
1742 */
1743
1744static int
1745genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp,
1746 off_t off, enum uio_rw rw)
1747{
1748 struct vm_map *map;
1749 struct pmap *upm, *kpm __unused;
1750 size_t klen = round_page(uva + len) - trunc_page(uva);
1751 off_t spoff, epoff;
1752 vaddr_t kva, puva;
1753 paddr_t pa;
1754 vm_prot_t prot;
1755 int error, rv __diagused, poff, koff;
1756 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED |
1757 (rw == UIO_WRITE ? PGO_FREE : 0);
1758
1759 /*
1760 * For writes, verify that this range of the file already has fully
1761 * allocated backing store. If there are any holes, just punt and
1762 * make the caller take the buffered write path.
1763 */
1764
1765 if (rw == UIO_WRITE) {
1766 daddr_t lbn, elbn, blkno;
1767 int bsize, bshift, run;
1768
1769 bshift = vp->v_mount->mnt_fs_bshift;
1770 bsize = 1 << bshift;
1771 lbn = off >> bshift;
1772 elbn = (off + len + bsize - 1) >> bshift;
1773 while (lbn < elbn) {
1774 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run);
1775 if (error) {
1776 return error;
1777 }
1778 if (blkno == (daddr_t)-1) {
1779 return ENOSPC;
1780 }
1781 lbn += 1 + run;
1782 }
1783 }
1784
1785 /*
1786 * Flush any cached pages for parts of the file that we're about to
1787 * access. If we're writing, invalidate pages as well.
1788 */
1789
1790 spoff = trunc_page(off);
1791 epoff = round_page(off + len);
1792 mutex_enter(vp->v_interlock);
1793 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags);
1794 if (error) {
1795 return error;
1796 }
1797
1798 /*
1799 * Wire the user pages and remap them into kernel memory.
1800 */
1801
1802 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ;
1803 error = uvm_vslock(vs, (void *)uva, len, prot);
1804 if (error) {
1805 return error;
1806 }
1807
1808 map = &vs->vm_map;
1809 upm = vm_map_pmap(map);
1810 kpm = vm_map_pmap(kernel_map);
1811 puva = trunc_page(uva);
1812 kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask,
1813 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
1814 for (poff = 0; poff < klen; poff += PAGE_SIZE) {
1815 rv = pmap_extract(upm, puva + poff, &pa);
1816 KASSERT(rv);
1817 pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED);
1818 }
1819 pmap_update(kpm);
1820
1821 /*
1822 * Do the I/O.
1823 */
1824
1825 koff = uva - trunc_page(uva);
1826 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw,
1827 genfs_dio_iodone);
1828
1829 /*
1830 * Tear down the kernel mapping.
1831 */
1832
1833 pmap_kremove(kva, klen);
1834 pmap_update(kpm);
1835 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY);
1836
1837 /*
1838 * Unwire the user pages.
1839 */
1840
1841 uvm_vsunlock(vs, (void *)uva, len);
1842 return error;
1843}
1844