1/* $NetBSD: rf_dagffrd.c,v 1.19 2013/09/15 12:23:06 martin Exp $ */
2/*
3 * Copyright (c) 1995 Carnegie-Mellon University.
4 * All rights reserved.
5 *
6 * Author: Mark Holland, Daniel Stodolsky, William V. Courtright II
7 *
8 * Permission to use, copy, modify and distribute this software and
9 * its documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17 *
18 * Carnegie Mellon requests users of this software to return to
19 *
20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
21 * School of Computer Science
22 * Carnegie Mellon University
23 * Pittsburgh PA 15213-3890
24 *
25 * any improvements or extensions that they make and grant Carnegie the
26 * rights to redistribute these changes.
27 */
28
29/*
30 * rf_dagffrd.c
31 *
32 * code for creating fault-free read DAGs
33 *
34 */
35
36#include <sys/cdefs.h>
37__KERNEL_RCSID(0, "$NetBSD: rf_dagffrd.c,v 1.19 2013/09/15 12:23:06 martin Exp $");
38
39#include <dev/raidframe/raidframevar.h>
40
41#include "rf_raid.h"
42#include "rf_dag.h"
43#include "rf_dagutils.h"
44#include "rf_dagfuncs.h"
45#include "rf_debugMem.h"
46#include "rf_general.h"
47#include "rf_dagffrd.h"
48
49/******************************************************************************
50 *
51 * General comments on DAG creation:
52 *
53 * All DAGs in this file use roll-away error recovery. Each DAG has a single
54 * commit node, usually called "Cmt." If an error occurs before the Cmt node
55 * is reached, the execution engine will halt forward execution and work
56 * backward through the graph, executing the undo functions. Assuming that
57 * each node in the graph prior to the Cmt node are undoable and atomic - or -
58 * does not make changes to permanent state, the graph will fail atomically.
59 * If an error occurs after the Cmt node executes, the engine will roll-forward
60 * through the graph, blindly executing nodes until it reaches the end.
61 * If a graph reaches the end, it is assumed to have completed successfully.
62 *
63 * A graph has only 1 Cmt node.
64 *
65 */
66
67
68/******************************************************************************
69 *
70 * The following wrappers map the standard DAG creation interface to the
71 * DAG creation routines. Additionally, these wrappers enable experimentation
72 * with new DAG structures by providing an extra level of indirection, allowing
73 * the DAG creation routines to be replaced at this single point.
74 */
75
76void
77rf_CreateFaultFreeReadDAG(RF_Raid_t *raidPtr, RF_AccessStripeMap_t *asmap,
78 RF_DagHeader_t *dag_h, void *bp,
79 RF_RaidAccessFlags_t flags,
80 RF_AllocListElem_t *allocList)
81{
82 rf_CreateNonredundantDAG(raidPtr, asmap, dag_h, bp, flags, allocList,
83 RF_IO_TYPE_READ);
84}
85
86
87/******************************************************************************
88 *
89 * DAG creation code begins here
90 */
91
92/******************************************************************************
93 *
94 * creates a DAG to perform a nonredundant read or write of data within one
95 * stripe.
96 * For reads, this DAG is as follows:
97 *
98 * /---- read ----\
99 * Header -- Block ---- read ---- Commit -- Terminate
100 * \---- read ----/
101 *
102 * For writes, this DAG is as follows:
103 *
104 * /---- write ----\
105 * Header -- Commit ---- write ---- Block -- Terminate
106 * \---- write ----/
107 *
108 * There is one disk node per stripe unit accessed, and all disk nodes are in
109 * parallel.
110 *
111 * Tricky point here: The first disk node (read or write) is created
112 * normally. Subsequent disk nodes are created by copying the first one,
113 * and modifying a few params. The "succedents" and "antecedents" fields are
114 * _not_ re-created in each node, but rather left pointing to the same array
115 * that was malloc'd when the first node was created. Thus, it's essential
116 * that when this DAG is freed, the succedents and antecedents fields be freed
117 * in ONLY ONE of the read nodes. This does not apply to the "params" field
118 * because it is recreated for each READ node.
119 *
120 * Note that normal-priority accesses do not need to be tagged with their
121 * parity stripe ID, because they will never be promoted. Hence, I've
122 * commented-out the code to do this, and marked it with UNNEEDED.
123 *
124 *****************************************************************************/
125
126void
127rf_CreateNonredundantDAG(RF_Raid_t *raidPtr,
128 RF_AccessStripeMap_t *asmap, RF_DagHeader_t *dag_h, void *bp,
129 RF_RaidAccessFlags_t flags, RF_AllocListElem_t *allocList,
130 RF_IoType_t type)
131{
132 RF_DagNode_t *diskNodes, *blockNode, *commitNode, *termNode;
133 RF_DagNode_t *tmpNode, *tmpdiskNode;
134 RF_PhysDiskAddr_t *pda = asmap->physInfo;
135 int (*doFunc) (RF_DagNode_t *), (*undoFunc) (RF_DagNode_t *);
136 int i, n;
137 const char *name;
138
139 n = asmap->numStripeUnitsAccessed;
140 dag_h->creator = "NonredundantDAG";
141
142 RF_ASSERT(RF_IO_IS_R_OR_W(type));
143 switch (type) {
144 case RF_IO_TYPE_READ:
145 doFunc = rf_DiskReadFunc;
146 undoFunc = rf_DiskReadUndoFunc;
147 name = "R ";
148#if RF_DEBUG_DAG
149 if (rf_dagDebug)
150 printf("[Creating non-redundant read DAG]\n");
151#endif
152 break;
153 case RF_IO_TYPE_WRITE:
154 doFunc = rf_DiskWriteFunc;
155 undoFunc = rf_DiskWriteUndoFunc;
156 name = "W ";
157#if RF_DEBUG_DAG
158 if (rf_dagDebug)
159 printf("[Creating non-redundant write DAG]\n");
160#endif
161 break;
162 default:
163 RF_PANIC();
164 }
165
166 /*
167 * For reads, the dag can not commit until the block node is reached.
168 * for writes, the dag commits immediately.
169 */
170 dag_h->numCommitNodes = 1;
171 dag_h->numCommits = 0;
172 dag_h->numSuccedents = 1;
173
174 /*
175 * Node count:
176 * 1 block node
177 * n data reads (or writes)
178 * 1 commit node
179 * 1 terminator node
180 */
181 RF_ASSERT(n > 0);
182
183 for (i = 0; i < n; i++) {
184 tmpNode = rf_AllocDAGNode();
185 tmpNode->list_next = dag_h->nodes;
186 dag_h->nodes = tmpNode;
187 }
188 diskNodes = dag_h->nodes;
189
190 blockNode = rf_AllocDAGNode();
191 blockNode->list_next = dag_h->nodes;
192 dag_h->nodes = blockNode;
193
194 commitNode = rf_AllocDAGNode();
195 commitNode->list_next = dag_h->nodes;
196 dag_h->nodes = commitNode;
197
198 termNode = rf_AllocDAGNode();
199 termNode->list_next = dag_h->nodes;
200 dag_h->nodes = termNode;
201
202 /* initialize nodes */
203 switch (type) {
204 case RF_IO_TYPE_READ:
205 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc,
206 NULL, n, 0, 0, 0, dag_h, "Nil", allocList);
207 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc,
208 NULL, 1, n, 0, 0, dag_h, "Cmt", allocList);
209 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc,
210 NULL, 0, 1, 0, 0, dag_h, "Trm", allocList);
211 break;
212 case RF_IO_TYPE_WRITE:
213 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc,
214 NULL, 1, 0, 0, 0, dag_h, "Nil", allocList);
215 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc,
216 NULL, n, 1, 0, 0, dag_h, "Cmt", allocList);
217 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc,
218 NULL, 0, n, 0, 0, dag_h, "Trm", allocList);
219 break;
220 default:
221 RF_PANIC();
222 }
223
224 tmpdiskNode = diskNodes;
225 for (i = 0; i < n; i++) {
226 RF_ASSERT(pda != NULL);
227 rf_InitNode(tmpdiskNode, rf_wait, RF_FALSE, doFunc, undoFunc, rf_GenericWakeupFunc,
228 1, 1, 4, 0, dag_h, name, allocList);
229 tmpdiskNode->params[0].p = pda;
230 tmpdiskNode->params[1].p = pda->bufPtr;
231 /* parity stripe id is not necessary */
232 tmpdiskNode->params[2].v = 0;
233 tmpdiskNode->params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0);
234 pda = pda->next;
235 tmpdiskNode = tmpdiskNode->list_next;
236 }
237
238 /*
239 * Connect nodes.
240 */
241
242 /* connect hdr to block node */
243 RF_ASSERT(blockNode->numAntecedents == 0);
244 dag_h->succedents[0] = blockNode;
245
246 if (type == RF_IO_TYPE_READ) {
247 /* connecting a nonredundant read DAG */
248 RF_ASSERT(blockNode->numSuccedents == n);
249 RF_ASSERT(commitNode->numAntecedents == n);
250 tmpdiskNode = diskNodes;
251 for (i = 0; i < n; i++) {
252 /* connect block node to each read node */
253 RF_ASSERT(tmpdiskNode->numAntecedents == 1);
254 blockNode->succedents[i] = tmpdiskNode;
255 tmpdiskNode->antecedents[0] = blockNode;
256 tmpdiskNode->antType[0] = rf_control;
257
258 /* connect each read node to the commit node */
259 RF_ASSERT(tmpdiskNode->numSuccedents == 1);
260 tmpdiskNode->succedents[0] = commitNode;
261 commitNode->antecedents[i] = tmpdiskNode;
262 commitNode->antType[i] = rf_control;
263 tmpdiskNode = tmpdiskNode->list_next;
264 }
265 /* connect the commit node to the term node */
266 RF_ASSERT(commitNode->numSuccedents == 1);
267 RF_ASSERT(termNode->numAntecedents == 1);
268 RF_ASSERT(termNode->numSuccedents == 0);
269 commitNode->succedents[0] = termNode;
270 termNode->antecedents[0] = commitNode;
271 termNode->antType[0] = rf_control;
272 } else {
273 /* connecting a nonredundant write DAG */
274 /* connect the block node to the commit node */
275 RF_ASSERT(blockNode->numSuccedents == 1);
276 RF_ASSERT(commitNode->numAntecedents == 1);
277 blockNode->succedents[0] = commitNode;
278 commitNode->antecedents[0] = blockNode;
279 commitNode->antType[0] = rf_control;
280
281 RF_ASSERT(commitNode->numSuccedents == n);
282 RF_ASSERT(termNode->numAntecedents == n);
283 RF_ASSERT(termNode->numSuccedents == 0);
284 tmpdiskNode = diskNodes;
285 for (i = 0; i < n; i++) {
286 /* connect the commit node to each write node */
287 RF_ASSERT(tmpdiskNode->numAntecedents == 1);
288 commitNode->succedents[i] = tmpdiskNode;
289 tmpdiskNode->antecedents[0] = commitNode;
290 tmpdiskNode->antType[0] = rf_control;
291
292 /* connect each write node to the term node */
293 RF_ASSERT(tmpdiskNode->numSuccedents == 1);
294 tmpdiskNode->succedents[0] = termNode;
295 termNode->antecedents[i] = tmpdiskNode;
296 termNode->antType[i] = rf_control;
297 tmpdiskNode = tmpdiskNode->list_next;
298 }
299 }
300}
301/******************************************************************************
302 * Create a fault-free read DAG for RAID level 1
303 *
304 * Hdr -> Nil -> Rmir -> Cmt -> Trm
305 *
306 * The "Rmir" node schedules a read from the disk in the mirror pair with the
307 * shortest disk queue. the proper queue is selected at Rmir execution. this
308 * deferred mapping is unlike other archs in RAIDframe which generally fix
309 * mapping at DAG creation time.
310 *
311 * Parameters: raidPtr - description of the physical array
312 * asmap - logical & physical addresses for this access
313 * bp - buffer ptr (for holding read data)
314 * flags - general flags (e.g. disk locking)
315 * allocList - list of memory allocated in DAG creation
316 *****************************************************************************/
317
318static void
319CreateMirrorReadDAG(RF_Raid_t *raidPtr, RF_AccessStripeMap_t *asmap,
320 RF_DagHeader_t *dag_h, void *bp,
321 RF_RaidAccessFlags_t flags, RF_AllocListElem_t *allocList,
322 int (*readfunc) (RF_DagNode_t * node))
323{
324 RF_DagNode_t *readNodes, *blockNode, *commitNode, *termNode;
325 RF_DagNode_t *tmpNode, *tmpreadNode;
326 RF_PhysDiskAddr_t *data_pda = asmap->physInfo;
327 RF_PhysDiskAddr_t *parity_pda = asmap->parityInfo;
328 int i, n;
329
330 n = asmap->numStripeUnitsAccessed;
331 dag_h->creator = "RaidOneReadDAG";
332#if RF_DEBUG_DAG
333 if (rf_dagDebug) {
334 printf("[Creating RAID level 1 read DAG]\n");
335 }
336#endif
337 /*
338 * This dag can not commit until the commit node is reached
339 * errors prior to the commit point imply the dag has failed.
340 */
341 dag_h->numCommitNodes = 1;
342 dag_h->numCommits = 0;
343 dag_h->numSuccedents = 1;
344
345 /*
346 * Node count:
347 * n data reads
348 * 1 block node
349 * 1 commit node
350 * 1 terminator node
351 */
352 RF_ASSERT(n > 0);
353
354 for (i = 0; i < n; i++) {
355 tmpNode = rf_AllocDAGNode();
356 tmpNode->list_next = dag_h->nodes;
357 dag_h->nodes = tmpNode;
358 }
359 readNodes = dag_h->nodes;
360
361 blockNode = rf_AllocDAGNode();
362 blockNode->list_next = dag_h->nodes;
363 dag_h->nodes = blockNode;
364
365 commitNode = rf_AllocDAGNode();
366 commitNode->list_next = dag_h->nodes;
367 dag_h->nodes = commitNode;
368
369 termNode = rf_AllocDAGNode();
370 termNode->list_next = dag_h->nodes;
371 dag_h->nodes = termNode;
372
373 /* initialize nodes */
374 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc,
375 rf_NullNodeUndoFunc, NULL, n, 0, 0, 0, dag_h, "Nil", allocList);
376 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc,
377 rf_NullNodeUndoFunc, NULL, 1, n, 0, 0, dag_h, "Cmt", allocList);
378 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc,
379 rf_TerminateUndoFunc, NULL, 0, 1, 0, 0, dag_h, "Trm", allocList);
380
381 tmpreadNode = readNodes;
382 for (i = 0; i < n; i++) {
383 RF_ASSERT(data_pda != NULL);
384 RF_ASSERT(parity_pda != NULL);
385 rf_InitNode(tmpreadNode, rf_wait, RF_FALSE, readfunc,
386 rf_DiskReadMirrorUndoFunc, rf_GenericWakeupFunc, 1, 1, 5, 0, dag_h,
387 "Rmir", allocList);
388 tmpreadNode->params[0].p = data_pda;
389 tmpreadNode->params[1].p = data_pda->bufPtr;
390 /* parity stripe id is not necessary */
391 tmpreadNode->params[2].p = 0;
392 tmpreadNode->params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0);
393 tmpreadNode->params[4].p = parity_pda;
394 data_pda = data_pda->next;
395 parity_pda = parity_pda->next;
396 tmpreadNode = tmpreadNode->list_next;
397 }
398
399 /*
400 * Connect nodes
401 */
402
403 /* connect hdr to block node */
404 RF_ASSERT(blockNode->numAntecedents == 0);
405 dag_h->succedents[0] = blockNode;
406
407 /* connect block node to read nodes */
408 RF_ASSERT(blockNode->numSuccedents == n);
409 tmpreadNode = readNodes;
410 for (i = 0; i < n; i++) {
411 RF_ASSERT(tmpreadNode->numAntecedents == 1);
412 blockNode->succedents[i] = tmpreadNode;
413 tmpreadNode->antecedents[0] = blockNode;
414 tmpreadNode->antType[0] = rf_control;
415 tmpreadNode = tmpreadNode->list_next;
416 }
417
418 /* connect read nodes to commit node */
419 RF_ASSERT(commitNode->numAntecedents == n);
420 tmpreadNode = readNodes;
421 for (i = 0; i < n; i++) {
422 RF_ASSERT(tmpreadNode->numSuccedents == 1);
423 tmpreadNode->succedents[0] = commitNode;
424 commitNode->antecedents[i] = tmpreadNode;
425 commitNode->antType[i] = rf_control;
426 tmpreadNode = tmpreadNode->list_next;
427 }
428
429 /* connect commit node to term node */
430 RF_ASSERT(commitNode->numSuccedents == 1);
431 RF_ASSERT(termNode->numAntecedents == 1);
432 RF_ASSERT(termNode->numSuccedents == 0);
433 commitNode->succedents[0] = termNode;
434 termNode->antecedents[0] = commitNode;
435 termNode->antType[0] = rf_control;
436}
437
438void
439rf_CreateMirrorIdleReadDAG(
440 RF_Raid_t * raidPtr,
441 RF_AccessStripeMap_t * asmap,
442 RF_DagHeader_t * dag_h,
443 void *bp,
444 RF_RaidAccessFlags_t flags,
445 RF_AllocListElem_t * allocList)
446{
447 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList,
448 rf_DiskReadMirrorIdleFunc);
449}
450
451#if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0)
452
453void
454rf_CreateMirrorPartitionReadDAG(RF_Raid_t *raidPtr,
455 RF_AccessStripeMap_t *asmap,
456 RF_DagHeader_t *dag_h, void *bp,
457 RF_RaidAccessFlags_t flags,
458 RF_AllocListElem_t *allocList)
459{
460 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList,
461 rf_DiskReadMirrorPartitionFunc);
462}
463#endif
464