Actual source code: blas_cyclic_cupm_impl.hpp
1: #pragma once
2: #include "blas_cyclic_cupm.h"
3: #include <petsc/private/cupminterface.hpp>
4: #include <petsc/private/cupmobject.hpp>
6: namespace Petsc
7: {
9: namespace device
10: {
12: namespace cupm
13: {
15: namespace impl
16: {
18: template <DeviceType T>
19: struct BLASCyclic : CUPMObject<T> {
20: PETSC_CUPMOBJECT_HEADER(T);
22: static PetscErrorCode axpby_dispatch(cupmBlasHandle_t, cupmBlasInt_t, PetscScalar, const PetscScalar[], PetscScalar, PetscScalar[], cupmBlasInt_t) noexcept;
23: static PetscErrorCode axpby(PetscDeviceContext, PetscInt, PetscInt, PetscInt, PetscScalar, const PetscScalar[], PetscScalar, PetscScalar[], PetscInt) noexcept;
24: static PetscErrorCode dmv(PetscDeviceContext, PetscBool, PetscInt, PetscInt, PetscInt, PetscScalar, const PetscScalar[], const PetscScalar[], PetscScalar, PetscScalar[]) noexcept;
25: static PetscErrorCode dsv(PetscDeviceContext, PetscBool, PetscInt, PetscInt, PetscInt, const PetscScalar[], const PetscScalar[], PetscScalar[]) noexcept;
26: static PetscErrorCode trsv(PetscDeviceContext, PetscBool, PetscInt, PetscInt, PetscInt, const PetscScalar[], PetscInt, const PetscScalar[], PetscScalar[]) noexcept;
27: static PetscErrorCode gemv(PetscDeviceContext, PetscBool, PetscInt, PetscInt, PetscInt, PetscScalar, const PetscScalar[], PetscInt, const PetscScalar[], PetscScalar, PetscScalar[]) noexcept;
28: static PetscErrorCode hemv(PetscDeviceContext, PetscInt, PetscInt, PetscInt, PetscScalar, const PetscScalar[], PetscInt, const PetscScalar[], PetscScalar, PetscScalar[]) noexcept;
29: };
31: template <DeviceType T>
32: PetscErrorCode BLASCyclic<T>::axpby_dispatch(cupmBlasHandle_t handle, cupmBlasInt_t n, PetscScalar alpha, const PetscScalar x[], PetscScalar beta, PetscScalar y[], cupmBlasInt_t y_stride) noexcept
33: {
34: auto x_ = cupmScalarPtrCast(x);
35: auto y_ = cupmScalarPtrCast(y);
36: const auto calpha = cupmScalarPtrCast(&alpha);
37: const auto cbeta = cupmScalarPtrCast(&beta);
39: PetscFunctionBegin;
40: if (alpha == 1.0 && beta == 0.0) {
41: PetscCallCUPMBLAS(cupmBlasXcopy(handle, n, x_, 1, y_, y_stride));
42: } else {
43: if (beta != 1.0) PetscCallCUPMBLAS(cupmBlasXscal(handle, n, cbeta, y_, y_stride));
44: if (alpha != 0.0) PetscCallCUPMBLAS(cupmBlasXaxpy(handle, n, calpha, x_, 1, y_, y_stride));
45: }
46: PetscFunctionReturn(PETSC_SUCCESS);
47: }
49: template <DeviceType T>
50: PetscErrorCode BLASCyclic<T>::axpby(PetscDeviceContext dctx, PetscInt M, PetscInt oldest, PetscInt next, PetscScalar alpha, const PetscScalar x[], PetscScalar beta, PetscScalar y[], PetscInt y_stride) noexcept
51: {
52: PetscInt N = next - oldest;
53: cupmBlasInt_t m, i_oldest, i_next, y_stride_;
54: cupmBlasPointerMode_t pointer_mode;
55: cupmBlasHandle_t handle;
57: PetscFunctionBegin;
58: if (!N) PetscFunctionReturn(PETSC_SUCCESS);
59: PetscCall(PetscCUPMBlasIntCast(M, &m));
60: PetscCall(PetscCUPMBlasIntCast(oldest % m, &i_oldest));
61: PetscCall(PetscCUPMBlasIntCast(((next - 1) % m) + 1, &i_next));
62: PetscCall(PetscCUPMBlasIntCast(y_stride, &y_stride_));
63: PetscCall(GetHandlesFrom_(dctx, &handle));
64: PetscCall(PetscLogGpuTimeBegin());
65: PetscCallCUPMBLAS(cupmBlasGetPointerMode(handle, &pointer_mode));
66: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, CUPMBLAS_POINTER_MODE_HOST));
67: if (N == m) {
68: PetscCall(axpby_dispatch(handle, m, alpha, x, beta, y, y_stride_));
69: } else if (i_next > i_oldest) {
70: cupmBlasInt_t diff = i_next - i_oldest;
72: PetscCall(axpby_dispatch(handle, diff, alpha, &x[i_oldest], beta, &y[i_oldest * y_stride], y_stride_));
73: } else {
74: cupmBlasInt_t diff = m - i_oldest;
76: if (i_next) PetscCall(axpby_dispatch(handle, i_next, alpha, x, beta, y, y_stride_));
77: if (diff) PetscCall(axpby_dispatch(handle, diff, alpha, &x[i_oldest], beta, &y[i_oldest * y_stride], y_stride_));
78: }
79: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, pointer_mode));
80: PetscCall(PetscLogGpuTimeEnd());
82: PetscCall(PetscLogGpuFlops(3.0 * N));
83: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
84: PetscFunctionReturn(PETSC_SUCCESS);
85: }
87: template <DeviceType T>
88: PetscErrorCode BLASCyclic<T>::dmv(PetscDeviceContext dctx, PetscBool hermitian_transpose, PetscInt M, PetscInt oldest, PetscInt next, PetscScalar alpha, const PetscScalar A[], const PetscScalar x[], PetscScalar beta, PetscScalar y[]) noexcept
89: {
90: PetscInt N = next - oldest;
91: cupmBlasInt_t m, i_oldest, i_next;
92: cupmBlasPointerMode_t pointer_mode;
93: cupmBlasHandle_t handle;
94: const auto A_ = cupmScalarPtrCast(A);
95: const auto x_ = cupmScalarPtrCast(x);
96: const auto y_ = cupmScalarPtrCast(y);
97: const auto calpha = cupmScalarPtrCast(&alpha);
98: const auto cbeta = cupmScalarPtrCast(&beta);
99: const auto trans = hermitian_transpose ? CUPMBLAS_OP_C : CUPMBLAS_OP_N;
101: PetscFunctionBegin;
102: if (!N) PetscFunctionReturn(PETSC_SUCCESS);
103: PetscCall(PetscCUPMBlasIntCast(M, &m));
104: PetscCall(PetscCUPMBlasIntCast(oldest % m, &i_oldest));
105: PetscCall(PetscCUPMBlasIntCast(((next - 1) % m) + 1, &i_next));
106: PetscCall(GetHandlesFrom_(dctx, &handle));
107: PetscCall(PetscLogGpuTimeBegin());
108: PetscCallCUPMBLAS(cupmBlasGetPointerMode(handle, &pointer_mode));
109: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, CUPMBLAS_POINTER_MODE_HOST));
110: if (N == m) {
111: PetscCallCUPMBLAS(cupmBlasXgbmv(handle, trans, m, m, 0, 0, calpha, A_, 1, x_, 1, cbeta, y_, 1));
112: } else if (i_next > i_oldest) {
113: cupmBlasInt_t diff = i_next - i_oldest;
115: PetscCallCUPMBLAS(cupmBlasXgbmv(handle, trans, diff, diff, 0, 0, calpha, &A_[i_oldest], 1, &x_[i_oldest], 1, cbeta, &y_[i_oldest], 1));
116: } else {
117: cupmBlasInt_t diff = m - i_oldest;
119: if (i_next) PetscCallCUPMBLAS(cupmBlasXgbmv(handle, trans, i_next, i_next, 0, 0, calpha, A_, 1, x_, 1, cbeta, y_, 1));
120: if (diff) PetscCallCUPMBLAS(cupmBlasXgbmv(handle, trans, diff, diff, 0, 0, calpha, &A_[i_oldest], 1, &x_[i_oldest], 1, cbeta, &y_[i_oldest], 1));
121: }
122: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, pointer_mode));
123: PetscCall(PetscLogGpuTimeEnd());
125: PetscCall(PetscLogGpuFlops(3.0 * N));
126: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
127: PetscFunctionReturn(PETSC_SUCCESS);
128: }
130: template <DeviceType T>
131: PetscErrorCode BLASCyclic<T>::dsv(PetscDeviceContext dctx, PetscBool hermitian_transpose, PetscInt M, PetscInt oldest, PetscInt next, const PetscScalar A[], const PetscScalar x[], PetscScalar y[]) noexcept
132: {
133: PetscInt N = next - oldest;
134: cupmBlasInt_t m, i_oldest, i_next;
135: cupmBlasPointerMode_t pointer_mode;
136: cupmBlasHandle_t handle;
137: cupmStream_t stream;
138: const auto A_ = cupmScalarPtrCast(A);
139: const auto y_ = cupmScalarPtrCast(y);
140: auto trans = hermitian_transpose ? CUPMBLAS_OP_C : CUPMBLAS_OP_N;
142: PetscFunctionBegin;
143: if (!N) PetscFunctionReturn(PETSC_SUCCESS);
144: PetscCall(PetscCUPMBlasIntCast(M, &m));
145: PetscCall(PetscCUPMBlasIntCast(oldest % m, &i_oldest));
146: PetscCall(PetscCUPMBlasIntCast(((next - 1) % m) + 1, &i_next));
147: PetscCall(GetHandlesFrom_(dctx, &handle, NULL, &stream));
148: PetscCall(PetscLogGpuTimeBegin());
149: PetscCallCUPMBLAS(cupmBlasGetPointerMode(handle, &pointer_mode));
150: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, CUPMBLAS_POINTER_MODE_HOST));
151: if (N == m) {
152: if (x != y) PetscCall(PetscCUPMMemcpyAsync(y, x, m, cupmMemcpyDeviceToDevice, stream));
153: PetscCallCUPMBLAS(cupmBlasXtbsv(handle, CUPMBLAS_FILL_MODE_UPPER, trans, CUPMBLAS_DIAG_NON_UNIT, m, 0, A_, 1, y_, 1));
154: } else if (i_next > i_oldest) {
155: cupmBlasInt_t diff = i_next - i_oldest;
157: if (x != y) PetscCall(PetscCUPMMemcpyAsync(&y[i_oldest], &x[i_oldest], diff, cupmMemcpyDeviceToDevice, stream));
158: PetscCallCUPMBLAS(cupmBlasXtbsv(handle, CUPMBLAS_FILL_MODE_UPPER, trans, CUPMBLAS_DIAG_NON_UNIT, diff, 0, &A_[i_oldest], 1, &y_[i_oldest], 1));
159: } else {
160: cupmBlasInt_t diff = m - i_oldest;
162: if (i_next) {
163: if (x != y) PetscCall(PetscCUPMMemcpyAsync(y, x, i_next, cupmMemcpyDeviceToDevice, stream));
164: PetscCallCUPMBLAS(cupmBlasXtbsv(handle, CUPMBLAS_FILL_MODE_UPPER, trans, CUPMBLAS_DIAG_NON_UNIT, i_next, 0, A_, 1, y_, 1));
165: }
166: if (diff) {
167: if (x != y) PetscCall(PetscCUPMMemcpyAsync(&y[i_oldest], &x[i_oldest], diff, cupmMemcpyDeviceToDevice, stream));
168: PetscCallCUPMBLAS(cupmBlasXtbsv(handle, CUPMBLAS_FILL_MODE_UPPER, trans, CUPMBLAS_DIAG_NON_UNIT, diff, 0, &A_[i_oldest], 1, &y_[i_oldest], 1));
169: }
170: }
171: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, pointer_mode));
172: PetscCall(PetscLogGpuTimeEnd());
174: PetscCall(PetscLogGpuFlops(3.0 * N));
175: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
176: PetscFunctionReturn(PETSC_SUCCESS);
177: }
179: template <DeviceType T>
180: PetscErrorCode BLASCyclic<T>::trsv(PetscDeviceContext dctx, PetscBool hermitian_transpose, PetscInt m, PetscInt oldest, PetscInt next, const PetscScalar A[], PetscInt lda, const PetscScalar x[], PetscScalar y[]) noexcept
181: {
182: PetscInt N = next - oldest;
183: PetscInt i_oldest = oldest % m;
184: PetscInt i_next = ((next - 1) % m) + 1;
185: cupmBlasInt_t n, n_old, n_new;
186: cupmBlasPointerMode_t pointer_mode;
187: cupmBlasHandle_t handle;
188: cupmStream_t stream;
189: auto sone = cupmScalarCast(1.0);
190: auto minus_one = cupmScalarCast(-1.0);
191: auto A_ = cupmScalarPtrCast(A);
192: auto y_ = cupmScalarPtrCast(y);
194: PetscFunctionBegin;
195: if (!N) PetscFunctionReturn(PETSC_SUCCESS);
196: PetscCall(PetscCUPMBlasIntCast(i_next - i_oldest, &n));
197: PetscCall(PetscCUPMBlasIntCast(m - i_oldest, &n_old));
198: PetscCall(PetscCUPMBlasIntCast(i_next, &n_new));
199: PetscCall(GetHandlesFrom_(dctx, &handle, NULL, &stream));
200: PetscCall(PetscLogGpuTimeBegin());
201: PetscCallCUPMBLAS(cupmBlasGetPointerMode(handle, &pointer_mode));
202: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, CUPMBLAS_POINTER_MODE_HOST));
203: if (n > 0) {
204: if (x != y) PetscCall(PetscCUPMMemcpyAsync(y, x, n, cupmMemcpyDeviceToDevice, stream));
205: PetscCallCUPMBLAS(cupmBlasXtrsv(handle, CUPMBLAS_FILL_MODE_UPPER, hermitian_transpose ? CUPMBLAS_OP_C : CUPMBLAS_OP_N, CUPMBLAS_DIAG_NON_UNIT, n, &A_[i_oldest * (lda + 1)], lda, y_, 1));
206: } else if (!hermitian_transpose) {
207: if (n_new > 0) PetscCallCUPMBLAS(cupmBlasXtrsv(handle, CUPMBLAS_FILL_MODE_UPPER, CUPMBLAS_OP_N, CUPMBLAS_DIAG_NON_UNIT, n_new, A_, lda, y_, 1));
208: if (n_new > 0 && n_old > 0) PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_N, n_old, n_new, &minus_one, &A_[i_oldest], lda, y_, 1, &sone, &y_[i_oldest], 1));
209: if (n_old > 0) PetscCallCUPMBLAS(cupmBlasXtrsv(handle, CUPMBLAS_FILL_MODE_UPPER, CUPMBLAS_OP_N, CUPMBLAS_DIAG_NON_UNIT, n_old, &A_[i_oldest * (lda + 1)], lda, &y_[i_oldest], 1));
210: } else {
211: if (n_old > 0) PetscCallCUPMBLAS(cupmBlasXtrsv(handle, CUPMBLAS_FILL_MODE_UPPER, CUPMBLAS_OP_C, CUPMBLAS_DIAG_NON_UNIT, n_old, &A_[i_oldest * (lda + 1)], lda, &y_[i_oldest], 1));
212: if (n_new > 0 && n_old > 0) PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_C, n_old, n_new, &minus_one, &A_[i_oldest], lda, &y_[i_oldest], 1, &sone, y_, 1));
213: if (n_new > 0) PetscCallCUPMBLAS(cupmBlasXtrsv(handle, CUPMBLAS_FILL_MODE_UPPER, CUPMBLAS_OP_C, CUPMBLAS_DIAG_NON_UNIT, n_new, A_, lda, y_, 1));
214: }
215: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, pointer_mode));
216: PetscCall(PetscLogGpuTimeEnd());
218: PetscCall(PetscLogGpuFlops(1.0 * N * N));
219: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
220: PetscFunctionReturn(PETSC_SUCCESS);
221: }
223: template <DeviceType T>
224: PetscErrorCode BLASCyclic<T>::hemv(PetscDeviceContext dctx, PetscInt m, PetscInt oldest, PetscInt next, PetscScalar alpha, const PetscScalar A[], PetscInt lda, const PetscScalar x[], PetscScalar beta, PetscScalar y[]) noexcept
225: {
226: PetscInt N = next - oldest;
227: PetscInt i_oldest = oldest % m;
228: PetscInt i_next = ((next - 1) % m) + 1;
229: cupmBlasInt_t n, n_old, n_new;
230: cupmBlasPointerMode_t pointer_mode;
231: cupmBlasHandle_t handle;
232: cupmStream_t stream;
233: auto sone = cupmScalarCast(1.0);
234: auto A_ = cupmScalarPtrCast(A);
235: auto x_ = cupmScalarPtrCast(x);
236: auto y_ = cupmScalarPtrCast(y);
237: const auto calpha = cupmScalarPtrCast(&alpha);
238: const auto cbeta = cupmScalarPtrCast(&beta);
240: PetscFunctionBegin;
241: if (!N) PetscFunctionReturn(PETSC_SUCCESS);
242: PetscCall(PetscCUPMBlasIntCast(i_next - i_oldest, &n));
243: PetscCall(PetscCUPMBlasIntCast(m - i_oldest, &n_old));
244: PetscCall(PetscCUPMBlasIntCast(i_next, &n_new));
245: PetscCall(GetHandlesFrom_(dctx, &handle, NULL, &stream));
246: PetscCall(PetscLogGpuTimeBegin());
247: PetscCallCUPMBLAS(cupmBlasGetPointerMode(handle, &pointer_mode));
248: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, CUPMBLAS_POINTER_MODE_HOST));
249: if (n > 0) PetscCallCUPMBLAS(cupmBlasXhemv(handle, CUPMBLAS_FILL_MODE_UPPER, n, calpha, &A_[i_oldest * (lda + 1)], lda, &x_[i_oldest], 1, cbeta, &y_[i_oldest], 1));
250: else {
251: if (n_new > 0) PetscCallCUPMBLAS(cupmBlasXhemv(handle, CUPMBLAS_FILL_MODE_UPPER, n_new, calpha, A_, lda, x_, 1, cbeta, y_, 1));
252: if (n_old > 0) PetscCallCUPMBLAS(cupmBlasXhemv(handle, CUPMBLAS_FILL_MODE_UPPER, n_old, calpha, &A_[i_oldest * (lda + 1)], lda, &x_[i_oldest], 1, cbeta, &y_[i_oldest], 1));
253: if (n_new > 0 && n_old > 0) {
254: PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_N, n_old, n_new, calpha, &A_[i_oldest], lda, x_, 1, &sone, &y_[i_oldest], 1));
255: PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_C, n_old, n_new, calpha, &A_[i_oldest], lda, &x_[i_oldest], 1, &sone, y_, 1));
256: }
257: }
258: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, pointer_mode));
259: PetscCall(PetscLogGpuTimeEnd());
261: PetscCall(PetscLogGpuFlops(2.0 * N * N));
262: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
263: PetscFunctionReturn(PETSC_SUCCESS);
264: }
266: template <DeviceType T>
267: PetscErrorCode BLASCyclic<T>::gemv(PetscDeviceContext dctx, PetscBool hermitian_transpose, PetscInt m, PetscInt oldest, PetscInt next, PetscScalar alpha, const PetscScalar A[], PetscInt lda, const PetscScalar x[], PetscScalar beta, PetscScalar y[]) noexcept
268: {
269: PetscInt N = next - oldest;
270: PetscInt i_oldest = oldest % m;
271: PetscInt i_next = ((next - 1) % m) + 1;
272: cupmBlasInt_t n, n_old, n_new;
273: cupmBlasPointerMode_t pointer_mode;
274: cupmBlasHandle_t handle;
275: cupmStream_t stream;
276: auto sone = cupmScalarCast(1.0);
277: auto A_ = cupmScalarPtrCast(A);
278: auto x_ = cupmScalarPtrCast(x);
279: auto y_ = cupmScalarPtrCast(y);
280: auto trans = hermitian_transpose ? CUPMBLAS_OP_C : CUPMBLAS_OP_N;
281: const auto calpha = cupmScalarPtrCast(&alpha);
282: const auto cbeta = cupmScalarPtrCast(&beta);
284: PetscFunctionBegin;
285: if (!N) PetscFunctionReturn(PETSC_SUCCESS);
286: PetscCall(PetscCUPMBlasIntCast(i_next - i_oldest, &n));
287: PetscCall(PetscCUPMBlasIntCast(m - i_oldest, &n_old));
288: PetscCall(PetscCUPMBlasIntCast(i_next, &n_new));
289: PetscCall(GetHandlesFrom_(dctx, &handle, NULL, &stream));
290: PetscCall(PetscLogGpuTimeBegin());
291: PetscCallCUPMBLAS(cupmBlasGetPointerMode(handle, &pointer_mode));
292: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, CUPMBLAS_POINTER_MODE_HOST));
293: if (N == m) {
294: PetscCallCUPMBLAS(cupmBlasXgemv(handle, trans, N, N, calpha, A_, lda, x_, 1, cbeta, y_, 1));
295: } else if (n > 0) {
296: PetscCallCUPMBLAS(cupmBlasXgemv(handle, trans, n, n, calpha, &A_[i_oldest * (lda + 1)], lda, &x_[i_oldest], 1, cbeta, &y_[i_oldest], 1));
297: } else {
298: if (n_new > 0) PetscCallCUPMBLAS(cupmBlasXgemv(handle, trans, n_new, n_new, calpha, A_, lda, x_, 1, cbeta, y_, 1));
299: if (n_old > 0) PetscCallCUPMBLAS(cupmBlasXgemv(handle, trans, n_old, n_old, calpha, &A_[i_oldest * (lda + 1)], lda, &x_[i_oldest], 1, cbeta, &y_[i_oldest], 1));
300: if (n_new > 0 && n_old > 0) {
301: if (!hermitian_transpose) {
302: PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_N, n_old, n_new, calpha, &A_[i_oldest], lda, x_, 1, &sone, &y_[i_oldest], 1));
303: PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_N, n_new, n_old, calpha, &A_[i_oldest * lda], lda, &x_[i_oldest], 1, &sone, y_, 1));
304: } else {
305: PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_C, n_new, n_old, calpha, &A_[i_oldest * lda], lda, x_, 1, &sone, &y_[i_oldest], 1));
306: PetscCallCUPMBLAS(cupmBlasXgemv(handle, CUPMBLAS_OP_C, n_old, n_new, calpha, &A_[i_oldest], lda, &x_[i_oldest], 1, &sone, y_, 1));
307: }
308: }
309: }
310: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, pointer_mode));
311: PetscCall(PetscLogGpuTimeEnd());
313: PetscCall(PetscLogGpuFlops(2.0 * N * N));
314: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
315: PetscFunctionReturn(PETSC_SUCCESS);
316: }
318: } // namespace impl
320: } // namespace cupm
322: } // namespace device
324: } // namespace Petsc