(git:374b731)
Loading...
Searching...
No Matches
offload_fft.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2024 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7#ifndef OFFLOAD_FFT_H
8#define OFFLOAD_FFT_H
9
10#include <stdio.h>
11#include <stdlib.h>
12
13#include "offload_library.h"
14#include "offload_runtime.h"
15
16#if (defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP))
17
18#if defined(__OFFLOAD_CUDA)
19#include <cufft.h>
20#elif defined(__OFFLOAD_HIP)
21#if (HIP_VERSION < 50600000)
22#include <hipfft.h>
23#else
24#include <hipfft/hipfft.h>
25#endif
26#endif
27
28#if defined(__OFFLOAD_CUDA)
29typedef cufftHandle offload_fftHandle;
30typedef cufftType offload_fftType;
31typedef cufftResult offload_fftResult;
32#elif defined(__OFFLOAD_HIP)
33typedef hipfftHandle offload_fftHandle;
34typedef hipfftType offload_fftType;
35typedef hipfftResult offload_fftResult;
36#endif
37
38#if defined(__OFFLOAD_CUDA)
39#define OFFLOAD_FFT_SUCCESS CUFFT_SUCCESS
40#define OFFLOAD_FFT_FORWARD CUFFT_FORWARD
41#define OFFLOAD_FFT_INVERSE CUFFT_INVERSE
42#define OFFLOAD_FFT_Z2Z CUFFT_Z2Z
43#elif defined(__OFFLOAD_HIP)
44#define OFFLOAD_FFT_SUCCESS HIPFFT_SUCCESS
45#define OFFLOAD_FFT_FORWARD HIPFFT_FORWARD
46#define OFFLOAD_FFT_INVERSE HIPFFT_BACKWARD // inconsistent
47#define OFFLOAD_FFT_Z2Z HIPFFT_Z2Z
48#endif
49
50/*******************************************************************************
51 * \brief Check given cufft status and upon failure abort with a nice message.
52 * \author Ole Schuett
53 ******************************************************************************/
54#define OFFLOAD_FFT_CHECK(status) \
55 if (status != OFFLOAD_FFT_SUCCESS) { \
56 fprintf(stderr, "ERROR: %s %s %d\n", offload_fftGetErrorString(status), \
57 __FILE__, __LINE__); \
58 abort(); \
59 }
60
61/*******************************************************************************
62 * \brief Equivalent to cudaGetErrorString for cufft.
63 ******************************************************************************/
64static inline const char *offload_fftGetErrorString(offload_fftResult error) {
65#if defined(__OFFLOAD_CUDA)
66 switch (error) {
67 case CUFFT_SUCCESS:
68 return "CUFFT_SUCCESS";
69
70 case CUFFT_INVALID_PLAN:
71 return "CUFFT_INVALID_PLAN";
72
73 case CUFFT_ALLOC_FAILED:
74 return "CUFFT_ALLOC_FAILED";
75
76 case CUFFT_INVALID_TYPE:
77 return "CUFFT_INVALID_TYPE";
78
79 case CUFFT_INVALID_VALUE:
80 return "CUFFT_INVALID_VALUE";
81
82 case CUFFT_INTERNAL_ERROR:
83 return "CUFFT_INTERNAL_ERROR";
84
85 case CUFFT_EXEC_FAILED:
86 return "CUFFT_EXEC_FAILED";
87
88 case CUFFT_SETUP_FAILED:
89 return "CUFFT_SETUP_FAILED";
90
91 case CUFFT_INVALID_SIZE:
92 return "CUFFT_INVALID_SIZE";
93
94 case CUFFT_INCOMPLETE_PARAMETER_LIST:
95 return "CUFFT_INCOMPLETE_PARAMETER_LIST";
96
97 case CUFFT_INVALID_DEVICE:
98 return "CUFFT_INVALID_DEVICE";
99
100 case CUFFT_PARSE_ERROR:
101 return "CUFFT_PARSE_ERROR";
102
103 case CUFFT_NO_WORKSPACE:
104 return "CUFFT_NO_WORKSPACE";
105
106 case CUFFT_NOT_IMPLEMENTED:
107 return "CUFFT_NOT_IMPLEMENTED";
108
109 case CUFFT_NOT_SUPPORTED:
110 return "CUFFT_NOT_SUPPORTED";
111
112 case CUFFT_UNALIGNED_DATA:
113 return "CUFFT_UNALIGNED_DATA";
114
115 case CUFFT_LICENSE_ERROR:
116 return "CUFFT_LICENSE_ERROR";
117 }
118
119#elif defined(__OFFLOAD_HIP)
120
121 switch (error) {
122 case HIPFFT_SUCCESS:
123 return "HIPFFT_SUCCESS";
124
125 case HIPFFT_INVALID_PLAN:
126 return "HIPFFT_INVALID_PLAN";
127
128 case HIPFFT_ALLOC_FAILED:
129 return "HIPFFT_ALLOC_FAILED";
130
131 case HIPFFT_INVALID_TYPE:
132 return "HIPFFT_INVALID_TYPE";
133
134 case HIPFFT_INVALID_VALUE:
135 return "HIPFFT_INVALID_VALUE";
136
137 case HIPFFT_INTERNAL_ERROR:
138 return "HIPFFT_INTERNAL_ERROR";
139
140 case HIPFFT_EXEC_FAILED:
141 return "HIPFFT_EXEC_FAILED";
142
143 case HIPFFT_SETUP_FAILED:
144 return "HIPFFT_SETUP_FAILED";
145
146 case HIPFFT_INVALID_SIZE:
147 return "HIPFFT_INVALID_SIZE";
148
149 case HIPFFT_INCOMPLETE_PARAMETER_LIST:
150 return "HIPFFT_INCOMPLETE_PARAMETER_LIST";
151
152 case HIPFFT_INVALID_DEVICE:
153 return "HIPFFT_INVALID_DEVICE";
154
155 case HIPFFT_PARSE_ERROR:
156 return "HIPFFT_PARSE_ERROR";
157
158 case HIPFFT_NO_WORKSPACE:
159 return "HIPFFT_NO_WORKSPACE";
160
161 case HIPFFT_NOT_IMPLEMENTED:
162 return "HIPFFT_NOT_IMPLEMENTED";
163
164 case HIPFFT_NOT_SUPPORTED:
165 return "HIPFFT_NOT_SUPPORTED";
166
167 case HIPFFT_UNALIGNED_DATA:
168 return "HIPFFT_UNALIGNED_DATA";
169
170 // case HIPFFT_LICENSE_ERROR:
171 // return "HIPFFT_LICENSE_ERROR";
172 }
173#endif
174
175 return "<unknown>";
176}
177
178/*******************************************************************************
179 * \brief Wrapper around cufftPlan3d.
180 ******************************************************************************/
181static inline void offload_fftPlan3d(offload_fftHandle *plan, int nx, int ny,
182 int nz, offload_fftType type) {
183#if defined(__OFFLOAD_CUDA)
184 OFFLOAD_FFT_CHECK(cufftPlan3d(plan, nx, ny, nz, type));
185#elif defined(__OFFLOAD_HIP)
186 OFFLOAD_FFT_CHECK(hipfftPlan3d(plan, nx, ny, nz, type));
187#endif
188}
189
190/*******************************************************************************
191 * \brief Wrapper around cufftPlanMany.
192 ******************************************************************************/
193static inline void offload_fftPlanMany(offload_fftHandle *plan, int rank,
194 int *n, int *inembed, int istride,
195 int idist, int *onembed, int ostride,
196 int odist, offload_fftType type,
197 int batch) {
198#if defined(__OFFLOAD_CUDA)
199 OFFLOAD_FFT_CHECK(cufftPlanMany(plan, rank, n, inembed, istride, idist,
200 onembed, ostride, odist, type, batch));
201#elif defined(__OFFLOAD_HIP)
202 OFFLOAD_FFT_CHECK(hipfftPlanMany(plan, rank, n, inembed, istride, idist,
203 onembed, ostride, odist, type, batch));
204#endif
205}
206
207/*******************************************************************************
208 * \brief Wrapper around cufftSetStream.
209 ******************************************************************************/
210static inline void offload_fftSetStream(offload_fftHandle plan,
211 offloadStream_t stream) {
212#if defined(__OFFLOAD_CUDA)
213 OFFLOAD_FFT_CHECK(cufftSetStream(plan, stream))
214#elif defined(__OFFLOAD_HIP)
215 OFFLOAD_FFT_CHECK(hipfftSetStream(plan, stream))
216#endif
217}
218
219/*******************************************************************************
220 * \brief Wrapper around cufftDestroy.
221 ******************************************************************************/
222static inline void offload_fftDestroy(offload_fftHandle plan) {
223#if defined(__OFFLOAD_CUDA)
224 OFFLOAD_FFT_CHECK(cufftDestroy(plan));
225#elif defined(__OFFLOAD_HIP)
226 OFFLOAD_FFT_CHECK(hipfftDestroy(plan));
227#endif
228}
229
230/*******************************************************************************
231 * \brief Wrapper around cufftExecZ2Z.
232 ******************************************************************************/
233static inline void offload_fftExecZ2Z(offload_fftHandle plan,
234 const double *idata, double *odata,
235 int direction) {
236#if defined(__OFFLOAD_CUDA)
237 OFFLOAD_FFT_CHECK(cufftExecZ2Z(plan, (cufftDoubleComplex *)idata,
238 (cufftDoubleComplex *)odata, direction));
239#elif defined(__OFFLOAD_HIP)
240 OFFLOAD_FFT_CHECK(hipfftExecZ2Z(plan, (hipfftDoubleComplex *)idata,
241 (hipfftDoubleComplex *)odata, direction));
242#endif
243}
244
245#endif // #if (defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP))
246
247#endif
248
249// EOF
offload_library.h
offload_runtime.h