(git:34ef472)
offload_fft.h
Go to the documentation of this file.
1 /*----------------------------------------------------------------------------*/
2 /* CP2K: A general program to perform molecular dynamics simulations */
3 /* Copyright 2000-2024 CP2K developers group <https://cp2k.org> */
4 /* */
5 /* SPDX-License-Identifier: BSD-3-Clause */
6 /*----------------------------------------------------------------------------*/
7 #ifndef OFFLOAD_FFT_H
8 #define OFFLOAD_FFT_H
9 
10 #include <stdio.h>
11 #include <stdlib.h>
12 
13 #include "offload_library.h"
14 #include "offload_runtime.h"
15 
16 #if (defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP))
17 
18 #if defined(__OFFLOAD_CUDA)
19 #include <cufft.h>
20 #elif defined(__OFFLOAD_HIP)
21 #if (HIP_VERSION < 50600000)
22 #include <hipfft.h>
23 #else
24 #include <hipfft/hipfft.h>
25 #endif
26 #endif
27 
28 #if defined(__OFFLOAD_CUDA)
29 typedef cufftHandle offload_fftHandle;
30 typedef cufftType offload_fftType;
31 typedef cufftResult offload_fftResult;
32 #elif defined(__OFFLOAD_HIP)
33 typedef hipfftHandle offload_fftHandle;
34 typedef hipfftType offload_fftType;
35 typedef hipfftResult offload_fftResult;
36 #endif
37 
38 #if defined(__OFFLOAD_CUDA)
39 #define OFFLOAD_FFT_SUCCESS CUFFT_SUCCESS
40 #define OFFLOAD_FFT_FORWARD CUFFT_FORWARD
41 #define OFFLOAD_FFT_INVERSE CUFFT_INVERSE
42 #define OFFLOAD_FFT_Z2Z CUFFT_Z2Z
43 #elif defined(__OFFLOAD_HIP)
44 #define OFFLOAD_FFT_SUCCESS HIPFFT_SUCCESS
45 #define OFFLOAD_FFT_FORWARD HIPFFT_FORWARD
46 #define OFFLOAD_FFT_INVERSE HIPFFT_BACKWARD // inconsistent
47 #define OFFLOAD_FFT_Z2Z HIPFFT_Z2Z
48 #endif
49 
50 /*******************************************************************************
51  * \brief Check given cufft status and upon failure abort with a nice message.
52  * \author Ole Schuett
53  ******************************************************************************/
54 #define OFFLOAD_FFT_CHECK(status) \
55  if (status != OFFLOAD_FFT_SUCCESS) { \
56  fprintf(stderr, "ERROR: %s %s %d\n", offload_fftGetErrorString(status), \
57  __FILE__, __LINE__); \
58  abort(); \
59  }
60 
61 /*******************************************************************************
62  * \brief Equivalent to cudaGetErrorString for cufft.
63  ******************************************************************************/
64 static inline const char *offload_fftGetErrorString(offload_fftResult error) {
65 #if defined(__OFFLOAD_CUDA)
66  switch (error) {
67  case CUFFT_SUCCESS:
68  return "CUFFT_SUCCESS";
69 
70  case CUFFT_INVALID_PLAN:
71  return "CUFFT_INVALID_PLAN";
72 
73  case CUFFT_ALLOC_FAILED:
74  return "CUFFT_ALLOC_FAILED";
75 
76  case CUFFT_INVALID_TYPE:
77  return "CUFFT_INVALID_TYPE";
78 
79  case CUFFT_INVALID_VALUE:
80  return "CUFFT_INVALID_VALUE";
81 
82  case CUFFT_INTERNAL_ERROR:
83  return "CUFFT_INTERNAL_ERROR";
84 
85  case CUFFT_EXEC_FAILED:
86  return "CUFFT_EXEC_FAILED";
87 
88  case CUFFT_SETUP_FAILED:
89  return "CUFFT_SETUP_FAILED";
90 
91  case CUFFT_INVALID_SIZE:
92  return "CUFFT_INVALID_SIZE";
93 
94  case CUFFT_INCOMPLETE_PARAMETER_LIST:
95  return "CUFFT_INCOMPLETE_PARAMETER_LIST";
96 
97  case CUFFT_INVALID_DEVICE:
98  return "CUFFT_INVALID_DEVICE";
99 
100  case CUFFT_PARSE_ERROR:
101  return "CUFFT_PARSE_ERROR";
102 
103  case CUFFT_NO_WORKSPACE:
104  return "CUFFT_NO_WORKSPACE";
105 
106  case CUFFT_NOT_IMPLEMENTED:
107  return "CUFFT_NOT_IMPLEMENTED";
108 
109  case CUFFT_NOT_SUPPORTED:
110  return "CUFFT_NOT_SUPPORTED";
111 
112  case CUFFT_UNALIGNED_DATA:
113  return "CUFFT_UNALIGNED_DATA";
114 
115  case CUFFT_LICENSE_ERROR:
116  return "CUFFT_LICENSE_ERROR";
117  }
118 
119 #elif defined(__OFFLOAD_HIP)
120 
121  switch (error) {
122  case HIPFFT_SUCCESS:
123  return "HIPFFT_SUCCESS";
124 
125  case HIPFFT_INVALID_PLAN:
126  return "HIPFFT_INVALID_PLAN";
127 
128  case HIPFFT_ALLOC_FAILED:
129  return "HIPFFT_ALLOC_FAILED";
130 
131  case HIPFFT_INVALID_TYPE:
132  return "HIPFFT_INVALID_TYPE";
133 
134  case HIPFFT_INVALID_VALUE:
135  return "HIPFFT_INVALID_VALUE";
136 
137  case HIPFFT_INTERNAL_ERROR:
138  return "HIPFFT_INTERNAL_ERROR";
139 
140  case HIPFFT_EXEC_FAILED:
141  return "HIPFFT_EXEC_FAILED";
142 
143  case HIPFFT_SETUP_FAILED:
144  return "HIPFFT_SETUP_FAILED";
145 
146  case HIPFFT_INVALID_SIZE:
147  return "HIPFFT_INVALID_SIZE";
148 
149  case HIPFFT_INCOMPLETE_PARAMETER_LIST:
150  return "HIPFFT_INCOMPLETE_PARAMETER_LIST";
151 
152  case HIPFFT_INVALID_DEVICE:
153  return "HIPFFT_INVALID_DEVICE";
154 
155  case HIPFFT_PARSE_ERROR:
156  return "HIPFFT_PARSE_ERROR";
157 
158  case HIPFFT_NO_WORKSPACE:
159  return "HIPFFT_NO_WORKSPACE";
160 
161  case HIPFFT_NOT_IMPLEMENTED:
162  return "HIPFFT_NOT_IMPLEMENTED";
163 
164  case HIPFFT_NOT_SUPPORTED:
165  return "HIPFFT_NOT_SUPPORTED";
166 
167  case HIPFFT_UNALIGNED_DATA:
168  return "HIPFFT_UNALIGNED_DATA";
169 
170  // case HIPFFT_LICENSE_ERROR:
171  // return "HIPFFT_LICENSE_ERROR";
172  }
173 #endif
174 
175  return "<unknown>";
176 }
177 
178 /*******************************************************************************
179  * \brief Wrapper around cufftPlan3d.
180  ******************************************************************************/
181 static inline void offload_fftPlan3d(offload_fftHandle *plan, int nx, int ny,
182  int nz, offload_fftType type) {
183 #if defined(__OFFLOAD_CUDA)
184  OFFLOAD_FFT_CHECK(cufftPlan3d(plan, nx, ny, nz, type));
185 #elif defined(__OFFLOAD_HIP)
186  OFFLOAD_FFT_CHECK(hipfftPlan3d(plan, nx, ny, nz, type));
187 #endif
188 }
189 
190 /*******************************************************************************
191  * \brief Wrapper around cufftPlanMany.
192  ******************************************************************************/
193 static inline void offload_fftPlanMany(offload_fftHandle *plan, int rank,
194  int *n, int *inembed, int istride,
195  int idist, int *onembed, int ostride,
196  int odist, offload_fftType type,
197  int batch) {
198 #if defined(__OFFLOAD_CUDA)
199  OFFLOAD_FFT_CHECK(cufftPlanMany(plan, rank, n, inembed, istride, idist,
200  onembed, ostride, odist, type, batch));
201 #elif defined(__OFFLOAD_HIP)
202  OFFLOAD_FFT_CHECK(hipfftPlanMany(plan, rank, n, inembed, istride, idist,
203  onembed, ostride, odist, type, batch));
204 #endif
205 }
206 
207 /*******************************************************************************
208  * \brief Wrapper around cufftSetStream.
209  ******************************************************************************/
210 static inline void offload_fftSetStream(offload_fftHandle plan,
211  offloadStream_t stream) {
212 #if defined(__OFFLOAD_CUDA)
213  OFFLOAD_FFT_CHECK(cufftSetStream(plan, stream))
214 #elif defined(__OFFLOAD_HIP)
215  OFFLOAD_FFT_CHECK(hipfftSetStream(plan, stream))
216 #endif
217 }
218 
219 /*******************************************************************************
220  * \brief Wrapper around cufftDestroy.
221  ******************************************************************************/
222 static inline void offload_fftDestroy(offload_fftHandle plan) {
223 #if defined(__OFFLOAD_CUDA)
224  OFFLOAD_FFT_CHECK(cufftDestroy(plan));
225 #elif defined(__OFFLOAD_HIP)
226  OFFLOAD_FFT_CHECK(hipfftDestroy(plan));
227 #endif
228 }
229 
230 /*******************************************************************************
231  * \brief Wrapper around cufftExecZ2Z.
232  ******************************************************************************/
233 static inline void offload_fftExecZ2Z(offload_fftHandle plan,
234  const double *idata, double *odata,
235  int direction) {
236 #if defined(__OFFLOAD_CUDA)
237  OFFLOAD_FFT_CHECK(cufftExecZ2Z(plan, (cufftDoubleComplex *)idata,
238  (cufftDoubleComplex *)odata, direction));
239 #elif defined(__OFFLOAD_HIP)
240  OFFLOAD_FFT_CHECK(hipfftExecZ2Z(plan, (hipfftDoubleComplex *)idata,
241  (hipfftDoubleComplex *)odata, direction));
242 #endif
243 }
244 
245 #endif // #if (defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP))
246 
247 #endif
248 
249 // EOF
offload_library.h
offload_runtime.h