(git:9c0f831)
Loading...
Searching...
No Matches
offload_fft.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2025 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7#ifndef OFFLOAD_FFT_H
8#define OFFLOAD_FFT_H
9
10#include "offload_library.h"
11#include "offload_runtime.h"
12
13#include <stdio.h>
14#include <stdlib.h>
15
16#if (defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP))
17#if defined(__OFFLOAD_CUDA)
18#include <cufft.h>
19#elif defined(__OFFLOAD_HIP)
20#if (HIP_VERSION < 50600000)
21#include <hipfft.h>
22#else
23#include <hipfft/hipfft.h>
24#endif
25#endif
26
27#if defined(__OFFLOAD_CUDA)
28typedef cufftHandle offload_fftHandle;
29typedef cufftType offload_fftType;
30typedef cufftResult offload_fftResult;
31#elif defined(__OFFLOAD_HIP)
32typedef hipfftHandle offload_fftHandle;
33typedef hipfftType offload_fftType;
34typedef hipfftResult offload_fftResult;
35#endif
36
37#if defined(__OFFLOAD_CUDA)
38#define OFFLOAD_FFT_SUCCESS CUFFT_SUCCESS
39#define OFFLOAD_FFT_FORWARD CUFFT_FORWARD
40#define OFFLOAD_FFT_INVERSE CUFFT_INVERSE
41#define OFFLOAD_FFT_Z2Z CUFFT_Z2Z
42#elif defined(__OFFLOAD_HIP)
43#define OFFLOAD_FFT_SUCCESS HIPFFT_SUCCESS
44#define OFFLOAD_FFT_FORWARD HIPFFT_FORWARD
45#define OFFLOAD_FFT_INVERSE HIPFFT_BACKWARD // inconsistent
46#define OFFLOAD_FFT_Z2Z HIPFFT_Z2Z
47#endif
48
49/*******************************************************************************
50 * \brief Check given cufft status and upon failure abort with a nice message.
51 * \author Ole Schuett
52 ******************************************************************************/
53#define OFFLOAD_FFT_CHECK(status) \
54 if (status != OFFLOAD_FFT_SUCCESS) { \
55 fprintf(stderr, "ERROR: %s %s %d\n", offload_fftGetErrorString(status), \
56 __FILE__, __LINE__); \
57 abort(); \
58 }
59
60/*******************************************************************************
61 * \brief Equivalent to cudaGetErrorString for cufft.
62 ******************************************************************************/
63static inline const char *offload_fftGetErrorString(offload_fftResult error) {
64#if defined(__OFFLOAD_CUDA)
65 switch (error) {
66 case CUFFT_SUCCESS:
67 return "CUFFT_SUCCESS";
68 case CUFFT_INVALID_PLAN:
69 return "CUFFT_INVALID_PLAN";
70 case CUFFT_ALLOC_FAILED:
71 return "CUFFT_ALLOC_FAILED";
72 case CUFFT_INVALID_TYPE:
73 return "CUFFT_INVALID_TYPE";
74 case CUFFT_INVALID_VALUE:
75 return "CUFFT_INVALID_VALUE";
76 case CUFFT_INTERNAL_ERROR:
77 return "CUFFT_INTERNAL_ERROR";
78 case CUFFT_EXEC_FAILED:
79 return "CUFFT_EXEC_FAILED";
80 case CUFFT_SETUP_FAILED:
81 return "CUFFT_SETUP_FAILED";
82 case CUFFT_INVALID_SIZE:
83 return "CUFFT_INVALID_SIZE";
84 /*case CUFFT_INCOMPLETE_PARAMETER_LIST:
85 return "CUFFT_INCOMPLETE_PARAMETER_LIST";*/
86 case CUFFT_INVALID_DEVICE:
87 return "CUFFT_INVALID_DEVICE";
88 /*case CUFFT_PARSE_ERROR:
89 return "CUFFT_PARSE_ERROR";*/
90 case CUFFT_NO_WORKSPACE:
91 return "CUFFT_NO_WORKSPACE";
92 case CUFFT_NOT_IMPLEMENTED:
93 return "CUFFT_NOT_IMPLEMENTED";
94 case CUFFT_NOT_SUPPORTED:
95 return "CUFFT_NOT_SUPPORTED";
96 case CUFFT_UNALIGNED_DATA:
97 return "CUFFT_UNALIGNED_DATA";
98 /*case CUFFT_LICENSE_ERROR:
99 return "CUFFT_LICENSE_ERROR";*/
100 default:; // <unknown>
101 }
102
103#elif defined(__OFFLOAD_HIP)
104 switch (error) {
105 case HIPFFT_SUCCESS:
106 return "HIPFFT_SUCCESS";
107 case HIPFFT_INVALID_PLAN:
108 return "HIPFFT_INVALID_PLAN";
109 case HIPFFT_ALLOC_FAILED:
110 return "HIPFFT_ALLOC_FAILED";
111 case HIPFFT_INVALID_TYPE:
112 return "HIPFFT_INVALID_TYPE";
113 case HIPFFT_INVALID_VALUE:
114 return "HIPFFT_INVALID_VALUE";
115 case HIPFFT_INTERNAL_ERROR:
116 return "HIPFFT_INTERNAL_ERROR";
117 case HIPFFT_EXEC_FAILED:
118 return "HIPFFT_EXEC_FAILED";
119 case HIPFFT_SETUP_FAILED:
120 return "HIPFFT_SETUP_FAILED";
121 case HIPFFT_INVALID_SIZE:
122 return "HIPFFT_INVALID_SIZE";
123 case HIPFFT_INCOMPLETE_PARAMETER_LIST:
124 return "HIPFFT_INCOMPLETE_PARAMETER_LIST";
125 case HIPFFT_INVALID_DEVICE:
126 return "HIPFFT_INVALID_DEVICE";
127 case HIPFFT_PARSE_ERROR:
128 return "HIPFFT_PARSE_ERROR";
129 case HIPFFT_NO_WORKSPACE:
130 return "HIPFFT_NO_WORKSPACE";
131 case HIPFFT_NOT_IMPLEMENTED:
132 return "HIPFFT_NOT_IMPLEMENTED";
133 case HIPFFT_NOT_SUPPORTED:
134 return "HIPFFT_NOT_SUPPORTED";
135 case HIPFFT_UNALIGNED_DATA:
136 return "HIPFFT_UNALIGNED_DATA";
137 default:; // <unknown>
138 }
139#endif
140
141 return "<unknown>";
142}
143
144/*******************************************************************************
145 * \brief Wrapper around cufftPlan3d.
146 ******************************************************************************/
147static inline void offload_fftPlan3d(offload_fftHandle *plan, int nx, int ny,
148 int nz, offload_fftType type) {
149#if defined(__OFFLOAD_CUDA)
150 OFFLOAD_FFT_CHECK(cufftPlan3d(plan, nx, ny, nz, type));
151#elif defined(__OFFLOAD_HIP)
152 OFFLOAD_FFT_CHECK(hipfftPlan3d(plan, nx, ny, nz, type));
153#endif
154}
155
156/*******************************************************************************
157 * \brief Wrapper around cufftPlanMany.
158 ******************************************************************************/
159static inline void offload_fftPlanMany(offload_fftHandle *plan, int rank,
160 int *n, int *inembed, int istride,
161 int idist, int *onembed, int ostride,
162 int odist, offload_fftType type,
163 int batch) {
164#if defined(__OFFLOAD_CUDA)
165 OFFLOAD_FFT_CHECK(cufftPlanMany(plan, rank, n, inembed, istride, idist,
166 onembed, ostride, odist, type, batch));
167#elif defined(__OFFLOAD_HIP)
168 OFFLOAD_FFT_CHECK(hipfftPlanMany(plan, rank, n, inembed, istride, idist,
169 onembed, ostride, odist, type, batch));
170#endif
171}
172
173/*******************************************************************************
174 * \brief Wrapper around cufftSetStream.
175 ******************************************************************************/
176static inline void offload_fftSetStream(offload_fftHandle plan,
177 offloadStream_t stream) {
178#if defined(__OFFLOAD_CUDA)
179 OFFLOAD_FFT_CHECK(cufftSetStream(plan, stream))
180#elif defined(__OFFLOAD_HIP)
181 OFFLOAD_FFT_CHECK(hipfftSetStream(plan, stream))
182#endif
183}
184
185/*******************************************************************************
186 * \brief Wrapper around cufftDestroy.
187 ******************************************************************************/
188static inline void offload_fftDestroy(offload_fftHandle plan) {
189#if defined(__OFFLOAD_CUDA)
190 OFFLOAD_FFT_CHECK(cufftDestroy(plan));
191#elif defined(__OFFLOAD_HIP)
192 OFFLOAD_FFT_CHECK(hipfftDestroy(plan));
193#endif
194}
195
196/*******************************************************************************
197 * \brief Wrapper around cufftExecZ2Z.
198 ******************************************************************************/
199static inline void offload_fftExecZ2Z(offload_fftHandle plan,
200 const double *idata, double *odata,
201 int direction) {
202#if defined(__OFFLOAD_CUDA)
203 OFFLOAD_FFT_CHECK(cufftExecZ2Z(plan, (cufftDoubleComplex *)idata,
204 (cufftDoubleComplex *)odata, direction));
205#elif defined(__OFFLOAD_HIP)
206 OFFLOAD_FFT_CHECK(hipfftExecZ2Z(plan, (hipfftDoubleComplex *)idata,
207 (hipfftDoubleComplex *)odata, direction));
208#endif
209}
210
211#endif // #if (defined(__OFFLOAD_CUDA) || defined(__OFFLOAD_HIP))
212
213#endif
214
215// EOF
offload_library.h
offload_runtime.h