(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_internal.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
8#ifndef NNP_GPU_INTERNAL_H
9#define NNP_GPU_INTERNAL_H
10
12#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
13
14#include <stdio.h>
15
16/*
17 * Shared helpers for the NNP GPU device code. Included by nnp_gpu_state.c and
18 * the device kernel translation units. Device runtime calls go through the
19 * backend-agnostic offload wrappers (offload_runtime.h), which abort on error;
20 * kernel-launch status is checked with OFFLOAD_CHECK(offloadPeekAtLastError()).
21 *
22 * Host-side error reporting: a launcher that rejects its arguments prints the
23 * offending sizes to stderr and returns -1, and the Fortran driver turns the
24 * code into a CPABORT. The stderr line carries the diagnostic detail; the
25 * fprintf-plus-fatal pattern matches OFFLOAD_CHECK itself and the fatal paths
26 * in src/grid/gpu. Nothing here prints on a success path.
27 *
28 * Warp width: every kernel uses 32-thread rows, so the butterfly reductions
29 * and the NNP_SYNCWARP barriers in the angular kernel are correct on a
30 * 64-wide AMD wavefront as well (a wavefront then carries two rows and the
31 * barrier orders both, a superset of what the callers need).
32 */
33
34/*
35 * Warp-level barrier ordering per-warp shared-memory traffic. CUDA provides
36 * the no-argument __syncwarp(); ROCm's device headers do not declare it for
37 * AMD targets, so there the same effect is an acquire-release fence at
38 * wavefront scope plus a wavefront execution barrier, which is how ROCm
39 * defines the intrinsic where it exists.
40 */
41#if defined(__OFFLOAD_HIP) && !defined(__HIP_PLATFORM_NVIDIA__)
42#define NNP_SYNCWARP() \
43 do { \
44 __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "wavefront"); \
45 __builtin_amdgcn_wave_barrier(); \
46 } while (0)
47#else
48#define NNP_SYNCWARP() __syncwarp()
49#endif
50
51/*
52 * Warp-wide XOR shuffle, portable across backends. CUDA's __shfl_xor_sync takes
53 * an active-lane mask; HIP's __shfl_xor has no mask argument. Both reduce over
54 * a 32-lane warp here (max lane offset 16), which is also correct on a 64-wide
55 * AMD wavefront when the block is 32 threads wide.
56 */
57#if defined(__OFFLOAD_HIP)
58#define NNP_SHFL_XOR(val, laneMask) __shfl_xor((val), (laneMask))
59#else
60#define NNP_SHFL_XOR(val, laneMask) \
61 __shfl_xor_sync(0xffffffffu, (val), (laneMask))
62#endif
63
64/*
65 * Double-precision atomicAdd is a device intrinsic only from compute capability
66 * 6.0 (Pascal). CP2K still supports older CUDA architectures, so provide the
67 * compare-and-swap fallback for them; the descriptor and force kernels then
68 * call atomicAdd on doubles unconditionally. The integer comparison in the loop
69 * is what makes it terminate when the accumulator holds a NaN. Mirrors the shim
70 * in src/grid/gpu/grid_gpu_internal_header.h.
71 */
72#if defined(__OFFLOAD_CUDA) || defined(__HIP_PLATFORM_NVIDIA__)
73#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600
74__device__ __inline__ double atomicAdd(double *address, double val) {
75 unsigned long long int *address_as_ull = (unsigned long long int *)address;
76 unsigned long long int old = *address_as_ull, assumed;
77 do {
78 assumed = old;
79 old = atomicCAS(address_as_ull, assumed,
80 __double_as_longlong(val + __longlong_as_double(assumed)));
81 } while (assumed != old);
82 return __longlong_as_double(old);
83}
84#endif
85#endif
86
87#endif /* __OFFLOAD && !__NO_OFFLOAD_NNP */
88#endif /* NNP_GPU_INTERNAL_H */