8#ifndef NNP_GPU_INTERNAL_H
9#define NNP_GPU_INTERNAL_H
12#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
41#if defined(__OFFLOAD_HIP) && !defined(__HIP_PLATFORM_NVIDIA__)
42#define NNP_SYNCWARP() \
44 __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "wavefront"); \
45 __builtin_amdgcn_wave_barrier(); \
48#define NNP_SYNCWARP() __syncwarp()
57#if defined(__OFFLOAD_HIP)
58#define NNP_SHFL_XOR(val, laneMask) __shfl_xor((val), (laneMask))
60#define NNP_SHFL_XOR(val, laneMask) \
61 __shfl_xor_sync(0xffffffffu, (val), (laneMask))
72#if defined(__OFFLOAD_CUDA) || defined(__HIP_PLATFORM_NVIDIA__)
73#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600
74__device__ __inline__
double atomicAdd(
double *address,
double val) {
75 unsigned long long int *address_as_ull = (
unsigned long long int *)address;
76 unsigned long long int old = *address_as_ull, assumed;
79 old = atomicCAS(address_as_ull, assumed,
80 __double_as_longlong(val + __longlong_as_double(assumed)));
81 }
while (assumed != old);
82 return __longlong_as_double(old);