14#ifndef GRID_GPU_INTERNAL_HEADER_H
15#define GRID_GPU_INTERNAL_HEADER_H
25#define GRID_DEVICE __device__
35#if defined(__OFFLOAD_CUDA) || defined(__HIP_PLATFORM_NVIDIA__)
36#if __CUDA_ARCH__ < 600
37__device__ __inline__
double atomicAdd(
double *address,
double val) {
38 unsigned long long int *address_as_ull = (
unsigned long long int *)address;
39 unsigned long long int old = *address_as_ull, assumed;
43 old = atomicCAS(address_as_ull, assumed,
44 __double_as_longlong(val + __longlong_as_double(assumed)));
48 }
while (assumed != old);
50 return __longlong_as_double(old);
63static_assert(
sizeof(
orbital) == 3,
"orbital is expected to be packed");
101 unsigned short int n1;
102 unsigned short int n2;
149__host__ __device__ __inline__
int ncoset(
const int l) {
150 static const int table[] = {1,
153 20, 35, 56, 84, 120, 165, 220, 286,
154 364, 455, 560, 680, 816, 969, 1140, 1330};
161__host__ __device__ __inline__
int coset(
int lx,
int ly,
int lz) {
162 const int l = lx + ly + lz;
166 return ncoset(l - 1) + ((l - lx) * (l - lx + 1)) / 2 + lz;
184 b.l[
i] = max(0, a.l[
i] - 1);
192 return coset(a.l[0], a.l[1], a.l[2]);
195__device__ __inline__
double power(
const double x,
const int expo) {
197 for (
int i = 1;
i <= expo;
i++)
205template <
typename T =
double>
207 const T value,
const int n, T *cab) {
208 atomicAdd(&cab[
idx(b) * n +
idx(a)], value);
216 static bool initialized =
false;
223 for (
int lx = 0; lx <= 18; lx++) {
224 for (
int ly = 0; ly <= 18 - lx; ly++) {
225 for (
int lz = 0; lz <= 18 - lx - ly; lz++) {
226 const int i =
coset(lx, ly, lz);
227 coset_inv_host[
i] = {{
static_cast<signed char>(lx),
228 static_cast<signed char>(ly),
229 static_cast<signed char>(lz)}};
234 offloadMemcpyToSymbol(
coset_inv, &coset_inv_host,
sizeof(coset_inv_host));
241 return blockIdx.x + gridDim.x * (blockIdx.y + gridDim.y * blockIdx.z);
246 return threadIdx.x + blockDim.x * (threadIdx.y + blockDim.y * threadIdx.z);
249__inline__ __host__ __device__ double3
251 const double y,
const double z) {
257 r3.x = z * dh_[6] + y * dh_[3] + x * dh_[0];
258 r3.y = z * dh_[7] + y * dh_[4] + x * dh_[1];
259 r3.z = z * dh_[8] + y * dh_[5] + x * dh_[2];
263template <
typename T,
typename T3,
bool orthorhombic_>
266 const T *
const __restrict__ dh_inv_,
const T3 &rp,
267 T3 &roffset, int3 &cubecenter, int3 &lb_cube,
278 cubecenter.x = std::floor(rp1.x);
279 cubecenter.y = std::floor(rp1.y);
280 cubecenter.z = std::floor(rp1.z);
297 T norm1, norm2, norm3;
298 norm1 = dh_[0] * dh_[0] + dh_[1] * dh_[1] + dh_[2] * dh_[2];
299 norm2 = dh_[3] * dh_[3] + dh_[4] * dh_[4] + dh_[5] * dh_[5];
300 norm3 = dh_[6] * dh_[6] + dh_[7] * dh_[7] + dh_[8] * dh_[8];
302 norm1 = std::sqrt(norm1);
303 norm2 = std::sqrt(norm2);
304 norm3 = std::sqrt(norm3);
306 const T disr_radius =
307 std::min(norm1, std::min(norm2, norm3)) *
309 (
int)ceil(radius / std::min(norm1, std::min(norm2, norm3)))));
311 rp2.x = cubecenter.x;
312 rp2.y = cubecenter.y;
313 rp2.z = cubecenter.z;
334 lb_cube.x = std::ceil(-1e-8 - rp3.x);
335 lb_cube.y = std::ceil(-1e-8 - rp3.y);
336 lb_cube.z = std::ceil(-1e-8 - rp3.z);
349 cube_size.x = 2 - 2 * lb_cube.x;
350 cube_size.y = 2 - 2 * lb_cube.y;
351 cube_size.z = 2 - 2 * lb_cube.z;
363 for (
int i = -1;
i <= 1;
i++) {
364 for (
int j = -1; j <= 1; j++) {
365 for (
int k = -1; k <= 1; k++) {
367 r.x = rp.x + ((T)
i) * radius;
368 r.y = rp.y + ((T)j) * radius;
369 r.z = rp.z + ((T)k) * radius;
372 lb_cube.x = std::min(lb_cube.x, (
int)std::floor(roffset.x));
373 ub_cube.x = std::max(ub_cube.x, (
int)std::ceil(roffset.x));
375 lb_cube.y = std::min(lb_cube.y, (
int)std::floor(roffset.y));
376 ub_cube.y = std::max(ub_cube.y, (
int)std::ceil(roffset.y));
378 lb_cube.z = std::min(lb_cube.z, (
int)std::floor(roffset.z));
379 ub_cube.z = std::max(ub_cube.z, (
int)std::ceil(roffset.z));
384 cube_size.x = ub_cube.x - lb_cube.x;
385 cube_size.y = ub_cube.y - lb_cube.y;
386 cube_size.z = ub_cube.z - lb_cube.z;
390 roffset.x = cubecenter.x - rp1.x;
391 roffset.y = cubecenter.y - rp1.y;
392 roffset.z = cubecenter.z - rp1.z;
396 lb_cube.x -= cubecenter.x;
397 lb_cube.y -= cubecenter.y;
398 lb_cube.z -= cubecenter.z;
405 const int border_mask,
406 const int3 &border_width,
408 int3 &window_shift) {
413 window_size.x = grid_size.x - 1;
414 window_size.y = grid_size.y - 1;
415 window_size.z = grid_size.z - 1;
417 if (border_mask & (1 << 0))
418 window_shift.x += border_width.x;
419 if (border_mask & (1 << 1))
420 window_size.x -= border_width.x;
421 if (border_mask & (1 << 2))
422 window_shift.y += border_width.y;
423 if (border_mask & (1 << 3))
424 window_size.y -= border_width.y;
425 if (border_mask & (1 << 4))
426 window_shift.z += border_width.z;
427 if (border_mask & (1 << 5))
428 window_size.z -= border_width.z;
449template <
typename T,
typename T3,
bool distributed__>
450__device__ __inline__
void
458 if (
task.apply_border_mask) {
496__device__ __inline__
int line_index(
const int dir,
const int m,
const int p,
497 const int q,
const bool on_a,
498 const int other,
const int n1) {
501 l[(dir + 1) % 3] = p;
502 l[(dir + 2) % 3] = q;
503 const int c =
coset(l[0], l[1], l[2]);
504 return on_a ? other * n1 + c : c * n1 + other;
523template <
typename T,
bool TRANSPOSE>
524__device__ __inline__
void move_center(
const int la,
const int lb,
const int n1,
525 const int dir,
const bool on_a,
526 const T dr, T *cab) {
528 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
529 const int l = on_a ? la : lb;
531 const int ntri = (l + 1) * (l + 2) / 2;
533 for (
int line =
tid; line < nother * ntri; line +=
nthreads) {
534 const int other = line / ntri;
537 const int len = l - p - q;
541 for (
int i = 0;
i < len;
i++) {
543 T carry = cab[
line_index(dir, len, p, q, on_a, other, n1)];
544 for (
int j = len - 1; j >=
i; j--) {
546 carry = cab[
idx] + dr * carry;
551 for (
int i = len - 1;
i >= 0;
i--) {
553 T carry = cab[
line_index(dir,
i, p, q, on_a, other, n1)];
554 for (
int j =
i + 1; j <= len; j++) {
556 carry = cab[
idx] + dr * carry;
572 T *__restrict__ cxyz) {
573 const int la =
task.la_max, lb =
task.lb_max, n1 =
task.n1;
574 for (
int dir = 0; dir < 3; dir++) {
579 for (
int dir = 0; dir < 3; dir++) {
587 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
591 for (
int ax = 0; ax <= min(c.l[0], la); ax++) {
592 for (
int ay = 0; ay <= min(c.l[1], la - ax); ay++) {
593 for (
int az = 0; az <= min(c.l[2], la - ax - ay); az++) {
594 const int bx = c.l[0] - ax, by = c.l[1] - ay, bz = c.l[2] - az;
595 if (bx + by + bz <= lb)
596 val += cab[
coset(bx, by, bz) * n1 +
coset(ax, ay, az)];
600 cxyz[ic] =
task.prefactor * val;
643 const T *__restrict__ cxyz,
644 T *__restrict__ cab) {
645 const int la =
task.la_max, lb =
task.lb_max, n1 =
task.n1;
647 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
651 const int jco = k / n1;
652 const int ico = k - jco * n1;
655 cab[k] =
task.prefactor *
656 cxyz[
coset(a.l[0] + b.l[0], a.l[1] + b.l[1], a.l[2] + b.l[2])];
660 for (
int dir = 0; dir < 3; dir++) {
665 for (
int dir = 0; dir < 3; dir++) {
680template <
typename T,
typename T3>
681__device__ __inline__
void
684 if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {
710 task.norm_lattice_vector_z_2 = dev.
dh_[6] * dev.
dh_[6] +
713 task.norm_lattice_vector_y_2 = dev.
dh_[3] * dev.
dh_[3] +
717 task.norm_inverse_lattice_vector_y =
720 task.norm_inverse_lattice_vector_x =
739 if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {
746 for (
int i = 0;
i < 3;
i++) {
758 const int la_max_basis =
glb_task.la_max;
759 const int lb_max_basis =
glb_task.lb_max;
760 const int la_min_basis =
glb_task.la_min;
761 const int lb_min_basis =
glb_task.lb_min;
771 task.first_coseta = (la_min_basis > 0) ?
ncoset(la_min_basis - 1) : 0;
772 task.first_cosetb = (lb_min_basis > 0) ?
ncoset(lb_min_basis - 1) : 0;
841template <
typename Kernel>
843#if defined(__OFFLOAD_HIP)
844 hipFuncAttributes attr;
846 hipFuncGetAttributes(&attr,
reinterpret_cast<const void *
>(kernel)));
848 cudaFuncAttributes attr;
849 OFFLOAD_CHECK(cudaFuncGetAttributes(&attr, kernel));
851 const size_t static_smem = (attr.sharedSizeBytes + 15) & ~
size_t(15);
852 if (static_smem >= budget)
854 return std::min(budget - static_smem,
855 (
size_t)std::max(attr.maxDynamicSharedSizeBytes, 0));
862#define GRID_GPU_CHECK_LAUNCH() OFFLOAD_CHECK(offloadGetLastError())
877 la_max_ = lmax +
ldiffs.la_max_diff;
878 lb_max_ = lmax +
ldiffs.lb_max_diff;
879 lp_max_ = la_max_ + lb_max_;
883 "ERROR: the GPU grid backend supports la + lb <= %d, this "
884 "operation needs %d (lmax %d, la_max_diff %d, lb_max_diff %d).\n",
903 inline int lp_diff()
const {
return lp_diff_; }
907 inline int lp_max()
const {
return lp_max_; }
915template <
typename T,
typename T3>
916__device__ __inline__ T
918 int &ymin,
int &ymax) {
920 ymin = (2 * (z +
task.lb_cube.z) - 1) / 2;
922 kremain =
task.discrete_radius *
task.discrete_radius -
923 ((T)ymin) *
task.norm_lattice_vector_z_2;
924 ymin = ceil(-1.0e-8 -
925 sqrt(fmax(0.0, kremain)) *
task.norm_inverse_lattice_vector_y);
926 ymax = 1 - ymin -
task.lb_cube.y;
927 ymin = ymin -
task.lb_cube.y;
931template <
typename T,
typename T3>
932__device__ __inline__
void
934 const T kremain,
int &xmin,
int &xmax) {
935 xmin = (2 * (y +
task.lb_cube.y) - 1) / 2;
937 xmin = ceil(-1.0e-8 -
938 sqrt(fmax(0.0, kremain - xmin *
task.norm_lattice_vector_y_2)) *
939 task.norm_inverse_lattice_vector_x);
940 xmax = 1 - xmin -
task.lb_cube.x;
941 xmin -=
task.lb_cube.x;
smem_parameters(const ldiffs_value ldiffs, const int lmax)
ldiffs_value ldiffs() const
static void const int const int i
fill_smem_task_reduced(dev_, dev_.first_task+block_index(), task)
constexpr size_t sphi_two_step_max_smem
int sphi_work_size(const int lmax, const int max_nsgf_set)
Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calcu...
__device__ __inline__ void prep_term(const orbital a, const orbital b, const T value, const int n, T *cab)
Adds given value to matrix element cab[idx(b)][idx(a)].
__inline__ __host__ __device__ double3 compute_coordinates(const double *__restrict__ dh_, const double x, const double y, const double z)
const task_info & glb_task
static void init_constant_memory()
Initializes the device's constant memory.
__inline__ __device__ unsigned int block_index()
__device__ __inline__ void setup_task_cube_center(const kernel_params &dev_, smem_task_reduced< T, T3 > &task)
Shifts task.cube_center into the local grid's coordinate system and, for distributed grids,...
__device__ __inline__ T calculate_ymix_ymax_boundaries(smem_task_reduced< T, T3 > &task, const int z, int &ymin, int &ymax)
__device__ __inline__ int line_index(const int dir, const int m, const int p, const int q, const bool on_a, const int other, const int n1)
__device__ __inline__ void fill_smem_task_coef(const kernel_params &dev, const int task_id, smem_task< T > &task)
Copies a task from global to shared memory and does precomputations.
__device__ __inline__ double power(const double x, const int expo)
__inline__ __device__ unsigned int thread_global_index()
__device__ __inline__ orbital up(const int i, const orbital &a)
Increase i'th component of given orbital angular momentum.
__inline__ __device__ int idx(const orbital a)
Return coset index of given orbital angular momentum.
__device__ __inline__ void cab_to_cxyz_factorized(const smem_task< T > &task, T *__restrict__ cab, T *__restrict__ cxyz)
Transforms the coefficients C_ab into C_xyz, cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a] ...
__inline__ T compute_cube_properties(const T radius, const T *const __restrict__ dh_, const T *const __restrict__ dh_inv_, const T3 &rp, T3 &roffset, int3 &cubecenter, int3 &lb_cube, int3 &cube_size)
__inline__ __device__ int wrap_grid_index(const int idx, const int full_size)
Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry share...
__constant__ orbital coset_inv[1330]
size_t max_dynamic_smem(Kernel kernel, const size_t budget)
Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memo...
__host__ __device__ __inline__ int coset(int lx, int ly, int lz)
Maps three angular momentum components to a single zero based index.
__global__ const int const int max_nsgf_set
__device__ __inline__ void move_center(const int la, const int lb, const int n1, const int dir, const bool on_a, const T dr, T *cab)
Moves the expansion center of the a (on_a) or b cartesian index of cab along direction dir,...
__device__ __inline__ void decode_pair(int t, int &p, int &q)
Factorized cab <-> cxyz transformations.
__inline__ __device__ T * allocate_workspace(const kernel_params &dev_)
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
__device__ __inline__ void cxyz_to_cab_factorized(const smem_task< T > &task, const T *__restrict__ cxyz, T *__restrict__ cab)
Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.
__device__ __inline__ void calculate_xmin_xmax_boundaries(smem_task_reduced< T, T3 > &task, const int y, const T kremain, int &xmin, int &xmax)
constexpr int grid_gpu_max_l
__inline__ __device__ orbital down(const int i, const orbital &a)
Decrease i'th component of given orbital angular momentum.
__shared__ smem_task_reduced< double, double3 > task
__inline__ __device__ void compute_window_size(const int3 grid_size, const int border_mask, const int3 &border_width, int3 &window_size, int3 &window_shift)
Parameters of the collocate kernel.
int * cab_block_offset_dev
kernel_buffers buffers_dev
Differences in angular momentum.
Orbital angular momentum.
data needed for collocate and integrate kernels
T norm_lattice_vector_y_2
T norm_lattice_vector_z_2
T norm_inverse_lattice_vector_x
T norm_inverse_lattice_vector_y
data needed for calculating the coefficients, forces, and stress
unsigned short int nsgf_seta
unsigned short int nsgf_setb