(git:27c966b)
Loading...
Searching...
No Matches
grid_gpu_internal_header.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7
8/*
9 * Authors :
10 - Dr Mathieu Taillefumier (ETH Zurich / CSCS)
11 - Advanced Micro Devices, Inc.
12*/
13
14#ifndef GRID_GPU_INTERNAL_HEADER_H
15#define GRID_GPU_INTERNAL_HEADER_H
16
17#include <algorithm>
18#include <assert.h>
19#include <limits.h>
20#include <math.h>
21#include <stdio.h>
22#include <stdlib.h>
23#include <string.h>
24
25#define GRID_DEVICE __device__
26extern "C" {
29}
30
31#include "grid_gpu_context.h"
32
33namespace grid_gpu_backend {
34
35#if defined(__OFFLOAD_CUDA) || defined(__HIP_PLATFORM_NVIDIA__)
36#if __CUDA_ARCH__ < 600
37__device__ __inline__ double atomicAdd(double *address, double val) {
38 unsigned long long int *address_as_ull = (unsigned long long int *)address;
39 unsigned long long int old = *address_as_ull, assumed;
40
41 do {
42 assumed = old;
43 old = atomicCAS(address_as_ull, assumed,
44 __double_as_longlong(val + __longlong_as_double(assumed)));
45
46 // Note: uses integer comparison to avoid hang in case of NaN (since NaN !=
47 // NaN)
48 } while (assumed != old);
49
50 return __longlong_as_double(old);
51}
52#endif
53#endif
54
55/*******************************************************************************
56 * \brief Orbital angular momentum.
57 ******************************************************************************/
58struct orbital {
59 // signed char keeps coset_inv at 4 kB of constant memory (16 kB with int).
60 // The components are promoted to int in any arithmetic expression.
61 signed char l[3];
62};
63static_assert(sizeof(orbital) == 3, "orbital is expected to be packed");
64
65// largest angular momentum covered by the tables of the backend (ncoset,
66// coset_inv). It bounds la + lb of every operation.
67constexpr int grid_gpu_max_l = 18;
68
69__constant__ orbital coset_inv[1330];
70
71/*******************************************************************************
72 * \brief Differences in angular momentum.
73 ******************************************************************************/
79};
80
81/*******************************************************************************
82 * \brief data needed for calculating the coefficients, forces, and stress
83 ******************************************************************************/
84template <typename T> struct smem_task {
86 T ra[3];
87 T rb[3];
88 T rp[3];
89 T rab[3];
95 char la_max;
96 char lb_max;
97 char la_min;
98 char lb_min;
99 char lp;
100 // size of the cab matrix
101 unsigned short int n1;
102 unsigned short int n2;
103 // size of entire spherical basis
104 unsigned short int nsgfa;
105 unsigned short int nsgfb;
106 // size of spherical set
107 unsigned short int nsgf_seta;
108 unsigned short int nsgf_setb;
109 // start of decontracted set, ie. pab and hab
112 // size of decontracted set, ie. pab and hab
115 // strides of the sphi transformation matrices
118 // pointers matrices
122 // integrate
126};
127
128/*******************************************************************************
129 * \brief data needed for collocate and integrate kernels
130 ******************************************************************************/
131template <typename T, typename T3> struct smem_task_reduced {
132 // radius: true cutoff used for the sphere-membership test.
133 // discrete_radius: box-aligned cutoff (always >= radius) used only for
134 // sizing the cube / trimming box boundaries.
141 char lp;
143};
144
145/*******************************************************************************
146 * \brief Number of Cartesian orbitals up to given angular momentum quantum.
147 * \author Ole Schuett
148 ******************************************************************************/
149__host__ __device__ __inline__ int ncoset(const int l) {
150 static const int table[] = {1, // l=0
151 4, // l=1
152 10, // l=2 ...
153 20, 35, 56, 84, 120, 165, 220, 286,
154 364, 455, 560, 680, 816, 969, 1140, 1330};
155 return table[l];
156}
157
158/*******************************************************************************
159 * \brief Maps three angular momentum components to a single zero based index.
160 ******************************************************************************/
161__host__ __device__ __inline__ int coset(int lx, int ly, int lz) {
162 const int l = lx + ly + lz;
163 if (l == 0) {
164 return 0;
165 } else {
166 return ncoset(l - 1) + ((l - lx) * (l - lx + 1)) / 2 + lz;
167 }
168}
169
170/*******************************************************************************
171 * \brief Increase i'th component of given orbital angular momentum.
172 ******************************************************************************/
173__device__ __inline__ orbital up(const int i, const orbital &a) {
174 orbital b = a;
175 b.l[i] += 1;
176 return b;
177}
178
179/*******************************************************************************
180 * \brief Decrease i'th component of given orbital angular momentum.
181 ******************************************************************************/
182__inline__ __device__ orbital down(const int i, const orbital &a) {
183 orbital b = a;
184 b.l[i] = max(0, a.l[i] - 1);
185 return b;
186}
187
188/*******************************************************************************
189 * \brief Return coset index of given orbital angular momentum.
190 ******************************************************************************/
191__inline__ __device__ int idx(const orbital a) {
192 return coset(a.l[0], a.l[1], a.l[2]);
193}
194
195__device__ __inline__ double power(const double x, const int expo) {
196 double tmp = 1.0;
197 for (int i = 1; i <= expo; i++)
198 tmp *= x;
199 return tmp;
200}
201
202/*******************************************************************************
203 * \brief Adds given value to matrix element cab[idx(b)][idx(a)].
204 ******************************************************************************/
205template <typename T = double>
206__device__ __inline__ void prep_term(const orbital a, const orbital b,
207 const T value, const int n, T *cab) {
208 atomicAdd(&cab[idx(b) * n + idx(a)], value);
209}
210
211/*******************************************************************************
212 * \brief Initializes the device's constant memory.
213 * \author Ole Schuett
214 ******************************************************************************/
215inline static void init_constant_memory() {
216 static bool initialized = false;
217 if (initialized) {
218 return; // constant memory has to be initialized only once
219 }
220
221 // Inverse coset mapping
222 orbital coset_inv_host[1330];
223 for (int lx = 0; lx <= 18; lx++) {
224 for (int ly = 0; ly <= 18 - lx; ly++) {
225 for (int lz = 0; lz <= 18 - lx - ly; lz++) {
226 const int i = coset(lx, ly, lz);
227 coset_inv_host[i] = {{static_cast<signed char>(lx),
228 static_cast<signed char>(ly),
229 static_cast<signed char>(lz)}};
230 }
231 }
232 }
233
234 offloadMemcpyToSymbol(coset_inv, &coset_inv_host, sizeof(coset_inv_host));
235
236 initialized = true;
237}
238
239// calculate the global index of a thread block
240__inline__ __device__ unsigned int block_index() {
241 return blockIdx.x + gridDim.x * (blockIdx.y + gridDim.y * blockIdx.z);
242}
243
244// Calculating the global index in the grid of any given device thread
245__inline__ __device__ unsigned int thread_global_index() {
246 return threadIdx.x + blockDim.x * (threadIdx.y + blockDim.y * threadIdx.z);
247}
248
249__inline__ __host__ __device__ double3
250compute_coordinates(const double *__restrict__ dh_, const double x,
251 const double y, const double z) {
252
253 double3 r3;
254 // I make no distinction between orthorhombic and non orthorhombic
255 // cases
256
257 r3.x = z * dh_[6] + y * dh_[3] + x * dh_[0];
258 r3.y = z * dh_[7] + y * dh_[4] + x * dh_[1];
259 r3.z = z * dh_[8] + y * dh_[5] + x * dh_[2];
260 return r3;
261}
262
263template <typename T, typename T3, bool orthorhombic_>
264__inline__ T
265compute_cube_properties(const T radius, const T *const __restrict__ dh_,
266 const T *const __restrict__ dh_inv_, const T3 &rp,
267 T3 &roffset, int3 &cubecenter, int3 &lb_cube,
268 int3 &cube_size) {
269
270 /* center of the gaussian in the lattice coordinates */
271 T3 rp1, rp2, rp3;
272
273 /* it is in the lattice vector frame */
274 rp1 = compute_coordinates(dh_inv_, rp.x, rp.y, rp.z);
275 // convert_to_lattice_coordinates(dh_inv_, rp, &rp1);
276
277 /* compute the grid point that is the closest to the sphere center. */
278 cubecenter.x = std::floor(rp1.x);
279 cubecenter.y = std::floor(rp1.y);
280 cubecenter.z = std::floor(rp1.z);
281
282 /* seting up the cube parameters */
283
284 if (orthorhombic_) {
285
286 // the cube is actually slightly bigger than the sphere of radius r. that's
287 // why we need to discretize it to get the cube size "right".
288
289 // disc_radius >= radius always. somehow despite the fact that we compile
290 // things with a c++ compiler on the host side, we need to use fmin instead
291 // of std::min since std::min is not allowed on the device side
292
293 // We assume no specific form for the orthogonal matrix. (can be diaognal or
294 // completely full, the only constraint is that the tree vectors are
295 // orthogonal)
296
297 T norm1, norm2, norm3;
298 norm1 = dh_[0] * dh_[0] + dh_[1] * dh_[1] + dh_[2] * dh_[2];
299 norm2 = dh_[3] * dh_[3] + dh_[4] * dh_[4] + dh_[5] * dh_[5];
300 norm3 = dh_[6] * dh_[6] + dh_[7] * dh_[7] + dh_[8] * dh_[8];
301
302 norm1 = std::sqrt(norm1);
303 norm2 = std::sqrt(norm2);
304 norm3 = std::sqrt(norm3);
305
306 const T disr_radius =
307 std::min(norm1, std::min(norm2, norm3)) *
308 (std::max(1,
309 (int)ceil(radius / std::min(norm1, std::min(norm2, norm3)))));
310
311 rp2.x = cubecenter.x;
312 rp2.y = cubecenter.y;
313 rp2.z = cubecenter.z;
314
315 /* convert the cube center from grid points coordinates to cartesian */
316 roffset = compute_coordinates(dh_, rp2.x, rp2.y, rp2.z);
317
318 /* cube center */
319 roffset.x -= rp.x;
320 roffset.y -= rp.y;
321 roffset.z -= rp.z;
322
323 /* This potentially makes the cube bigger than necessary because disc_radius
324 is derived with the smallest lattice parameter. It is not an issue when
325 the lattice is cubic but could lead to more calculations than necessary
326 when the lattice is orthorhombic */
327 rp2.x = disr_radius;
328 rp2.y = disr_radius;
329 rp2.z = disr_radius;
330
331 /* it is in the lattice vector frame */
332 rp3 = compute_coordinates(dh_inv_, rp2.x, rp2.y, rp2.z);
333 /* lower and upper bounds */
334 lb_cube.x = std::ceil(-1e-8 - rp3.x);
335 lb_cube.y = std::ceil(-1e-8 - rp3.y);
336 lb_cube.z = std::ceil(-1e-8 - rp3.z);
337
338 /* it is in the lattice vector frame */
339 rp2 = compute_coordinates(dh_inv_, roffset.x, roffset.y, roffset.z);
340
341 /* Express the offset in lattice coordinates */
342 roffset.x = rp2.x;
343 roffset.y = rp2.y;
344 roffset.z = rp2.z;
345
346 /* compute the cube size ignoring periodicity */
347 /* the interval is not symmetrical for some curious reasons. it should go
348 * from [-L..L+1] so the number of points is multiple of two */
349 cube_size.x = 2 - 2 * lb_cube.x;
350 cube_size.y = 2 - 2 * lb_cube.y;
351 cube_size.z = 2 - 2 * lb_cube.z;
352 return disr_radius;
353 } else {
354 int3 ub_cube;
355
356 lb_cube.x = INT_MAX;
357 ub_cube.x = INT_MIN;
358 lb_cube.y = INT_MAX;
359 ub_cube.y = INT_MIN;
360 lb_cube.z = INT_MAX;
361 ub_cube.z = INT_MIN;
362
363 for (int i = -1; i <= 1; i++) {
364 for (int j = -1; j <= 1; j++) {
365 for (int k = -1; k <= 1; k++) {
366 T3 r;
367 r.x = rp.x + ((T)i) * radius;
368 r.y = rp.y + ((T)j) * radius;
369 r.z = rp.z + ((T)k) * radius;
370 roffset = compute_coordinates(dh_inv_, r.x, r.y, r.z);
371
372 lb_cube.x = std::min(lb_cube.x, (int)std::floor(roffset.x));
373 ub_cube.x = std::max(ub_cube.x, (int)std::ceil(roffset.x));
374
375 lb_cube.y = std::min(lb_cube.y, (int)std::floor(roffset.y));
376 ub_cube.y = std::max(ub_cube.y, (int)std::ceil(roffset.y));
377
378 lb_cube.z = std::min(lb_cube.z, (int)std::floor(roffset.z));
379 ub_cube.z = std::max(ub_cube.z, (int)std::ceil(roffset.z));
380 }
381 }
382 }
383 /* compute the cube size ignoring periodicity */
384 cube_size.x = ub_cube.x - lb_cube.x;
385 cube_size.y = ub_cube.y - lb_cube.y;
386 cube_size.z = ub_cube.z - lb_cube.z;
387
388 /* compute the offset in lattice coordinates */
389
390 roffset.x = cubecenter.x - rp1.x;
391 roffset.y = cubecenter.y - rp1.y;
392 roffset.z = cubecenter.z - rp1.z;
393
394 // shift the boundaries compared to the cube center so that the
395 // specialization ortho / non ortho is minimal
396 lb_cube.x -= cubecenter.x;
397 lb_cube.y -= cubecenter.y;
398 lb_cube.z -= cubecenter.z;
399
400 return radius;
401 }
402}
403
404__inline__ __device__ void compute_window_size(const int3 grid_size,
405 const int border_mask,
406 const int3 &border_width,
407 int3 &window_size,
408 int3 &window_shift) {
409 window_shift.x = 0;
410 window_shift.y = 0;
411 window_shift.z = 0;
412
413 window_size.x = grid_size.x - 1;
414 window_size.y = grid_size.y - 1;
415 window_size.z = grid_size.z - 1;
416
417 if (border_mask & (1 << 0))
418 window_shift.x += border_width.x;
419 if (border_mask & (1 << 1))
420 window_size.x -= border_width.x;
421 if (border_mask & (1 << 2))
422 window_shift.y += border_width.y;
423 if (border_mask & (1 << 3))
424 window_size.y -= border_width.y;
425 if (border_mask & (1 << 4))
426 window_shift.z += border_width.z;
427 if (border_mask & (1 << 5))
428 window_size.z -= border_width.z;
429}
430
431/*******************************************************************************
432 * \brief Wraps a cube-local coordinate into the periodic full grid along one
433 * axis. Single point of entry shared by collocate_kernel and
434 * integrate_kernel.
435 ******************************************************************************/
436__inline__ __device__ int wrap_grid_index(const int idx, const int full_size) {
437 int idx2 = idx % full_size;
438 if (idx2 < 0)
439 idx2 += full_size;
440 return idx2;
441}
442
443/*******************************************************************************
444 * \brief Shifts task.cube_center into the local grid's coordinate system and,
445 * for distributed grids, derives the border window. Single point of
446 * entry shared by collocate_kernel and integrate_kernel so the two
447 * cannot drift apart on which axis maps to which.
448 ******************************************************************************/
449template <typename T, typename T3, bool distributed__>
450__device__ __inline__ void
453 task.cube_center.x += task.lb_cube.x - dev_.grid_lower_corner_.x;
454 task.cube_center.y += task.lb_cube.y - dev_.grid_lower_corner_.y;
455 task.cube_center.z += task.lb_cube.z - dev_.grid_lower_corner_.z;
456
457 if (distributed__) {
458 if (task.apply_border_mask) {
460 dev_.grid_local_size_,
461 dev_.tasks[dev_.first_task + block_index()].border_mask,
462 dev_.grid_border_width_, task.window_size, task.window_shift);
463 }
464 }
465}
466
467/*******************************************************************************
468 * \brief Factorized cab <-> cxyz transformations.
469 *
470 * cab_to_cxyz expands (x - a)^la (x - b)^lb around p in each direction.
471 * It is the product of three steps: move the expansion center of the a
472 * index from a to p, of the b index from b to p, then multiply the
473 * polynomials,
474 *
475 * (x - a)^n = sum_k C(n, k) (p - a)^k (x - p)^(n - k)
476 * cxyz[c] = prefactor sum_{a' + b' = c} cab_p[b'][a']
477 *
478 * The center moves are triangular, separable in x, y and z and map cab
479 * onto itself, so they are done in place, one 1D line of cab per
480 * thread. cxyz_to_cab is the transpose: a gather followed by the
481 * transposed moves. The expansion coefficients alpha of the product are
482 * never formed.
483 ******************************************************************************/
484
485// decodes t in [0, (l + 1) (l + 2) / 2) into (p, q) with p + q <= l
486__device__ __inline__ void decode_pair(int t, int &p, int &q) {
487 int s = 0;
488 while (t > s) {
489 t -= s + 1;
490 s++;
491 }
492 p = t;
493 q = s - t;
494}
495
496__device__ __inline__ int line_index(const int dir, const int m, const int p,
497 const int q, const bool on_a,
498 const int other, const int n1) {
499 int l[3];
500 l[dir] = m;
501 l[(dir + 1) % 3] = p;
502 l[(dir + 2) % 3] = q;
503 const int c = coset(l[0], l[1], l[2]);
504 return on_a ? other * n1 + c : c * n1 + other;
505}
506
507/*******************************************************************************
508 * \brief Moves the expansion center of the a (on_a) or b cartesian index of
509 * cab along direction dir, in place:
510 *
511 * forward: new[m] = sum_{k >= 0} C(m + k, k) dr^k old[m + k]
512 * transpose: new[m] = sum_{k <= m} C(m, k) dr^k old[m - k]
513 *
514 * Each thread owns a whole line (all other indices fixed). The forward
515 * update is a Taylor shift of the polynomial sum_m old[m] x^m by dr,
516 * done in place by repeated synthetic division (Ruffini-Horner):
517 * for i = 0 .. len-1, for j = len-1 down to i: c[j] += dr c[j+1]
518 * The transpose applies the transposed elementary updates in reverse
519 * order:
520 * for i = len-1 down to 0, for j = i .. len-1: c[j+1] += dr c[j]
521 * No binomial coefficient is needed, len (len + 1) / 2 FMA per line.
522 ******************************************************************************/
523template <typename T, bool TRANSPOSE>
524__device__ __inline__ void move_center(const int la, const int lb, const int n1,
525 const int dir, const bool on_a,
526 const T dr, T *cab) {
527 const int tid = thread_global_index();
528 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
529 const int l = on_a ? la : lb;
530 const int nother = on_a ? ncoset(lb) : ncoset(la);
531 const int ntri = (l + 1) * (l + 2) / 2;
532
533 for (int line = tid; line < nother * ntri; line += nthreads) {
534 const int other = line / ntri;
535 int p, q;
536 decode_pair(line - other * ntri, p, q);
537 const int len = l - p - q;
538 if (len == 0)
539 continue; // new[0] = old[0]
540 if (!TRANSPOSE) {
541 for (int i = 0; i < len; i++) {
542 // c[j] += dr c[j + 1] for j descending, c[j + 1] carried in a register
543 T carry = cab[line_index(dir, len, p, q, on_a, other, n1)];
544 for (int j = len - 1; j >= i; j--) {
545 const int idx = line_index(dir, j, p, q, on_a, other, n1);
546 carry = cab[idx] + dr * carry;
547 cab[idx] = carry;
548 }
549 }
550 } else {
551 for (int i = len - 1; i >= 0; i--) {
552 // c[j + 1] += dr c[j] for j ascending, c[j] carried in a register
553 T carry = cab[line_index(dir, i, p, q, on_a, other, n1)];
554 for (int j = i + 1; j <= len; j++) {
555 const int idx = line_index(dir, j, p, q, on_a, other, n1);
556 carry = cab[idx] + dr * carry;
557 cab[idx] = carry;
558 }
559 }
560 }
561 }
562}
563
564/*******************************************************************************
565 * \brief Transforms the coefficients C_ab into C_xyz,
566 * cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a]
567 * (see cxyz_to_cab_factorized). cab is overwritten.
568 ******************************************************************************/
569template <typename T>
570__device__ __inline__ void cab_to_cxyz_factorized(const smem_task<T> &task,
571 T *__restrict__ cab,
572 T *__restrict__ cxyz) {
573 const int la = task.la_max, lb = task.lb_max, n1 = task.n1;
574 for (int dir = 0; dir < 3; dir++) {
575 move_center<T, false>(la, lb, n1, dir, true, task.rp[dir] - task.ra[dir],
576 cab);
578 }
579 for (int dir = 0; dir < 3; dir++) {
580 move_center<T, false>(la, lb, n1, dir, false, task.rp[dir] - task.rb[dir],
581 cab);
583 }
584
585 // cxyz[c] = prefactor sum_{a' + b' = c} cab_p[b'][a']
586 const int tid = thread_global_index();
587 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
588 for (int ic = tid; ic < ncoset(task.lp); ic += nthreads) {
589 const auto &c = coset_inv[ic];
590 T val = 0.0;
591 for (int ax = 0; ax <= min(c.l[0], la); ax++) {
592 for (int ay = 0; ay <= min(c.l[1], la - ax); ay++) {
593 for (int az = 0; az <= min(c.l[2], la - ax - ay); az++) {
594 const int bx = c.l[0] - ax, by = c.l[1] - ay, bz = c.l[2] - az;
595 if (bx + by + bz <= lb)
596 val += cab[coset(bx, by, bz) * n1 + coset(ax, ay, az)];
597 }
598 }
599 }
600 cxyz[ic] = task.prefactor * val;
601 }
603}
604
605/*******************************************************************************
606 * \brief Transforms the coefficients C_xyz into C_ab, with the 3D
607 * multi-indices a, b, c,
608 *
609 * cab[b][a] = prefactor sum_c alpha[a][b][c] cxyz[c]
610 *
611 * where alpha expands the product of the two cartesian monomials
612 * around p, (x - A)^a (x - B)^b = sum_c alpha[a][b][c] (x - P)^c.
613 * In each direction, with dA = P - A and dB = P - B,
614 *
615 * (x - A)^a = sum_{a' <= a} TA[a][a'] (x - P)^a',
616 * TA[a][a'] = C(a, a') dA^(a - a')
617 *
618 * and similarly TB for b. Multiplying the two expansions,
619 *
620 * alpha[a][b][c] = sum_{a' + b' = c} TA[a][a'] TB[b][b'],
621 *
622 * hence
623 *
624 * cab[b][a] = sum_{a', b'} TB[b][b'] TA[a][a'] G[b'][a'],
625 * G[b'][a'] = prefactor cxyz[a' + b'].
626 *
627 * The sum is done in three steps without forming alpha:
628 *
629 * 1. gather G from cxyz (a' + b' summed per direction);
630 * 2. apply TA to the a index, one direction at a time, since the 3D
631 * TA is the product of the x, y and z 1D matrices:
632 * new[m] = sum_{k <= m} C(m, k) dA^k old[m - k]
633 * (move_center<T, true>, in place, one 1D line per thread);
634 * 3. apply TB to the b index in the same way.
635 *
636 * TA and TB are lower triangular, so both steps map cab onto itself.
637 * cab_to_cxyz_factorized is the transpose of the same product: TA^T
638 * and TB^T (move_center<T, false>), then the scatter
639 * cxyz[c] = prefactor sum_{a' + b' = c} cab[b'][a'].
640 ******************************************************************************/
641template <typename T>
642__device__ __inline__ void cxyz_to_cab_factorized(const smem_task<T> &task,
643 const T *__restrict__ cxyz,
644 T *__restrict__ cab) {
645 const int la = task.la_max, lb = task.lb_max, n1 = task.n1;
646 const int tid = thread_global_index();
647 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
648
649 // cab_p[b'][a'] = prefactor cxyz[a' + b']
650 for (int k = tid; k < n1 * task.n2; k += nthreads) {
651 const int jco = k / n1;
652 const int ico = k - jco * n1;
653 const auto &a = coset_inv[ico];
654 const auto &b = coset_inv[jco];
655 cab[k] = task.prefactor *
656 cxyz[coset(a.l[0] + b.l[0], a.l[1] + b.l[1], a.l[2] + b.l[2])];
657 }
659
660 for (int dir = 0; dir < 3; dir++) {
661 move_center<T, true>(la, lb, n1, dir, true, task.rp[dir] - task.ra[dir],
662 cab);
664 }
665 for (int dir = 0; dir < 3; dir++) {
666 move_center<T, true>(la, lb, n1, dir, false, task.rp[dir] - task.rb[dir],
667 cab);
669 }
670}
671
672/*******************************************************************************
673 * \brief Copies a task from global to shared memory.
674 ******************************************************************************/
675
676/* Collocate and integrate do not care about the sphi etc... computing the
677 * coefficients do not care about the exponentials (the parameters are still
678 * needed), so simplify the amount of information to save shared memory (it is
679 * crucial on AMD GPU to max out occupancy) */
680template <typename T, typename T3>
681__device__ __inline__ void
684 if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {
685 const auto &glb_task = dev.tasks[task_id];
686 task.zetp = glb_task.zeta + glb_task.zetb;
687 task.radius = glb_task.radius;
688 task.discrete_radius = glb_task.discrete_radius;
689
690 // angular momentum range for the actual collocate/integrate operation.
691 task.lp =
692 glb_task.la_max + dev.la_max_diff + glb_task.lb_max + dev.lb_max_diff;
693
694 task.cube_size.x = glb_task.cube_size.x;
695 task.cube_size.y = glb_task.cube_size.y;
696 task.cube_size.z = glb_task.cube_size.z;
697
698 task.cube_center.x = glb_task.cube_center.x;
699 task.cube_center.y = glb_task.cube_center.y;
700 task.cube_center.z = glb_task.cube_center.z;
701
702 task.roffset.x = glb_task.roffset.x;
703 task.roffset.y = glb_task.roffset.y;
704 task.roffset.z = glb_task.roffset.z;
705
706 task.lb_cube.x = glb_task.lb_cube.x;
707 task.lb_cube.y = glb_task.lb_cube.y;
708 task.lb_cube.z = glb_task.lb_cube.z;
709
710 task.norm_lattice_vector_z_2 = dev.dh_[6] * dev.dh_[6] +
711 dev.dh_[7] * dev.dh_[7] +
712 dev.dh_[8] * dev.dh_[8];
713 task.norm_lattice_vector_y_2 = dev.dh_[3] * dev.dh_[3] +
714 dev.dh_[4] * dev.dh_[4] +
715 dev.dh_[5] * dev.dh_[5];
716
717 task.norm_inverse_lattice_vector_y =
718 sqrt(dev.dh_inv_[3] * dev.dh_inv_[3] + dev.dh_inv_[4] * dev.dh_inv_[4] +
719 dev.dh_inv_[5] * dev.dh_inv_[5]);
720 task.norm_inverse_lattice_vector_x =
721 sqrt(dev.dh_inv_[0] * dev.dh_inv_[0] + dev.dh_inv_[1] * dev.dh_inv_[1] +
722 dev.dh_inv_[2] * dev.dh_inv_[2]);
723
724 task.apply_border_mask = glb_task.apply_border_mask;
725 }
727}
728
729/*******************************************************************************
730 * \brief Copies a task from global to shared memory and does precomputations.
731 ******************************************************************************/
732
733/* computing the coefficients do not care about many of the exponentials
734 * parameters, etc.. so */
735template <typename T>
736__device__ __inline__ void fill_smem_task_coef(const kernel_params &dev,
737 const int task_id,
739 if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {
740 const auto &glb_task = dev.tasks[task_id];
741 const int iatom = glb_task.iatom;
742 const int jatom = glb_task.jatom;
743 task.zeta = glb_task.zeta;
744 task.zetb = glb_task.zetb;
745
746 for (int i = 0; i < 3; i++) {
747 task.rab[i] = glb_task.rab[i];
748 task.ra[i] = glb_task.ra[i];
749 task.rb[i] = task.ra[i] + task.rab[i];
750 task.rp[i] =
751 task.ra[i] + task.rab[i] * task.zetb / (task.zeta + task.zetb);
752 }
753
754 task.prefactor = glb_task.prefactor;
755 task.off_diag_twice = glb_task.off_diag_twice;
756
757 // angular momentum range of basis set
758 const int la_max_basis = glb_task.la_max;
759 const int lb_max_basis = glb_task.lb_max;
760 const int la_min_basis = glb_task.la_min;
761 const int lb_min_basis = glb_task.lb_min;
762
763 // angular momentum range for the actual collocate/integrate opteration.
764 task.la_max = la_max_basis + dev.la_max_diff;
765 task.lb_max = lb_max_basis + dev.lb_max_diff;
766 task.la_min = max(la_min_basis + (int)dev.la_min_diff, 0);
767 task.lb_min = max(lb_min_basis + (int)dev.lb_min_diff, 0);
768 task.lp = task.la_max + task.lb_max;
769
770 // start of decontracted set, ie. pab and hab
771 task.first_coseta = (la_min_basis > 0) ? ncoset(la_min_basis - 1) : 0;
772 task.first_cosetb = (lb_min_basis > 0) ? ncoset(lb_min_basis - 1) : 0;
773
774 // size of decontracted set, ie. pab and hab
775 task.ncoseta = ncoset(la_max_basis);
776 task.ncosetb = ncoset(lb_max_basis);
777
778 // size of the cab matrix
779 task.n1 = ncoset(task.la_max);
780 task.n2 = ncoset(task.lb_max);
781
782 // size of entire spherical basis
783 task.nsgfa = glb_task.nsgfa; // ibasis.nsgf;
784 task.nsgfb = glb_task.nsgfb;
785
786 // size of spherical set
787 task.nsgf_seta = glb_task.nsgf_seta;
788 task.nsgf_setb = glb_task.nsgf_setb;
789
790 // strides of the sphi transformation matrices
791 task.maxcoa = glb_task.maxcoa;
792 task.maxcob = glb_task.maxcob;
793
794 // transformations from contracted spherical to primitive cartesian basis
795 task.sphia = &dev.sphi_dev[glb_task.ikind][glb_task.sgfa * task.maxcoa +
796 glb_task.ipgf * task.ncoseta];
797 task.sphib = &dev.sphi_dev[glb_task.jkind][glb_task.sgfb * task.maxcob +
798 glb_task.jpgf * task.ncosetb];
799
800 // Locate current matrix block within the buffer.
801 const int block_offset = dev.block_offsets[glb_task.block_num];
802 task.block_transposed = glb_task.block_transposed;
803 task.pab_block =
804 dev.buffers_dev.pab_block + block_offset + glb_task.subblock_offset;
805
806 if (dev.buffers_dev.hab_block != nullptr) {
807 task.hab_block =
808 dev.buffers_dev.hab_block + block_offset + glb_task.subblock_offset;
809 if (dev.buffers_dev.forces != nullptr) {
810 task.forces_a = &dev.buffers_dev.forces[3 * iatom];
811 task.forces_b = &dev.buffers_dev.forces[3 * jatom];
812 }
813 }
814 }
816}
817
818/*******************************************************************************
819 * \brief Size (in doubles) of the work arrays of the two step spherical <->
820 * cartesian transformations of calculate_coefficients_two_steps and
821 * compute_hab_two_steps: [max_nsgf_set][ld] and [ld][ld] with
822 * ld = ncoset(lmax).
823 ******************************************************************************/
824inline int sphi_work_size(const int lmax, const int max_nsgf_set) {
825 const int ld = ncoset(lmax);
826 return max_nsgf_set * ld + ld * ld;
827}
828
829// shared memory budget of the two step kernels (default limit without opting
830// in on NVIDIA hardware), static and dynamic. Above, the fused kernels are
831// used.
832constexpr size_t sphi_two_step_max_smem = 48 * 1024;
833
834/*******************************************************************************
835 * \brief Largest dynamic shared memory the kernel can be launched with when
836 * its static and dynamic shared memory together must fit in budget
837 * bytes. The static part (e.g. __shared__ task structures) is taken
838 * from the compiled kernel and rounded up to 16 bytes for the
839 * alignment of the dynamic array placed after it.
840 ******************************************************************************/
841template <typename Kernel>
842inline size_t max_dynamic_smem(Kernel kernel, const size_t budget) {
843#if defined(__OFFLOAD_HIP)
844 hipFuncAttributes attr;
845 OFFLOAD_CHECK(
846 hipFuncGetAttributes(&attr, reinterpret_cast<const void *>(kernel)));
847#else
848 cudaFuncAttributes attr;
849 OFFLOAD_CHECK(cudaFuncGetAttributes(&attr, kernel));
850#endif
851 const size_t static_smem = (attr.sharedSizeBytes + 15) & ~size_t(15);
852 if (static_smem >= budget)
853 return 0;
854 return std::min(budget - static_smem,
855 (size_t)std::max(attr.maxDynamicSharedSizeBytes, 0));
856}
857
858/*******************************************************************************
859 * \brief Aborts with the location if the last kernel launch failed, e.g.
860 * because of an invalid launch configuration.
861 ******************************************************************************/
862#define GRID_GPU_CHECK_LAUNCH() OFFLOAD_CHECK(offloadGetLastError())
863
865private:
866 int la_max_{-1};
867 int lb_max_{-1};
868 int cab_size_{-1};
869 int lp_max_{-1};
870 ldiffs_value ldiffs_;
871 int lp_diff_{-1};
872
873public:
874 smem_parameters(const ldiffs_value ldiffs, const int lmax) {
875 ldiffs_ = ldiffs;
876 lp_diff_ = ldiffs.la_max_diff + ldiffs.lb_max_diff;
877 la_max_ = lmax + ldiffs.la_max_diff;
878 lb_max_ = lmax + ldiffs.lb_max_diff;
879 lp_max_ = la_max_ + lb_max_;
880
881 if (lp_max_ > grid_gpu_max_l) {
882 fprintf(stderr,
883 "ERROR: the GPU grid backend supports la + lb <= %d, this "
884 "operation needs %d (lmax %d, la_max_diff %d, lb_max_diff %d).\n",
885 grid_gpu_max_l, lp_max_, lmax, ldiffs.la_max_diff,
886 ldiffs.lb_max_diff);
887 abort();
888 }
889
890 // NB: cab is allocated in global memory not shared memory. Each block has
891 // its own cab space
892
893 cab_size_ =
895 }
896
898
899 // copy and move are trivial
900
901 inline int cab_size() const { return cab_size_; }
902
903 inline int lp_diff() const { return lp_diff_; }
904
905 inline ldiffs_value ldiffs() const { return ldiffs_; }
906
907 inline int lp_max() const { return lp_max_; }
908};
909template <typename T>
910__inline__ __device__ T *allocate_workspace(const kernel_params &dev_) {
911 unsigned int offset = dev_.cab_block_offset_dev[block_index()];
912 return (T *)(dev_.buffers_dev.cab + offset);
913}
914
915template <typename T, typename T3>
916__device__ __inline__ T
918 int &ymin, int &ymax) {
919 T kremain = 0.0;
920 ymin = (2 * (z + task.lb_cube.z) - 1) / 2;
921 ymin *= ymin;
922 kremain = task.discrete_radius * task.discrete_radius -
923 ((T)ymin) * task.norm_lattice_vector_z_2;
924 ymin = ceil(-1.0e-8 -
925 sqrt(fmax(0.0, kremain)) * task.norm_inverse_lattice_vector_y);
926 ymax = 1 - ymin - task.lb_cube.y;
927 ymin = ymin - task.lb_cube.y;
928 return kremain;
929}
930
931template <typename T, typename T3>
932__device__ __inline__ void
934 const T kremain, int &xmin, int &xmax) {
935 xmin = (2 * (y + task.lb_cube.y) - 1) / 2;
936 xmin *= xmin;
937 xmin = ceil(-1.0e-8 -
938 sqrt(fmax(0.0, kremain - xmin * task.norm_lattice_vector_y_2)) *
939 task.norm_inverse_lattice_vector_x);
940 xmax = 1 - xmin - task.lb_cube.x;
941 xmin -= task.lb_cube.x;
942}
943
944} // namespace grid_gpu_backend
945#endif
smem_parameters(const ldiffs_value ldiffs, const int lmax)
static void const int const int i
#define idx2(a, i, j)
fill_smem_task_reduced(dev_, dev_.first_task+block_index(), task)
constexpr size_t sphi_two_step_max_smem
int sphi_work_size(const int lmax, const int max_nsgf_set)
Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calcu...
__device__ __inline__ void prep_term(const orbital a, const orbital b, const T value, const int n, T *cab)
Adds given value to matrix element cab[idx(b)][idx(a)].
__inline__ __host__ __device__ double3 compute_coordinates(const double *__restrict__ dh_, const double x, const double y, const double z)
const task_info & glb_task
static void init_constant_memory()
Initializes the device's constant memory.
__inline__ __device__ unsigned int block_index()
__global__ const int ld
__device__ __inline__ void setup_task_cube_center(const kernel_params &dev_, smem_task_reduced< T, T3 > &task)
Shifts task.cube_center into the local grid's coordinate system and, for distributed grids,...
__device__ __inline__ T calculate_ymix_ymax_boundaries(smem_task_reduced< T, T3 > &task, const int z, int &ymin, int &ymax)
__device__ __inline__ int line_index(const int dir, const int m, const int p, const int q, const bool on_a, const int other, const int n1)
__device__ __inline__ void fill_smem_task_coef(const kernel_params &dev, const int task_id, smem_task< T > &task)
Copies a task from global to shared memory and does precomputations.
__device__ __inline__ double power(const double x, const int expo)
__inline__ __device__ unsigned int thread_global_index()
__device__ __inline__ orbital up(const int i, const orbital &a)
Increase i'th component of given orbital angular momentum.
__inline__ __device__ int idx(const orbital a)
Return coset index of given orbital angular momentum.
__device__ __inline__ void cab_to_cxyz_factorized(const smem_task< T > &task, T *__restrict__ cab, T *__restrict__ cxyz)
Transforms the coefficients C_ab into C_xyz, cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a] ...
__inline__ T compute_cube_properties(const T radius, const T *const __restrict__ dh_, const T *const __restrict__ dh_inv_, const T3 &rp, T3 &roffset, int3 &cubecenter, int3 &lb_cube, int3 &cube_size)
__inline__ __device__ int wrap_grid_index(const int idx, const int full_size)
Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry share...
__constant__ orbital coset_inv[1330]
size_t max_dynamic_smem(Kernel kernel, const size_t budget)
Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memo...
__host__ __device__ __inline__ int coset(int lx, int ly, int lz)
Maps three angular momentum components to a single zero based index.
__global__ const int const int max_nsgf_set
__device__ __inline__ void move_center(const int la, const int lb, const int n1, const int dir, const bool on_a, const T dr, T *cab)
Moves the expansion center of the a (on_a) or b cartesian index of cab along direction dir,...
__device__ __inline__ void decode_pair(int t, int &p, int &q)
Factorized cab <-> cxyz transformations.
__inline__ __device__ T * allocate_workspace(const kernel_params &dev_)
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
__device__ __inline__ void cxyz_to_cab_factorized(const smem_task< T > &task, const T *__restrict__ cxyz, T *__restrict__ cab)
Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.
__device__ __inline__ void calculate_xmin_xmax_boundaries(smem_task_reduced< T, T3 > &task, const int y, const T kremain, int &xmin, int &xmax)
__inline__ __device__ orbital down(const int i, const orbital &a)
Decrease i'th component of given orbital angular momentum.
__shared__ smem_task_reduced< double, double3 > task
__inline__ __device__ void compute_window_size(const int3 grid_size, const int border_mask, const int3 &border_width, int3 &window_size, int3 &window_shift)
Parameters of the collocate kernel.
Differences in angular momentum.
Orbital angular momentum.
data needed for collocate and integrate kernels
data needed for calculating the coefficients, forces, and stress