(git:27c966b)
Loading...
Searching...
No Matches
grid_gpu_integrate.cu
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7
8/*
9 * inspirations from the gpu backend
10 * Authors :
11 - Mathieu Taillefumier (ETH Zurich / CSCS)
12 - Advanced Micro Devices, Inc.
13 - Ole Schuett
14*/
15
16#include <algorithm>
17#include <assert.h>
18#include <limits.h>
19#include <math.h>
20#include <stdio.h>
21#include <stdlib.h>
22#include <string.h>
23
24#include "grid_gpu_context.h"
27#include "grid_gpu_separable.h"
28
29#if defined(_OMP_H)
30#error "OpenMP should not be used in .cu files to accommodate HIP."
31#endif
32
33namespace grid_gpu_backend {
34
35// tasks with lp >= this value go to integrate_separable_kernel, the others to
36// integrate_kernel. The standalone driver shows the separable kernel 20-37%
37// faster than integrate_kernel for lp = 0..3 as well (V100), so all tasks go
38// to it and integrate_kernel is only launched as a fallback.
40
41__inline__ __device__ double sep_row_reduce(double val) {
42 for (int offset = GRID_SEP_INTEGRATE_ROW_LANES / 2; offset > 0;
43 offset >>= 1) {
44#if defined(__CUDACC__)
45 val +=
46 __shfl_down_sync(0xffffffff, val, offset, GRID_SEP_INTEGRATE_ROW_LANES);
47#else
48 val += __shfl_down(val, offset, GRID_SEP_INTEGRATE_ROW_LANES);
49#endif
50 }
51 return val;
52}
53
54// do a block reduction for a block of size 64 and return the final sum to
55// thread_id = 0. table is 64 elements of the dynamic shared memory, see
56// block_reduce_64_smem.
57constexpr size_t block_reduce_64_smem = 64 * sizeof(double);
58
59template <typename T>
60__device__ __inline__ T block_reduce_64(T *table, const T val_, const int tid) {
61 // AMD GPU have warp size of 64 while nvidia GPUs have warpSize of 32 so the
62 // first step is common to both platforms.
63 table[tid] = val_;
65
66 T val = 0.0;
67
68 if (tid < 32) {
69 val = table[tid] + table[tid + 32];
70
71 for (int offset = 16; offset > 0; offset >>= 1) {
72#if defined(__CUDACC__)
73 val += __shfl_down_sync(0xffffffff, val, offset);
74#else
75 val += __shfl_down(val, offset);
76#endif
77 }
78 }
79
80 // prevents threads >= 32 (which never touch table[]) from racing ahead and
81 // overwriting it for the next call before the reduction above has read it.
83 return val;
84}
85
86template <typename T, bool COMPUTE_TAU>
87__global__ __launch_bounds__(64) void compute_hab(const kernel_params dev_) {
88 const int number_of_tasks = dev_.num_tasks_per_block_dev[block_index()];
89
90 if (number_of_tasks == 0)
91 return;
92
93 const int tid = thread_global_index();
94 const int offset = dev_.sorted_blocks_offset_dev[block_index()];
95
96 T *__restrict__ smem_cab = nullptr;
97
99
100 for (int tk = 0; tk < number_of_tasks; tk++) {
101 __shared__ smem_task<T> task;
102 const int task_id = dev_.task_sorted_by_blocks_dev[offset + tk];
103 if (dev_.tasks[task_id].skip_task)
104 continue;
105
106 // all warps need to be synchronized here before modifying the task
107 // information.
110
111 T *__restrict__ coef_ = reinterpret_cast<T *>(__builtin_assume_aligned(
112 &dev_.buffers_dev.coef[dev_.tasks[task_id].coef_offset], 32));
113
117
118 for (int i = tid / 8; i < task.nsgf_setb; i += 8) {
119 for (int j = tid % 8; j < task.nsgf_seta; j += 8) {
120 T tmp = 0.0;
121 for (int jco = task.first_cosetb; jco < task.ncosetb; jco++) {
122 const T sphib = task.sphib[i * task.maxcob + jco];
123 const auto &b = coset_inv[jco];
124 for (int ico = task.first_coseta; ico < task.ncoseta; ico++) {
125 const auto &a = coset_inv[ico];
126 const T hab = get_hab<COMPUTE_TAU, T>(a, b, task.zeta, task.zetb,
127 task.n1, smem_cab);
128 const T sphia_times_sphib =
129 task.sphia[j * task.maxcoa + ico] * sphib;
130 tmp += hab * sphia_times_sphib;
131 }
132 }
133 if (task.block_transposed) {
134 task.hab_block[j * task.nsgfb + i] += tmp;
135 } else {
136 task.hab_block[i * task.nsgfa + j] += tmp;
137 }
138 }
139 }
140 }
141}
142
143/*******************************************************************************
144 * \brief Same as compute_hab, with the cartesian to spherical transformation
145 * done in two steps,
146 *
147 * H[jco][ico] = get_hab(a, b, cab) (ncob ncoa)
148 * U[i][ico] = sum_jco sphib[i][jco] H[jco][ico] (nsgfb ncob ncoa)
149 * hab[i][j] += sum_ico U[i][ico] sphia[j][ico] (nsgfb nsgfa ncoa)
150 *
151 * instead of evaluating get_hab and both sphi factors in the innermost
152 * loop of a nsgfb nsgfa ncob ncoa loop. The work arrays H [ld][ld] and
153 * U [max_nsgf_set][ld] live in the dynamic shared memory, see
154 * sphi_work_size.
155 ******************************************************************************/
156template <typename T, bool COMPUTE_TAU>
157__global__ __launch_bounds__(64) void compute_hab_two_steps(
158 const kernel_params dev_, const int ld, const int max_nsgf_set) {
159 extern __shared__ T shared_memory[];
160 const int number_of_tasks = dev_.num_tasks_per_block_dev[block_index()];
161
162 if (number_of_tasks == 0)
163 return;
164
165 T *smem_h = &shared_memory[0]; // [ld][ld], H[jco][ico]
166 T *smem_u = smem_h + ld * ld; // [max_nsgf_set][ld]
167 const int tid = thread_global_index();
168 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
169 const int offset = dev_.sorted_blocks_offset_dev[block_index()];
170
171 T *__restrict__ smem_cab = allocate_workspace<T>(dev_);
172
173 for (int tk = 0; tk < number_of_tasks; tk++) {
174 __shared__ smem_task<T> task;
175 const int task_id = dev_.task_sorted_by_blocks_dev[offset + tk];
176 if (dev_.tasks[task_id].skip_task)
177 continue;
178
181
182 T *__restrict__ coef_ = reinterpret_cast<T *>(__builtin_assume_aligned(
183 &dev_.buffers_dev.coef[dev_.tasks[task_id].coef_offset], 32));
184
188
189 const int nco_a = task.ncoseta - task.first_coseta;
190 const int nco_b = task.ncosetb - task.first_cosetb;
191
192 // H[jco][ico] = get_hab(a, b)
193 for (int k = tid; k < nco_b * nco_a; k += nthreads) {
194 const int jj = k / nco_a;
195 const int jco = task.first_cosetb + jj;
196 const int ico = task.first_coseta + (k - jj * nco_a);
197 smem_h[jco * ld + ico] =
199 task.zetb, task.n1, smem_cab);
200 }
202
203 // U[i][ico] = sum_jco sphib[i][jco] H[jco][ico]
204 for (int k = tid; k < task.nsgf_setb * nco_a; k += nthreads) {
205 const int i = k / nco_a;
206 const int ico = task.first_coseta + (k - i * nco_a);
207 T val = 0.0;
208 for (int jco = task.first_cosetb; jco < task.ncosetb; jco++)
209 val += task.sphib[i * task.maxcob + jco] * smem_h[jco * ld + ico];
210 smem_u[i * ld + ico] = val;
211 }
213
214 // hab[i][j] += sum_ico U[i][ico] sphia[j][ico]
215 for (int k = tid; k < task.nsgf_setb * task.nsgf_seta; k += nthreads) {
216 const int i = k / task.nsgf_seta;
217 const int j = k - i * task.nsgf_seta;
218 T val = 0.0;
219 for (int ico = task.first_coseta; ico < task.ncoseta; ico++)
220 val += smem_u[i * ld + ico] * task.sphia[j * task.maxcoa + ico];
221 if (task.block_transposed) {
222 task.hab_block[j * task.nsgfb + i] += val;
223 } else {
224 task.hab_block[i * task.nsgfa + j] += val;
225 }
226 }
227 }
228}
229
230/*******************************************************************************
231 * \brief Adds the contribution of the cartesian pair (a, b), weighted by the
232 * cartesian density p, to the forces and the virial.
233 ******************************************************************************/
234template <typename T, bool COMPUTE_TAU>
235__device__ __inline__ void
237 const orbital &b, const T p, const bool compute_virial,
238 const T *__restrict__ cab, T *fa, T *fb, T *virial) {
239 for (int d = 0; d < 3; d++) {
240 fa[d] += p * get_force_a<COMPUTE_TAU, T>(a, b, d, task.zeta, task.zetb,
241 task.n1, cab);
242 fb[d] += p * get_force_b<COMPUTE_TAU, T>(a, b, d, task.zeta, task.zetb,
243 task.rab, task.n1, cab);
244 }
245 if (compute_virial) {
246 for (int d1 = 0; d1 < 3; d1++)
247 for (int d2 = 0; d2 < 3; d2++)
248 virial[3 * d1 + d2] +=
249 p *
250 (get_virial_a<COMPUTE_TAU, T>(a, b, d1, d2, task.zeta, task.zetb,
251 task.n1, cab) +
252 get_virial_b<COMPUTE_TAU, T>(a, b, d1, d2, task.zeta, task.zetb,
253 task.rab, task.n1, cab));
254 }
255}
256
257/*******************************************************************************
258 * \brief Fallback of compute_hab_forces_two_steps when its work arrays do not
259 * fit in the shared memory. hab is computed as in compute_hab. The
260 * forces and the virial only depend on the cartesian pair (a, b), so
261 * they are evaluated once per pair with the cartesian density
262 *
263 * p[jco][ico] = sum_i sphib[i][jco] sum_j pab[i][j] sphia[j][ico]
264 *
265 * computed on the fly (nsgfb (nsgfa + 1) FMA per pair), instead of
266 * once per (i, j, a, b) in the hab loop.
267 ******************************************************************************/
268template <typename T, typename T3, bool COMPUTE_TAU>
269__global__
270__launch_bounds__(64) void compute_hab_forces(const kernel_params dev_) {
271 // only used by the final block reductions, block_reduce_64_smem bytes
272 extern __shared__ T shared_memory[];
273 const int number_of_tasks = dev_.num_tasks_per_block_dev[block_index()];
274
275 if (number_of_tasks == 0)
276 return;
277
278 const int tid = thread_global_index();
279 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
280 const int offset = dev_.sorted_blocks_offset_dev[block_index()];
281 const bool compute_virial = (dev_.buffers_dev.virial != nullptr);
282
283 T *__restrict__ smem_cab = allocate_workspace<T>(dev_);
284
285 T fa[3] = {0.0, 0.0, 0.0}, fb[3] = {0.0, 0.0, 0.0};
286 T virial[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
287
288 for (int tk = 0; tk < number_of_tasks; tk++) {
289 __shared__ smem_task<T> task;
290 const int task_id = dev_.task_sorted_by_blocks_dev[offset + tk];
291 if (dev_.tasks[task_id].skip_task)
292 continue;
294
295 T *__restrict__ coef_ =
296 &dev_.buffers_dev.coef[dev_.tasks[task_id].coef_offset];
300
301 // hab[i][j] += sum_{jco, ico} sphib[i][jco] get_hab(a, b) sphia[j][ico]
302 for (int i = tid / 8; i < task.nsgf_setb; i += 8) {
303 for (int j = tid % 8; j < task.nsgf_seta; j += 8) {
304 T tmp = 0.0;
305 for (int jco = task.first_cosetb; jco < task.ncosetb; jco++) {
306 const T sphib = task.sphib[i * task.maxcob + jco];
307 const auto &b = coset_inv[jco];
308 for (int ico = task.first_coseta; ico < task.ncoseta; ico++) {
309 const auto &a = coset_inv[ico];
310 const T hab = get_hab<COMPUTE_TAU, T>(a, b, task.zeta, task.zetb,
311 task.n1, smem_cab);
312 tmp += hab * task.sphia[j * task.maxcoa + ico] * sphib;
313 }
314 }
315 if (task.block_transposed) {
316 task.hab_block[j * task.nsgfb + i] += tmp;
317 } else {
318 task.hab_block[i * task.nsgfa + j] += tmp;
319 }
320 }
321 }
322
323 // forces and virial, once per cartesian pair
324 const int nco_a = task.ncoseta - task.first_coseta;
325 const int nco_b = task.ncosetb - task.first_cosetb;
326 for (int k = tid; k < nco_b * nco_a; k += nthreads) {
327 const int jj = k / nco_a;
328 const int jco = task.first_cosetb + jj;
329 const int ico = task.first_coseta + (k - jj * nco_a);
330
331 T p = 0.0;
332 for (int i = 0; i < task.nsgf_setb; i++) {
333 T w = 0.0;
334 for (int j = 0; j < task.nsgf_seta; j++) {
335 const T block_value = task.block_transposed
336 ? task.pab_block[j * task.nsgfb + i]
337 : task.pab_block[i * task.nsgfa + j];
338 w += block_value * task.sphia[j * task.maxcoa + ico];
339 }
340 p += task.sphib[i * task.maxcob + jco] * w;
341 }
343 task, coset_inv[ico], coset_inv[jco], p * task.off_diag_twice,
345 }
347 }
348
349 // theoretically not needed
351
352 const int task_id = dev_.task_sorted_by_blocks_dev[offset];
353 const auto &glb_task = dev_.tasks[task_id];
354 const int iatom = glb_task.iatom;
355 const int jatom = glb_task.jatom;
356 T *forces_a = &dev_.buffers_dev.forces[3 * iatom];
357 T *forces_b = &dev_.buffers_dev.forces[3 * jatom];
358
359 T *sum = (T *)shared_memory;
360 if (dev_.buffers_dev.virial != nullptr) {
361
362 for (int i = 0; i < 9; i++) {
364
365 if (tid == 0)
366 atomicAdd(dev_.buffers_dev.virial + i, virial[i]);
367 }
368 }
369
370 for (int i = 0; i < 3; i++) {
372
373 if (tid == 0)
374 atomicAdd(forces_a + i, fa[i]);
375
377
378 if (tid == 0)
379 atomicAdd(forces_b + i, fb[i]);
380 }
381}
382
383/*******************************************************************************
384 * \brief Same as compute_hab_forces with the spherical to cartesian
385 * transformations done as matrix products. The forces and the virial
386 * only depend on the cartesian pair (a, b), so they are contracted with
387 * the cartesian density
388 *
389 * P[jco][ico] = sum_ij sphib[i][jco] pab[i][j] sphia[j][ico]
390 *
391 * and evaluated once per pair instead of once per (i, j, a, b):
392 *
393 * W[i][ico] = sum_j pab[i][j] sphia[j][ico] (nsgfb nsgfa ncoa)
394 * for each (jco, ico):
395 * p = sum_i sphib[i][jco] W[i][ico] (nsgfb ncob ncoa)
396 * fa, fb, virial += p * get_force / get_virial (a, b)
397 * H[jco][ico] = get_hab(a, b)
398 * U[i][ico] = sum_jco sphib[i][jco] H[jco][ico] (nsgfb ncob ncoa)
399 * hab[i][j] += sum_ico U[i][ico] sphia[j][ico] (nsgfb nsgfa ncoa)
400 *
401 * P is never stored. The work arrays H [ld][ld] and W / U
402 * [max_nsgf_set][ld] (U overwrites W) live in the dynamic shared
403 * memory, see sphi_work_size.
404 ******************************************************************************/
405template <typename T, bool COMPUTE_TAU>
406__global__ __launch_bounds__(64) void compute_hab_forces_two_steps(
407 const kernel_params dev_, const int ld, const int max_nsgf_set) {
408 extern __shared__ T shared_memory[];
409 const int number_of_tasks = dev_.num_tasks_per_block_dev[block_index()];
410
411 if (number_of_tasks == 0)
412 return;
413
414 T *smem_h = &shared_memory[0]; // [ld][ld], H[jco][ico]
415 T *smem_w = smem_h + ld * ld; // [max_nsgf_set][ld], W then U
416 const int tid = thread_global_index();
417 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
418 const int offset = dev_.sorted_blocks_offset_dev[block_index()];
419 const bool compute_virial = (dev_.buffers_dev.virial != nullptr);
420
421 T *__restrict__ smem_cab = allocate_workspace<T>(dev_);
422
423 T fa[3] = {0.0, 0.0, 0.0}, fb[3] = {0.0, 0.0, 0.0};
424 T virial[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
425
426 for (int tk = 0; tk < number_of_tasks; tk++) {
427 __shared__ smem_task<T> task;
428 const int task_id = dev_.task_sorted_by_blocks_dev[offset + tk];
429 if (dev_.tasks[task_id].skip_task)
430 continue;
431
432 // the previous task may still be reading task, W / U and H
435
436 T *__restrict__ coef_ = reinterpret_cast<T *>(__builtin_assume_aligned(
437 &dev_.buffers_dev.coef[dev_.tasks[task_id].coef_offset], 32));
438
441
442 const int nco_a = task.ncoseta - task.first_coseta;
443 const int nco_b = task.ncosetb - task.first_cosetb;
444
445 // W[i][ico] = sum_j pab[i][j] sphia[j][ico]
446 for (int k = tid; k < task.nsgf_setb * nco_a; k += nthreads) {
447 const int i = k / nco_a;
448 const int ico = task.first_coseta + (k - i * nco_a);
449 T val = 0.0;
450 for (int j = 0; j < task.nsgf_seta; j++) {
451 const T block_value = task.block_transposed
452 ? task.pab_block[j * task.nsgfb + i]
453 : task.pab_block[i * task.nsgfa + j];
454 val += block_value * task.sphia[j * task.maxcoa + ico];
455 }
456 smem_w[i * ld + ico] = val * task.off_diag_twice;
457 }
458 __syncthreads(); // W and cab are complete
459
460 // one pass over the cartesian pairs: density, forces, virial and H
461 for (int k = tid; k < nco_b * nco_a; k += nthreads) {
462 const int jj = k / nco_a;
463 const int jco = task.first_cosetb + jj;
464 const int ico = task.first_coseta + (k - jj * nco_a);
465 const auto &a = coset_inv[ico];
466 const auto &b = coset_inv[jco];
467
468 T p = 0.0;
469 for (int i = 0; i < task.nsgf_setb; i++)
470 p += task.sphib[i * task.maxcob + jco] * smem_w[i * ld + ico];
471
473 smem_cab, fa, fb, virial);
474
475 smem_h[jco * ld + ico] = get_hab<COMPUTE_TAU, T>(
476 a, b, task.zeta, task.zetb, task.n1, smem_cab);
477 }
478 __syncthreads(); // H is complete, W is no longer read
479
480 // U[i][ico] = sum_jco sphib[i][jco] H[jco][ico], overwrites W
481 for (int k = tid; k < task.nsgf_setb * nco_a; k += nthreads) {
482 const int i = k / nco_a;
483 const int ico = task.first_coseta + (k - i * nco_a);
484 T val = 0.0;
485 for (int jco = task.first_cosetb; jco < task.ncosetb; jco++)
486 val += task.sphib[i * task.maxcob + jco] * smem_h[jco * ld + ico];
487 smem_w[i * ld + ico] = val;
488 }
490
491 // hab[i][j] += sum_ico U[i][ico] sphia[j][ico]
492 for (int k = tid; k < task.nsgf_setb * task.nsgf_seta; k += nthreads) {
493 const int i = k / task.nsgf_seta;
494 const int j = k - i * task.nsgf_seta;
495 T val = 0.0;
496 for (int ico = task.first_coseta; ico < task.ncoseta; ico++)
497 val += smem_w[i * ld + ico] * task.sphia[j * task.maxcoa + ico];
498 if (task.block_transposed) {
499 task.hab_block[j * task.nsgfb + i] += val;
500 } else {
501 task.hab_block[i * task.nsgfa + j] += val;
502 }
503 }
504 }
505
506 // all tasks of a block belong to the same atom pair
508
509 const int task_id = dev_.task_sorted_by_blocks_dev[offset];
510 const auto &glb_task = dev_.tasks[task_id];
511 T *forces_a = &dev_.buffers_dev.forces[3 * glb_task.iatom];
512 T *forces_b = &dev_.buffers_dev.forces[3 * glb_task.jatom];
513
514 T *sum = (T *)shared_memory;
516 for (int i = 0; i < 9; i++) {
518 if (tid == 0)
519 atomicAdd(dev_.buffers_dev.virial + i, virial[i]);
520 }
521 }
522
523 for (int i = 0; i < 3; i++) {
525 if (tid == 0)
526 atomicAdd(forces_a + i, fa[i]);
528 if (tid == 0)
529 atomicAdd(forces_b + i, fb[i]);
530 }
531}
532
533/*******************************************************************************
534 * Cuda kernel for calculating the coefficients of a potential (density,
535 etc...) for a given pair of gaussian basis sets
536
537We compute the discretized version of the following integral
538
539$$\int _\infty ^\infty V_{ijk} P^\alpha_iP^beta_j P ^ \gamma _ k Exp(- \eta
540|r_{ijk} - r_c|^2)$$
541
542where in practice the summation is over a finite domain. The discrete form has
543this shape
544
545$$
546\sum_{ijk < domain} V_{ijk} (x_i - x_c)^\alpha (y_j - y_c)^\beta (z_k - z_c) ^
547\gamma Exp(- \eta |r_{ijk} - r_c|^2)
548$$
549
550where $0 \le \alpha + \beta + \gamma \le lmax$
551
552It is formely the same operation than collocate (from a discrete point of view)
553but the implementation differ because of technical reasons.
554
555So most of the code is the same except the core of the routine that is
556specialized to the integration.
557
558******************************************************************************/
559template <typename T, typename T3, bool distributed__, bool orthogonal_>
560__global__
561__launch_bounds__(64) void integrate_kernel(const kernel_params dev_) {
562 const int lbatch = 20;
563 const task_info &glb_task = dev_.tasks[dev_.first_task + block_index()];
564 if (glb_task.skip_task)
565 return;
566
567 // handled by integrate_separable_kernel
568 if (glb_task.la_max + dev_.la_max_diff + glb_task.lb_max + dev_.lb_max_diff >=
569 dev_.separable_lp_min)
570 return;
571
572 const int tid = thread_global_index();
573
574 __shared__ T dh_[9];
575
576 if (tid < 9)
577 dh_[tid] = dev_.dh_[tid];
578
580 fill_smem_task_reduced(dev_, dev_.first_task + block_index(), task);
581
582 if (tid == 0) {
584 }
586
587 __shared__ T accumulator[20][64];
588
589 // we use a multi pass algorithm here because shared memory usage (or
590 // register) would become too high for high angular momentum
591
592 const short int size_loop =
593 (ncoset(task.lp) / lbatch + ((ncoset(task.lp) % lbatch) != 0)) * lbatch;
594 const short int length = ncoset(task.lp);
595 for (int ico = 0; ico < size_loop; ico += lbatch) {
596#pragma unroll lbatch
597 for (int i = 0; i < lbatch; i++)
598 accumulator[i][tid] = 0.0;
599
601
602 for (int z = threadIdx.z; z < task.cube_size.z; z += blockDim.z) {
603 int z2 = wrap_grid_index(z + task.cube_center.z, dev_.grid_full_size_.z);
604
605 if (distributed__) {
606 // known at compile time. Will be stripped away
607 if (task.apply_border_mask) {
608 /* check if the point is within the window */
609 if ((z2 < task.window_shift.z) || (z2 > task.window_size.z)) {
610 continue;
611 }
612 }
613 }
614
615 /* compute the coordinates of the point in atomic coordinates */
616 int ymin = 0;
617 int ymax = task.cube_size.y - 1;
618 T kremain = 0.0;
619
620 if (orthogonal_ && !task.apply_border_mask) {
621 kremain = calculate_ymix_ymax_boundaries(task, z, ymin, ymax);
622 }
623
624 for (int y = ymin + threadIdx.y; y <= ymax; y += blockDim.y) {
625 int y2 =
626 wrap_grid_index(y + task.cube_center.y, dev_.grid_full_size_.y);
627
628 if (distributed__) {
629 /* check if the point is within the window */
630 if (task.apply_border_mask) {
631 if ((y2 < task.window_shift.y) || (y2 > task.window_size.y)) {
632 continue;
633 }
634 }
635 }
636
637 int xmin = 0;
638 int xmax = task.cube_size.x - 1;
639
640 if (orthogonal_ && !task.apply_border_mask) {
641 calculate_xmin_xmax_boundaries<T, T3>(task, y, kremain, xmin, xmax);
642 }
643
644 for (int x = xmin + threadIdx.x; x <= xmax; x += blockDim.x) {
645 int x2 =
646 wrap_grid_index(x + task.cube_center.x, dev_.grid_full_size_.x);
647
648 if (distributed__) {
649 /* check if the point is within the window */
650 if (task.apply_border_mask) {
651 if ((x2 < task.window_shift.x) || (x2 > task.window_size.x)) {
652 continue;
653 }
654 }
655 }
656
657 // I make no distinction between orthogonal and non orthogonal
658 // cases
659 T3 r3;
660
661 r3 = compute_coordinates(dh_, (x + task.lb_cube.x + task.roffset.x),
662 (y + task.lb_cube.y + task.roffset.y),
663 (z + task.lb_cube.z + task.roffset.z));
664 // check if the point is inside the sphere or not. Note that it does
665 // not apply for the orthogonal case when the full sphere is inside
666 // the region of interest.
667 const T r3x2 = r3.x * r3.x;
668 const T r3y2 = r3.y * r3.y;
669 const T r3z2 = r3.z * r3.z;
670
671 if (distributed__) {
672 // check if the point is inside the sphere or not. Note that it does
673 // not apply for the orthorhombic case when the full sphere is
674 // inside the region of interest.
675
676 if (((task.radius * task.radius) <= (r3x2 + r3y2 + r3z2)) &&
677 (!orthogonal_ || task.apply_border_mask))
678 continue;
679 } else {
680 // we do not need to do this test for the orthorhombic case
681 if ((!orthogonal_) &&
682 ((task.radius * task.radius) <= (r3x2 + r3y2 + r3z2)))
683 continue;
684 }
685
686 // read the next point assuming that reading is non blocking until
687 // the register is actually needed for computation. This is true on
688 // NVIDIA hardware
689
690 const int grid_index =
691 (z2 * dev_.grid_local_size_.y + y2) * dev_.grid_local_size_.x +
692 x2;
693 T grid_value = __ldg(&dev_.buffers_dev.grid[grid_index]);
694
695 const T r3xy = r3.x * r3.y;
696 const T r3xz = r3.x * r3.z;
697 const T r3yz = r3.y * r3.z;
698
699 grid_value *= exp(-(r3x2 + r3y2 + r3z2) * task.zetp);
700
701 switch (ico / lbatch) {
702 case 0: {
703 accumulator[0][tid] += grid_value;
704
705 if (task.lp >= 1) {
706 accumulator[1][tid] += grid_value * r3.x;
707 accumulator[2][tid] += grid_value * r3.y;
708 accumulator[3][tid] += grid_value * r3.z;
709 }
710
711 if (task.lp >= 2) {
712 accumulator[4][tid] += grid_value * r3x2;
713 accumulator[5][tid] += grid_value * r3xy;
714 accumulator[6][tid] += grid_value * r3xz;
715 accumulator[7][tid] += grid_value * r3y2;
716 accumulator[8][tid] += grid_value * r3yz;
717 accumulator[9][tid] += grid_value * r3z2;
718 }
719 if (task.lp >= 3) {
720 T tmp = grid_value * r3x2;
721 accumulator[10][tid] += tmp * r3.x;
722 accumulator[11][tid] += tmp * r3.y;
723 accumulator[12][tid] += tmp * r3.z;
724 tmp = grid_value * r3.x;
725 accumulator[13][tid] += tmp * r3y2;
726 accumulator[14][tid] += tmp * r3yz;
727 accumulator[15][tid] += tmp * r3z2;
728 tmp = grid_value * r3y2;
729 accumulator[16][tid] += tmp * r3.y;
730 accumulator[17][tid] += tmp * r3.z;
731 tmp = grid_value * r3z2;
732 accumulator[18][tid] += tmp * r3.y;
733 accumulator[19][tid] += tmp * r3.z;
734 }
735 } break;
736 case 1: {
737 if (task.lp >= 4) {
738 T tmp = grid_value * r3x2;
739 accumulator[0][tid] += tmp * r3x2;
740 accumulator[1][tid] += tmp * r3xy;
741 accumulator[2][tid] += tmp * r3xz;
742 accumulator[3][tid] += tmp * r3y2;
743 accumulator[4][tid] += tmp * r3yz;
744 accumulator[5][tid] += tmp * r3z2;
745 tmp = grid_value * r3y2;
746 accumulator[6][tid] += tmp * r3xy;
747 accumulator[7][tid] += tmp * r3xz;
748 tmp = grid_value * r3z2;
749 accumulator[8][tid] += tmp * r3xy;
750 accumulator[9][tid] += tmp * r3xz;
751 }
752 if (task.lp >= 4) {
753 T tmp = grid_value * r3y2;
754 accumulator[10][tid] += tmp * r3y2;
755 accumulator[11][tid] += tmp * r3yz;
756 accumulator[12][tid] += tmp * r3z2;
757 accumulator[13][tid] += grid_value * r3yz * r3z2;
758 accumulator[14][tid] += grid_value * r3z2 * r3z2;
759 }
760 if (task.lp >= 5) {
761 T tmp = grid_value * r3x2 * r3x2;
762 accumulator[15][tid] += tmp * r3.x;
763 accumulator[16][tid] += tmp * r3.y;
764 accumulator[17][tid] += tmp * r3.z;
765 tmp = grid_value * r3x2;
766 accumulator[18][tid] += tmp * r3.x * r3y2;
767 accumulator[19][tid] += tmp * r3xy * r3.z;
768 }
769 } break;
770 case 2: {
771 T tmp = grid_value * r3x2;
772 accumulator[0][tid] += tmp * r3.x * r3z2;
773 accumulator[1][tid] += tmp * r3y2 * r3.y;
774 accumulator[2][tid] += tmp * r3y2 * r3.z;
775 accumulator[3][tid] += tmp * r3.y * r3z2;
776 accumulator[4][tid] += tmp * r3z2 * r3.z;
777 tmp = grid_value * r3.x * r3y2;
778 accumulator[5][tid] += tmp * r3y2;
779 accumulator[6][tid] += tmp * r3yz;
780 accumulator[7][tid] += tmp * r3z2;
781 tmp = grid_value * r3.x * r3z2;
782 accumulator[8][tid] += tmp * r3yz;
783 accumulator[9][tid] += tmp * r3z2;
784 tmp = grid_value * r3y2 * r3.y;
785 accumulator[10][tid] += tmp * r3y2;
786 accumulator[11][tid] += tmp * r3yz;
787 accumulator[12][tid] += tmp * r3z2;
788 accumulator[13][tid] += grid_value * r3y2 * r3z2 * r3.z;
789 accumulator[14][tid] += grid_value * r3.y * r3z2 * r3z2;
790 accumulator[15][tid] += grid_value * r3z2 * r3z2 * r3.z;
791 if (task.lp >= 6) {
792 tmp = grid_value * r3x2 * r3x2;
793 accumulator[16][tid] += tmp * r3x2; // x^6
794 accumulator[17][tid] += tmp * r3xy; // x^5 y
795 accumulator[18][tid] += tmp * r3xz; // x^5 z
796 accumulator[19][tid] += tmp * r3y2; // x^4 y^2
797 }
798 } break;
799 case 3: {
800 T tmp = grid_value * r3x2;
801 accumulator[0][tid] += tmp * r3x2 * r3yz; // x^4 y z
802 accumulator[1][tid] += tmp * r3x2 * r3z2; // x^4 z^2
803 accumulator[2][tid] += tmp * r3y2 * r3xy; // x^3 y^3
804 accumulator[3][tid] += tmp * r3y2 * r3xz; // x^3 y^2 z
805 accumulator[4][tid] += tmp * r3xy * r3z2; // x^3 y z^2
806 accumulator[5][tid] += tmp * r3z2 * r3xz; // x^3 z^3
807 accumulator[6][tid] += tmp * r3y2 * r3y2; // x^2 y^4
808 accumulator[7][tid] += tmp * r3y2 * r3yz; // x^3 y^2 z
809 accumulator[8][tid] += tmp * r3y2 * r3z2; // x^2 y^2 z^2
810 accumulator[9][tid] += tmp * r3z2 * r3yz; // x^2 y z^3
811 accumulator[10][tid] += tmp * r3z2 * r3z2; // x^2 z^4
812 tmp = grid_value * r3y2 * r3y2;
813 accumulator[11][tid] += tmp * r3xy; // x y^5
814 accumulator[12][tid] += tmp * r3xz; // x y^4 z
815 accumulator[13][tid] +=
816 grid_value * r3y2 * r3xy * r3z2; // x y^3 z^2
817 accumulator[14][tid] +=
818 grid_value * r3y2 * r3z2 * r3xz; // x y^2 z^3
819 accumulator[15][tid] += grid_value * r3xy * r3z2 * r3z2; // x y z^4
820 accumulator[16][tid] += grid_value * r3z2 * r3z2 * r3xz; // x z^5
821 accumulator[17][tid] += tmp * r3y2; // y^6
822 accumulator[18][tid] += tmp * r3yz; // y^5 z
823 accumulator[19][tid] += tmp * r3z2; // y^4 z^2
824 } break;
825 default:
826 for (int ic = 0; (ic < lbatch) && ((ic + ico) < length); ic++) {
827 auto &co = coset_inv[ic + ico];
828 T tmp = 1.0;
829 for (int po = 0; po < (co.l[2] >> 1); po++)
830 tmp *= r3z2;
831 if (co.l[2] & 0x1)
832 tmp *= r3.z;
833 for (int po = 0; po < (co.l[1] >> 1); po++)
834 tmp *= r3y2;
835 if (co.l[1] & 0x1)
836 tmp *= r3.y;
837 for (int po = 0; po < (co.l[0] >> 1); po++)
838 tmp *= r3x2;
839 if (co.l[0] & 0x1)
840 tmp *= r3.x;
841 accumulator[ic][tid] += tmp * grid_value;
842 }
843 break;
844 }
845 }
846 }
847 }
848
849 const int max_i = min(length - ico, lbatch);
850
852
853 // we know there is only 1 wavefront in each block lbatch threads could
854 // reduce the values saved by all threads of the warp and save results do a
855 // shuffle_down by hand
856
857 if (tid < 32) {
858 for (int i = 0; i < max_i; i++) {
859 // Load local value
860 T val = accumulator[i][tid] + accumulator[i][tid + 32];
861
862 // Warp-level reduction (32 lanes) After this loop, lane 0 of warp 0
863 // holds the result All threads should execute this operation so
864 // __activemask() is incorrect here.
865
866 // The ifdef statement is needed because hip does not necessarily
867 // support the instruction either. Reverting back to __shfl_down is
868 // required.
869 //
870 for (int offset = 16; offset > 0; offset >>= 1) {
871#if defined(__CUDACC__)
872 val += __shfl_down_sync(0xffffffff, val, offset);
873#else
874 val += __shfl_down(val, offset);
875#endif
876 }
877
878 if (tid == 0)
879 accumulator[i][0] = val;
880 }
881#if defined(__CUDACC__)
882 __syncwarp();
883#endif
884 }
885
886#if !defined(__CUDACC__)
888#endif
889
890 if (tid < min(length - ico, lbatch)) {
891 const size_t coef_offset =
892 dev_.tasks[dev_.first_task + block_index()].coef_offset;
893 dev_.buffers_dev.coef[coef_offset + tid + ico] = accumulator[tid][0];
894 }
896 }
897}
898
899/*******************************************************************************
900 * Separable (plane streaming) integrate kernel.
901 *
902 * Computes the same coefficients as integrate_kernel, but with a cost per grid
903 * point linear in lp instead of proportional to ncoset(lp), and reading the
904 * cube only once whatever lp is.
905 *
906 * It uses the same underlying method than the DGEMM backend which is a tensor
907 * contraction. Speedup comes from different areas:
908 *
909 * Axis aligned cell (axis_aligned_ = true): r3.x, r3.y, r3.z only depend on
910 * x, y and z respectively, so that
911 *
912 * x^a y^b z^c exp(-zetp r^2) = [x^a e^{-zetp x^2}] [y^b e^{-zetp y^2}]
913 * [z^c e^{-zetp z^2}]
914 *
915 * Non orthogonal cell (axis_aligned_ = false): the moments are accumulated in
916 * lattice monomials u^i v^j w^k (u, v, w are the lattice coordinates of the
917 * point relative to the gaussian center) with the full gaussian evaluated per
918 * point, and transformed to cartesian monomials at the end with the matrix
919 * built by build_lattice_to_cartesian_matrix. It is the same transformation
920 * as grid_transform_coef_jik_to_yxz in the dgemm backend, precomputed once per
921 * grid level.
922 *
923 * For each z plane of the cube:
924 *
925 * A[y][a] = sum_x V[z][y][x] px[a][x] (step A)
926 * B[a][b] = sum_y A[y][a] py[b][y] a + b <= lp (step B)
927 * acc[coset(a, b, c)] += B[a][b] pz[c] c <= lp - a - b (step C)
928 *
929 * B only lives in registers and the reduction over z is a running sum in acc,
930 * so the shared memory does not depend on the z extent of the cube. The set of
931 * grid points visited is the same as for integrate_kernel (window masks, y / x
932 * trimming in the orthogonal case, sphere test).
933 *
934 * Shared memory layout: mc = max cube size, ls = lp_max + 1:
935 * ex[mc], py[ls][mc], pz[ls], A[mc][ls], acc[ncoset(lp_max)]
936 * pz only holds the values of the current plane, ex is only used for an axis
937 * aligned cell.
938 ******************************************************************************/
939
940template <bool distributed__, bool axis_aligned_, int lmax_>
941__global__ __launch_bounds__(64) void integrate_separable_kernel(
942 const kernel_params dev_, const int mc, const int lp_max,
943 const double *__restrict__ lattice_to_cartesian) {
944 // an axis aligned cell is orthogonal, otherwise the cell is treated as non
945 // orthogonal (see integrate_one_grid_level).
946 constexpr bool orthogonal_ = axis_aligned_;
947
948 // the per thread accumulators have lmax_ + 1 entries, lmax_ >= lp_max (see
949 // separable_lmax_bucket)
950 static_assert(lmax_ <= GRID_SEP_MAX_L, "lmax_ larger than GRID_SEP_MAX_L");
951
952 const task_info &glb_task = dev_.tasks[dev_.first_task + block_index()];
953 if (glb_task.skip_task)
954 return;
955
956 // handled by integrate_kernel
957 if (glb_task.la_max + dev_.la_max_diff + glb_task.lb_max + dev_.lb_max_diff <
958 dev_.separable_lp_min)
959 return;
960
961 const int tid = thread_global_index();
962 const int ls = lp_max + 1;
963
964 extern __shared__ double sep_smem[];
965 // exponetials along the x direction (note that it is the lattice direction
966 // not necessarily the cartesian coordinate)
967 double *ex = sep_smem;
968 // y coordinate and its powers (note that it is the lattice direction not
969 // necessarily the cartesian coordinate)
970 double *py = ex + mc;
971 // z coordinate and its powers common to all threads and single point (note
972 // that it is the lattice direction not necessarily the cartesian coordinate)
973 double *pz = py + ls * mc;
974 double *A = pz + ls;
975 double *acc = A + mc * ls;
976
978 fill_smem_task_reduced(dev_, dev_.first_task + block_index(), task);
979 if (tid == 0) {
981 }
983
984 const int lp = task.lp;
985 const int ncoef = ncoset(lp);
986 const double zetp = task.zetp;
987 const double r2max = task.radius * task.radius;
988 const bool trim = orthogonal_ && !task.apply_border_mask;
989
990 // 1D tables. Gaussian along x for an axis aligned cell (the x coordinates
991 // are computed on the fly) and powers of y in py.
992 for (int x = tid; x < task.cube_size.x; x += 64) {
993 const double u = x + task.lb_cube.x + task.roffset.x;
994 if (axis_aligned_) {
995 const double r = u * dev_.dh_[0];
996 ex[x] = exp(-zetp * r * r);
997 }
998 }
999
1000 // p[beta][y]
1001 for (int y = tid; y < task.cube_size.y; y += 64) {
1002 const double v = y + task.lb_cube.y + task.roffset.y;
1003 const double r = axis_aligned_ ? v * dev_.dh_[4] : v;
1004 double p = axis_aligned_ ? exp(-zetp * r * r) : 1.0;
1005 for (int b = 0; b <= lp; b++, p *= r)
1006 py[b * mc + y] = p;
1007 }
1008 for (int i = tid; i < ncoef; i += 64)
1009 acc[i] = 0.0;
1010 __syncthreads();
1011
1012 const int npairs = (lp + 1) * (lp + 2) / 2;
1015 constexpr int nrow_groups = 64 / GRID_SEP_INTEGRATE_ROW_LANES;
1016
1017 for (int z = 0; z < task.cube_size.z; z++) {
1018 // the plane decisions below are uniform over the block
1019 const int z2 =
1020 wrap_grid_index(z + task.cube_center.z, dev_.grid_full_size_.z);
1021 if (distributed__ && task.apply_border_mask &&
1022 ((z2 < task.window_shift.z) || (z2 > task.window_size.z)))
1023 continue;
1024
1025 int ymin = 0, ymax = task.cube_size.y - 1;
1026 double kremain = 0.0;
1027 if (trim)
1028 kremain = calculate_ymix_ymax_boundaries(task, z, ymin, ymax);
1029 if (ymin > ymax)
1030 continue;
1031
1032 const double w = z + task.lb_cube.z + task.roffset.z;
1033 const double rz = axis_aligned_ ? w * dev_.dh_[8] : w;
1034
1035 // pz for this plane. Made visible to step C by the barrier after step A,
1036 // and only overwritten after the barrier that ends step C.
1037
1038 // Calculate 1, z, z^2, .., z^lp recursively
1039
1040 if (tid == 0) {
1041 double p = axis_aligned_ ? exp(-zetp * rz * rz) : 1.0;
1042 for (int c = 0; c <= lp; c++, p *= rz)
1043 pz[c] = p;
1044 }
1045
1046 // step A: one group of GRID_SEP_INTEGRATE_ROW_LANES lanes per row y. The
1047 // trip count of the y0 loop is uniform over the block so that every lane
1048 // takes part in the shuffle reduction, even when its row is past ymax.
1049 for (int y0 = ymin; y0 <= ymax; y0 += nrow_groups) {
1050 const int y = y0 + row_group;
1051 const bool row_valid = (y <= ymax);
1052 double part[lmax_ + 1];
1053#pragma unroll
1054 for (int a = 0; a <= lmax_; a++)
1055 part[a] = 0.0;
1056
1057 const int y2 =
1058 wrap_grid_index(y + task.cube_center.y, dev_.grid_full_size_.y);
1059 const bool row_masked =
1060 distributed__ && task.apply_border_mask &&
1061 ((y2 < task.window_shift.y) || (y2 > task.window_size.y));
1062
1063 if (row_valid && !row_masked) {
1064 int xmin = 0, xmax = task.cube_size.x - 1;
1065 if (trim)
1067 xmin, xmax);
1068 const double v = y + task.lb_cube.y + task.roffset.y;
1069 const double ry = v * dev_.dh_[4];
1070 const double r2yz = ry * ry + rz * rz; // only used when axis aligned
1071 const double *__restrict__ row =
1072 &dev_.buffers_dev.grid[(z2 * dev_.grid_local_size_.y + y2) *
1073 dev_.grid_local_size_.x];
1074
1075 // periodic index of x, updated incrementally
1076 const int nx_full = dev_.grid_full_size_.x;
1077 int x2 = wrap_grid_index(xmin + lane + task.cube_center.x, nx_full);
1078 for (int x = xmin + lane; x <= xmax; x += GRID_SEP_INTEGRATE_ROW_LANES,
1080 x2, nx_full)) {
1081 if (distributed__ && task.apply_border_mask &&
1082 ((x2 < task.window_shift.x) || (x2 > task.window_size.x)))
1083 continue;
1084
1085 const double u = x + task.lb_cube.x + task.roffset.x;
1086 const double r = axis_aligned_ ? u * dev_.dh_[0] : u;
1087 double r2;
1088 if (axis_aligned_) {
1089 r2 = r * r + r2yz;
1090 } else {
1091 const double3 r3 = compute_coordinates(dev_.dh_, r, v, w);
1092 r2 = r3.x * r3.x + r3.y * r3.y + r3.z * r3.z;
1093 }
1094
1095 // same sphere test as integrate_kernel
1096 const bool outside = (r2max <= r2);
1097 if (distributed__) {
1098 if (outside && (!orthogonal_ || task.apply_border_mask))
1099 continue;
1100 } else if (!orthogonal_ && outside) {
1101 continue;
1102 }
1103
1104 // Calculate x_i^\alpha \exp (-x_i^2) V_{ijk}
1105
1106 double p = __ldg(&row[x2]);
1107 p *= axis_aligned_ ? ex[x] : exp(-zetp * r2);
1108#pragma unroll
1109 for (int a = 0; a <= lmax_; a++) {
1110 if (a <= lp) {
1111 part[a] += p;
1112 p *= r;
1113 }
1114 }
1115 }
1116 }
1117
1118 // now calculate B_{j k \alpha} = \sum_{i} V_{ijk} x_i ^ \alpha \exp(
1119 // -x_i^2)
1120#pragma unroll
1121 for (int a = 0; a <= lmax_; a++) {
1122 if (a <= lp) {
1123 const double val = sep_row_reduce(part[a]);
1124 if (row_valid && lane == 0)
1125 A[(y - ymin) * ls + a] = val;
1126 }
1127 }
1128 }
1129 __syncthreads();
1130
1131 // steps B and C: each thread owns the (a, b, *) coefficients of its pairs,
1132 // so no atomics are needed.
1133
1134 // finally calculate
1135 //
1136 // C_{\alpha\beta\gamma} = \sum_{jk} B_{jk\alpha} p^y_{j\beta}
1137 // p^z_{k,\gamma}
1138 //
1139 // with \alpha+\beta+\gamma<=l. The pair list contains the \alpha,\beta
1140 // pairs that are required. $\gamma$ can be derived from the constraint over
1141 // angular momentum.
1142
1143 for (int ip = tid; ip < npairs; ip += 64) {
1144 const int a = sep_pair_ab[ip].x;
1145 const int b = sep_pair_ab[ip].y;
1146 double bab = 0.0;
1147 for (int y = ymin; y <= ymax; y++)
1148 bab += A[(y - ymin) * ls + a] * py[b * mc + y];
1149 for (int c = 0; c <= lp - a - b; c++)
1150 acc[coset(a, b, c)] += bab * pz[c];
1151 }
1152 __syncthreads();
1153 }
1154
1155 // finally store the results directly when the lattice is orthorhombic or
1156 // rotate the coefficients when the lattice is generic.
1157 //
1158 // the dgemm backend uses the same trick.
1159 //
1160
1161 double *coef = &dev_.buffers_dev.coef[glb_task.coef_offset];
1162 if (axis_aligned_) {
1163 for (int i = tid; i < ncoef; i += 64)
1164 coef[i] = acc[i];
1165 } else {
1166 // lattice -> cartesian monomials, one block per total angular momentum
1167 for (int i = tid; i < ncoef; i += 64) {
1168 const orbital &co = coset_inv[i];
1169 const int l = co.l[0] + co.l[1] + co.l[2];
1170 const int base = (l > 0) ? ncoset(l - 1) : 0;
1171 const int nl = ncoset(l) - base;
1172 const double *__restrict__ m =
1173 lattice_to_cartesian + sep_m_offset[l] + (i - base) * nl;
1174 double val = 0.0;
1175 for (int j = 0; j < nl; j++)
1176 val += m[j] * acc[base + j];
1177 coef[i] = val;
1178 }
1179 }
1180}
1181
1182static size_t integrate_separable_smem_size(const int mc, const int lp_max) {
1183 const int ls = lp_max + 1;
1184 return sizeof(double) * (mc + ls * mc + ls + mc * ls + ncoset(lp_max));
1185}
1186
1187using integrate_separable_kernel_t = void (*)(const kernel_params, const int,
1188 const int, const double *);
1189
1190/*******************************************************************************
1191 * \brief Selects the instance of integrate_separable_kernel for one value of
1192 * lmax_.
1193 ******************************************************************************/
1194template <int lmax_>
1195static integrate_separable_kernel_t
1196select_integrate_separable(const bool distributed, const bool axis_aligned) {
1197 if (distributed)
1198 return axis_aligned ? integrate_separable_kernel<true, true, lmax_>
1199 : integrate_separable_kernel<true, false, lmax_>;
1200 return axis_aligned ? integrate_separable_kernel<false, true, lmax_>
1201 : integrate_separable_kernel<false, false, lmax_>;
1202}
1203
1204/*******************************************************************************
1205 * \brief Selects the instance of integrate_separable_kernel with the smallest
1206 * bucket of the angular momentum covering lp_max.
1207 ******************************************************************************/
1208static integrate_separable_kernel_t
1209select_integrate_separable(const int lp_max, const bool distributed,
1210 const bool axis_aligned) {
1211 switch (separable_lmax_bucket(lp_max)) {
1212 case 4:
1213 return select_integrate_separable<4>(distributed, axis_aligned);
1214 case 8:
1215 return select_integrate_separable<8>(distributed, axis_aligned);
1216 case 12:
1217 return select_integrate_separable<12>(distributed, axis_aligned);
1218 default:
1220 axis_aligned);
1221 }
1222}
1223
1224/*******************************************************************************
1225 * \brief Launches the Cuda kernel that integrates all tasks of one grid level.
1226 ******************************************************************************/
1227void context_info::integrate_one_grid_level(const int level, int *lp_diff) {
1228 if (number_of_tasks_per_level_[level] == 0)
1229 return;
1231
1232 // Compute max angular momentum.
1233 const ldiffs_value ldiffs =
1235
1236 smem_parameters smem_params(ldiffs, lmax());
1237
1238 *lp_diff = smem_params.lp_diff();
1240
1241 kernel_params params = set_kernel_parameters(level, smem_params);
1242
1243 // Tasks with lp >= integrate_separable_lp_threshold go to the separable
1244 // kernel when it applies:
1245 // - axis aligned cells (cartesian factorization) and non orthogonal cells
1246 // (lattice coordinates + transformation). Rotated orthogonal cells keep
1247 // integrate_kernel since their cube is set up by the orthogonal code path.
1248 // - the angular momentum fits the tables and the shared memory the budget.
1249 auto &grid = grid_[level];
1250 const int lp_max = smem_params.lp_max();
1251 const int max_cube_size = grid.max_cube_size();
1252 const bool axis_aligned = grid.is_axis_aligned();
1253
1254 // fall back to the old integrate if shared memory is not large enough or
1255 // anything else happens
1256 const integrate_separable_kernel_t separable_kernel =
1257 select_integrate_separable(lp_max, grid.is_distributed(), axis_aligned);
1258 const bool use_separable =
1260 (lp_max <= GRID_SEP_MAX_L) && (max_cube_size > 0) &&
1261 (axis_aligned || !grid.is_orthogonal()) &&
1262 (integrate_separable_smem_size(max_cube_size, lp_max) <=
1263 max_dynamic_smem(separable_kernel, separable_max_smem));
1264
1265 if (use_separable)
1267
1268 /* WARNING : if you change the block size please be aware of that the number
1269 * of warps is hardcoded when we do the block reduction in the integrate
1270 * kernel. The number of warps should be explicitly indicated in the
1271 * templating parameters or simply adjust the switch statements inside the
1272 * integrate kernels */
1273
1274 // integrate_kernel handles the tasks the separable kernel does not take
1275 const bool use_integrate_kernel =
1276 !use_separable || (integrate_separable_lp_threshold > 0);
1277 const dim3 threads_per_block(4, 4, 4);
1278 if (!use_integrate_kernel) {
1279 // nothing to do
1280 } else if (grid_[level].is_distributed()) {
1281 if (grid_[level].is_orthogonal()) {
1282 integrate_kernel<double, double3, true, true>
1283 <<<number_of_tasks_per_level_[level], threads_per_block, 0,
1284 level_streams[level]>>>(params);
1285 } else {
1286 integrate_kernel<double, double3, true, false>
1287 <<<number_of_tasks_per_level_[level], threads_per_block, 0,
1288 level_streams[level]>>>(params);
1289 }
1290 } else {
1291 if (grid_[level].is_orthogonal()) {
1292 integrate_kernel<double, double3, false, true>
1293 <<<number_of_tasks_per_level_[level], threads_per_block, 0,
1294 level_streams[level]>>>(params);
1295 } else {
1296 integrate_kernel<double, double3, false, false>
1297 <<<number_of_tasks_per_level_[level], threads_per_block, 0,
1298 level_streams[level]>>>(params);
1299 }
1300 }
1301
1302 if (use_integrate_kernel)
1304
1305 if (!use_separable)
1306 return;
1307
1308 // both kernels write the coefficients of disjoint sets of tasks.
1310 const double *lattice_to_cartesian =
1311 axis_aligned ? nullptr : get_lattice_to_cartesian(grid, lp_max);
1312 const size_t smem = integrate_separable_smem_size(max_cube_size, lp_max);
1313 const int nblocks_level = number_of_tasks_per_level_[level];
1314 separable_kernel<<<nblocks_level, 64, smem, level_streams[level]>>>(
1315 params, max_cube_size, lp_max, lattice_to_cartesian);
1317}
1318
1320
1322
1323 // Compute max angular momentum.
1324 const ldiffs_value ldiffs =
1326
1327 smem_parameters smem_params(ldiffs, lmax());
1330
1331 kernel_params params = set_kernel_parameters(-1, smem_params);
1332
1333 /* WARNING if you change the block size. The number
1334 * of warps is hardcoded when we do the block reduction in the integrate
1335 * kernel. */
1336
1337 const dim3 threads_per_block(4, 4, 4);
1338
1339 // two step sphi transformation when its work arrays fit in the shared memory
1340 // next to the static shared memory of the kernel
1341 const int ld = ncoset(lmax());
1342 // (the forces kernels reuse the first 64 doubles for the block reductions)
1343 const size_t smem_two_steps =
1344 std::max(sizeof(double) * sphi_work_size(lmax(), max_nsgf_set()),
1346
1347 const auto two_steps_kernel =
1349 ? (compute_tau ? compute_hab_forces_two_steps<double, true>
1350 : compute_hab_forces_two_steps<double, false>)
1351 : (compute_tau ? compute_hab_two_steps<double, true>
1352 : compute_hab_two_steps<double, false>);
1353 if (smem_two_steps <=
1354 max_dynamic_smem(two_steps_kernel, sphi_two_step_max_smem)) {
1355 two_steps_kernel<<<this->nblocks, threads_per_block, smem_two_steps,
1356 this->main_stream>>>(params, ld, max_nsgf_set());
1358 return;
1359 }
1360
1361 // fused fallback, the forces kernels need block_reduce_64_smem
1362 const auto fused_kernel =
1364 ? (compute_tau ? compute_hab_forces<double, double3, true>
1365 : compute_hab_forces<double, double3, false>)
1366 : (compute_tau ? compute_hab<double, true>
1367 : compute_hab<double, false>);
1368 const size_t smem_fused = calculate_forces ? block_reduce_64_smem : 0;
1369 fused_kernel<<<this->nblocks, threads_per_block, smem_fused,
1370 this->main_stream>>>(params);
1372}
1373}; // namespace grid_gpu_backend
std::vector< offloadStream_t > level_streams
std::vector< int > number_of_tasks_per_level_
void integrate_one_grid_level(const int level, int *lp_diff)
Launches the Cuda kernel that integrates all tasks of one grid level.
void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff)
Lays out the cab workspace of the coefficient kernels for the angular momentum offsets of the current...
std::vector< grid_info< double > > grid_
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i
#define GRID_GPU_CHECK_LAUNCH()
Aborts with the location if the last kernel launch failed, e.g. because of an invalid launch configur...
#define GRID_SEP_MAX_L
#define GRID_SEP_INTEGRATE_ROW_LANES
real(dp), dimension(3) w
real(dp), dimension(3) c
real(dp), dimension(3) a
real(dp), dimension(3) b
real(dp), dimension(3) p
fill_smem_task_reduced(dev_, dev_.first_task+block_index(), task)
__device__ __inline__ T block_reduce_64(T *table, const T val_, const int tid)
__device__ __inline__ int sep_advance_wrapped_index(const int x2, const int n)
int separable_lmax_bucket(const int lp_max)
__device__ __inline__ T get_hab(const orbital &a, const orbital &b, const T zeta, const T zetb, const int n, const T *cab)
Returns element i,j of hab matrix.
constexpr size_t sphi_two_step_max_smem
int sphi_work_size(const int lmax, const int max_nsgf_set)
Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calcu...
static integrate_separable_kernel_t select_integrate_separable(const bool distributed, const bool axis_aligned)
Selects the instance of integrate_separable_kernel for one value of lmax_.
__inline__ __host__ __device__ double3 compute_coordinates(const double *__restrict__ dh_, const double x, const double y, const double z)
const task_info & glb_task
static void init_constant_memory()
Initializes the device's constant memory.
__device__ __inline__ T get_force_b(const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const T rab[3], const int n, const T *cab)
Returns i'th component of force on atom b.
__inline__ __device__ unsigned int block_index()
__global__ const int ld
__device__ __inline__ void setup_task_cube_center(const kernel_params &dev_, smem_task_reduced< T, T3 > &task)
Shifts task.cube_center into the local grid's coordinate system and, for distributed grids,...
__device__ __inline__ T calculate_ymix_ymax_boundaries(smem_task_reduced< T, T3 > &task, const int z, int &ymin, int &ymax)
__shared__ double sep_smem[]
__device__ __inline__ void fill_smem_task_coef(const kernel_params &dev, const int task_id, smem_task< T > &task)
Copies a task from global to shared memory and does precomputations.
constexpr int nrow_groups
__inline__ __device__ unsigned int thread_global_index()
constexpr size_t block_reduce_64_smem
constexpr int integrate_separable_lp_threshold
ldiffs_value process_get_ldiffs(bool calculate_forces, bool calculate_virial, bool compute_tau)
Returns difference in angular momentum range for given flags.
__device__ __inline__ T get_virial_a(const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const int n, const T *cab)
Returns element i,j of virial on atom a.
__device__ __inline__ void accumulate_pair_forces(const smem_task< T > &task, const orbital &a, const orbital &b, const T p, const bool compute_virial, const T *__restrict__ cab, T *fa, T *fb, T *virial)
Adds the contribution of the cartesian pair (a, b), weighted by the cartesian density p,...
static void init_separable_constant_memory()
Initializes the constant memory of integrate_separable_kernel.
__inline__ __device__ int wrap_grid_index(const int idx, const int full_size)
Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry share...
__constant__ orbital coset_inv[1330]
size_t max_dynamic_smem(Kernel kernel, const size_t budget)
Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memo...
__host__ __device__ __inline__ int coset(int lx, int ly, int lz)
Maps three angular momentum components to a single zero based index.
__global__ const int const int max_nsgf_set
static const double * get_lattice_to_cartesian(grid_info< double > &grid, const int lmax)
Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the latt...
__global__ const int mc
__global__ __launch_bounds__(64) void calculate_coefficients(const kernel_params dev_)
Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done i...
__inline__ __device__ T * allocate_workspace(const kernel_params &dev_)
__device__ __inline__ double get_force_a(const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const int n, const T *cab)
Returns i'th component of force on atom a.
if(number_of_tasks==0) return
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
__constant__ int sep_m_offset[GRID_SEP_MAX_L+1]
__device__ __inline__ void cxyz_to_cab_factorized(const smem_task< T > &task, const T *__restrict__ cxyz, T *__restrict__ cab)
Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.
__inline__ __device__ double sep_row_reduce(double val)
__device__ __inline__ void calculate_xmin_xmax_boundaries(smem_task_reduced< T, T3 > &task, const int y, const T kremain, int &xmin, int &xmax)
__global__ const int const int const double *__restrict__ lattice_to_cartesian
__shared__ smem_task_reduced< double, double3 > task
const double *__restrict__ coef
__constant__ char2 sep_pair_ab[(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2]
__device__ __inline__ double get_virial_b(const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const T rab[3], const int n, const T *cab)
Returns element i,j of virial on atom b.
constexpr size_t separable_max_smem
__global__ const int const int lp_max
Parameters of the collocate kernel.
Differences in angular momentum.
Orbital angular momentum.
data needed for collocate and integrate kernels
data needed for calculating the coefficients, forces, and stress
Internal representation of a task.
Orbital angular momentum.