(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_acsf.cu
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
10
11#include <stdio.h>
12#include <stdlib.h>
13#include <string.h>
14
15#include "nnp_gpu_internal.h"
16#include "nnp_gpu_pack.h"
17#include "nnp_gpu_state.h"
18
19#if defined(_OMP_H)
20#error "OpenMP should not be used in .cu files to accommodate HIP."
21#endif
22
23/*
24 * Production symmetry-function (descriptor) pipeline for the NNP GPU backend:
25 * the device-resident path that runs on every MD step once the on-device
26 * cell-list walk (nnp_gpu_walk.cu) has filled the per-(atom, group, element)
27 * neighbour buffers.
28 *
29 * The descriptor G_dev is aggregated on device: nnp_gpu_begin_descriptor_step
30 * zeroes it, the radial and angular launchers atomicAdd each symmetry function
31 * into it, and nnp_gpu_scale_G_dev_c produces the scaled network input
32 * G_dev_scaled. G_dev is laid out as G[m + i_local * n_input_max] (descriptor
33 * row m fastest, rank-local atom i_local outermost); radial descriptors occupy
34 * the first n_rad(element) rows of an atom, angular descriptors follow them.
35 *
36 * Each launcher processes one (symmetry-function group, element) slot,
37 * identified by se_idx = s0 + max_grp * e0 (s0 the 0-based group index, e0 the
38 * 0-based element index, max_grp the per-kind group bound). The launchers read
39 * the walk output and the once-uploaded per-(group, element) tables, rearrange
40 * them into the per-call pbuf_* layouts on device, and evaluate the symmetry
41 * functions plus their force partials into the per-slot state buffers that the
42 * force-scatter kernels later consume.
43 *
44 * Compute precision follows nnp_gpu_precision_fp32(): the angular symmetry
45 * functions run in single precision under MIXED (the launcher instantiates the
46 * <float> kernel), everything else stays double precision. The four device
47 * behaviour knobs are compile-time constants here (see below), so there is no
48 * per-step host-to-device push of a runtime flag.
49 */
50
51/*******************************************************************************
52 * \brief Upload the rank-local 0-based element index of every atom once, then
53 * reuse it. The buffer is refreshed when the rank-local atom count or
54 * the contents change against an exact host shadow copy; the guard
55 * cannot use the host pointer because the Fortran callers reallocate
56 * their staging arrays every step. The count alone would not do: a
57 * composition change that preserves the atom count (an
58 * identity-exchange Monte Carlo move, a resorted subsystem) would then
59 * keep the stale map and route every atom to the wrong element's weights
60 * and scaling table, with no diagnostic. The copy is synchronous: it
61 * fires once per run (plus once per change), so stream placement is
62 * irrelevant. Shared by the scaling and network kernels. Returns 0 on
63 * success, -1 on failure.
64 * \author Dhruv Sharma (ds2173@cam.ac.uk)
65 ******************************************************************************/
66int nnp_gpu_step_ele_ind_ensure(nnp_gpu_state_t *state, const int *h_ele_ind,
67 int n_atoms) {
68 const size_t n_bytes = sizeof(int) * (size_t)n_atoms;
69 if (state->step_ele_ind_n == n_atoms &&
70 memcmp(state->step_ele_ind_host, h_ele_ind, n_bytes) == 0)
71 return 0;
72 if (state->step_ele_ind_host_cap < n_bytes) {
73 free(state->step_ele_ind_host);
74 state->step_ele_ind_host = (int *)malloc(n_bytes);
75 if (state->step_ele_ind_host == NULL) {
76 state->step_ele_ind_host_cap = 0;
77 state->step_ele_ind_n = 0;
78 fprintf(stderr, "ERROR: nnp_gpu_step_ele_ind_ensure: malloc failed\n");
79 return -1;
80 }
81 state->step_ele_ind_host_cap = n_bytes;
82 }
83 if (nnp_gpu_pbuf_ensure((void **)&state->step_ele_ind_dev,
84 &state->step_ele_ind_cap, n_bytes))
85 return -1;
86 memcpy(state->step_ele_ind_host, h_ele_ind, n_bytes);
87 offloadMemcpyHtoD(state->step_ele_ind_dev, h_ele_ind, n_bytes);
88 state->step_ele_ind_n = n_atoms;
89 return 0;
90}
91
92/*******************************************************************************
93 * \brief Device cutoff function fc(r) and its derivative dfc(r), mirroring the
94 * host routine nnp_fill_fc_dfc_cache in src/nnp_acsf.F. cut_type selects
95 * the cutoff shape (0 cosine, 1 tanh^3), matching the C-side convention
96 * cut_int = nnp%cut_type - 1. The operation order follows the host
97 * SELECT CASE branches exactly.
98 *
99 * Templated on the compute type CT so the angular kernel can evaluate
100 * the cutoff in single precision (MIXED) or double precision from one
101 * source. CT literals keep the arithmetic in CT; a bare double literal
102 * would promote the whole expression and negate the single-precision
103 * path. Callers passing double arguments deduce CT=double and are
104 * byte-identical to a plain double implementation. The cos/sin/tanh
105 * calls resolve to the CT overload.
106 * \author Dhruv Sharma (ds2173@cam.ac.uk)
107 ******************************************************************************/
108template <typename CT>
109static __device__ __forceinline__ void
110nnp_compute_fc_dfc(CT r, CT cutoff_s, int cut_type, CT *fc, CT *dfc) {
111 if (cut_type == 0) {
112 /* cosine: fc = 0.5 * (cos(pi*r/c) + 1); dfc = -0.5 * sin(pi*r/c) * (pi/c).
113 * Local pi literal (not M_PI) so the constant stays in CT and the device
114 * translation unit does not depend on M_PI being defined. */
115 const CT pi_over_c = (CT)3.141592653589793238462643383279502884 / cutoff_s;
116 const CT arg = pi_over_c * r;
117 *fc = (CT)0.5 * (cos(arg) + (CT)1.0);
118 *dfc = (CT)(-0.5) * sin(arg) * pi_over_c;
119 } else {
120 /* tanh: fc = th^3; dfc = (-3/c) * (th^2 - th^4)
121 * where th = tanh(1 - r/c). Match host order. */
122 const CT th = tanh((CT)1.0 - r / cutoff_s);
123 const CT th2 = th * th;
124 *fc = th2 * th;
125 *dfc = ((CT)(-3.0) / cutoff_s) * (th2 - th2 * th2);
126 }
127}
128
129/*******************************************************************************
130 * \brief Batched radial symmetry-function kernel with per-neighbour force
131 * partials, reading atomic positions and periodic-image translations
132 * from device-resident state instead of host-packed distance arrays.
133 * The Fortran caller ships only the per-(group, element) integer
134 * neighbour tables (rad_ind, rad_image_idx, atoms_global); the kernel
135 * reconstructs each displacement inline:
136 *
137 * i_glob = atoms_global[atom_idx]
138 * j_glob = rad_ind[r_off + j]
139 * img = rad_image_idx[r_off + j]
140 * image_pos = coord(j_glob) + image_table(img)
141 * rvect = coord(i_glob) - image_pos
142 * r = norm(rvect)
143 *
144 * which matches the host order in nnp_compute_neighbors_cell_list, so
145 * the doubles produced here are bit-identical to the host-packed path.
146 * The Hermite cubic spline value and radial derivative mirror the CPU
147 * routine nnp_calc_rad in src/nnp_acsf.F. spline_y and spline_dy are
148 * column-major [n_symf, spline_n] with sf the fastest index. The per-sf
149 * descriptor is atomicAdd'd into G_dev[symf_map[sf] + n_input_max *
150 * atom_map[atom_idx]]; the per-neighbour force partial is written into
151 * force_flat and the self term summed into self_partial_flat.
152 *
153 * Determinism: G_dev is zeroed by nnp_gpu_begin_descriptor_step before
154 * this launch, and within one launch each (sf, atom_idx) maps to a
155 * unique (descriptor row, atom slot) so there is no intra-launch
156 * contention on the atomicAdd. Each thread accumulates over neighbours
157 * in fixed order, so the result agrees with the host at the FP64 ULP
158 * level, subject only to host/device FMA-contraction differences.
159 * \author Dhruv Sharma (ds2173@cam.ac.uk)
160 ******************************************************************************/
161__global__ void nnp_radial_sf_force_kernel(
162 const int n_atoms, const int n_symf, const int max_n_neigh,
163 const int spline_n, const double spline_dx, const double spline_dx_inv,
164 const double spline_x_max, const double *__restrict__ spline_y,
165 const double *__restrict__ spline_dy,
166 const int *__restrict__ n_neigh_per_atom,
167 const double *__restrict__ coord_dev,
168 const double *__restrict__ image_table_dev, const int *__restrict__ rad_ind,
169 const int *__restrict__ rad_image_idx, const int *__restrict__ atoms_global,
170 double *__restrict__ self_partial_flat, double *__restrict__ force_flat,
171 double *__restrict__ G_dev, const int *__restrict__ symf_map,
172 const int *__restrict__ atom_map, const int n_input_max) {
173 const int sf = blockIdx.y * blockDim.x + threadIdx.x;
174 if (sf >= n_symf)
175 return;
176 const int atom_idx = blockIdx.x;
177 if (atom_idx >= n_atoms)
178 return;
179
180 const int n_neigh_a = n_neigh_per_atom[atom_idx];
181 if (n_neigh_a <= 0)
182 return;
183
184 const int i_glob = atoms_global[atom_idx];
185 const double cx = coord_dev[3 * i_glob + 0];
186 const double cy = coord_dev[3 * i_glob + 1];
187 const double cz = coord_dev[3 * i_glob + 2];
188
189 /* 64-bit per-atom base offsets, matching nnp_gpu_pack.h: the per-neighbour
190 * buffers exceed 2^31 entries on large ranks. */
191 const long long r_off = (long long)max_n_neigh * (long long)atom_idx;
192 const long long force_atom_off =
193 3LL * (long long)n_symf * (long long)max_n_neigh * (long long)atom_idx;
194
195 double Gacc = 0.0;
196 double sx = 0.0, sy = 0.0, sz = 0.0;
197
198 for (int j = 0; j < n_neigh_a; j++) {
199 const int j_glob = rad_ind[r_off + j];
200 const int img = rad_image_idx[r_off + j];
201
202 /* Match host order from nnp_compute_neighbors_cell_list:
203 * image_pos = coord_primary(:, j) + image_translation(:, img)
204 * dr = center(:) - image_pos(:)
205 * r = NORM2(dr)
206 */
207 const double image_pos_x =
208 coord_dev[3 * j_glob + 0] + image_table_dev[3 * img + 0];
209 const double image_pos_y =
210 coord_dev[3 * j_glob + 1] + image_table_dev[3 * img + 1];
211 const double image_pos_z =
212 coord_dev[3 * j_glob + 2] + image_table_dev[3 * img + 2];
213 const double rvect_x = cx - image_pos_x;
214 const double rvect_y = cy - image_pos_y;
215 const double rvect_z = cz - image_pos_z;
216 const double r =
217 sqrt(rvect_x * rvect_x + rvect_y * rvect_y + rvect_z * rvect_z);
218
219 const long long force_base = force_atom_off + 3 * sf + 3 * n_symf * j;
220
221 if (r >= spline_x_max) {
222 force_flat[force_base + 0] = 0.0;
223 force_flat[force_base + 1] = 0.0;
224 force_flat[force_base + 2] = 0.0;
225 continue;
226 }
227
228 int idx = (int)(r * spline_dx_inv);
229 if (idx < 0)
230 idx = 0;
231 if (idx > spline_n - 2)
232 idx = spline_n - 2;
233
234 const double t = (r - (double)idx * spline_dx) * spline_dx_inv;
235 const double t2 = t * t;
236 const double t3 = t2 * t;
237 const double h00 = 2.0 * t3 - 3.0 * t2 + 1.0;
238 const double h10 = t3 - 2.0 * t2 + t;
239 const double h01 = -2.0 * t3 + 3.0 * t2;
240 const double h11 = t3 - t2;
241 const double h10_dx = h10 * spline_dx;
242 const double h11_dx = h11 * spline_dx;
243 const double dh00 = 6.0 * (t2 - t);
244 const double dh10 = 3.0 * t2 - 4.0 * t + 1.0;
245 const double dh01 = -dh00;
246 const double dh11 = 3.0 * t2 - 2.0 * t;
247
248 const double yi = spline_y[sf + idx * n_symf];
249 const double yi1 = spline_y[sf + (idx + 1) * n_symf];
250 const double dyi = spline_dy[sf + idx * n_symf];
251 const double dyi1 = spline_dy[sf + (idx + 1) * n_symf];
252
253 const double sym = h00 * yi + h10_dx * dyi + h01 * yi1 + h11_dx * dyi1;
254 Gacc += sym;
255
256 const double r_inv = 1.0 / r;
257 const double dsymdr_full =
258 (dh00 * yi + dh01 * yi1) * spline_dx_inv + dh10 * dyi + dh11 * dyi1;
259 const double drdx_x = rvect_x * r_inv;
260 const double drdx_y = rvect_y * r_inv;
261 const double drdx_z = rvect_z * r_inv;
262
263 const double f_x = dsymdr_full * drdx_x;
264 const double f_y = dsymdr_full * drdx_y;
265 const double f_z = dsymdr_full * drdx_z;
266
267 force_flat[force_base + 0] = f_x;
268 force_flat[force_base + 1] = f_y;
269 force_flat[force_base + 2] = f_z;
270
271 sx += f_x;
272 sy += f_y;
273 sz += f_z;
274 }
275
276 /* atomicAdd contract: nnp_gpu_begin_descriptor_step zeroes G_dev before this
277 * launch, and within one launch each (sf, atom_idx) maps to a unique
278 * (m, i_local), so there is no intra-launch contention. */
279 const int g_row = symf_map[sf];
280 const int g_atom = atom_map[atom_idx];
281 atomicAdd(&G_dev[g_row + (long long)n_input_max * g_atom], Gacc);
282
283 const long long self_base = 3LL * sf + 3LL * n_symf * atom_idx;
284 self_partial_flat[self_base + 0] = sx;
285 self_partial_flat[self_base + 1] = sy;
286 self_partial_flat[self_base + 2] = sz;
287}
288
289/*******************************************************************************
290 * \brief Templated transcendental wrappers used by the angular triplet math.
291 * Both instantiations use the accurate libm functions of their own
292 * precision, so the single-precision path differs from the double one
293 * only by the width of the arithmetic and not by the function used.
294 * \author Dhruv Sharma (ds2173@cam.ac.uk)
295 ******************************************************************************/
296template <typename CT> __device__ __forceinline__ CT nnp_fast_exp(CT x) {
297 return exp(x);
298}
299template <> __device__ __forceinline__ float nnp_fast_exp<float>(float x) {
300 return expf(x);
301}
302template <typename CT>
303__device__ __forceinline__ void nnp_fast_sincos(CT x, CT &s, CT &c) {
304 s = sin(x);
305 c = cos(x);
306}
307template <>
308__device__ __forceinline__ void nnp_fast_sincos<float>(float x, float &s,
309 float &c) {
310 s = sinf(x);
311 c = cosf(x);
312}
313template <typename CT> __device__ __forceinline__ CT nnp_fast_tanh(CT x) {
314 return tanh(x);
315}
316template <> __device__ __forceinline__ float nnp_fast_tanh<float>(float x) {
317 return tanhf(x);
318}
319template <typename CT> __device__ __forceinline__ CT nnp_fast_pow(CT b, CT e) {
320 return pow(b, e);
321}
322template <>
323__device__ __forceinline__ float nnp_fast_pow<float>(float b, float e) {
324 return powf(b, e);
325}
326
327/*******************************************************************************
328 * \brief Angular symmetry-function inner triplet math, factored as a device
329 * inline so the two-phase angular kernel can call it once in the
330 * j-parallel phase (producing jj / G / self) and once in the k-parallel
331 * phase (producing kk) without duplicating the arithmetic. Mirrors the
332 * host per-triplet angular routine in src/nnp_acsf.F. The caller must
333 * already have verified r3_sqr < cutoff_sqr.
334 *
335 * Templated on the compute type CT: under MIXED the launcher
336 * instantiates it with CT=float and the whole evaluation (distances,
337 * cutoff, angular term, force partials) runs in single precision; with
338 * CT=double it is byte-identical to a plain double implementation.
339 * \author Dhruv Sharma (ds2173@cam.ac.uk)
340 ******************************************************************************/
341template <typename CT>
342__device__ __forceinline__ void nnp_angular_triplet_math(
343 const CT r1, const CT rv1_x, const CT rv1_y, const CT rv1_z, const CT fc1,
344 const CT dfc1, const CT r2, const CT rv2_x, const CT rv2_y, const CT rv2_z,
345 const CT fc2, const CT dfc2, const CT r3, const CT rvect3_x,
346 const CT rvect3_y, const CT rvect3_z, const CT r3_sqr, const CT cutoff_s,
347 const int cut_type, const CT lam_sf, const CT zeta_sf, const int izeta_sf,
348 const int use_int_zeta_sf, const CT eta_sf, const CT prefzeta_sf,
349 CT &sym_out, CT &fjj_x, CT &fjj_y, CT &fjj_z, CT &fkk_x, CT &fkk_y,
350 CT &fkk_z, CT &self_x_inc, CT &self_y_inc, CT &self_z_inc) {
351
352 const CT M_PI_DP = (CT)3.141592653589793238462643383279502884;
353
354 const CT rsqr1 = r1 * r1;
355 const CT rsqr2 = r2 * r2;
356 const CT r2sum = rsqr1 + rsqr2 + r3_sqr;
357 const CT f_il = r3_sqr - rsqr1 - rsqr2;
358 const CT g_il = (CT)(-2.0) * r1 * r2;
359 const CT costheta = f_il / g_il;
360
361 CT fcut3, dfcut3;
362 if (cut_type == 0) {
363 const CT arg = M_PI_DP * r3 / cutoff_s;
364 CT sarg, carg;
365 nnp_fast_sincos<CT>(arg, sarg, carg);
366 fcut3 = (CT)0.5 * (carg + (CT)1.0);
367 dfcut3 = (CT)(-0.5) * sarg * (M_PI_DP / cutoff_s);
368 } else {
369 const CT th = nnp_fast_tanh<CT>((CT)1.0 - r3 / cutoff_s);
370 fcut3 = th * th * th;
371 dfcut3 = ((CT)(-3.0) / cutoff_s) * (th * th - th * th * th * th);
372 }
373
374 const CT ftot = fc1 * fc2 * fcut3;
375 const CT dfcdr1 = dfc1 * fc2 * fcut3;
376 const CT dfcdr2 = fc1 * dfc2 * fcut3;
377 const CT dfcdr3 = fc1 * fc2 * dfcut3;
378
379 const CT r1_inv = (CT)1.0 / r1;
380 const CT r2_inv = (CT)1.0 / r2;
381 const CT r3_inv = (CT)1.0 / r3;
382 const CT dr1dx_x = rv1_x * r1_inv;
383 const CT dr1dx_y = rv1_y * r1_inv;
384 const CT dr1dx_z = rv1_z * r1_inv;
385 const CT dr2dx_x = rv2_x * r2_inv;
386 const CT dr2dx_y = rv2_y * r2_inv;
387 const CT dr2dx_z = rv2_z * r2_inv;
388 const CT dr3dx_x = rvect3_x * r3_inv;
389 const CT dr3dx_y = rvect3_y * r3_inv;
390 const CT dr3dx_z = rvect3_z * r3_inv;
391
392 const CT inv_g2 = (CT)1.0 / (g_il * g_il);
393 const CT dgdx_t1_x = (CT)2.0 * r2 * dr1dx_x;
394 const CT dgdx_t1_y = (CT)2.0 * r2 * dr1dx_y;
395 const CT dgdx_t1_z = (CT)2.0 * r2 * dr1dx_z;
396 const CT dgdx_t2_x = (CT)2.0 * r1 * dr2dx_x;
397 const CT dgdx_t2_y = (CT)2.0 * r1 * dr2dx_y;
398 const CT dgdx_t2_z = (CT)2.0 * r1 * dr2dx_z;
399 const CT dcb1_x =
400 (CT)(-2.0) * (rv1_x + rv2_x) * g_il - f_il * (-(dgdx_t1_x + dgdx_t2_x));
401 const CT dcb1_y =
402 (CT)(-2.0) * (rv1_y + rv2_y) * g_il - f_il * (-(dgdx_t1_y + dgdx_t2_y));
403 const CT dcb1_z =
404 (CT)(-2.0) * (rv1_z + rv2_z) * g_il - f_il * (-(dgdx_t1_z + dgdx_t2_z));
405 const CT dcb2_x = (CT)2.0 * (rvect3_x + rv1_x) * g_il - f_il * dgdx_t1_x;
406 const CT dcb2_y = (CT)2.0 * (rvect3_y + rv1_y) * g_il - f_il * dgdx_t1_y;
407 const CT dcb2_z = (CT)2.0 * (rvect3_z + rv1_z) * g_il - f_il * dgdx_t1_z;
408 const CT dcb3_x = (CT)2.0 * (rv2_x - rvect3_x) * g_il - f_il * dgdx_t2_x;
409 const CT dcb3_y = (CT)2.0 * (rv2_y - rvect3_y) * g_il - f_il * dgdx_t2_y;
410 const CT dcb3_z = (CT)2.0 * (rv2_z - rvect3_z) * g_il - f_il * dgdx_t2_z;
411
412 CT tmp = (CT)1.0 + lam_sf * costheta;
413 CT tmpzeta;
414 CT angular;
415 if (tmp <= (CT)0.0) {
416 tmpzeta = (CT)0.0;
417 angular = (CT)0.0;
418 } else {
419 if (use_int_zeta_sf) {
420 CT base = tmp;
421 int e = izeta_sf - 1;
422 CT acc = (CT)1.0;
423 if (e < 0) {
424 base = (CT)1.0 / tmp;
425 e = -e;
426 }
427 while (e > 0) {
428 if (e & 1)
429 acc *= base;
430 base *= base;
431 e >>= 1;
432 }
433 tmpzeta = acc;
434 } else {
435 tmpzeta = nnp_fast_pow<CT>(tmp, zeta_sf - (CT)1.0);
436 }
437 angular = tmpzeta * tmp;
438 }
439
440 const CT symtmp = nnp_fast_exp<CT>(-eta_sf * r2sum);
441 sym_out = prefzeta_sf * angular * symtmp * ftot;
442
443 const CT pref_lam = zeta_sf * tmpzeta * lam_sf * inv_g2;
444 const CT tmp_dsymdr = (CT)(-2.0) * symtmp * eta_sf;
445 const CT dsymdr1 = tmp_dsymdr * r1;
446 const CT dsymdr2 = tmp_dsymdr * r2;
447 const CT dsymdr3 = tmp_dsymdr * r3;
448
449 const CT pref_local = prefzeta_sf * symtmp * ftot;
450 const CT tmp1 = prefzeta_sf * angular * (ftot * dsymdr1 + dfcdr1 * symtmp);
451 const CT tmp2 = prefzeta_sf * angular * (ftot * dsymdr2 + dfcdr2 * symtmp);
452 const CT tmp3 = prefzeta_sf * angular * (ftot * dsymdr3 + dfcdr3 * symtmp);
453
454 const CT pl_dcb2_x = pref_local * pref_lam * dcb2_x;
455 const CT pl_dcb2_y = pref_local * pref_lam * dcb2_y;
456 const CT pl_dcb2_z = pref_local * pref_lam * dcb2_z;
457 const CT pl_dcb3_x = pref_local * pref_lam * dcb3_x;
458 const CT pl_dcb3_y = pref_local * pref_lam * dcb3_y;
459 const CT pl_dcb3_z = pref_local * pref_lam * dcb3_z;
460
461 fjj_x = pl_dcb2_x - tmp1 * dr1dx_x + tmp3 * dr3dx_x;
462 fjj_y = pl_dcb2_y - tmp1 * dr1dx_y + tmp3 * dr3dx_y;
463 fjj_z = pl_dcb2_z - tmp1 * dr1dx_z + tmp3 * dr3dx_z;
464 fkk_x = pl_dcb3_x - tmp2 * dr2dx_x - tmp3 * dr3dx_x;
465 fkk_y = pl_dcb3_y - tmp2 * dr2dx_y - tmp3 * dr3dx_y;
466 fkk_z = pl_dcb3_z - tmp2 * dr2dx_z - tmp3 * dr3dx_z;
467
468 self_x_inc = pref_local * pref_lam * dcb1_x + tmp1 * dr1dx_x + tmp2 * dr2dx_x;
469 self_y_inc = pref_local * pref_lam * dcb1_y + tmp1 * dr1dx_y + tmp2 * dr2dx_y;
470 self_z_inc = pref_local * pref_lam * dcb1_z + tmp1 * dr1dx_z + tmp2 * dr2dx_z;
471}
472
473/*******************************************************************************
474 * \brief Production angular symmetry-function kernel with force partials,
475 * reading atomic positions and periodic-image translations from
476 * device-resident state. One warp handles one (symmetry function, atom)
477 * pair; the 32 lanes split the neighbour work. The displacement
478 * reconstruction matches nnp_compute_neighbors_cell_list and the cutoff
479 * matches nnp_fill_fc_dfc_cache, so the outputs are bit-identical to the
480 * host-packed path. pass 1 is j-parallel and k-serial (writing jj, the
481 * warp-reduced G into G_dev via atomicAdd, and the self term); the
482 * k-side force kk is produced either by the fused shared-memory path or
483 * by the k-parallel pass 2.
484 *
485 * Cutoff guards. Under Verlet-skin neighbour reuse the walk hands this
486 * kernel a (cutoff + skin) superset, so an r >= cutoff_s neighbour must
487 * contribute exactly zero. The tanh^3 cutoff fc = tanh^3(1 - r/c) goes
488 * NEGATIVE for r > c, so fc alone cannot be relied on to zero those
489 * contributions the way the host's strict cutoff did; the explicit
490 * r < cutoff_s guards do it instead.
491 *
492 * K3 fused path (fused_kk != 0). Each warp owns a private 3*max_n_kk
493 * shared-memory slice keyed by threadIdx.y. pass 1 computes each
494 * triplet's kk force once and shared-memory-atomicAdds it into the k
495 * slot, then the warp flushes the slice to kk_flat and skips pass 2.
496 * The k slot index matches pass 2's kk_flat row exactly. The launcher
497 * passes the dynamic shared size only when fused_kk != 0; with
498 * fused_kk == 0 it passes zero and the two-phase path runs, leaving the
499 * shared accumulator untouched.
500 *
501 * Precision. Templated on CT: under MIXED the launcher instantiates
502 * <float>, and the distances (read from the FP32 displacement buffers),
503 * the cutoff and the triplet math follow CT. The accumulators that sum
504 * over the triplet loop (Gacc, self, jj / kk, the warp butterfly) stay
505 * FP64 regardless, because a thousand-term FP32 sum loses more than the
506 * per-term rounding does; only the shared kk slice follows CT, to keep
507 * its shared-memory footprint down.
508 * <double> is byte-identical to a plain double kernel. G_dev is zeroed
509 * by nnp_gpu_begin_descriptor_step; each (sf, atom) atomicAdds a unique
510 * (descriptor row, atom slot), so there is no intra-launch contention.
511 *
512 * Occupancy lever (OCC, compile-time). The <float> occupancy shell
513 * instantiates OCC = true, which stages the per-j geometry
514 * {rvect1 xyz, r1, fc1, dfc1} into a per-(warp, lane) shared-memory
515 * slot once per j-iteration and reloads it, giving the compiler a
516 * cheap home for those values under a tight register cap instead of
517 * spilling them across the inner k-loop. The store/reload is
518 * CT->shared->CT, so every value is bit-identical to the
519 * register-resident original. Shared-memory layout with OCC: the
520 * geometry cache occupies the first GEOM_SLOTS*32*blockDim.y elements
521 * (slot base (threadIdx.y*32 + lane)*GEOM_SLOTS) and the K3 kk
522 * accumulator follows it, so the kk offset is independent of fused_kk;
523 * the launcher sizes the region as geom_bytes + (fused_kk ? kk_bytes :
524 * 0). With OCC = false the geometry cache has zero extent and the
525 * layout is the kk slice alone.
526 * \author Dhruv Sharma (ds2173@cam.ac.uk)
527 ******************************************************************************/
528template <typename CT, bool OCC>
529__device__ __forceinline__ void nnp_angular_sf_force_body(
530 const int homo, const int cut_type, const int max_n_ang1,
531 const int max_n_ang2, const int max_n_kk, const int n_symf,
532 const int n_atoms, const double cutoff_s, const double cutoff_sqr,
533 const int *__restrict__ n_ang1_per_atom,
534 const int *__restrict__ n_ang2_per_atom,
535 const double *__restrict__ coord_dev,
536 const double *__restrict__ image_table_dev,
537 const int *__restrict__ ang1_ind, const int *__restrict__ ang2_ind,
538 const int *__restrict__ ang1_image_idx,
539 const int *__restrict__ ang2_image_idx,
540 const int *__restrict__ atoms_global,
541 /* FP32-geometry path inputs (read only when sizeof(CT)==4):
542 * precomputed rvect = center - neighbour_image, FP32,
543 * layout 3*(max_n_ang1*atom + m) for ang1, 3*(max_n_ang2*atom + m) for
544 * ang2 (homo k reuses ang1_disp). NULL on the FP64/DOUBLE path. */
545 const float *__restrict__ ang1_disp, const float *__restrict__ ang2_disp,
546 const double *__restrict__ pack_lam, const double *__restrict__ pack_zeta,
547 const int *__restrict__ pack_izeta,
548 const char *__restrict__ pack_use_int_zeta,
549 const double *__restrict__ pack_eta,
550 const double *__restrict__ pack_prefzeta,
551 double *__restrict__ self_partial_flat, double *__restrict__ jj_flat,
552 double *__restrict__ kk_flat, double *__restrict__ G_dev,
553 const int *__restrict__ symf_map, const int *__restrict__ atom_map,
554 const int n_input_max, const int fused_kk) {
555 const int lane = threadIdx.x;
556 const int sf = blockIdx.y * blockDim.y + threadIdx.y;
557 if (sf >= n_symf)
558 return;
559 const int atom_idx = blockIdx.x;
560 if (atom_idx >= n_atoms)
561 return;
562
563 const int n_ang1_a = n_ang1_per_atom[atom_idx];
564 const int n_ang2_a = homo ? 0 : n_ang2_per_atom[atom_idx];
565 if (n_ang1_a <= 0)
566 return;
567 if (!homo && n_ang2_a <= 0)
568 return;
569
570 const double lam_sf = pack_lam[sf];
571 const double zeta_sf = pack_zeta[sf];
572 const int izeta_sf = pack_izeta[sf];
573 const int use_int_zeta_sf = pack_use_int_zeta[sf] ? 1 : 0;
574 const double eta_sf = pack_eta[sf];
575 const double prefzeta_sf = pack_prefzeta[sf];
576
577 /* Hoisted center-atom position (one global load per (sf, atom)). */
578 const int i_glob = atoms_global[atom_idx];
579 const double cx = coord_dev[3 * i_glob + 0];
580 const double cy = coord_dev[3 * i_glob + 1];
581 const double cz = coord_dev[3 * i_glob + 2];
582
583 /* Per-atom strides, 64-bit where they scale with the per-neighbour buffers
584 * (matching nnp_gpu_pack.h: those tables exceed 2^31 entries on large
585 * ranks). */
586 const long long r_off = (long long)max_n_ang1 * (long long)atom_idx;
587 const long long r2_off = (long long)max_n_ang2 * (long long)atom_idx;
588 const long long jj_atom_off =
589 3LL * (long long)n_symf * (long long)max_n_ang1 * (long long)atom_idx;
590 const long long kk_atom_off =
591 3LL * (long long)n_symf * (long long)max_n_kk * (long long)atom_idx;
592
593 /* The per-triplet math runs in CT, which is where the single-precision
594 * speed-up lives, but the accumulators stay FP64: an angular G sums on the
595 * order of a thousand triplets, and the self term the same, so accumulating
596 * those in FP32 would cost more accuracy than the per-term rounding does.
597 * These are a handful of registers, so keeping them wide is close to free. */
598 double Gacc = 0.0;
599 double self_x = 0.0, self_y = 0.0, self_z = 0.0;
600 const int stride = blockDim.x;
601
602 /* K3 fusion: per-warp shared-memory kk accumulator. Each warp is one sf row
603 * of the block (all 32 lanes share threadIdx.y), so it owns a private
604 * 3*max_n_kk slice, so there is no cross-warp contention, only contention
605 * across the 32 j-lanes of this warp. The launcher passes the dynamic shared
606 * size only when
607 * fused_kk == 1; with fused_kk == 0 it passes zero and kk_sh is never
608 * touched (the kernel runs the two-phase path). Under OCC the geometry
609 * cache precedes the kk slice in the same dynamic region. */
610 const int GEOM_SLOTS = 6;
611 extern __shared__ unsigned char nnp_kk_raw[];
612 CT *sh = reinterpret_cast<CT *>(nnp_kk_raw);
613 const int geom_total = OCC ? GEOM_SLOTS * 32 * blockDim.y : 0;
614 const int geom_base = OCC ? (threadIdx.y * 32 + lane) * GEOM_SLOTS : 0;
615 CT *kk_sh = sh + geom_total;
616 const int kk_sh_base = threadIdx.y * 3 * max_n_kk;
617 if (fused_kk) {
618 for (int k = lane; k < max_n_kk; k += stride) {
619 kk_sh[kk_sh_base + 3 * k + 0] = (CT)0;
620 kk_sh[kk_sh_base + 3 * k + 1] = (CT)0;
621 kk_sh[kk_sh_base + 3 * k + 2] = (CT)0;
622 }
623 NNP_SYNCWARP();
624 }
625
626 /* pass 1: j-parallel, k-serial. */
627 for (int j = lane; j < n_ang1_a; j += stride) {
628 /* Geometry in CT: FP32-disp path reads the precomputed rvect (MIXED
629 * only); FP64 path recomputes from coords (DOUBLE bit-identical since
630 * sizeof(CT)==4 is false and the disp branch is dropped). */
631 CT rvect1_x, rvect1_y, rvect1_z, r1;
632 if (sizeof(CT) == 4) {
633 rvect1_x = (CT)ang1_disp[3 * (r_off + j) + 0];
634 rvect1_y = (CT)ang1_disp[3 * (r_off + j) + 1];
635 rvect1_z = (CT)ang1_disp[3 * (r_off + j) + 2];
636 r1 =
637 sqrt(rvect1_x * rvect1_x + rvect1_y * rvect1_y + rvect1_z * rvect1_z);
638 } else {
639 const int j_glob = ang1_ind[r_off + j];
640 const int img1 = ang1_image_idx[r_off + j];
641 const double image_pos_j_x =
642 coord_dev[3 * j_glob + 0] + image_table_dev[3 * img1 + 0];
643 const double image_pos_j_y =
644 coord_dev[3 * j_glob + 1] + image_table_dev[3 * img1 + 1];
645 const double image_pos_j_z =
646 coord_dev[3 * j_glob + 2] + image_table_dev[3 * img1 + 2];
647 const double d_rvect1_x = cx - image_pos_j_x;
648 const double d_rvect1_y = cy - image_pos_j_y;
649 const double d_rvect1_z = cz - image_pos_j_z;
650 const double d_r1 =
651 sqrt(d_rvect1_x * d_rvect1_x + d_rvect1_y * d_rvect1_y +
652 d_rvect1_z * d_rvect1_z);
653 rvect1_x = (CT)d_rvect1_x;
654 rvect1_y = (CT)d_rvect1_y;
655 rvect1_z = (CT)d_rvect1_z;
656 r1 = (CT)d_r1;
657 }
658 if (OCC) {
659 /* Stage the j-side geometry that stays live across the k-loop. */
660 sh[geom_base + 0] = rvect1_x;
661 sh[geom_base + 1] = rvect1_y;
662 sh[geom_base + 2] = rvect1_z;
663 sh[geom_base + 3] = r1;
664 rvect1_x = sh[geom_base + 0];
665 rvect1_y = sh[geom_base + 1];
666 rvect1_z = sh[geom_base + 2];
667 r1 = sh[geom_base + 3];
668 }
669
670 double jj_acc_x = 0.0, jj_acc_y = 0.0, jj_acc_z = 0.0;
671
672 /* r1 >= cutoff_s must contribute exactly 0: fc1 = tanh^3(1 - r/c) is
673 * negative past the cutoff, so skip the k-loop entirely (jj_acc stays 0,
674 * jj_flat is written 0 below). */
675 if (r1 < (CT)cutoff_s) {
676 CT fc1, dfc1;
677 nnp_compute_fc_dfc<CT>(r1, (CT)cutoff_s, cut_type, &fc1, &dfc1);
678 if (OCC) {
679 sh[geom_base + 4] = fc1;
680 sh[geom_base + 5] = dfc1;
681 fc1 = sh[geom_base + 4];
682 dfc1 = sh[geom_base + 5];
683 }
684
685 const int k_start = homo ? j + 1 : 0;
686 const int k_end = homo ? n_ang1_a : n_ang2_a;
687 for (int k = k_start; k < k_end; k++) {
688 /* k-neighbour disp: homo reuses the ang1 list (3*(r_off + k)),
689 * hetero reads the ang2 list (3*(r2_off + k)). */
690 CT rvect2_x, rvect2_y, rvect2_z, r2;
691 if (sizeof(CT) == 4) {
692 const float *disp_k = homo ? ang1_disp : ang2_disp;
693 const int koff = homo ? (r_off + k) : (r2_off + k);
694 rvect2_x = (CT)disp_k[3 * koff + 0];
695 rvect2_y = (CT)disp_k[3 * koff + 1];
696 rvect2_z = (CT)disp_k[3 * koff + 2];
697 r2 = sqrt(rvect2_x * rvect2_x + rvect2_y * rvect2_y +
698 rvect2_z * rvect2_z);
699 } else {
700 const int k_glob = homo ? ang1_ind[r_off + k] : ang2_ind[r2_off + k];
701 const int img2 =
702 homo ? ang1_image_idx[r_off + k] : ang2_image_idx[r2_off + k];
703 const double image_pos_k_x =
704 coord_dev[3 * k_glob + 0] + image_table_dev[3 * img2 + 0];
705 const double image_pos_k_y =
706 coord_dev[3 * k_glob + 1] + image_table_dev[3 * img2 + 1];
707 const double image_pos_k_z =
708 coord_dev[3 * k_glob + 2] + image_table_dev[3 * img2 + 2];
709 const double d_rvect2_x = cx - image_pos_k_x;
710 const double d_rvect2_y = cy - image_pos_k_y;
711 const double d_rvect2_z = cz - image_pos_k_z;
712 const double d_r2 =
713 sqrt(d_rvect2_x * d_rvect2_x + d_rvect2_y * d_rvect2_y +
714 d_rvect2_z * d_rvect2_z);
715 rvect2_x = (CT)d_rvect2_x;
716 rvect2_y = (CT)d_rvect2_y;
717 rvect2_z = (CT)d_rvect2_z;
718 r2 = (CT)d_r2;
719 }
720 /* Same r-cutoff guard for the k-side (ang2 list also widened by skin
721 * at the walk level). */
722 if (r2 >= (CT)cutoff_s)
723 continue;
724 CT fc2, dfc2;
725 nnp_compute_fc_dfc<CT>(r2, (CT)cutoff_s, cut_type, &fc2, &dfc2);
726
727 const CT rvect3_x = rvect2_x - rvect1_x;
728 const CT rvect3_y = rvect2_y - rvect1_y;
729 const CT rvect3_z = rvect2_z - rvect1_z;
730 const CT r3_sqr =
731 rvect3_x * rvect3_x + rvect3_y * rvect3_y + rvect3_z * rvect3_z;
732 if (r3_sqr >= (CT)cutoff_sqr)
733 continue;
734 const CT r3 = sqrt(r3_sqr);
735
736 CT sym_out;
737 CT f_jj_x, f_jj_y, f_jj_z;
738 CT f_kk_x, f_kk_y, f_kk_z;
739 CT sx_inc, sy_inc, sz_inc;
740 nnp_angular_triplet_math<CT>(
741 r1, rvect1_x, rvect1_y, rvect1_z, fc1, dfc1, r2, rvect2_x, rvect2_y,
742 rvect2_z, fc2, dfc2, r3, rvect3_x, rvect3_y, rvect3_z, r3_sqr,
743 (CT)cutoff_s, cut_type, (CT)lam_sf, (CT)zeta_sf, izeta_sf,
744 use_int_zeta_sf, (CT)eta_sf, (CT)prefzeta_sf, sym_out, f_jj_x,
745 f_jj_y, f_jj_z, f_kk_x, f_kk_y, f_kk_z, sx_inc, sy_inc, sz_inc);
746
747 Gacc += sym_out;
748 self_x += sx_inc;
749 self_y += sy_inc;
750 self_z += sz_inc;
751 jj_acc_x += f_jj_x;
752 jj_acc_y += f_jj_y;
753 jj_acc_z += f_jj_z;
754 if (fused_kk) {
755 /* f_kk for this (j,k) is already computed above. Accumulate it into
756 * the warp's shared k-slot (a shared-memory atomicAdd in CT, not a
757 * global atomicAdd). The kk slot index k matches pass 2's kk_flat
758 * row exactly; pass 2 is skipped below. */
759 atomicAdd(&kk_sh[kk_sh_base + 3 * k + 0], f_kk_x);
760 atomicAdd(&kk_sh[kk_sh_base + 3 * k + 1], f_kk_y);
761 atomicAdd(&kk_sh[kk_sh_base + 3 * k + 2], f_kk_z);
762 }
763 }
764 } /* end r1 < cutoff_s guard */
765
766 const long long jj_base = jj_atom_off + 3 * sf + 3 * n_symf * j;
767 jj_flat[jj_base + 0] = (double)jj_acc_x;
768 jj_flat[jj_base + 1] = (double)jj_acc_y;
769 jj_flat[jj_base + 2] = (double)jj_acc_z;
770 }
771
772 /* Butterfly-reduce G / self_partial across the warp. Max lane offset is 16,
773 * so the reduction spans a 32-lane warp and is correct on a 64-wide AMD
774 * wavefront given the 32-thread block width. */
775 for (int offset = 16; offset > 0; offset /= 2) {
776 Gacc += NNP_SHFL_XOR(Gacc, offset);
777 self_x += NNP_SHFL_XOR(self_x, offset);
778 self_y += NNP_SHFL_XOR(self_y, offset);
779 self_z += NNP_SHFL_XOR(self_z, offset);
780 }
781 if (lane == 0) {
782 const int g_row = symf_map[sf];
783 const int g_atom = atom_map[atom_idx];
784 atomicAdd(&G_dev[g_row + (long long)n_input_max * g_atom], (double)Gacc);
785
786 const long long self_base = 3LL * sf + 3LL * n_symf * atom_idx;
787 self_partial_flat[self_base + 0] = (double)self_x;
788 self_partial_flat[self_base + 1] = (double)self_y;
789 self_partial_flat[self_base + 2] = (double)self_z;
790 }
791
792 const int k_count = homo ? n_ang1_a : n_ang2_a;
793
794 /* K3 fused flush: every j-lane has atomicAdd'd its f_kk into this warp's
795 * shared slice. All 32 lanes reach here (the same warp-convergence point the
796 * butterfly above relies on, since the in-loop r1 < cutoff_s guard never
797 * gated reaching it, and the per-(atom, sf) early returns are warp-uniform).
798 * __syncwarp makes the shared writes visible, then kk_flat is written
799 * coalesced and pass 2 is skipped. Slots k in [k_count, max_n_kk) keep the
800 * launcher's pre-launch memset 0, matching the two-phase path. */
801 if (fused_kk) {
802 NNP_SYNCWARP();
803 for (int k = lane; k < k_count; k += stride) {
804 const long long kk_base = kk_atom_off + 3 * sf + 3 * n_symf * k;
805 kk_flat[kk_base + 0] = (double)kk_sh[kk_sh_base + 3 * k + 0];
806 kk_flat[kk_base + 1] = (double)kk_sh[kk_sh_base + 3 * k + 1];
807 kk_flat[kk_base + 2] = (double)kk_sh[kk_sh_base + 3 * k + 2];
808 }
809 return;
810 }
811
812 /* pass 2: k-parallel, j-serial (only when not fused). */
813 for (int k = lane; k < k_count; k += stride) {
814 /* k-neighbour disp: homo reuses ang1 list (3*(r_off + k)), hetero
815 * reads ang2 list (3*(r2_off + k)). Same selection as pass 1. */
816 CT rvect2_x, rvect2_y, rvect2_z, r2;
817 if (sizeof(CT) == 4) {
818 const float *disp_k = homo ? ang1_disp : ang2_disp;
819 const int koff = homo ? (r_off + k) : (r2_off + k);
820 rvect2_x = (CT)disp_k[3 * koff + 0];
821 rvect2_y = (CT)disp_k[3 * koff + 1];
822 rvect2_z = (CT)disp_k[3 * koff + 2];
823 r2 =
824 sqrt(rvect2_x * rvect2_x + rvect2_y * rvect2_y + rvect2_z * rvect2_z);
825 } else {
826 const int k_glob = homo ? ang1_ind[r_off + k] : ang2_ind[r2_off + k];
827 const int img2 =
828 homo ? ang1_image_idx[r_off + k] : ang2_image_idx[r2_off + k];
829 const double image_pos_k_x =
830 coord_dev[3 * k_glob + 0] + image_table_dev[3 * img2 + 0];
831 const double image_pos_k_y =
832 coord_dev[3 * k_glob + 1] + image_table_dev[3 * img2 + 1];
833 const double image_pos_k_z =
834 coord_dev[3 * k_glob + 2] + image_table_dev[3 * img2 + 2];
835 const double d_rvect2_x = cx - image_pos_k_x;
836 const double d_rvect2_y = cy - image_pos_k_y;
837 const double d_rvect2_z = cz - image_pos_k_z;
838 const double d_r2 =
839 sqrt(d_rvect2_x * d_rvect2_x + d_rvect2_y * d_rvect2_y +
840 d_rvect2_z * d_rvect2_z);
841 rvect2_x = (CT)d_rvect2_x;
842 rvect2_y = (CT)d_rvect2_y;
843 rvect2_z = (CT)d_rvect2_z;
844 r2 = (CT)d_r2;
845 }
846
847 double kk_acc_x = 0.0, kk_acc_y = 0.0, kk_acc_z = 0.0;
848
849 /* Skip the j-loop entirely if r2 is in the Verlet-skin margin
850 * [cutoff, cutoff+skin): fc2 would be negative, polluting kk_acc.
851 * kk_flat is still written 0 below. */
852 if (r2 < (CT)cutoff_s) {
853 CT fc2, dfc2;
854 nnp_compute_fc_dfc<CT>(r2, (CT)cutoff_s, cut_type, &fc2, &dfc2);
855
856 const int j_end = homo ? k : n_ang1_a;
857 for (int j = 0; j < j_end; j++) {
858 /* j-neighbour disp always reads the ang1 list (3*(r_off + j)). */
859 CT rvect1_x, rvect1_y, rvect1_z, r1;
860 if (sizeof(CT) == 4) {
861 rvect1_x = (CT)ang1_disp[3 * (r_off + j) + 0];
862 rvect1_y = (CT)ang1_disp[3 * (r_off + j) + 1];
863 rvect1_z = (CT)ang1_disp[3 * (r_off + j) + 2];
864 r1 = sqrt(rvect1_x * rvect1_x + rvect1_y * rvect1_y +
865 rvect1_z * rvect1_z);
866 } else {
867 const int j_glob = ang1_ind[r_off + j];
868 const int img1 = ang1_image_idx[r_off + j];
869 const double image_pos_j_x =
870 coord_dev[3 * j_glob + 0] + image_table_dev[3 * img1 + 0];
871 const double image_pos_j_y =
872 coord_dev[3 * j_glob + 1] + image_table_dev[3 * img1 + 1];
873 const double image_pos_j_z =
874 coord_dev[3 * j_glob + 2] + image_table_dev[3 * img1 + 2];
875 const double d_rvect1_x = cx - image_pos_j_x;
876 const double d_rvect1_y = cy - image_pos_j_y;
877 const double d_rvect1_z = cz - image_pos_j_z;
878 const double d_r1 =
879 sqrt(d_rvect1_x * d_rvect1_x + d_rvect1_y * d_rvect1_y +
880 d_rvect1_z * d_rvect1_z);
881 rvect1_x = (CT)d_rvect1_x;
882 rvect1_y = (CT)d_rvect1_y;
883 rvect1_z = (CT)d_rvect1_z;
884 r1 = (CT)d_r1;
885 }
886 /* Same r1 cutoff guard. */
887 if (r1 >= (CT)cutoff_s)
888 continue;
889 CT fc1, dfc1;
890 nnp_compute_fc_dfc<CT>(r1, (CT)cutoff_s, cut_type, &fc1, &dfc1);
891
892 const CT rvect3_x = rvect2_x - rvect1_x;
893 const CT rvect3_y = rvect2_y - rvect1_y;
894 const CT rvect3_z = rvect2_z - rvect1_z;
895 const CT r3_sqr =
896 rvect3_x * rvect3_x + rvect3_y * rvect3_y + rvect3_z * rvect3_z;
897 if (r3_sqr >= (CT)cutoff_sqr)
898 continue;
899 const CT r3 = sqrt(r3_sqr);
900
901 CT sym_out;
902 CT f_jj_x, f_jj_y, f_jj_z;
903 CT f_kk_x, f_kk_y, f_kk_z;
904 CT sx_inc, sy_inc, sz_inc;
905 nnp_angular_triplet_math<CT>(
906 r1, rvect1_x, rvect1_y, rvect1_z, fc1, dfc1, r2, rvect2_x, rvect2_y,
907 rvect2_z, fc2, dfc2, r3, rvect3_x, rvect3_y, rvect3_z, r3_sqr,
908 (CT)cutoff_s, cut_type, (CT)lam_sf, (CT)zeta_sf, izeta_sf,
909 use_int_zeta_sf, (CT)eta_sf, (CT)prefzeta_sf, sym_out, f_jj_x,
910 f_jj_y, f_jj_z, f_kk_x, f_kk_y, f_kk_z, sx_inc, sy_inc, sz_inc);
911
912 kk_acc_x += f_kk_x;
913 kk_acc_y += f_kk_y;
914 kk_acc_z += f_kk_z;
915 }
916 } /* end r2 < cutoff_s guard */
917
918 const long long kk_base = kk_atom_off + 3 * sf + 3 * n_symf * k;
919 kk_flat[kk_base + 0] = (double)kk_acc_x;
920 kk_flat[kk_base + 1] = (double)kk_acc_y;
921 kk_flat[kk_base + 2] = (double)kk_acc_z;
922 }
923}
924
925/*******************************************************************************
926 * \brief Default __global__ shell of nnp_angular_sf_force_body (OCC = false):
927 * register-resident geometry, no launch-bounds cap.
928 * \author Dhruv Sharma (ds2173@cam.ac.uk)
929 ******************************************************************************/
930template <typename CT>
931__global__ void nnp_angular_sf_force_kernel(
932 const int homo, const int cut_type, const int max_n_ang1,
933 const int max_n_ang2, const int max_n_kk, const int n_symf,
934 const int n_atoms, const double cutoff_s, const double cutoff_sqr,
935 const int *__restrict__ n_ang1_per_atom,
936 const int *__restrict__ n_ang2_per_atom,
937 const double *__restrict__ coord_dev,
938 const double *__restrict__ image_table_dev,
939 const int *__restrict__ ang1_ind, const int *__restrict__ ang2_ind,
940 const int *__restrict__ ang1_image_idx,
941 const int *__restrict__ ang2_image_idx,
942 const int *__restrict__ atoms_global, const float *__restrict__ ang1_disp,
943 const float *__restrict__ ang2_disp, const double *__restrict__ pack_lam,
944 const double *__restrict__ pack_zeta, const int *__restrict__ pack_izeta,
945 const char *__restrict__ pack_use_int_zeta,
946 const double *__restrict__ pack_eta,
947 const double *__restrict__ pack_prefzeta,
948 double *__restrict__ self_partial_flat, double *__restrict__ jj_flat,
949 double *__restrict__ kk_flat, double *__restrict__ G_dev,
950 const int *__restrict__ symf_map, const int *__restrict__ atom_map,
951 const int n_input_max, const int fused_kk) {
952 nnp_angular_sf_force_body<CT, false>(
953 homo, cut_type, max_n_ang1, max_n_ang2, max_n_kk, n_symf, n_atoms,
954 cutoff_s, cutoff_sqr, n_ang1_per_atom, n_ang2_per_atom, coord_dev,
955 image_table_dev, ang1_ind, ang2_ind, ang1_image_idx, ang2_image_idx,
956 atoms_global, ang1_disp, ang2_disp, pack_lam, pack_zeta, pack_izeta,
957 pack_use_int_zeta, pack_eta, pack_prefzeta, self_partial_flat, jj_flat,
958 kk_flat, G_dev, symf_map, atom_map, n_input_max, fused_kk);
959}
960
961/*******************************************************************************
962 * \brief Occupancy __global__ shell of nnp_angular_sf_force_body (OCC =
963 * true). __launch_bounds__(1024, 1) caps the per-thread register count
964 * so at least one full 1024-thread block (32 warps) stays resident per
965 * SM on this register-bound kernel; 1024 is the largest block the
966 * launcher uses (32 * sf_per_block with sf_per_block <= 32), and a
967 * smaller block is still valid under that upper bound. Launched on
968 * either precision whenever the shared budget fits; measured on an A100
969 * it cuts the FP64 descriptor stage by a third (see the launcher).
970 * \author Dhruv Sharma (ds2173@cam.ac.uk)
971 ******************************************************************************/
972template <typename CT>
973__global__ void __launch_bounds__(1024, 1) nnp_angular_sf_force_kernel_occ(
974 const int homo, const int cut_type, const int max_n_ang1,
975 const int max_n_ang2, const int max_n_kk, const int n_symf,
976 const int n_atoms, const double cutoff_s, const double cutoff_sqr,
977 const int *__restrict__ n_ang1_per_atom,
978 const int *__restrict__ n_ang2_per_atom,
979 const double *__restrict__ coord_dev,
980 const double *__restrict__ image_table_dev,
981 const int *__restrict__ ang1_ind, const int *__restrict__ ang2_ind,
982 const int *__restrict__ ang1_image_idx,
983 const int *__restrict__ ang2_image_idx,
984 const int *__restrict__ atoms_global, const float *__restrict__ ang1_disp,
985 const float *__restrict__ ang2_disp, const double *__restrict__ pack_lam,
986 const double *__restrict__ pack_zeta, const int *__restrict__ pack_izeta,
987 const char *__restrict__ pack_use_int_zeta,
988 const double *__restrict__ pack_eta,
989 const double *__restrict__ pack_prefzeta,
990 double *__restrict__ self_partial_flat, double *__restrict__ jj_flat,
991 double *__restrict__ kk_flat, double *__restrict__ G_dev,
992 const int *__restrict__ symf_map, const int *__restrict__ atom_map,
993 const int n_input_max, const int fused_kk) {
994 nnp_angular_sf_force_body<CT, true>(
995 homo, cut_type, max_n_ang1, max_n_ang2, max_n_kk, n_symf, n_atoms,
996 cutoff_s, cutoff_sqr, n_ang1_per_atom, n_ang2_per_atom, coord_dev,
997 image_table_dev, ang1_ind, ang2_ind, ang1_image_idx, ang2_image_idx,
998 atoms_global, ang1_disp, ang2_disp, pack_lam, pack_zeta, pack_izeta,
999 pack_use_int_zeta, pack_eta, pack_prefzeta, self_partial_flat, jj_flat,
1000 kk_flat, G_dev, symf_map, atom_map, n_input_max, fused_kk);
1001}
1002
1003/*******************************************************************************
1004 * \brief Precompute the per-(atom, neighbour) displacement rvect = center -
1005 * neighbour_image in FP64 (so the large-coordinate cancellation happens
1006 * once, off the angular kernel's O(n^2) triplet loop) and store it as
1007 * FP32 for the single-precision angular path. The layout mirrors
1008 * pbuf_*_ind / pbuf_image_idx exactly: index 3*(max_n_per_se*atom + m).
1009 * The sign matches the symmetry-function kernel's rvect = center -
1010 * neighbour_image. Padding slots (m >= per-atom count) write zero; the
1011 * kernel's neighbour-count guards keep them inert.
1012 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1013 ******************************************************************************/
1014__global__ void nnp_pack_angular_disp_kernel(
1015 int n_atoms_in_e, int max_n_per_se, const int *__restrict__ pbuf_n_pa,
1016 const int *__restrict__ pbuf_ind, const int *__restrict__ pbuf_image_idx,
1017 const int *__restrict__ pbuf_atoms_global0,
1018 const double *__restrict__ coord_dev,
1019 const double *__restrict__ image_table_dev, float *__restrict__ pbuf_disp) {
1020 const int atom = blockIdx.x;
1021 const int m = blockIdx.y * blockDim.x + threadIdx.x;
1022 if (atom >= n_atoms_in_e || m >= max_n_per_se)
1023 return;
1024 const long long off = (long long)max_n_per_se * atom + m;
1025 const int n = pbuf_n_pa[atom];
1026 if (m >= n) {
1027 pbuf_disp[3 * off + 0] = 0.f;
1028 pbuf_disp[3 * off + 1] = 0.f;
1029 pbuf_disp[3 * off + 2] = 0.f;
1030 return;
1031 }
1032 const int ci = pbuf_atoms_global0[atom];
1033 const int ni = pbuf_ind[off];
1034 const int im = pbuf_image_idx[off];
1035 const double rx = coord_dev[3 * ci + 0] -
1036 (coord_dev[3 * ni + 0] + image_table_dev[3 * im + 0]);
1037 const double ry = coord_dev[3 * ci + 1] -
1038 (coord_dev[3 * ni + 1] + image_table_dev[3 * im + 1]);
1039 const double rz = coord_dev[3 * ci + 2] -
1040 (coord_dev[3 * ni + 2] + image_table_dev[3 * im + 2]);
1041 pbuf_disp[3 * off + 0] = (float)rx;
1042 pbuf_disp[3 * off + 1] = (float)ry;
1043 pbuf_disp[3 * off + 2] = (float)rz;
1044}
1045
1046/*******************************************************************************
1047 * \brief Descriptor scaling kernel: one thread per atom, looping over the
1048 * atom's n_input_nodes(element) descriptors and applying one of five
1049 * scaling modes (matching the four Fortran branches in
1050 * nnp_aggregate_G_from_batches plus identity). Reads the raw aggregated
1051 * descriptor G_in (state->G_dev) and writes the scaled network input
1052 * G_out (state->G_dev_scaled).
1053 *
1054 * Bit-exact with the Fortran path: the same FP operations on the same
1055 * values. Per-element scaling parameters (loc_av, loc_min, loc_max,
1056 * sigma) are concatenated radial-then-angular and indexed
1057 * [ind * n_input_max + m] to match the per-atom
1058 * [m + i_local * n_input_max] descriptor layout the upstream
1059 * symmetry-function launchers write. Descriptors above n_input_nodes are
1060 * zero-padded, mirroring the Fortran pad.
1061 *
1062 * As a side effect the kernel sets *expol_flag when an unscaled
1063 * descriptor lies outside its training range, the device analogue of
1064 * nnp_check_extrapolation on the CPU path; the launcher zeroes the flag
1065 * beforehand and copies it back stream-ordered.
1066 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1067 ******************************************************************************/
1068__global__ void nnp_scale_G_dev_kernel(
1069 const int n_atoms, const int n_input_max, const int scaling_mode,
1070 const double scmin, const double scmax,
1071 const int *__restrict__ ele_ind_for_atoms,
1072 const int *__restrict__ n_input_nodes_per_ele,
1073 const double *__restrict__ loc_av_per_ele,
1074 const double *__restrict__ loc_min_per_ele,
1075 const double *__restrict__ loc_max_per_ele,
1076 const double *__restrict__ sigma_per_ele, const double *__restrict__ G_in,
1077 double *__restrict__ G_out, int *__restrict__ expol_flag) {
1078 /* Out-of-training-range threshold on the unscaled descriptor, the same
1079 * predicate and value as nnp_check_extrapolation on the CPU path
1080 * (src/nnp_acsf.F). */
1081 const double expol_threshold = 1.0e-4;
1082
1083 const int i_local = blockIdx.x * blockDim.x + threadIdx.x;
1084 if (i_local >= n_atoms)
1085 return;
1086
1087 const int ind = ele_ind_for_atoms[i_local];
1088 const int n_in = n_input_nodes_per_ele[ind];
1089 const int param_off = ind * n_input_max;
1090 const long long atom_off = (long long)i_local * n_input_max;
1091 const double range = scmax - scmin;
1092
1093 const double *loc_av = loc_av_per_ele + param_off;
1094 const double *loc_min = loc_min_per_ele + param_off;
1095 const double *loc_max = loc_max_per_ele + param_off;
1096 const double *sigma = sigma_per_ele + param_off;
1097
1098 for (int m = 0; m < n_in; ++m) {
1099 const double y = G_in[m + atom_off];
1100 /* Input rows above the model's symmetry-function count carry zero bounds
1101 * and zero values, so they never trip the flag. */
1102 if (y - loc_max[m] > expol_threshold || loc_min[m] - y > expol_threshold) {
1103 atomicOr(expol_flag, 1);
1104 }
1105 double out;
1106 switch (scaling_mode) {
1107 case 1: /* scale_acsf only: (y - loc_min)/(loc_max - loc_min)*range +
1108 scmin */
1109 out = (y - loc_min[m]) / (loc_max[m] - loc_min[m]) * range + scmin;
1110 break;
1111 case 2: /* center_acsf only: y - loc_av */
1112 out = y - loc_av[m];
1113 break;
1114 case 3: /* center + scale: (y - loc_av)/(loc_max - loc_min)*range +
1115 scmin */
1116 out = (y - loc_av[m]) / (loc_max[m] - loc_min[m]) * range + scmin;
1117 break;
1118 case 4: /* scale_sigma_acsf: (y - loc_av)/sigma * range + scmin */
1119 out = (y - loc_av[m]) / sigma[m] * range + scmin;
1120 break;
1121 case 0: /* identity */
1122 default:
1123 out = y;
1124 break;
1125 }
1126 G_out[m + atom_off] = out;
1127 }
1128
1129 /* Pad above n_input_nodes, mirroring the Fortran pad in
1130 * nnp_aggregate_G_from_batches (src/nnp_force.F). */
1131 for (int m = n_in; m < n_input_max; ++m) {
1132 G_out[m + atom_off] = 0.0;
1133 }
1134}
1135
1136/*******************************************************************************
1137 * \brief Begin a descriptor step: ensure the device-resident descriptor
1138 * G_dev and its scaled copy G_dev_scaled are large enough for
1139 * n_atoms x n_input_max doubles (grow-only via nnp_gpu_pbuf_ensure) and
1140 * zero G_dev so the symmetry-function launchers' atomicAdds accumulate
1141 * from a known-zero accumulator. G_dev_scaled is fully overwritten by
1142 * the scaling kernel, so it is not zeroed here. The memset is queued on
1143 * state->stream; the launcher returns immediately. Exported with C
1144 * linkage for the Fortran driver.
1145 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1146 ******************************************************************************/
1147extern "C" int nnp_gpu_begin_descriptor_step(int n_atoms, int n_input_max) {
1148 if (n_atoms <= 0 || n_input_max <= 0) {
1149 fprintf(stderr,
1150 "ERROR: nnp_gpu_begin_descriptor_step: bad sizes "
1151 "n_atoms=%d n_input_max=%d\n",
1152 n_atoms, n_input_max);
1153 return -1;
1154 }
1155 nnp_gpu_state_t *state = nnp_gpu_state_get();
1156 if (state == NULL)
1157 return -1;
1158
1159 const size_t n_bytes = sizeof(double) * (size_t)n_input_max * (size_t)n_atoms;
1160 if (nnp_gpu_pbuf_ensure((void **)&state->G_dev, &state->G_dev_cap, n_bytes))
1161 return -1;
1162 if (nnp_gpu_pbuf_ensure((void **)&state->G_dev_scaled,
1163 &state->G_dev_scaled_cap, n_bytes))
1164 return -1;
1165
1166 offloadMemsetAsync(state->G_dev, 0, n_bytes, state->stream);
1167 return 0;
1168}
1169
1170/*******************************************************************************
1171 * \brief Once-at-NNP-init upload of the per-element descriptor scaling tables
1172 * (loc_av, loc_min, loc_max, sigma, each n_ele x n_input_max, and the
1173 * per-element input-node counts). Grows the persistent state buffers,
1174 * queues the copies on state->stream, synchronizes so the host buffers
1175 * may be freed on return, and sets state->scaling_uploaded. Exported
1176 * with C linkage for the Fortran driver.
1177 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1178 ******************************************************************************/
1179extern "C" int nnp_gpu_upload_scaling_tables_c(
1180 int n_ele, int n_input_max, const double *h_loc_av_per_ele,
1181 const double *h_loc_min_per_ele, const double *h_loc_max_per_ele,
1182 const double *h_sigma_per_ele, const int *h_n_input_nodes_per_ele) {
1183 if (n_ele <= 0 || n_input_max <= 0) {
1184 fprintf(stderr,
1185 "ERROR: nnp_gpu_upload_scaling_tables: bad sizes "
1186 "n_ele=%d n_input_max=%d\n",
1187 n_ele, n_input_max);
1188 return -1;
1189 }
1190 nnp_gpu_state_t *state = nnp_gpu_state_get();
1191 if (state == NULL)
1192 return -1;
1193
1194 const size_t param_doubles = (size_t)n_ele * (size_t)n_input_max;
1195 const size_t param_bytes = sizeof(double) * param_doubles;
1196 const size_t n_in_bytes = sizeof(int) * (size_t)n_ele;
1197
1198 if (nnp_gpu_pbuf_ensure((void **)&state->scaling_loc_av_dev,
1199 &state->scaling_loc_av_cap, param_bytes))
1200 return -1;
1201 if (nnp_gpu_pbuf_ensure((void **)&state->scaling_loc_min_dev,
1202 &state->scaling_loc_min_cap, param_bytes))
1203 return -1;
1204 if (nnp_gpu_pbuf_ensure((void **)&state->scaling_loc_max_dev,
1205 &state->scaling_loc_max_cap, param_bytes))
1206 return -1;
1207 if (nnp_gpu_pbuf_ensure((void **)&state->scaling_sigma_dev,
1208 &state->scaling_sigma_cap, param_bytes))
1209 return -1;
1210 if (nnp_gpu_pbuf_ensure((void **)&state->scaling_n_input_nodes_dev,
1211 &state->scaling_n_input_nodes_cap, n_in_bytes))
1212 return -1;
1213
1214 offloadMemcpyAsyncHtoD(state->scaling_loc_av_dev, h_loc_av_per_ele,
1215 param_bytes, state->stream);
1216 offloadMemcpyAsyncHtoD(state->scaling_loc_min_dev, h_loc_min_per_ele,
1217 param_bytes, state->stream);
1218 offloadMemcpyAsyncHtoD(state->scaling_loc_max_dev, h_loc_max_per_ele,
1219 param_bytes, state->stream);
1220 offloadMemcpyAsyncHtoD(state->scaling_sigma_dev, h_sigma_per_ele, param_bytes,
1221 state->stream);
1222 offloadMemcpyAsyncHtoD(state->scaling_n_input_nodes_dev,
1223 h_n_input_nodes_per_ele, n_in_bytes, state->stream);
1224
1225 /* Sync before return so the host buffers can be freed by the caller. */
1226 offloadStreamSynchronize(state->stream);
1227
1228 state->scaling_uploaded = 1;
1229 state->scaling_n_ele = n_ele;
1230 state->scaling_n_input_max = n_input_max;
1231 return 0;
1232}
1233
1234/*******************************************************************************
1235 * \brief Once-at-NNP-init upload of the per-(group, element) static
1236 * symmetry-function map tables. The caller packs every radial / angular
1237 * symmetry-function group's symf array into row-major contiguous buffers
1238 * with prefix-sum offset tables (see nnp_gpu_state.h for the layout).
1239 * After this returns, the device copies are resident and
1240 * state->se_static_uploaded is set, so the descriptor launchers read the
1241 * maps directly from state without a per-step upload. Sizes can be zero
1242 * for a kind with no groups (the H2D is skipped but the shape fields are
1243 * still recorded). The stream is synchronized before return so the host
1244 * buffers may be freed. Exported with C linkage for the Fortran driver.
1245 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1246 ******************************************************************************/
1247extern "C" int nnp_gpu_upload_se_static_tables_c(
1248 int n_ele, int max_n_radgrp, int max_n_anggrp, int total_rad_symf,
1249 int total_ang_symf, const int *h_rad_symf_packed,
1250 const int *h_rad_symf_offsets, const int *h_ang_symf_packed,
1251 const int *h_ang_symf_offsets) {
1252 if (n_ele <= 0) {
1253 fprintf(stderr, "ERROR: nnp_gpu_upload_se_static_tables: bad n_ele=%d\n",
1254 n_ele);
1255 return -1;
1256 }
1257 if (max_n_radgrp < 0 || max_n_anggrp < 0 || total_rad_symf < 0 ||
1258 total_ang_symf < 0) {
1259 fprintf(stderr,
1260 "ERROR: nnp_gpu_upload_se_static_tables: negative size "
1261 "(max_n_radgrp=%d max_n_anggrp=%d total_rad_symf=%d "
1262 "total_ang_symf=%d)\n",
1263 max_n_radgrp, max_n_anggrp, total_rad_symf, total_ang_symf);
1264 return -1;
1265 }
1266 nnp_gpu_state_t *state = nnp_gpu_state_get();
1267 if (state == NULL)
1268 return -1;
1269
1270 const size_t n_rad_se = (size_t)max_n_radgrp * (size_t)n_ele;
1271 const size_t n_ang_se = (size_t)max_n_anggrp * (size_t)n_ele;
1272 const size_t rad_off_bytes = sizeof(int) * (n_rad_se + 1);
1273 const size_t ang_off_bytes = sizeof(int) * (n_ang_se + 1);
1274 const size_t rad_pack_bytes = sizeof(int) * (size_t)total_rad_symf;
1275 const size_t ang_pack_bytes = sizeof(int) * (size_t)total_ang_symf;
1276
1277 if (max_n_radgrp > 0) {
1278 if (h_rad_symf_offsets == NULL) {
1279 fprintf(stderr,
1280 "ERROR: nnp_gpu_upload_se_static_tables: "
1281 "h_rad_symf_offsets NULL with max_n_radgrp=%d\n",
1282 max_n_radgrp);
1283 return -1;
1284 }
1285 if (nnp_gpu_pbuf_ensure((void **)&state->se_static_rad_symf_offsets_dev,
1286 &state->se_static_rad_symf_offsets_cap,
1287 rad_off_bytes))
1288 return -1;
1289 offloadMemcpyAsyncHtoD(state->se_static_rad_symf_offsets_dev,
1290 h_rad_symf_offsets, rad_off_bytes, state->stream);
1291 if (total_rad_symf > 0) {
1292 if (h_rad_symf_packed == NULL) {
1293 fprintf(stderr,
1294 "ERROR: nnp_gpu_upload_se_static_tables: "
1295 "h_rad_symf_packed NULL with total_rad_symf=%d\n",
1296 total_rad_symf);
1297 return -1;
1298 }
1299 if (nnp_gpu_pbuf_ensure((void **)&state->se_static_rad_symf_packed_dev,
1300 &state->se_static_rad_symf_packed_cap,
1301 rad_pack_bytes))
1302 return -1;
1303 offloadMemcpyAsyncHtoD(state->se_static_rad_symf_packed_dev,
1304 h_rad_symf_packed, rad_pack_bytes, state->stream);
1305 }
1306 }
1307
1308 if (max_n_anggrp > 0) {
1309 if (h_ang_symf_offsets == NULL) {
1310 fprintf(stderr,
1311 "ERROR: nnp_gpu_upload_se_static_tables: "
1312 "h_ang_symf_offsets NULL with max_n_anggrp=%d\n",
1313 max_n_anggrp);
1314 return -1;
1315 }
1316 if (nnp_gpu_pbuf_ensure((void **)&state->se_static_ang_symf_offsets_dev,
1317 &state->se_static_ang_symf_offsets_cap,
1318 ang_off_bytes))
1319 return -1;
1320 offloadMemcpyAsyncHtoD(state->se_static_ang_symf_offsets_dev,
1321 h_ang_symf_offsets, ang_off_bytes, state->stream);
1322 if (total_ang_symf > 0) {
1323 if (h_ang_symf_packed == NULL) {
1324 fprintf(stderr,
1325 "ERROR: nnp_gpu_upload_se_static_tables: "
1326 "h_ang_symf_packed NULL with total_ang_symf=%d\n",
1327 total_ang_symf);
1328 return -1;
1329 }
1330 if (nnp_gpu_pbuf_ensure((void **)&state->se_static_ang_symf_packed_dev,
1331 &state->se_static_ang_symf_packed_cap,
1332 ang_pack_bytes))
1333 return -1;
1334 offloadMemcpyAsyncHtoD(state->se_static_ang_symf_packed_dev,
1335 h_ang_symf_packed, ang_pack_bytes, state->stream);
1336 }
1337 }
1338
1339 offloadStreamSynchronize(state->stream);
1340
1341 state->se_static_uploaded = 1;
1342 state->se_static_n_ele = n_ele;
1343 state->se_static_max_n_radgrp = max_n_radgrp;
1344 state->se_static_max_n_anggrp = max_n_anggrp;
1345 state->se_static_total_rad_symf = total_rad_symf;
1346 state->se_static_total_ang_symf = total_ang_symf;
1347 return 0;
1348}
1349
1350/*******************************************************************************
1351 * \brief Scale G_dev into G_dev_scaled with nnp_scale_G_dev_kernel (five
1352 * modes). Requires the scaling tables (nnp_gpu_upload_scaling_tables_c)
1353 * and G_dev (nnp_gpu_begin_descriptor_step) to be in place. The rank's
1354 * 0-based element index is static during MD, so it is uploaded once and
1355 * shared with the network launcher. Also zeroes the per-step
1356 * extrapolation flag the kernel may set and queues its copy back to the
1357 * host. Everything is stream-ordered, so no synchronization is done
1358 * here; the flag is read through nnp_gpu_extrapolation_flag_c after the
1359 * step's final synchronization. Exported with C linkage for the Fortran
1360 * driver.
1361 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1362 ******************************************************************************/
1363extern "C" int nnp_gpu_scale_G_dev_c(int n_atoms, int n_input_max, int n_ele,
1364 int scaling_mode, double scmin,
1365 double scmax,
1366 const int *h_ele_ind_for_atoms) {
1367 if (n_atoms <= 0 || n_input_max <= 0 || n_ele <= 0) {
1368 fprintf(stderr,
1369 "ERROR: nnp_gpu_scale_G_dev: bad sizes n_atoms=%d "
1370 "n_input_max=%d n_ele=%d\n",
1371 n_atoms, n_input_max, n_ele);
1372 return -1;
1373 }
1374 if (scaling_mode < 0 || scaling_mode > 4) {
1375 fprintf(stderr,
1376 "ERROR: nnp_gpu_scale_G_dev: bad scaling_mode=%d (expect 0..4)\n",
1377 scaling_mode);
1378 return -1;
1379 }
1380
1381 nnp_gpu_state_t *state = nnp_gpu_state_get();
1382 if (state == NULL)
1383 return -1;
1384
1385 if (!state->scaling_uploaded) {
1386 fprintf(stderr, "ERROR: nnp_gpu_scale_G_dev: scaling tables not uploaded "
1387 "(nnp_gpu_upload_scaling_tables_c must run at NNP init)\n");
1388 return -1;
1389 }
1390
1391 /* The kernel strides the per-element slabs by n_input_max, so a launch shape
1392 * that disagrees with the upload would read a wrong descriptor row and scale
1393 * silently by the wrong factors. */
1394 if (state->scaling_n_ele != n_ele ||
1395 state->scaling_n_input_max != n_input_max) {
1396 fprintf(stderr,
1397 "ERROR: nnp_gpu_scale_G_dev: scaling tables were uploaded for "
1398 "n_ele=%d n_input_max=%d but the kernel was called with %d/%d\n",
1399 state->scaling_n_ele, state->scaling_n_input_max, n_ele,
1400 n_input_max);
1401 return -1;
1402 }
1403
1404 if (state->G_dev == NULL || state->G_dev_scaled == NULL) {
1405 fprintf(stderr, "ERROR: nnp_gpu_scale_G_dev: G_dev not initialised "
1406 "(call nnp_gpu_begin_descriptor_step first)\n");
1407 return -1;
1408 }
1409
1410 /* ele_ind is static during MD: one upload, shared with the atomwise
1411 * network launcher (both consume the same rank-local 0-based element
1412 * index). */
1413 if (nnp_gpu_step_ele_ind_ensure(state, h_ele_ind_for_atoms, n_atoms))
1414 return -1;
1415
1416 /* Per-step extrapolation flag: re-zeroed here, set by the kernel when a
1417 * descriptor leaves its training range, and copied back stream-ordered so
1418 * the host value is valid after the step's final synchronization. */
1419 if (nnp_gpu_pbuf_ensure((void **)&state->expol_flag_dev,
1420 &state->expol_flag_dev_cap, sizeof(int)))
1421 return -1;
1422 offloadMemsetAsync(state->expol_flag_dev, 0, sizeof(int), state->stream);
1423
1424 {
1425 const int block = 64;
1426 const int grid = (n_atoms + block - 1) / block;
1427 nnp_scale_G_dev_kernel<<<grid, block, 0, state->stream>>>(
1428 n_atoms, n_input_max, scaling_mode, scmin, scmax,
1429 state->step_ele_ind_dev, state->scaling_n_input_nodes_dev,
1430 state->scaling_loc_av_dev, state->scaling_loc_min_dev,
1431 state->scaling_loc_max_dev, state->scaling_sigma_dev, state->G_dev,
1432 state->G_dev_scaled, state->expol_flag_dev);
1433 OFFLOAD_CHECK(offloadPeekAtLastError());
1434 }
1435
1436 offloadMemcpyAsyncDtoH(&state->expol_flag_host, state->expol_flag_dev,
1437 sizeof(int), state->stream);
1438 return 0;
1439}
1440
1441/*******************************************************************************
1442 * \brief Report whether the last scaling pass saw a descriptor outside its
1443 * training range. Valid once the step's device work has been
1444 * synchronized (the force read-back does that); the value belongs to
1445 * the current MD step because the scaling launcher re-zeroes the flag
1446 * every call. Exported with C linkage for the Fortran driver.
1447 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1448 ******************************************************************************/
1449extern "C" int nnp_gpu_extrapolation_flag_c(void) {
1450 nnp_gpu_state_t *state = nnp_gpu_state_get();
1451 if (state == NULL)
1452 return 0;
1453 return state->expol_flag_host;
1454}
1455
1456/*******************************************************************************
1457 * \brief Radial symmetry-function launcher for one (group, element) slot. A
1458 * device-side rearrange materialises the per-call pbuf_* layouts from
1459 * the walk output (state->walk_*_dev) and the once-uploaded per-(group,
1460 * element) tables (state->se_static_*_dev), then
1461 * nnp_radial_sf_force_kernel evaluates the symmetry functions into
1462 * G_dev and the per-slot self/force buffers (per_se_rad_*[se_idx]) the
1463 * radial scatter reads in place. se_idx = s_0based + max_n_radgrp_walk *
1464 * e_0based. No device-to-host copy and no synchronization: the outputs
1465 * stay resident for the scatter stage. Exported with C linkage for the
1466 * Fortran driver.
1467 *
1468 * The spline tables are cached by host pointer via
1469 * nnp_gpu_get_or_upload_spline (model-constant, host-pointer-stable).
1470 * pbuf_izeta is reused here as int scratch for the 1-based symf map (its
1471 * only other use is on the angular path, which does not run radial
1472 * scatter); scatter_neigh_ind holds the 1-based neighbour indices for
1473 * the scatter kernel.
1474 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1475 ******************************************************************************/
1476extern "C" int nnp_gpu_radial_sf_to_G_dev_c(
1477 const int n_atoms_in_e, const int max_n_neigh_per_se, const int n_symf,
1478 const int spline_n, const double spline_dx, const double spline_dx_inv,
1479 const double spline_x_max, const double *h_spline_y,
1480 const double *h_spline_dy, const int s_0based, const int e_0based,
1481 const int max_n_radgrp_walk, const int n_atoms_in_e_max_walk,
1482 const int max_n_neigh_walk, const int istart_1based, const int n_input_max,
1483 const int se_idx) {
1484 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_neigh_per_se <= 0 ||
1485 spline_n < 2 || n_input_max <= 0) {
1486 fprintf(stderr, "ERROR: nnp_gpu_radial_sf_to_G_dev: bad sizes\n");
1487 return -1;
1488 }
1489 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
1490 fprintf(stderr,
1491 "ERROR: nnp_gpu_radial_sf_to_G_dev: se_idx=%d out of range\n",
1492 se_idx);
1493 return -1;
1494 }
1495 nnp_gpu_state_t *state = nnp_gpu_state_get();
1496 if (state == NULL)
1497 return -1;
1498 if (state->G_dev == NULL || state->coord_dev == NULL ||
1499 state->image_table_dev == NULL) {
1500 fprintf(stderr, "ERROR: nnp_gpu_radial_sf_to_G_dev: G_dev / "
1501 "coord_dev / image_table_dev not initialised\n");
1502 return -1;
1503 }
1504 if (state->walk_n_rad_pa_dev == NULL || state->walk_rad_ind_dev == NULL ||
1505 state->walk_rad_image_idx_dev == NULL ||
1506 state->walk_atoms_global_dev == NULL) {
1507 fprintf(stderr, "ERROR: nnp_gpu_radial_sf_to_G_dev: walk buffers "
1508 "not initialised (call nnp_gpu_cell_list_walk_c first)\n");
1509 return -1;
1510 }
1511 if (!state->se_static_uploaded) {
1512 fprintf(stderr, "ERROR: nnp_gpu_radial_sf_to_G_dev: se_static "
1513 "tables not uploaded (call nnp_gpu_upload_se_static_"
1514 "tables_c at NNP init)\n");
1515 return -1;
1516 }
1517 /* se_idx_in_kind below strides by max_n_radgrp_walk, so a stride other than
1518 * the one the prefix-sum offsets were built with would select the wrong
1519 * symmetry-function row. */
1520 if (state->se_static_max_n_radgrp != max_n_radgrp_walk) {
1521 fprintf(stderr,
1522 "ERROR: nnp_gpu_radial_sf_to_G_dev: se_static tables were "
1523 "uploaded with max_n_radgrp=%d but the kernel was called with %d\n",
1524 state->se_static_max_n_radgrp, max_n_radgrp_walk);
1525 return -1;
1526 }
1527
1528 const size_t neigh_count = (size_t)max_n_neigh_per_se * (size_t)n_atoms_in_e;
1529 const size_t self_doubles = (size_t)3 * (size_t)n_symf * (size_t)n_atoms_in_e;
1530 const size_t force_doubles = (size_t)3 * (size_t)n_symf * neigh_count;
1531
1532 double *d_spline_y = nnp_gpu_get_or_upload_spline(
1533 h_spline_y, (size_t)n_symf * (size_t)spline_n);
1534 double *d_spline_dy = nnp_gpu_get_or_upload_spline(
1535 h_spline_dy, (size_t)n_symf * (size_t)spline_n);
1536 if (d_spline_y == NULL || d_spline_dy == NULL)
1537 return -1;
1538
1539 /* Per-(group, element) outputs the downstream scatter reads. */
1540 if (nnp_gpu_pbuf_ensure((void **)&state->per_se_rad_self[se_idx],
1541 &state->per_se_rad_self_cap[se_idx],
1542 sizeof(double) * self_doubles))
1543 return -1;
1544 if (nnp_gpu_pbuf_ensure((void **)&state->per_se_rad_force[se_idx],
1545 &state->per_se_rad_force_cap[se_idx],
1546 sizeof(double) * force_doubles))
1547 return -1;
1548
1549 /* Per-call pbufs, populated by the rearrange kernel. */
1550 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_n_neigh_pa,
1551 &state->pbuf_n_neigh_pa_cap,
1552 sizeof(int) * (size_t)n_atoms_in_e))
1553 return -1;
1554 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_symf_map,
1555 &state->pbuf_symf_map_cap,
1556 sizeof(int) * (size_t)n_symf))
1557 return -1;
1558 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atom_map,
1559 &state->pbuf_atom_map_cap,
1560 sizeof(int) * (size_t)n_atoms_in_e))
1561 return -1;
1562 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_rad_ind,
1563 &state->pbuf_rad_ind_cap, sizeof(int) * neigh_count))
1564 return -1;
1565 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_rad_image_idx,
1566 &state->pbuf_rad_image_idx_cap,
1567 sizeof(int) * neigh_count))
1568 return -1;
1569 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atoms_global_rad,
1570 &state->pbuf_atoms_global_rad_cap,
1571 sizeof(int) * (size_t)n_atoms_in_e))
1572 return -1;
1573 /* scatter_neigh_ind doubles as the 1-based output for the scatter kernel,
1574 * filled here so scatter does not re-rearrange. */
1575 if (nnp_gpu_pbuf_ensure((void **)&state->scatter_neigh_ind,
1576 &state->scatter_neigh_ind_cap,
1577 sizeof(int) * neigh_count))
1578 return -1;
1579 /* Side scratch: 1-based symf_map for the scatter kernel. pbuf_izeta is
1580 * reused as a small int scratch buffer here (its only other use is on the
1581 * angular path, which does not run radial scatter). */
1582 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_izeta, &state->pbuf_izeta_cap,
1583 sizeof(int) * (size_t)n_symf))
1584 return -1;
1585
1586 {
1587 const int block = 32;
1588 const int j_blocks = (max_n_neigh_per_se + block - 1) / block;
1589 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1590 dim3 blk(block, 1, 1);
1591 nnp_pack_walk_to_pbuf_radial_kernel<<<grid, blk, 0, state->stream>>>(
1592 n_atoms_in_e, max_n_neigh_per_se, s_0based, e_0based, max_n_radgrp_walk,
1593 n_atoms_in_e_max_walk, max_n_neigh_walk, istart_1based,
1594 state->walk_n_rad_pa_dev, state->walk_rad_ind_dev,
1595 state->walk_rad_image_idx_dev, state->walk_atoms_global_dev,
1596 state->pbuf_n_neigh_pa, state->pbuf_rad_ind, state->pbuf_rad_image_idx,
1597 state->pbuf_atoms_global_rad, state->pbuf_atom_map,
1598 state->scatter_neigh_ind);
1599 OFFLOAD_CHECK(offloadPeekAtLastError());
1600 }
1601 {
1602 const int block = 32;
1603 const int sf_blocks = (n_symf + block - 1) / block;
1604 dim3 grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
1605 dim3 blk(block, 1, 1);
1606 const int se_idx_in_kind = s_0based + max_n_radgrp_walk * e_0based;
1607 /* Radial: pbuf_symf_map is 0-based with no n_rad offset (m_offset=0). */
1608 nnp_pack_se_static_symf_map_kernel<<<grid, blk, 0, state->stream>>>(
1609 se_idx_in_kind, n_symf, /*m_offset=*/0,
1610 state->se_static_rad_symf_packed_dev,
1611 state->se_static_rad_symf_offsets_dev, state->pbuf_symf_map,
1612 state->pbuf_izeta);
1613 OFFLOAD_CHECK(offloadPeekAtLastError());
1614 }
1615
1616 offloadMemsetAsync(state->per_se_rad_self[se_idx], 0,
1617 sizeof(double) * self_doubles, state->stream);
1618 offloadMemsetAsync(state->per_se_rad_force[se_idx], 0,
1619 sizeof(double) * force_doubles, state->stream);
1620
1621 {
1622 const int block = 64;
1623 dim3 grid(n_atoms_in_e, (n_symf + block - 1) / block, 1);
1624 dim3 blk(block, 1, 1);
1625 nnp_radial_sf_force_kernel<<<grid, blk, 0, state->stream>>>(
1626 n_atoms_in_e, n_symf, max_n_neigh_per_se, spline_n, spline_dx,
1627 spline_dx_inv, spline_x_max, d_spline_y, d_spline_dy,
1628 state->pbuf_n_neigh_pa, state->coord_dev, state->image_table_dev,
1629 state->pbuf_rad_ind, state->pbuf_rad_image_idx,
1630 state->pbuf_atoms_global_rad, state->per_se_rad_self[se_idx],
1631 state->per_se_rad_force[se_idx], state->G_dev, state->pbuf_symf_map,
1632 state->pbuf_atom_map, n_input_max);
1633 }
1634 OFFLOAD_CHECK(offloadPeekAtLastError());
1635 /* No sync, no D2H: the scatter reads per_se_rad_*[se_idx] +
1636 * scatter_neigh_ind + pbuf_izeta (1-based symf) in place. */
1637 return 0;
1638}
1639
1640/*******************************************************************************
1641 * \brief Angular symmetry-function launcher for one (group, element) slot.
1642 * Two rearrange kernels materialise the per-atom and per-neighbour
1643 * pbuf_* layouts from the walk output (state->walk_ang*_dev) and the
1644 * once-uploaded per-(group, element) tables
1645 * (state->se_static_ang_*_dev). On the single-precision path with FP32
1646 * geometry, a displacement-precompute kernel fills the FP32 rvect
1647 * buffers. Then one angular kernel evaluates the symmetry functions into
1648 * G_dev and the per-slot self/jj/kk buffers (per_se_ang_*[se_idx]) the
1649 * angular scatter reads in place. se_idx = s_0based + max_n_anggrp_walk
1650 ** e_0based.
1651 *
1652 * Kernel selection follows nnp_gpu_precision_fp32(): the occupancy
1653 * variant for the single-precision path when the occupancy knob is on
1654 * and its shared budget fits the device opt-in cap; otherwise the
1655 * default <float> kernel for single precision; otherwise the <double>
1656 * kernel. The K3 fused shared-memory kk path is used when its shared
1657 * budget fits, in both precisions. The dynamic-shared opt-in for the
1658 * kernels is requested once (floored to 48 KB on any query failure).
1659 *
1660 * The angular parameter tables pack_lam/zeta/eta/prefzeta are cached by
1661 * host pointer via nnp_gpu_get_or_upload_spline (model-constant,
1662 * host-pointer-stable). pack_izeta / pack_use_int_zeta are keyed by
1663 * se_idx in ang_izeta_dev/ang_uiz_dev instead, because the Fortran
1664 * packing buffers are reallocated every step, so a host-pointer cache
1665 * would alias a recycled address; the per-slot upload is guarded by
1666 * ang_zeta_n[se_idx]. scatter_neigh_ind is reused here as int scratch
1667 * for the 1-based symf map (the angular symmetry-function kernel does
1668 * not read it). Exported with C linkage for the Fortran driver.
1669 * \author Dhruv Sharma (ds2173@cam.ac.uk)
1670 ******************************************************************************/
1671extern "C" int nnp_gpu_angular_sf_to_G_dev_c(
1672 const int homo, const int cut_type, const int max_n_ang1_per_se,
1673 const int max_n_ang2_per_se, const int max_n_kk_per_se, const int n_symf,
1674 const int n_atoms_in_e, const double cutoff_s, const double cutoff_sqr,
1675 const double *h_pack_lam, const double *h_pack_zeta,
1676 const double *h_pack_eta, const double *h_pack_prefzeta,
1677 const int *h_pack_izeta, const char *h_pack_use_int_zeta,
1678 const int s_0based, const int e_0based, const int max_n_anggrp_walk,
1679 const int n_atoms_in_e_max_walk, const int max_n_neigh_ang1_walk,
1680 const int max_n_neigh_ang2_walk, const int istart_1based,
1681 const int n_rad_for_e, const int n_input_max, const int se_idx) {
1682 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_ang1_per_se <= 0 ||
1683 n_input_max <= 0) {
1684 fprintf(stderr, "ERROR: nnp_gpu_angular_sf_to_G_dev: bad sizes\n");
1685 return -1;
1686 }
1687 if (!homo && max_n_ang2_per_se <= 0) {
1688 fprintf(stderr, "ERROR: nnp_gpu_angular_sf_to_G_dev: hetero needs "
1689 "max_n_ang2_per_se>0\n");
1690 return -1;
1691 }
1692 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
1693 fprintf(stderr,
1694 "ERROR: nnp_gpu_angular_sf_to_G_dev: se_idx=%d out of range\n",
1695 se_idx);
1696 return -1;
1697 }
1698 /* max_n_kk_per_se sizes the per-warp kk slice in shared memory, and the
1699 * kernels index it with the k-side neighbour count, which is bounded by
1700 * max_n_ang1_per_se (homonuclear) or max_n_ang2_per_se (heteronuclear). A
1701 * caller that passed a smaller value would write past this warp's slice into
1702 * the neighbouring warp's accumulator, so check it here rather than trust the
1703 * convention. */
1704 {
1705 const int k_bound = homo ? max_n_ang1_per_se : max_n_ang2_per_se;
1706 if (max_n_kk_per_se < k_bound) {
1707 fprintf(stderr,
1708 "ERROR: nnp_gpu_angular_sf_to_G_dev: max_n_kk_per_se=%d is "
1709 "below the k-side neighbour bound %d\n",
1710 max_n_kk_per_se, k_bound);
1711 return -1;
1712 }
1713 }
1714 nnp_gpu_state_t *state = nnp_gpu_state_get();
1715 if (state == NULL)
1716 return -1;
1717 if (state->G_dev == NULL || state->coord_dev == NULL ||
1718 state->image_table_dev == NULL) {
1719 fprintf(stderr, "ERROR: nnp_gpu_angular_sf_to_G_dev: "
1720 "G_dev / coord_dev / image_table_dev not initialised\n");
1721 return -1;
1722 }
1723 if (state->walk_n_ang1_pa_dev == NULL || state->walk_ang1_ind_dev == NULL ||
1724 state->walk_ang1_image_idx_dev == NULL ||
1725 state->walk_atoms_global_dev == NULL) {
1726 fprintf(stderr, "ERROR: nnp_gpu_angular_sf_to_G_dev: walk buffers "
1727 "not initialised (call nnp_gpu_cell_list_walk_c first)\n");
1728 return -1;
1729 }
1730 if (!homo &&
1731 (state->walk_n_ang2_pa_dev == NULL || state->walk_ang2_ind_dev == NULL ||
1732 state->walk_ang2_image_idx_dev == NULL)) {
1733 fprintf(stderr, "ERROR: nnp_gpu_angular_sf_to_G_dev: hetero needs "
1734 "walk ang2 buffers initialised\n");
1735 return -1;
1736 }
1737 if (!state->se_static_uploaded) {
1738 fprintf(stderr, "ERROR: nnp_gpu_angular_sf_to_G_dev: se_static "
1739 "tables not uploaded\n");
1740 return -1;
1741 }
1742 /* As in the radial launcher: se_idx_in_kind strides by max_n_anggrp_walk, so
1743 * a mismatched stride would select the wrong symmetry-function row. */
1744 if (state->se_static_max_n_anggrp != max_n_anggrp_walk) {
1745 fprintf(stderr,
1746 "ERROR: nnp_gpu_angular_sf_to_G_dev: se_static tables were "
1747 "uploaded with max_n_anggrp=%d but the kernel was called with %d\n",
1748 state->se_static_max_n_anggrp, max_n_anggrp_walk);
1749 return -1;
1750 }
1751
1752 const size_t neigh1_count = (size_t)max_n_ang1_per_se * (size_t)n_atoms_in_e;
1753 const size_t neigh2_count =
1754 (size_t)(max_n_ang2_per_se > 0 ? max_n_ang2_per_se : 1) *
1755 (size_t)n_atoms_in_e;
1756 const size_t self_doubles = (size_t)3 * (size_t)n_symf * (size_t)n_atoms_in_e;
1757 const size_t jj_doubles = (size_t)3 * (size_t)n_symf * neigh1_count;
1758 const size_t kk_doubles = (size_t)3 * (size_t)n_symf *
1759 (size_t)max_n_kk_per_se * (size_t)n_atoms_in_e;
1760
1761 const size_t n_pack = (size_t)n_symf;
1762 double *d_pack_lam = nnp_gpu_get_or_upload_spline(h_pack_lam, n_pack);
1763 double *d_pack_zeta = nnp_gpu_get_or_upload_spline(h_pack_zeta, n_pack);
1764 double *d_pack_eta = nnp_gpu_get_or_upload_spline(h_pack_eta, n_pack);
1765 double *d_pack_prefzeta =
1766 nnp_gpu_get_or_upload_spline(h_pack_prefzeta, n_pack);
1767 if (d_pack_lam == NULL || d_pack_zeta == NULL || d_pack_eta == NULL ||
1768 d_pack_prefzeta == NULL)
1769 return -1;
1770
1771 /* Per-(group, element) outputs. */
1772 if (nnp_gpu_pbuf_ensure((void **)&state->per_se_ang_self[se_idx],
1773 &state->per_se_ang_self_cap[se_idx],
1774 sizeof(double) * self_doubles))
1775 return -1;
1776 if (nnp_gpu_pbuf_ensure((void **)&state->per_se_ang_jj[se_idx],
1777 &state->per_se_ang_jj_cap[se_idx],
1778 sizeof(double) * jj_doubles))
1779 return -1;
1780 if (nnp_gpu_pbuf_ensure((void **)&state->per_se_ang_kk[se_idx],
1781 &state->per_se_ang_kk_cap[se_idx],
1782 sizeof(double) * kk_doubles))
1783 return -1;
1784
1785 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_n_ang1_pa,
1786 &state->pbuf_n_ang1_pa_cap,
1787 sizeof(int) * (size_t)n_atoms_in_e))
1788 return -1;
1789 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_n_ang2_pa,
1790 &state->pbuf_n_ang2_pa_cap,
1791 sizeof(int) * (size_t)n_atoms_in_e))
1792 return -1;
1793 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_symf_map,
1794 &state->pbuf_symf_map_cap,
1795 sizeof(int) * (size_t)n_symf))
1796 return -1;
1797 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atom_map,
1798 &state->pbuf_atom_map_cap,
1799 sizeof(int) * (size_t)n_atoms_in_e))
1800 return -1;
1801 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang1_ind,
1802 &state->pbuf_ang1_ind_cap,
1803 sizeof(int) * neigh1_count))
1804 return -1;
1805 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang1_image_idx,
1806 &state->pbuf_ang1_image_idx_cap,
1807 sizeof(int) * neigh1_count))
1808 return -1;
1809 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang2_ind,
1810 &state->pbuf_ang2_ind_cap,
1811 sizeof(int) * neigh2_count))
1812 return -1;
1813 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang2_image_idx,
1814 &state->pbuf_ang2_image_idx_cap,
1815 sizeof(int) * neigh2_count))
1816 return -1;
1817 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atoms_global_ang,
1818 &state->pbuf_atoms_global_ang_cap,
1819 sizeof(int) * (size_t)n_atoms_in_e))
1820 return -1;
1821
1822 /* pack_izeta + pack_use_int_zeta are model constants uploaded once into the
1823 * per-(group, element) slot, guarded by ang_zeta_n[se_idx]. Keyed by se_idx,
1824 * never by host pointer, because the Fortran packing buffers are reallocated
1825 * every MD step (a host-pointer cache would alias a recycled address).
1826 * Synchronous memcpy: the transient host buffer must not be referenced by an
1827 * async op. */
1828 if (state->ang_zeta_n[se_idx] != n_symf) {
1829 if (nnp_gpu_pbuf_ensure((void **)&state->ang_izeta_dev[se_idx],
1830 &state->ang_izeta_cap[se_idx],
1831 sizeof(int) * (size_t)n_symf))
1832 return -1;
1833 if (nnp_gpu_pbuf_ensure((void **)&state->ang_uiz_dev[se_idx],
1834 &state->ang_uiz_cap[se_idx],
1835 sizeof(char) * (size_t)n_symf))
1836 return -1;
1837 offloadMemcpyHtoD(state->ang_izeta_dev[se_idx], h_pack_izeta,
1838 sizeof(int) * (size_t)n_symf);
1839 offloadMemcpyHtoD(state->ang_uiz_dev[se_idx], h_pack_use_int_zeta,
1840 sizeof(char) * (size_t)n_symf);
1841 state->ang_zeta_n[se_idx] = n_symf;
1842 }
1843
1844 /* Per-atom rearrange. */
1845 {
1846 const int block = 32;
1847 dim3 grid((n_atoms_in_e + block - 1) / block, 1, 1);
1848 dim3 blk(block, 1, 1);
1849 nnp_pack_walk_to_pbuf_angular_per_atom_kernel<<<grid, blk, 0,
1850 state->stream>>>(
1851 n_atoms_in_e, homo, s_0based, e_0based, max_n_anggrp_walk,
1852 n_atoms_in_e_max_walk, istart_1based, state->walk_n_ang1_pa_dev,
1853 state->walk_n_ang2_pa_dev, state->walk_atoms_global_dev,
1854 state->pbuf_n_ang1_pa, state->pbuf_n_ang2_pa,
1855 state->pbuf_atoms_global_ang, state->pbuf_atom_map);
1856 OFFLOAD_CHECK(offloadPeekAtLastError());
1857 }
1858
1859 /* Per-(atom, j) rearrange: ang1 always; ang2 only for hetero. ind_offset = 0
1860 * (the symmetry-function kernel reads 0-based). */
1861 {
1862 const int block = 32;
1863 const int j_blocks = (max_n_ang1_per_se + block - 1) / block;
1864 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1865 dim3 blk(block, 1, 1);
1866 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<grid, blk, 0,
1867 state->stream>>>(
1868 n_atoms_in_e, max_n_ang1_per_se, s_0based, e_0based, max_n_anggrp_walk,
1869 n_atoms_in_e_max_walk, max_n_neigh_ang1_walk,
1870 /*ind_offset=*/0, state->walk_n_ang1_pa_dev, state->walk_ang1_ind_dev,
1871 state->walk_ang1_image_idx_dev, state->pbuf_ang1_ind,
1872 state->pbuf_ang1_image_idx);
1873 OFFLOAD_CHECK(offloadPeekAtLastError());
1874 }
1875 if (!homo && max_n_ang2_per_se > 0) {
1876 const int block = 32;
1877 const int j_blocks = (max_n_ang2_per_se + block - 1) / block;
1878 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1879 dim3 blk(block, 1, 1);
1880 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<grid, blk, 0,
1881 state->stream>>>(
1882 n_atoms_in_e, max_n_ang2_per_se, s_0based, e_0based, max_n_anggrp_walk,
1883 n_atoms_in_e_max_walk, max_n_neigh_ang2_walk,
1884 /*ind_offset=*/0, state->walk_n_ang2_pa_dev, state->walk_ang2_ind_dev,
1885 state->walk_ang2_image_idx_dev, state->pbuf_ang2_ind,
1886 state->pbuf_ang2_image_idx);
1887 OFFLOAD_CHECK(offloadPeekAtLastError());
1888 }
1889
1890 /* FP32-geometry path (single precision only): precompute the per-(atom, j)
1891 * displacement rvect = center - neighbour_image in FP64 and store FP32, so
1892 * the angular kernel's O(n^2) triplet loop reads it instead of recomputing
1893 * the geometry in FP64. The pack kernels above have filled pbuf_ang1_ind /
1894 * pbuf_ang1_image_idx (+ ang2) and pbuf_atoms_global_ang; stream ordering
1895 * makes them visible here. Skipped on the FP64 path, which never reads the
1896 * disp buffers. */
1897 if (nnp_gpu_precision_fp32()) {
1898 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang1_disp,
1899 &state->pbuf_ang1_disp_cap,
1900 sizeof(float) * 3 * neigh1_count))
1901 return -1;
1902 {
1903 const int block = 256;
1904 const int j_blocks = (max_n_ang1_per_se + block - 1) / block;
1905 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1906 dim3 blk(block, 1, 1);
1907 nnp_pack_angular_disp_kernel<<<grid, blk, 0, state->stream>>>(
1908 n_atoms_in_e, max_n_ang1_per_se, state->pbuf_n_ang1_pa,
1909 state->pbuf_ang1_ind, state->pbuf_ang1_image_idx,
1910 state->pbuf_atoms_global_ang, state->coord_dev,
1911 state->image_table_dev, state->pbuf_ang1_disp);
1912 OFFLOAD_CHECK(offloadPeekAtLastError());
1913 }
1914 if (!homo && max_n_ang2_per_se > 0) {
1915 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang2_disp,
1916 &state->pbuf_ang2_disp_cap,
1917 sizeof(float) * 3 * neigh2_count))
1918 return -1;
1919 const int block = 256;
1920 const int j_blocks = (max_n_ang2_per_se + block - 1) / block;
1921 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1922 dim3 blk(block, 1, 1);
1923 nnp_pack_angular_disp_kernel<<<grid, blk, 0, state->stream>>>(
1924 n_atoms_in_e, max_n_ang2_per_se, state->pbuf_n_ang2_pa,
1925 state->pbuf_ang2_ind, state->pbuf_ang2_image_idx,
1926 state->pbuf_atoms_global_ang, state->coord_dev,
1927 state->image_table_dev, state->pbuf_ang2_disp);
1928 OFFLOAD_CHECK(offloadPeekAtLastError());
1929 }
1930 }
1931
1932 /* Angular symf_map: pbuf_symf_map is 0-based with the n_rad(e) offset baked
1933 * in (angular descriptors live after the radial block in G_dev's per-atom
1934 * row and the angular kernel takes no separate offset parameter). Reuse the
1935 * shared helper kernel; its 1-based scratch output goes to scatter_neigh_ind
1936 * (int, sized here; not read by the angular symmetry-function kernel). */
1937 if (nnp_gpu_pbuf_ensure((void **)&state->scatter_neigh_ind,
1938 &state->scatter_neigh_ind_cap,
1939 sizeof(int) * (size_t)n_symf))
1940 return -1;
1941 {
1942 const int block = 32;
1943 const int sf_blocks = (n_symf + block - 1) / block;
1944 dim3 grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
1945 dim3 blk(block, 1, 1);
1946 const int se_idx_in_kind = s_0based + max_n_anggrp_walk * e_0based;
1947 nnp_pack_se_static_symf_map_kernel<<<grid, blk, 0, state->stream>>>(
1948 se_idx_in_kind, n_symf, /*m_offset=*/n_rad_for_e,
1949 state->se_static_ang_symf_packed_dev,
1950 state->se_static_ang_symf_offsets_dev, state->pbuf_symf_map,
1951 state->scatter_neigh_ind);
1952 OFFLOAD_CHECK(offloadPeekAtLastError());
1953 }
1954
1955 offloadMemsetAsync(state->per_se_ang_self[se_idx], 0,
1956 sizeof(double) * self_doubles, state->stream);
1957 offloadMemsetAsync(state->per_se_ang_jj[se_idx], 0,
1958 sizeof(double) * jj_doubles, state->stream);
1959 offloadMemsetAsync(state->per_se_ang_kk[se_idx], 0,
1960 sizeof(double) * kk_doubles, state->stream);
1961
1962 {
1963 /* Local block scoping the d_ang2_* aliases and the K3 shared-memory
1964 * bookkeeping to the kernel launch. */
1965 int *d_ang2_ind_use = homo ? state->pbuf_ang1_ind : state->pbuf_ang2_ind;
1966 int *d_ang2_img_use =
1967 homo ? state->pbuf_ang1_image_idx : state->pbuf_ang2_image_idx;
1968 /* Same homo/hetero aliasing for the FP32-geom disp: homo's k-list reuses
1969 * ang1_disp, hetero reads ang2_disp. Inert (and may be NULL) unless the
1970 * FP32-geometry path filled the buffers on a single-precision run. */
1971 float *d_ang2_disp_use =
1972 homo ? state->pbuf_ang1_disp : state->pbuf_ang2_disp;
1973 const int lanes = 32;
1974 int sf_per_block = n_symf < 32 ? n_symf : 32;
1975 if (sf_per_block < 1)
1976 sf_per_block = 1;
1977 dim3 grid(n_atoms_in_e, (n_symf + sf_per_block - 1) / sf_per_block, 1);
1978 dim3 blk(lanes, sf_per_block, 1);
1979 /* The kk slice is allocated in the kernel's own compute type, so size it
1980 * that way: charging FP64 bytes for the <float> instantiation would halve
1981 * occupancy on the MIXED path and, worse, push k3_fused and ang_occ below
1982 * their caps at half the neighbour count they can actually handle. */
1983 const size_t ct_bytes =
1984 nnp_gpu_precision_fp32() ? sizeof(float) : sizeof(double);
1985 const size_t k3_smem =
1986 (size_t)3 * (size_t)max_n_kk_per_se * ct_bytes * (size_t)blk.y;
1987 /* K3 carveout: opt the fused kernels into the device's full dynamic
1988 * shared-memory budget once, so the single-pass path survives large
1989 * neighbour counts instead of falling back to the two-phase recompute at
1990 * the 48 KB default cap. On any query or attribute failure the cap floors
1991 * to 48 KB, reproducing the default behaviour.
1992 * Held in a process-lifetime static: the attribute is per device, and the
1993 * device a process uses is chosen once at CP2K startup and never changed
1994 * afterwards, so one query answers for the whole run. */
1995 static size_t k3_cap = 0;
1996 if (k3_cap == 0) {
1997 int optin = 48 * 1024;
1998#if defined(__OFFLOAD_CUDA)
1999 /* The dynamic-shared opt-in above 48 KB is CUDA-specific; AMD's 64 KB LDS
2000 * serves the default allocation without an opt-in, so this whole query is
2001 * skipped on HIP and k3_cap stays at the 48 KB floor. */
2002 int dev = 0;
2003 const cudaError_t e_dev = cudaGetDevice(&dev);
2004 const cudaError_t e_att = cudaDeviceGetAttribute(
2005 &optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
2006 if (e_dev != cudaSuccess || e_att != cudaSuccess || optin < 48 * 1024)
2007 optin = 48 * 1024;
2008 /* Every kernel that can be launched with the carveout has to be opted in,
2009 * including the occupancy variant, whose dynamic region is the geometry
2010 * cache plus kk. If any opt-in is refused, fall back to the 48 KB floor
2011 * for all of them rather than clear the error and launch against a cap
2012 * the device did not grant. */
2013 if (optin > 48 * 1024) {
2014 const cudaError_t e1 = cudaFuncSetAttribute(
2015 (const void *)nnp_angular_sf_force_kernel<float>,
2016 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2017 const cudaError_t e2 = cudaFuncSetAttribute(
2018 (const void *)nnp_angular_sf_force_kernel<double>,
2019 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2020 const cudaError_t e3 = cudaFuncSetAttribute(
2021 (const void *)nnp_angular_sf_force_kernel_occ<float>,
2022 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2023 const cudaError_t e4 = cudaFuncSetAttribute(
2024 (const void *)nnp_angular_sf_force_kernel_occ<double>,
2025 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2026 if (e1 != cudaSuccess || e2 != cudaSuccess || e3 != cudaSuccess ||
2027 e4 != cudaSuccess)
2028 optin = 48 * 1024;
2029 (void)cudaGetLastError();
2030 }
2031#endif
2032 k3_cap = (size_t)optin;
2033 }
2034 /* The fused and two-phase paths accumulate the k-side force in different
2035 * orders, so they do not agree to the last bit. Decide between them from
2036 * the walk's allocated list width, not from this step's neighbour maximum:
2037 * the width is what the walk buffers are sized for and only moves when an
2038 * overflow grows them, whereas the per-step maximum tracks density
2039 * fluctuations and would move a slot between the two paths from one MD step
2040 * to the next. The allocation below still uses the (smaller or equal)
2041 * per-step size, so a fused launch never exceeds the budget it was cleared
2042 * for. */
2043 const size_t k3_decide_smem =
2044 (size_t)3 *
2045 (size_t)(homo ? max_n_neigh_ang1_walk : max_n_neigh_ang2_walk) *
2046 ct_bytes * (size_t)blk.y;
2047 const int k3_fused =
2048 (max_n_kk_per_se > 0 && k3_decide_smem <= k3_cap) ? 1 : 0;
2049 /* Occupancy decision, for both precisions. The occ kernel's dynamic region
2050 * is the geometry cache (GEOM_SLOTS=6 CT per lane, 32 lanes per warp) plus
2051 * the same kk slice when fused, in the kernel's own compute type. It must
2052 * fit the same opt-in cap; if it would overflow, fall back to the default
2053 * kernel so the launch can never exceed the device budget. Decided on the
2054 * same step-invariant width as k3_fused, for the same reason.
2055 *
2056 * The launch-bounds cap trades registers for occupancy -- on the FP64 path
2057 * it takes the angular kernel from 152 registers with a 48 B spill to 64
2058 * registers with 264 B -- and the occupancy wins by a wide margin. A100,
2059 * medians of three interleaved repeats, descriptor stage: 0.753 -> 0.511 s
2060 * at 12,288 atoms and 2.938 -> 1.968 s at 49,152, i.e. a third off, with
2061 * the whole device force evaluation 21% faster. The staged values make the
2062 * round trip CT -> shared -> CT, so results are bit-identical: step-zero
2063 * energies agree to every printed digit. */
2064 const size_t occ_geom_smem = (size_t)6 * 32 * (size_t)blk.y * ct_bytes;
2065 const size_t occ_kk_smem =
2066 (size_t)3 * (size_t)max_n_kk_per_se * (size_t)blk.y * ct_bytes;
2067 const size_t occ_decide_smem =
2068 occ_geom_smem + (k3_fused ? k3_decide_smem : 0);
2069 const size_t occ_smem = occ_geom_smem + (k3_fused ? occ_kk_smem : 0);
2070 const int ang_occ = (occ_decide_smem <= k3_cap) ? 1 : 0;
2071 /* Precision dispatch: the occupancy shell when the shared budget allows,
2072 * else the default register-resident kernel. The per-(group, element)
2073 * device-resident argument list is shared across all four launches. */
2074#define NNP_ANG_ARGS \
2075 homo, cut_type, max_n_ang1_per_se, max_n_ang2_per_se, max_n_kk_per_se, \
2076 n_symf, n_atoms_in_e, cutoff_s, cutoff_sqr, state->pbuf_n_ang1_pa, \
2077 state->pbuf_n_ang2_pa, state->coord_dev, state->image_table_dev, \
2078 state->pbuf_ang1_ind, d_ang2_ind_use, state->pbuf_ang1_image_idx, \
2079 d_ang2_img_use, state->pbuf_atoms_global_ang, state->pbuf_ang1_disp, \
2080 d_ang2_disp_use, d_pack_lam, d_pack_zeta, state->ang_izeta_dev[se_idx], \
2081 state->ang_uiz_dev[se_idx], d_pack_eta, d_pack_prefzeta, \
2082 state->per_se_ang_self[se_idx], state->per_se_ang_jj[se_idx], \
2083 state->per_se_ang_kk[se_idx], state->G_dev, state->pbuf_symf_map, \
2084 state->pbuf_atom_map, n_input_max, k3_fused
2085 if (nnp_gpu_precision_fp32()) {
2086 if (ang_occ)
2087 nnp_angular_sf_force_kernel_occ<float>
2088 <<<grid, blk, occ_smem, state->stream>>>(NNP_ANG_ARGS);
2089 else
2090 nnp_angular_sf_force_kernel<float>
2091 <<<grid, blk, k3_fused ? k3_smem : 0, state->stream>>>(
2092 NNP_ANG_ARGS);
2093 } else {
2094 if (ang_occ)
2095 nnp_angular_sf_force_kernel_occ<double>
2096 <<<grid, blk, occ_smem, state->stream>>>(NNP_ANG_ARGS);
2097 else
2098 nnp_angular_sf_force_kernel<double>
2099 <<<grid, blk, k3_fused ? k3_smem : 0, state->stream>>>(
2100 NNP_ANG_ARGS);
2101 }
2102#undef NNP_ANG_ARGS
2103 }
2104 OFFLOAD_CHECK(offloadPeekAtLastError());
2105 return 0;
2106}
2107
2108#endif /* defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP) */
static GRID_HOST_DEVICE int idx(const orbital a)
Return coset index of given orbital angular momentum.
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int const double GRID_CONST_WHEN_COLLOCATE double * cx
real(dp), dimension(3) c
Definition atom.F:9
__global__ __launch_bounds__(64) void calculate_coefficients(const kernel_params dev_)
Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done i...