(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_scatter.cu
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
10
11#include <stdio.h>
12
13#include "nnp_gpu_internal.h"
14#include "nnp_gpu_state.h"
15
16#if defined(_OMP_H)
17#error "OpenMP should not be used in .cu files to accommodate HIP."
18#endif
19
20#include "nnp_gpu_pack.h"
21
22/*
23 * Force assembly. The descriptor gradient the network leaves on the device
24 * is multiplied by the descriptor derivatives the symmetry-function pass
25 * stored per (group, element), and the products are accumulated into the
26 * force array. When the caller asks for a stress tensor the same kernels
27 * accumulate the analytic virial alongside the forces.
28 */
29
30/*******************************************************************************
31 * \brief Descriptor-derivative to force scale factor for one descriptor row.
32 * Reproduces the host scaling applied when the network's descriptor
33 * gradient is turned into a force contribution. Scaling modes 1 and 3
34 * map the descriptor range [loc_min, loc_max] onto [scmin, scmax];
35 * mode 4 uses the descriptor standard deviation; any other mode leaves
36 * the gradient unscaled.
37 * \author Dhruv Sharma (ds2173@cam.ac.uk)
38 ******************************************************************************/
39static __device__ __forceinline__ double
40nnp_compute_force_scale(int scaling_mode, double scmin, double scmax, int idx,
41 const double *__restrict__ scaling_loc_max,
42 const double *__restrict__ scaling_loc_min,
43 const double *__restrict__ scaling_sigma) {
44 if (scaling_mode == 1 || scaling_mode == 3) {
45 const double diff = scaling_loc_max[idx] - scaling_loc_min[idx];
46 return (scmax - scmin) / diff;
47 }
48 if (scaling_mode == 4) {
49 return (scmax - scmin) / scaling_sigma[idx];
50 }
51 return 1.0;
52}
53
54/*******************************************************************************
55 * \brief Warp-reduce the 9 per-thread analytical-virial components across the
56 * 32-lane scatter block and have lane 0 commit a single atomicAdd set
57 * into the per-committee accumulator virial_dev[9*c + 3*a + b]. Keeps
58 * global atomic traffic at 9 adds per (atom, sf, committee) block
59 * instead of 9 per neighbour. Assumes blockDim.x == warpSize (the
60 * scatter launches use blk(32, 1, 1)); only called from the scatter
61 * launchers, which do.
62 * \author Dhruv Sharma (ds2173@cam.ac.uk)
63 ******************************************************************************/
64__device__ __forceinline__ void
65nnp_virial_warp_flush(const double vir[3][3], double *__restrict__ virial_dev,
66 const int c) {
67#pragma unroll
68 for (int a = 0; a < 3; ++a) {
69#pragma unroll
70 for (int b = 0; b < 3; ++b) {
71 /* Butterfly XOR reduction (backend-portable) leaves the full 32-lane sum
72 * in every lane; lane 0 commits the single atomicAdd. */
73 double v = vir[a][b];
74 for (int mask = 16; mask > 0; mask >>= 1)
75 v += NNP_SHFL_XOR(v, mask);
76 if (threadIdx.x == 0)
77 atomicAdd(&virial_dev[9 * c + 3 * a + b], v);
78 }
79 }
80}
81
82/*******************************************************************************
83 * \brief Radial force-scatter kernel: one block per (atom in element, symmetry
84 * function, committee member). Thread 0 adds the self contribution to
85 * the central atom; the block's threads share the per-neighbour loop,
86 * each atomic-adding one neighbour's contribution per Cartesian
87 * component into the device force accumulator. Sign convention:
88 * f[i] -= de * self, f[k_atom] += de * force. When accumulate_virial is
89 * set, each per-neighbour force contribution also accumulates
90 * -r_ij (outer) f_k into the per-committee analytical virial.
91 * \author Dhruv Sharma (ds2173@cam.ac.uk)
92 ******************************************************************************/
93__global__ void nnp_force_scatter_radial_kernel(
94 const int n_atoms_in_e, const int n_symf, const int max_n_neigh,
95 const int n_committee, const int n_input_max, const int num_atoms_global,
96 const int ele_idx, const int scaling_mode, const int n_rad_offset,
97 const double scmin, const double scmax,
98 const int *__restrict__ symf_map_one_based,
99 const int *__restrict__ atom_map, const int *__restrict__ atoms_global,
100 const int *__restrict__ n_neigh_per_atom,
101 const int *__restrict__ rad_ind_one_based,
102 const double *__restrict__ self_partial,
103 const double *__restrict__ force_rad, const double *__restrict__ dE_dG,
104 const double *__restrict__ scaling_loc_max,
105 const double *__restrict__ scaling_loc_min,
106 const double *__restrict__ scaling_sigma, double *__restrict__ f_dev,
107 /* Virial (read only when accumulate_virial != 0): rad_image_idx pairs
108 * 1:1 with rad_ind_one_based; coord_dev + image_table_dev rebuild the
109 * bond vector r_ij = c_i - (c_k + image), matching the SF kernel. */
110 const int *__restrict__ rad_image_idx, const double *__restrict__ coord_dev,
111 const double *__restrict__ image_table_dev, double *__restrict__ virial_dev,
112 const int accumulate_virial) {
113 const int atom_in_e = blockIdx.x;
114 const int sf = blockIdx.y;
115 const int c = blockIdx.z;
116 if (atom_in_e >= n_atoms_in_e || sf >= n_symf || c >= n_committee)
117 return;
118
119 /* (void) suppress unused-arg warning when n_rad_offset is 0 (radial). */
120 (void)n_rad_offset;
121
122 const int i_local = atom_map[atom_in_e];
123 const int i_global = atoms_global[atom_in_e];
124 /* m is the descriptor row (1-based in symfgrp%symf, 0-based in
125 * dE_dG / scaling tables). Radial: m = symf(sf). */
126 const int m = symf_map_one_based[sf] - 1;
127
128 /* 64-bit offsets on every per-atom-scaled buffer, matching nnp_gpu_pack.h:
129 * these tables exceed 2^31 entries on large ranks. */
130 const double dEdG = dE_dG[m + n_input_max * c +
131 (long long)n_input_max * n_committee * i_local];
132 const int scale_idx = ele_idx * n_input_max + m;
133 const double scale =
134 nnp_compute_force_scale(scaling_mode, scmin, scmax, scale_idx,
135 scaling_loc_max, scaling_loc_min, scaling_sigma);
136 const double de = dEdG * scale;
137
138 /* Self contribution to atom_i: f[i] -= de * self_partial(d, sf, atom_in_e).
139 * r_ii = 0, so the self term makes no virial contribution. */
140 const long long self_off = 3 * (sf + (long long)n_symf * atom_in_e);
141 const long long f_i_off = 3 * (i_global + (long long)num_atoms_global * c);
142 if (threadIdx.x == 0) {
143 atomicAdd(&f_dev[f_i_off + 0], -de * self_partial[self_off + 0]);
144 atomicAdd(&f_dev[f_i_off + 1], -de * self_partial[self_off + 1]);
145 atomicAdd(&f_dev[f_i_off + 2], -de * self_partial[self_off + 2]);
146 }
147
148 /* Per-thread virial accumulator vir[a][b], a = force component,
149 * b = bond component. Folds onto committee_stress(b, a) via the identity
150 * committee_stress(b, a) = -sum r_ij[b] * F_k[a] over neighbours. */
151 double vir[3][3] = {{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}};
152 const double ci_x = accumulate_virial ? coord_dev[3 * i_global + 0] : 0.0;
153 const double ci_y = accumulate_virial ? coord_dev[3 * i_global + 1] : 0.0;
154 const double ci_z = accumulate_virial ? coord_dev[3 * i_global + 2] : 0.0;
155
156 /* Per-neighbour contribution: f[k_atom] += de * force(d, sf, j, atom_in_e).
157 * One global atomicAdd per neighbour per component. */
158 const int n_neigh = n_neigh_per_atom[atom_in_e];
159 for (int j = threadIdx.x; j < n_neigh; j += blockDim.x) {
160 const long long nb_off = j + (long long)max_n_neigh * atom_in_e;
161 const int k_atom = rad_ind_one_based[nb_off] - 1;
162 const long long force_off = 3 * (sf + (long long)n_symf * nb_off);
163 const long long f_k_off = 3 * (k_atom + (long long)num_atoms_global * c);
164 const double fk0 = +de * force_rad[force_off + 0];
165 const double fk1 = +de * force_rad[force_off + 1];
166 const double fk2 = +de * force_rad[force_off + 2];
167 atomicAdd(&f_dev[f_k_off + 0], fk0);
168 atomicAdd(&f_dev[f_k_off + 1], fk1);
169 atomicAdd(&f_dev[f_k_off + 2], fk2);
170 if (accumulate_virial) {
171 const int img = rad_image_idx[nb_off];
172 const double rb0 =
173 ci_x - (coord_dev[3 * k_atom + 0] + image_table_dev[3 * img + 0]);
174 const double rb1 =
175 ci_y - (coord_dev[3 * k_atom + 1] + image_table_dev[3 * img + 1]);
176 const double rb2 =
177 ci_z - (coord_dev[3 * k_atom + 2] + image_table_dev[3 * img + 2]);
178 vir[0][0] += -rb0 * fk0;
179 vir[0][1] += -rb1 * fk0;
180 vir[0][2] += -rb2 * fk0;
181 vir[1][0] += -rb0 * fk1;
182 vir[1][1] += -rb1 * fk1;
183 vir[1][2] += -rb2 * fk1;
184 vir[2][0] += -rb0 * fk2;
185 vir[2][1] += -rb1 * fk2;
186 vir[2][2] += -rb2 * fk2;
187 }
188 }
189
190 if (accumulate_virial)
191 nnp_virial_warp_flush(vir, virial_dev, c);
192}
193
194/*******************************************************************************
195 * \brief Angular force-scatter kernel: the same block layout as the radial
196 * kernel but with two neighbour lists (ang1 and ang2). Thread 0 adds
197 * the self contribution; the threads share the ang1 loop (partials jj)
198 * and the ang2 loop (partials kk), each atomic-adding one contribution
199 * per Cartesian component. In the homonuclear case both loops read the
200 * ang1 index list. Sign convention: f[i] -= de * self,
201 * f[k_atom_j] -= de * jj, f[k_atom_k] -= de * kk.
202 * \author Dhruv Sharma (ds2173@cam.ac.uk)
203 ******************************************************************************/
204__global__ void nnp_force_scatter_angular_kernel(
205 const int n_atoms_in_e, const int n_symf, const int max_n_ang1,
206 const int max_n_kk, const int n_committee, const int n_input_max,
207 const int num_atoms_global, const int ele_idx, const int scaling_mode,
208 const int homo, const int n_rad_offset, const double scmin,
209 const double scmax, const int *__restrict__ symf_map_one_based,
210 const int *__restrict__ atom_map, const int *__restrict__ atoms_global,
211 const int *__restrict__ n_ang1_per_atom,
212 const int *__restrict__ n_ang2_per_atom,
213 const int *__restrict__ ang1_ind_one_based,
214 const int *__restrict__ ang2_ind_one_based,
215 const double *__restrict__ self_partial, const double *__restrict__ jj,
216 const double *__restrict__ kk, const double *__restrict__ dE_dG,
217 const double *__restrict__ scaling_loc_max,
218 const double *__restrict__ scaling_loc_min,
219 const double *__restrict__ scaling_sigma, double *__restrict__ f_dev,
220 /* Virial (read only when accumulate_virial != 0): ang1_image_idx pairs
221 * with ang1_ind (bond r_ij), ang2_image_idx with ang2_ind (bond r_ik);
222 * in homo mode the kk side reads ang1_image_idx to match ang1_ind. */
223 const int *__restrict__ ang1_image_idx,
224 const int *__restrict__ ang2_image_idx,
225 const double *__restrict__ coord_dev,
226 const double *__restrict__ image_table_dev, double *__restrict__ virial_dev,
227 const int accumulate_virial) {
228 const int atom_in_e = blockIdx.x;
229 const int sf = blockIdx.y;
230 const int c = blockIdx.z;
231 if (atom_in_e >= n_atoms_in_e || sf >= n_symf || c >= n_committee)
232 return;
233
234 const int i_local = atom_map[atom_in_e];
235 const int i_global = atoms_global[atom_in_e];
236 /* Angular: m = n_rad(ele) + symf(sf), 1-based; convert to 0-based. */
237 const int m = n_rad_offset + symf_map_one_based[sf] - 1;
238
239 /* 64-bit offsets on every per-atom-scaled buffer, matching nnp_gpu_pack.h:
240 * these tables exceed 2^31 entries on large ranks. */
241 const double dEdG = dE_dG[m + n_input_max * c +
242 (long long)n_input_max * n_committee * i_local];
243 const int scale_idx = ele_idx * n_input_max + m;
244 const double scale =
245 nnp_compute_force_scale(scaling_mode, scmin, scmax, scale_idx,
246 scaling_loc_max, scaling_loc_min, scaling_sigma);
247 const double de = dEdG * scale;
248
249 /* Self term: r_ii = 0, so no virial contribution. */
250 const long long self_off = 3 * (sf + (long long)n_symf * atom_in_e);
251 const long long f_i_off = 3 * (i_global + (long long)num_atoms_global * c);
252 if (threadIdx.x == 0) {
253 atomicAdd(&f_dev[f_i_off + 0], -de * self_partial[self_off + 0]);
254 atomicAdd(&f_dev[f_i_off + 1], -de * self_partial[self_off + 1]);
255 atomicAdd(&f_dev[f_i_off + 2], -de * self_partial[self_off + 2]);
256 }
257
258 /* Per-thread virial accumulator vir[a][b], a = force component,
259 * b = bond component; folds onto committee_stress(b, a). Both the jj
260 * (bond r_ij) and kk (bond r_ik) sides accumulate into it. */
261 double vir[3][3] = {{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}};
262 const double ci_x = accumulate_virial ? coord_dev[3 * i_global + 0] : 0.0;
263 const double ci_y = accumulate_virial ? coord_dev[3 * i_global + 1] : 0.0;
264 const double ci_z = accumulate_virial ? coord_dev[3 * i_global + 2] : 0.0;
265
266 const int n_ang1 = n_ang1_per_atom[atom_in_e];
267 const int n_ang2 = n_ang2_per_atom[atom_in_e];
268 /* One global atomicAdd per neighbour per component. */
269 /* ang1 neighbours j: f[k_atom] -= de * jj(d, sf, j, atom). */
270 for (int j = threadIdx.x; j < n_ang1; j += blockDim.x) {
271 const long long nb_off = j + (long long)max_n_ang1 * atom_in_e;
272 const int k_atom = ang1_ind_one_based[nb_off] - 1;
273 const long long jj_off = 3 * (sf + (long long)n_symf * nb_off);
274 const long long f_k_off = 3 * (k_atom + (long long)num_atoms_global * c);
275 const double fk0 = -de * jj[jj_off + 0];
276 const double fk1 = -de * jj[jj_off + 1];
277 const double fk2 = -de * jj[jj_off + 2];
278 atomicAdd(&f_dev[f_k_off + 0], fk0);
279 atomicAdd(&f_dev[f_k_off + 1], fk1);
280 atomicAdd(&f_dev[f_k_off + 2], fk2);
281 if (accumulate_virial) {
282 const int img = ang1_image_idx[nb_off];
283 const double rb0 =
284 ci_x - (coord_dev[3 * k_atom + 0] + image_table_dev[3 * img + 0]);
285 const double rb1 =
286 ci_y - (coord_dev[3 * k_atom + 1] + image_table_dev[3 * img + 1]);
287 const double rb2 =
288 ci_z - (coord_dev[3 * k_atom + 2] + image_table_dev[3 * img + 2]);
289 vir[0][0] += -rb0 * fk0;
290 vir[0][1] += -rb1 * fk0;
291 vir[0][2] += -rb2 * fk0;
292 vir[1][0] += -rb0 * fk1;
293 vir[1][1] += -rb1 * fk1;
294 vir[1][2] += -rb2 * fk1;
295 vir[2][0] += -rb0 * fk2;
296 vir[2][1] += -rb1 * fk2;
297 vir[2][2] += -rb2 * fk2;
298 }
299 }
300 /* ang2 neighbours k (or ang1 in homo): f[k_atom] -= de * kk(d, sf, k, atom).
301 */
302 for (int kk_j = threadIdx.x; kk_j < n_ang2; kk_j += blockDim.x) {
303 const int *idx_src = homo ? ang1_ind_one_based : ang2_ind_one_based;
304 const long long nb_off = kk_j + (long long)max_n_kk * atom_in_e;
305 const int k_atom = idx_src[nb_off] - 1;
306 const long long kk_off = 3 * (sf + (long long)n_symf * nb_off);
307 const long long f_k_off = 3 * (k_atom + (long long)num_atoms_global * c);
308 const double fk0 = -de * kk[kk_off + 0];
309 const double fk1 = -de * kk[kk_off + 1];
310 const double fk2 = -de * kk[kk_off + 2];
311 atomicAdd(&f_dev[f_k_off + 0], fk0);
312 atomicAdd(&f_dev[f_k_off + 1], fk1);
313 atomicAdd(&f_dev[f_k_off + 2], fk2);
314 if (accumulate_virial) {
315 const int *img_src = homo ? ang1_image_idx : ang2_image_idx;
316 const int img = img_src[nb_off];
317 const double rb0 =
318 ci_x - (coord_dev[3 * k_atom + 0] + image_table_dev[3 * img + 0]);
319 const double rb1 =
320 ci_y - (coord_dev[3 * k_atom + 1] + image_table_dev[3 * img + 1]);
321 const double rb2 =
322 ci_z - (coord_dev[3 * k_atom + 2] + image_table_dev[3 * img + 2]);
323 vir[0][0] += -rb0 * fk0;
324 vir[0][1] += -rb1 * fk0;
325 vir[0][2] += -rb2 * fk0;
326 vir[1][0] += -rb0 * fk1;
327 vir[1][1] += -rb1 * fk1;
328 vir[1][2] += -rb2 * fk1;
329 vir[2][0] += -rb0 * fk2;
330 vir[2][1] += -rb1 * fk2;
331 vir[2][2] += -rb2 * fk2;
332 }
333 }
334
335 if (accumulate_virial)
336 nnp_virial_warp_flush(vir, virial_dev, c);
337}
338
339/*******************************************************************************
340 * \brief Seed the device force accumulator for a step. Ensures f_dev holds
341 * 3 * num_atoms_global * n_committee doubles, then copies the current
342 * host force array onto the device so the scatter kernels accumulate on
343 * top of it, matching the host loop's += semantics.
344 * \author Dhruv Sharma (ds2173@cam.ac.uk)
345 ******************************************************************************/
346extern "C" int nnp_gpu_force_scatter_begin_c(int num_atoms_global,
347 int n_committee,
348 const double *h_force) {
349 if (num_atoms_global <= 0 || n_committee <= 0) {
350 fprintf(stderr,
351 "ERROR: nnp_gpu_force_scatter_begin: bad sizes (num_atoms=%d, "
352 "n_committee=%d)\n",
353 num_atoms_global, n_committee);
354 return -1;
355 }
356 nnp_gpu_state_t *state = nnp_gpu_state_get();
357 if (state == NULL)
358 return -1;
359 const size_t bytes =
360 sizeof(double) * 3 * (size_t)num_atoms_global * (size_t)n_committee;
361 if (nnp_gpu_pbuf_ensure((void **)&state->f_dev, &state->f_dev_cap, bytes))
362 return -1;
363 /* Copy the host nnp%myforce so the scatter kernel accumulates on top,
364 * matching the host loop's += semantics. */
365 offloadMemcpyAsyncHtoD(state->f_dev, h_force, bytes, state->stream);
366 /* Zero the per-committee analytical virial accumulator. The scatter kernels
367 * atomic-add onto it when virial accumulation is requested;
368 * nnp_gpu_force_scatter_virial_d2h_c reads it back. Always maintained (tiny:
369 * 9 * n_committee doubles) so the buffer exists whenever a stress-requesting
370 * step D2Hs it. */
371 const size_t vbytes = sizeof(double) * 9 * (size_t)n_committee;
372 if (nnp_gpu_pbuf_ensure((void **)&state->virial_dev, &state->virial_dev_cap,
373 vbytes))
374 return -1;
375 offloadMemsetAsync(state->virial_dev, 0, vbytes, state->stream);
376 return 0;
377}
378
379/*******************************************************************************
380 * \brief Read the accumulated device force back to the host at the end of a
381 * step. The stream synchronize here is the single per-step
382 * synchronization point: it guarantees every kernel queued this step
383 * has completed before the host reads the result.
384 * \author Dhruv Sharma (ds2173@cam.ac.uk)
385 ******************************************************************************/
386extern "C" int nnp_gpu_force_scatter_end_c(int num_atoms_global,
387 int n_committee, double *h_force) {
388 if (num_atoms_global <= 0 || n_committee <= 0)
389 return -1;
390 nnp_gpu_state_t *state = nnp_gpu_state_get();
391 if (state == NULL || state->f_dev == NULL)
392 return -1;
393 const size_t bytes =
394 sizeof(double) * 3 * (size_t)num_atoms_global * (size_t)n_committee;
395 offloadMemcpyAsyncDtoH(h_force, state->f_dev, bytes, state->stream);
396 offloadStreamSynchronize(state->stream);
397 return 0;
398}
399
400/*******************************************************************************
401 * \brief D2H the per-committee analytical virial accumulator into a host
402 * buffer laid out [9 * n_committee] (virial[9*c + 3*a + b], a = force
403 * component, b = bond component). Call immediately after
404 * nnp_gpu_force_scatter_end_c: its stream synchronize has already
405 * committed every scatter kernel's atomic adds, so a plain synchronous
406 * D2H of this tiny buffer is sufficient.
407 * \author Dhruv Sharma (ds2173@cam.ac.uk)
408 ******************************************************************************/
409extern "C" int nnp_gpu_force_scatter_virial_d2h_c(int n_committee,
410 double *h_virial) {
411 if (n_committee <= 0)
412 return -1;
413 nnp_gpu_state_t *state = nnp_gpu_state_get();
414 if (state == NULL || state->virial_dev == NULL)
415 return -1;
416 const size_t vbytes = sizeof(double) * 9 * (size_t)n_committee;
417 offloadMemcpyDtoH(h_virial, state->virial_dev, vbytes);
418 return 0;
419}
420
421/*******************************************************************************
422 * \brief Radial force-scatter launcher for one (group, element) slot,
423 * companion to the radial symmetry-function launcher. Rebuilds the
424 * per-neighbour index and per-atom mapping tables (the shared index
425 * buffers were overwritten by later descriptor groups, so the
426 * rearrange is repeated here) and the 1-based symmetry-function map,
427 * then launches the radial scatter kernel to accumulate this slot's
428 * force contribution into f_dev.
429 * \author Dhruv Sharma (ds2173@cam.ac.uk)
430 ******************************************************************************/
431extern "C" int nnp_gpu_force_scatter_radial_c(
432 const int n_atoms_in_e, const int n_symf, const int max_n_neigh_per_se,
433 const int n_committee, const int n_input_max, const int num_atoms_global,
434 const int ele_idx, const int scaling_mode, const double scmin,
435 const double scmax, const int s_0based, const int max_n_radgrp_walk,
436 const int n_atoms_in_e_max_walk, const int max_n_neigh_walk,
437 const int istart_1based, const int se_idx, const int accumulate_virial) {
438 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_neigh_per_se <= 0 ||
439 n_committee <= 0 || n_input_max <= 0 || num_atoms_global <= 0)
440 return -1;
441 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
442 fprintf(stderr,
443 "ERROR: nnp_gpu_force_scatter_radial: se_idx=%d out of range\n",
444 se_idx);
445 return -1;
446 }
447 nnp_gpu_state_t *state = nnp_gpu_state_get();
448 if (state == NULL || state->f_dev == NULL)
449 return -1;
450 if (!state->scaling_uploaded)
451 return -1;
452 if (state->per_se_rad_self[se_idx] == NULL ||
453 state->per_se_rad_force[se_idx] == NULL) {
454 fprintf(stderr, "ERROR: nnp_gpu_force_scatter_radial: per_se buffers "
455 "not populated (call the symmetry-function stage first)\n");
456 return -1;
457 }
458 if (state->walk_n_rad_pa_dev == NULL || state->walk_rad_ind_dev == NULL ||
459 state->walk_atoms_global_dev == NULL || !state->se_static_uploaded) {
460 fprintf(stderr, "ERROR: nnp_gpu_force_scatter_radial: walk / "
461 "se_static prerequisites not met\n");
462 return -1;
463 }
464 /* The descriptor gradient is the network stage's output and the one input
465 * this launcher does not produce itself. */
466 if (state->pbuf_dEdG_out == NULL) {
467 fprintf(stderr, "ERROR: nnp_gpu_force_scatter_radial: descriptor "
468 "gradient not on the device (run the network stage "
469 "first)\n");
470 return -1;
471 }
472
473 const size_t neigh_count = (size_t)max_n_neigh_per_se * (size_t)n_atoms_in_e;
474
475 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atom_map,
476 &state->pbuf_atom_map_cap,
477 sizeof(int) * (size_t)n_atoms_in_e))
478 return -1;
479 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atoms_global_rad,
480 &state->pbuf_atoms_global_rad_cap,
481 sizeof(int) * (size_t)n_atoms_in_e))
482 return -1;
483 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_symf_map,
484 &state->pbuf_symf_map_cap,
485 sizeof(int) * (size_t)n_symf))
486 return -1;
487 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_n_neigh_pa,
488 &state->pbuf_n_neigh_pa_cap,
489 sizeof(int) * (size_t)n_atoms_in_e))
490 return -1;
491 if (nnp_gpu_pbuf_ensure((void **)&state->scatter_neigh_ind,
492 &state->scatter_neigh_ind_cap,
493 sizeof(int) * neigh_count))
494 return -1;
495 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_izeta, &state->pbuf_izeta_cap,
496 sizeof(int) * (size_t)n_symf))
497 return -1;
498 /* pbuf_rad_ind / pbuf_rad_image_idx are written too but unused by
499 * scatter; keep their allocations alive in case the symmetry-function stage
500 * has not run. */
501 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_rad_ind,
502 &state->pbuf_rad_ind_cap, sizeof(int) * neigh_count))
503 return -1;
504 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_rad_image_idx,
505 &state->pbuf_rad_image_idx_cap,
506 sizeof(int) * neigh_count))
507 return -1;
508
509 {
510 const int block = 32;
511 const int j_blocks = (max_n_neigh_per_se + block - 1) / block;
512 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
513 dim3 blk(block, 1, 1);
514 nnp_pack_walk_to_pbuf_radial_kernel<<<grid, blk, 0, state->stream>>>(
515 n_atoms_in_e, max_n_neigh_per_se, s_0based, ele_idx, max_n_radgrp_walk,
516 n_atoms_in_e_max_walk, max_n_neigh_walk, istart_1based,
517 state->walk_n_rad_pa_dev, state->walk_rad_ind_dev,
518 state->walk_rad_image_idx_dev, state->walk_atoms_global_dev,
519 state->pbuf_n_neigh_pa, state->pbuf_rad_ind, state->pbuf_rad_image_idx,
520 state->pbuf_atoms_global_rad, state->pbuf_atom_map,
521 state->scatter_neigh_ind);
522 OFFLOAD_CHECK(offloadPeekAtLastError());
523 }
524 {
525 const int block = 32;
526 const int sf_blocks = (n_symf + block - 1) / block;
527 dim3 grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
528 dim3 blk(block, 1, 1);
529 const int se_idx_in_kind = s_0based + max_n_radgrp_walk * ele_idx;
530 /* Radial scatter: pbuf_izeta = 1-based; pbuf_symf_map = 0-based scratch
531 * (kernel reads pbuf_izeta and applies n_rad_offset itself). m_offset=0. */
532 nnp_pack_se_static_symf_map_kernel<<<grid, blk, 0, state->stream>>>(
533 se_idx_in_kind, n_symf, /*m_offset=*/0,
534 state->se_static_rad_symf_packed_dev,
535 state->se_static_rad_symf_offsets_dev, state->pbuf_symf_map,
536 state->pbuf_izeta);
537 OFFLOAD_CHECK(offloadPeekAtLastError());
538 }
539
540 {
541 dim3 grid(n_atoms_in_e, n_symf, n_committee);
542 dim3 blk(32, 1, 1);
543 /* Scatter kernel takes 1-based symf_map (pbuf_izeta) and 1-based
544 * neigh_ind (scatter_neigh_ind). */
545 nnp_force_scatter_radial_kernel<<<grid, blk, 0, state->stream>>>(
546 n_atoms_in_e, n_symf, max_n_neigh_per_se, n_committee, n_input_max,
547 num_atoms_global, ele_idx, scaling_mode, /*n_rad_offset=*/0, scmin,
548 scmax, state->pbuf_izeta, state->pbuf_atom_map,
549 state->pbuf_atoms_global_rad, state->pbuf_n_neigh_pa,
550 state->scatter_neigh_ind, state->per_se_rad_self[se_idx],
551 state->per_se_rad_force[se_idx], state->pbuf_dEdG_out,
552 state->scaling_loc_max_dev, state->scaling_loc_min_dev,
553 state->scaling_sigma_dev, state->f_dev,
554 /* Virial: pbuf_rad_image_idx pairs 1:1 (same out_off) with the
555 * scatter_neigh_ind neighbour list the kernel reads. */
556 state->pbuf_rad_image_idx, state->coord_dev, state->image_table_dev,
557 state->virial_dev, accumulate_virial);
558 }
559 OFFLOAD_CHECK(offloadPeekAtLastError());
560 return 0;
561}
562
563/*******************************************************************************
564 * \brief Angular force-scatter launcher for one (group, element) slot,
565 * companion to the angular symmetry-function launcher. Rebuilds the
566 * per-atom counts, the two 1-based angular neighbour index lists and
567 * the 1-based symmetry-function map (the shared index buffers were
568 * overwritten by later descriptor groups, so the rearrange is repeated
569 * here), then launches the angular scatter kernel to accumulate this
570 * slot's force contribution into f_dev.
571 * \author Dhruv Sharma (ds2173@cam.ac.uk)
572 ******************************************************************************/
573extern "C" int nnp_gpu_force_scatter_angular_c(
574 const int n_atoms_in_e, const int n_symf, const int max_n_ang1_per_se,
575 const int max_n_kk_per_se, const int n_committee, const int n_input_max,
576 const int num_atoms_global, const int ele_idx, const int n_rad_offset,
577 const int scaling_mode, const int homo, const double scmin,
578 const double scmax, const int s_0based, const int max_n_anggrp_walk,
579 const int n_atoms_in_e_max_walk, const int max_n_neigh_ang1_walk,
580 const int max_n_neigh_ang2_walk, const int istart_1based, const int se_idx,
581 const int accumulate_virial) {
582 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_ang1_per_se <= 0 ||
583 n_committee <= 0 || n_input_max <= 0 || num_atoms_global <= 0)
584 return -1;
585 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
586 fprintf(stderr,
587 "ERROR: nnp_gpu_force_scatter_angular: se_idx=%d out of range\n",
588 se_idx);
589 return -1;
590 }
591 nnp_gpu_state_t *state = nnp_gpu_state_get();
592 if (state == NULL || state->f_dev == NULL)
593 return -1;
594 if (!state->scaling_uploaded)
595 return -1;
596 if (state->per_se_ang_self[se_idx] == NULL ||
597 state->per_se_ang_jj[se_idx] == NULL ||
598 state->per_se_ang_kk[se_idx] == NULL) {
599 fprintf(stderr,
600 "ERROR: nnp_gpu_force_scatter_angular: per_se_ang "
601 "buffers not populated (call the symmetry-function stage first)\n");
602 return -1;
603 }
604 if (state->walk_n_ang1_pa_dev == NULL || state->walk_ang1_ind_dev == NULL ||
605 state->walk_atoms_global_dev == NULL || !state->se_static_uploaded) {
606 fprintf(stderr, "ERROR: nnp_gpu_force_scatter_angular: walk / "
607 "se_static prerequisites not met\n");
608 return -1;
609 }
610 /* The descriptor gradient is the network stage's output and the one input
611 * this launcher does not produce itself. */
612 if (state->pbuf_dEdG_out == NULL) {
613 fprintf(stderr, "ERROR: nnp_gpu_force_scatter_angular: descriptor "
614 "gradient not on the device (run the network stage "
615 "first)\n");
616 return -1;
617 }
618 /* Mirrors the bound the SF launcher enforces: the scatter reads the k-side
619 * index list with the max_n_kk stride, so it must not exceed the width the
620 * pack kernel wrote. */
621 if (homo && max_n_kk_per_se != max_n_ang1_per_se) {
622 fprintf(stderr,
623 "ERROR: nnp_gpu_force_scatter_angular: homonuclear slot needs "
624 "max_n_kk_per_se == max_n_ang1_per_se (%d vs %d); the k-side list "
625 "is the ang1 list and is packed with that stride\n",
626 max_n_kk_per_se, max_n_ang1_per_se);
627 return -1;
628 }
629
630 const size_t ang1_count = (size_t)max_n_ang1_per_se * (size_t)n_atoms_in_e;
631 const size_t ang2_count = (size_t)max_n_kk_per_se * (size_t)n_atoms_in_e;
632
633 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atom_map,
634 &state->pbuf_atom_map_cap,
635 sizeof(int) * (size_t)n_atoms_in_e))
636 return -1;
637 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atoms_global_ang,
638 &state->pbuf_atoms_global_ang_cap,
639 sizeof(int) * (size_t)n_atoms_in_e))
640 return -1;
641 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_symf_map,
642 &state->pbuf_symf_map_cap,
643 sizeof(int) * (size_t)n_symf))
644 return -1;
645 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_n_ang1_pa,
646 &state->pbuf_n_ang1_pa_cap,
647 sizeof(int) * (size_t)n_atoms_in_e))
648 return -1;
649 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_n_ang2_pa,
650 &state->pbuf_n_ang2_pa_cap,
651 sizeof(int) * (size_t)n_atoms_in_e))
652 return -1;
653 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang1_ind,
654 &state->pbuf_ang1_ind_cap, sizeof(int) * ang1_count))
655 return -1;
656 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang1_image_idx,
657 &state->pbuf_ang1_image_idx_cap,
658 sizeof(int) * ang1_count))
659 return -1;
660 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang2_ind,
661 &state->pbuf_ang2_ind_cap,
662 sizeof(int) * (ang2_count > 0 ? ang2_count : 1)))
663 return -1;
664 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_ang2_image_idx,
665 &state->pbuf_ang2_image_idx_cap,
666 sizeof(int) * (ang2_count > 0 ? ang2_count : 1)))
667 return -1;
668 /* scatter scratch for 0-based symf_map (unused by scatter kernel; the
669 * rearrange writes both 0-based and 1-based, we use only 1-based). */
670 if (nnp_gpu_pbuf_ensure((void **)&state->scatter_neigh_ind,
671 &state->scatter_neigh_ind_cap,
672 sizeof(int) * (size_t)n_symf))
673 return -1;
674
675 /* Per-atom rearrange (writes pbuf_n_ang*, atom_map, atoms_global). */
676 {
677 const int block = 32;
678 dim3 grid((n_atoms_in_e + block - 1) / block, 1, 1);
679 dim3 blk(block, 1, 1);
680 nnp_pack_walk_to_pbuf_angular_per_atom_kernel<<<grid, blk, 0,
681 state->stream>>>(
682 n_atoms_in_e, homo, s_0based, ele_idx, max_n_anggrp_walk,
683 n_atoms_in_e_max_walk, istart_1based, state->walk_n_ang1_pa_dev,
684 state->walk_n_ang2_pa_dev, state->walk_atoms_global_dev,
685 state->pbuf_n_ang1_pa, state->pbuf_n_ang2_pa,
686 state->pbuf_atoms_global_ang, state->pbuf_atom_map);
687 OFFLOAD_CHECK(offloadPeekAtLastError());
688 }
689
690 /* Per-(atom, j) rearrange with ind_offset = 1 (1-based for scatter). */
691 {
692 const int block = 32;
693 const int j_blocks = (max_n_ang1_per_se + block - 1) / block;
694 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
695 dim3 blk(block, 1, 1);
696 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<grid, blk, 0,
697 state->stream>>>(
698 n_atoms_in_e, max_n_ang1_per_se, s_0based, ele_idx, max_n_anggrp_walk,
699 n_atoms_in_e_max_walk, max_n_neigh_ang1_walk,
700 /*ind_offset=*/1, state->walk_n_ang1_pa_dev, state->walk_ang1_ind_dev,
701 state->walk_ang1_image_idx_dev, state->pbuf_ang1_ind,
702 state->pbuf_ang1_image_idx);
703 OFFLOAD_CHECK(offloadPeekAtLastError());
704 }
705 if (!homo && max_n_kk_per_se > 0) {
706 const int block = 32;
707 const int j_blocks = (max_n_kk_per_se + block - 1) / block;
708 dim3 grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
709 dim3 blk(block, 1, 1);
710 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<grid, blk, 0,
711 state->stream>>>(
712 n_atoms_in_e, max_n_kk_per_se, s_0based, ele_idx, max_n_anggrp_walk,
713 n_atoms_in_e_max_walk, max_n_neigh_ang2_walk,
714 /*ind_offset=*/1, state->walk_n_ang2_pa_dev, state->walk_ang2_ind_dev,
715 state->walk_ang2_image_idx_dev, state->pbuf_ang2_ind,
716 state->pbuf_ang2_image_idx);
717 OFFLOAD_CHECK(offloadPeekAtLastError());
718 }
719
720 /* Materialise the symmetry-function map: scatter wants 1-based. The kernel
721 * writes both (0-based to first ptr, 1-based to second); we put 1-based
722 * directly into pbuf_symf_map this time and discard 0-based to scratch. */
723 {
724 const int block = 32;
725 const int sf_blocks = (n_symf + block - 1) / block;
726 dim3 grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
727 dim3 blk(block, 1, 1);
728 const int se_idx_in_kind = s_0based + max_n_anggrp_walk * ele_idx;
729 /* Angular scatter: scatter_neigh_ind = 0-based scratch (discarded);
730 * pbuf_symf_map = 1-based (consumed by scatter kernel which applies
731 * n_rad_offset itself). m_offset=0 only affects the discarded scratch. */
732 nnp_pack_se_static_symf_map_kernel<<<grid, blk, 0, state->stream>>>(
733 se_idx_in_kind, n_symf, /*m_offset=*/0,
734 state->se_static_ang_symf_packed_dev,
735 state->se_static_ang_symf_offsets_dev, state->scatter_neigh_ind,
736 state->pbuf_symf_map);
737 OFFLOAD_CHECK(offloadPeekAtLastError());
738 }
739
740 {
741 dim3 grid(n_atoms_in_e, n_symf, n_committee);
742 dim3 blk(32, 1, 1);
743 nnp_force_scatter_angular_kernel<<<grid, blk, 0, state->stream>>>(
744 n_atoms_in_e, n_symf, max_n_ang1_per_se, max_n_kk_per_se, n_committee,
745 n_input_max, num_atoms_global, ele_idx, scaling_mode, homo,
746 n_rad_offset, scmin, scmax, state->pbuf_symf_map, state->pbuf_atom_map,
747 state->pbuf_atoms_global_ang, state->pbuf_n_ang1_pa,
748 state->pbuf_n_ang2_pa, state->pbuf_ang1_ind, state->pbuf_ang2_ind,
749 state->per_se_ang_self[se_idx], state->per_se_ang_jj[se_idx],
750 state->per_se_ang_kk[se_idx], state->pbuf_dEdG_out,
751 state->scaling_loc_max_dev, state->scaling_loc_min_dev,
752 state->scaling_sigma_dev, state->f_dev,
753 /* Virial: pbuf_ang1/2_image_idx pair 1:1 (same out_off) with the
754 * pbuf_ang1/2_ind neighbour lists the kernel reads; in homo mode the
755 * kk side reads pbuf_ang1_image_idx to match its ang1_ind read. */
756 state->pbuf_ang1_image_idx, state->pbuf_ang2_image_idx,
757 state->coord_dev, state->image_table_dev, state->virial_dev,
758 accumulate_virial);
759 }
760 OFFLOAD_CHECK(offloadPeekAtLastError());
761 return 0;
762}
763
764#endif /* defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP) */
static GRID_HOST_DEVICE int idx(const orbital a)
Return coset index of given orbital angular momentum.
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
real(dp), dimension(3) c
real(dp), dimension(3) a
real(dp), dimension(3) b