(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_state.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
8#ifndef NNP_GPU_STATE_H
9#define NNP_GPU_STATE_H
10
12#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
13
14#include <stddef.h>
15
16#ifdef __cplusplus
17extern "C" {
18#endif
19
20/* Capacity of the host-pointer-keyed spline/parameter device cache. */
21#define NNP_SPLINE_CACHE_MAX 256
22
23/* Upper bound on the number of (symmetry-function group, element) slots for
24 * which per-slot device buffers are kept. */
25#define NNP_MAX_SE 64
26
27/* The two capacities above, exported so the Fortran driver can test a model
28 * against them at initialization, before the device is claimed. The per-call
29 * checks in the launchers remain as a defense against inconsistent callers. */
30int nnp_gpu_max_se_c(void);
31int nnp_gpu_spline_cache_max_c(void);
32
33/*
34 * One entry of the host-pointer-keyed device cache: a device copy of a host
35 * array that is constant over an MD run (spline tables, packed angular
36 * parameters). The key is the host pointer identity; see the lifetime contract
37 * on nnp_gpu_get_or_upload_spline.
38 */
39struct nnp_gpu_cache_entry {
40 const double *h_ptr; /* cache key: source host pointer */
41 double *d_ptr; /* device copy */
42 size_t n_doubles; /* element count recorded at insert */
43};
44
45/*
46 * Per-process device state for the NNP GPU pipeline. Created lazily on first
47 * use and owned for the lifetime of the process (released explicitly when an
48 * NNP environment is torn down). All device work is queued on a single stream
49 * so the force evaluation needs only one synchronization per MD step. The host
50 * side is single-threaded; none of these routines are called from an OpenMP
51 * region.
52 *
53 * Grow-only scratch buffers (added by the kernel translation units) live
54 * alongside this state and are resized through nnp_gpu_pbuf_ensure.
55 */
56typedef struct nnp_gpu_state {
57 offloadStream_t stream; /* single work stream for all NNP device work */
58 int md_step; /* MD step counter, incremented on each position upload */
59
60 /* Device copies of run-constant host arrays, keyed by host pointer. */
61 struct nnp_gpu_cache_entry spline_cache[NNP_SPLINE_CACHE_MAX];
62 int spline_cache_size;
63
64 /* Per-step atomic positions and periodic-image translation vectors. */
65 double *coord_dev;
66 size_t coord_dev_cap;
67 double *image_table_dev;
68 size_t image_table_dev_cap;
69
70 /* Cell-list inputs to the on-device walk: scaled coordinates, the global
71 * element index of each atom, the linked-cell chain and image bookkeeping. */
72 double *coord_scaled_dev;
73 size_t coord_scaled_dev_cap;
74 int *ele_ind_global_dev;
75 size_t ele_ind_global_dev_cap;
76 int *cell_image_atom_dev;
77 size_t cell_image_atom_dev_cap;
78 int *cell_image_shift_dev;
79 size_t cell_image_shift_dev_cap;
80 int *cell_head_dev;
81 size_t cell_head_dev_cap;
82 int *cell_next_dev;
83 size_t cell_next_dev_cap;
84
85 /* Species-pair routing tables, uploaded once and flagged by
86 * pair_map_uploaded. The shape fields guard against a later re-upload with
87 * incompatible dimensions. */
88 int *pair_n_rad_dev;
89 size_t pair_n_rad_dev_cap;
90 int *pair_n_ang1_dev;
91 size_t pair_n_ang1_dev_cap;
92 int *pair_n_ang2_dev;
93 size_t pair_n_ang2_dev_cap;
94 double *pair_max_cutoff_dev;
95 size_t pair_max_cutoff_dev_cap;
96 int *pair_rad_groups_dev;
97 size_t pair_rad_groups_dev_cap;
98 int *pair_ang1_groups_dev;
99 size_t pair_ang1_groups_dev_cap;
100 int *pair_ang2_groups_dev;
101 size_t pair_ang2_groups_dev_cap;
102 double *rad_cutoff_dev;
103 size_t rad_cutoff_dev_cap;
104 double *ang_cutoff_dev;
105 size_t ang_cutoff_dev_cap;
106 int pair_map_uploaded;
107 int pair_map_n_ele;
108 int pair_map_max_pair_grps;
109 int pair_map_max_n_radgrp;
110 int pair_map_max_n_anggrp;
111
112 /* On-device cell-list walk outputs: per-(atom-slot, group, element) neighbour
113 * index and image-column lists, and the per-slot neighbour counts. The walk
114 * appends with an atomic counter; the overflow flag trips when a per-slot
115 * list exceeds its allocated width, prompting the host to grow and retry. */
116 int *walk_atoms_global_dev;
117 size_t walk_atoms_global_dev_cap;
118 int *walk_n_atoms_in_e_dev;
119 size_t walk_n_atoms_in_e_dev_cap;
120 int *walk_rad_ind_dev;
121 size_t walk_rad_ind_dev_cap;
122 int *walk_rad_image_idx_dev;
123 size_t walk_rad_image_idx_dev_cap;
124 int *walk_ang1_ind_dev;
125 size_t walk_ang1_ind_dev_cap;
126 int *walk_ang1_image_idx_dev;
127 size_t walk_ang1_image_idx_dev_cap;
128 int *walk_ang2_ind_dev;
129 size_t walk_ang2_ind_dev_cap;
130 int *walk_ang2_image_idx_dev;
131 size_t walk_ang2_image_idx_dev_cap;
132 int *walk_n_rad_pa_dev;
133 size_t walk_n_rad_pa_dev_cap;
134 int *walk_n_ang1_pa_dev;
135 size_t walk_n_ang1_pa_dev_cap;
136 int *walk_n_ang2_pa_dev;
137 size_t walk_n_ang2_pa_dev_cap;
138 int *walk_overflow_dev;
139 size_t walk_overflow_cap;
140
141 /* Device-resident aggregated descriptor G (one row per symmetry function per
142 * rank-local atom) and its scaled copy, the network input. Layout is
143 * G[m + i_local * n_input_max] (descriptor-major within an atom, atoms
144 * outermost). */
145 double *G_dev;
146 size_t G_dev_cap;
147 double *G_dev_scaled;
148 size_t G_dev_scaled_cap;
149
150 /* One-int device flag set by the scaling kernel when a descriptor leaves its
151 * training range, re-zeroed every scaling pass, and its stream-ordered host
152 * copy, valid after the step's final synchronization. */
153 int *expol_flag_dev;
154 size_t expol_flag_dev_cap;
155 int expol_flag_host;
156
157 /* Per-call mapping tables used when the descriptor kernels write into G_dev:
158 * symf_map[sf] is the descriptor row a group's symmetry function writes to,
159 * atom_map[atom_in_batch] is the atom slot in G_dev. */
160 int *pbuf_symf_map;
161 size_t pbuf_symf_map_cap;
162 int *pbuf_atom_map;
163 size_t pbuf_atom_map_cap;
164
165 /* Per-atom neighbour counts and per-group angular zeta scratch, filled on
166 * device from the walk output. */
167 int *pbuf_n_neigh_pa;
168 size_t pbuf_n_neigh_pa_cap;
169 int *pbuf_n_ang1_pa;
170 size_t pbuf_n_ang1_pa_cap;
171 int *pbuf_n_ang2_pa;
172 size_t pbuf_n_ang2_pa_cap;
173 int *pbuf_izeta;
174 size_t pbuf_izeta_cap;
175
176 /* Per-(symmetry-function group, element) neighbour index and image-column
177 * lists, rearranged from the walk output for the descriptor kernels. */
178 int *pbuf_rad_ind;
179 size_t pbuf_rad_ind_cap;
180 int *pbuf_rad_image_idx;
181 size_t pbuf_rad_image_idx_cap;
182 int *pbuf_atoms_global_rad;
183 size_t pbuf_atoms_global_rad_cap;
184 int *pbuf_ang1_ind;
185 size_t pbuf_ang1_ind_cap;
186 int *pbuf_ang2_ind;
187 size_t pbuf_ang2_ind_cap;
188 int *pbuf_ang1_image_idx;
189 size_t pbuf_ang1_image_idx_cap;
190 int *pbuf_ang2_image_idx;
191 size_t pbuf_ang2_image_idx_cap;
192 int *pbuf_atoms_global_ang;
193 size_t pbuf_atoms_global_ang_cap;
194 int *scatter_neigh_ind;
195 size_t scatter_neigh_ind_cap;
196
197 /* Single-precision per-neighbour displacements for the mixed-precision
198 * angular path; populated only when single precision is selected. */
199 float *pbuf_ang1_disp;
200 size_t pbuf_ang1_disp_cap;
201 float *pbuf_ang2_disp;
202 size_t pbuf_ang2_disp_cap;
203
204 /* Per-element descriptor scaling tables (average, min, max, sigma) and the
205 * per-element input-node counts, uploaded once and flagged by
206 * scaling_uploaded. */
207 double *scaling_loc_av_dev;
208 size_t scaling_loc_av_cap;
209 double *scaling_loc_min_dev;
210 size_t scaling_loc_min_cap;
211 double *scaling_loc_max_dev;
212 size_t scaling_loc_max_cap;
213 double *scaling_sigma_dev;
214 size_t scaling_sigma_cap;
215 int *scaling_n_input_nodes_dev;
216 size_t scaling_n_input_nodes_cap;
217 int scaling_uploaded;
218 /* Shape the tables above were uploaded with. The scale kernel strides the
219 * per-element slabs by n_input_max, so a launch that disagrees with the
220 * upload would read the wrong descriptor row; these let it be rejected
221 * instead. */
222 int scaling_n_ele;
223 int scaling_n_input_max;
224
225 /* Per-(group, element) symmetry-function to descriptor-row maps, packed with
226 * prefix-sum offsets and uploaded once (model-static). */
227 int *se_static_rad_symf_packed_dev;
228 size_t se_static_rad_symf_packed_cap;
229 int *se_static_rad_symf_offsets_dev;
230 size_t se_static_rad_symf_offsets_cap;
231 int *se_static_ang_symf_packed_dev;
232 size_t se_static_ang_symf_packed_cap;
233 int *se_static_ang_symf_offsets_dev;
234 size_t se_static_ang_symf_offsets_cap;
235 int se_static_uploaded;
236 /* Shape the maps above were built with. The descriptor launchers stride
237 * se_idx_in_kind by max_n_radgrp / max_n_anggrp, so they compare against
238 * these and refuse a launch whose stride would select a wrong row. */
239 int se_static_n_ele;
240 int se_static_max_n_radgrp;
241 int se_static_max_n_anggrp;
242 int se_static_total_rad_symf;
243 int se_static_total_ang_symf;
244
245 /* Per-(group, element) slot buffers holding the descriptor kernels' partial
246 * outputs, read back by the force-scatter kernels. Each slot is grown on
247 * demand and freed on release. */
248 double *per_se_rad_self[NNP_MAX_SE];
249 size_t per_se_rad_self_cap[NNP_MAX_SE];
250 double *per_se_rad_force[NNP_MAX_SE];
251 size_t per_se_rad_force_cap[NNP_MAX_SE];
252 double *per_se_ang_self[NNP_MAX_SE];
253 size_t per_se_ang_self_cap[NNP_MAX_SE];
254 double *per_se_ang_jj[NNP_MAX_SE];
255 size_t per_se_ang_jj_cap[NNP_MAX_SE];
256 double *per_se_ang_kk[NNP_MAX_SE];
257 size_t per_se_ang_kk_cap[NNP_MAX_SE];
258
259 /* Per-(group, element) angular zeta tables, keyed by slot index rather than
260 * host pointer (the host packing buffers are reallocated every step).
261 * ang_zeta_n is 0 for a slot that has not been uploaded. */
262 int *ang_izeta_dev[NNP_MAX_SE];
263 size_t ang_izeta_cap[NNP_MAX_SE];
264 char *ang_uiz_dev[NNP_MAX_SE];
265 size_t ang_uiz_cap[NNP_MAX_SE];
266 int ang_zeta_n[NNP_MAX_SE];
267
268 /* Rank-local 0-based element index per atom, shared by the scaling and
269 * network kernels. Uploaded once and re-uploaded when the rank-local atom
270 * count or the contents change against the exact host shadow copy
271 * (step_ele_ind_n is 0 until first upload). The shadow is what catches a
272 * composition change at constant atom count, such as an identity-exchange
273 * Monte Carlo move or a resorted subsystem: without it the run would keep
274 * routing every atom to the previous element's weights and scaling table.
275 * The shadow lives in host memory (plain malloc, freed on release). */
276 int *step_ele_ind_dev;
277 size_t step_ele_ind_cap;
278 int *step_ele_ind_host;
279 size_t step_ele_ind_host_cap;
280 int step_ele_ind_n;
281
282 /* Packed network topology, activation functions, atomic energies and
283 * weights and biases, uploaded once and flagged by nn_packed_uploaded. The
284 * shape fields guard a later call against an incompatible network. */
285 int *nn_n_nodes_dev;
286 size_t nn_n_nodes_dev_cap;
287 int *nn_act_fnct_dev;
288 size_t nn_act_fnct_dev_cap;
289 double *nn_atom_energies_dev;
290 size_t nn_atom_energies_dev_cap;
291 double *nn_W_packed_dev;
292 size_t nn_W_packed_dev_cap;
293 double *nn_b_packed_dev;
294 size_t nn_b_packed_dev_cap;
295 int nn_packed_uploaded;
296 int nn_packed_n_committee;
297 int nn_packed_n_layer;
298 int nn_packed_max_n_nodes;
299 int nn_packed_n_ele;
300
301 /* Network outputs: per-atom per-committee energy and the descriptor
302 * gradient, read back to the host each step. */
303 double *pbuf_atomE_out;
304 size_t pbuf_atomE_out_cap;
305 double *pbuf_dEdG_out;
306 size_t pbuf_dEdG_out_cap;
307
308 /* Device force accumulator, sized (3, num_atoms, n_committee). Seeded from
309 * the host force array, accumulated into by the scatter kernels and read
310 * back at the end of the step. */
311 double *f_dev;
312 size_t f_dev_cap;
313
314 /*
315 * Per-committee analytical virial accumulator (9 * n_committee doubles),
316 * layout virial_dev[9*c + 3*a + b] with a = force component, b = bond
317 * (r_ij) component. This equals the host committee_stress(b+1, a+1, c+1)
318 * (column-major 3x3) after the fold in nnp_gpu_energy_force. Zeroed at
319 * scatter begin, atomic-added by the radial/angular scatter kernels when
320 * virial accumulation is requested, D2H'd by
321 * nnp_gpu_force_scatter_virial_d2h_c. Stored full 3x3 (not the 6 Voigt
322 * components) because the raw NNP virial is not symmetric before
323 * symmetrisation.
324 */
325 double *virial_dev;
326 size_t virial_dev_cap;
327} nnp_gpu_state_t;
328
329/*
330 * Return the singleton device state, creating it (and its work stream) on
331 * first call. Returns NULL if the state allocation fails; a stream-creation
332 * failure aborts inside the offload wrappers.
333 */
334nnp_gpu_state_t *nnp_gpu_state_get(void);
335
336/*
337 * Release the singleton device state: free every scratch buffer and cached
338 * device copy, destroy the work stream and free the state itself. Idempotent
339 * and safe to call when the state was never created.
340 */
341void nnp_gpu_state_release(void);
342
343/*
344 * Invalidate the host-pointer-keyed caches and the once-per-run upload flags,
345 * without freeing scratch buffers or destroying the stream, so a later
346 * environment that receives recycled host addresses cannot hit stale device
347 * copies. Safe no-op when the state was never created. Environment teardown
348 * goes through nnp_gpu_release_env, which calls this.
349 */
350void nnp_gpu_reset_caches(void);
351
352/*
353 * Count this NNP environment as a user of the device state and return the
354 * number now live. The device state is one per process and its once-per-run
355 * uploads (network weights, pair map, scaling and symmetry-function tables) are
356 * keyed only by shape, so two environments that are live at the same time and
357 * happen to share an architecture would overwrite each other's tables while
358 * keeping the first one's weights. The caller declines the device for a second
359 * live environment rather than run that combination.
360 */
361int nnp_gpu_register_env(void);
362
363/*
364 * Undo a nnp_gpu_register_env whose caller declined the device, without
365 * disturbing the state the owning environment is using.
366 */
367void nnp_gpu_unregister_env(void);
368
369/*
370 * Drop one environment's claim on the device state. Invalidates the caches
371 * while other environments remain, and releases the state (scratch buffers and
372 * work stream included) once the last one goes away. Must be called when a
373 * GPU-enabled NNP environment is released, before its host arrays are freed.
374 */
375void nnp_gpu_release_env(void);
376
377/*
378 * Ensure *pbuf points at a device allocation of at least n_bytes, growing it
379 * (never shrinking) with slack so that small step-to-step size drifts do not
380 * force a reallocation. The buffer contents are not preserved across a grow,
381 * so callers must fully rewrite a buffer after any ensure. Returns 0 on
382 * success; a device allocation failure aborts inside the offload wrappers.
383 * A request of 0 bytes is satisfied by any existing capacity and never frees;
384 * buffers are released together by nnp_gpu_state_release.
385 */
386int nnp_gpu_pbuf_ensure_named(void **pbuf, size_t *pcap, size_t n_bytes,
387 const char *bufname);
388#define nnp_gpu_pbuf_ensure(pbuf, pcap, n_bytes) \
389 nnp_gpu_pbuf_ensure_named((pbuf), (pcap), (n_bytes), #pbuf)
390
391/* Number of scratch-buffer reallocations since process start (diagnostics). */
392unsigned long nnp_gpu_pbuf_grow_events(void);
393
394/* Current total bytes held in grow-only scratch buffers (diagnostics). */
395size_t nnp_gpu_pbuf_total_bytes(void);
396
397/*
398 * Return the device copy of a run-constant host array, uploading it on first
399 * request and caching it by host pointer. Returns NULL if the cache is full,
400 * which the callers report as a launcher error; the Fortran driver sizes the
401 * model against NNP_SPLINE_CACHE_MAX at initialization so a full cache is not
402 * reached in a normal run. A device runtime error aborts inside the offload
403 * wrappers.
404 *
405 * Lifetime contract: entries are keyed by host pointer identity, so callers
406 * must only pass host arrays whose lifetime spans the NNP environment and
407 * whose contents do not change. nnp_gpu_reset_caches must be called when the
408 * environment is released, before the host arrays are freed, otherwise a later
409 * environment reusing the same heap addresses would silently receive stale
410 * device copies.
411 */
412double *nnp_gpu_get_or_upload_spline(const double *h_ptr, size_t n_doubles);
413
414/*
415 * Select the GPU compute precision: use_fp32 != 0 runs the angular symmetry
416 * functions in single precision (radial functions, network evaluation and
417 * accumulation stay double precision). Returns 0.
418 */
419int nnp_gpu_set_precision(int use_fp32);
420
421/* Report whether single-precision angular symmetry functions are selected. */
422int nnp_gpu_precision_fp32(void);
423
424/*
425 * Per-step and once-at-init uploads feeding the on-device cell-list walk, and
426 * the walk itself. See nnp_gpu_walk.cu. All return 0 on success or a negative
427 * error code; the walk returns -2 when a per-slot neighbour list overflows and
428 * the host must grow the per-list widths and retry.
429 */
430int nnp_gpu_upload_positions_step_c(int num_atoms, const double *h_coord);
431
432int nnp_gpu_upload_image_table_c(int n_images, const double *h_image_table);
433
434int nnp_gpu_upload_cell_list_c(int n_images, int n_cells,
435 const int *h_image_atom,
436 const int *h_image_shift, const int *h_head,
437 const int *h_next);
438
439int nnp_gpu_upload_pair_map_c(int n_ele, int max_pair_grps, int max_n_radgrp,
440 int max_n_anggrp, const int *h_n_rad,
441 const int *h_n_ang1, const int *h_n_ang2,
442 const double *h_pair_max_cutoff,
443 const int *h_rad_groups, const int *h_ang1_groups,
444 const int *h_ang2_groups,
445 const double *h_rad_cutoff,
446 const double *h_ang_cutoff);
447
448int nnp_gpu_cell_list_walk_c(
449 int num_atoms_global, int n_ele, int n_images, int n_cells,
450 int n_atoms_in_e_max, int max_n_neigh_rad, int max_n_neigh_ang1,
451 int max_n_neigh_ang2, int max_n_radgrp, int max_n_anggrp, const int *h_nbin,
452 const double *h_bin_lower, const double *h_bin_width, const int *h_bin_span,
453 const int *h_perd, const int *h_exact_pbc_copies,
454 const double *h_coord_scaled, const int *h_ele_ind_global,
455 const int *h_atoms_global_per_e, const int *h_n_atoms_in_e);
456
457int nnp_gpu_walk_counts_d2h_c(int n_atoms_in_e_max, int max_n_radgrp,
458 int max_n_anggrp, int n_ele, int *h_n_rad_pa,
459 int *h_n_ang1_pa, int *h_n_ang2_pa);
460
461/*
462 * Descriptor pipeline: once-at-init uploads of the per-element scaling tables
463 * and the per-(group, element) symmetry-function maps; the per-step
464 * device-resident descriptor build (radial and angular symmetry functions into
465 * G_dev), its zeroing, and the scaling into G_dev_scaled. See nnp_gpu_acsf.cu.
466 * All return 0 on success or a negative error code.
467 */
468int nnp_gpu_begin_descriptor_step(int n_atoms, int n_input_max);
469
470int nnp_gpu_upload_scaling_tables_c(int n_ele, int n_input_max,
471 const double *h_loc_av_per_ele,
472 const double *h_loc_min_per_ele,
473 const double *h_loc_max_per_ele,
474 const double *h_sigma_per_ele,
475 const int *h_n_input_nodes_per_ele);
476
477int nnp_gpu_upload_se_static_tables_c(int n_ele, int max_n_radgrp,
478 int max_n_anggrp, int total_rad_symf,
479 int total_ang_symf,
480 const int *h_rad_symf_packed,
481 const int *h_rad_symf_offsets,
482 const int *h_ang_symf_packed,
483 const int *h_ang_symf_offsets);
484
485int nnp_gpu_scale_G_dev_c(int n_atoms, int n_input_max, int n_ele,
486 int scaling_mode, double scmin, double scmax,
487 const int *h_ele_ind_for_atoms);
488
489/* Whether the last scaling pass saw a descriptor outside its training range.
490 * Valid once the step's device work has been synchronized (the force
491 * read-back does that). */
492int nnp_gpu_extrapolation_flag_c(void);
493
494int nnp_gpu_radial_sf_to_G_dev_c(
495 const int n_atoms_in_e, const int max_n_neigh_per_se, const int n_symf,
496 const int spline_n, const double spline_dx, const double spline_dx_inv,
497 const double spline_x_max, const double *h_spline_y,
498 const double *h_spline_dy, const int s_0based, const int e_0based,
499 const int max_n_radgrp_walk, const int n_atoms_in_e_max_walk,
500 const int max_n_neigh_walk, const int istart_1based, const int n_input_max,
501 const int se_idx);
502
503int nnp_gpu_angular_sf_to_G_dev_c(
504 const int homo, const int cut_type, const int max_n_ang1_per_se,
505 const int max_n_ang2_per_se, const int max_n_kk_per_se, const int n_symf,
506 const int n_atoms_in_e, const double cutoff_s, const double cutoff_sqr,
507 const double *h_pack_lam, const double *h_pack_zeta,
508 const double *h_pack_eta, const double *h_pack_prefzeta,
509 const int *h_pack_izeta, const char *h_pack_use_int_zeta,
510 const int s_0based, const int e_0based, const int max_n_anggrp_walk,
511 const int n_atoms_in_e_max_walk, const int max_n_neigh_ang1_walk,
512 const int max_n_neigh_ang2_walk, const int istart_1based,
513 const int n_rad_for_e, const int n_input_max, const int se_idx);
514
515/*
516 * Committee network evaluation. The topology, activation functions, atomic
517 * energies and weights are uploaded once (nnp_gpu_upload_nn_packed_c); the
518 * per-step launcher evaluates every atom and committee member from the scaled
519 * device descriptor and reads back the per-atom energies and descriptor
520 * gradients. See nnp_gpu_nn.cu. Both return 0 on success or a negative error.
521 */
522int nnp_gpu_upload_nn_packed_c(int n_committee, int n_layer, int max_n_nodes,
523 int n_ele, const int *h_n_nodes,
524 const int *h_act_fnct,
525 const double *h_atom_energies,
526 const double *h_W_packed,
527 const double *h_b_packed);
528
529/*
530 * Ensure the rank-local 0-based element index of every atom is on the device,
531 * re-uploading when the count or the contents have changed. Defined in
532 * nnp_gpu_acsf.cu and shared with the network launcher. Returns 0 on success.
533 */
534int nnp_gpu_step_ele_ind_ensure(nnp_gpu_state_t *state, const int *h_ele_ind,
535 int n_atoms);
536
537/* h_dE_dG may be NULL: the gradient then stays on the device for the force
538 * scatter to consume, and only the energies are read back. */
539int nnp_gpu_nn_atomwise_with_G_dev_c(const int n_atoms, const int n_committee,
540 const int n_layer, const int max_n_nodes,
541 const int n_input_max, const int normnodes,
542 const int n_ele, const int *h_ele_ind,
543 double *h_atomic_energy, double *h_dE_dG);
544
545/*
546 * Force assembly. begin seeds the device accumulator from the host force
547 * array; the per-(group, element) radial and angular scatter kernels multiply
548 * the descriptor gradient by the descriptor derivatives and accumulate into
549 * it; end reads it back. See nnp_gpu_acsf.cu. All return 0 on success or a
550 * negative error code.
551 */
552int nnp_gpu_force_scatter_begin_c(int num_atoms_global, int n_committee,
553 const double *h_force);
554
555int nnp_gpu_force_scatter_end_c(int num_atoms_global, int n_committee,
556 double *h_force);
557
558/*
559 * D2H the per-committee analytical virial accumulator into h_virial
560 * ([9 * n_committee] doubles, layout [9*c + 3*a + b]). Call after
561 * nnp_gpu_force_scatter_end_c has synchronised the stream.
562 */
563int nnp_gpu_force_scatter_virial_d2h_c(int n_committee, double *h_virial);
564
565int nnp_gpu_force_scatter_radial_c(
566 const int n_atoms_in_e, const int n_symf, const int max_n_neigh_per_se,
567 const int n_committee, const int n_input_max, const int num_atoms_global,
568 const int ele_idx, const int scaling_mode, const double scmin,
569 const double scmax, const int s_0based, const int max_n_radgrp_walk,
570 const int n_atoms_in_e_max_walk, const int max_n_neigh_walk,
571 const int istart_1based, const int se_idx, const int accumulate_virial);
572
573int nnp_gpu_force_scatter_angular_c(
574 const int n_atoms_in_e, const int n_symf, const int max_n_ang1_per_se,
575 const int max_n_kk_per_se, const int n_committee, const int n_input_max,
576 const int num_atoms_global, const int ele_idx, const int n_rad_offset,
577 const int scaling_mode, const int homo, const double scmin,
578 const double scmax, const int s_0based, const int max_n_anggrp_walk,
579 const int n_atoms_in_e_max_walk, const int max_n_neigh_ang1_walk,
580 const int max_n_neigh_ang2_walk, const int istart_1based, const int se_idx,
581 const int accumulate_virial);
582
583#ifdef __cplusplus
584}
585#endif
586
587#endif /* __OFFLOAD && !__NO_OFFLOAD_NNP */
588#endif /* NNP_GPU_STATE_H */