(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_nn.cu
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
10
11#include <stdio.h>
12
13#include "nnp_gpu_internal.h"
14#include "nnp_gpu_state.h"
15
16#if defined(_OMP_H)
17#error "OpenMP should not be used in .cu files to accommodate HIP."
18#endif
19
20/*
21 * Committee neural-network evaluation on the device. The topology, activation
22 * functions, atomic energies and packed weights and biases are uploaded once
23 * per run (nnp_gpu_upload_nn_packed_c); the per-step launcher
24 * (nnp_gpu_nn_atomwise_with_G_dev_c) evaluates every atom and committee member
25 * straight from the scaled device descriptor G_dev_scaled and reads the
26 * per-atom energies and descriptor gradients back to the host. The single
27 * kernel below performs both the forward pass (the network energy of each atom)
28 * and the backward pass (the gradient of that energy with respect to each input
29 * descriptor), mirroring the host routines that evaluate the same feed-forward
30 * network and its activation derivatives.
31 */
32
33/*******************************************************************************
34 * \brief Atom-batched committee-network kernel: one thread block per
35 * (atom, committee) pair does a full forward pass followed by a backward
36 * pass. blockIdx.x selects the 0-based atom, blockIdx.y the 0-based
37 * committee member, and the block's threads split each layer's nodes by
38 * threadIdx.x stride.
39 *
40 * Input and output layouts:
41 * - G_batch[i + n_input_max * atom]: scaled descriptor i of atom.
42 * - atomic_energy[c + n_committee * atom]: network energy of the atom
43 * for committee member c. The kernel adds atom_energies[ele] to the
44 * network output itself, so the result is the total atomic energy.
45 * - dE_dG[m + n_input_max * c + n_input_max * n_committee * atom]:
46 * gradient of that energy with respect to input descriptor m.
47 *
48 * The weights are packed as
49 * W[i + max_n_nodes * j
50 * + max_n_nodes * max_n_nodes * c
51 * + max_n_nodes * max_n_nodes * n_committee * (L - 2)
52 * + max_n_nodes * max_n_nodes * n_committee * (n_layer - 1) * ele]
53 * and the biases as
54 * b[j + max_n_nodes * c
55 * + max_n_nodes * n_committee * (L - 2)
56 * + max_n_nodes * n_committee * (n_layer - 1) * ele],
57 * with n_nodes[(L - 1) + n_layer * ele] the node count of layer L for
58 * element ele. Activation codes match the enum in
59 * nnp_environment_types.F; normnodes divides each node's pre-activation
60 * (and its derivative) by the previous layer's node count.
61 *
62 * Shared memory holds the per-layer pre- and post-activation node values
63 * and two alternating node vectors for the backpropagated energy
64 * gradient, 2*max_n_nodes*(n_layer + 1) doubles per block (6.4 KB for
65 * an 80-25-25-1 net). The launcher opts into the device's per-block
66 * shared-memory maximum only when that exceeds the 48 KB default, and
67 * rejects only what the device cannot hold. The final layer is the
68 * scalar energy output (node 0).
69 * \author Dhruv Sharma (ds2173@cam.ac.uk)
70 ******************************************************************************/
71__global__ void nnp_nn_kernel_atomwise(
72 const int n_atoms, const int n_committee, const int n_layer,
73 const int max_n_nodes, const int n_input_max, const int normnodes,
74 const int n_ele, const int *__restrict__ n_nodes,
75 const int *__restrict__ act_fnct, const int *__restrict__ ele_ind,
76 const double *__restrict__ atom_energies,
77 const double *__restrict__ weights, const double *__restrict__ biases,
78 const double *__restrict__ G_batch, double *__restrict__ atomic_energy,
79 double *__restrict__ dE_dG) {
80
81 const int atom = blockIdx.x;
82 const int c = blockIdx.y;
83 const int tid = threadIdx.x;
84 const int nth = blockDim.x;
85 if (atom >= n_atoms || c >= n_committee)
86 return;
87
88 const int ele = ele_ind[atom];
89 const int n_input = n_nodes[0 + n_layer * ele];
90
91 /* Shared memory layout. */
92 extern __shared__ double smem[];
93 double *node_pre = smem;
94 double *node_post = node_pre + (size_t)max_n_nodes * n_layer;
95 double *delta_curr = node_post + (size_t)max_n_nodes * n_layer;
96 double *delta_prev = delta_curr + max_n_nodes;
97
98 /* Init layer 1 (input descriptor) into node_post. 64-bit atom offset,
99 * matching nnp_gpu_pack.h: G_batch scales with the rank's atom count. */
100 for (int j = tid; j < n_input; j += nth) {
101 node_post[j /* + 0*max_n_nodes */] =
102 G_batch[j + (long long)n_input_max * atom];
103 }
105
106 /* ---------- Forward ---------- */
107 for (int L = 2; L <= n_layer; L++) {
108 const int n_in = n_nodes[(L - 2) + n_layer * ele];
109 const int n_out = n_nodes[(L - 1) + n_layer * ele];
110 const size_t W_off =
111 (size_t)max_n_nodes * max_n_nodes * c +
112 (size_t)max_n_nodes * max_n_nodes * n_committee * (L - 2) +
113 (size_t)max_n_nodes * max_n_nodes * n_committee * (n_layer - 1) * ele;
114 const size_t b_off =
115 (size_t)max_n_nodes * c + (size_t)max_n_nodes * n_committee * (L - 2) +
116 (size_t)max_n_nodes * n_committee * (n_layer - 1) * ele;
117 const double *W = weights + W_off;
118 const double *b = biases + b_off;
119
120 /* Each thread computes one output node j (stride nth). */
121 for (int j = tid; j < n_out; j += nth) {
122 double y = 0.0;
123 for (int i = 0; i < n_in; i++) {
124 y += W[i + max_n_nodes * j] * node_post[i + max_n_nodes * (L - 2)];
125 }
126 y += b[j];
127 if (normnodes)
128 y /= (double)n_in;
129
130 /* Save pre-activation. */
131 node_pre[j + max_n_nodes * (L - 1)] = y;
132
133 /* Activation enum per nnp_environment_types.F. */
134 const int act = act_fnct[L - 2];
135 double y_post;
136 switch (act) {
137 case 1: /* tanh */
138 y_post = tanh(y);
139 break;
140 case 2: /* gaus f(x) = exp(-0.5 x^2) */
141 y_post = exp(-0.5 * y * y);
142 break;
143 case 3: /* lin */
144 y_post = y;
145 break;
146 case 4: /* cos */
147 y_post = cos(y);
148 break;
149 case 5: /* sig f(x) = 1 / (1 + exp(-x)) */
150 y_post = 1.0 / (1.0 + exp(-y));
151 break;
152 case 6: /* invsig f(x) = 1 - sig(x) */
153 y_post = 1.0 - 1.0 / (1.0 + exp(-y));
154 break;
155 case 7: /* exp f(x) = exp(-x) */
156 y_post = exp(-y);
157 break;
158 case 8: /* softplus f(x) = log(exp(x) + 1) */
159 y_post = log(exp(y) + 1.0);
160 break;
161 case 9: /* quad f(x) = x^2 */
162 y_post = y * y;
163 break;
164 default:
165 y_post = y;
166 break;
167 }
168 node_post[j + max_n_nodes * (L - 1)] = y_post;
169 }
171 }
172
173 /* atomic_energy = node_post[layer n_layer, node 0] + atom_energies[ele]. */
174 if (tid == 0) {
175 atomic_energy[c + (long long)n_committee * atom] =
176 node_post[0 + max_n_nodes * (n_layer - 1)] + atom_energies[ele];
177 }
178
179 /* ---------- Backward ---------- */
180 /* Step 1: convert node_pre/node_post into activation derivative,
181 * stored in node_pre (overwrite, pre-activation not needed
182 * after this). Mirrors nnp_gradients_apply_actderiv. */
183 for (int L = 2; L <= n_layer; L++) {
184 const int n_out = n_nodes[(L - 1) + n_layer * ele];
185 const int act = act_fnct[L - 2];
186 for (int j = tid; j < n_out; j += nth) {
187 const double pre = node_pre[j + max_n_nodes * (L - 1)];
188 const double post = node_post[j + max_n_nodes * (L - 1)];
189 double deriv;
190 switch (act) {
191 case 1: /* tanh: 1 - post^2 */
192 deriv = 1.0 - post * post;
193 break;
194 case 2: /* gaus: -post * pre */
195 deriv = -post * pre;
196 break;
197 case 3: /* lin: 1 */
198 deriv = 1.0;
199 break;
200 case 4: /* cos: -sin(pre) */
201 deriv = -sin(pre);
202 break;
203 case 5: /* sig: exp(-pre) / (1+exp(-pre))^2 */
204 {
205 const double e = exp(-pre);
206 const double d = 1.0 + e;
207 deriv = e / (d * d);
208 break;
209 }
210 case 6: /* invsig: -sig'(pre) */
211 {
212 const double e = exp(-pre);
213 const double d = 1.0 + e;
214 deriv = -e / (d * d);
215 break;
216 }
217 case 7: /* exp: -post */
218 deriv = -post;
219 break;
220 case 8: /* softplus: (exp(post) + 1) / exp(post), uses POST per CPU code
221 */
222 deriv = (exp(post) + 1.0) / exp(post);
223 break;
224 case 9: /* quad: 2 * pre */
225 deriv = 2.0 * pre;
226 break;
227 default:
228 deriv = 1.0;
229 break;
230 }
231 if (normnodes)
232 deriv /= (double)n_nodes[(L - 2) + n_layer * ele];
233 node_pre[j + max_n_nodes * (L - 1)] = deriv;
234 }
235 }
237
238 /* Step 2: reverse-mode backpropagation of the scalar energy, node 0 of the
239 * output layer. delta_curr holds dE/d(post-activation) of layer L, seeded
240 * with the unit vector e_0 at the output layer. One step through layer L
241 * gives layer L-1:
242 * delta[L-1][i] = sum_j weights[L][i, j, c] * deriv[L, j] * delta[L][j]
243 * and the delta of the input layer is dE/dG. This yields the same gradient
244 * as chaining the full (n_input x n_nodes) Jacobian forward through the
245 * layers, but needs one node vector per layer instead of a Jacobian tile. */
246 {
247 const int n_out_last = n_nodes[(n_layer - 1) + n_layer * ele];
248 for (int j = tid; j < n_out_last; j += nth)
249 delta_curr[j] = (j == 0) ? 1.0 : 0.0;
250 }
252
253 for (int L = n_layer; L >= 2; L--) {
254 const int n_in_L = n_nodes[(L - 2) + n_layer * ele];
255 const int n_out_L = n_nodes[(L - 1) + n_layer * ele];
256 const size_t WL_off =
257 (size_t)max_n_nodes * max_n_nodes * c +
258 (size_t)max_n_nodes * max_n_nodes * n_committee * (L - 2) +
259 (size_t)max_n_nodes * max_n_nodes * n_committee * (n_layer - 1) * ele;
260 const double *WL = weights + WL_off;
261
262 /* Fold the activation derivative of layer L into its delta. */
263 for (int j = tid; j < n_out_L; j += nth)
264 delta_curr[j] *= node_pre[j + max_n_nodes * (L - 1)];
266
267 /* Each thread computes one node i of layer L-1 (stride nth). */
268 for (int i = tid; i < n_in_L; i += nth) {
269 double acc = 0.0;
270 for (int j = 0; j < n_out_L; j++)
271 acc += WL[i + max_n_nodes * j] * delta_curr[j];
272 delta_prev[i] = acc;
273 }
275
276 double *tmp_swap = delta_curr;
277 delta_curr = delta_prev;
278 delta_prev = tmp_swap;
279 }
280
281 /* Step 3: dE_dG[m] is the delta of the input layer. */
282 for (int m = tid; m < n_input; m += nth) {
283 dE_dG[m + n_input_max * c + (long long)n_input_max * n_committee * atom] =
284 delta_curr[m];
285 }
286}
287
288/*******************************************************************************
289 * \brief Once-at-init upload of the committee-network weights, biases,
290 * topology, activation functions and per-element atomic energies into
291 * the persistent state buffers (nn_n_nodes_dev, nn_act_fnct_dev,
292 * nn_atom_energies_dev, nn_W_packed_dev, nn_b_packed_dev), so the
293 * per-step launcher skips these host-to-device copies. Idempotent on the
294 * network shape: a second call with the same (n_committee, n_layer,
295 * max_n_nodes, n_ele) returns immediately, and a call with a different
296 * shape is an error. The packed layouts match the kernel's indexing
297 * (see nnp_nn_kernel_atomwise). The stream is synchronized before return
298 * so the caller's host staging arrays can go out of scope. Returns 0 on
299 * success, -1 on failure.
300 * \author Dhruv Sharma (ds2173@cam.ac.uk)
301 ******************************************************************************/
302extern "C" int
303nnp_gpu_upload_nn_packed_c(int n_committee, int n_layer, int max_n_nodes,
304 int n_ele, const int *h_n_nodes,
305 const int *h_act_fnct, const double *h_atom_energies,
306 const double *h_W_packed, const double *h_b_packed) {
307 if (n_committee <= 0 || n_layer < 2 || max_n_nodes <= 0 || n_ele <= 0) {
308 fprintf(stderr,
309 "ERROR: nnp_gpu_upload_nn_packed: bad sizes "
310 "n_committee=%d n_layer=%d max_n_nodes=%d n_ele=%d\n",
311 n_committee, n_layer, max_n_nodes, n_ele);
312 return -1;
313 }
314 nnp_gpu_state_t *state = nnp_gpu_state_get();
315 if (state == NULL)
316 return -1;
317
318 if (state->nn_packed_uploaded) {
319 if (state->nn_packed_n_committee == n_committee &&
320 state->nn_packed_n_layer == n_layer &&
321 state->nn_packed_max_n_nodes == max_n_nodes &&
322 state->nn_packed_n_ele == n_ele) {
323 return 0;
324 }
325 fprintf(stderr,
326 "ERROR: nnp_gpu_upload_nn_packed: shape mismatch on second "
327 "upload (n_committee %d->%d, n_layer %d->%d, "
328 "max_n_nodes %d->%d, n_ele %d->%d)\n",
329 state->nn_packed_n_committee, n_committee, state->nn_packed_n_layer,
330 n_layer, state->nn_packed_max_n_nodes, max_n_nodes,
331 state->nn_packed_n_ele, n_ele);
332 return -1;
333 }
334
335 const size_t n_nodes_cnt = (size_t)n_layer * (size_t)n_ele;
336 const size_t act_cnt = (size_t)(n_layer - 1);
337 const size_t atom_e_cnt = (size_t)n_ele;
338 const size_t W_cnt = (size_t)max_n_nodes * (size_t)max_n_nodes *
339 (size_t)n_committee * (size_t)(n_layer - 1) *
340 (size_t)n_ele;
341 const size_t b_cnt = (size_t)max_n_nodes * (size_t)n_committee *
342 (size_t)(n_layer - 1) * (size_t)n_ele;
343
344 if (nnp_gpu_pbuf_ensure((void **)&state->nn_n_nodes_dev,
345 &state->nn_n_nodes_dev_cap,
346 sizeof(int) * n_nodes_cnt))
347 return -1;
348 if (nnp_gpu_pbuf_ensure((void **)&state->nn_act_fnct_dev,
349 &state->nn_act_fnct_dev_cap, sizeof(int) * act_cnt))
350 return -1;
351 if (nnp_gpu_pbuf_ensure((void **)&state->nn_atom_energies_dev,
352 &state->nn_atom_energies_dev_cap,
353 sizeof(double) * atom_e_cnt))
354 return -1;
355 if (nnp_gpu_pbuf_ensure((void **)&state->nn_W_packed_dev,
356 &state->nn_W_packed_dev_cap, sizeof(double) * W_cnt))
357 return -1;
358 if (nnp_gpu_pbuf_ensure((void **)&state->nn_b_packed_dev,
359 &state->nn_b_packed_dev_cap, sizeof(double) * b_cnt))
360 return -1;
361
362 offloadMemcpyAsyncHtoD(state->nn_n_nodes_dev, h_n_nodes,
363 sizeof(int) * n_nodes_cnt, state->stream);
364 offloadMemcpyAsyncHtoD(state->nn_act_fnct_dev, h_act_fnct,
365 sizeof(int) * act_cnt, state->stream);
366 offloadMemcpyAsyncHtoD(state->nn_atom_energies_dev, h_atom_energies,
367 sizeof(double) * atom_e_cnt, state->stream);
368 offloadMemcpyAsyncHtoD(state->nn_W_packed_dev, h_W_packed,
369 sizeof(double) * W_cnt, state->stream);
370 offloadMemcpyAsyncHtoD(state->nn_b_packed_dev, h_b_packed,
371 sizeof(double) * b_cnt, state->stream);
372 offloadStreamSynchronize(state->stream);
373
374 state->nn_packed_uploaded = 1;
375 state->nn_packed_n_committee = n_committee;
376 state->nn_packed_n_layer = n_layer;
377 state->nn_packed_max_n_nodes = max_n_nodes;
378 state->nn_packed_n_ele = n_ele;
379 return 0;
380}
381
382/*******************************************************************************
383 * \brief Per-step committee-network launcher: evaluate every atom and committee
384 * member straight from the scaled device descriptor state->G_dev_scaled,
385 * then read the per-atom energies and descriptor gradients back to the
386 * host. The network weights, biases, topology, activation functions and
387 * atomic energies are read from the once-at-init device cache
388 * (state->nn_*_dev, populated by nnp_gpu_upload_nn_packed_c), so the
389 *only per-step input that varies is the rank-local element index of each atom
390 *(h_ele_ind), which is uploaded once and shared with the scaling launcher. The
391 *grid is one block per (atom, committee) pair; the kernel writes
392 *h_atomic_energy[c + n_committee * atom] and, when the caller passes a buffer
393 *for it, h_dE_dG[m + n_input_max * c + n_input_max * n_committee * atom].
394 * h_dE_dG may be NULL, which leaves the gradient on the device for the
395 * force scatter and skips a read-back of several megabytes per step.
396 * The stream is synchronized after the read-backs so the caller can read
397 * the host arrays it asked for on return. Networks larger than the 48 KB
398 * default shared-memory window opt into the device per-block maximum;
399 * only sizes beyond that limit are rejected. Returns 0 on success, -1 on
400 * failure.
401 * \author Dhruv Sharma (ds2173@cam.ac.uk)
402 ******************************************************************************/
403extern "C" int
404nnp_gpu_nn_atomwise_with_G_dev_c(const int n_atoms, const int n_committee,
405 const int n_layer, const int max_n_nodes,
406 const int n_input_max, const int normnodes,
407 const int n_ele, const int *h_ele_ind,
408 double *h_atomic_energy, double *h_dE_dG) {
409
410 if (n_atoms <= 0 || n_committee <= 0 || n_layer < 2 || max_n_nodes <= 0 ||
411 n_input_max <= 0 || n_ele <= 0) {
412 fprintf(stderr, "ERROR: nnp_gpu_nn_atomwise_with_G_dev: bad sizes\n");
413 return -1;
414 }
415 /* Shared-memory footprint of nnp_nn_kernel_atomwise for these sizes:
416 * per-layer pre/post node values plus the two backpropagation vectors. */
417 const size_t smem_bytes =
418 sizeof(double) * 2ULL * (size_t)max_n_nodes * ((size_t)n_layer + 1ULL);
419
420 /* Networks beyond the 48 KB default dynamic-shared-memory window (more
421 * than ~3000 nodes summed over the layers) opt into the device's
422 * per-block maximum instead of being rejected.
423 * The granted size is a property of the network, not of the device, so what
424 * is remembered is the number of bytes already opted into rather than a
425 * boolean. A process that runs NNP environments in sequence can raise the
426 * requirement, and launching against a smaller earlier grant would fail; a
427 * request at or below the standing grant needs no new opt-in. */
428 static size_t smem_optin_bytes = 0;
429 if (smem_bytes > 48ULL * 1024ULL && smem_bytes > smem_optin_bytes) {
430#if defined(__OFFLOAD_CUDA)
431 int dev = 0, smem_max = 0;
432 OFFLOAD_CHECK(cudaGetDevice(&dev));
433 OFFLOAD_CHECK(cudaDeviceGetAttribute(
434 &smem_max, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev));
435 if ((size_t)smem_max < smem_bytes ||
436 cudaFuncSetAttribute(nnp_nn_kernel_atomwise,
437 cudaFuncAttributeMaxDynamicSharedMemorySize,
438 (int)smem_bytes) != cudaSuccess) {
439 fprintf(stderr,
440 "ERROR: nnp_gpu_nn_atomwise_with_G_dev: max_n_nodes=%d "
441 "n_input_max=%d needs %zu bytes of shared memory per block; "
442 "device allows %d.\n",
443 max_n_nodes, n_input_max, smem_bytes, smem_max);
444 return -1;
445 }
446#elif defined(__OFFLOAD_HIP)
447 int dev = 0, smem_max = 0;
448 OFFLOAD_CHECK(hipGetDevice(&dev));
449 OFFLOAD_CHECK(hipDeviceGetAttribute(
450 &smem_max, hipDeviceAttributeMaxSharedMemoryPerBlock, dev));
451 if ((size_t)smem_max < smem_bytes) {
452 fprintf(stderr,
453 "ERROR: nnp_gpu_nn_atomwise_with_G_dev: max_n_nodes=%d "
454 "n_input_max=%d needs %zu bytes of shared memory per block; "
455 "device allows %d.\n",
456 max_n_nodes, n_input_max, smem_bytes, smem_max);
457 return -1;
458 }
459#endif
460 smem_optin_bytes = smem_bytes;
461 }
462
463 const size_t E_cnt = (size_t)n_committee * (size_t)n_atoms;
464 const size_t dEdG_cnt =
465 (size_t)n_input_max * (size_t)n_committee * (size_t)n_atoms;
466
467 nnp_gpu_state_t *state = nnp_gpu_state_get();
468 if (state == NULL)
469 return -1;
470 if (state->G_dev_scaled == NULL) {
471 fprintf(stderr, "ERROR: nnp_gpu_nn_atomwise_with_G_dev: G_dev_scaled "
472 "not initialised (call nnp_gpu_begin_descriptor_step + "
473 "nnp_gpu_scale_G_dev_c first)\n");
474 return -1;
475 }
476 if (!state->nn_packed_uploaded) {
477 fprintf(stderr, "ERROR: nnp_gpu_nn_atomwise_with_G_dev: NN weights not "
478 "uploaded (call nnp_gpu_upload_nn_packed_c at NNP init)\n");
479 return -1;
480 }
481 if (state->nn_packed_n_committee != n_committee ||
482 state->nn_packed_n_layer != n_layer ||
483 state->nn_packed_max_n_nodes != max_n_nodes ||
484 state->nn_packed_n_ele != n_ele) {
485 fprintf(stderr,
486 "ERROR: nnp_gpu_nn_atomwise_with_G_dev: cached shape "
487 "(n_committee=%d n_layer=%d max_n_nodes=%d n_ele=%d) "
488 "differs from kernel call (%d/%d/%d/%d)\n",
489 state->nn_packed_n_committee, state->nn_packed_n_layer,
490 state->nn_packed_max_n_nodes, state->nn_packed_n_ele, n_committee,
491 n_layer, max_n_nodes, n_ele);
492 return -1;
493 }
494
495 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_atomE_out,
496 &state->pbuf_atomE_out_cap, sizeof(double) * E_cnt))
497 return -1;
498 if (nnp_gpu_pbuf_ensure((void **)&state->pbuf_dEdG_out,
499 &state->pbuf_dEdG_out_cap, sizeof(double) * dEdG_cnt))
500 return -1;
501 /* The kernel writes only the rows below each element's own input width, so
502 * for a model whose elements have different descriptor counts the rows in
503 * between stay untouched. The force scatter never reads them, so the zeroing
504 * is worth its bandwidth only when the caller is going to copy the whole
505 * buffer back and would otherwise receive uninitialised device memory. */
506 if (h_dE_dG != NULL)
507 offloadMemsetAsync(state->pbuf_dEdG_out, 0, sizeof(double) * dEdG_cnt,
508 state->stream);
509
510 /* Ensure the rank-local element index is on the device. Shared with the
511 * scaling launcher, which applies the same guard, so only one host-to-device
512 * copy happens per step across both. */
513 if (nnp_gpu_step_ele_ind_ensure(state, h_ele_ind, n_atoms))
514 return -1;
515
516 /* Every kernel input except step_ele_ind_dev lives in state->nn_*_dev,
517 * uploaded once by nnp_gpu_upload_nn_packed_c. */
518 {
519 dim3 grid(n_atoms, n_committee, 1);
520 dim3 blk(32, 1, 1);
521 nnp_nn_kernel_atomwise<<<grid, blk, smem_bytes, state->stream>>>(
522 n_atoms, n_committee, n_layer, max_n_nodes, n_input_max, normnodes,
523 n_ele, state->nn_n_nodes_dev, state->nn_act_fnct_dev,
524 state->step_ele_ind_dev, state->nn_atom_energies_dev,
525 state->nn_W_packed_dev, state->nn_b_packed_dev, state->G_dev_scaled,
526 state->pbuf_atomE_out, state->pbuf_dEdG_out);
527 }
528 OFFLOAD_CHECK(offloadPeekAtLastError());
529
530 offloadMemcpyAsyncDtoH(h_atomic_energy, state->pbuf_atomE_out,
531 sizeof(double) * E_cnt, state->stream);
532 /* The descriptor gradient is optional: the force scatter reads the device
533 * copy, so the whole-step driver passes NULL and skips a read-back that runs
534 * to megabytes per step on the one synchronisation point of the step. */
535 if (h_dE_dG != NULL)
536 offloadMemcpyAsyncDtoH(h_dE_dG, state->pbuf_dEdG_out,
537 sizeof(double) * dEdG_cnt, state->stream);
538 /* Sync after the D2Hs: the caller reads h_atomic_energy (and h_dE_dG when it
539 * asked for it) on return, so the read-after-return contract requires the
540 * copies to complete here. */
541 offloadStreamSynchronize(state->stream);
542 return 0;
543}
544
545#endif /* defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP) */
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i
real(dp), dimension(3) c
real(dp), dimension(3) d
real(dp), dimension(3) b
Definition atom.F:9