(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_walk.cu
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
10
11#include <stdio.h>
12
13#include "nnp_gpu_internal.h"
14#include "nnp_gpu_state.h"
15
16#if defined(_OMP_H)
17#error "OpenMP should not be used in .cu files to accommodate HIP."
18#endif
19
20/*
21 * On-device cell-list walk and the per-step host-to-device uploads that feed
22 * it. The walk kernel reproduces the host routine
23 * nnp_compute_neighbors_cell_list: for every atom it visits the bins of its
24 * local stencil, follows each bin's linked list of periodic images, applies
25 * the same self-skip, image-match and cutoff filters as the host, and appends
26 * the surviving neighbours into the per-(atom, group) walk buffers. The result
27 * is the exact same neighbour SET the host produces; only the append order
28 * within a set differs (see the kernel banner). The upload launchers copy the
29 * coordinates, image table, cell-list bins and the species-pair routing table
30 * the kernel reads, all onto the single NNP work stream so downstream kernels
31 * observe the data through stream ordering.
32 */
33
34/*******************************************************************************
35 * \brief Device helper for the 1-based linear bin index, matching the host
36 * routine nnp_cell_list_linear_index. bx, by, bz are 1-based bin
37 * coordinates; nbx, nby are the bin counts along x and y.
38 * \author Dhruv Sharma (ds2173@cam.ac.uk)
39 ******************************************************************************/
40__device__ __forceinline__ int nnp_walk_lin_idx_1based(int bx, int by, int bz,
41 int nbx, int nby) {
42 return bx + nbx * ((by - 1) + nby * (bz - 1));
43}
44
45/*******************************************************************************
46 * \brief Device version of the host routine nnp_cell_list_exact_image_match.
47 * perd is 0/1 per dimension (periodic flag); exact_pbc_copies bounds
48 * the accepted shift along each periodic dimension. Returns 1 to accept
49 * the image, 0 to reject it. coord_scaled is laid out as
50 * [3 * num_atoms_global], xyz-interleaved, matching Fortran
51 * column-major (3, n) storage when uploaded as a flat array. i_global
52 * and j_global are 0-based atom indices.
53 *
54 * The Fortran NINT rounds half away from zero, whereas the CUDA
55 * intrinsic __double2int_rn rounds half to even. The two disagree only
56 * when the coordinate difference lands exactly on a half-integer, which
57 * continuous coordinates essentially never do; a portable
58 * round-half-away-from-zero is used here so the device result is
59 * bit-exact with the host.
60 * \author Dhruv Sharma (ds2173@cam.ac.uk)
61 ******************************************************************************/
62__device__ __forceinline__ int
63nnp_walk_image_match(const double *coord_scaled, int i_global, int j_global,
64 int sx, int sy, int sz, const int *perd,
65 const int *exact_pbc_copies) {
66 int shifts[3] = {sx, sy, sz};
67 for (int d = 0; d < 3; d++) {
68 if (perd[d] == 1) {
69 double diff =
70 coord_scaled[3 * i_global + d] - coord_scaled[3 * j_global + d];
71 double rounded = (diff >= 0.0) ? floor(diff + 0.5) : -floor(-diff + 0.5);
72 int minimal_shift = (int)rounded;
73 int delta = minimal_shift - shifts[d];
74 if (delta < 0)
75 delta = -delta;
76 if (delta > exact_pbc_copies[d])
77 return 0;
78 } else if (shifts[d] != 0) {
79 return 0;
80 }
81 }
82 return 1;
83}
84
85/*******************************************************************************
86 * \brief On-device cell-list walk kernel. One warp handles one
87 * (atom_in_e_slot, e_idx) pair: blockIdx.x selects the 0-based atom
88 * slot within element e, blockIdx.y selects the 0-based element index,
89 * and the 32 lanes of the block share the work for that central atom.
90 * The kernel reproduces the host routine nnp_compute_neighbors_cell_list
91 * and appends the surviving neighbours into the per-(atom, group) flat
92 * buffers walk_*_ind_dev / walk_*_image_idx_dev, tracking per-(slot, s,
93 * e) counts in walk_n_*_pa_dev.
94 *
95 * Walk math (line-for-line with the host):
96 * - Bin index from an INT-and-clamp on the central atom's primary
97 * coordinate (nnp_cell_list_bin_from_position), with a small width
98 * floor to avoid dividing by a degenerate bin width.
99 * - Linear bin index bx + nbx * ((by - 1) + nby * (bz - 1)) via
100 * nnp_walk_lin_idx_1based.
101 * - image_match: NINT(coord_scaled_i - coord_scaled_j) minus the
102 * image shift, bounded per periodic dimension by exact_pbc_copies
103 * (nnp_walk_image_match).
104 * - Displacement reconstruction:
105 * image_pos = coord(j) + image_translation(img),
106 * dr = coord(i) - image_pos,
107 * norm = sqrt(dr . dr).
108 * - Filtering: reject beyond the pair maximum cutoff, then per radial
109 * / ang1 / ang2 group with its per-(s, ele) cutoff.
110 *
111 * Each lane strides over the flattened bin stencil and follows the
112 * linked-list chains; matches are appended with atomicAdd on the
113 * per-(atom, group) counter, so the neighbour append ORDER differs from
114 * the serial host walk. The downstream symmetry functions sum over the
115 * neighbour SET, so this only reorders the floating-point reduction and
116 * the results match the host. On a per-(slot, s, e) overflow the kernel
117 * sets overflow_flag_dev via atomicExch; the launcher reads the flag
118 * back and asks the caller to grow the buffers and retry.
119 * \author Dhruv Sharma (ds2173@cam.ac.uk)
120 ******************************************************************************/
121__global__ void nnp_cell_list_walk_kernel(
122 int num_atoms_global, int n_ele, int n_atoms_in_e_max, int max_n_neigh_rad,
123 int max_n_neigh_ang1, int max_n_neigh_ang2, int max_n_radgrp,
124 int max_n_anggrp, int max_pair_grps, int nbx, int nby, int nbz,
125 double bx_lo, double by_lo, double bz_lo, double bw_x, double bw_y,
126 double bw_z, int span_x, int span_y, int span_z, int perd_x, int perd_y,
127 int perd_z, int epbc_x, int epbc_y, int epbc_z,
128 const double *__restrict__ coord_dev,
129 const double *__restrict__ image_table_dev,
130 const double *__restrict__ coord_scaled_dev,
131 const int *__restrict__ ele_ind_global_dev,
132 const int *__restrict__ image_atom_dev,
133 const int *__restrict__ image_shift_dev, const int *__restrict__ head_dev,
134 const int *__restrict__ next_dev,
135 const int *__restrict__ atoms_global_per_e_dev,
136 const int *__restrict__ n_atoms_in_e_dev,
137 const int *__restrict__ pair_n_rad_dev,
138 const int *__restrict__ pair_n_ang1_dev,
139 const int *__restrict__ pair_n_ang2_dev,
140 const double *__restrict__ pair_max_cutoff_dev,
141 const int *__restrict__ pair_rad_groups_dev,
142 const int *__restrict__ pair_ang1_groups_dev,
143 const int *__restrict__ pair_ang2_groups_dev,
144 const double *__restrict__ rad_cutoff_dev,
145 const double *__restrict__ ang_cutoff_dev,
146 int *__restrict__ walk_rad_ind_dev,
147 int *__restrict__ walk_rad_image_idx_dev,
148 int *__restrict__ walk_ang1_ind_dev,
149 int *__restrict__ walk_ang1_image_idx_dev,
150 int *__restrict__ walk_ang2_ind_dev,
151 int *__restrict__ walk_ang2_image_idx_dev,
152 int *__restrict__ walk_n_rad_pa_dev, int *__restrict__ walk_n_ang1_pa_dev,
153 int *__restrict__ walk_n_ang2_pa_dev, int *__restrict__ overflow_flag_dev) {
154
155 /* Block layout: blockIdx.x = atom_in_e_slot (0-based),
156 * blockIdx.y = e_idx (0-based). One warp per atom: every
157 * lane recomputes the (cheap) central-atom setup, then the
158 * stencil bins are split across threadIdx.x and matches are
159 * appended with atomicAdd on the per-(atom, group) counter. */
160 const int slot = blockIdx.x;
161 const int e0 = blockIdx.y;
162 if (e0 >= n_ele)
163 return;
164 if (slot >= n_atoms_in_e_dev[e0])
165 return;
166
167 /* Pull i_global (1-based in host, 0-based on device for indexing). */
168 const int i1 = atoms_global_per_e_dev[slot + n_atoms_in_e_max * e0];
169 if (i1 <= 0)
170 return;
171 const int i0 = i1 - 1;
172 const int i_ele1 = ele_ind_global_dev[i0]; /* 1-based */
173 const int i_ele0 = i_ele1 - 1;
174
175 const double cx = coord_dev[3 * i0 + 0];
176 const double cy = coord_dev[3 * i0 + 1];
177 const double cz = coord_dev[3 * i0 + 2];
178
179 /* Bin index, matching host nnp_cell_list_bin_from_position. */
180 const double rebuild_eps = 1.0e-12;
181 const double bw_xs = (bw_x > rebuild_eps) ? bw_x : rebuild_eps;
182 const double bw_ys = (bw_y > rebuild_eps) ? bw_y : rebuild_eps;
183 const double bw_zs = (bw_z > rebuild_eps) ? bw_z : rebuild_eps;
184 int bin_ix = (int)((cx - bx_lo) / bw_xs) + 1;
185 int bin_iy = (int)((cy - by_lo) / bw_ys) + 1;
186 int bin_iz = (int)((cz - bz_lo) / bw_zs) + 1;
187 if (bin_ix < 1)
188 bin_ix = 1;
189 else if (bin_ix > nbx)
190 bin_ix = nbx;
191 if (bin_iy < 1)
192 bin_iy = 1;
193 else if (bin_iy > nby)
194 bin_iy = nby;
195 if (bin_iz < 1)
196 bin_iz = 1;
197 else if (bin_iz > nbz)
198 bin_iz = nbz;
199
200 const int bx_lo_i = max(1, bin_ix - span_x);
201 const int bx_hi_i = min(nbx, bin_ix + span_x);
202 const int by_lo_i = max(1, bin_iy - span_y);
203 const int by_hi_i = min(nby, bin_iy + span_y);
204 const int bz_lo_i = max(1, bin_iz - span_z);
205 const int bz_hi_i = min(nbz, bin_iz + span_z);
206
207 /* Bin walk: flatten the stencil and stride the bins across the warp.
208 * Each lane scans its bins' chains; matches append via atomicAdd, so the
209 * neighbour append order differs from the serial host walk order. The SF
210 * sums over the neighbour SET, so this only reorders the FP reduction. */
211 const int nbx_w = bx_hi_i - bx_lo_i + 1;
212 const int nby_w = by_hi_i - by_lo_i + 1;
213 const int nbz_w = bz_hi_i - bz_lo_i + 1;
214 const int n_bins_w = nbx_w * nby_w * nbz_w;
215 for (int bsel = threadIdx.x; bsel < n_bins_w; bsel += blockDim.x) {
216 const int bx = bx_lo_i + (bsel % nbx_w);
217 const int by = by_lo_i + ((bsel / nbx_w) % nby_w);
218 const int bz = bz_lo_i + (bsel / (nbx_w * nby_w));
219 const int lin = nnp_walk_lin_idx_1based(bx, by, bz, nbx, nby);
220 int img1 = head_dev[lin - 1]; /* head is 1-based-indexed Fortran array */
221 while (img1 > 0) {
222 const int j1 = image_atom_dev[img1 - 1];
223 const int j0 = j1 - 1;
224 const int sx = image_shift_dev[3 * (img1 - 1) + 0];
225 const int sy_ = image_shift_dev[3 * (img1 - 1) + 1];
226 const int sz_ = image_shift_dev[3 * (img1 - 1) + 2];
227
228 /* Self-skip at zero shift */
229 if (j1 == i1 && sx == 0 && sy_ == 0 && sz_ == 0) {
230 img1 = next_dev[img1 - 1];
231 continue;
232 }
233 /* Image-match check */
234 const int perd_arr[3] = {perd_x, perd_y, perd_z};
235 const int epbc_arr[3] = {epbc_x, epbc_y, epbc_z};
236 if (!nnp_walk_image_match(coord_scaled_dev, i0, j0, sx, sy_, sz_,
237 perd_arr, epbc_arr)) {
238 img1 = next_dev[img1 - 1];
239 continue;
240 }
241
242 const double image_x =
243 coord_dev[3 * j0 + 0] + image_table_dev[3 * (img1 - 1) + 0];
244 const double image_y =
245 coord_dev[3 * j0 + 1] + image_table_dev[3 * (img1 - 1) + 1];
246 const double image_z =
247 coord_dev[3 * j0 + 2] + image_table_dev[3 * (img1 - 1) + 2];
248 const double drx = cx - image_x;
249 const double dry = cy - image_y;
250 const double drz = cz - image_z;
251 const double norm = sqrt(drx * drx + dry * dry + drz * drz);
252
253 const int j_ele1 = ele_ind_global_dev[j0];
254 const int j_ele0 = j_ele1 - 1;
255 const int pair_idx = i_ele0 + n_ele * j_ele0;
256 const double pair_max = pair_max_cutoff_dev[pair_idx];
257
258 if (norm < pair_max) {
259 /* Radial */
260 const int n_rad_pair = pair_n_rad_dev[pair_idx];
261 for (int g = 0; g < n_rad_pair; g++) {
262 const int s1 = pair_rad_groups_dev[g + max_pair_grps * pair_idx];
263 const int s0 = s1 - 1;
264 const double cutoff = rad_cutoff_dev[s0 + max_n_radgrp * i_ele0];
265 if (norm < cutoff) {
266 /* 64-bit offsets, matching the reader side in nnp_gpu_pack.h:
267 * the neighbour tables exceed 2^31 entries on large ranks. */
268 const long long pa_off =
269 (long long)slot + (long long)n_atoms_in_e_max *
270 ((long long)s0 + (long long)max_n_radgrp *
271 (long long)i_ele0);
272 const int k = atomicAdd(&walk_n_rad_pa_dev[pa_off], 1);
273 if (k >= max_n_neigh_rad) {
274 atomicExch(overflow_flag_dev, 1);
275 } else {
276 const long long out_off =
277 (long long)k + (long long)max_n_neigh_rad * pa_off;
278 walk_rad_ind_dev[out_off] = j0; /* 0-based */
279 walk_rad_image_idx_dev[out_off] = img1 - 1; /* 0-based */
280 }
281 }
282 }
283 /* Ang1 */
284 const int n_ang1_pair = pair_n_ang1_dev[pair_idx];
285 for (int g = 0; g < n_ang1_pair; g++) {
286 const int s1 = pair_ang1_groups_dev[g + max_pair_grps * pair_idx];
287 const int s0 = s1 - 1;
288 const double cutoff = ang_cutoff_dev[s0 + max_n_anggrp * i_ele0];
289 if (norm < cutoff) {
290 const long long pa_off =
291 (long long)slot + (long long)n_atoms_in_e_max *
292 ((long long)s0 + (long long)max_n_anggrp *
293 (long long)i_ele0);
294 const int k = atomicAdd(&walk_n_ang1_pa_dev[pa_off], 1);
295 if (k >= max_n_neigh_ang1) {
296 atomicExch(overflow_flag_dev, 1);
297 } else {
298 const long long out_off =
299 (long long)k + (long long)max_n_neigh_ang1 * pa_off;
300 walk_ang1_ind_dev[out_off] = j0; /* 0-based */
301 walk_ang1_image_idx_dev[out_off] = img1 - 1; /* 0-based */
302 }
303 }
304 }
305 /* Ang2 */
306 const int n_ang2_pair = pair_n_ang2_dev[pair_idx];
307 for (int g = 0; g < n_ang2_pair; g++) {
308 const int s1 = pair_ang2_groups_dev[g + max_pair_grps * pair_idx];
309 const int s0 = s1 - 1;
310 const double cutoff = ang_cutoff_dev[s0 + max_n_anggrp * i_ele0];
311 if (norm < cutoff) {
312 const long long pa_off =
313 (long long)slot + (long long)n_atoms_in_e_max *
314 ((long long)s0 + (long long)max_n_anggrp *
315 (long long)i_ele0);
316 const int k = atomicAdd(&walk_n_ang2_pa_dev[pa_off], 1);
317 if (k >= max_n_neigh_ang2) {
318 atomicExch(overflow_flag_dev, 1);
319 } else {
320 const long long out_off =
321 (long long)k + (long long)max_n_neigh_ang2 * pa_off;
322 walk_ang2_ind_dev[out_off] = j0; /* 0-based */
323 walk_ang2_image_idx_dev[out_off] = img1 - 1; /* 0-based */
324 }
325 }
326 }
327 }
328
329 img1 = next_dev[img1 - 1];
330 }
331 }
332}
333
334/*******************************************************************************
335 * \brief Per-MD-step host-to-device copy of cell_list_cache%coord_primary into
336 * state->coord_dev. h_coord is a contiguous 3*num_atoms host buffer
337 * (xyz interleaved). The device buffer is grow-only via
338 * nnp_gpu_pbuf_ensure. The copy is queued on state->stream and the
339 * launcher returns immediately; downstream kernels on the same stream
340 * observe the updated positions in order.
341 *
342 * This is the first GPU call of every MD step, so the observability step
343 * counter state->md_step is incremented here.
344 * \author Dhruv Sharma (ds2173@cam.ac.uk)
345 ******************************************************************************/
346extern "C" int nnp_gpu_upload_positions_step_c(int num_atoms,
347 const double *h_coord) {
348 if (num_atoms <= 0) {
349 fprintf(stderr, "ERROR: nnp_gpu_upload_positions_step: bad num_atoms=%d\n",
350 num_atoms);
351 return -1;
352 }
353 nnp_gpu_state_t *state = nnp_gpu_state_get();
354 if (state == NULL)
355 return -1;
356
357 /* First GPU call of every MD step. The observability step counter is
358 * advanced here so that per-step device work can be attributed. */
359 state->md_step++;
360
361 const size_t bytes = sizeof(double) * 3 * (size_t)num_atoms;
362 if (nnp_gpu_pbuf_ensure((void **)&state->coord_dev, &state->coord_dev_cap,
363 bytes))
364 return -1;
365 offloadMemcpyAsyncHtoD(state->coord_dev, h_coord, bytes, state->stream);
366 return 0;
367}
368
369/*******************************************************************************
370 * \brief Per-MD-step host-to-device copy of cell_list_cache%image_translation
371 * into state->image_table_dev. h_image_table is a contiguous
372 * 3*n_images host buffer (xyz per image, the columns of the Fortran
373 * image_translation(3, n_images) array). The device buffer is grow-only
374 * and the copy is queued on state->stream. The image table changes only
375 * when the cell-list bins rebuild, so the driver calls this only on
376 * rebuild steps.
377 * \author Dhruv Sharma (ds2173@cam.ac.uk)
378 ******************************************************************************/
379extern "C" int nnp_gpu_upload_image_table_c(int n_images,
380 const double *h_image_table) {
381 if (n_images <= 0) {
382 fprintf(stderr, "ERROR: nnp_gpu_upload_image_table: bad n_images=%d\n",
383 n_images);
384 return -1;
385 }
386 nnp_gpu_state_t *state = nnp_gpu_state_get();
387 if (state == NULL)
388 return -1;
389
390 const size_t bytes = sizeof(double) * 3 * (size_t)n_images;
391 if (nnp_gpu_pbuf_ensure((void **)&state->image_table_dev,
392 &state->image_table_dev_cap, bytes))
393 return -1;
394 offloadMemcpyAsyncHtoD(state->image_table_dev, h_image_table, bytes,
395 state->stream);
396 return 0;
397}
398
399/*******************************************************************************
400 * \brief Host-to-device copy of the cell-list bins (image_atom, image_shift,
401 * head, next) into state, read by the walk kernel. Sizes:
402 * image_atom: n_images ints
403 * image_shift: 3 * n_images ints
404 * head: n_cells ints
405 * next: n_images ints
406 * The device buffers are grow-only via nnp_gpu_pbuf_ensure. The four
407 * copies are queued on state->stream and the stream is synchronized
408 * before returning: the caller passes kind-converted temporaries whose
409 * lifetime ends at the end of the call statement. Called whenever the
410 * cell-list is rebinned.
411 * \author Dhruv Sharma (ds2173@cam.ac.uk)
412 ******************************************************************************/
413extern "C" int nnp_gpu_upload_cell_list_c(int n_images, int n_cells,
414 const int *h_image_atom,
415 const int *h_image_shift,
416 const int *h_head,
417 const int *h_next) {
418 if (n_images <= 0 || n_cells <= 0) {
419 fprintf(stderr,
420 "ERROR: nnp_gpu_upload_cell_list: bad sizes "
421 "n_images=%d n_cells=%d\n",
422 n_images, n_cells);
423 return -1;
424 }
425 nnp_gpu_state_t *state = nnp_gpu_state_get();
426 if (state == NULL)
427 return -1;
428
429 const size_t bytes_imgs = sizeof(int) * (size_t)n_images;
430 const size_t bytes_shifts = sizeof(int) * 3 * (size_t)n_images;
431 const size_t bytes_head = sizeof(int) * (size_t)n_cells;
432
433 if (nnp_gpu_pbuf_ensure((void **)&state->cell_image_atom_dev,
434 &state->cell_image_atom_dev_cap, bytes_imgs))
435 return -1;
436 if (nnp_gpu_pbuf_ensure((void **)&state->cell_image_shift_dev,
437 &state->cell_image_shift_dev_cap, bytes_shifts))
438 return -1;
439 if (nnp_gpu_pbuf_ensure((void **)&state->cell_head_dev,
440 &state->cell_head_dev_cap, bytes_head))
441 return -1;
442 if (nnp_gpu_pbuf_ensure((void **)&state->cell_next_dev,
443 &state->cell_next_dev_cap, bytes_imgs))
444 return -1;
445
446 offloadMemcpyAsyncHtoD(state->cell_image_atom_dev, h_image_atom, bytes_imgs,
447 state->stream);
448 offloadMemcpyAsyncHtoD(state->cell_image_shift_dev, h_image_shift,
449 bytes_shifts, state->stream);
450 offloadMemcpyAsyncHtoD(state->cell_head_dev, h_head, bytes_head,
451 state->stream);
452 offloadMemcpyAsyncHtoD(state->cell_next_dev, h_next, bytes_imgs,
453 state->stream);
454 /* Sync so the host buffers can go out of scope safely after return. */
455 offloadStreamSynchronize(state->stream);
456 return 0;
457}
458
459/*******************************************************************************
460 * \brief Once-at-NNP-init host-to-device copy of the species-pair routing
461 * table. The caller passes host arrays already flat in the layouts the
462 * walk kernel expects. The upload is idempotent: a second call with the
463 * same shape detects state->pair_map_uploaded and returns 0 without
464 * re-uploading, and a call with a different shape is rejected with an
465 * error. On the first call the device buffers are grown, the tables are
466 * copied on state->stream, the stream is synchronized so the host
467 * buffers may be freed on return, and the recorded shape fields are set.
468 * \author Dhruv Sharma (ds2173@cam.ac.uk)
469 ******************************************************************************/
470extern "C" int nnp_gpu_upload_pair_map_c(
471 int n_ele, int max_pair_grps, int max_n_radgrp, int max_n_anggrp,
472 const int *h_n_rad, const int *h_n_ang1, const int *h_n_ang2,
473 const double *h_pair_max_cutoff, const int *h_rad_groups,
474 const int *h_ang1_groups, const int *h_ang2_groups,
475 const double *h_rad_cutoff, const double *h_ang_cutoff) {
476 if (n_ele <= 0 || max_pair_grps <= 0 || max_n_radgrp <= 0 ||
477 max_n_anggrp <= 0) {
478 fprintf(stderr,
479 "ERROR: nnp_gpu_upload_pair_map: bad sizes "
480 "n_ele=%d max_pair_grps=%d max_n_radgrp=%d max_n_anggrp=%d\n",
481 n_ele, max_pair_grps, max_n_radgrp, max_n_anggrp);
482 return -1;
483 }
484 nnp_gpu_state_t *state = nnp_gpu_state_get();
485 if (state == NULL)
486 return -1;
487
488 if (state->pair_map_uploaded) {
489 if (state->pair_map_n_ele == n_ele &&
490 state->pair_map_max_pair_grps == max_pair_grps &&
491 state->pair_map_max_n_radgrp == max_n_radgrp &&
492 state->pair_map_max_n_anggrp == max_n_anggrp) {
493 return 0;
494 }
495 fprintf(stderr,
496 "ERROR: nnp_gpu_upload_pair_map: shape mismatch on second "
497 "upload (n_ele %d->%d, max_pair_grps %d->%d, "
498 "max_n_radgrp %d->%d, max_n_anggrp %d->%d)\n",
499 state->pair_map_n_ele, n_ele, state->pair_map_max_pair_grps,
500 max_pair_grps, state->pair_map_max_n_radgrp, max_n_radgrp,
501 state->pair_map_max_n_anggrp, max_n_anggrp);
502 return -1;
503 }
504
505 const size_t n_pairs = (size_t)n_ele * (size_t)n_ele;
506 const size_t bytes_counts = sizeof(int) * n_pairs;
507 const size_t bytes_max_cut = sizeof(double) * n_pairs;
508 const size_t bytes_groups = sizeof(int) * (size_t)max_pair_grps * n_pairs;
509 const size_t bytes_rcut =
510 sizeof(double) * (size_t)max_n_radgrp * (size_t)n_ele;
511 const size_t bytes_acut =
512 sizeof(double) * (size_t)max_n_anggrp * (size_t)n_ele;
513
514 if (nnp_gpu_pbuf_ensure((void **)&state->pair_n_rad_dev,
515 &state->pair_n_rad_dev_cap, bytes_counts))
516 return -1;
517 if (nnp_gpu_pbuf_ensure((void **)&state->pair_n_ang1_dev,
518 &state->pair_n_ang1_dev_cap, bytes_counts))
519 return -1;
520 if (nnp_gpu_pbuf_ensure((void **)&state->pair_n_ang2_dev,
521 &state->pair_n_ang2_dev_cap, bytes_counts))
522 return -1;
523 if (nnp_gpu_pbuf_ensure((void **)&state->pair_max_cutoff_dev,
524 &state->pair_max_cutoff_dev_cap, bytes_max_cut))
525 return -1;
526 if (nnp_gpu_pbuf_ensure((void **)&state->pair_rad_groups_dev,
527 &state->pair_rad_groups_dev_cap, bytes_groups))
528 return -1;
529 if (nnp_gpu_pbuf_ensure((void **)&state->pair_ang1_groups_dev,
530 &state->pair_ang1_groups_dev_cap, bytes_groups))
531 return -1;
532 if (nnp_gpu_pbuf_ensure((void **)&state->pair_ang2_groups_dev,
533 &state->pair_ang2_groups_dev_cap, bytes_groups))
534 return -1;
535 if (nnp_gpu_pbuf_ensure((void **)&state->rad_cutoff_dev,
536 &state->rad_cutoff_dev_cap, bytes_rcut))
537 return -1;
538 if (nnp_gpu_pbuf_ensure((void **)&state->ang_cutoff_dev,
539 &state->ang_cutoff_dev_cap, bytes_acut))
540 return -1;
541
542 offloadMemcpyAsyncHtoD(state->pair_n_rad_dev, h_n_rad, bytes_counts,
543 state->stream);
544 offloadMemcpyAsyncHtoD(state->pair_n_ang1_dev, h_n_ang1, bytes_counts,
545 state->stream);
546 offloadMemcpyAsyncHtoD(state->pair_n_ang2_dev, h_n_ang2, bytes_counts,
547 state->stream);
548 offloadMemcpyAsyncHtoD(state->pair_max_cutoff_dev, h_pair_max_cutoff,
549 bytes_max_cut, state->stream);
550 offloadMemcpyAsyncHtoD(state->pair_rad_groups_dev, h_rad_groups, bytes_groups,
551 state->stream);
552 offloadMemcpyAsyncHtoD(state->pair_ang1_groups_dev, h_ang1_groups,
553 bytes_groups, state->stream);
554 offloadMemcpyAsyncHtoD(state->pair_ang2_groups_dev, h_ang2_groups,
555 bytes_groups, state->stream);
556 offloadMemcpyAsyncHtoD(state->rad_cutoff_dev, h_rad_cutoff, bytes_rcut,
557 state->stream);
558 offloadMemcpyAsyncHtoD(state->ang_cutoff_dev, h_ang_cutoff, bytes_acut,
559 state->stream);
560
561 /* Sync so the host buffers can go out of scope safely after return. */
562 offloadStreamSynchronize(state->stream);
563
564 state->pair_map_uploaded = 1;
565 state->pair_map_n_ele = n_ele;
566 state->pair_map_max_pair_grps = max_pair_grps;
567 state->pair_map_max_n_radgrp = max_n_radgrp;
568 state->pair_map_max_n_anggrp = max_n_anggrp;
569 return 0;
570}
571
572/*******************************************************************************
573 * \brief Launcher for nnp_cell_list_walk_kernel. Exported with C linkage so
574 * Fortran calls it via ISO_C_BINDING. Uploads the per-step support
575 * buffers (scaled coordinates, global element indices, the per-element
576 * atom lists and counts), grows and zeroes the per-(slot, s, e) walk
577 * output and count buffers, clears the overflow flag, and launches the
578 * walk with grid (n_atoms_in_e_max, n_ele) and one warp per block. It
579 * then synchronizes the stream and reads the overflow flag back: on
580 * overflow it returns -2 so the caller can grow the per-atom neighbour
581 * capacities and retry, otherwise it returns 0. The overflow flag lives
582 * in state->walk_overflow_dev, a small grow-only device buffer.
583 * \author Dhruv Sharma (ds2173@cam.ac.uk)
584 ******************************************************************************/
585extern "C" int nnp_gpu_cell_list_walk_c(
586 int num_atoms_global, int n_ele, int n_images, int n_cells,
587 int n_atoms_in_e_max, int max_n_neigh_rad, int max_n_neigh_ang1,
588 int max_n_neigh_ang2, int max_n_radgrp, int max_n_anggrp, const int *h_nbin,
589 const double *h_bin_lower, const double *h_bin_width, const int *h_bin_span,
590 const int *h_perd, const int *h_exact_pbc_copies,
591 const double *h_coord_scaled, const int *h_ele_ind_global,
592 const int *h_atoms_global_per_e, const int *h_n_atoms_in_e) {
593 if (num_atoms_global <= 0 || n_ele <= 0 || n_images <= 0 || n_cells <= 0 ||
594 n_atoms_in_e_max <= 0 || max_n_neigh_rad <= 0 || max_n_neigh_ang1 <= 0 ||
595 max_n_neigh_ang2 <= 0 || max_n_radgrp <= 0 || max_n_anggrp <= 0) {
596 fprintf(stderr,
597 "ERROR: nnp_gpu_cell_list_walk: bad sizes "
598 "num_atoms_global=%d n_ele=%d n_images=%d n_cells=%d\n",
599 num_atoms_global, n_ele, n_images, n_cells);
600 return -1;
601 }
602 nnp_gpu_state_t *state = nnp_gpu_state_get();
603 if (state == NULL)
604 return -1;
605 if (!state->pair_map_uploaded) {
606 fprintf(stderr, "ERROR: nnp_gpu_cell_list_walk: pair_map not uploaded "
607 "(call nnp_gpu_upload_pair_map_c first)\n");
608 return -1;
609 }
610 if (state->pair_map_max_pair_grps <= 0) {
611 fprintf(stderr,
612 "ERROR: nnp_gpu_cell_list_walk: pair_map_max_pair_grps=%d\n",
613 state->pair_map_max_pair_grps);
614 return -1;
615 }
616 /* The walk chases the cell-list chain and the image table, both of which are
617 * uploaded by earlier launchers. Check them here rather than let a NULL or a
618 * short buffer surface as an illegal access at the next stream sync, which
619 * would blame the sync instead of the missing upload. */
620 if (state->coord_dev == NULL || state->image_table_dev == NULL ||
621 state->cell_head_dev == NULL || state->cell_next_dev == NULL ||
622 state->cell_image_atom_dev == NULL ||
623 state->cell_image_shift_dev == NULL) {
624 fprintf(stderr, "ERROR: nnp_gpu_cell_list_walk: positions, image table or "
625 "cell-list chain not uploaded for this step\n");
626 return -1;
627 }
628 if (state->cell_head_dev_cap < sizeof(int) * (size_t)n_cells ||
629 state->image_table_dev_cap < sizeof(double) * 3 * (size_t)n_images ||
630 state->cell_image_atom_dev_cap < sizeof(int) * (size_t)n_images ||
631 state->cell_image_shift_dev_cap < sizeof(int) * 3 * (size_t)n_images) {
632 fprintf(stderr,
633 "ERROR: nnp_gpu_cell_list_walk: device cell list is sized for a "
634 "smaller geometry than this call's n_cells=%d n_images=%d\n",
635 n_cells, n_images);
636 return -1;
637 }
638 const int max_pair_grps = state->pair_map_max_pair_grps;
639
640 const size_t bytes_coord_scaled =
641 sizeof(double) * 3 * (size_t)num_atoms_global;
642 const size_t bytes_ele_ind = sizeof(int) * (size_t)num_atoms_global;
643 const size_t bytes_atoms_pe =
644 sizeof(int) * (size_t)n_atoms_in_e_max * (size_t)n_ele;
645 const size_t bytes_n_in_e = sizeof(int) * (size_t)n_ele;
646 const size_t pa_count_rad =
647 (size_t)n_atoms_in_e_max * (size_t)max_n_radgrp * (size_t)n_ele;
648 const size_t pa_count_ang =
649 (size_t)n_atoms_in_e_max * (size_t)max_n_anggrp * (size_t)n_ele;
650 const size_t bytes_rad_ind =
651 sizeof(int) * (size_t)max_n_neigh_rad * pa_count_rad;
652 const size_t bytes_ang1_ind =
653 sizeof(int) * (size_t)max_n_neigh_ang1 * pa_count_ang;
654 const size_t bytes_ang2_ind =
655 sizeof(int) * (size_t)max_n_neigh_ang2 * pa_count_ang;
656 const size_t bytes_n_rad_pa = sizeof(int) * pa_count_rad;
657 const size_t bytes_n_ang1_pa = sizeof(int) * pa_count_ang;
658 const size_t bytes_n_ang2_pa = sizeof(int) * pa_count_ang;
659 int h_overflow = 0;
660
661 /* Per-step support buffer uploads. */
662 if (nnp_gpu_pbuf_ensure((void **)&state->coord_scaled_dev,
663 &state->coord_scaled_dev_cap, bytes_coord_scaled))
664 return -1;
665 if (nnp_gpu_pbuf_ensure((void **)&state->ele_ind_global_dev,
666 &state->ele_ind_global_dev_cap, bytes_ele_ind))
667 return -1;
668 if (nnp_gpu_pbuf_ensure((void **)&state->walk_atoms_global_dev,
669 &state->walk_atoms_global_dev_cap, bytes_atoms_pe))
670 return -1;
671 if (nnp_gpu_pbuf_ensure((void **)&state->walk_n_atoms_in_e_dev,
672 &state->walk_n_atoms_in_e_dev_cap, bytes_n_in_e))
673 return -1;
674
675 offloadMemcpyAsyncHtoD(state->coord_scaled_dev, h_coord_scaled,
676 bytes_coord_scaled, state->stream);
677 offloadMemcpyAsyncHtoD(state->ele_ind_global_dev, h_ele_ind_global,
678 bytes_ele_ind, state->stream);
679 offloadMemcpyAsyncHtoD(state->walk_atoms_global_dev, h_atoms_global_per_e,
680 bytes_atoms_pe, state->stream);
681 offloadMemcpyAsyncHtoD(state->walk_n_atoms_in_e_dev, h_n_atoms_in_e,
682 bytes_n_in_e, state->stream);
683
684 /* Output buffer ensure + zero. */
685 if (nnp_gpu_pbuf_ensure((void **)&state->walk_rad_ind_dev,
686 &state->walk_rad_ind_dev_cap, bytes_rad_ind))
687 return -1;
688 if (nnp_gpu_pbuf_ensure((void **)&state->walk_rad_image_idx_dev,
689 &state->walk_rad_image_idx_dev_cap, bytes_rad_ind))
690 return -1;
691 if (nnp_gpu_pbuf_ensure((void **)&state->walk_ang1_ind_dev,
692 &state->walk_ang1_ind_dev_cap, bytes_ang1_ind))
693 return -1;
694 if (nnp_gpu_pbuf_ensure((void **)&state->walk_ang1_image_idx_dev,
695 &state->walk_ang1_image_idx_dev_cap, bytes_ang1_ind))
696 return -1;
697 if (nnp_gpu_pbuf_ensure((void **)&state->walk_ang2_ind_dev,
698 &state->walk_ang2_ind_dev_cap, bytes_ang2_ind))
699 return -1;
700 if (nnp_gpu_pbuf_ensure((void **)&state->walk_ang2_image_idx_dev,
701 &state->walk_ang2_image_idx_dev_cap, bytes_ang2_ind))
702 return -1;
703 if (nnp_gpu_pbuf_ensure((void **)&state->walk_n_rad_pa_dev,
704 &state->walk_n_rad_pa_dev_cap, bytes_n_rad_pa))
705 return -1;
706 if (nnp_gpu_pbuf_ensure((void **)&state->walk_n_ang1_pa_dev,
707 &state->walk_n_ang1_pa_dev_cap, bytes_n_ang1_pa))
708 return -1;
709 if (nnp_gpu_pbuf_ensure((void **)&state->walk_n_ang2_pa_dev,
710 &state->walk_n_ang2_pa_dev_cap, bytes_n_ang2_pa))
711 return -1;
712
713 offloadMemsetAsync(state->walk_n_rad_pa_dev, 0, bytes_n_rad_pa,
714 state->stream);
715 offloadMemsetAsync(state->walk_n_ang1_pa_dev, 0, bytes_n_ang1_pa,
716 state->stream);
717 offloadMemsetAsync(state->walk_n_ang2_pa_dev, 0, bytes_n_ang2_pa,
718 state->stream);
719
720 /* Overflow flag: small persistent buffer, grown once and reused. */
721 if (nnp_gpu_pbuf_ensure((void **)&state->walk_overflow_dev,
722 &state->walk_overflow_cap, sizeof(int)))
723 return -1;
724 offloadMemsetAsync(state->walk_overflow_dev, 0, sizeof(int), state->stream);
725
726 /* Launch */
727 {
728 dim3 grid(n_atoms_in_e_max, n_ele, 1);
729 dim3 block(32, 1, 1);
730 nnp_cell_list_walk_kernel<<<grid, block, 0, state->stream>>>(
731 num_atoms_global, n_ele, n_atoms_in_e_max, max_n_neigh_rad,
732 max_n_neigh_ang1, max_n_neigh_ang2, max_n_radgrp, max_n_anggrp,
733 max_pair_grps, h_nbin[0], h_nbin[1], h_nbin[2], h_bin_lower[0],
734 h_bin_lower[1], h_bin_lower[2], h_bin_width[0], h_bin_width[1],
735 h_bin_width[2], h_bin_span[0], h_bin_span[1], h_bin_span[2], h_perd[0],
736 h_perd[1], h_perd[2], h_exact_pbc_copies[0], h_exact_pbc_copies[1],
737 h_exact_pbc_copies[2], state->coord_dev, state->image_table_dev,
738 state->coord_scaled_dev, state->ele_ind_global_dev,
739 state->cell_image_atom_dev, state->cell_image_shift_dev,
740 state->cell_head_dev, state->cell_next_dev,
741 state->walk_atoms_global_dev, state->walk_n_atoms_in_e_dev,
742 state->pair_n_rad_dev, state->pair_n_ang1_dev, state->pair_n_ang2_dev,
743 state->pair_max_cutoff_dev, state->pair_rad_groups_dev,
744 state->pair_ang1_groups_dev, state->pair_ang2_groups_dev,
745 state->rad_cutoff_dev, state->ang_cutoff_dev, state->walk_rad_ind_dev,
746 state->walk_rad_image_idx_dev, state->walk_ang1_ind_dev,
747 state->walk_ang1_image_idx_dev, state->walk_ang2_ind_dev,
748 state->walk_ang2_image_idx_dev, state->walk_n_rad_pa_dev,
749 state->walk_n_ang1_pa_dev, state->walk_n_ang2_pa_dev,
750 state->walk_overflow_dev);
751 OFFLOAD_CHECK(offloadPeekAtLastError());
752 }
753
754 /* Sync + read overflow flag back. */
755 offloadStreamSynchronize(state->stream);
756 offloadMemcpyDtoH(&h_overflow, state->walk_overflow_dev, sizeof(int));
757 if (h_overflow) {
758 fprintf(stderr, "ERROR: nnp_gpu_cell_list_walk: per-(slot, s, e) overflow "
759 "(grow max_n_neigh_rad/ang1/ang2 and retry)\n");
760 return -2;
761 }
762 return 0;
763}
764
765/*******************************************************************************
766 * \brief Device-to-host copy of the per-(slot, s, e) walk count buffers into
767 * the caller's host arrays, then synchronizes the stream so the host
768 * buffers are valid on return. Used by the Fortran host-versus-device
769 * walk comparison. Cheap (well under a megabyte for typical systems).
770 * \author Dhruv Sharma (ds2173@cam.ac.uk)
771 ******************************************************************************/
772extern "C" int nnp_gpu_walk_counts_d2h_c(int n_atoms_in_e_max, int max_n_radgrp,
773 int max_n_anggrp, int n_ele,
774 int *h_n_rad_pa, int *h_n_ang1_pa,
775 int *h_n_ang2_pa) {
776 if (n_atoms_in_e_max <= 0 || max_n_radgrp <= 0 || max_n_anggrp <= 0 ||
777 n_ele <= 0) {
778 fprintf(stderr, "ERROR: nnp_gpu_walk_counts_d2h: bad sizes\n");
779 return -1;
780 }
781 nnp_gpu_state_t *state = nnp_gpu_state_get();
782 if (state == NULL)
783 return -1;
784 if (state->walk_n_rad_pa_dev == NULL || state->walk_n_ang1_pa_dev == NULL ||
785 state->walk_n_ang2_pa_dev == NULL) {
786 fprintf(stderr, "ERROR: nnp_gpu_walk_counts_d2h: walk count buffers not "
787 "allocated (call nnp_gpu_cell_list_walk_c first)\n");
788 return -1;
789 }
790 const size_t pa_count_rad =
791 (size_t)n_atoms_in_e_max * (size_t)max_n_radgrp * (size_t)n_ele;
792 const size_t pa_count_ang =
793 (size_t)n_atoms_in_e_max * (size_t)max_n_anggrp * (size_t)n_ele;
794
795 offloadMemcpyAsyncDtoH(h_n_rad_pa, state->walk_n_rad_pa_dev,
796 sizeof(int) * pa_count_rad, state->stream);
797 offloadMemcpyAsyncDtoH(h_n_ang1_pa, state->walk_n_ang1_pa_dev,
798 sizeof(int) * pa_count_ang, state->stream);
799 offloadMemcpyAsyncDtoH(h_n_ang2_pa, state->walk_n_ang2_pa_dev,
800 sizeof(int) * pa_count_ang, state->stream);
801 offloadStreamSynchronize(state->stream);
802 return 0;
803}
804
805#endif /* defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP) */
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int j1
static void const int const int const double GRID_CONST_WHEN_COLLOCATE double * cx
real(dp), dimension(3) d
real(dp), dimension(3) g