9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
17#error "OpenMP should not be used in .cu files to accommodate HIP."
40__device__ __forceinline__
int nnp_walk_lin_idx_1based(
int bx,
int by,
int bz,
42 return bx + nbx * ((by - 1) + nby * (bz - 1));
62__device__ __forceinline__
int
63nnp_walk_image_match(
const double *coord_scaled,
int i_global,
int j_global,
64 int sx,
int sy,
int sz,
const int *perd,
65 const int *exact_pbc_copies) {
66 int shifts[3] = {sx, sy, sz};
67 for (
int d = 0;
d < 3;
d++) {
70 coord_scaled[3 * i_global +
d] - coord_scaled[3 * j_global +
d];
71 double rounded = (diff >= 0.0) ? floor(diff + 0.5) : -floor(-diff + 0.5);
72 int minimal_shift = (int)rounded;
73 int delta = minimal_shift - shifts[
d];
76 if (delta > exact_pbc_copies[d])
78 }
else if (shifts[d] != 0) {
121__global__
void nnp_cell_list_walk_kernel(
122 int num_atoms_global,
int n_ele,
int n_atoms_in_e_max,
int max_n_neigh_rad,
123 int max_n_neigh_ang1,
int max_n_neigh_ang2,
int max_n_radgrp,
124 int max_n_anggrp,
int max_pair_grps,
int nbx,
int nby,
int nbz,
125 double bx_lo,
double by_lo,
double bz_lo,
double bw_x,
double bw_y,
126 double bw_z,
int span_x,
int span_y,
int span_z,
int perd_x,
int perd_y,
127 int perd_z,
int epbc_x,
int epbc_y,
int epbc_z,
128 const double *__restrict__ coord_dev,
129 const double *__restrict__ image_table_dev,
130 const double *__restrict__ coord_scaled_dev,
131 const int *__restrict__ ele_ind_global_dev,
132 const int *__restrict__ image_atom_dev,
133 const int *__restrict__ image_shift_dev,
const int *__restrict__ head_dev,
134 const int *__restrict__ next_dev,
135 const int *__restrict__ atoms_global_per_e_dev,
136 const int *__restrict__ n_atoms_in_e_dev,
137 const int *__restrict__ pair_n_rad_dev,
138 const int *__restrict__ pair_n_ang1_dev,
139 const int *__restrict__ pair_n_ang2_dev,
140 const double *__restrict__ pair_max_cutoff_dev,
141 const int *__restrict__ pair_rad_groups_dev,
142 const int *__restrict__ pair_ang1_groups_dev,
143 const int *__restrict__ pair_ang2_groups_dev,
144 const double *__restrict__ rad_cutoff_dev,
145 const double *__restrict__ ang_cutoff_dev,
146 int *__restrict__ walk_rad_ind_dev,
147 int *__restrict__ walk_rad_image_idx_dev,
148 int *__restrict__ walk_ang1_ind_dev,
149 int *__restrict__ walk_ang1_image_idx_dev,
150 int *__restrict__ walk_ang2_ind_dev,
151 int *__restrict__ walk_ang2_image_idx_dev,
152 int *__restrict__ walk_n_rad_pa_dev,
int *__restrict__ walk_n_ang1_pa_dev,
153 int *__restrict__ walk_n_ang2_pa_dev,
int *__restrict__ overflow_flag_dev) {
160 const int slot = blockIdx.x;
161 const int e0 = blockIdx.y;
164 if (slot >= n_atoms_in_e_dev[e0])
168 const int i1 = atoms_global_per_e_dev[slot + n_atoms_in_e_max * e0];
171 const int i0 = i1 - 1;
172 const int i_ele1 = ele_ind_global_dev[i0];
173 const int i_ele0 = i_ele1 - 1;
175 const double cx = coord_dev[3 * i0 + 0];
176 const double cy = coord_dev[3 * i0 + 1];
177 const double cz = coord_dev[3 * i0 + 2];
180 const double rebuild_eps = 1.0e-12;
181 const double bw_xs = (bw_x > rebuild_eps) ? bw_x : rebuild_eps;
182 const double bw_ys = (bw_y > rebuild_eps) ? bw_y : rebuild_eps;
183 const double bw_zs = (bw_z > rebuild_eps) ? bw_z : rebuild_eps;
184 int bin_ix = (int)((
cx - bx_lo) / bw_xs) + 1;
185 int bin_iy = (int)((cy - by_lo) / bw_ys) + 1;
186 int bin_iz = (int)((cz - bz_lo) / bw_zs) + 1;
189 else if (bin_ix > nbx)
193 else if (bin_iy > nby)
197 else if (bin_iz > nbz)
200 const int bx_lo_i = max(1, bin_ix - span_x);
201 const int bx_hi_i = min(nbx, bin_ix + span_x);
202 const int by_lo_i = max(1, bin_iy - span_y);
203 const int by_hi_i = min(nby, bin_iy + span_y);
204 const int bz_lo_i = max(1, bin_iz - span_z);
205 const int bz_hi_i = min(nbz, bin_iz + span_z);
211 const int nbx_w = bx_hi_i - bx_lo_i + 1;
212 const int nby_w = by_hi_i - by_lo_i + 1;
213 const int nbz_w = bz_hi_i - bz_lo_i + 1;
214 const int n_bins_w = nbx_w * nby_w * nbz_w;
215 for (
int bsel = threadIdx.x; bsel < n_bins_w; bsel += blockDim.x) {
216 const int bx = bx_lo_i + (bsel % nbx_w);
217 const int by = by_lo_i + ((bsel / nbx_w) % nby_w);
218 const int bz = bz_lo_i + (bsel / (nbx_w * nby_w));
219 const int lin = nnp_walk_lin_idx_1based(bx, by, bz, nbx, nby);
220 int img1 = head_dev[lin - 1];
222 const int j1 = image_atom_dev[img1 - 1];
223 const int j0 =
j1 - 1;
224 const int sx = image_shift_dev[3 * (img1 - 1) + 0];
225 const int sy_ = image_shift_dev[3 * (img1 - 1) + 1];
226 const int sz_ = image_shift_dev[3 * (img1 - 1) + 2];
229 if (
j1 == i1 && sx == 0 && sy_ == 0 && sz_ == 0) {
230 img1 = next_dev[img1 - 1];
234 const int perd_arr[3] = {perd_x, perd_y, perd_z};
235 const int epbc_arr[3] = {epbc_x, epbc_y, epbc_z};
236 if (!nnp_walk_image_match(coord_scaled_dev, i0, j0, sx, sy_, sz_,
237 perd_arr, epbc_arr)) {
238 img1 = next_dev[img1 - 1];
242 const double image_x =
243 coord_dev[3 * j0 + 0] + image_table_dev[3 * (img1 - 1) + 0];
244 const double image_y =
245 coord_dev[3 * j0 + 1] + image_table_dev[3 * (img1 - 1) + 1];
246 const double image_z =
247 coord_dev[3 * j0 + 2] + image_table_dev[3 * (img1 - 1) + 2];
248 const double drx =
cx - image_x;
249 const double dry = cy - image_y;
250 const double drz = cz - image_z;
251 const double norm = sqrt(drx * drx + dry * dry + drz * drz);
253 const int j_ele1 = ele_ind_global_dev[j0];
254 const int j_ele0 = j_ele1 - 1;
255 const int pair_idx = i_ele0 + n_ele * j_ele0;
256 const double pair_max = pair_max_cutoff_dev[pair_idx];
258 if (norm < pair_max) {
260 const int n_rad_pair = pair_n_rad_dev[pair_idx];
261 for (
int g = 0;
g < n_rad_pair;
g++) {
262 const int s1 = pair_rad_groups_dev[
g + max_pair_grps * pair_idx];
263 const int s0 = s1 - 1;
264 const double cutoff = rad_cutoff_dev[s0 + max_n_radgrp * i_ele0];
268 const long long pa_off =
269 (
long long)slot + (
long long)n_atoms_in_e_max *
270 ((
long long)s0 + (
long long)max_n_radgrp *
272 const int k = atomicAdd(&walk_n_rad_pa_dev[pa_off], 1);
273 if (k >= max_n_neigh_rad) {
274 atomicExch(overflow_flag_dev, 1);
276 const long long out_off =
277 (
long long)k + (
long long)max_n_neigh_rad * pa_off;
278 walk_rad_ind_dev[out_off] = j0;
279 walk_rad_image_idx_dev[out_off] = img1 - 1;
284 const int n_ang1_pair = pair_n_ang1_dev[pair_idx];
285 for (
int g = 0;
g < n_ang1_pair;
g++) {
286 const int s1 = pair_ang1_groups_dev[
g + max_pair_grps * pair_idx];
287 const int s0 = s1 - 1;
288 const double cutoff = ang_cutoff_dev[s0 + max_n_anggrp * i_ele0];
290 const long long pa_off =
291 (
long long)slot + (
long long)n_atoms_in_e_max *
292 ((
long long)s0 + (
long long)max_n_anggrp *
294 const int k = atomicAdd(&walk_n_ang1_pa_dev[pa_off], 1);
295 if (k >= max_n_neigh_ang1) {
296 atomicExch(overflow_flag_dev, 1);
298 const long long out_off =
299 (
long long)k + (
long long)max_n_neigh_ang1 * pa_off;
300 walk_ang1_ind_dev[out_off] = j0;
301 walk_ang1_image_idx_dev[out_off] = img1 - 1;
306 const int n_ang2_pair = pair_n_ang2_dev[pair_idx];
307 for (
int g = 0;
g < n_ang2_pair;
g++) {
308 const int s1 = pair_ang2_groups_dev[
g + max_pair_grps * pair_idx];
309 const int s0 = s1 - 1;
310 const double cutoff = ang_cutoff_dev[s0 + max_n_anggrp * i_ele0];
312 const long long pa_off =
313 (
long long)slot + (
long long)n_atoms_in_e_max *
314 ((
long long)s0 + (
long long)max_n_anggrp *
316 const int k = atomicAdd(&walk_n_ang2_pa_dev[pa_off], 1);
317 if (k >= max_n_neigh_ang2) {
318 atomicExch(overflow_flag_dev, 1);
320 const long long out_off =
321 (
long long)k + (
long long)max_n_neigh_ang2 * pa_off;
322 walk_ang2_ind_dev[out_off] = j0;
323 walk_ang2_image_idx_dev[out_off] = img1 - 1;
329 img1 = next_dev[img1 - 1];
346extern "C" int nnp_gpu_upload_positions_step_c(
int num_atoms,
347 const double *h_coord) {
348 if (num_atoms <= 0) {
349 fprintf(stderr,
"ERROR: nnp_gpu_upload_positions_step: bad num_atoms=%d\n",
353 nnp_gpu_state_t *state = nnp_gpu_state_get();
361 const size_t bytes =
sizeof(double) * 3 * (
size_t)num_atoms;
362 if (nnp_gpu_pbuf_ensure((
void **)&state->coord_dev, &state->coord_dev_cap,
365 offloadMemcpyAsyncHtoD(state->coord_dev, h_coord, bytes, state->stream);
379extern "C" int nnp_gpu_upload_image_table_c(
int n_images,
380 const double *h_image_table) {
382 fprintf(stderr,
"ERROR: nnp_gpu_upload_image_table: bad n_images=%d\n",
386 nnp_gpu_state_t *state = nnp_gpu_state_get();
390 const size_t bytes =
sizeof(double) * 3 * (
size_t)n_images;
391 if (nnp_gpu_pbuf_ensure((
void **)&state->image_table_dev,
392 &state->image_table_dev_cap, bytes))
394 offloadMemcpyAsyncHtoD(state->image_table_dev, h_image_table, bytes,
413extern "C" int nnp_gpu_upload_cell_list_c(
int n_images,
int n_cells,
414 const int *h_image_atom,
415 const int *h_image_shift,
418 if (n_images <= 0 || n_cells <= 0) {
420 "ERROR: nnp_gpu_upload_cell_list: bad sizes "
421 "n_images=%d n_cells=%d\n",
425 nnp_gpu_state_t *state = nnp_gpu_state_get();
429 const size_t bytes_imgs =
sizeof(int) * (
size_t)n_images;
430 const size_t bytes_shifts =
sizeof(int) * 3 * (
size_t)n_images;
431 const size_t bytes_head =
sizeof(int) * (
size_t)n_cells;
433 if (nnp_gpu_pbuf_ensure((
void **)&state->cell_image_atom_dev,
434 &state->cell_image_atom_dev_cap, bytes_imgs))
436 if (nnp_gpu_pbuf_ensure((
void **)&state->cell_image_shift_dev,
437 &state->cell_image_shift_dev_cap, bytes_shifts))
439 if (nnp_gpu_pbuf_ensure((
void **)&state->cell_head_dev,
440 &state->cell_head_dev_cap, bytes_head))
442 if (nnp_gpu_pbuf_ensure((
void **)&state->cell_next_dev,
443 &state->cell_next_dev_cap, bytes_imgs))
446 offloadMemcpyAsyncHtoD(state->cell_image_atom_dev, h_image_atom, bytes_imgs,
448 offloadMemcpyAsyncHtoD(state->cell_image_shift_dev, h_image_shift,
449 bytes_shifts, state->stream);
450 offloadMemcpyAsyncHtoD(state->cell_head_dev, h_head, bytes_head,
452 offloadMemcpyAsyncHtoD(state->cell_next_dev, h_next, bytes_imgs,
455 offloadStreamSynchronize(state->stream);
470extern "C" int nnp_gpu_upload_pair_map_c(
471 int n_ele,
int max_pair_grps,
int max_n_radgrp,
int max_n_anggrp,
472 const int *h_n_rad,
const int *h_n_ang1,
const int *h_n_ang2,
473 const double *h_pair_max_cutoff,
const int *h_rad_groups,
474 const int *h_ang1_groups,
const int *h_ang2_groups,
475 const double *h_rad_cutoff,
const double *h_ang_cutoff) {
476 if (n_ele <= 0 || max_pair_grps <= 0 || max_n_radgrp <= 0 ||
479 "ERROR: nnp_gpu_upload_pair_map: bad sizes "
480 "n_ele=%d max_pair_grps=%d max_n_radgrp=%d max_n_anggrp=%d\n",
481 n_ele, max_pair_grps, max_n_radgrp, max_n_anggrp);
484 nnp_gpu_state_t *state = nnp_gpu_state_get();
488 if (state->pair_map_uploaded) {
489 if (state->pair_map_n_ele == n_ele &&
490 state->pair_map_max_pair_grps == max_pair_grps &&
491 state->pair_map_max_n_radgrp == max_n_radgrp &&
492 state->pair_map_max_n_anggrp == max_n_anggrp) {
496 "ERROR: nnp_gpu_upload_pair_map: shape mismatch on second "
497 "upload (n_ele %d->%d, max_pair_grps %d->%d, "
498 "max_n_radgrp %d->%d, max_n_anggrp %d->%d)\n",
499 state->pair_map_n_ele, n_ele, state->pair_map_max_pair_grps,
500 max_pair_grps, state->pair_map_max_n_radgrp, max_n_radgrp,
501 state->pair_map_max_n_anggrp, max_n_anggrp);
505 const size_t n_pairs = (size_t)n_ele * (
size_t)n_ele;
506 const size_t bytes_counts =
sizeof(int) * n_pairs;
507 const size_t bytes_max_cut =
sizeof(double) * n_pairs;
508 const size_t bytes_groups =
sizeof(int) * (
size_t)max_pair_grps * n_pairs;
509 const size_t bytes_rcut =
510 sizeof(double) * (
size_t)max_n_radgrp * (size_t)n_ele;
511 const size_t bytes_acut =
512 sizeof(double) * (
size_t)max_n_anggrp * (size_t)n_ele;
514 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_n_rad_dev,
515 &state->pair_n_rad_dev_cap, bytes_counts))
517 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_n_ang1_dev,
518 &state->pair_n_ang1_dev_cap, bytes_counts))
520 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_n_ang2_dev,
521 &state->pair_n_ang2_dev_cap, bytes_counts))
523 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_max_cutoff_dev,
524 &state->pair_max_cutoff_dev_cap, bytes_max_cut))
526 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_rad_groups_dev,
527 &state->pair_rad_groups_dev_cap, bytes_groups))
529 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_ang1_groups_dev,
530 &state->pair_ang1_groups_dev_cap, bytes_groups))
532 if (nnp_gpu_pbuf_ensure((
void **)&state->pair_ang2_groups_dev,
533 &state->pair_ang2_groups_dev_cap, bytes_groups))
535 if (nnp_gpu_pbuf_ensure((
void **)&state->rad_cutoff_dev,
536 &state->rad_cutoff_dev_cap, bytes_rcut))
538 if (nnp_gpu_pbuf_ensure((
void **)&state->ang_cutoff_dev,
539 &state->ang_cutoff_dev_cap, bytes_acut))
542 offloadMemcpyAsyncHtoD(state->pair_n_rad_dev, h_n_rad, bytes_counts,
544 offloadMemcpyAsyncHtoD(state->pair_n_ang1_dev, h_n_ang1, bytes_counts,
546 offloadMemcpyAsyncHtoD(state->pair_n_ang2_dev, h_n_ang2, bytes_counts,
548 offloadMemcpyAsyncHtoD(state->pair_max_cutoff_dev, h_pair_max_cutoff,
549 bytes_max_cut, state->stream);
550 offloadMemcpyAsyncHtoD(state->pair_rad_groups_dev, h_rad_groups, bytes_groups,
552 offloadMemcpyAsyncHtoD(state->pair_ang1_groups_dev, h_ang1_groups,
553 bytes_groups, state->stream);
554 offloadMemcpyAsyncHtoD(state->pair_ang2_groups_dev, h_ang2_groups,
555 bytes_groups, state->stream);
556 offloadMemcpyAsyncHtoD(state->rad_cutoff_dev, h_rad_cutoff, bytes_rcut,
558 offloadMemcpyAsyncHtoD(state->ang_cutoff_dev, h_ang_cutoff, bytes_acut,
562 offloadStreamSynchronize(state->stream);
564 state->pair_map_uploaded = 1;
565 state->pair_map_n_ele = n_ele;
566 state->pair_map_max_pair_grps = max_pair_grps;
567 state->pair_map_max_n_radgrp = max_n_radgrp;
568 state->pair_map_max_n_anggrp = max_n_anggrp;
585extern "C" int nnp_gpu_cell_list_walk_c(
586 int num_atoms_global,
int n_ele,
int n_images,
int n_cells,
587 int n_atoms_in_e_max,
int max_n_neigh_rad,
int max_n_neigh_ang1,
588 int max_n_neigh_ang2,
int max_n_radgrp,
int max_n_anggrp,
const int *h_nbin,
589 const double *h_bin_lower,
const double *h_bin_width,
const int *h_bin_span,
590 const int *h_perd,
const int *h_exact_pbc_copies,
591 const double *h_coord_scaled,
const int *h_ele_ind_global,
592 const int *h_atoms_global_per_e,
const int *h_n_atoms_in_e) {
593 if (num_atoms_global <= 0 || n_ele <= 0 || n_images <= 0 || n_cells <= 0 ||
594 n_atoms_in_e_max <= 0 || max_n_neigh_rad <= 0 || max_n_neigh_ang1 <= 0 ||
595 max_n_neigh_ang2 <= 0 || max_n_radgrp <= 0 || max_n_anggrp <= 0) {
597 "ERROR: nnp_gpu_cell_list_walk: bad sizes "
598 "num_atoms_global=%d n_ele=%d n_images=%d n_cells=%d\n",
599 num_atoms_global, n_ele, n_images, n_cells);
602 nnp_gpu_state_t *state = nnp_gpu_state_get();
605 if (!state->pair_map_uploaded) {
606 fprintf(stderr,
"ERROR: nnp_gpu_cell_list_walk: pair_map not uploaded "
607 "(call nnp_gpu_upload_pair_map_c first)\n");
610 if (state->pair_map_max_pair_grps <= 0) {
612 "ERROR: nnp_gpu_cell_list_walk: pair_map_max_pair_grps=%d\n",
613 state->pair_map_max_pair_grps);
620 if (state->coord_dev == NULL || state->image_table_dev == NULL ||
621 state->cell_head_dev == NULL || state->cell_next_dev == NULL ||
622 state->cell_image_atom_dev == NULL ||
623 state->cell_image_shift_dev == NULL) {
624 fprintf(stderr,
"ERROR: nnp_gpu_cell_list_walk: positions, image table or "
625 "cell-list chain not uploaded for this step\n");
628 if (state->cell_head_dev_cap <
sizeof(
int) * (
size_t)n_cells ||
629 state->image_table_dev_cap <
sizeof(
double) * 3 * (
size_t)n_images ||
630 state->cell_image_atom_dev_cap <
sizeof(
int) * (
size_t)n_images ||
631 state->cell_image_shift_dev_cap <
sizeof(
int) * 3 * (
size_t)n_images) {
633 "ERROR: nnp_gpu_cell_list_walk: device cell list is sized for a "
634 "smaller geometry than this call's n_cells=%d n_images=%d\n",
638 const int max_pair_grps = state->pair_map_max_pair_grps;
640 const size_t bytes_coord_scaled =
641 sizeof(double) * 3 * (
size_t)num_atoms_global;
642 const size_t bytes_ele_ind =
sizeof(int) * (
size_t)num_atoms_global;
643 const size_t bytes_atoms_pe =
644 sizeof(int) * (
size_t)n_atoms_in_e_max * (size_t)n_ele;
645 const size_t bytes_n_in_e =
sizeof(int) * (
size_t)n_ele;
646 const size_t pa_count_rad =
647 (size_t)n_atoms_in_e_max * (
size_t)max_n_radgrp * (size_t)n_ele;
648 const size_t pa_count_ang =
649 (size_t)n_atoms_in_e_max * (
size_t)max_n_anggrp * (size_t)n_ele;
650 const size_t bytes_rad_ind =
651 sizeof(int) * (
size_t)max_n_neigh_rad * pa_count_rad;
652 const size_t bytes_ang1_ind =
653 sizeof(int) * (
size_t)max_n_neigh_ang1 * pa_count_ang;
654 const size_t bytes_ang2_ind =
655 sizeof(int) * (
size_t)max_n_neigh_ang2 * pa_count_ang;
656 const size_t bytes_n_rad_pa =
sizeof(int) * pa_count_rad;
657 const size_t bytes_n_ang1_pa =
sizeof(int) * pa_count_ang;
658 const size_t bytes_n_ang2_pa =
sizeof(int) * pa_count_ang;
662 if (nnp_gpu_pbuf_ensure((
void **)&state->coord_scaled_dev,
663 &state->coord_scaled_dev_cap, bytes_coord_scaled))
665 if (nnp_gpu_pbuf_ensure((
void **)&state->ele_ind_global_dev,
666 &state->ele_ind_global_dev_cap, bytes_ele_ind))
668 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_atoms_global_dev,
669 &state->walk_atoms_global_dev_cap, bytes_atoms_pe))
671 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_n_atoms_in_e_dev,
672 &state->walk_n_atoms_in_e_dev_cap, bytes_n_in_e))
675 offloadMemcpyAsyncHtoD(state->coord_scaled_dev, h_coord_scaled,
676 bytes_coord_scaled, state->stream);
677 offloadMemcpyAsyncHtoD(state->ele_ind_global_dev, h_ele_ind_global,
678 bytes_ele_ind, state->stream);
679 offloadMemcpyAsyncHtoD(state->walk_atoms_global_dev, h_atoms_global_per_e,
680 bytes_atoms_pe, state->stream);
681 offloadMemcpyAsyncHtoD(state->walk_n_atoms_in_e_dev, h_n_atoms_in_e,
682 bytes_n_in_e, state->stream);
685 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_rad_ind_dev,
686 &state->walk_rad_ind_dev_cap, bytes_rad_ind))
688 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_rad_image_idx_dev,
689 &state->walk_rad_image_idx_dev_cap, bytes_rad_ind))
691 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_ang1_ind_dev,
692 &state->walk_ang1_ind_dev_cap, bytes_ang1_ind))
694 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_ang1_image_idx_dev,
695 &state->walk_ang1_image_idx_dev_cap, bytes_ang1_ind))
697 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_ang2_ind_dev,
698 &state->walk_ang2_ind_dev_cap, bytes_ang2_ind))
700 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_ang2_image_idx_dev,
701 &state->walk_ang2_image_idx_dev_cap, bytes_ang2_ind))
703 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_n_rad_pa_dev,
704 &state->walk_n_rad_pa_dev_cap, bytes_n_rad_pa))
706 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_n_ang1_pa_dev,
707 &state->walk_n_ang1_pa_dev_cap, bytes_n_ang1_pa))
709 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_n_ang2_pa_dev,
710 &state->walk_n_ang2_pa_dev_cap, bytes_n_ang2_pa))
713 offloadMemsetAsync(state->walk_n_rad_pa_dev, 0, bytes_n_rad_pa,
715 offloadMemsetAsync(state->walk_n_ang1_pa_dev, 0, bytes_n_ang1_pa,
717 offloadMemsetAsync(state->walk_n_ang2_pa_dev, 0, bytes_n_ang2_pa,
721 if (nnp_gpu_pbuf_ensure((
void **)&state->walk_overflow_dev,
722 &state->walk_overflow_cap,
sizeof(
int)))
724 offloadMemsetAsync(state->walk_overflow_dev, 0,
sizeof(
int), state->stream);
728 dim3
grid(n_atoms_in_e_max, n_ele, 1);
729 dim3 block(32, 1, 1);
730 nnp_cell_list_walk_kernel<<<
grid, block, 0, state->stream>>>(
731 num_atoms_global, n_ele, n_atoms_in_e_max, max_n_neigh_rad,
732 max_n_neigh_ang1, max_n_neigh_ang2, max_n_radgrp, max_n_anggrp,
733 max_pair_grps, h_nbin[0], h_nbin[1], h_nbin[2], h_bin_lower[0],
734 h_bin_lower[1], h_bin_lower[2], h_bin_width[0], h_bin_width[1],
735 h_bin_width[2], h_bin_span[0], h_bin_span[1], h_bin_span[2], h_perd[0],
736 h_perd[1], h_perd[2], h_exact_pbc_copies[0], h_exact_pbc_copies[1],
737 h_exact_pbc_copies[2], state->coord_dev, state->image_table_dev,
738 state->coord_scaled_dev, state->ele_ind_global_dev,
739 state->cell_image_atom_dev, state->cell_image_shift_dev,
740 state->cell_head_dev, state->cell_next_dev,
741 state->walk_atoms_global_dev, state->walk_n_atoms_in_e_dev,
742 state->pair_n_rad_dev, state->pair_n_ang1_dev, state->pair_n_ang2_dev,
743 state->pair_max_cutoff_dev, state->pair_rad_groups_dev,
744 state->pair_ang1_groups_dev, state->pair_ang2_groups_dev,
745 state->rad_cutoff_dev, state->ang_cutoff_dev, state->walk_rad_ind_dev,
746 state->walk_rad_image_idx_dev, state->walk_ang1_ind_dev,
747 state->walk_ang1_image_idx_dev, state->walk_ang2_ind_dev,
748 state->walk_ang2_image_idx_dev, state->walk_n_rad_pa_dev,
749 state->walk_n_ang1_pa_dev, state->walk_n_ang2_pa_dev,
750 state->walk_overflow_dev);
751 OFFLOAD_CHECK(offloadPeekAtLastError());
755 offloadStreamSynchronize(state->stream);
756 offloadMemcpyDtoH(&h_overflow, state->walk_overflow_dev,
sizeof(
int));
758 fprintf(stderr,
"ERROR: nnp_gpu_cell_list_walk: per-(slot, s, e) overflow "
759 "(grow max_n_neigh_rad/ang1/ang2 and retry)\n");
772extern "C" int nnp_gpu_walk_counts_d2h_c(
int n_atoms_in_e_max,
int max_n_radgrp,
773 int max_n_anggrp,
int n_ele,
774 int *h_n_rad_pa,
int *h_n_ang1_pa,
776 if (n_atoms_in_e_max <= 0 || max_n_radgrp <= 0 || max_n_anggrp <= 0 ||
778 fprintf(stderr,
"ERROR: nnp_gpu_walk_counts_d2h: bad sizes\n");
781 nnp_gpu_state_t *state = nnp_gpu_state_get();
784 if (state->walk_n_rad_pa_dev == NULL || state->walk_n_ang1_pa_dev == NULL ||
785 state->walk_n_ang2_pa_dev == NULL) {
786 fprintf(stderr,
"ERROR: nnp_gpu_walk_counts_d2h: walk count buffers not "
787 "allocated (call nnp_gpu_cell_list_walk_c first)\n");
790 const size_t pa_count_rad =
791 (size_t)n_atoms_in_e_max * (
size_t)max_n_radgrp * (size_t)n_ele;
792 const size_t pa_count_ang =
793 (size_t)n_atoms_in_e_max * (
size_t)max_n_anggrp * (size_t)n_ele;
795 offloadMemcpyAsyncDtoH(h_n_rad_pa, state->walk_n_rad_pa_dev,
796 sizeof(
int) * pa_count_rad, state->stream);
797 offloadMemcpyAsyncDtoH(h_n_ang1_pa, state->walk_n_ang1_pa_dev,
798 sizeof(
int) * pa_count_ang, state->stream);
799 offloadMemcpyAsyncDtoH(h_n_ang2_pa, state->walk_n_ang2_pa_dev,
800 sizeof(
int) * pa_count_ang, state->stream);
801 offloadStreamSynchronize(state->stream);
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int const double GRID_CONST_WHEN_COLLOCATE double * cx