9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
17#error "OpenMP should not be used in .cu files to accommodate HIP."
39static __device__ __forceinline__
double
40nnp_compute_force_scale(
int scaling_mode,
double scmin,
double scmax,
int idx,
41 const double *__restrict__ scaling_loc_max,
42 const double *__restrict__ scaling_loc_min,
43 const double *__restrict__ scaling_sigma) {
44 if (scaling_mode == 1 || scaling_mode == 3) {
45 const double diff = scaling_loc_max[
idx] - scaling_loc_min[
idx];
46 return (scmax - scmin) / diff;
48 if (scaling_mode == 4) {
49 return (scmax - scmin) / scaling_sigma[
idx];
64__device__ __forceinline__
void
65nnp_virial_warp_flush(
const double vir[3][3],
double *__restrict__ virial_dev,
68 for (
int a = 0;
a < 3; ++
a) {
70 for (
int b = 0;
b < 3; ++
b) {
74 for (
int mask = 16; mask > 0; mask >>= 1)
75 v += NNP_SHFL_XOR(v, mask);
77 atomicAdd(&virial_dev[9 * c + 3 * a + b], v);
93__global__
void nnp_force_scatter_radial_kernel(
94 const int n_atoms_in_e,
const int n_symf,
const int max_n_neigh,
95 const int n_committee,
const int n_input_max,
const int num_atoms_global,
96 const int ele_idx,
const int scaling_mode,
const int n_rad_offset,
97 const double scmin,
const double scmax,
98 const int *__restrict__ symf_map_one_based,
99 const int *__restrict__ atom_map,
const int *__restrict__ atoms_global,
100 const int *__restrict__ n_neigh_per_atom,
101 const int *__restrict__ rad_ind_one_based,
102 const double *__restrict__ self_partial,
103 const double *__restrict__ force_rad,
const double *__restrict__ dE_dG,
104 const double *__restrict__ scaling_loc_max,
105 const double *__restrict__ scaling_loc_min,
106 const double *__restrict__ scaling_sigma,
double *__restrict__ f_dev,
110 const int *__restrict__ rad_image_idx,
const double *__restrict__ coord_dev,
111 const double *__restrict__ image_table_dev,
double *__restrict__ virial_dev,
112 const int accumulate_virial) {
113 const int atom_in_e = blockIdx.x;
114 const int sf = blockIdx.y;
115 const int c = blockIdx.z;
116 if (atom_in_e >= n_atoms_in_e || sf >= n_symf || c >= n_committee)
122 const int i_local = atom_map[atom_in_e];
123 const int i_global = atoms_global[atom_in_e];
126 const int m = symf_map_one_based[sf] - 1;
130 const double dEdG = dE_dG[m + n_input_max *
c +
131 (
long long)n_input_max * n_committee * i_local];
132 const int scale_idx = ele_idx * n_input_max + m;
134 nnp_compute_force_scale(scaling_mode, scmin, scmax, scale_idx,
135 scaling_loc_max, scaling_loc_min, scaling_sigma);
136 const double de = dEdG * scale;
140 const long long self_off = 3 * (sf + (
long long)n_symf * atom_in_e);
141 const long long f_i_off = 3 * (i_global + (
long long)num_atoms_global *
c);
142 if (threadIdx.x == 0) {
143 atomicAdd(&f_dev[f_i_off + 0], -de * self_partial[self_off + 0]);
144 atomicAdd(&f_dev[f_i_off + 1], -de * self_partial[self_off + 1]);
145 atomicAdd(&f_dev[f_i_off + 2], -de * self_partial[self_off + 2]);
151 double vir[3][3] = {{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}};
152 const double ci_x = accumulate_virial ? coord_dev[3 * i_global + 0] : 0.0;
153 const double ci_y = accumulate_virial ? coord_dev[3 * i_global + 1] : 0.0;
154 const double ci_z = accumulate_virial ? coord_dev[3 * i_global + 2] : 0.0;
158 const int n_neigh = n_neigh_per_atom[atom_in_e];
159 for (
int j = threadIdx.x; j < n_neigh; j += blockDim.x) {
160 const long long nb_off = j + (
long long)max_n_neigh * atom_in_e;
161 const int k_atom = rad_ind_one_based[nb_off] - 1;
162 const long long force_off = 3 * (sf + (
long long)n_symf * nb_off);
163 const long long f_k_off = 3 * (k_atom + (
long long)num_atoms_global *
c);
164 const double fk0 = +de * force_rad[force_off + 0];
165 const double fk1 = +de * force_rad[force_off + 1];
166 const double fk2 = +de * force_rad[force_off + 2];
167 atomicAdd(&f_dev[f_k_off + 0], fk0);
168 atomicAdd(&f_dev[f_k_off + 1], fk1);
169 atomicAdd(&f_dev[f_k_off + 2], fk2);
170 if (accumulate_virial) {
171 const int img = rad_image_idx[nb_off];
173 ci_x - (coord_dev[3 * k_atom + 0] + image_table_dev[3 * img + 0]);
175 ci_y - (coord_dev[3 * k_atom + 1] + image_table_dev[3 * img + 1]);
177 ci_z - (coord_dev[3 * k_atom + 2] + image_table_dev[3 * img + 2]);
178 vir[0][0] += -rb0 * fk0;
179 vir[0][1] += -rb1 * fk0;
180 vir[0][2] += -rb2 * fk0;
181 vir[1][0] += -rb0 * fk1;
182 vir[1][1] += -rb1 * fk1;
183 vir[1][2] += -rb2 * fk1;
184 vir[2][0] += -rb0 * fk2;
185 vir[2][1] += -rb1 * fk2;
186 vir[2][2] += -rb2 * fk2;
190 if (accumulate_virial)
191 nnp_virial_warp_flush(vir, virial_dev, c);
204__global__
void nnp_force_scatter_angular_kernel(
205 const int n_atoms_in_e,
const int n_symf,
const int max_n_ang1,
206 const int max_n_kk,
const int n_committee,
const int n_input_max,
207 const int num_atoms_global,
const int ele_idx,
const int scaling_mode,
208 const int homo,
const int n_rad_offset,
const double scmin,
209 const double scmax,
const int *__restrict__ symf_map_one_based,
210 const int *__restrict__ atom_map,
const int *__restrict__ atoms_global,
211 const int *__restrict__ n_ang1_per_atom,
212 const int *__restrict__ n_ang2_per_atom,
213 const int *__restrict__ ang1_ind_one_based,
214 const int *__restrict__ ang2_ind_one_based,
215 const double *__restrict__ self_partial,
const double *__restrict__ jj,
216 const double *__restrict__ kk,
const double *__restrict__ dE_dG,
217 const double *__restrict__ scaling_loc_max,
218 const double *__restrict__ scaling_loc_min,
219 const double *__restrict__ scaling_sigma,
double *__restrict__ f_dev,
223 const int *__restrict__ ang1_image_idx,
224 const int *__restrict__ ang2_image_idx,
225 const double *__restrict__ coord_dev,
226 const double *__restrict__ image_table_dev,
double *__restrict__ virial_dev,
227 const int accumulate_virial) {
228 const int atom_in_e = blockIdx.x;
229 const int sf = blockIdx.y;
230 const int c = blockIdx.z;
231 if (atom_in_e >= n_atoms_in_e || sf >= n_symf || c >= n_committee)
234 const int i_local = atom_map[atom_in_e];
235 const int i_global = atoms_global[atom_in_e];
237 const int m = n_rad_offset + symf_map_one_based[sf] - 1;
241 const double dEdG = dE_dG[m + n_input_max *
c +
242 (
long long)n_input_max * n_committee * i_local];
243 const int scale_idx = ele_idx * n_input_max + m;
245 nnp_compute_force_scale(scaling_mode, scmin, scmax, scale_idx,
246 scaling_loc_max, scaling_loc_min, scaling_sigma);
247 const double de = dEdG * scale;
250 const long long self_off = 3 * (sf + (
long long)n_symf * atom_in_e);
251 const long long f_i_off = 3 * (i_global + (
long long)num_atoms_global *
c);
252 if (threadIdx.x == 0) {
253 atomicAdd(&f_dev[f_i_off + 0], -de * self_partial[self_off + 0]);
254 atomicAdd(&f_dev[f_i_off + 1], -de * self_partial[self_off + 1]);
255 atomicAdd(&f_dev[f_i_off + 2], -de * self_partial[self_off + 2]);
261 double vir[3][3] = {{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}};
262 const double ci_x = accumulate_virial ? coord_dev[3 * i_global + 0] : 0.0;
263 const double ci_y = accumulate_virial ? coord_dev[3 * i_global + 1] : 0.0;
264 const double ci_z = accumulate_virial ? coord_dev[3 * i_global + 2] : 0.0;
266 const int n_ang1 = n_ang1_per_atom[atom_in_e];
267 const int n_ang2 = n_ang2_per_atom[atom_in_e];
270 for (
int j = threadIdx.x; j < n_ang1; j += blockDim.x) {
271 const long long nb_off = j + (
long long)max_n_ang1 * atom_in_e;
272 const int k_atom = ang1_ind_one_based[nb_off] - 1;
273 const long long jj_off = 3 * (sf + (
long long)n_symf * nb_off);
274 const long long f_k_off = 3 * (k_atom + (
long long)num_atoms_global *
c);
275 const double fk0 = -de * jj[jj_off + 0];
276 const double fk1 = -de * jj[jj_off + 1];
277 const double fk2 = -de * jj[jj_off + 2];
278 atomicAdd(&f_dev[f_k_off + 0], fk0);
279 atomicAdd(&f_dev[f_k_off + 1], fk1);
280 atomicAdd(&f_dev[f_k_off + 2], fk2);
281 if (accumulate_virial) {
282 const int img = ang1_image_idx[nb_off];
284 ci_x - (coord_dev[3 * k_atom + 0] + image_table_dev[3 * img + 0]);
286 ci_y - (coord_dev[3 * k_atom + 1] + image_table_dev[3 * img + 1]);
288 ci_z - (coord_dev[3 * k_atom + 2] + image_table_dev[3 * img + 2]);
289 vir[0][0] += -rb0 * fk0;
290 vir[0][1] += -rb1 * fk0;
291 vir[0][2] += -rb2 * fk0;
292 vir[1][0] += -rb0 * fk1;
293 vir[1][1] += -rb1 * fk1;
294 vir[1][2] += -rb2 * fk1;
295 vir[2][0] += -rb0 * fk2;
296 vir[2][1] += -rb1 * fk2;
297 vir[2][2] += -rb2 * fk2;
302 for (
int kk_j = threadIdx.x; kk_j < n_ang2; kk_j += blockDim.x) {
303 const int *idx_src = homo ? ang1_ind_one_based : ang2_ind_one_based;
304 const long long nb_off = kk_j + (
long long)max_n_kk * atom_in_e;
305 const int k_atom = idx_src[nb_off] - 1;
306 const long long kk_off = 3 * (sf + (
long long)n_symf * nb_off);
307 const long long f_k_off = 3 * (k_atom + (
long long)num_atoms_global *
c);
308 const double fk0 = -de * kk[kk_off + 0];
309 const double fk1 = -de * kk[kk_off + 1];
310 const double fk2 = -de * kk[kk_off + 2];
311 atomicAdd(&f_dev[f_k_off + 0], fk0);
312 atomicAdd(&f_dev[f_k_off + 1], fk1);
313 atomicAdd(&f_dev[f_k_off + 2], fk2);
314 if (accumulate_virial) {
315 const int *img_src = homo ? ang1_image_idx : ang2_image_idx;
316 const int img = img_src[nb_off];
318 ci_x - (coord_dev[3 * k_atom + 0] + image_table_dev[3 * img + 0]);
320 ci_y - (coord_dev[3 * k_atom + 1] + image_table_dev[3 * img + 1]);
322 ci_z - (coord_dev[3 * k_atom + 2] + image_table_dev[3 * img + 2]);
323 vir[0][0] += -rb0 * fk0;
324 vir[0][1] += -rb1 * fk0;
325 vir[0][2] += -rb2 * fk0;
326 vir[1][0] += -rb0 * fk1;
327 vir[1][1] += -rb1 * fk1;
328 vir[1][2] += -rb2 * fk1;
329 vir[2][0] += -rb0 * fk2;
330 vir[2][1] += -rb1 * fk2;
331 vir[2][2] += -rb2 * fk2;
335 if (accumulate_virial)
336 nnp_virial_warp_flush(vir, virial_dev, c);
346extern "C" int nnp_gpu_force_scatter_begin_c(
int num_atoms_global,
348 const double *h_force) {
349 if (num_atoms_global <= 0 || n_committee <= 0) {
351 "ERROR: nnp_gpu_force_scatter_begin: bad sizes (num_atoms=%d, "
353 num_atoms_global, n_committee);
356 nnp_gpu_state_t *state = nnp_gpu_state_get();
360 sizeof(double) * 3 * (
size_t)num_atoms_global * (size_t)n_committee;
361 if (nnp_gpu_pbuf_ensure((
void **)&state->f_dev, &state->f_dev_cap, bytes))
365 offloadMemcpyAsyncHtoD(state->f_dev, h_force, bytes, state->stream);
371 const size_t vbytes =
sizeof(double) * 9 * (
size_t)n_committee;
372 if (nnp_gpu_pbuf_ensure((
void **)&state->virial_dev, &state->virial_dev_cap,
375 offloadMemsetAsync(state->virial_dev, 0, vbytes, state->stream);
386extern "C" int nnp_gpu_force_scatter_end_c(
int num_atoms_global,
387 int n_committee,
double *h_force) {
388 if (num_atoms_global <= 0 || n_committee <= 0)
390 nnp_gpu_state_t *state = nnp_gpu_state_get();
391 if (state == NULL || state->f_dev == NULL)
394 sizeof(double) * 3 * (
size_t)num_atoms_global * (size_t)n_committee;
395 offloadMemcpyAsyncDtoH(h_force, state->f_dev, bytes, state->stream);
396 offloadStreamSynchronize(state->stream);
409extern "C" int nnp_gpu_force_scatter_virial_d2h_c(
int n_committee,
411 if (n_committee <= 0)
413 nnp_gpu_state_t *state = nnp_gpu_state_get();
414 if (state == NULL || state->virial_dev == NULL)
416 const size_t vbytes =
sizeof(double) * 9 * (
size_t)n_committee;
417 offloadMemcpyDtoH(h_virial, state->virial_dev, vbytes);
431extern "C" int nnp_gpu_force_scatter_radial_c(
432 const int n_atoms_in_e,
const int n_symf,
const int max_n_neigh_per_se,
433 const int n_committee,
const int n_input_max,
const int num_atoms_global,
434 const int ele_idx,
const int scaling_mode,
const double scmin,
435 const double scmax,
const int s_0based,
const int max_n_radgrp_walk,
436 const int n_atoms_in_e_max_walk,
const int max_n_neigh_walk,
437 const int istart_1based,
const int se_idx,
const int accumulate_virial) {
438 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_neigh_per_se <= 0 ||
439 n_committee <= 0 || n_input_max <= 0 || num_atoms_global <= 0)
441 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
443 "ERROR: nnp_gpu_force_scatter_radial: se_idx=%d out of range\n",
447 nnp_gpu_state_t *state = nnp_gpu_state_get();
448 if (state == NULL || state->f_dev == NULL)
450 if (!state->scaling_uploaded)
452 if (state->per_se_rad_self[se_idx] == NULL ||
453 state->per_se_rad_force[se_idx] == NULL) {
454 fprintf(stderr,
"ERROR: nnp_gpu_force_scatter_radial: per_se buffers "
455 "not populated (call the symmetry-function stage first)\n");
458 if (state->walk_n_rad_pa_dev == NULL || state->walk_rad_ind_dev == NULL ||
459 state->walk_atoms_global_dev == NULL || !state->se_static_uploaded) {
460 fprintf(stderr,
"ERROR: nnp_gpu_force_scatter_radial: walk / "
461 "se_static prerequisites not met\n");
466 if (state->pbuf_dEdG_out == NULL) {
467 fprintf(stderr,
"ERROR: nnp_gpu_force_scatter_radial: descriptor "
468 "gradient not on the device (run the network stage "
473 const size_t neigh_count = (size_t)max_n_neigh_per_se * (
size_t)n_atoms_in_e;
475 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atom_map,
476 &state->pbuf_atom_map_cap,
477 sizeof(
int) * (
size_t)n_atoms_in_e))
479 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atoms_global_rad,
480 &state->pbuf_atoms_global_rad_cap,
481 sizeof(
int) * (
size_t)n_atoms_in_e))
483 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_symf_map,
484 &state->pbuf_symf_map_cap,
485 sizeof(
int) * (
size_t)n_symf))
487 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_n_neigh_pa,
488 &state->pbuf_n_neigh_pa_cap,
489 sizeof(
int) * (
size_t)n_atoms_in_e))
491 if (nnp_gpu_pbuf_ensure((
void **)&state->scatter_neigh_ind,
492 &state->scatter_neigh_ind_cap,
493 sizeof(
int) * neigh_count))
495 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_izeta, &state->pbuf_izeta_cap,
496 sizeof(
int) * (
size_t)n_symf))
501 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_rad_ind,
502 &state->pbuf_rad_ind_cap,
sizeof(
int) * neigh_count))
504 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_rad_image_idx,
505 &state->pbuf_rad_image_idx_cap,
506 sizeof(
int) * neigh_count))
510 const int block = 32;
511 const int j_blocks = (max_n_neigh_per_se + block - 1) / block;
512 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
513 dim3 blk(block, 1, 1);
514 nnp_pack_walk_to_pbuf_radial_kernel<<<
grid, blk, 0, state->stream>>>(
515 n_atoms_in_e, max_n_neigh_per_se, s_0based, ele_idx, max_n_radgrp_walk,
516 n_atoms_in_e_max_walk, max_n_neigh_walk, istart_1based,
517 state->walk_n_rad_pa_dev, state->walk_rad_ind_dev,
518 state->walk_rad_image_idx_dev, state->walk_atoms_global_dev,
519 state->pbuf_n_neigh_pa, state->pbuf_rad_ind, state->pbuf_rad_image_idx,
520 state->pbuf_atoms_global_rad, state->pbuf_atom_map,
521 state->scatter_neigh_ind);
522 OFFLOAD_CHECK(offloadPeekAtLastError());
525 const int block = 32;
526 const int sf_blocks = (n_symf + block - 1) / block;
527 dim3
grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
528 dim3 blk(block, 1, 1);
529 const int se_idx_in_kind = s_0based + max_n_radgrp_walk * ele_idx;
532 nnp_pack_se_static_symf_map_kernel<<<
grid, blk, 0, state->stream>>>(
533 se_idx_in_kind, n_symf, 0,
534 state->se_static_rad_symf_packed_dev,
535 state->se_static_rad_symf_offsets_dev, state->pbuf_symf_map,
537 OFFLOAD_CHECK(offloadPeekAtLastError());
541 dim3
grid(n_atoms_in_e, n_symf, n_committee);
545 nnp_force_scatter_radial_kernel<<<
grid, blk, 0, state->stream>>>(
546 n_atoms_in_e, n_symf, max_n_neigh_per_se, n_committee, n_input_max,
547 num_atoms_global, ele_idx, scaling_mode, 0, scmin,
548 scmax, state->pbuf_izeta, state->pbuf_atom_map,
549 state->pbuf_atoms_global_rad, state->pbuf_n_neigh_pa,
550 state->scatter_neigh_ind, state->per_se_rad_self[se_idx],
551 state->per_se_rad_force[se_idx], state->pbuf_dEdG_out,
552 state->scaling_loc_max_dev, state->scaling_loc_min_dev,
553 state->scaling_sigma_dev, state->f_dev,
556 state->pbuf_rad_image_idx, state->coord_dev, state->image_table_dev,
557 state->virial_dev, accumulate_virial);
559 OFFLOAD_CHECK(offloadPeekAtLastError());
573extern "C" int nnp_gpu_force_scatter_angular_c(
574 const int n_atoms_in_e,
const int n_symf,
const int max_n_ang1_per_se,
575 const int max_n_kk_per_se,
const int n_committee,
const int n_input_max,
576 const int num_atoms_global,
const int ele_idx,
const int n_rad_offset,
577 const int scaling_mode,
const int homo,
const double scmin,
578 const double scmax,
const int s_0based,
const int max_n_anggrp_walk,
579 const int n_atoms_in_e_max_walk,
const int max_n_neigh_ang1_walk,
580 const int max_n_neigh_ang2_walk,
const int istart_1based,
const int se_idx,
581 const int accumulate_virial) {
582 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_ang1_per_se <= 0 ||
583 n_committee <= 0 || n_input_max <= 0 || num_atoms_global <= 0)
585 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
587 "ERROR: nnp_gpu_force_scatter_angular: se_idx=%d out of range\n",
591 nnp_gpu_state_t *state = nnp_gpu_state_get();
592 if (state == NULL || state->f_dev == NULL)
594 if (!state->scaling_uploaded)
596 if (state->per_se_ang_self[se_idx] == NULL ||
597 state->per_se_ang_jj[se_idx] == NULL ||
598 state->per_se_ang_kk[se_idx] == NULL) {
600 "ERROR: nnp_gpu_force_scatter_angular: per_se_ang "
601 "buffers not populated (call the symmetry-function stage first)\n");
604 if (state->walk_n_ang1_pa_dev == NULL || state->walk_ang1_ind_dev == NULL ||
605 state->walk_atoms_global_dev == NULL || !state->se_static_uploaded) {
606 fprintf(stderr,
"ERROR: nnp_gpu_force_scatter_angular: walk / "
607 "se_static prerequisites not met\n");
612 if (state->pbuf_dEdG_out == NULL) {
613 fprintf(stderr,
"ERROR: nnp_gpu_force_scatter_angular: descriptor "
614 "gradient not on the device (run the network stage "
621 if (homo && max_n_kk_per_se != max_n_ang1_per_se) {
623 "ERROR: nnp_gpu_force_scatter_angular: homonuclear slot needs "
624 "max_n_kk_per_se == max_n_ang1_per_se (%d vs %d); the k-side list "
625 "is the ang1 list and is packed with that stride\n",
626 max_n_kk_per_se, max_n_ang1_per_se);
630 const size_t ang1_count = (size_t)max_n_ang1_per_se * (
size_t)n_atoms_in_e;
631 const size_t ang2_count = (size_t)max_n_kk_per_se * (
size_t)n_atoms_in_e;
633 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atom_map,
634 &state->pbuf_atom_map_cap,
635 sizeof(
int) * (
size_t)n_atoms_in_e))
637 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atoms_global_ang,
638 &state->pbuf_atoms_global_ang_cap,
639 sizeof(
int) * (
size_t)n_atoms_in_e))
641 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_symf_map,
642 &state->pbuf_symf_map_cap,
643 sizeof(
int) * (
size_t)n_symf))
645 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_n_ang1_pa,
646 &state->pbuf_n_ang1_pa_cap,
647 sizeof(
int) * (
size_t)n_atoms_in_e))
649 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_n_ang2_pa,
650 &state->pbuf_n_ang2_pa_cap,
651 sizeof(
int) * (
size_t)n_atoms_in_e))
653 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang1_ind,
654 &state->pbuf_ang1_ind_cap,
sizeof(
int) * ang1_count))
656 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang1_image_idx,
657 &state->pbuf_ang1_image_idx_cap,
658 sizeof(
int) * ang1_count))
660 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang2_ind,
661 &state->pbuf_ang2_ind_cap,
662 sizeof(
int) * (ang2_count > 0 ? ang2_count : 1)))
664 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang2_image_idx,
665 &state->pbuf_ang2_image_idx_cap,
666 sizeof(
int) * (ang2_count > 0 ? ang2_count : 1)))
670 if (nnp_gpu_pbuf_ensure((
void **)&state->scatter_neigh_ind,
671 &state->scatter_neigh_ind_cap,
672 sizeof(
int) * (
size_t)n_symf))
677 const int block = 32;
678 dim3
grid((n_atoms_in_e + block - 1) / block, 1, 1);
679 dim3 blk(block, 1, 1);
680 nnp_pack_walk_to_pbuf_angular_per_atom_kernel<<<
grid, blk, 0,
682 n_atoms_in_e, homo, s_0based, ele_idx, max_n_anggrp_walk,
683 n_atoms_in_e_max_walk, istart_1based, state->walk_n_ang1_pa_dev,
684 state->walk_n_ang2_pa_dev, state->walk_atoms_global_dev,
685 state->pbuf_n_ang1_pa, state->pbuf_n_ang2_pa,
686 state->pbuf_atoms_global_ang, state->pbuf_atom_map);
687 OFFLOAD_CHECK(offloadPeekAtLastError());
692 const int block = 32;
693 const int j_blocks = (max_n_ang1_per_se + block - 1) / block;
694 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
695 dim3 blk(block, 1, 1);
696 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<
grid, blk, 0,
698 n_atoms_in_e, max_n_ang1_per_se, s_0based, ele_idx, max_n_anggrp_walk,
699 n_atoms_in_e_max_walk, max_n_neigh_ang1_walk,
700 1, state->walk_n_ang1_pa_dev, state->walk_ang1_ind_dev,
701 state->walk_ang1_image_idx_dev, state->pbuf_ang1_ind,
702 state->pbuf_ang1_image_idx);
703 OFFLOAD_CHECK(offloadPeekAtLastError());
705 if (!homo && max_n_kk_per_se > 0) {
706 const int block = 32;
707 const int j_blocks = (max_n_kk_per_se + block - 1) / block;
708 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
709 dim3 blk(block, 1, 1);
710 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<
grid, blk, 0,
712 n_atoms_in_e, max_n_kk_per_se, s_0based, ele_idx, max_n_anggrp_walk,
713 n_atoms_in_e_max_walk, max_n_neigh_ang2_walk,
714 1, state->walk_n_ang2_pa_dev, state->walk_ang2_ind_dev,
715 state->walk_ang2_image_idx_dev, state->pbuf_ang2_ind,
716 state->pbuf_ang2_image_idx);
717 OFFLOAD_CHECK(offloadPeekAtLastError());
724 const int block = 32;
725 const int sf_blocks = (n_symf + block - 1) / block;
726 dim3
grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
727 dim3 blk(block, 1, 1);
728 const int se_idx_in_kind = s_0based + max_n_anggrp_walk * ele_idx;
732 nnp_pack_se_static_symf_map_kernel<<<
grid, blk, 0, state->stream>>>(
733 se_idx_in_kind, n_symf, 0,
734 state->se_static_ang_symf_packed_dev,
735 state->se_static_ang_symf_offsets_dev, state->scatter_neigh_ind,
736 state->pbuf_symf_map);
737 OFFLOAD_CHECK(offloadPeekAtLastError());
741 dim3
grid(n_atoms_in_e, n_symf, n_committee);
743 nnp_force_scatter_angular_kernel<<<
grid, blk, 0, state->stream>>>(
744 n_atoms_in_e, n_symf, max_n_ang1_per_se, max_n_kk_per_se, n_committee,
745 n_input_max, num_atoms_global, ele_idx, scaling_mode, homo,
746 n_rad_offset, scmin, scmax, state->pbuf_symf_map, state->pbuf_atom_map,
747 state->pbuf_atoms_global_ang, state->pbuf_n_ang1_pa,
748 state->pbuf_n_ang2_pa, state->pbuf_ang1_ind, state->pbuf_ang2_ind,
749 state->per_se_ang_self[se_idx], state->per_se_ang_jj[se_idx],
750 state->per_se_ang_kk[se_idx], state->pbuf_dEdG_out,
751 state->scaling_loc_max_dev, state->scaling_loc_min_dev,
752 state->scaling_sigma_dev, state->f_dev,
756 state->pbuf_ang1_image_idx, state->pbuf_ang2_image_idx,
757 state->coord_dev, state->image_table_dev, state->virial_dev,
760 OFFLOAD_CHECK(offloadPeekAtLastError());
static GRID_HOST_DEVICE int idx(const orbital a)
Return coset index of given orbital angular momentum.
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid