9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
20#error "OpenMP should not be used in .cu files to accommodate HIP."
66int nnp_gpu_step_ele_ind_ensure(nnp_gpu_state_t *state,
const int *h_ele_ind,
68 const size_t n_bytes =
sizeof(int) * (
size_t)n_atoms;
69 if (state->step_ele_ind_n == n_atoms &&
70 memcmp(state->step_ele_ind_host, h_ele_ind, n_bytes) == 0)
72 if (state->step_ele_ind_host_cap < n_bytes) {
73 free(state->step_ele_ind_host);
74 state->step_ele_ind_host = (
int *)malloc(n_bytes);
75 if (state->step_ele_ind_host == NULL) {
76 state->step_ele_ind_host_cap = 0;
77 state->step_ele_ind_n = 0;
78 fprintf(stderr,
"ERROR: nnp_gpu_step_ele_ind_ensure: malloc failed\n");
81 state->step_ele_ind_host_cap = n_bytes;
83 if (nnp_gpu_pbuf_ensure((
void **)&state->step_ele_ind_dev,
84 &state->step_ele_ind_cap, n_bytes))
86 memcpy(state->step_ele_ind_host, h_ele_ind, n_bytes);
87 offloadMemcpyHtoD(state->step_ele_ind_dev, h_ele_ind, n_bytes);
88 state->step_ele_ind_n = n_atoms;
108template <
typename CT>
109static __device__ __forceinline__
void
110nnp_compute_fc_dfc(CT r, CT cutoff_s,
int cut_type, CT *fc, CT *dfc) {
115 const CT pi_over_c = (CT)3.141592653589793238462643383279502884 / cutoff_s;
116 const CT arg = pi_over_c * r;
117 *fc = (CT)0.5 * (cos(arg) + (CT)1.0);
118 *dfc = (CT)(-0.5) * sin(arg) * pi_over_c;
122 const CT th = tanh((CT)1.0 - r / cutoff_s);
123 const CT th2 = th * th;
125 *dfc = ((CT)(-3.0) / cutoff_s) * (th2 - th2 * th2);
161__global__
void nnp_radial_sf_force_kernel(
162 const int n_atoms,
const int n_symf,
const int max_n_neigh,
163 const int spline_n,
const double spline_dx,
const double spline_dx_inv,
164 const double spline_x_max,
const double *__restrict__ spline_y,
165 const double *__restrict__ spline_dy,
166 const int *__restrict__ n_neigh_per_atom,
167 const double *__restrict__ coord_dev,
168 const double *__restrict__ image_table_dev,
const int *__restrict__ rad_ind,
169 const int *__restrict__ rad_image_idx,
const int *__restrict__ atoms_global,
170 double *__restrict__ self_partial_flat,
double *__restrict__ force_flat,
171 double *__restrict__ G_dev,
const int *__restrict__ symf_map,
172 const int *__restrict__ atom_map,
const int n_input_max) {
173 const int sf = blockIdx.y * blockDim.x + threadIdx.x;
176 const int atom_idx = blockIdx.x;
177 if (atom_idx >= n_atoms)
180 const int n_neigh_a = n_neigh_per_atom[atom_idx];
184 const int i_glob = atoms_global[atom_idx];
185 const double cx = coord_dev[3 * i_glob + 0];
186 const double cy = coord_dev[3 * i_glob + 1];
187 const double cz = coord_dev[3 * i_glob + 2];
191 const long long r_off = (
long long)max_n_neigh * (
long long)atom_idx;
192 const long long force_atom_off =
193 3LL * (
long long)n_symf * (
long long)max_n_neigh * (
long long)atom_idx;
196 double sx = 0.0, sy = 0.0, sz = 0.0;
198 for (
int j = 0; j < n_neigh_a; j++) {
199 const int j_glob = rad_ind[r_off + j];
200 const int img = rad_image_idx[r_off + j];
207 const double image_pos_x =
208 coord_dev[3 * j_glob + 0] + image_table_dev[3 * img + 0];
209 const double image_pos_y =
210 coord_dev[3 * j_glob + 1] + image_table_dev[3 * img + 1];
211 const double image_pos_z =
212 coord_dev[3 * j_glob + 2] + image_table_dev[3 * img + 2];
213 const double rvect_x =
cx - image_pos_x;
214 const double rvect_y = cy - image_pos_y;
215 const double rvect_z = cz - image_pos_z;
217 sqrt(rvect_x * rvect_x + rvect_y * rvect_y + rvect_z * rvect_z);
219 const long long force_base = force_atom_off + 3 * sf + 3 * n_symf * j;
221 if (r >= spline_x_max) {
222 force_flat[force_base + 0] = 0.0;
223 force_flat[force_base + 1] = 0.0;
224 force_flat[force_base + 2] = 0.0;
228 int idx = (int)(r * spline_dx_inv);
231 if (
idx > spline_n - 2)
234 const double t = (r - (double)
idx * spline_dx) * spline_dx_inv;
235 const double t2 =
t *
t;
236 const double t3 = t2 *
t;
237 const double h00 = 2.0 * t3 - 3.0 * t2 + 1.0;
238 const double h10 = t3 - 2.0 * t2 +
t;
239 const double h01 = -2.0 * t3 + 3.0 * t2;
240 const double h11 = t3 - t2;
241 const double h10_dx = h10 * spline_dx;
242 const double h11_dx = h11 * spline_dx;
243 const double dh00 = 6.0 * (t2 -
t);
244 const double dh10 = 3.0 * t2 - 4.0 *
t + 1.0;
245 const double dh01 = -dh00;
246 const double dh11 = 3.0 * t2 - 2.0 *
t;
248 const double yi = spline_y[sf +
idx * n_symf];
249 const double yi1 = spline_y[sf + (
idx + 1) * n_symf];
250 const double dyi = spline_dy[sf +
idx * n_symf];
251 const double dyi1 = spline_dy[sf + (
idx + 1) * n_symf];
253 const double sym = h00 * yi + h10_dx * dyi + h01 * yi1 + h11_dx * dyi1;
256 const double r_inv = 1.0 / r;
257 const double dsymdr_full =
258 (dh00 * yi + dh01 * yi1) * spline_dx_inv + dh10 * dyi + dh11 * dyi1;
259 const double drdx_x = rvect_x * r_inv;
260 const double drdx_y = rvect_y * r_inv;
261 const double drdx_z = rvect_z * r_inv;
263 const double f_x = dsymdr_full * drdx_x;
264 const double f_y = dsymdr_full * drdx_y;
265 const double f_z = dsymdr_full * drdx_z;
267 force_flat[force_base + 0] = f_x;
268 force_flat[force_base + 1] = f_y;
269 force_flat[force_base + 2] = f_z;
279 const int g_row = symf_map[sf];
280 const int g_atom = atom_map[atom_idx];
281 atomicAdd(&G_dev[g_row + (
long long)n_input_max * g_atom], Gacc);
283 const long long self_base = 3LL * sf + 3LL * n_symf * atom_idx;
284 self_partial_flat[self_base + 0] = sx;
285 self_partial_flat[self_base + 1] = sy;
286 self_partial_flat[self_base + 2] = sz;
296template <
typename CT> __device__ __forceinline__ CT nnp_fast_exp(CT x) {
299template <> __device__ __forceinline__
float nnp_fast_exp<float>(
float x) {
302template <
typename CT>
303__device__ __forceinline__
void nnp_fast_sincos(CT x, CT &s, CT &c) {
308__device__ __forceinline__
void nnp_fast_sincos<float>(
float x,
float &s,
313template <
typename CT> __device__ __forceinline__ CT nnp_fast_tanh(CT x) {
316template <> __device__ __forceinline__
float nnp_fast_tanh<float>(
float x) {
319template <
typename CT> __device__ __forceinline__ CT nnp_fast_pow(CT b, CT e) {
323__device__ __forceinline__
float nnp_fast_pow<float>(
float b,
float e) {
341template <
typename CT>
342__device__ __forceinline__
void nnp_angular_triplet_math(
343 const CT r1,
const CT rv1_x,
const CT rv1_y,
const CT rv1_z,
const CT fc1,
344 const CT dfc1,
const CT r2,
const CT rv2_x,
const CT rv2_y,
const CT rv2_z,
345 const CT fc2,
const CT dfc2,
const CT r3,
const CT rvect3_x,
346 const CT rvect3_y,
const CT rvect3_z,
const CT r3_sqr,
const CT cutoff_s,
347 const int cut_type,
const CT lam_sf,
const CT zeta_sf,
const int izeta_sf,
348 const int use_int_zeta_sf,
const CT eta_sf,
const CT prefzeta_sf,
349 CT &sym_out, CT &fjj_x, CT &fjj_y, CT &fjj_z, CT &fkk_x, CT &fkk_y,
350 CT &fkk_z, CT &self_x_inc, CT &self_y_inc, CT &self_z_inc) {
352 const CT M_PI_DP = (CT)3.141592653589793238462643383279502884;
354 const CT rsqr1 = r1 * r1;
355 const CT rsqr2 = r2 * r2;
356 const CT r2sum = rsqr1 + rsqr2 + r3_sqr;
357 const CT f_il = r3_sqr - rsqr1 - rsqr2;
358 const CT g_il = (CT)(-2.0) * r1 * r2;
359 const CT costheta = f_il / g_il;
363 const CT arg = M_PI_DP * r3 / cutoff_s;
365 nnp_fast_sincos<CT>(arg, sarg, carg);
366 fcut3 = (CT)0.5 * (carg + (CT)1.0);
367 dfcut3 = (CT)(-0.5) * sarg * (M_PI_DP / cutoff_s);
369 const CT th = nnp_fast_tanh<CT>((CT)1.0 - r3 / cutoff_s);
370 fcut3 = th * th * th;
371 dfcut3 = ((CT)(-3.0) / cutoff_s) * (th * th - th * th * th * th);
374 const CT ftot = fc1 * fc2 * fcut3;
375 const CT dfcdr1 = dfc1 * fc2 * fcut3;
376 const CT dfcdr2 = fc1 * dfc2 * fcut3;
377 const CT dfcdr3 = fc1 * fc2 * dfcut3;
379 const CT r1_inv = (CT)1.0 / r1;
380 const CT r2_inv = (CT)1.0 / r2;
381 const CT r3_inv = (CT)1.0 / r3;
382 const CT dr1dx_x = rv1_x * r1_inv;
383 const CT dr1dx_y = rv1_y * r1_inv;
384 const CT dr1dx_z = rv1_z * r1_inv;
385 const CT dr2dx_x = rv2_x * r2_inv;
386 const CT dr2dx_y = rv2_y * r2_inv;
387 const CT dr2dx_z = rv2_z * r2_inv;
388 const CT dr3dx_x = rvect3_x * r3_inv;
389 const CT dr3dx_y = rvect3_y * r3_inv;
390 const CT dr3dx_z = rvect3_z * r3_inv;
392 const CT inv_g2 = (CT)1.0 / (g_il * g_il);
393 const CT dgdx_t1_x = (CT)2.0 * r2 * dr1dx_x;
394 const CT dgdx_t1_y = (CT)2.0 * r2 * dr1dx_y;
395 const CT dgdx_t1_z = (CT)2.0 * r2 * dr1dx_z;
396 const CT dgdx_t2_x = (CT)2.0 * r1 * dr2dx_x;
397 const CT dgdx_t2_y = (CT)2.0 * r1 * dr2dx_y;
398 const CT dgdx_t2_z = (CT)2.0 * r1 * dr2dx_z;
400 (CT)(-2.0) * (rv1_x + rv2_x) * g_il - f_il * (-(dgdx_t1_x + dgdx_t2_x));
402 (CT)(-2.0) * (rv1_y + rv2_y) * g_il - f_il * (-(dgdx_t1_y + dgdx_t2_y));
404 (CT)(-2.0) * (rv1_z + rv2_z) * g_il - f_il * (-(dgdx_t1_z + dgdx_t2_z));
405 const CT dcb2_x = (CT)2.0 * (rvect3_x + rv1_x) * g_il - f_il * dgdx_t1_x;
406 const CT dcb2_y = (CT)2.0 * (rvect3_y + rv1_y) * g_il - f_il * dgdx_t1_y;
407 const CT dcb2_z = (CT)2.0 * (rvect3_z + rv1_z) * g_il - f_il * dgdx_t1_z;
408 const CT dcb3_x = (CT)2.0 * (rv2_x - rvect3_x) * g_il - f_il * dgdx_t2_x;
409 const CT dcb3_y = (CT)2.0 * (rv2_y - rvect3_y) * g_il - f_il * dgdx_t2_y;
410 const CT dcb3_z = (CT)2.0 * (rv2_z - rvect3_z) * g_il - f_il * dgdx_t2_z;
412 CT tmp = (CT)1.0 + lam_sf * costheta;
415 if (tmp <= (CT)0.0) {
419 if (use_int_zeta_sf) {
421 int e = izeta_sf - 1;
424 base = (CT)1.0 / tmp;
435 tmpzeta = nnp_fast_pow<CT>(tmp, zeta_sf - (CT)1.0);
437 angular = tmpzeta * tmp;
440 const CT symtmp = nnp_fast_exp<CT>(-eta_sf * r2sum);
441 sym_out = prefzeta_sf * angular * symtmp * ftot;
443 const CT pref_lam = zeta_sf * tmpzeta * lam_sf * inv_g2;
444 const CT tmp_dsymdr = (CT)(-2.0) * symtmp * eta_sf;
445 const CT dsymdr1 = tmp_dsymdr * r1;
446 const CT dsymdr2 = tmp_dsymdr * r2;
447 const CT dsymdr3 = tmp_dsymdr * r3;
449 const CT pref_local = prefzeta_sf * symtmp * ftot;
450 const CT tmp1 = prefzeta_sf * angular * (ftot * dsymdr1 + dfcdr1 * symtmp);
451 const CT tmp2 = prefzeta_sf * angular * (ftot * dsymdr2 + dfcdr2 * symtmp);
452 const CT tmp3 = prefzeta_sf * angular * (ftot * dsymdr3 + dfcdr3 * symtmp);
454 const CT pl_dcb2_x = pref_local * pref_lam * dcb2_x;
455 const CT pl_dcb2_y = pref_local * pref_lam * dcb2_y;
456 const CT pl_dcb2_z = pref_local * pref_lam * dcb2_z;
457 const CT pl_dcb3_x = pref_local * pref_lam * dcb3_x;
458 const CT pl_dcb3_y = pref_local * pref_lam * dcb3_y;
459 const CT pl_dcb3_z = pref_local * pref_lam * dcb3_z;
461 fjj_x = pl_dcb2_x - tmp1 * dr1dx_x + tmp3 * dr3dx_x;
462 fjj_y = pl_dcb2_y - tmp1 * dr1dx_y + tmp3 * dr3dx_y;
463 fjj_z = pl_dcb2_z - tmp1 * dr1dx_z + tmp3 * dr3dx_z;
464 fkk_x = pl_dcb3_x - tmp2 * dr2dx_x - tmp3 * dr3dx_x;
465 fkk_y = pl_dcb3_y - tmp2 * dr2dx_y - tmp3 * dr3dx_y;
466 fkk_z = pl_dcb3_z - tmp2 * dr2dx_z - tmp3 * dr3dx_z;
468 self_x_inc = pref_local * pref_lam * dcb1_x + tmp1 * dr1dx_x + tmp2 * dr2dx_x;
469 self_y_inc = pref_local * pref_lam * dcb1_y + tmp1 * dr1dx_y + tmp2 * dr2dx_y;
470 self_z_inc = pref_local * pref_lam * dcb1_z + tmp1 * dr1dx_z + tmp2 * dr2dx_z;
528template <
typename CT,
bool OCC>
529__device__ __forceinline__
void nnp_angular_sf_force_body(
530 const int homo,
const int cut_type,
const int max_n_ang1,
531 const int max_n_ang2,
const int max_n_kk,
const int n_symf,
532 const int n_atoms,
const double cutoff_s,
const double cutoff_sqr,
533 const int *__restrict__ n_ang1_per_atom,
534 const int *__restrict__ n_ang2_per_atom,
535 const double *__restrict__ coord_dev,
536 const double *__restrict__ image_table_dev,
537 const int *__restrict__ ang1_ind,
const int *__restrict__ ang2_ind,
538 const int *__restrict__ ang1_image_idx,
539 const int *__restrict__ ang2_image_idx,
540 const int *__restrict__ atoms_global,
545 const float *__restrict__ ang1_disp,
const float *__restrict__ ang2_disp,
546 const double *__restrict__ pack_lam,
const double *__restrict__ pack_zeta,
547 const int *__restrict__ pack_izeta,
548 const char *__restrict__ pack_use_int_zeta,
549 const double *__restrict__ pack_eta,
550 const double *__restrict__ pack_prefzeta,
551 double *__restrict__ self_partial_flat,
double *__restrict__ jj_flat,
552 double *__restrict__ kk_flat,
double *__restrict__ G_dev,
553 const int *__restrict__ symf_map,
const int *__restrict__ atom_map,
554 const int n_input_max,
const int fused_kk) {
555 const int lane = threadIdx.x;
556 const int sf = blockIdx.y * blockDim.y + threadIdx.y;
559 const int atom_idx = blockIdx.x;
560 if (atom_idx >= n_atoms)
563 const int n_ang1_a = n_ang1_per_atom[atom_idx];
564 const int n_ang2_a = homo ? 0 : n_ang2_per_atom[atom_idx];
567 if (!homo && n_ang2_a <= 0)
570 const double lam_sf = pack_lam[sf];
571 const double zeta_sf = pack_zeta[sf];
572 const int izeta_sf = pack_izeta[sf];
573 const int use_int_zeta_sf = pack_use_int_zeta[sf] ? 1 : 0;
574 const double eta_sf = pack_eta[sf];
575 const double prefzeta_sf = pack_prefzeta[sf];
578 const int i_glob = atoms_global[atom_idx];
579 const double cx = coord_dev[3 * i_glob + 0];
580 const double cy = coord_dev[3 * i_glob + 1];
581 const double cz = coord_dev[3 * i_glob + 2];
586 const long long r_off = (
long long)max_n_ang1 * (
long long)atom_idx;
587 const long long r2_off = (
long long)max_n_ang2 * (
long long)atom_idx;
588 const long long jj_atom_off =
589 3LL * (
long long)n_symf * (
long long)max_n_ang1 * (
long long)atom_idx;
590 const long long kk_atom_off =
591 3LL * (
long long)n_symf * (
long long)max_n_kk * (
long long)atom_idx;
599 double self_x = 0.0, self_y = 0.0, self_z = 0.0;
600 const int stride = blockDim.x;
610 const int GEOM_SLOTS = 6;
611 extern __shared__
unsigned char nnp_kk_raw[];
612 CT *sh =
reinterpret_cast<CT *
>(nnp_kk_raw);
613 const int geom_total = OCC ? GEOM_SLOTS * 32 * blockDim.y : 0;
614 const int geom_base = OCC ? (threadIdx.y * 32 +
lane) * GEOM_SLOTS : 0;
615 CT *kk_sh = sh + geom_total;
616 const int kk_sh_base = threadIdx.y * 3 * max_n_kk;
618 for (
int k = lane; k < max_n_kk; k += stride) {
619 kk_sh[kk_sh_base + 3 * k + 0] = (CT)0;
620 kk_sh[kk_sh_base + 3 * k + 1] = (CT)0;
621 kk_sh[kk_sh_base + 3 * k + 2] = (CT)0;
627 for (
int j = lane; j < n_ang1_a; j += stride) {
631 CT rvect1_x, rvect1_y, rvect1_z, r1;
632 if (
sizeof(CT) == 4) {
633 rvect1_x = (CT)ang1_disp[3 * (r_off + j) + 0];
634 rvect1_y = (CT)ang1_disp[3 * (r_off + j) + 1];
635 rvect1_z = (CT)ang1_disp[3 * (r_off + j) + 2];
637 sqrt(rvect1_x * rvect1_x + rvect1_y * rvect1_y + rvect1_z * rvect1_z);
639 const int j_glob = ang1_ind[r_off + j];
640 const int img1 = ang1_image_idx[r_off + j];
641 const double image_pos_j_x =
642 coord_dev[3 * j_glob + 0] + image_table_dev[3 * img1 + 0];
643 const double image_pos_j_y =
644 coord_dev[3 * j_glob + 1] + image_table_dev[3 * img1 + 1];
645 const double image_pos_j_z =
646 coord_dev[3 * j_glob + 2] + image_table_dev[3 * img1 + 2];
647 const double d_rvect1_x =
cx - image_pos_j_x;
648 const double d_rvect1_y = cy - image_pos_j_y;
649 const double d_rvect1_z = cz - image_pos_j_z;
651 sqrt(d_rvect1_x * d_rvect1_x + d_rvect1_y * d_rvect1_y +
652 d_rvect1_z * d_rvect1_z);
653 rvect1_x = (CT)d_rvect1_x;
654 rvect1_y = (CT)d_rvect1_y;
655 rvect1_z = (CT)d_rvect1_z;
660 sh[geom_base + 0] = rvect1_x;
661 sh[geom_base + 1] = rvect1_y;
662 sh[geom_base + 2] = rvect1_z;
663 sh[geom_base + 3] = r1;
664 rvect1_x = sh[geom_base + 0];
665 rvect1_y = sh[geom_base + 1];
666 rvect1_z = sh[geom_base + 2];
667 r1 = sh[geom_base + 3];
670 double jj_acc_x = 0.0, jj_acc_y = 0.0, jj_acc_z = 0.0;
675 if (r1 < (CT)cutoff_s) {
677 nnp_compute_fc_dfc<CT>(r1, (CT)cutoff_s, cut_type, &fc1, &dfc1);
679 sh[geom_base + 4] = fc1;
680 sh[geom_base + 5] = dfc1;
681 fc1 = sh[geom_base + 4];
682 dfc1 = sh[geom_base + 5];
685 const int k_start = homo ? j + 1 : 0;
686 const int k_end = homo ? n_ang1_a : n_ang2_a;
687 for (
int k = k_start; k < k_end; k++) {
690 CT rvect2_x, rvect2_y, rvect2_z, r2;
691 if (
sizeof(CT) == 4) {
692 const float *disp_k = homo ? ang1_disp : ang2_disp;
693 const int koff = homo ? (r_off + k) : (r2_off + k);
694 rvect2_x = (CT)disp_k[3 * koff + 0];
695 rvect2_y = (CT)disp_k[3 * koff + 1];
696 rvect2_z = (CT)disp_k[3 * koff + 2];
697 r2 = sqrt(rvect2_x * rvect2_x + rvect2_y * rvect2_y +
698 rvect2_z * rvect2_z);
700 const int k_glob = homo ? ang1_ind[r_off + k] : ang2_ind[r2_off + k];
702 homo ? ang1_image_idx[r_off + k] : ang2_image_idx[r2_off + k];
703 const double image_pos_k_x =
704 coord_dev[3 * k_glob + 0] + image_table_dev[3 * img2 + 0];
705 const double image_pos_k_y =
706 coord_dev[3 * k_glob + 1] + image_table_dev[3 * img2 + 1];
707 const double image_pos_k_z =
708 coord_dev[3 * k_glob + 2] + image_table_dev[3 * img2 + 2];
709 const double d_rvect2_x =
cx - image_pos_k_x;
710 const double d_rvect2_y = cy - image_pos_k_y;
711 const double d_rvect2_z = cz - image_pos_k_z;
713 sqrt(d_rvect2_x * d_rvect2_x + d_rvect2_y * d_rvect2_y +
714 d_rvect2_z * d_rvect2_z);
715 rvect2_x = (CT)d_rvect2_x;
716 rvect2_y = (CT)d_rvect2_y;
717 rvect2_z = (CT)d_rvect2_z;
722 if (r2 >= (CT)cutoff_s)
725 nnp_compute_fc_dfc<CT>(r2, (CT)cutoff_s, cut_type, &fc2, &dfc2);
727 const CT rvect3_x = rvect2_x - rvect1_x;
728 const CT rvect3_y = rvect2_y - rvect1_y;
729 const CT rvect3_z = rvect2_z - rvect1_z;
731 rvect3_x * rvect3_x + rvect3_y * rvect3_y + rvect3_z * rvect3_z;
732 if (r3_sqr >= (CT)cutoff_sqr)
734 const CT r3 = sqrt(r3_sqr);
737 CT f_jj_x, f_jj_y, f_jj_z;
738 CT f_kk_x, f_kk_y, f_kk_z;
739 CT sx_inc, sy_inc, sz_inc;
740 nnp_angular_triplet_math<CT>(
741 r1, rvect1_x, rvect1_y, rvect1_z, fc1, dfc1, r2, rvect2_x, rvect2_y,
742 rvect2_z, fc2, dfc2, r3, rvect3_x, rvect3_y, rvect3_z, r3_sqr,
743 (CT)cutoff_s, cut_type, (CT)lam_sf, (CT)zeta_sf, izeta_sf,
744 use_int_zeta_sf, (CT)eta_sf, (CT)prefzeta_sf, sym_out, f_jj_x,
745 f_jj_y, f_jj_z, f_kk_x, f_kk_y, f_kk_z, sx_inc, sy_inc, sz_inc);
759 atomicAdd(&kk_sh[kk_sh_base + 3 * k + 0], f_kk_x);
760 atomicAdd(&kk_sh[kk_sh_base + 3 * k + 1], f_kk_y);
761 atomicAdd(&kk_sh[kk_sh_base + 3 * k + 2], f_kk_z);
766 const long long jj_base = jj_atom_off + 3 * sf + 3 * n_symf * j;
767 jj_flat[jj_base + 0] = (double)jj_acc_x;
768 jj_flat[jj_base + 1] = (double)jj_acc_y;
769 jj_flat[jj_base + 2] = (double)jj_acc_z;
776 Gacc += NNP_SHFL_XOR(Gacc, offset);
777 self_x += NNP_SHFL_XOR(self_x, offset);
778 self_y += NNP_SHFL_XOR(self_y, offset);
779 self_z += NNP_SHFL_XOR(self_z, offset);
782 const int g_row = symf_map[sf];
783 const int g_atom = atom_map[atom_idx];
784 atomicAdd(&G_dev[g_row + (
long long)n_input_max * g_atom], (
double)Gacc);
786 const long long self_base = 3LL * sf + 3LL * n_symf * atom_idx;
787 self_partial_flat[self_base + 0] = (double)self_x;
788 self_partial_flat[self_base + 1] = (double)self_y;
789 self_partial_flat[self_base + 2] = (double)self_z;
792 const int k_count = homo ? n_ang1_a : n_ang2_a;
803 for (
int k = lane; k < k_count; k += stride) {
804 const long long kk_base = kk_atom_off + 3 * sf + 3 * n_symf * k;
805 kk_flat[kk_base + 0] = (double)kk_sh[kk_sh_base + 3 * k + 0];
806 kk_flat[kk_base + 1] = (double)kk_sh[kk_sh_base + 3 * k + 1];
807 kk_flat[kk_base + 2] = (double)kk_sh[kk_sh_base + 3 * k + 2];
813 for (
int k = lane; k < k_count; k += stride) {
816 CT rvect2_x, rvect2_y, rvect2_z, r2;
817 if (
sizeof(CT) == 4) {
818 const float *disp_k = homo ? ang1_disp : ang2_disp;
819 const int koff = homo ? (r_off + k) : (r2_off + k);
820 rvect2_x = (CT)disp_k[3 * koff + 0];
821 rvect2_y = (CT)disp_k[3 * koff + 1];
822 rvect2_z = (CT)disp_k[3 * koff + 2];
824 sqrt(rvect2_x * rvect2_x + rvect2_y * rvect2_y + rvect2_z * rvect2_z);
826 const int k_glob = homo ? ang1_ind[r_off + k] : ang2_ind[r2_off + k];
828 homo ? ang1_image_idx[r_off + k] : ang2_image_idx[r2_off + k];
829 const double image_pos_k_x =
830 coord_dev[3 * k_glob + 0] + image_table_dev[3 * img2 + 0];
831 const double image_pos_k_y =
832 coord_dev[3 * k_glob + 1] + image_table_dev[3 * img2 + 1];
833 const double image_pos_k_z =
834 coord_dev[3 * k_glob + 2] + image_table_dev[3 * img2 + 2];
835 const double d_rvect2_x =
cx - image_pos_k_x;
836 const double d_rvect2_y = cy - image_pos_k_y;
837 const double d_rvect2_z = cz - image_pos_k_z;
839 sqrt(d_rvect2_x * d_rvect2_x + d_rvect2_y * d_rvect2_y +
840 d_rvect2_z * d_rvect2_z);
841 rvect2_x = (CT)d_rvect2_x;
842 rvect2_y = (CT)d_rvect2_y;
843 rvect2_z = (CT)d_rvect2_z;
847 double kk_acc_x = 0.0, kk_acc_y = 0.0, kk_acc_z = 0.0;
852 if (r2 < (CT)cutoff_s) {
854 nnp_compute_fc_dfc<CT>(r2, (CT)cutoff_s, cut_type, &fc2, &dfc2);
856 const int j_end = homo ? k : n_ang1_a;
857 for (
int j = 0; j < j_end; j++) {
859 CT rvect1_x, rvect1_y, rvect1_z, r1;
860 if (
sizeof(CT) == 4) {
861 rvect1_x = (CT)ang1_disp[3 * (r_off + j) + 0];
862 rvect1_y = (CT)ang1_disp[3 * (r_off + j) + 1];
863 rvect1_z = (CT)ang1_disp[3 * (r_off + j) + 2];
864 r1 = sqrt(rvect1_x * rvect1_x + rvect1_y * rvect1_y +
865 rvect1_z * rvect1_z);
867 const int j_glob = ang1_ind[r_off + j];
868 const int img1 = ang1_image_idx[r_off + j];
869 const double image_pos_j_x =
870 coord_dev[3 * j_glob + 0] + image_table_dev[3 * img1 + 0];
871 const double image_pos_j_y =
872 coord_dev[3 * j_glob + 1] + image_table_dev[3 * img1 + 1];
873 const double image_pos_j_z =
874 coord_dev[3 * j_glob + 2] + image_table_dev[3 * img1 + 2];
875 const double d_rvect1_x =
cx - image_pos_j_x;
876 const double d_rvect1_y = cy - image_pos_j_y;
877 const double d_rvect1_z = cz - image_pos_j_z;
879 sqrt(d_rvect1_x * d_rvect1_x + d_rvect1_y * d_rvect1_y +
880 d_rvect1_z * d_rvect1_z);
881 rvect1_x = (CT)d_rvect1_x;
882 rvect1_y = (CT)d_rvect1_y;
883 rvect1_z = (CT)d_rvect1_z;
887 if (r1 >= (CT)cutoff_s)
890 nnp_compute_fc_dfc<CT>(r1, (CT)cutoff_s, cut_type, &fc1, &dfc1);
892 const CT rvect3_x = rvect2_x - rvect1_x;
893 const CT rvect3_y = rvect2_y - rvect1_y;
894 const CT rvect3_z = rvect2_z - rvect1_z;
896 rvect3_x * rvect3_x + rvect3_y * rvect3_y + rvect3_z * rvect3_z;
897 if (r3_sqr >= (CT)cutoff_sqr)
899 const CT r3 = sqrt(r3_sqr);
902 CT f_jj_x, f_jj_y, f_jj_z;
903 CT f_kk_x, f_kk_y, f_kk_z;
904 CT sx_inc, sy_inc, sz_inc;
905 nnp_angular_triplet_math<CT>(
906 r1, rvect1_x, rvect1_y, rvect1_z, fc1, dfc1, r2, rvect2_x, rvect2_y,
907 rvect2_z, fc2, dfc2, r3, rvect3_x, rvect3_y, rvect3_z, r3_sqr,
908 (CT)cutoff_s, cut_type, (CT)lam_sf, (CT)zeta_sf, izeta_sf,
909 use_int_zeta_sf, (CT)eta_sf, (CT)prefzeta_sf, sym_out, f_jj_x,
910 f_jj_y, f_jj_z, f_kk_x, f_kk_y, f_kk_z, sx_inc, sy_inc, sz_inc);
918 const long long kk_base = kk_atom_off + 3 * sf + 3 * n_symf * k;
919 kk_flat[kk_base + 0] = (double)kk_acc_x;
920 kk_flat[kk_base + 1] = (double)kk_acc_y;
921 kk_flat[kk_base + 2] = (double)kk_acc_z;
930template <
typename CT>
931__global__
void nnp_angular_sf_force_kernel(
932 const int homo,
const int cut_type,
const int max_n_ang1,
933 const int max_n_ang2,
const int max_n_kk,
const int n_symf,
934 const int n_atoms,
const double cutoff_s,
const double cutoff_sqr,
935 const int *__restrict__ n_ang1_per_atom,
936 const int *__restrict__ n_ang2_per_atom,
937 const double *__restrict__ coord_dev,
938 const double *__restrict__ image_table_dev,
939 const int *__restrict__ ang1_ind,
const int *__restrict__ ang2_ind,
940 const int *__restrict__ ang1_image_idx,
941 const int *__restrict__ ang2_image_idx,
942 const int *__restrict__ atoms_global,
const float *__restrict__ ang1_disp,
943 const float *__restrict__ ang2_disp,
const double *__restrict__ pack_lam,
944 const double *__restrict__ pack_zeta,
const int *__restrict__ pack_izeta,
945 const char *__restrict__ pack_use_int_zeta,
946 const double *__restrict__ pack_eta,
947 const double *__restrict__ pack_prefzeta,
948 double *__restrict__ self_partial_flat,
double *__restrict__ jj_flat,
949 double *__restrict__ kk_flat,
double *__restrict__ G_dev,
950 const int *__restrict__ symf_map,
const int *__restrict__ atom_map,
951 const int n_input_max,
const int fused_kk) {
952 nnp_angular_sf_force_body<CT, false>(
953 homo, cut_type, max_n_ang1, max_n_ang2, max_n_kk, n_symf, n_atoms,
954 cutoff_s, cutoff_sqr, n_ang1_per_atom, n_ang2_per_atom, coord_dev,
955 image_table_dev, ang1_ind, ang2_ind, ang1_image_idx, ang2_image_idx,
956 atoms_global, ang1_disp, ang2_disp, pack_lam, pack_zeta, pack_izeta,
957 pack_use_int_zeta, pack_eta, pack_prefzeta, self_partial_flat, jj_flat,
958 kk_flat, G_dev, symf_map, atom_map, n_input_max, fused_kk);
972template <
typename CT>
974 const
int homo, const
int cut_type, const
int max_n_ang1,
975 const
int max_n_ang2, const
int max_n_kk, const
int n_symf,
976 const
int n_atoms, const
double cutoff_s, const
double cutoff_sqr,
977 const
int *__restrict__ n_ang1_per_atom,
978 const
int *__restrict__ n_ang2_per_atom,
979 const
double *__restrict__ coord_dev,
980 const
double *__restrict__ image_table_dev,
981 const
int *__restrict__ ang1_ind, const
int *__restrict__ ang2_ind,
982 const
int *__restrict__ ang1_image_idx,
983 const
int *__restrict__ ang2_image_idx,
984 const
int *__restrict__ atoms_global, const
float *__restrict__ ang1_disp,
985 const
float *__restrict__ ang2_disp, const
double *__restrict__ pack_lam,
986 const
double *__restrict__ pack_zeta, const
int *__restrict__ pack_izeta,
987 const
char *__restrict__ pack_use_int_zeta,
988 const
double *__restrict__ pack_eta,
989 const
double *__restrict__ pack_prefzeta,
990 double *__restrict__ self_partial_flat,
double *__restrict__ jj_flat,
991 double *__restrict__ kk_flat,
double *__restrict__ G_dev,
992 const
int *__restrict__ symf_map, const
int *__restrict__ atom_map,
993 const
int n_input_max, const
int fused_kk) {
994 nnp_angular_sf_force_body<CT, true>(
995 homo, cut_type, max_n_ang1, max_n_ang2, max_n_kk, n_symf, n_atoms,
996 cutoff_s, cutoff_sqr, n_ang1_per_atom, n_ang2_per_atom, coord_dev,
997 image_table_dev, ang1_ind, ang2_ind, ang1_image_idx, ang2_image_idx,
998 atoms_global, ang1_disp, ang2_disp, pack_lam, pack_zeta, pack_izeta,
999 pack_use_int_zeta, pack_eta, pack_prefzeta, self_partial_flat, jj_flat,
1000 kk_flat, G_dev, symf_map, atom_map, n_input_max, fused_kk);
1014__global__
void nnp_pack_angular_disp_kernel(
1015 int n_atoms_in_e,
int max_n_per_se,
const int *__restrict__ pbuf_n_pa,
1016 const int *__restrict__ pbuf_ind,
const int *__restrict__ pbuf_image_idx,
1017 const int *__restrict__ pbuf_atoms_global0,
1018 const double *__restrict__ coord_dev,
1019 const double *__restrict__ image_table_dev,
float *__restrict__ pbuf_disp) {
1020 const int atom = blockIdx.x;
1021 const int m = blockIdx.y * blockDim.x + threadIdx.x;
1022 if (
atom >= n_atoms_in_e || m >= max_n_per_se)
1024 const long long off = (
long long)max_n_per_se *
atom + m;
1025 const int n = pbuf_n_pa[
atom];
1027 pbuf_disp[3 * off + 0] = 0.f;
1028 pbuf_disp[3 * off + 1] = 0.f;
1029 pbuf_disp[3 * off + 2] = 0.f;
1032 const int ci = pbuf_atoms_global0[
atom];
1033 const int ni = pbuf_ind[off];
1034 const int im = pbuf_image_idx[off];
1035 const double rx = coord_dev[3 * ci + 0] -
1036 (coord_dev[3 * ni + 0] + image_table_dev[3 * im + 0]);
1037 const double ry = coord_dev[3 * ci + 1] -
1038 (coord_dev[3 * ni + 1] + image_table_dev[3 * im + 1]);
1039 const double rz = coord_dev[3 * ci + 2] -
1040 (coord_dev[3 * ni + 2] + image_table_dev[3 * im + 2]);
1041 pbuf_disp[3 * off + 0] = (float)rx;
1042 pbuf_disp[3 * off + 1] = (float)ry;
1043 pbuf_disp[3 * off + 2] = (float)rz;
1068__global__
void nnp_scale_G_dev_kernel(
1069 const int n_atoms,
const int n_input_max,
const int scaling_mode,
1070 const double scmin,
const double scmax,
1071 const int *__restrict__ ele_ind_for_atoms,
1072 const int *__restrict__ n_input_nodes_per_ele,
1073 const double *__restrict__ loc_av_per_ele,
1074 const double *__restrict__ loc_min_per_ele,
1075 const double *__restrict__ loc_max_per_ele,
1076 const double *__restrict__ sigma_per_ele,
const double *__restrict__ G_in,
1077 double *__restrict__ G_out,
int *__restrict__ expol_flag) {
1081 const double expol_threshold = 1.0e-4;
1083 const int i_local = blockIdx.x * blockDim.x + threadIdx.x;
1084 if (i_local >= n_atoms)
1087 const int ind = ele_ind_for_atoms[i_local];
1088 const int n_in = n_input_nodes_per_ele[ind];
1089 const int param_off = ind * n_input_max;
1090 const long long atom_off = (
long long)i_local * n_input_max;
1091 const double range = scmax - scmin;
1093 const double *loc_av = loc_av_per_ele + param_off;
1094 const double *loc_min = loc_min_per_ele + param_off;
1095 const double *loc_max = loc_max_per_ele + param_off;
1096 const double *sigma = sigma_per_ele + param_off;
1098 for (
int m = 0; m < n_in; ++m) {
1099 const double y = G_in[m + atom_off];
1102 if (y - loc_max[m] > expol_threshold || loc_min[m] - y > expol_threshold) {
1103 atomicOr(expol_flag, 1);
1106 switch (scaling_mode) {
1109 out = (y - loc_min[m]) / (loc_max[m] - loc_min[m]) * range + scmin;
1112 out = y - loc_av[m];
1116 out = (y - loc_av[m]) / (loc_max[m] - loc_min[m]) * range + scmin;
1119 out = (y - loc_av[m]) / sigma[m] * range + scmin;
1126 G_out[m + atom_off] = out;
1131 for (
int m = n_in; m < n_input_max; ++m) {
1132 G_out[m + atom_off] = 0.0;
1147extern "C" int nnp_gpu_begin_descriptor_step(
int n_atoms,
int n_input_max) {
1148 if (n_atoms <= 0 || n_input_max <= 0) {
1150 "ERROR: nnp_gpu_begin_descriptor_step: bad sizes "
1151 "n_atoms=%d n_input_max=%d\n",
1152 n_atoms, n_input_max);
1155 nnp_gpu_state_t *state = nnp_gpu_state_get();
1159 const size_t n_bytes =
sizeof(double) * (
size_t)n_input_max * (size_t)n_atoms;
1160 if (nnp_gpu_pbuf_ensure((
void **)&state->G_dev, &state->G_dev_cap, n_bytes))
1162 if (nnp_gpu_pbuf_ensure((
void **)&state->G_dev_scaled,
1163 &state->G_dev_scaled_cap, n_bytes))
1166 offloadMemsetAsync(state->G_dev, 0, n_bytes, state->stream);
1179extern "C" int nnp_gpu_upload_scaling_tables_c(
1180 int n_ele,
int n_input_max,
const double *h_loc_av_per_ele,
1181 const double *h_loc_min_per_ele,
const double *h_loc_max_per_ele,
1182 const double *h_sigma_per_ele,
const int *h_n_input_nodes_per_ele) {
1183 if (n_ele <= 0 || n_input_max <= 0) {
1185 "ERROR: nnp_gpu_upload_scaling_tables: bad sizes "
1186 "n_ele=%d n_input_max=%d\n",
1187 n_ele, n_input_max);
1190 nnp_gpu_state_t *state = nnp_gpu_state_get();
1194 const size_t param_doubles = (size_t)n_ele * (
size_t)n_input_max;
1195 const size_t param_bytes =
sizeof(double) * param_doubles;
1196 const size_t n_in_bytes =
sizeof(int) * (
size_t)n_ele;
1198 if (nnp_gpu_pbuf_ensure((
void **)&state->scaling_loc_av_dev,
1199 &state->scaling_loc_av_cap, param_bytes))
1201 if (nnp_gpu_pbuf_ensure((
void **)&state->scaling_loc_min_dev,
1202 &state->scaling_loc_min_cap, param_bytes))
1204 if (nnp_gpu_pbuf_ensure((
void **)&state->scaling_loc_max_dev,
1205 &state->scaling_loc_max_cap, param_bytes))
1207 if (nnp_gpu_pbuf_ensure((
void **)&state->scaling_sigma_dev,
1208 &state->scaling_sigma_cap, param_bytes))
1210 if (nnp_gpu_pbuf_ensure((
void **)&state->scaling_n_input_nodes_dev,
1211 &state->scaling_n_input_nodes_cap, n_in_bytes))
1214 offloadMemcpyAsyncHtoD(state->scaling_loc_av_dev, h_loc_av_per_ele,
1215 param_bytes, state->stream);
1216 offloadMemcpyAsyncHtoD(state->scaling_loc_min_dev, h_loc_min_per_ele,
1217 param_bytes, state->stream);
1218 offloadMemcpyAsyncHtoD(state->scaling_loc_max_dev, h_loc_max_per_ele,
1219 param_bytes, state->stream);
1220 offloadMemcpyAsyncHtoD(state->scaling_sigma_dev, h_sigma_per_ele, param_bytes,
1222 offloadMemcpyAsyncHtoD(state->scaling_n_input_nodes_dev,
1223 h_n_input_nodes_per_ele, n_in_bytes, state->stream);
1226 offloadStreamSynchronize(state->stream);
1228 state->scaling_uploaded = 1;
1229 state->scaling_n_ele = n_ele;
1230 state->scaling_n_input_max = n_input_max;
1247extern "C" int nnp_gpu_upload_se_static_tables_c(
1248 int n_ele,
int max_n_radgrp,
int max_n_anggrp,
int total_rad_symf,
1249 int total_ang_symf,
const int *h_rad_symf_packed,
1250 const int *h_rad_symf_offsets,
const int *h_ang_symf_packed,
1251 const int *h_ang_symf_offsets) {
1253 fprintf(stderr,
"ERROR: nnp_gpu_upload_se_static_tables: bad n_ele=%d\n",
1257 if (max_n_radgrp < 0 || max_n_anggrp < 0 || total_rad_symf < 0 ||
1258 total_ang_symf < 0) {
1260 "ERROR: nnp_gpu_upload_se_static_tables: negative size "
1261 "(max_n_radgrp=%d max_n_anggrp=%d total_rad_symf=%d "
1262 "total_ang_symf=%d)\n",
1263 max_n_radgrp, max_n_anggrp, total_rad_symf, total_ang_symf);
1266 nnp_gpu_state_t *state = nnp_gpu_state_get();
1270 const size_t n_rad_se = (size_t)max_n_radgrp * (
size_t)n_ele;
1271 const size_t n_ang_se = (size_t)max_n_anggrp * (
size_t)n_ele;
1272 const size_t rad_off_bytes =
sizeof(int) * (n_rad_se + 1);
1273 const size_t ang_off_bytes =
sizeof(int) * (n_ang_se + 1);
1274 const size_t rad_pack_bytes =
sizeof(int) * (
size_t)total_rad_symf;
1275 const size_t ang_pack_bytes =
sizeof(int) * (
size_t)total_ang_symf;
1277 if (max_n_radgrp > 0) {
1278 if (h_rad_symf_offsets == NULL) {
1280 "ERROR: nnp_gpu_upload_se_static_tables: "
1281 "h_rad_symf_offsets NULL with max_n_radgrp=%d\n",
1285 if (nnp_gpu_pbuf_ensure((
void **)&state->se_static_rad_symf_offsets_dev,
1286 &state->se_static_rad_symf_offsets_cap,
1289 offloadMemcpyAsyncHtoD(state->se_static_rad_symf_offsets_dev,
1290 h_rad_symf_offsets, rad_off_bytes, state->stream);
1291 if (total_rad_symf > 0) {
1292 if (h_rad_symf_packed == NULL) {
1294 "ERROR: nnp_gpu_upload_se_static_tables: "
1295 "h_rad_symf_packed NULL with total_rad_symf=%d\n",
1299 if (nnp_gpu_pbuf_ensure((
void **)&state->se_static_rad_symf_packed_dev,
1300 &state->se_static_rad_symf_packed_cap,
1303 offloadMemcpyAsyncHtoD(state->se_static_rad_symf_packed_dev,
1304 h_rad_symf_packed, rad_pack_bytes, state->stream);
1308 if (max_n_anggrp > 0) {
1309 if (h_ang_symf_offsets == NULL) {
1311 "ERROR: nnp_gpu_upload_se_static_tables: "
1312 "h_ang_symf_offsets NULL with max_n_anggrp=%d\n",
1316 if (nnp_gpu_pbuf_ensure((
void **)&state->se_static_ang_symf_offsets_dev,
1317 &state->se_static_ang_symf_offsets_cap,
1320 offloadMemcpyAsyncHtoD(state->se_static_ang_symf_offsets_dev,
1321 h_ang_symf_offsets, ang_off_bytes, state->stream);
1322 if (total_ang_symf > 0) {
1323 if (h_ang_symf_packed == NULL) {
1325 "ERROR: nnp_gpu_upload_se_static_tables: "
1326 "h_ang_symf_packed NULL with total_ang_symf=%d\n",
1330 if (nnp_gpu_pbuf_ensure((
void **)&state->se_static_ang_symf_packed_dev,
1331 &state->se_static_ang_symf_packed_cap,
1334 offloadMemcpyAsyncHtoD(state->se_static_ang_symf_packed_dev,
1335 h_ang_symf_packed, ang_pack_bytes, state->stream);
1339 offloadStreamSynchronize(state->stream);
1341 state->se_static_uploaded = 1;
1342 state->se_static_n_ele = n_ele;
1343 state->se_static_max_n_radgrp = max_n_radgrp;
1344 state->se_static_max_n_anggrp = max_n_anggrp;
1345 state->se_static_total_rad_symf = total_rad_symf;
1346 state->se_static_total_ang_symf = total_ang_symf;
1363extern "C" int nnp_gpu_scale_G_dev_c(
int n_atoms,
int n_input_max,
int n_ele,
1364 int scaling_mode,
double scmin,
1366 const int *h_ele_ind_for_atoms) {
1367 if (n_atoms <= 0 || n_input_max <= 0 || n_ele <= 0) {
1369 "ERROR: nnp_gpu_scale_G_dev: bad sizes n_atoms=%d "
1370 "n_input_max=%d n_ele=%d\n",
1371 n_atoms, n_input_max, n_ele);
1374 if (scaling_mode < 0 || scaling_mode > 4) {
1376 "ERROR: nnp_gpu_scale_G_dev: bad scaling_mode=%d (expect 0..4)\n",
1381 nnp_gpu_state_t *state = nnp_gpu_state_get();
1385 if (!state->scaling_uploaded) {
1386 fprintf(stderr,
"ERROR: nnp_gpu_scale_G_dev: scaling tables not uploaded "
1387 "(nnp_gpu_upload_scaling_tables_c must run at NNP init)\n");
1394 if (state->scaling_n_ele != n_ele ||
1395 state->scaling_n_input_max != n_input_max) {
1397 "ERROR: nnp_gpu_scale_G_dev: scaling tables were uploaded for "
1398 "n_ele=%d n_input_max=%d but the kernel was called with %d/%d\n",
1399 state->scaling_n_ele, state->scaling_n_input_max, n_ele,
1404 if (state->G_dev == NULL || state->G_dev_scaled == NULL) {
1405 fprintf(stderr,
"ERROR: nnp_gpu_scale_G_dev: G_dev not initialised "
1406 "(call nnp_gpu_begin_descriptor_step first)\n");
1413 if (nnp_gpu_step_ele_ind_ensure(state, h_ele_ind_for_atoms, n_atoms))
1419 if (nnp_gpu_pbuf_ensure((
void **)&state->expol_flag_dev,
1420 &state->expol_flag_dev_cap,
sizeof(
int)))
1422 offloadMemsetAsync(state->expol_flag_dev, 0,
sizeof(
int), state->stream);
1425 const int block = 64;
1426 const int grid = (n_atoms + block - 1) / block;
1427 nnp_scale_G_dev_kernel<<<
grid, block, 0, state->stream>>>(
1428 n_atoms, n_input_max, scaling_mode, scmin, scmax,
1429 state->step_ele_ind_dev, state->scaling_n_input_nodes_dev,
1430 state->scaling_loc_av_dev, state->scaling_loc_min_dev,
1431 state->scaling_loc_max_dev, state->scaling_sigma_dev, state->G_dev,
1432 state->G_dev_scaled, state->expol_flag_dev);
1433 OFFLOAD_CHECK(offloadPeekAtLastError());
1436 offloadMemcpyAsyncDtoH(&state->expol_flag_host, state->expol_flag_dev,
1437 sizeof(
int), state->stream);
1449extern "C" int nnp_gpu_extrapolation_flag_c(
void) {
1450 nnp_gpu_state_t *state = nnp_gpu_state_get();
1453 return state->expol_flag_host;
1476extern "C" int nnp_gpu_radial_sf_to_G_dev_c(
1477 const int n_atoms_in_e,
const int max_n_neigh_per_se,
const int n_symf,
1478 const int spline_n,
const double spline_dx,
const double spline_dx_inv,
1479 const double spline_x_max,
const double *h_spline_y,
1480 const double *h_spline_dy,
const int s_0based,
const int e_0based,
1481 const int max_n_radgrp_walk,
const int n_atoms_in_e_max_walk,
1482 const int max_n_neigh_walk,
const int istart_1based,
const int n_input_max,
1484 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_neigh_per_se <= 0 ||
1485 spline_n < 2 || n_input_max <= 0) {
1486 fprintf(stderr,
"ERROR: nnp_gpu_radial_sf_to_G_dev: bad sizes\n");
1489 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
1491 "ERROR: nnp_gpu_radial_sf_to_G_dev: se_idx=%d out of range\n",
1495 nnp_gpu_state_t *state = nnp_gpu_state_get();
1498 if (state->G_dev == NULL || state->coord_dev == NULL ||
1499 state->image_table_dev == NULL) {
1500 fprintf(stderr,
"ERROR: nnp_gpu_radial_sf_to_G_dev: G_dev / "
1501 "coord_dev / image_table_dev not initialised\n");
1504 if (state->walk_n_rad_pa_dev == NULL || state->walk_rad_ind_dev == NULL ||
1505 state->walk_rad_image_idx_dev == NULL ||
1506 state->walk_atoms_global_dev == NULL) {
1507 fprintf(stderr,
"ERROR: nnp_gpu_radial_sf_to_G_dev: walk buffers "
1508 "not initialised (call nnp_gpu_cell_list_walk_c first)\n");
1511 if (!state->se_static_uploaded) {
1512 fprintf(stderr,
"ERROR: nnp_gpu_radial_sf_to_G_dev: se_static "
1513 "tables not uploaded (call nnp_gpu_upload_se_static_"
1514 "tables_c at NNP init)\n");
1520 if (state->se_static_max_n_radgrp != max_n_radgrp_walk) {
1522 "ERROR: nnp_gpu_radial_sf_to_G_dev: se_static tables were "
1523 "uploaded with max_n_radgrp=%d but the kernel was called with %d\n",
1524 state->se_static_max_n_radgrp, max_n_radgrp_walk);
1528 const size_t neigh_count = (size_t)max_n_neigh_per_se * (
size_t)n_atoms_in_e;
1529 const size_t self_doubles = (size_t)3 * (
size_t)n_symf * (size_t)n_atoms_in_e;
1530 const size_t force_doubles = (size_t)3 * (
size_t)n_symf * neigh_count;
1532 double *d_spline_y = nnp_gpu_get_or_upload_spline(
1533 h_spline_y, (
size_t)n_symf * (
size_t)spline_n);
1534 double *d_spline_dy = nnp_gpu_get_or_upload_spline(
1535 h_spline_dy, (
size_t)n_symf * (
size_t)spline_n);
1536 if (d_spline_y == NULL || d_spline_dy == NULL)
1540 if (nnp_gpu_pbuf_ensure((
void **)&state->per_se_rad_self[se_idx],
1541 &state->per_se_rad_self_cap[se_idx],
1542 sizeof(
double) * self_doubles))
1544 if (nnp_gpu_pbuf_ensure((
void **)&state->per_se_rad_force[se_idx],
1545 &state->per_se_rad_force_cap[se_idx],
1546 sizeof(
double) * force_doubles))
1550 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_n_neigh_pa,
1551 &state->pbuf_n_neigh_pa_cap,
1552 sizeof(
int) * (
size_t)n_atoms_in_e))
1554 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_symf_map,
1555 &state->pbuf_symf_map_cap,
1556 sizeof(
int) * (
size_t)n_symf))
1558 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atom_map,
1559 &state->pbuf_atom_map_cap,
1560 sizeof(
int) * (
size_t)n_atoms_in_e))
1562 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_rad_ind,
1563 &state->pbuf_rad_ind_cap,
sizeof(
int) * neigh_count))
1565 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_rad_image_idx,
1566 &state->pbuf_rad_image_idx_cap,
1567 sizeof(
int) * neigh_count))
1569 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atoms_global_rad,
1570 &state->pbuf_atoms_global_rad_cap,
1571 sizeof(
int) * (
size_t)n_atoms_in_e))
1575 if (nnp_gpu_pbuf_ensure((
void **)&state->scatter_neigh_ind,
1576 &state->scatter_neigh_ind_cap,
1577 sizeof(
int) * neigh_count))
1582 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_izeta, &state->pbuf_izeta_cap,
1583 sizeof(
int) * (
size_t)n_symf))
1587 const int block = 32;
1588 const int j_blocks = (max_n_neigh_per_se + block - 1) / block;
1589 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1590 dim3 blk(block, 1, 1);
1591 nnp_pack_walk_to_pbuf_radial_kernel<<<
grid, blk, 0, state->stream>>>(
1592 n_atoms_in_e, max_n_neigh_per_se, s_0based, e_0based, max_n_radgrp_walk,
1593 n_atoms_in_e_max_walk, max_n_neigh_walk, istart_1based,
1594 state->walk_n_rad_pa_dev, state->walk_rad_ind_dev,
1595 state->walk_rad_image_idx_dev, state->walk_atoms_global_dev,
1596 state->pbuf_n_neigh_pa, state->pbuf_rad_ind, state->pbuf_rad_image_idx,
1597 state->pbuf_atoms_global_rad, state->pbuf_atom_map,
1598 state->scatter_neigh_ind);
1599 OFFLOAD_CHECK(offloadPeekAtLastError());
1602 const int block = 32;
1603 const int sf_blocks = (n_symf + block - 1) / block;
1604 dim3
grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
1605 dim3 blk(block, 1, 1);
1606 const int se_idx_in_kind = s_0based + max_n_radgrp_walk * e_0based;
1608 nnp_pack_se_static_symf_map_kernel<<<
grid, blk, 0, state->stream>>>(
1609 se_idx_in_kind, n_symf, 0,
1610 state->se_static_rad_symf_packed_dev,
1611 state->se_static_rad_symf_offsets_dev, state->pbuf_symf_map,
1613 OFFLOAD_CHECK(offloadPeekAtLastError());
1616 offloadMemsetAsync(state->per_se_rad_self[se_idx], 0,
1617 sizeof(
double) * self_doubles, state->stream);
1618 offloadMemsetAsync(state->per_se_rad_force[se_idx], 0,
1619 sizeof(
double) * force_doubles, state->stream);
1622 const int block = 64;
1623 dim3
grid(n_atoms_in_e, (n_symf + block - 1) / block, 1);
1624 dim3 blk(block, 1, 1);
1625 nnp_radial_sf_force_kernel<<<
grid, blk, 0, state->stream>>>(
1626 n_atoms_in_e, n_symf, max_n_neigh_per_se, spline_n, spline_dx,
1627 spline_dx_inv, spline_x_max, d_spline_y, d_spline_dy,
1628 state->pbuf_n_neigh_pa, state->coord_dev, state->image_table_dev,
1629 state->pbuf_rad_ind, state->pbuf_rad_image_idx,
1630 state->pbuf_atoms_global_rad, state->per_se_rad_self[se_idx],
1631 state->per_se_rad_force[se_idx], state->G_dev, state->pbuf_symf_map,
1632 state->pbuf_atom_map, n_input_max);
1634 OFFLOAD_CHECK(offloadPeekAtLastError());
1671extern "C" int nnp_gpu_angular_sf_to_G_dev_c(
1672 const int homo,
const int cut_type,
const int max_n_ang1_per_se,
1673 const int max_n_ang2_per_se,
const int max_n_kk_per_se,
const int n_symf,
1674 const int n_atoms_in_e,
const double cutoff_s,
const double cutoff_sqr,
1675 const double *h_pack_lam,
const double *h_pack_zeta,
1676 const double *h_pack_eta,
const double *h_pack_prefzeta,
1677 const int *h_pack_izeta,
const char *h_pack_use_int_zeta,
1678 const int s_0based,
const int e_0based,
const int max_n_anggrp_walk,
1679 const int n_atoms_in_e_max_walk,
const int max_n_neigh_ang1_walk,
1680 const int max_n_neigh_ang2_walk,
const int istart_1based,
1681 const int n_rad_for_e,
const int n_input_max,
const int se_idx) {
1682 if (n_atoms_in_e <= 0 || n_symf <= 0 || max_n_ang1_per_se <= 0 ||
1684 fprintf(stderr,
"ERROR: nnp_gpu_angular_sf_to_G_dev: bad sizes\n");
1687 if (!homo && max_n_ang2_per_se <= 0) {
1688 fprintf(stderr,
"ERROR: nnp_gpu_angular_sf_to_G_dev: hetero needs "
1689 "max_n_ang2_per_se>0\n");
1692 if (se_idx < 0 || se_idx >= NNP_MAX_SE) {
1694 "ERROR: nnp_gpu_angular_sf_to_G_dev: se_idx=%d out of range\n",
1705 const int k_bound = homo ? max_n_ang1_per_se : max_n_ang2_per_se;
1706 if (max_n_kk_per_se < k_bound) {
1708 "ERROR: nnp_gpu_angular_sf_to_G_dev: max_n_kk_per_se=%d is "
1709 "below the k-side neighbour bound %d\n",
1710 max_n_kk_per_se, k_bound);
1714 nnp_gpu_state_t *state = nnp_gpu_state_get();
1717 if (state->G_dev == NULL || state->coord_dev == NULL ||
1718 state->image_table_dev == NULL) {
1719 fprintf(stderr,
"ERROR: nnp_gpu_angular_sf_to_G_dev: "
1720 "G_dev / coord_dev / image_table_dev not initialised\n");
1723 if (state->walk_n_ang1_pa_dev == NULL || state->walk_ang1_ind_dev == NULL ||
1724 state->walk_ang1_image_idx_dev == NULL ||
1725 state->walk_atoms_global_dev == NULL) {
1726 fprintf(stderr,
"ERROR: nnp_gpu_angular_sf_to_G_dev: walk buffers "
1727 "not initialised (call nnp_gpu_cell_list_walk_c first)\n");
1731 (state->walk_n_ang2_pa_dev == NULL || state->walk_ang2_ind_dev == NULL ||
1732 state->walk_ang2_image_idx_dev == NULL)) {
1733 fprintf(stderr,
"ERROR: nnp_gpu_angular_sf_to_G_dev: hetero needs "
1734 "walk ang2 buffers initialised\n");
1737 if (!state->se_static_uploaded) {
1738 fprintf(stderr,
"ERROR: nnp_gpu_angular_sf_to_G_dev: se_static "
1739 "tables not uploaded\n");
1744 if (state->se_static_max_n_anggrp != max_n_anggrp_walk) {
1746 "ERROR: nnp_gpu_angular_sf_to_G_dev: se_static tables were "
1747 "uploaded with max_n_anggrp=%d but the kernel was called with %d\n",
1748 state->se_static_max_n_anggrp, max_n_anggrp_walk);
1752 const size_t neigh1_count = (size_t)max_n_ang1_per_se * (
size_t)n_atoms_in_e;
1753 const size_t neigh2_count =
1754 (size_t)(max_n_ang2_per_se > 0 ? max_n_ang2_per_se : 1) *
1755 (size_t)n_atoms_in_e;
1756 const size_t self_doubles = (size_t)3 * (
size_t)n_symf * (size_t)n_atoms_in_e;
1757 const size_t jj_doubles = (size_t)3 * (
size_t)n_symf * neigh1_count;
1758 const size_t kk_doubles = (size_t)3 * (
size_t)n_symf *
1759 (size_t)max_n_kk_per_se * (
size_t)n_atoms_in_e;
1761 const size_t n_pack = (size_t)n_symf;
1762 double *d_pack_lam = nnp_gpu_get_or_upload_spline(h_pack_lam, n_pack);
1763 double *d_pack_zeta = nnp_gpu_get_or_upload_spline(h_pack_zeta, n_pack);
1764 double *d_pack_eta = nnp_gpu_get_or_upload_spline(h_pack_eta, n_pack);
1765 double *d_pack_prefzeta =
1766 nnp_gpu_get_or_upload_spline(h_pack_prefzeta, n_pack);
1767 if (d_pack_lam == NULL || d_pack_zeta == NULL || d_pack_eta == NULL ||
1768 d_pack_prefzeta == NULL)
1772 if (nnp_gpu_pbuf_ensure((
void **)&state->per_se_ang_self[se_idx],
1773 &state->per_se_ang_self_cap[se_idx],
1774 sizeof(
double) * self_doubles))
1776 if (nnp_gpu_pbuf_ensure((
void **)&state->per_se_ang_jj[se_idx],
1777 &state->per_se_ang_jj_cap[se_idx],
1778 sizeof(
double) * jj_doubles))
1780 if (nnp_gpu_pbuf_ensure((
void **)&state->per_se_ang_kk[se_idx],
1781 &state->per_se_ang_kk_cap[se_idx],
1782 sizeof(
double) * kk_doubles))
1785 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_n_ang1_pa,
1786 &state->pbuf_n_ang1_pa_cap,
1787 sizeof(
int) * (
size_t)n_atoms_in_e))
1789 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_n_ang2_pa,
1790 &state->pbuf_n_ang2_pa_cap,
1791 sizeof(
int) * (
size_t)n_atoms_in_e))
1793 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_symf_map,
1794 &state->pbuf_symf_map_cap,
1795 sizeof(
int) * (
size_t)n_symf))
1797 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atom_map,
1798 &state->pbuf_atom_map_cap,
1799 sizeof(
int) * (
size_t)n_atoms_in_e))
1801 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang1_ind,
1802 &state->pbuf_ang1_ind_cap,
1803 sizeof(
int) * neigh1_count))
1805 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang1_image_idx,
1806 &state->pbuf_ang1_image_idx_cap,
1807 sizeof(
int) * neigh1_count))
1809 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang2_ind,
1810 &state->pbuf_ang2_ind_cap,
1811 sizeof(
int) * neigh2_count))
1813 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang2_image_idx,
1814 &state->pbuf_ang2_image_idx_cap,
1815 sizeof(
int) * neigh2_count))
1817 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atoms_global_ang,
1818 &state->pbuf_atoms_global_ang_cap,
1819 sizeof(
int) * (
size_t)n_atoms_in_e))
1828 if (state->ang_zeta_n[se_idx] != n_symf) {
1829 if (nnp_gpu_pbuf_ensure((
void **)&state->ang_izeta_dev[se_idx],
1830 &state->ang_izeta_cap[se_idx],
1831 sizeof(
int) * (
size_t)n_symf))
1833 if (nnp_gpu_pbuf_ensure((
void **)&state->ang_uiz_dev[se_idx],
1834 &state->ang_uiz_cap[se_idx],
1835 sizeof(
char) * (
size_t)n_symf))
1837 offloadMemcpyHtoD(state->ang_izeta_dev[se_idx], h_pack_izeta,
1838 sizeof(
int) * (
size_t)n_symf);
1839 offloadMemcpyHtoD(state->ang_uiz_dev[se_idx], h_pack_use_int_zeta,
1840 sizeof(
char) * (
size_t)n_symf);
1841 state->ang_zeta_n[se_idx] = n_symf;
1846 const int block = 32;
1847 dim3
grid((n_atoms_in_e + block - 1) / block, 1, 1);
1848 dim3 blk(block, 1, 1);
1849 nnp_pack_walk_to_pbuf_angular_per_atom_kernel<<<
grid, blk, 0,
1851 n_atoms_in_e, homo, s_0based, e_0based, max_n_anggrp_walk,
1852 n_atoms_in_e_max_walk, istart_1based, state->walk_n_ang1_pa_dev,
1853 state->walk_n_ang2_pa_dev, state->walk_atoms_global_dev,
1854 state->pbuf_n_ang1_pa, state->pbuf_n_ang2_pa,
1855 state->pbuf_atoms_global_ang, state->pbuf_atom_map);
1856 OFFLOAD_CHECK(offloadPeekAtLastError());
1862 const int block = 32;
1863 const int j_blocks = (max_n_ang1_per_se + block - 1) / block;
1864 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1865 dim3 blk(block, 1, 1);
1866 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<
grid, blk, 0,
1868 n_atoms_in_e, max_n_ang1_per_se, s_0based, e_0based, max_n_anggrp_walk,
1869 n_atoms_in_e_max_walk, max_n_neigh_ang1_walk,
1870 0, state->walk_n_ang1_pa_dev, state->walk_ang1_ind_dev,
1871 state->walk_ang1_image_idx_dev, state->pbuf_ang1_ind,
1872 state->pbuf_ang1_image_idx);
1873 OFFLOAD_CHECK(offloadPeekAtLastError());
1875 if (!homo && max_n_ang2_per_se > 0) {
1876 const int block = 32;
1877 const int j_blocks = (max_n_ang2_per_se + block - 1) / block;
1878 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1879 dim3 blk(block, 1, 1);
1880 nnp_pack_walk_to_pbuf_angular_per_neigh_kernel<<<
grid, blk, 0,
1882 n_atoms_in_e, max_n_ang2_per_se, s_0based, e_0based, max_n_anggrp_walk,
1883 n_atoms_in_e_max_walk, max_n_neigh_ang2_walk,
1884 0, state->walk_n_ang2_pa_dev, state->walk_ang2_ind_dev,
1885 state->walk_ang2_image_idx_dev, state->pbuf_ang2_ind,
1886 state->pbuf_ang2_image_idx);
1887 OFFLOAD_CHECK(offloadPeekAtLastError());
1897 if (nnp_gpu_precision_fp32()) {
1898 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang1_disp,
1899 &state->pbuf_ang1_disp_cap,
1900 sizeof(
float) * 3 * neigh1_count))
1903 const int block = 256;
1904 const int j_blocks = (max_n_ang1_per_se + block - 1) / block;
1905 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1906 dim3 blk(block, 1, 1);
1907 nnp_pack_angular_disp_kernel<<<
grid, blk, 0, state->stream>>>(
1908 n_atoms_in_e, max_n_ang1_per_se, state->pbuf_n_ang1_pa,
1909 state->pbuf_ang1_ind, state->pbuf_ang1_image_idx,
1910 state->pbuf_atoms_global_ang, state->coord_dev,
1911 state->image_table_dev, state->pbuf_ang1_disp);
1912 OFFLOAD_CHECK(offloadPeekAtLastError());
1914 if (!homo && max_n_ang2_per_se > 0) {
1915 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_ang2_disp,
1916 &state->pbuf_ang2_disp_cap,
1917 sizeof(
float) * 3 * neigh2_count))
1919 const int block = 256;
1920 const int j_blocks = (max_n_ang2_per_se + block - 1) / block;
1921 dim3
grid(n_atoms_in_e, j_blocks > 0 ? j_blocks : 1, 1);
1922 dim3 blk(block, 1, 1);
1923 nnp_pack_angular_disp_kernel<<<
grid, blk, 0, state->stream>>>(
1924 n_atoms_in_e, max_n_ang2_per_se, state->pbuf_n_ang2_pa,
1925 state->pbuf_ang2_ind, state->pbuf_ang2_image_idx,
1926 state->pbuf_atoms_global_ang, state->coord_dev,
1927 state->image_table_dev, state->pbuf_ang2_disp);
1928 OFFLOAD_CHECK(offloadPeekAtLastError());
1937 if (nnp_gpu_pbuf_ensure((
void **)&state->scatter_neigh_ind,
1938 &state->scatter_neigh_ind_cap,
1939 sizeof(
int) * (
size_t)n_symf))
1942 const int block = 32;
1943 const int sf_blocks = (n_symf + block - 1) / block;
1944 dim3
grid(sf_blocks > 0 ? sf_blocks : 1, 1, 1);
1945 dim3 blk(block, 1, 1);
1946 const int se_idx_in_kind = s_0based + max_n_anggrp_walk * e_0based;
1947 nnp_pack_se_static_symf_map_kernel<<<
grid, blk, 0, state->stream>>>(
1948 se_idx_in_kind, n_symf, n_rad_for_e,
1949 state->se_static_ang_symf_packed_dev,
1950 state->se_static_ang_symf_offsets_dev, state->pbuf_symf_map,
1951 state->scatter_neigh_ind);
1952 OFFLOAD_CHECK(offloadPeekAtLastError());
1955 offloadMemsetAsync(state->per_se_ang_self[se_idx], 0,
1956 sizeof(
double) * self_doubles, state->stream);
1957 offloadMemsetAsync(state->per_se_ang_jj[se_idx], 0,
1958 sizeof(
double) * jj_doubles, state->stream);
1959 offloadMemsetAsync(state->per_se_ang_kk[se_idx], 0,
1960 sizeof(
double) * kk_doubles, state->stream);
1965 int *d_ang2_ind_use = homo ? state->pbuf_ang1_ind : state->pbuf_ang2_ind;
1966 int *d_ang2_img_use =
1967 homo ? state->pbuf_ang1_image_idx : state->pbuf_ang2_image_idx;
1971 float *d_ang2_disp_use =
1972 homo ? state->pbuf_ang1_disp : state->pbuf_ang2_disp;
1973 const int lanes = 32;
1974 int sf_per_block = n_symf < 32 ? n_symf : 32;
1975 if (sf_per_block < 1)
1977 dim3
grid(n_atoms_in_e, (n_symf + sf_per_block - 1) / sf_per_block, 1);
1978 dim3 blk(lanes, sf_per_block, 1);
1983 const size_t ct_bytes =
1984 nnp_gpu_precision_fp32() ?
sizeof(float) : sizeof(double);
1985 const size_t k3_smem =
1986 (size_t)3 * (size_t)max_n_kk_per_se * ct_bytes * (size_t)blk.y;
1995 static size_t k3_cap = 0;
1997 int optin = 48 * 1024;
1998#if defined(__OFFLOAD_CUDA)
2003 const cudaError_t e_dev = cudaGetDevice(&dev);
2004 const cudaError_t e_att = cudaDeviceGetAttribute(
2005 &optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
2006 if (e_dev != cudaSuccess || e_att != cudaSuccess || optin < 48 * 1024)
2013 if (optin > 48 * 1024) {
2014 const cudaError_t e1 = cudaFuncSetAttribute(
2015 (
const void *)nnp_angular_sf_force_kernel<float>,
2016 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2017 const cudaError_t e2 = cudaFuncSetAttribute(
2018 (
const void *)nnp_angular_sf_force_kernel<double>,
2019 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2020 const cudaError_t e3 = cudaFuncSetAttribute(
2021 (
const void *)nnp_angular_sf_force_kernel_occ<float>,
2022 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2023 const cudaError_t e4 = cudaFuncSetAttribute(
2024 (
const void *)nnp_angular_sf_force_kernel_occ<double>,
2025 cudaFuncAttributeMaxDynamicSharedMemorySize, optin);
2026 if (e1 != cudaSuccess || e2 != cudaSuccess || e3 != cudaSuccess ||
2029 (void)cudaGetLastError();
2032 k3_cap = (size_t)optin;
2043 const size_t k3_decide_smem =
2045 (size_t)(homo ? max_n_neigh_ang1_walk : max_n_neigh_ang2_walk) *
2046 ct_bytes * (size_t)blk.y;
2047 const int k3_fused =
2048 (max_n_kk_per_se > 0 && k3_decide_smem <= k3_cap) ? 1 : 0;
2064 const size_t occ_geom_smem = (size_t)6 * 32 * (size_t)blk.y * ct_bytes;
2065 const size_t occ_kk_smem =
2066 (size_t)3 * (size_t)max_n_kk_per_se * (size_t)blk.y * ct_bytes;
2067 const size_t occ_decide_smem =
2068 occ_geom_smem + (k3_fused ? k3_decide_smem : 0);
2069 const size_t occ_smem = occ_geom_smem + (k3_fused ? occ_kk_smem : 0);
2070 const int ang_occ = (occ_decide_smem <= k3_cap) ? 1 : 0;
2074#define NNP_ANG_ARGS \
2075 homo, cut_type, max_n_ang1_per_se, max_n_ang2_per_se, max_n_kk_per_se, \
2076 n_symf, n_atoms_in_e, cutoff_s, cutoff_sqr, state->pbuf_n_ang1_pa, \
2077 state->pbuf_n_ang2_pa, state->coord_dev, state->image_table_dev, \
2078 state->pbuf_ang1_ind, d_ang2_ind_use, state->pbuf_ang1_image_idx, \
2079 d_ang2_img_use, state->pbuf_atoms_global_ang, state->pbuf_ang1_disp, \
2080 d_ang2_disp_use, d_pack_lam, d_pack_zeta, state->ang_izeta_dev[se_idx], \
2081 state->ang_uiz_dev[se_idx], d_pack_eta, d_pack_prefzeta, \
2082 state->per_se_ang_self[se_idx], state->per_se_ang_jj[se_idx], \
2083 state->per_se_ang_kk[se_idx], state->G_dev, state->pbuf_symf_map, \
2084 state->pbuf_atom_map, n_input_max, k3_fused
2085 if (nnp_gpu_precision_fp32()) {
2087 nnp_angular_sf_force_kernel_occ<float>
2088 <<<
grid, blk, occ_smem, state->stream>>>(NNP_ANG_ARGS);
2090 nnp_angular_sf_force_kernel<float>
2091 <<<
grid, blk, k3_fused ? k3_smem : 0, state->stream>>>(
2095 nnp_angular_sf_force_kernel_occ<double>
2096 <<<
grid, blk, occ_smem, state->stream>>>(NNP_ANG_ARGS);
2098 nnp_angular_sf_force_kernel<double>
2099 <<<
grid, blk, k3_fused ? k3_smem : 0, state->stream>>>(
2104 OFFLOAD_CHECK(offloadPeekAtLastError());
static GRID_HOST_DEVICE int idx(const orbital a)
Return coset index of given orbital angular momentum.
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int const double GRID_CONST_WHEN_COLLOCATE double * cx
__global__ __launch_bounds__(64) void calculate_coefficients(const kernel_params dev_)
Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done i...