9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
17#error "OpenMP should not be used in .cu files to accommodate HIP."
71__global__
void nnp_nn_kernel_atomwise(
72 const int n_atoms,
const int n_committee,
const int n_layer,
73 const int max_n_nodes,
const int n_input_max,
const int normnodes,
74 const int n_ele,
const int *__restrict__ n_nodes,
75 const int *__restrict__ act_fnct,
const int *__restrict__ ele_ind,
76 const double *__restrict__ atom_energies,
77 const double *__restrict__ weights,
const double *__restrict__ biases,
78 const double *__restrict__ G_batch,
double *__restrict__ atomic_energy,
79 double *__restrict__ dE_dG) {
81 const int atom = blockIdx.x;
82 const int c = blockIdx.y;
83 const int tid = threadIdx.x;
84 const int nth = blockDim.x;
85 if (
atom >= n_atoms || c >= n_committee)
88 const int ele = ele_ind[
atom];
89 const int n_input = n_nodes[0 + n_layer * ele];
92 extern __shared__
double smem[];
93 double *node_pre = smem;
94 double *node_post = node_pre + (size_t)max_n_nodes * n_layer;
95 double *delta_curr = node_post + (size_t)max_n_nodes * n_layer;
96 double *delta_prev = delta_curr + max_n_nodes;
100 for (
int j = tid; j < n_input; j += nth) {
102 G_batch[j + (
long long)n_input_max *
atom];
107 for (
int L = 2; L <= n_layer; L++) {
108 const int n_in = n_nodes[(L - 2) + n_layer * ele];
109 const int n_out = n_nodes[(L - 1) + n_layer * ele];
111 (size_t)max_n_nodes * max_n_nodes * c +
112 (
size_t)max_n_nodes * max_n_nodes * n_committee * (L - 2) +
113 (
size_t)max_n_nodes * max_n_nodes * n_committee * (n_layer - 1) * ele;
115 (size_t)max_n_nodes * c + (
size_t)max_n_nodes * n_committee * (L - 2) +
116 (
size_t)max_n_nodes * n_committee * (n_layer - 1) * ele;
117 const double *W = weights + W_off;
118 const double *
b = biases + b_off;
121 for (
int j = tid; j < n_out; j += nth) {
123 for (
int i = 0;
i < n_in;
i++) {
124 y += W[
i + max_n_nodes * j] * node_post[
i + max_n_nodes * (L - 2)];
131 node_pre[j + max_n_nodes * (L - 1)] = y;
134 const int act = act_fnct[L - 2];
141 y_post = exp(-0.5 * y * y);
150 y_post = 1.0 / (1.0 + exp(-y));
153 y_post = 1.0 - 1.0 / (1.0 + exp(-y));
159 y_post = log(exp(y) + 1.0);
168 node_post[j + max_n_nodes * (L - 1)] = y_post;
175 atomic_energy[
c + (
long long)n_committee *
atom] =
176 node_post[0 + max_n_nodes * (n_layer - 1)] + atom_energies[ele];
183 for (
int L = 2; L <= n_layer; L++) {
184 const int n_out = n_nodes[(L - 1) + n_layer * ele];
185 const int act = act_fnct[L - 2];
186 for (
int j = tid; j < n_out; j += nth) {
187 const double pre = node_pre[j + max_n_nodes * (L - 1)];
188 const double post = node_post[j + max_n_nodes * (L - 1)];
192 deriv = 1.0 - post * post;
205 const double e = exp(-pre);
206 const double d = 1.0 + e;
212 const double e = exp(-pre);
213 const double d = 1.0 + e;
214 deriv = -e / (
d *
d);
222 deriv = (exp(post) + 1.0) / exp(post);
232 deriv /= (double)n_nodes[(L - 2) + n_layer * ele];
233 node_pre[j + max_n_nodes * (L - 1)] = deriv;
247 const int n_out_last = n_nodes[(n_layer - 1) + n_layer * ele];
248 for (
int j = tid; j < n_out_last; j += nth)
249 delta_curr[j] = (j == 0) ? 1.0 : 0.0;
253 for (
int L = n_layer; L >= 2; L--) {
254 const int n_in_L = n_nodes[(L - 2) + n_layer * ele];
255 const int n_out_L = n_nodes[(L - 1) + n_layer * ele];
256 const size_t WL_off =
257 (size_t)max_n_nodes * max_n_nodes * c +
258 (
size_t)max_n_nodes * max_n_nodes * n_committee * (L - 2) +
259 (
size_t)max_n_nodes * max_n_nodes * n_committee * (n_layer - 1) * ele;
260 const double *WL = weights + WL_off;
263 for (
int j = tid; j < n_out_L; j += nth)
264 delta_curr[j] *= node_pre[j + max_n_nodes * (L - 1)];
268 for (
int i = tid;
i < n_in_L;
i += nth) {
270 for (
int j = 0; j < n_out_L; j++)
271 acc += WL[
i + max_n_nodes * j] * delta_curr[j];
276 double *tmp_swap = delta_curr;
277 delta_curr = delta_prev;
278 delta_prev = tmp_swap;
282 for (
int m = tid; m < n_input; m += nth) {
283 dE_dG[m + n_input_max *
c + (
long long)n_input_max * n_committee *
atom] =
303nnp_gpu_upload_nn_packed_c(
int n_committee,
int n_layer,
int max_n_nodes,
304 int n_ele,
const int *h_n_nodes,
305 const int *h_act_fnct,
const double *h_atom_energies,
306 const double *h_W_packed,
const double *h_b_packed) {
307 if (n_committee <= 0 || n_layer < 2 || max_n_nodes <= 0 || n_ele <= 0) {
309 "ERROR: nnp_gpu_upload_nn_packed: bad sizes "
310 "n_committee=%d n_layer=%d max_n_nodes=%d n_ele=%d\n",
311 n_committee, n_layer, max_n_nodes, n_ele);
314 nnp_gpu_state_t *state = nnp_gpu_state_get();
318 if (state->nn_packed_uploaded) {
319 if (state->nn_packed_n_committee == n_committee &&
320 state->nn_packed_n_layer == n_layer &&
321 state->nn_packed_max_n_nodes == max_n_nodes &&
322 state->nn_packed_n_ele == n_ele) {
326 "ERROR: nnp_gpu_upload_nn_packed: shape mismatch on second "
327 "upload (n_committee %d->%d, n_layer %d->%d, "
328 "max_n_nodes %d->%d, n_ele %d->%d)\n",
329 state->nn_packed_n_committee, n_committee, state->nn_packed_n_layer,
330 n_layer, state->nn_packed_max_n_nodes, max_n_nodes,
331 state->nn_packed_n_ele, n_ele);
335 const size_t n_nodes_cnt = (size_t)n_layer * (
size_t)n_ele;
336 const size_t act_cnt = (size_t)(n_layer - 1);
337 const size_t atom_e_cnt = (size_t)n_ele;
338 const size_t W_cnt = (size_t)max_n_nodes * (
size_t)max_n_nodes *
339 (size_t)n_committee * (
size_t)(n_layer - 1) *
341 const size_t b_cnt = (size_t)max_n_nodes * (
size_t)n_committee *
342 (size_t)(n_layer - 1) * (size_t)n_ele;
344 if (nnp_gpu_pbuf_ensure((
void **)&state->nn_n_nodes_dev,
345 &state->nn_n_nodes_dev_cap,
346 sizeof(
int) * n_nodes_cnt))
348 if (nnp_gpu_pbuf_ensure((
void **)&state->nn_act_fnct_dev,
349 &state->nn_act_fnct_dev_cap,
sizeof(
int) * act_cnt))
351 if (nnp_gpu_pbuf_ensure((
void **)&state->nn_atom_energies_dev,
352 &state->nn_atom_energies_dev_cap,
353 sizeof(
double) * atom_e_cnt))
355 if (nnp_gpu_pbuf_ensure((
void **)&state->nn_W_packed_dev,
356 &state->nn_W_packed_dev_cap,
sizeof(
double) * W_cnt))
358 if (nnp_gpu_pbuf_ensure((
void **)&state->nn_b_packed_dev,
359 &state->nn_b_packed_dev_cap,
sizeof(
double) * b_cnt))
362 offloadMemcpyAsyncHtoD(state->nn_n_nodes_dev, h_n_nodes,
363 sizeof(
int) * n_nodes_cnt, state->stream);
364 offloadMemcpyAsyncHtoD(state->nn_act_fnct_dev, h_act_fnct,
365 sizeof(
int) * act_cnt, state->stream);
366 offloadMemcpyAsyncHtoD(state->nn_atom_energies_dev, h_atom_energies,
367 sizeof(
double) * atom_e_cnt, state->stream);
368 offloadMemcpyAsyncHtoD(state->nn_W_packed_dev, h_W_packed,
369 sizeof(
double) * W_cnt, state->stream);
370 offloadMemcpyAsyncHtoD(state->nn_b_packed_dev, h_b_packed,
371 sizeof(
double) * b_cnt, state->stream);
372 offloadStreamSynchronize(state->stream);
374 state->nn_packed_uploaded = 1;
375 state->nn_packed_n_committee = n_committee;
376 state->nn_packed_n_layer = n_layer;
377 state->nn_packed_max_n_nodes = max_n_nodes;
378 state->nn_packed_n_ele = n_ele;
404nnp_gpu_nn_atomwise_with_G_dev_c(
const int n_atoms,
const int n_committee,
405 const int n_layer,
const int max_n_nodes,
406 const int n_input_max,
const int normnodes,
407 const int n_ele,
const int *h_ele_ind,
408 double *h_atomic_energy,
double *h_dE_dG) {
410 if (n_atoms <= 0 || n_committee <= 0 || n_layer < 2 || max_n_nodes <= 0 ||
411 n_input_max <= 0 || n_ele <= 0) {
412 fprintf(stderr,
"ERROR: nnp_gpu_nn_atomwise_with_G_dev: bad sizes\n");
417 const size_t smem_bytes =
418 sizeof(double) * 2ULL * (
size_t)max_n_nodes * ((size_t)n_layer + 1ULL);
428 static size_t smem_optin_bytes = 0;
429 if (smem_bytes > 48ULL * 1024ULL && smem_bytes > smem_optin_bytes) {
430#if defined(__OFFLOAD_CUDA)
431 int dev = 0, smem_max = 0;
432 OFFLOAD_CHECK(cudaGetDevice(&dev));
433 OFFLOAD_CHECK(cudaDeviceGetAttribute(
434 &smem_max, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev));
435 if ((
size_t)smem_max < smem_bytes ||
436 cudaFuncSetAttribute(nnp_nn_kernel_atomwise,
437 cudaFuncAttributeMaxDynamicSharedMemorySize,
438 (
int)smem_bytes) != cudaSuccess) {
440 "ERROR: nnp_gpu_nn_atomwise_with_G_dev: max_n_nodes=%d "
441 "n_input_max=%d needs %zu bytes of shared memory per block; "
442 "device allows %d.\n",
443 max_n_nodes, n_input_max, smem_bytes, smem_max);
446#elif defined(__OFFLOAD_HIP)
447 int dev = 0, smem_max = 0;
448 OFFLOAD_CHECK(hipGetDevice(&dev));
449 OFFLOAD_CHECK(hipDeviceGetAttribute(
450 &smem_max, hipDeviceAttributeMaxSharedMemoryPerBlock, dev));
451 if ((
size_t)smem_max < smem_bytes) {
453 "ERROR: nnp_gpu_nn_atomwise_with_G_dev: max_n_nodes=%d "
454 "n_input_max=%d needs %zu bytes of shared memory per block; "
455 "device allows %d.\n",
456 max_n_nodes, n_input_max, smem_bytes, smem_max);
460 smem_optin_bytes = smem_bytes;
463 const size_t E_cnt = (size_t)n_committee * (
size_t)n_atoms;
464 const size_t dEdG_cnt =
465 (size_t)n_input_max * (
size_t)n_committee * (size_t)n_atoms;
467 nnp_gpu_state_t *state = nnp_gpu_state_get();
470 if (state->G_dev_scaled == NULL) {
471 fprintf(stderr,
"ERROR: nnp_gpu_nn_atomwise_with_G_dev: G_dev_scaled "
472 "not initialised (call nnp_gpu_begin_descriptor_step + "
473 "nnp_gpu_scale_G_dev_c first)\n");
476 if (!state->nn_packed_uploaded) {
477 fprintf(stderr,
"ERROR: nnp_gpu_nn_atomwise_with_G_dev: NN weights not "
478 "uploaded (call nnp_gpu_upload_nn_packed_c at NNP init)\n");
481 if (state->nn_packed_n_committee != n_committee ||
482 state->nn_packed_n_layer != n_layer ||
483 state->nn_packed_max_n_nodes != max_n_nodes ||
484 state->nn_packed_n_ele != n_ele) {
486 "ERROR: nnp_gpu_nn_atomwise_with_G_dev: cached shape "
487 "(n_committee=%d n_layer=%d max_n_nodes=%d n_ele=%d) "
488 "differs from kernel call (%d/%d/%d/%d)\n",
489 state->nn_packed_n_committee, state->nn_packed_n_layer,
490 state->nn_packed_max_n_nodes, state->nn_packed_n_ele, n_committee,
491 n_layer, max_n_nodes, n_ele);
495 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_atomE_out,
496 &state->pbuf_atomE_out_cap,
sizeof(
double) * E_cnt))
498 if (nnp_gpu_pbuf_ensure((
void **)&state->pbuf_dEdG_out,
499 &state->pbuf_dEdG_out_cap,
sizeof(
double) * dEdG_cnt))
507 offloadMemsetAsync(state->pbuf_dEdG_out, 0,
sizeof(
double) * dEdG_cnt,
513 if (nnp_gpu_step_ele_ind_ensure(state, h_ele_ind, n_atoms))
519 dim3
grid(n_atoms, n_committee, 1);
521 nnp_nn_kernel_atomwise<<<
grid, blk, smem_bytes, state->stream>>>(
522 n_atoms, n_committee, n_layer, max_n_nodes, n_input_max, normnodes,
523 n_ele, state->nn_n_nodes_dev, state->nn_act_fnct_dev,
524 state->step_ele_ind_dev, state->nn_atom_energies_dev,
525 state->nn_W_packed_dev, state->nn_b_packed_dev, state->G_dev_scaled,
526 state->pbuf_atomE_out, state->pbuf_dEdG_out);
528 OFFLOAD_CHECK(offloadPeekAtLastError());
530 offloadMemcpyAsyncDtoH(h_atomic_energy, state->pbuf_atomE_out,
531 sizeof(
double) * E_cnt, state->stream);
536 offloadMemcpyAsyncDtoH(h_dE_dG, state->pbuf_dEdG_out,
537 sizeof(
double) * dEdG_cnt, state->stream);
541 offloadStreamSynchronize(state->stream);
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i