9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
24static nnp_gpu_state_t *g_nnp_state = NULL;
28static size_t g_pbuf_total_bytes = 0;
29static unsigned long g_pbuf_grow_events = 0;
33static int g_nnp_gpu_precision_fp32 = 0;
39static int g_nnp_live_envs = 0;
50nnp_gpu_state_t *nnp_gpu_state_get(
void) {
51 if (g_nnp_state != NULL)
54 g_nnp_state = (nnp_gpu_state_t *)calloc(1,
sizeof(nnp_gpu_state_t));
55 if (g_nnp_state == NULL) {
56 fprintf(stderr,
"ERROR: nnp_gpu_state_get: calloc failed\n");
60 offloadStreamCreate(&g_nnp_state->stream);
62 g_nnp_state->md_step = 0;
63 g_nnp_state->spline_cache_size = 0;
76static void nnp_gpu_clear_cache(
struct nnp_gpu_cache_entry *cache,
int *size) {
77 for (
int i = 0;
i < *size;
i++) {
78 if (cache[
i].d_ptr != NULL) {
79 offloadFree(cache[
i].d_ptr);
80 cache[
i].d_ptr = NULL;
82 cache[
i].h_ptr = NULL;
83 cache[
i].n_doubles = 0;
94void nnp_gpu_state_release(
void) {
95 if (g_nnp_state == NULL)
100 offloadStreamSynchronize(g_nnp_state->stream);
102 nnp_gpu_clear_cache(g_nnp_state->spline_cache,
103 &g_nnp_state->spline_cache_size);
106#define NNP_PBUF_FREE(buf, cap) \
108 if (g_nnp_state->buf != NULL) { \
109 offloadFree(g_nnp_state->buf); \
110 g_nnp_state->buf = NULL; \
112 g_nnp_state->cap = 0; \
114 NNP_PBUF_FREE(coord_dev, coord_dev_cap);
115 NNP_PBUF_FREE(image_table_dev, image_table_dev_cap);
116 NNP_PBUF_FREE(coord_scaled_dev, coord_scaled_dev_cap);
117 NNP_PBUF_FREE(ele_ind_global_dev, ele_ind_global_dev_cap);
118 NNP_PBUF_FREE(cell_image_atom_dev, cell_image_atom_dev_cap);
119 NNP_PBUF_FREE(cell_image_shift_dev, cell_image_shift_dev_cap);
120 NNP_PBUF_FREE(cell_head_dev, cell_head_dev_cap);
121 NNP_PBUF_FREE(cell_next_dev, cell_next_dev_cap);
122 NNP_PBUF_FREE(pair_n_rad_dev, pair_n_rad_dev_cap);
123 NNP_PBUF_FREE(pair_n_ang1_dev, pair_n_ang1_dev_cap);
124 NNP_PBUF_FREE(pair_n_ang2_dev, pair_n_ang2_dev_cap);
125 NNP_PBUF_FREE(pair_max_cutoff_dev, pair_max_cutoff_dev_cap);
126 NNP_PBUF_FREE(pair_rad_groups_dev, pair_rad_groups_dev_cap);
127 NNP_PBUF_FREE(pair_ang1_groups_dev, pair_ang1_groups_dev_cap);
128 NNP_PBUF_FREE(pair_ang2_groups_dev, pair_ang2_groups_dev_cap);
129 NNP_PBUF_FREE(rad_cutoff_dev, rad_cutoff_dev_cap);
130 NNP_PBUF_FREE(ang_cutoff_dev, ang_cutoff_dev_cap);
131 NNP_PBUF_FREE(walk_atoms_global_dev, walk_atoms_global_dev_cap);
132 NNP_PBUF_FREE(walk_n_atoms_in_e_dev, walk_n_atoms_in_e_dev_cap);
133 NNP_PBUF_FREE(walk_rad_ind_dev, walk_rad_ind_dev_cap);
134 NNP_PBUF_FREE(walk_rad_image_idx_dev, walk_rad_image_idx_dev_cap);
135 NNP_PBUF_FREE(walk_ang1_ind_dev, walk_ang1_ind_dev_cap);
136 NNP_PBUF_FREE(walk_ang1_image_idx_dev, walk_ang1_image_idx_dev_cap);
137 NNP_PBUF_FREE(walk_ang2_ind_dev, walk_ang2_ind_dev_cap);
138 NNP_PBUF_FREE(walk_ang2_image_idx_dev, walk_ang2_image_idx_dev_cap);
139 NNP_PBUF_FREE(walk_n_rad_pa_dev, walk_n_rad_pa_dev_cap);
140 NNP_PBUF_FREE(walk_n_ang1_pa_dev, walk_n_ang1_pa_dev_cap);
141 NNP_PBUF_FREE(walk_n_ang2_pa_dev, walk_n_ang2_pa_dev_cap);
142 NNP_PBUF_FREE(walk_overflow_dev, walk_overflow_cap);
143 NNP_PBUF_FREE(G_dev, G_dev_cap);
144 NNP_PBUF_FREE(G_dev_scaled, G_dev_scaled_cap);
145 NNP_PBUF_FREE(expol_flag_dev, expol_flag_dev_cap);
146 NNP_PBUF_FREE(pbuf_symf_map, pbuf_symf_map_cap);
147 NNP_PBUF_FREE(pbuf_atom_map, pbuf_atom_map_cap);
148 NNP_PBUF_FREE(pbuf_n_neigh_pa, pbuf_n_neigh_pa_cap);
149 NNP_PBUF_FREE(pbuf_n_ang1_pa, pbuf_n_ang1_pa_cap);
150 NNP_PBUF_FREE(pbuf_n_ang2_pa, pbuf_n_ang2_pa_cap);
151 NNP_PBUF_FREE(pbuf_izeta, pbuf_izeta_cap);
152 NNP_PBUF_FREE(pbuf_rad_ind, pbuf_rad_ind_cap);
153 NNP_PBUF_FREE(pbuf_rad_image_idx, pbuf_rad_image_idx_cap);
154 NNP_PBUF_FREE(pbuf_atoms_global_rad, pbuf_atoms_global_rad_cap);
155 NNP_PBUF_FREE(pbuf_ang1_ind, pbuf_ang1_ind_cap);
156 NNP_PBUF_FREE(pbuf_ang2_ind, pbuf_ang2_ind_cap);
157 NNP_PBUF_FREE(pbuf_ang1_image_idx, pbuf_ang1_image_idx_cap);
158 NNP_PBUF_FREE(pbuf_ang2_image_idx, pbuf_ang2_image_idx_cap);
159 NNP_PBUF_FREE(pbuf_atoms_global_ang, pbuf_atoms_global_ang_cap);
160 NNP_PBUF_FREE(scatter_neigh_ind, scatter_neigh_ind_cap);
161 NNP_PBUF_FREE(pbuf_ang1_disp, pbuf_ang1_disp_cap);
162 NNP_PBUF_FREE(pbuf_ang2_disp, pbuf_ang2_disp_cap);
163 NNP_PBUF_FREE(scaling_loc_av_dev, scaling_loc_av_cap);
164 NNP_PBUF_FREE(scaling_loc_min_dev, scaling_loc_min_cap);
165 NNP_PBUF_FREE(scaling_loc_max_dev, scaling_loc_max_cap);
166 NNP_PBUF_FREE(scaling_sigma_dev, scaling_sigma_cap);
167 NNP_PBUF_FREE(scaling_n_input_nodes_dev, scaling_n_input_nodes_cap);
168 NNP_PBUF_FREE(se_static_rad_symf_packed_dev, se_static_rad_symf_packed_cap);
169 NNP_PBUF_FREE(se_static_rad_symf_offsets_dev, se_static_rad_symf_offsets_cap);
170 NNP_PBUF_FREE(se_static_ang_symf_packed_dev, se_static_ang_symf_packed_cap);
171 NNP_PBUF_FREE(se_static_ang_symf_offsets_dev, se_static_ang_symf_offsets_cap);
172 NNP_PBUF_FREE(step_ele_ind_dev, step_ele_ind_cap);
174 free(g_nnp_state->step_ele_ind_host);
175 g_nnp_state->step_ele_ind_host = NULL;
176 g_nnp_state->step_ele_ind_host_cap = 0;
177 NNP_PBUF_FREE(nn_n_nodes_dev, nn_n_nodes_dev_cap);
178 NNP_PBUF_FREE(nn_act_fnct_dev, nn_act_fnct_dev_cap);
179 NNP_PBUF_FREE(nn_atom_energies_dev, nn_atom_energies_dev_cap);
180 NNP_PBUF_FREE(nn_W_packed_dev, nn_W_packed_dev_cap);
181 NNP_PBUF_FREE(nn_b_packed_dev, nn_b_packed_dev_cap);
182 NNP_PBUF_FREE(pbuf_atomE_out, pbuf_atomE_out_cap);
183 NNP_PBUF_FREE(pbuf_dEdG_out, pbuf_dEdG_out_cap);
184 NNP_PBUF_FREE(f_dev, f_dev_cap);
185 NNP_PBUF_FREE(virial_dev, virial_dev_cap);
189#define NNP_PBUF_FREE_AT(buf, cap, i) \
191 if (g_nnp_state->buf[i] != NULL) { \
192 offloadFree(g_nnp_state->buf[i]); \
193 g_nnp_state->buf[i] = NULL; \
195 g_nnp_state->cap[i] = 0; \
197 for (
int s = 0; s < NNP_MAX_SE; s++) {
198 NNP_PBUF_FREE_AT(per_se_rad_self, per_se_rad_self_cap, s);
199 NNP_PBUF_FREE_AT(per_se_rad_force, per_se_rad_force_cap, s);
200 NNP_PBUF_FREE_AT(per_se_ang_self, per_se_ang_self_cap, s);
201 NNP_PBUF_FREE_AT(per_se_ang_jj, per_se_ang_jj_cap, s);
202 NNP_PBUF_FREE_AT(per_se_ang_kk, per_se_ang_kk_cap, s);
203 NNP_PBUF_FREE_AT(ang_izeta_dev, ang_izeta_cap, s);
204 NNP_PBUF_FREE_AT(ang_uiz_dev, ang_uiz_cap, s);
206#undef NNP_PBUF_FREE_AT
208 offloadStreamDestroy(g_nnp_state->stream);
212 g_pbuf_total_bytes = 0;
223void nnp_gpu_reset_caches(
void) {
224 if (g_nnp_state == NULL)
226 nnp_gpu_clear_cache(g_nnp_state->spline_cache,
227 &g_nnp_state->spline_cache_size);
229 g_nnp_state->pair_map_uploaded = 0;
230 g_nnp_state->scaling_uploaded = 0;
231 g_nnp_state->scaling_n_ele = 0;
232 g_nnp_state->scaling_n_input_max = 0;
233 g_nnp_state->se_static_uploaded = 0;
234 g_nnp_state->se_static_max_n_radgrp = 0;
235 g_nnp_state->se_static_max_n_anggrp = 0;
236 g_nnp_state->nn_packed_uploaded = 0;
237 g_nnp_state->step_ele_ind_n = 0;
238 for (
int s = 0; s < NNP_MAX_SE; s++)
239 g_nnp_state->ang_zeta_n[s] = 0;
255int nnp_gpu_pbuf_ensure_named(
void **pbuf,
size_t *pcap,
size_t n_bytes,
256 const char *bufname) {
259 if (*pcap >= n_bytes)
262 const size_t old_cap = *pcap;
271 size_t alloc_bytes = n_bytes + n_bytes / 2;
272 if (alloc_bytes < n_bytes)
273 alloc_bytes = n_bytes;
278 g_pbuf_total_bytes -= old_cap;
280 offloadMalloc(pbuf, alloc_bytes);
282 g_pbuf_total_bytes += alloc_bytes;
283 g_pbuf_grow_events++;
294unsigned long nnp_gpu_pbuf_grow_events(
void) {
return g_pbuf_grow_events; }
303size_t nnp_gpu_pbuf_total_bytes(
void) {
return g_pbuf_total_bytes; }
314int nnp_gpu_max_se_c(
void) {
return NNP_MAX_SE; }
324int nnp_gpu_spline_cache_max_c(
void) {
return NNP_SPLINE_CACHE_MAX; }
342static double *nnp_gpu_cache_get_or_upload(
struct nnp_gpu_cache_entry *cache,
343 int *size,
int max,
const char *what,
346 for (
int i = 0;
i < *size;
i++) {
347 if (cache[
i].h_ptr == h_ptr) {
351 if (cache[
i].n_doubles < n_doubles) {
353 "ERROR: nnp_gpu %s: cached device copy holds %zu doubles but "
354 "%zu were requested for the same host array\n",
355 what, cache[
i].n_doubles, n_doubles);
358 return cache[
i].d_ptr;
363 "ERROR: nnp_gpu %s: device cache is full (%d entries). This model "
364 "needs more than the compiled-in limit; raise "
365 "NNP_SPLINE_CACHE_MAX in nnp_gpu_state.h and rebuild, or run this "
366 "model with USE_GPU OFF.\n",
371 double *d_ptr = NULL;
372 offloadMalloc((
void **)&d_ptr,
sizeof(
double) * n_doubles);
373 offloadMemcpyHtoD(d_ptr, h_ptr,
sizeof(
double) * n_doubles);
374 cache[*size].h_ptr = h_ptr;
375 cache[*size].d_ptr = d_ptr;
376 cache[*size].n_doubles = n_doubles;
392double *nnp_gpu_get_or_upload_spline(
const double *h_ptr,
size_t n_doubles) {
393 nnp_gpu_state_t *state = nnp_gpu_state_get();
396 return nnp_gpu_cache_get_or_upload(
397 state->spline_cache, &state->spline_cache_size, NNP_SPLINE_CACHE_MAX,
398 "spline_cache", h_ptr, n_doubles);
408int nnp_gpu_register_env(
void) {
return ++g_nnp_live_envs; }
417void nnp_gpu_unregister_env(
void) {
418 if (g_nnp_live_envs > 0)
432void nnp_gpu_release_env(
void) {
433 if (g_nnp_live_envs > 0)
435 if (g_nnp_live_envs == 0) {
436 nnp_gpu_state_release();
438 nnp_gpu_reset_caches();
450int nnp_gpu_set_precision(
int use_fp32) {
451 g_nnp_gpu_precision_fp32 = (use_fp32 != 0) ? 1 : 0;
463int nnp_gpu_precision_fp32(
void) {
return g_nnp_gpu_precision_fp32; }
static void const int const int i