(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_state.c
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
9#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
10
11#include <stddef.h>
12#include <stdio.h>
13#include <stdlib.h>
14
15#include "nnp_gpu_state.h"
16
17/*
18 * Lifetime, scratch-buffer and cache management for the NNP GPU pipeline. The
19 * device state is a single per-process object; the host side is
20 * single-threaded, so there is no locking.
21 */
22
23/* The per-process device-state singleton. */
24static nnp_gpu_state_t *g_nnp_state = NULL;
25
26/* Total bytes currently held in grow-only scratch buffers, and the number of
27 * reallocations since process start. Diagnostics only. */
28static size_t g_pbuf_total_bytes = 0;
29static unsigned long g_pbuf_grow_events = 0;
30
31/* Selected GPU compute precision: nonzero selects single-precision angular
32 * symmetry functions. Set once at initialization. */
33static int g_nnp_gpu_precision_fp32 = 0;
34
35/* Number of NNP environments currently using the device state. The
36 * once-per-run uploads are keyed by shape only, so two live environments of the
37 * same architecture would share weights while overwriting each other's
38 * descriptor tables; the Fortran side refuses that rather than run it. */
39static int g_nnp_live_envs = 0;
40
41/* ****************************************************************************
42 */
43/* \brief Return the singleton device state, creating it and its work stream
44 * on first call.
45 * \return pointer to the device state, or NULL if the state allocation fails
46 * (a stream-creation failure aborts inside the offload wrappers)
47 * \author Dhruv Sharma (ds2173@cam.ac.uk)
48 * ****************************************************************************
49 */
50nnp_gpu_state_t *nnp_gpu_state_get(void) {
51 if (g_nnp_state != NULL)
52 return g_nnp_state;
53
54 g_nnp_state = (nnp_gpu_state_t *)calloc(1, sizeof(nnp_gpu_state_t));
55 if (g_nnp_state == NULL) {
56 fprintf(stderr, "ERROR: nnp_gpu_state_get: calloc failed\n");
57 return NULL;
58 }
59
60 offloadStreamCreate(&g_nnp_state->stream);
61
62 g_nnp_state->md_step = 0;
63 g_nnp_state->spline_cache_size = 0;
64 return g_nnp_state;
65}
66
67/* ****************************************************************************
68 */
69/* \brief Free every device copy held in a host-pointer-keyed cache and reset
70 * its entry count.
71 * \param cache the cache array
72 * \param size in/out entry count, set to 0 on return
73 * \author Dhruv Sharma (ds2173@cam.ac.uk)
74 * ****************************************************************************
75 */
76static void nnp_gpu_clear_cache(struct nnp_gpu_cache_entry *cache, int *size) {
77 for (int i = 0; i < *size; i++) {
78 if (cache[i].d_ptr != NULL) {
79 offloadFree(cache[i].d_ptr);
80 cache[i].d_ptr = NULL;
81 }
82 cache[i].h_ptr = NULL;
83 cache[i].n_doubles = 0;
84 }
85 *size = 0;
86}
87
88/* ****************************************************************************
89 */
90/* \brief Release the singleton device state and everything it owns.
91 * \author Dhruv Sharma (ds2173@cam.ac.uk)
92 * ****************************************************************************
93 */
94void nnp_gpu_state_release(void) {
95 if (g_nnp_state == NULL)
96 return;
97
98 /* Drain the work stream before anything is freed: a kernel still in flight
99 * holds pointers into the buffers below. */
100 offloadStreamSynchronize(g_nnp_state->stream);
101
102 nnp_gpu_clear_cache(g_nnp_state->spline_cache,
103 &g_nnp_state->spline_cache_size);
104
105 /* Free every grow-only scratch buffer. */
106#define NNP_PBUF_FREE(buf, cap) \
107 do { \
108 if (g_nnp_state->buf != NULL) { \
109 offloadFree(g_nnp_state->buf); \
110 g_nnp_state->buf = NULL; \
111 } \
112 g_nnp_state->cap = 0; \
113 } while (0)
114 NNP_PBUF_FREE(coord_dev, coord_dev_cap);
115 NNP_PBUF_FREE(image_table_dev, image_table_dev_cap);
116 NNP_PBUF_FREE(coord_scaled_dev, coord_scaled_dev_cap);
117 NNP_PBUF_FREE(ele_ind_global_dev, ele_ind_global_dev_cap);
118 NNP_PBUF_FREE(cell_image_atom_dev, cell_image_atom_dev_cap);
119 NNP_PBUF_FREE(cell_image_shift_dev, cell_image_shift_dev_cap);
120 NNP_PBUF_FREE(cell_head_dev, cell_head_dev_cap);
121 NNP_PBUF_FREE(cell_next_dev, cell_next_dev_cap);
122 NNP_PBUF_FREE(pair_n_rad_dev, pair_n_rad_dev_cap);
123 NNP_PBUF_FREE(pair_n_ang1_dev, pair_n_ang1_dev_cap);
124 NNP_PBUF_FREE(pair_n_ang2_dev, pair_n_ang2_dev_cap);
125 NNP_PBUF_FREE(pair_max_cutoff_dev, pair_max_cutoff_dev_cap);
126 NNP_PBUF_FREE(pair_rad_groups_dev, pair_rad_groups_dev_cap);
127 NNP_PBUF_FREE(pair_ang1_groups_dev, pair_ang1_groups_dev_cap);
128 NNP_PBUF_FREE(pair_ang2_groups_dev, pair_ang2_groups_dev_cap);
129 NNP_PBUF_FREE(rad_cutoff_dev, rad_cutoff_dev_cap);
130 NNP_PBUF_FREE(ang_cutoff_dev, ang_cutoff_dev_cap);
131 NNP_PBUF_FREE(walk_atoms_global_dev, walk_atoms_global_dev_cap);
132 NNP_PBUF_FREE(walk_n_atoms_in_e_dev, walk_n_atoms_in_e_dev_cap);
133 NNP_PBUF_FREE(walk_rad_ind_dev, walk_rad_ind_dev_cap);
134 NNP_PBUF_FREE(walk_rad_image_idx_dev, walk_rad_image_idx_dev_cap);
135 NNP_PBUF_FREE(walk_ang1_ind_dev, walk_ang1_ind_dev_cap);
136 NNP_PBUF_FREE(walk_ang1_image_idx_dev, walk_ang1_image_idx_dev_cap);
137 NNP_PBUF_FREE(walk_ang2_ind_dev, walk_ang2_ind_dev_cap);
138 NNP_PBUF_FREE(walk_ang2_image_idx_dev, walk_ang2_image_idx_dev_cap);
139 NNP_PBUF_FREE(walk_n_rad_pa_dev, walk_n_rad_pa_dev_cap);
140 NNP_PBUF_FREE(walk_n_ang1_pa_dev, walk_n_ang1_pa_dev_cap);
141 NNP_PBUF_FREE(walk_n_ang2_pa_dev, walk_n_ang2_pa_dev_cap);
142 NNP_PBUF_FREE(walk_overflow_dev, walk_overflow_cap);
143 NNP_PBUF_FREE(G_dev, G_dev_cap);
144 NNP_PBUF_FREE(G_dev_scaled, G_dev_scaled_cap);
145 NNP_PBUF_FREE(expol_flag_dev, expol_flag_dev_cap);
146 NNP_PBUF_FREE(pbuf_symf_map, pbuf_symf_map_cap);
147 NNP_PBUF_FREE(pbuf_atom_map, pbuf_atom_map_cap);
148 NNP_PBUF_FREE(pbuf_n_neigh_pa, pbuf_n_neigh_pa_cap);
149 NNP_PBUF_FREE(pbuf_n_ang1_pa, pbuf_n_ang1_pa_cap);
150 NNP_PBUF_FREE(pbuf_n_ang2_pa, pbuf_n_ang2_pa_cap);
151 NNP_PBUF_FREE(pbuf_izeta, pbuf_izeta_cap);
152 NNP_PBUF_FREE(pbuf_rad_ind, pbuf_rad_ind_cap);
153 NNP_PBUF_FREE(pbuf_rad_image_idx, pbuf_rad_image_idx_cap);
154 NNP_PBUF_FREE(pbuf_atoms_global_rad, pbuf_atoms_global_rad_cap);
155 NNP_PBUF_FREE(pbuf_ang1_ind, pbuf_ang1_ind_cap);
156 NNP_PBUF_FREE(pbuf_ang2_ind, pbuf_ang2_ind_cap);
157 NNP_PBUF_FREE(pbuf_ang1_image_idx, pbuf_ang1_image_idx_cap);
158 NNP_PBUF_FREE(pbuf_ang2_image_idx, pbuf_ang2_image_idx_cap);
159 NNP_PBUF_FREE(pbuf_atoms_global_ang, pbuf_atoms_global_ang_cap);
160 NNP_PBUF_FREE(scatter_neigh_ind, scatter_neigh_ind_cap);
161 NNP_PBUF_FREE(pbuf_ang1_disp, pbuf_ang1_disp_cap);
162 NNP_PBUF_FREE(pbuf_ang2_disp, pbuf_ang2_disp_cap);
163 NNP_PBUF_FREE(scaling_loc_av_dev, scaling_loc_av_cap);
164 NNP_PBUF_FREE(scaling_loc_min_dev, scaling_loc_min_cap);
165 NNP_PBUF_FREE(scaling_loc_max_dev, scaling_loc_max_cap);
166 NNP_PBUF_FREE(scaling_sigma_dev, scaling_sigma_cap);
167 NNP_PBUF_FREE(scaling_n_input_nodes_dev, scaling_n_input_nodes_cap);
168 NNP_PBUF_FREE(se_static_rad_symf_packed_dev, se_static_rad_symf_packed_cap);
169 NNP_PBUF_FREE(se_static_rad_symf_offsets_dev, se_static_rad_symf_offsets_cap);
170 NNP_PBUF_FREE(se_static_ang_symf_packed_dev, se_static_ang_symf_packed_cap);
171 NNP_PBUF_FREE(se_static_ang_symf_offsets_dev, se_static_ang_symf_offsets_cap);
172 NNP_PBUF_FREE(step_ele_ind_dev, step_ele_ind_cap);
173 /* Host shadow of the element map: plain host memory, not a device buffer. */
174 free(g_nnp_state->step_ele_ind_host);
175 g_nnp_state->step_ele_ind_host = NULL;
176 g_nnp_state->step_ele_ind_host_cap = 0;
177 NNP_PBUF_FREE(nn_n_nodes_dev, nn_n_nodes_dev_cap);
178 NNP_PBUF_FREE(nn_act_fnct_dev, nn_act_fnct_dev_cap);
179 NNP_PBUF_FREE(nn_atom_energies_dev, nn_atom_energies_dev_cap);
180 NNP_PBUF_FREE(nn_W_packed_dev, nn_W_packed_dev_cap);
181 NNP_PBUF_FREE(nn_b_packed_dev, nn_b_packed_dev_cap);
182 NNP_PBUF_FREE(pbuf_atomE_out, pbuf_atomE_out_cap);
183 NNP_PBUF_FREE(pbuf_dEdG_out, pbuf_dEdG_out_cap);
184 NNP_PBUF_FREE(f_dev, f_dev_cap);
185 NNP_PBUF_FREE(virial_dev, virial_dev_cap);
186#undef NNP_PBUF_FREE
187
188 /* Per-(group, element) slot buffers. */
189#define NNP_PBUF_FREE_AT(buf, cap, i) \
190 do { \
191 if (g_nnp_state->buf[i] != NULL) { \
192 offloadFree(g_nnp_state->buf[i]); \
193 g_nnp_state->buf[i] = NULL; \
194 } \
195 g_nnp_state->cap[i] = 0; \
196 } while (0)
197 for (int s = 0; s < NNP_MAX_SE; s++) {
198 NNP_PBUF_FREE_AT(per_se_rad_self, per_se_rad_self_cap, s);
199 NNP_PBUF_FREE_AT(per_se_rad_force, per_se_rad_force_cap, s);
200 NNP_PBUF_FREE_AT(per_se_ang_self, per_se_ang_self_cap, s);
201 NNP_PBUF_FREE_AT(per_se_ang_jj, per_se_ang_jj_cap, s);
202 NNP_PBUF_FREE_AT(per_se_ang_kk, per_se_ang_kk_cap, s);
203 NNP_PBUF_FREE_AT(ang_izeta_dev, ang_izeta_cap, s);
204 NNP_PBUF_FREE_AT(ang_uiz_dev, ang_uiz_cap, s);
205 }
206#undef NNP_PBUF_FREE_AT
207
208 offloadStreamDestroy(g_nnp_state->stream);
209
210 free(g_nnp_state);
211 g_nnp_state = NULL;
212 g_pbuf_total_bytes = 0;
213}
214
215/* ****************************************************************************
216 */
217/* \brief Invalidate the host-pointer-keyed caches without freeing scratch
218 * buffers or destroying the stream, so a later NNP environment that
219 * reuses recycled host addresses cannot hit stale device copies.
220 * \author Dhruv Sharma (ds2173@cam.ac.uk)
221 * ****************************************************************************
222 */
223void nnp_gpu_reset_caches(void) {
224 if (g_nnp_state == NULL)
225 return;
226 nnp_gpu_clear_cache(g_nnp_state->spline_cache,
227 &g_nnp_state->spline_cache_size);
228 /* Force the once-at-init uploads to run again for the next environment. */
229 g_nnp_state->pair_map_uploaded = 0;
230 g_nnp_state->scaling_uploaded = 0;
231 g_nnp_state->scaling_n_ele = 0;
232 g_nnp_state->scaling_n_input_max = 0;
233 g_nnp_state->se_static_uploaded = 0;
234 g_nnp_state->se_static_max_n_radgrp = 0;
235 g_nnp_state->se_static_max_n_anggrp = 0;
236 g_nnp_state->nn_packed_uploaded = 0;
237 g_nnp_state->step_ele_ind_n = 0;
238 for (int s = 0; s < NNP_MAX_SE; s++)
239 g_nnp_state->ang_zeta_n[s] = 0;
240}
241
242/* ****************************************************************************
243 */
244/* \brief Ensure a grow-only scratch buffer is at least n_bytes, reallocating
245 * with slack when it must grow.
246 * \param pbuf in/out device pointer
247 * \param pcap in/out capacity in bytes
248 * \param n_bytes requested minimum size in bytes
249 * \param bufname buffer name (unused; kept for call-site symmetry)
250 * \return 0 on success; a device allocation failure aborts inside the offload
251 * wrappers
252 * \author Dhruv Sharma (ds2173@cam.ac.uk)
253 * ****************************************************************************
254 */
255int nnp_gpu_pbuf_ensure_named(void **pbuf, size_t *pcap, size_t n_bytes,
256 const char *bufname) {
257 (void)bufname;
258
259 if (*pcap >= n_bytes)
260 return 0;
261
262 const size_t old_cap = *pcap;
263
264 /* High-water-mark ratchet: grow with 1.5x slack so a per-(s,e) neighbour or
265 * symmetry-function count that drifts up by a few elements between MD steps
266 * is absorbed by spare capacity instead of forcing an offloadFree +
267 * offloadMalloc (and its implicit serialization) on nearly every step.
268 * Capacity only ever
269 * grows; the slack bytes are never read or written by any kernel, so results
270 * are bit-identical to an exact-size allocation. */
271 size_t alloc_bytes = n_bytes + n_bytes / 2;
272 if (alloc_bytes < n_bytes) /* overflow guard */
273 alloc_bytes = n_bytes;
274
275 if (*pbuf != NULL) {
276 offloadFree(*pbuf);
277 *pbuf = NULL;
278 g_pbuf_total_bytes -= old_cap;
279 }
280 offloadMalloc(pbuf, alloc_bytes);
281 *pcap = alloc_bytes;
282 g_pbuf_total_bytes += alloc_bytes;
283 g_pbuf_grow_events++;
284 return 0;
285}
286
287/* ****************************************************************************
288 */
289/* \brief Number of scratch-buffer reallocations since process start.
290 * \return the grow-event count
291 * \author Dhruv Sharma (ds2173@cam.ac.uk)
292 * ****************************************************************************
293 */
294unsigned long nnp_gpu_pbuf_grow_events(void) { return g_pbuf_grow_events; }
295
296/* ****************************************************************************
297 */
298/* \brief Total bytes currently held in grow-only scratch buffers.
299 * \return the byte count
300 * \author Dhruv Sharma (ds2173@cam.ac.uk)
301 * ****************************************************************************
302 */
303size_t nnp_gpu_pbuf_total_bytes(void) { return g_pbuf_total_bytes; }
304
305/* ****************************************************************************
306 */
307/* \brief Compiled-in capacity of the per-(group, element) device buffers,
308 * exported for the initialization-time model check in the Fortran
309 * driver.
310 * \return NNP_MAX_SE
311 * \author Dhruv Sharma (ds2173@cam.ac.uk)
312 * ****************************************************************************
313 */
314int nnp_gpu_max_se_c(void) { return NNP_MAX_SE; }
315
316/* ****************************************************************************
317 */
318/* \brief Compiled-in capacity of the host-pointer-keyed device cache, exported
319 * for the initialization-time model check in the Fortran driver.
320 * \return NNP_SPLINE_CACHE_MAX
321 * \author Dhruv Sharma (ds2173@cam.ac.uk)
322 * ****************************************************************************
323 */
324int nnp_gpu_spline_cache_max_c(void) { return NNP_SPLINE_CACHE_MAX; }
325
326/* ****************************************************************************
327 */
328/* \brief Look up a host array in a device cache, uploading and inserting it on
329 * a miss.
330 * \param cache the cache array
331 * \param size in/out entry count
332 * \param max cache capacity
333 * \param what label used in error messages
334 * \param h_ptr host source pointer (the cache key)
335 * \param n_doubles element count to upload on a miss
336 * \return the device pointer, or NULL when the cache is full or a hit is too
337 * small for the request (a device runtime error aborts inside the
338 * offload wrappers)
339 * \author Dhruv Sharma (ds2173@cam.ac.uk)
340 * ****************************************************************************
341 */
342static double *nnp_gpu_cache_get_or_upload(struct nnp_gpu_cache_entry *cache,
343 int *size, int max, const char *what,
344 const double *h_ptr,
345 size_t n_doubles) {
346 for (int i = 0; i < *size; i++) {
347 if (cache[i].h_ptr == h_ptr) {
348 /* A hit is keyed on the host address alone, so check the size too: an
349 * array reallocated to a larger extent at the same address would
350 * otherwise be read past the end of its device copy. */
351 if (cache[i].n_doubles < n_doubles) {
352 fprintf(stderr,
353 "ERROR: nnp_gpu %s: cached device copy holds %zu doubles but "
354 "%zu were requested for the same host array\n",
355 what, cache[i].n_doubles, n_doubles);
356 return NULL;
357 }
358 return cache[i].d_ptr;
359 }
360 }
361 if (*size >= max) {
362 fprintf(stderr,
363 "ERROR: nnp_gpu %s: device cache is full (%d entries). This model "
364 "needs more than the compiled-in limit; raise "
365 "NNP_SPLINE_CACHE_MAX in nnp_gpu_state.h and rebuild, or run this "
366 "model with USE_GPU OFF.\n",
367 what, max);
368 return NULL;
369 }
370
371 double *d_ptr = NULL;
372 offloadMalloc((void **)&d_ptr, sizeof(double) * n_doubles);
373 offloadMemcpyHtoD(d_ptr, h_ptr, sizeof(double) * n_doubles);
374 cache[*size].h_ptr = h_ptr;
375 cache[*size].d_ptr = d_ptr;
376 cache[*size].n_doubles = n_doubles;
377 (*size)++;
378 return d_ptr;
379}
380
381/* ****************************************************************************
382 */
383/* \brief Return the device copy of a run-constant host array, uploading and
384 * caching it on first request.
385 * \param h_ptr host source pointer (the cache key)
386 * \param n_doubles element count
387 * \return the device pointer, or NULL if the cache is full (a device runtime
388 * error aborts inside the offload wrappers)
389 * \author Dhruv Sharma (ds2173@cam.ac.uk)
390 * ****************************************************************************
391 */
392double *nnp_gpu_get_or_upload_spline(const double *h_ptr, size_t n_doubles) {
393 nnp_gpu_state_t *state = nnp_gpu_state_get();
394 if (state == NULL)
395 return NULL;
396 return nnp_gpu_cache_get_or_upload(
397 state->spline_cache, &state->spline_cache_size, NNP_SPLINE_CACHE_MAX,
398 "spline_cache", h_ptr, n_doubles);
399}
400
401/* ****************************************************************************
402 */
403/* \brief Count one more NNP environment as a user of the device state.
404 * \return the number of environments now live
405 * \author Dhruv Sharma (ds2173@cam.ac.uk)
406 * ****************************************************************************
407 */
408int nnp_gpu_register_env(void) { return ++g_nnp_live_envs; }
409
410/* ****************************************************************************
411 */
412/* \brief Undo a nnp_gpu_register_env that the caller decided not to honour,
413 * leaving the device state untouched for the environment that owns it.
414 * \author Dhruv Sharma (ds2173@cam.ac.uk)
415 * ****************************************************************************
416 */
417void nnp_gpu_unregister_env(void) {
418 if (g_nnp_live_envs > 0)
419 g_nnp_live_envs--;
420}
421
422/* ****************************************************************************
423 */
424/* \brief Drop one NNP environment's claim on the device state: invalidate the
425 * host-pointer-keyed caches, and release the state outright once no
426 * environment is left, so a process that creates and destroys many of
427 * them (FARMING, TMC, libcp2k) does not retain the scratch buffers and
428 * the work stream to the end of the run.
429 * \author Dhruv Sharma (ds2173@cam.ac.uk)
430 * ****************************************************************************
431 */
432void nnp_gpu_release_env(void) {
433 if (g_nnp_live_envs > 0)
434 g_nnp_live_envs--;
435 if (g_nnp_live_envs == 0) {
436 nnp_gpu_state_release();
437 } else {
438 nnp_gpu_reset_caches();
439 }
440}
441
442/* ****************************************************************************
443 */
444/* \brief Select the GPU compute precision.
445 * \param use_fp32 nonzero selects single-precision angular symmetry functions
446 * \return 0
447 * \author Dhruv Sharma (ds2173@cam.ac.uk)
448 * ****************************************************************************
449 */
450int nnp_gpu_set_precision(int use_fp32) {
451 g_nnp_gpu_precision_fp32 = (use_fp32 != 0) ? 1 : 0;
452 return 0;
453}
454
455/* ****************************************************************************
456 */
457/* \brief Report whether single-precision angular symmetry functions are
458 * selected.
459 * \return 1 for single precision, 0 for double precision
460 * \author Dhruv Sharma (ds2173@cam.ac.uk)
461 * ****************************************************************************
462 */
463int nnp_gpu_precision_fp32(void) { return g_nnp_gpu_precision_fp32; }
464
465#endif /* __OFFLOAD && !__NO_OFFLOAD_NNP */
static void const int const int i