(git:27c966b)
Loading...
Searching...
No Matches
nnp_gpu_pack.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: GPL-2.0-or-later */
6/*----------------------------------------------------------------------------*/
7
8#ifndef NNP_GPU_PACK_H
9#define NNP_GPU_PACK_H
10
12#if defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP)
13
14#include "nnp_gpu_state.h"
15
16/*
17 * Kernels that rearrange the on-device cell-list walk output into the
18 * per-(group, element) buffers the descriptor and force-assembly kernels
19 * read. Both nnp_gpu_acsf.cu and nnp_gpu_scatter.cu launch them: the
20 * descriptor pass fills the buffers, and the force pass rebuilds them
21 * because the descriptor groups that follow overwrite the shared index
22 * scratch. A kernel cannot be launched from a translation unit that does
23 * not define it unless the build enables relocatable device code, so these
24 * are static and each unit compiles its own copy.
25 */
26
27/*******************************************************************************
28 * \brief Rearrange one (group, element) slice of the on-device walk output
29 * into the per-call pbuf_* layouts the radial symmetry-function and
30 * scatter kernels consume, removing both the device-to-host readback of
31 * the walk buffers and the host packing loop. The grid is sized so each
32 * thread handles one (atom, neighbour) pair; the j == 0 lane of each
33 * atom writes the per-atom scalar outputs.
34 *
35 * Index bases. The walk stores rad_ind / rad_image_idx 0-BASED (the
36 * same convention the host uses post-conversion, where grp%rad_ind =
37 * walk + 1 is 1-based). walk_atoms_global is 1-based. The radial
38 * symmetry-function kernel expects 0-based atom and image indices, so
39 * pbuf_rad_ind / pbuf_rad_image_idx are copied through unchanged. The
40 * scatter kernel expects 1-based atom indices, so
41 * scatter_neigh_ind = walk + 1. Padding slots (j >= n_neigh) get zeros
42 * (and 0 + 1 = 1 in the 1-based buffer), harmless because downstream
43 * kernels guard on the per-atom neighbour count.
44 * \author Dhruv Sharma (ds2173@cam.ac.uk)
45 ******************************************************************************/
46static __global__ void nnp_pack_walk_to_pbuf_radial_kernel(
47 const int n_atoms_in_e, const int max_n_neigh_per_se, const int s_0based,
48 const int e_0based, const int max_n_radgrp_walk,
49 const int n_atoms_in_e_max_walk, const int max_n_neigh_walk,
50 const int istart_1based, const int *__restrict__ walk_n_rad_pa,
51 const int *__restrict__ walk_rad_ind,
52 const int *__restrict__ walk_rad_image_idx,
53 const int *__restrict__ walk_atoms_global,
54 int *__restrict__ pbuf_n_neigh_pa,
55 int *__restrict__ pbuf_rad_ind_zero_based,
56 int *__restrict__ pbuf_rad_image_idx_zero_based,
57 int *__restrict__ pbuf_atoms_global_zero_based,
58 int *__restrict__ pbuf_atom_map,
59 int *__restrict__ scatter_neigh_ind_one_based) {
60 const int atom = blockIdx.x;
61 const int j = blockIdx.y * blockDim.x + threadIdx.x;
62 if (atom >= n_atoms_in_e)
63 return;
64
65 /* Walk-side strides (column-major, matching state field comments). */
66 const long long count_off =
67 (long long)atom +
68 (long long)n_atoms_in_e_max_walk *
69 ((long long)s_0based +
70 (long long)max_n_radgrp_walk * (long long)e_0based);
71 const long long ij_base =
72 (long long)max_n_neigh_walk *
73 ((long long)atom +
74 (long long)n_atoms_in_e_max_walk *
75 ((long long)s_0based +
76 (long long)max_n_radgrp_walk * (long long)e_0based));
77
78 const int n_neigh = walk_n_rad_pa[count_off];
79
80 /* Per-atom scalar outputs: one thread (j == 0) per atom row. */
81 if (j == 0) {
82 pbuf_n_neigh_pa[atom] = n_neigh;
83 const int atom_global_1b =
84 walk_atoms_global[atom + n_atoms_in_e_max_walk * e_0based];
85 pbuf_atoms_global_zero_based[atom] = atom_global_1b - 1;
86 pbuf_atom_map[atom] = atom_global_1b - istart_1based;
87 }
88
89 /* Per-(atom, j) outputs. Out-of-range slots get pad zeros so that
90 * downstream kernels (which guard on n_neigh) never read garbage. */
91 if (j < max_n_neigh_per_se) {
92 int j_global_0b = 0;
93 int img_0b = 0;
94 if (j < n_neigh) {
95 j_global_0b = walk_rad_ind[ij_base + j];
96 img_0b = walk_rad_image_idx[ij_base + j];
97 }
98 const long long out_off =
99 (long long)atom * (long long)max_n_neigh_per_se + (long long)j;
100 /* Walk is already 0-based, so copy through directly. */
101 pbuf_rad_ind_zero_based[out_off] = j_global_0b;
102 pbuf_rad_image_idx_zero_based[out_off] = img_0b;
103 /* Scatter wants 1-based (walk + 1). Padding (j >= n_neigh) stays 0 + 1 =
104 * 1, harmless because scatter guards on n_neigh_per_atom. */
105 scatter_neigh_ind_one_based[out_off] = j_global_0b + 1;
106 }
107}
108
109/*******************************************************************************
110 * \brief Materialise the per-(group, element) symmetry-function map into both
111 * 0-based (for the symmetry-function kernel) and 1-based (for the
112 * scatter kernel) layouts in one launch, reading the once-uploaded
113 * packed table (se_symf_packed) and its prefix-sum offsets
114 * (se_symf_offsets). The packed table stores raw 1-based symf values
115 * with no descriptor-row offset. Shared by the radial and angular
116 * launchers.
117 *
118 * m_offset is added to the 0-based output only. For radial groups it is
119 * 0 (the descriptor row is just symf - 1). For angular groups it is
120 * n_rad(element), because angular descriptors live after all radial
121 * descriptors in an atom's G_dev row and the angular symmetry-function
122 * kernel takes no separate offset parameter, so the offset must be baked
123 * into the map. The 1-based output is never offset: both scatter kernels
124 * read it as the raw symf value and apply n_rad themselves.
125 * \author Dhruv Sharma (ds2173@cam.ac.uk)
126 ******************************************************************************/
127static __global__ void
128nnp_pack_se_static_symf_map_kernel(const int se_idx_in_kind, const int n_symf,
129 const int m_offset,
130 const int *__restrict__ se_symf_packed,
131 const int *__restrict__ se_symf_offsets,
132 int *__restrict__ symf_map_zero_based,
133 int *__restrict__ symf_map_one_based) {
134 const int sf = blockIdx.x * blockDim.x + threadIdx.x;
135 if (sf >= n_symf)
136 return;
137 const int off = se_symf_offsets[se_idx_in_kind];
138 const int v_one = se_symf_packed[off + sf];
139 symf_map_zero_based[sf] = v_one - 1 + m_offset;
140 symf_map_one_based[sf] = v_one;
141}
142
143/*******************************************************************************
144 * \brief Rearrange the per-atom scalar outputs (neighbour counts, 0-based
145 * center index, atom-slot map) for one angular (group, element) slice
146 * of the walk output. One block per atom row; thread 0 writes
147 * everything for that atom. homo == 1 sets pbuf_n_ang2_pa =
148 * pbuf_n_ang1_pa. walk_atoms_global is 1-based; the 0-based center index
149 * and the atom_map (atom_global - istart) are derived from it.
150 * \author Dhruv Sharma (ds2173@cam.ac.uk)
151 ******************************************************************************/
152static __global__ void nnp_pack_walk_to_pbuf_angular_per_atom_kernel(
153 const int n_atoms_in_e, const int homo, const int s_0based,
154 const int e_0based, const int max_n_anggrp_walk,
155 const int n_atoms_in_e_max_walk, const int istart_1based,
156 const int *__restrict__ walk_n_ang1_pa,
157 const int *__restrict__ walk_n_ang2_pa,
158 const int *__restrict__ walk_atoms_global, int *__restrict__ pbuf_n_ang1_pa,
159 int *__restrict__ pbuf_n_ang2_pa,
160 int *__restrict__ pbuf_atoms_global_zero_based,
161 int *__restrict__ pbuf_atom_map) {
162 const int atom = blockIdx.x * blockDim.x + threadIdx.x;
163 if (atom >= n_atoms_in_e)
164 return;
165
166 const long long count_off =
167 (long long)atom +
168 (long long)n_atoms_in_e_max_walk *
169 ((long long)s_0based +
170 (long long)max_n_anggrp_walk * (long long)e_0based);
171
172 const int n1 = walk_n_ang1_pa[count_off];
173 const int n2 = homo ? n1 : walk_n_ang2_pa[count_off];
174 pbuf_n_ang1_pa[atom] = n1;
175 pbuf_n_ang2_pa[atom] = n2;
176
177 const int atom_global_1b =
178 walk_atoms_global[atom + n_atoms_in_e_max_walk * e_0based];
179 pbuf_atoms_global_zero_based[atom] = atom_global_1b - 1;
180 pbuf_atom_map[atom] = atom_global_1b - istart_1based;
181}
182
183/*******************************************************************************
184 * \brief Rearrange one (group, element) slice of a single angular neighbour
185 * list (ang1 or ang2, selected by the device pointers and walk strides
186 * the caller passes) from the walk output into the per-call pbuf layout
187 * [max_n_per_se x n_atoms_in_e]. ind_offset is added to the walk index:
188 * 0 for the symmetry-function kernel (0-based), 1 for the scatter kernel
189 * (1-based). image_idx is always 0-based (only the symmetry-function
190 * kernel reads it). Padding slots (j >= n_neigh) write ind = ind_offset
191 * and image = 0, inert because downstream kernels guard on the per-atom
192 * neighbour count.
193 * \author Dhruv Sharma (ds2173@cam.ac.uk)
194 ******************************************************************************/
195static __global__ void nnp_pack_walk_to_pbuf_angular_per_neigh_kernel(
196 const int n_atoms_in_e, const int max_n_per_se, const int s_0based,
197 const int e_0based, const int max_n_anggrp_walk,
198 const int n_atoms_in_e_max_walk, const int max_n_neigh_walk,
199 const int ind_offset, const int *__restrict__ walk_n_pa,
200 const int *__restrict__ walk_ind, const int *__restrict__ walk_image_idx,
201 int *__restrict__ pbuf_ind, int *__restrict__ pbuf_image_idx) {
202 const int atom = blockIdx.x;
203 const int j = blockIdx.y * blockDim.x + threadIdx.x;
204 if (atom >= n_atoms_in_e)
205 return;
206 if (j >= max_n_per_se)
207 return;
208
209 const long long count_off =
210 (long long)atom +
211 (long long)n_atoms_in_e_max_walk *
212 ((long long)s_0based +
213 (long long)max_n_anggrp_walk * (long long)e_0based);
214 const long long ij_base =
215 (long long)max_n_neigh_walk *
216 ((long long)atom +
217 (long long)n_atoms_in_e_max_walk *
218 ((long long)s_0based +
219 (long long)max_n_anggrp_walk * (long long)e_0based));
220
221 const int n_neigh = walk_n_pa[count_off];
222 int j_global_0b = 0;
223 int img_0b = 0;
224 if (j < n_neigh) {
225 j_global_0b = walk_ind[ij_base + j];
226 img_0b = walk_image_idx[ij_base + j];
227 }
228 const long long out_off =
229 (long long)atom * (long long)max_n_per_se + (long long)j;
230 pbuf_ind[out_off] = j_global_0b + ind_offset;
231 pbuf_image_idx[out_off] = img_0b;
232}
233
234#endif /* defined(__OFFLOAD) && !defined(__NO_OFFLOAD_NNP) */
235#endif /* NNP_GPU_PACK_H */
Definition atom.F:9