(git:27c966b)
Loading...
Searching...
No Matches
grid_gpu_separable.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7
8/*
9 * Helpers shared by the separable (plane streaming) integrate and collocate
10 * kernels (grid_gpu_integrate.cu and grid_gpu_collocate.cu).
11 *
12 * The constant memory is defined in every translation unit that includes this
13 * header (as coset_inv in grid_gpu_internal_header.h), so that
14 * init_separable_constant_memory has to be called in each of them.
15 */
16
17#ifndef GRID_GPU_SEPARABLE_H
18#define GRID_GPU_SEPARABLE_H
19
20#include <algorithm>
21#include <vector>
22
23#include "grid_gpu_context.h"
25
26namespace grid_gpu_backend {
27
28// lanes cooperating on one row of the cube, set separately for each kernel.
29// 64 / <lanes> rows are processed concurrently. Must be a power of two not
30// larger than the warp size. The values come from a sweep over lmax and the
31// cell type on V100: integrate prefers short rows (fewer shuffle steps in the
32// row reduction, more points per thread), collocate prefers 8 lanes (8
33// contiguous atomicAdd per row).
34#ifndef GRID_SEP_COLLOCATE_ROW_LANES
35#define GRID_SEP_COLLOCATE_ROW_LANES 8
36#endif
37#ifndef GRID_SEP_INTEGRATE_ROW_LANES
38#define GRID_SEP_INTEGRATE_ROW_LANES 4
39#endif
40static_assert((GRID_SEP_COLLOCATE_ROW_LANES &
41 (GRID_SEP_COLLOCATE_ROW_LANES - 1)) == 0 &&
43 "GRID_SEP_COLLOCATE_ROW_LANES must be a power of two <= 32");
44static_assert((GRID_SEP_INTEGRATE_ROW_LANES &
45 (GRID_SEP_INTEGRATE_ROW_LANES - 1)) == 0 &&
47 "GRID_SEP_INTEGRATE_ROW_LANES must be a power of two <= 32");
48// largest angular momentum handled by the separable kernels
49#define GRID_SEP_MAX_L 18
50
51// advances the periodic index x2 in [0, n) by STRIDE. The modulo is only
52// evaluated when wrapping around, which also covers n < STRIDE.
53template <int STRIDE>
54__device__ __inline__ int sep_advance_wrapped_index(const int x2, const int n) {
55 const int next = x2 + STRIDE;
56 return (next >= n) ? next % n : next;
57}
58
59// smallest instantiated upper bound of the angular momentum covering lp_max.
60// The separable integrate kernel keeps lmax_ + 1 accumulators per thread in
61// registers, so a tight bound saves registers and dead iterations.
62inline int separable_lmax_bucket(const int lp_max) {
63 if (lp_max <= 4)
64 return 4;
65 if (lp_max <= 8)
66 return 8;
67 if (lp_max <= 12)
68 return 12;
69 return GRID_SEP_MAX_L;
70}
71
72// shared memory budget of the separable kernels, static and dynamic (default
73// limit without opting in on NVIDIA hardware)
74constexpr size_t separable_max_smem = 48 * 1024;
75
76// (a, b) pairs ordered by a + b so that the first (lp+1)(lp+2)/2 entries are
77// exactly the pairs with a + b <= lp.
78__constant__ char2 sep_pair_ab[(GRID_SEP_MAX_L + 1) * (GRID_SEP_MAX_L + 2) / 2];
79
80// offset of the block of total angular momentum l in the lattice to cartesian
81// matrix
82__constant__ int sep_m_offset[GRID_SEP_MAX_L + 1];
83
84/*******************************************************************************
85 * \brief Initializes the constant memory of integrate_separable_kernel.
86 ******************************************************************************/
88 static bool initialized = false;
89 if (initialized) {
90 return;
91 }
92
93 char2 pairs[(GRID_SEP_MAX_L + 1) * (GRID_SEP_MAX_L + 2) / 2];
94 int ip = 0;
95 for (int s = 0; s <= GRID_SEP_MAX_L; s++)
96 for (int a = s; a >= 0; a--)
97 pairs[ip++] = make_char2(a, s - a);
98 offloadMemcpyToSymbol(sep_pair_ab, pairs, sizeof(pairs));
99
100 int m_offset[GRID_SEP_MAX_L + 1];
101 int offset = 0;
102 for (int l = 0; l <= GRID_SEP_MAX_L; l++) {
103 const int nl = (l + 1) * (l + 2) / 2;
104 m_offset[l] = offset;
105 offset += nl * nl;
106 }
107 offloadMemcpyToSymbol(sep_m_offset, m_offset, sizeof(m_offset));
108 initialized = true;
109}
110
111/*******************************************************************************
112 * \brief Builds the matrix expanding the cartesian monomials x^a y^b z^c in
113 * lattice monomials u^i v^j w^k, where x = dh[0] u + dh[3] v + dh[6] w,
114 * y = dh[1] u + dh[4] v + dh[7] w, z = dh[2] u + dh[5] v + dh[8] w.
115 * Only the blocks a + b + c = i + j + k = l are non zero. They are
116 * stored one after the other for l = 0..lmax, block l being row major
117 * of size nl x nl, nl = (l + 1)(l + 2) / 2, rows and columns in coset
118 * order.
119 *
120 * With H[d][e] = dh[3 e + d] and p = (a, b, c), the entries are given
121 * by the multinomial expansion of prod_d (sum_e H[d][e] u_e)^p_d,
122 *
123 * M[(abc)][(ijk)] = sum_n prod_d p_d! / (n_d0! n_d1! n_d2!)
124 * prod_e H[d][e]^n_de
125 *
126 * the sum running over the 3x3 non negative integer matrices n with
127 * row sums (a, b, c) and column sums (i, j, k). Block l is the
128 * symmetric power Sym^l(H) of the linear map in the monomial basis,
129 * so that M(H1 H2) = M(H1) M(H2) and M(H)^-1 = M(H^-1).
130 *
131 * The factorials are never evaluated. Each row is built by
132 * multiplying the polynomial 1 by one linear form at a time,
133 *
134 * x_d u^i v^j w^k = H[d][0] u^(i+1) v^j w^k + H[d][1] u^i v^(j+1) w^k
135 * + H[d][2] u^i v^j w^(k+1)
136 *
137 * a times with d = 0, b times with d = 1 and c times with d = 2, the
138 * multinomial coefficients being generated by the repeated additions
139 * as in Pascal's triangle. Equivalently, for a > 0 (similarly for b,
140 * c with rows 1, 2 of H),
141 *
142 * M[(abc)][(ijk)] = H[0][0] M[(a-1,b,c)][(i-1,j,k)]
143 * + H[0][1] M[(a-1,b,c)][(i,j-1,k)]
144 * + H[0][2] M[(a-1,b,c)][(i,j,k-1)]
145 *
146 * Integrate maps lattice moments to cartesian moments with M, collocate
147 * maps cartesian coefficients to lattice coefficients with M^T. The
148 * matrix is built on the host once per grid level and cached.
149 ******************************************************************************/
150static std::vector<double> build_lattice_to_cartesian_matrix(const double *dh,
151 const int lmax) {
152 std::vector<double> matrix;
153 const int n = ncoset(lmax);
154 std::vector<double> poly(n), next(n);
155
156 for (int l = 0; l <= lmax; l++) {
157 for (int a = l; a >= 0; a--) {
158 for (int b = l - a; b >= 0; b--) {
159 // coset order within a block: decreasing a, then decreasing b
160 const int pw[3] = {a, b, l - a - b};
161
162 // poly = x^a y^b z^c as a homogeneous polynomial in (u, v, w)
163 std::fill(poly.begin(), poly.end(), 0.0);
164 poly[0] = 1.0;
165 int deg = 0;
166 for (int d = 0; d < 3; d++) {
167 for (int p = 0; p < pw[d]; p++) {
168 std::fill(next.begin(), next.end(), 0.0);
169 for (int i = 0; i <= deg; i++) {
170 for (int j = 0; j <= deg - i; j++) {
171 const int k = deg - i - j;
172 const double val = poly[coset(i, j, k)];
173 next[coset(i + 1, j, k)] += dh[0 + d] * val;
174 next[coset(i, j + 1, k)] += dh[3 + d] * val;
175 next[coset(i, j, k + 1)] += dh[6 + d] * val;
176 }
177 }
178 poly.swap(next);
179 deg++;
180 }
181 }
182 const int base = (l > 0) ? ncoset(l - 1) : 0;
183 matrix.insert(matrix.end(), poly.begin() + base,
184 poly.begin() + ncoset(l));
185 }
186 }
187 }
188 return matrix;
189}
190
191/*******************************************************************************
192 * \brief Returns the lattice to cartesian matrix of a grid level on the
193 * device, (re)computing it when the lattice vectors changed or a higher
194 * angular momentum is needed.
195 ******************************************************************************/
197 const int lmax) {
198 if (grid.lattice_to_cartesian_lmax() < lmax) {
199 const std::vector<double> matrix =
201 grid.lattice_to_cartesian().resize(matrix.size());
202 grid.lattice_to_cartesian().copy_to_gpu(matrix);
203 grid.lattice_to_cartesian_lmax(lmax);
204 }
205 return grid.lattice_to_cartesian().data();
206}
207
208} // namespace grid_gpu_backend
209#endif
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i
#define GRID_SEP_MAX_L
#define GRID_SEP_INTEGRATE_ROW_LANES
#define GRID_SEP_COLLOCATE_ROW_LANES
__device__ __inline__ int sep_advance_wrapped_index(const int x2, const int n)
int separable_lmax_bucket(const int lp_max)
static std::vector< double > build_lattice_to_cartesian_matrix(const double *dh, const int lmax)
Builds the matrix expanding the cartesian monomials x^a y^b z^c in lattice monomials u^i v^j w^k,...
static void init_separable_constant_memory()
Initializes the constant memory of integrate_separable_kernel.
__host__ __device__ __inline__ int coset(int lx, int ly, int lz)
Maps three angular momentum components to a single zero based index.
static const double * get_lattice_to_cartesian(grid_info< double > &grid, const int lmax)
Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the latt...
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
__constant__ int sep_m_offset[GRID_SEP_MAX_L+1]
__constant__ char2 sep_pair_ab[(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2]
constexpr size_t separable_max_smem
__global__ const int const int lp_max