(git:27c966b)
Loading...
Searching...
No Matches
grid_gpu_backend Namespace Reference

Data Structures

class  gpu_vector
class  grid_info
struct  task_info
 Internal representation of a task. More...
struct  kernel_buffers
 Device buffers used by the collocate and integrate kernels. Which fields are non-null depends on which kernel is launched. More...
struct  kernel_params
 Parameters of the collocate kernel. More...
class  context_info
struct  orbital
 Orbital angular momentum. More...
struct  ldiffs_value
 Differences in angular momentum. More...
struct  smem_task
 data needed for calculating the coefficients, forces, and stress More...
struct  smem_task_reduced
 data needed for collocate and integrate kernels More...
class  smem_parameters

Functions

template<typename T, bool IS_FUNC_AB>
__device__ __inline__ void block_to_cab (const kernel_params &params, const smem_task< T > &task, T *cab)
 Decontracts the subblock, going from spherical to cartesian harmonics.
template<typename T, bool IS_FUNC_AB>
__global__ __launch_bounds__ (64) void calculate_coefficients_two_steps(const kernel_params dev_
 Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done in two steps,.
 if (number_of_tasks==0) return
 for (int tk=0;tk< number_of_tasks;tk++)
template<bool distributed__, bool axis_aligned_>
__global__ __launch_bounds__ (64) void collocate_separable_kernel(const kernel_params dev_
 if (glb_task.skip_task) return
 fill_smem_task_reduced (dev_, dev_.first_task+block_index(), task)
 if (tid==0)
 __syncthreads ()
 if (axis_aligned_)
static size_t collocate_separable_smem_size (const int mc, const int lp_max, const int erows)
static int collocate_separable_erows (const int mc, const int lp_max, const size_t smem_max)
constexpr size_t align_up_elems (size_t n_elems, size_t elem_alignment)
size_t round_up16 (size_t n)
__inline__ __device__ double sep_row_reduce (double val)
template<typename T>
__device__ __inline__ T block_reduce_64 (T *table, const T val_, const int tid)
template<typename T, bool COMPUTE_TAU>
__device__ __inline__ void accumulate_pair_forces (const smem_task< T > &task, const orbital &a, const orbital &b, const T p, const bool compute_virial, const T *__restrict__ cab, T *fa, T *fb, T *virial)
 Adds the contribution of the cartesian pair (a, b), weighted by the cartesian density p, to the forces and the virial.
template<typename T, typename T3, bool COMPUTE_TAU>
__global__ __launch_bounds__ (64) void compute_hab_forces(const kernel_params dev_)
 Fallback of compute_hab_forces_two_steps when its work arrays do not fit in the shared memory. hab is computed as in compute_hab. The forces and the virial only depend on the cartesian pair (a, b), so they are evaluated once per pair with the cartesian density.
 if (compute_virial)
template<typename T, typename T3, bool distributed__, bool orthogonal_>
__global__ __launch_bounds__ (64) void integrate_kernel(const kernel_params dev_)
template<bool distributed__, bool axis_aligned_, int lmax_>
__global__ __launch_bounds__ (64) void integrate_separable_kernel(const kernel_params dev_
 if (glb_task.la_max+dev_.la_max_diff+glb_task.lb_max+dev_.lb_max_diff< dev_.separable_lp_min) return
template<int lmax_>
static integrate_separable_kernel_t select_integrate_separable (const bool distributed, const bool axis_aligned)
 Selects the instance of integrate_separable_kernel for one value of lmax_.
static integrate_separable_kernel_t select_integrate_separable (const int lp_max, const bool distributed, const bool axis_aligned)
 Selects the instance of integrate_separable_kernel with the smallest bucket of the angular momentum covering lp_max.
__host__ __device__ __inline__ int ncoset (const int l)
 Number of Cartesian orbitals up to given angular momentum quantum.
__host__ __device__ __inline__ int coset (int lx, int ly, int lz)
 Maps three angular momentum components to a single zero based index.
__device__ __inline__ orbital up (const int i, const orbital &a)
 Increase i'th component of given orbital angular momentum.
__inline__ __device__ orbital down (const int i, const orbital &a)
 Decrease i'th component of given orbital angular momentum.
__inline__ __device__ int idx (const orbital a)
 Return coset index of given orbital angular momentum.
__device__ __inline__ double power (const double x, const int expo)
template<typename T = double>
__device__ __inline__ void prep_term (const orbital a, const orbital b, const T value, const int n, T *cab)
 Adds given value to matrix element cab[idx(b)][idx(a)].
static void init_constant_memory ()
 Initializes the device's constant memory.
__inline__ __device__ unsigned int block_index ()
__inline__ __device__ unsigned int thread_global_index ()
__inline__ __host__ __device__ double3 compute_coordinates (const double *__restrict__ dh_, const double x, const double y, const double z)
template<typename T, typename T3, bool orthorhombic_>
__inline__ T compute_cube_properties (const T radius, const T *const __restrict__ dh_, const T *const __restrict__ dh_inv_, const T3 &rp, T3 &roffset, int3 &cubecenter, int3 &lb_cube, int3 &cube_size)
__inline__ __device__ void compute_window_size (const int3 grid_size, const int border_mask, const int3 &border_width, int3 &window_size, int3 &window_shift)
__inline__ __device__ int wrap_grid_index (const int idx, const int full_size)
 Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry shared by collocate_kernel and integrate_kernel.
template<typename T, typename T3, bool distributed__>
__device__ __inline__ void setup_task_cube_center (const kernel_params &dev_, smem_task_reduced< T, T3 > &task)
 Shifts task.cube_center into the local grid's coordinate system and, for distributed grids, derives the border window. Single point of entry shared by collocate_kernel and integrate_kernel so the two cannot drift apart on which axis maps to which.
__device__ __inline__ void decode_pair (int t, int &p, int &q)
 Factorized cab <-> cxyz transformations.
__device__ __inline__ int line_index (const int dir, const int m, const int p, const int q, const bool on_a, const int other, const int n1)
template<typename T, bool TRANSPOSE>
__device__ __inline__ void move_center (const int la, const int lb, const int n1, const int dir, const bool on_a, const T dr, T *cab)
 Moves the expansion center of the a (on_a) or b cartesian index of cab along direction dir, in place:
template<typename T>
__device__ __inline__ void cab_to_cxyz_factorized (const smem_task< T > &task, T *__restrict__ cab, T *__restrict__ cxyz)
 Transforms the coefficients C_ab into C_xyz, cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a] (see cxyz_to_cab_factorized). cab is overwritten.
template<typename T>
__device__ __inline__ void cxyz_to_cab_factorized (const smem_task< T > &task, const T *__restrict__ cxyz, T *__restrict__ cab)
 Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.
template<typename T, typename T3>
__device__ __inline__ void fill_smem_task_reduced (const kernel_params &dev, const int task_id, smem_task_reduced< T, T3 > &task)
 Copies a task from global to shared memory.
template<typename T>
__device__ __inline__ void fill_smem_task_coef (const kernel_params &dev, const int task_id, smem_task< T > &task)
 Copies a task from global to shared memory and does precomputations.
int sphi_work_size (const int lmax, const int max_nsgf_set)
 Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calculate_coefficients_two_steps and compute_hab_two_steps: [max_nsgf_set][ld] and [ld][ld] with ld = ncoset(lmax).
template<typename Kernel>
size_t max_dynamic_smem (Kernel kernel, const size_t budget)
 Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memory together must fit in budget bytes. The static part (e.g. shared task structures) is taken from the compiled kernel and rounded up to 16 bytes for the alignment of the dynamic array placed after it.
template<typename T>
__inline__ __device__ T * allocate_workspace (const kernel_params &dev_)
template<typename T, typename T3>
__device__ __inline__ T calculate_ymix_ymax_boundaries (smem_task_reduced< T, T3 > &task, const int z, int &ymin, int &ymax)
template<typename T, typename T3>
__device__ __inline__ void calculate_xmin_xmax_boundaries (smem_task_reduced< T, T3 > &task, const int y, const T kremain, int &xmin, int &xmax)
template<typename T>
__device__ __inline__ void prepare_pab_AB (const orbital a, const orbital b, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_AB, ie. identity transformation.
template<typename T>
__device__ __inline__ void prepare_pab_DADB (const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_DADB.
template<typename T>
__device__ void prepare_pab_ADBmDAB (const int idir, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_ADBmDAB_{X,Y,Z}.
template<typename T>
__device__ void prepare_pab_ARDBmDARB (const int idir, const int ir, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_ARDBmDARB_{X,Y,Z}{X,Y,Z}.
template<typename T>
__device__ void prepare_pab_DABpADB (const int idir, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_DABpADB_{X,Y,Z}.
template<typename T>
__device__ __inline__ void prepare_pab_Di (const int ider, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_{DX,DY,DZ}.
template<typename T>
__device__ __inline__ void oneterm_dijdij (const int idir, const T func_a, const orbital a, const orbital b, const T zetb, const int n, T *cab)
 Helper for grid_prepare_pab_DiDj.
template<typename T>
__device__ __inline__ void prepare_pab_DiDj (const int ider1, const int ider2, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_{DXDY,DYDZ,DZDX}.
template<typename T>
__device__ void oneterm_diidii (const int idir, const T func_a, const orbital a, const orbital b, const T zetb, const int n, T *cab)
 Helper for grid_prepare_pab_Di2.
template<typename T>
__device__ __inline__ void prepare_pab_Di2 (const int ider, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Implementation of function GRID_FUNC_{DXDX,DYDY,DZDZ}.
template<typename T>
__device__ __inline__ void prepare_pab (const enum grid_func func, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab)
 Transforms a given element of the density matrix according to func.
ldiffs_value prepare_get_ldiffs (const enum grid_func func)
 Returns difference in angular momentum range for given func.
template<typename T>
__device__ __inline__ T get_term (const orbital &a, const orbital &b, const int n, const T *cab)
 Returns matrix element cab[idx(b)][idx(a)].
template<typename T>
__device__ __inline__ T get_force_a_normal (const orbital &a, const orbital &b, const int i, const T zeta, const int n, const T *cab)
 Returns i'th component of force on atom a for compute_tau=false.
template<bool compute_tau, typename T>
__device__ __inline__ double get_force_a (const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const int n, const T *cab)
 Returns i'th component of force on atom a.
template<typename T>
__device__ __inline__ double get_force_b_normal (const orbital &a, const orbital &b, const int i, const T zetb, const T rab[3], const int n, const T *cab)
 Returns i'th component of force on atom b for compute_tau=false.
template<bool compute_tau, typename T>
__device__ __inline__ T get_force_b (const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const T rab[3], const int n, const T *cab)
 Returns i'th component of force on atom b.
template<typename T>
__device__ __inline__ double get_virial_a_normal (const orbital &a, const orbital &b, const int i, const int j, const T zeta, const int n, const T *cab)
 Returns element i,j of virial on atom a for compute_tau=false.
template<bool compute_tau, typename T>
__device__ __inline__ T get_virial_a (const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const int n, const T *cab)
 Returns element i,j of virial on atom a.
template<typename T>
__device__ __inline__ double get_virial_b_normal (const orbital &a, const orbital &b, const int i, const int j, const T zetb, const T rab[3], const int n, const T *cab)
 Returns element i,j of virial on atom b for compute_tau=false.
template<bool compute_tau, typename T>
__device__ __inline__ double get_virial_b (const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const T rab[3], const int n, const T *cab)
 Returns element i,j of virial on atom b.
template<bool compute_tau, typename T>
__device__ __inline__ T get_hab (const orbital &a, const orbital &b, const T zeta, const T zetb, const int n, const T *cab)
 Returns element i,j of hab matrix.
ldiffs_value process_get_ldiffs (bool calculate_forces, bool calculate_virial, bool compute_tau)
 Returns difference in angular momentum range for given flags.
template<int STRIDE>
__device__ __inline__ int sep_advance_wrapped_index (const int x2, const int n)
int separable_lmax_bucket (const int lp_max)
static void init_separable_constant_memory ()
 Initializes the constant memory of integrate_separable_kernel.
static std::vector< double > build_lattice_to_cartesian_matrix (const double *dh, const int lmax)
 Builds the matrix expanding the cartesian monomials x^a y^b z^c in lattice monomials u^i v^j w^k, where x = dh[0] u + dh[3] v + dh[6] w, y = dh[1] u + dh[4] v + dh[7] w, z = dh[2] u + dh[5] v + dh[8] w. Only the blocks a + b + c = i + j + k = l are non zero. They are stored one after the other for l = 0..lmax, block l being row major of size nl x nl, nl = (l + 1)(l + 2) / 2, rows and columns in coset order.
static const double * get_lattice_to_cartesian (grid_info< double > &grid, const int lmax)
 Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the lattice vectors changed or a higher angular momentum is needed.

Variables

__global__ const int ld
__global__ const int const int max_nsgf_set
const int number_of_tasks = dev_.num_tasks_per_block_dev[block_index()]
T * smem_t = &shared_memory[0]
T * smem_p = smem_t + max_nsgf_set * ld
const int tid = thread_global_index()
const int nthreads = blockDim.x * blockDim.y * blockDim.z
const int offset = dev_.sorted_blocks_offset_dev[block_index()]
T * smem_cab
__global__ const int mc
__global__ const int const int lp_max
__global__ const int const int const int erows
__global__ const int const int const int const double *__restrict__ M
const task_info & glb_task = dev_.tasks[dev_.first_task + block_index()]
const int ls = lp_max + 1
__shared__ double sep_smem []
double * ex = sep_smem
double * ey = ex + mc
double * D = ey + mc
double * C = D + ls * ls
double * E = C + ncoset(lp_max)
__shared__ smem_task_reduced< double, double3 > task
const int lp = task.lp
const int ncoef = ncoset(lp)
const double zetp = task.zetp
const double r2max = task.radius * task.radius
const bool trim = orthogonal_ && !task.apply_border_mask
const double *__restrict__ coef
const int npairs = (lp + 1) * (lp + 2) / 2
const int row_group = tid / GRID_SEP_COLLOCATE_ROW_LANES
const int lane = tid % GRID_SEP_COLLOCATE_ROW_LANES
constexpr int nrow_groups = 64 / GRID_SEP_COLLOCATE_ROW_LANES
constexpr int integrate_separable_lp_threshold = 0
constexpr size_t block_reduce_64_smem = 64 * sizeof(double)
T * smem_h = &shared_memory[0]
T * smem_u = smem_h + ld * ld
T * smem_w = smem_h + ld * ld
const bool compute_virial = (dev_.buffers_dev.virial != nullptr)
T fa [3] = {0.0, 0.0, 0.0}
T fb [3] = {0.0, 0.0, 0.0}
T virial [9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}
const int task_id = dev_.task_sorted_by_blocks_dev[offset]
T * forces_a = &dev_.buffers_dev.forces[3 * glb_task.iatom]
T * forces_b = &dev_.buffers_dev.forces[3 * glb_task.jatom]
T * sum = (T *)shared_memory
__global__ const int const int const double *__restrict__ lattice_to_cartesian
double * py = ex + mc
double * pz = py + ls * mc
double * A = pz + ls
double * acc = A + mc * ls
 else
constexpr int grid_gpu_max_l = 18
__constant__ orbital coset_inv [1330]
constexpr size_t sphi_two_step_max_smem = 48 * 1024
constexpr size_t separable_max_smem = 48 * 1024
__constant__ char2 sep_pair_ab [(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2]
__constant__ int sep_m_offset [GRID_SEP_MAX_L+1]

Function Documentation

◆ block_to_cab()

template<typename T, bool IS_FUNC_AB>
__device__ __inline__ void grid_gpu_backend::block_to_cab ( const kernel_params & params,
const smem_task< T > & task,
T * cab )

Decontracts the subblock, going from spherical to cartesian harmonics.

Definition at line 36 of file grid_gpu_collocate.cu.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ __launch_bounds__() [1/5]

template<typename T, bool IS_FUNC_AB>
__global__ grid_gpu_backend::__launch_bounds__ ( 64 ) const

Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done in two steps,.

Same as compute_hab_forces with the spherical to cartesian transformations done as matrix products. The forces and the virial only depend on the cartesian pair (a, b), so they are contracted with the cartesian density.

Same as compute_hab, with the cartesian to spherical transformation done in two steps,.

T[i][ico] = sum_j pab[i][j] sphia[j][ico] (nsgfb nsgfa ncoa) P[jco][ico] = sum_i sphib[i][jco] T[i][ico] (ncob nsgfb ncoa)

 instead of the fused loop of cost ncoa ncob nsgfb (nsgfa + 1). The
 work arrays T [max_nsgf_set][ld] and P [ld][ld] live in the dynamic
 shared memory, see sphi_work_size.

H[jco][ico] = get_hab(a, b, cab) (ncob ncoa) U[i][ico] = sum_jco sphib[i][jco] H[jco][ico] (nsgfb ncob ncoa) hab[i][j] += sum_ico U[i][ico] sphia[j][ico] (nsgfb nsgfa ncoa)

 instead of evaluating get_hab and both sphi factors in the innermost
 loop of a nsgfb nsgfa ncob ncoa loop. The work arrays H [ld][ld] and
 U [max_nsgf_set][ld] live in the dynamic shared memory, see
 sphi_work_size.

P[jco][ico] = sum_ij sphib[i][jco] pab[i][j] sphia[j][ico]

 and evaluated once per pair instead of once per (i, j, a, b):

W[i][ico] = sum_j pab[i][j] sphia[j][ico] (nsgfb nsgfa ncoa) for each (jco, ico): p = sum_i sphib[i][jco] W[i][ico] (nsgfb ncob ncoa) fa, fb, virial += p * get_force / get_virial (a, b) H[jco][ico] = get_hab(a, b) U[i][ico] = sum_jco sphib[i][jco] H[jco][ico] (nsgfb ncob ncoa) hab[i][j] += sum_ico U[i][ico] sphia[j][ico] (nsgfb nsgfa ncoa)

 P is never stored. The work arrays H [ld][ld] and W / U
 [max_nsgf_set][ld] (U overwrites W) live in the dynamic shared
 memory, see sphi_work_size.

Definition at line 84 of file grid_gpu_collocate.cu.

Here is the call graph for this function:

◆ if() [1/6]

grid_gpu_backend::if ( number_of_tasks = =0)
Here is the caller graph for this function:

◆ for()

grid_gpu_backend::for ( )

Definition at line 145 of file grid_gpu_collocate.cu.

Here is the call graph for this function:

◆ __launch_bounds__() [2/5]

template<bool distributed__, bool axis_aligned_>
__global__ grid_gpu_backend::__launch_bounds__ ( 64 ) const

Separable (plane streaming) collocate kernel. This is integrate_separable_kernel (grid_gpu_integrate.cu) run backwards.

Axis aligned orthorhombic cell (axis_aligned_ = true):

sum_{abc} C[abc] x^a y^b z^c exp(-zetp r^2) = sum_a [x^a e^{-zetp x^2}] sum_b [y^b e^{-zetp y^2}] sum_c [z^c e^{-zetp z^2}] C[abc]

Other cells (axis_aligned_ = false): the cartesian coefficients are first expressed in lattice monomials u^i v^j w^k (u, v, w lattice coordinates of the point relative to the gaussian center),

C_ijk[(i,j,k)] = sum_{a+b+c=i+j+k} M[(a,b,c)][(i,j,k)] C[(a,b,c)]

with the matrix M of build_lattice_to_cartesian_matrix (the transpose of the integrate transformation). Only the polynomial is separated, the gaussian exp(-zetp r^2) is evaluated per grid point from the r^2 of the sphere test, as in integrate_separable_kernel. Factoring it as

exp(-zetp g00 u^2) exp(-zetp g11 v^2) exp(-zetp g22 w^2) exp(-2 zetp g12 v w) exp(-2 zetp u (g01 v + g02 w))

with the metric g_ij = a_i . a_j is not numerically stable: on strongly sheared cells the diagonal factors underflow while the cross terms overflow, even though their product is finite.

For each z plane of the cube:

D[a][b] = sum_c C[coset(a, b, c)] pz[c] a + b <= lp (step 1) E[y][a] = sum_b D[a][b] py[b][y] (step 2) grid[z][y][x] += sum_a E[y][a] px[a][x] (step 3)

so that a grid point costs lp + 1 FMAs (Horner in x) instead of ncoset(lp). Steps 1 and 2 are O(lp^3) and O(ny lp^2) per plane. The set of grid points visited is the same as for the original collocate kernel: same window masks, same y / x trimming in the orthogonal case, same sphere test.

Dynamic shared memory layout (doubles), mc = max cube size, ls = lp_max + 1: ex[mc], ey[mc], D[ls][ls], C[ncoset(lp_max)], E[erows][ls]

The powers of y are computed on the fly in step 2 and the rows of a plane are processed by chunks of erows rows, so that the shared memory is bounded for any cube size (see collocate_separable_erows), the x coordinates are computed on the fly. ex and ey are only used when axis_aligned_ = true.

◆ if() [2/6]

grid_gpu_backend::if ( glb_task. skip_task)

◆ fill_smem_task_reduced() [1/2]

grid_gpu_backend::fill_smem_task_reduced ( dev_ ,
dev_.first_task+ block_index(),
task  )
Here is the call graph for this function:
Here is the caller graph for this function:

◆ if() [3/6]

grid_gpu_backend::if ( tid = = 0)

Definition at line 280 of file grid_gpu_collocate.cu.

Here is the call graph for this function:

◆ __syncthreads()

grid_gpu_backend::__syncthreads ( )
Here is the caller graph for this function:

◆ if() [4/6]

grid_gpu_backend::if ( axis_aligned_ )

Definition at line 313 of file grid_gpu_collocate.cu.

◆ collocate_separable_smem_size()

size_t grid_gpu_backend::collocate_separable_smem_size ( const int mc,
const int lp_max,
const int erows )
static

Definition at line 451 of file grid_gpu_collocate.cu.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ collocate_separable_erows()

int grid_gpu_backend::collocate_separable_erows ( const int mc,
const int lp_max,
const size_t smem_max )
static

Definition at line 458 of file grid_gpu_collocate.cu.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ align_up_elems()

size_t grid_gpu_backend::align_up_elems ( size_t n_elems,
size_t elem_alignment )
constexpr

Definition at line 40 of file grid_gpu_context.cu.

Here is the caller graph for this function:

◆ round_up16()

size_t grid_gpu_backend::round_up16 ( size_t n)
inline

Definition at line 39 of file grid_gpu_context.h.

Here is the caller graph for this function:

◆ sep_row_reduce()

__inline__ __device__ double grid_gpu_backend::sep_row_reduce ( double val)

Definition at line 41 of file grid_gpu_integrate.cu.

◆ block_reduce_64()

template<typename T>
__device__ __inline__ T grid_gpu_backend::block_reduce_64 ( T * table,
const T val_,
const int tid )

Definition at line 60 of file grid_gpu_integrate.cu.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ accumulate_pair_forces()

template<typename T, bool COMPUTE_TAU>
__device__ __inline__ void grid_gpu_backend::accumulate_pair_forces ( const smem_task< T > & task,
const orbital & a,
const orbital & b,
const T p,
const bool compute_virial,
const T *__restrict__ cab,
T * fa,
T * fb,
T * virial )

Adds the contribution of the cartesian pair (a, b), weighted by the cartesian density p, to the forces and the virial.

Definition at line 236 of file grid_gpu_integrate.cu.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ __launch_bounds__() [3/5]

template<typename T, typename T3, bool COMPUTE_TAU>
__global__ grid_gpu_backend::__launch_bounds__ ( 64 ) const

Fallback of compute_hab_forces_two_steps when its work arrays do not fit in the shared memory. hab is computed as in compute_hab. The forces and the virial only depend on the cartesian pair (a, b), so they are evaluated once per pair with the cartesian density.

p[jco][ico] = sum_i sphib[i][jco] sum_j pab[i][j] sphia[j][ico]

computed on the fly (nsgfb (nsgfa + 1) FMA per pair), instead of once per (i, j, a, b) in the hab loop.

Definition at line 270 of file grid_gpu_integrate.cu.

Here is the call graph for this function:

◆ if() [5/6]

grid_gpu_backend::if ( compute_virial )

Definition at line 515 of file grid_gpu_integrate.cu.

Here is the call graph for this function:

◆ __launch_bounds__() [4/5]

template<typename T, typename T3, bool distributed__, bool orthogonal_>
__global__ grid_gpu_backend::__launch_bounds__ ( 64 ) const

Cuda kernel for calculating the coefficients of a potential (density, etc...) for a given pair of gaussian basis sets

We compute the discretized version of the following integral

$$\int _\infty ^\infty V_{ijk} P^\alpha_iP^beta_j P ^ \gamma _ k Exp(- \eta |r_{ijk} - r_c|^2)$$

where in practice the summation is over a finite domain. The discrete form has this shape

$$ \sum_{ijk < domain} V_{ijk} (x_i - x_c)^\alpha (y_j - y_c)^\beta (z_k - z_c) ^ \gamma Exp(- \eta |r_{ijk} - r_c|^2) $$

where $0 \le \alpha + \beta + \gamma \le lmax$

It is formely the same operation than collocate (from a discrete point of view) but the implementation differ because of technical reasons.

So most of the code is the same except the core of the routine that is specialized to the integration.

Definition at line 561 of file grid_gpu_integrate.cu.

Here is the call graph for this function:

◆ __launch_bounds__() [5/5]

template<bool distributed__, bool axis_aligned_, int lmax_>
__global__ grid_gpu_backend::__launch_bounds__ ( 64 ) const

Separable (plane streaming) integrate kernel.

Computes the same coefficients as integrate_kernel, but with a cost per grid point linear in lp instead of proportional to ncoset(lp), and reading the cube only once whatever lp is.

It uses the same underlying method than the DGEMM backend which is a tensor contraction. Speedup comes from different areas:

Axis aligned cell (axis_aligned_ = true): r3.x, r3.y, r3.z only depend on x, y and z respectively, so that

x^a y^b z^c exp(-zetp r^2) = [x^a e^{-zetp x^2}] [y^b e^{-zetp y^2}] [z^c e^{-zetp z^2}]

Non orthogonal cell (axis_aligned_ = false): the moments are accumulated in lattice monomials u^i v^j w^k (u, v, w are the lattice coordinates of the point relative to the gaussian center) with the full gaussian evaluated per point, and transformed to cartesian monomials at the end with the matrix built by build_lattice_to_cartesian_matrix. It is the same transformation as grid_transform_coef_jik_to_yxz in the dgemm backend, precomputed once per grid level.

For each z plane of the cube:

A[y][a] = sum_x V[z][y][x] px[a][x] (step A) B[a][b] = sum_y A[y][a] py[b][y] a + b <= lp (step B) acc[coset(a, b, c)] += B[a][b] pz[c] c <= lp - a - b (step C)

B only lives in registers and the reduction over z is a running sum in acc, so the shared memory does not depend on the z extent of the cube. The set of grid points visited is the same as for integrate_kernel (window masks, y / x trimming in the orthogonal case, sphere test).

Shared memory layout: mc = max cube size, ls = lp_max + 1: ex[mc], py[ls][mc], pz[ls], A[mc][ls], acc[ncoset(lp_max)] pz only holds the values of the current plane, ex is only used for an axis aligned cell.

◆ if() [6/6]

grid_gpu_backend::if ( )
Here is the call graph for this function:

◆ select_integrate_separable() [1/2]

template<int lmax_>
integrate_separable_kernel_t grid_gpu_backend::select_integrate_separable ( const bool distributed,
const bool axis_aligned )
static

Selects the instance of integrate_separable_kernel for one value of lmax_.

Definition at line 1196 of file grid_gpu_integrate.cu.

Here is the caller graph for this function:

◆ select_integrate_separable() [2/2]

integrate_separable_kernel_t grid_gpu_backend::select_integrate_separable ( const int lp_max,
const bool distributed,
const bool axis_aligned )
static

Selects the instance of integrate_separable_kernel with the smallest bucket of the angular momentum covering lp_max.

Definition at line 1209 of file grid_gpu_integrate.cu.

Here is the call graph for this function:

◆ ncoset()

__host__ __device__ __inline__ int grid_gpu_backend::ncoset ( const int l)

Number of Cartesian orbitals up to given angular momentum quantum.

Author
Ole Schuett

Definition at line 149 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ coset()

__host__ __device__ __inline__ int grid_gpu_backend::coset ( int lx,
int ly,
int lz )

Maps three angular momentum components to a single zero based index.

Definition at line 161 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ up()

__device__ __inline__ orbital grid_gpu_backend::up ( const int i,
const orbital & a )

Increase i'th component of given orbital angular momentum.

Definition at line 173 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ down()

__inline__ __device__ orbital grid_gpu_backend::down ( const int i,
const orbital & a )

Decrease i'th component of given orbital angular momentum.

Definition at line 182 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ idx()

__inline__ __device__ int grid_gpu_backend::idx ( const orbital a)

Return coset index of given orbital angular momentum.

Definition at line 191 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ power()

__device__ __inline__ double grid_gpu_backend::power ( const double x,
const int expo )

Definition at line 195 of file grid_gpu_internal_header.h.

◆ prep_term()

template<typename T = double>
__device__ __inline__ void grid_gpu_backend::prep_term ( const orbital a,
const orbital b,
const T value,
const int n,
T * cab )

Adds given value to matrix element cab[idx(b)][idx(a)].

Definition at line 206 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ init_constant_memory()

void grid_gpu_backend::init_constant_memory ( )
inlinestatic

Initializes the device's constant memory.

Author
Ole Schuett

Definition at line 215 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ block_index()

__inline__ __device__ unsigned int grid_gpu_backend::block_index ( )

Definition at line 240 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ thread_global_index()

__inline__ __device__ unsigned int grid_gpu_backend::thread_global_index ( )

Definition at line 245 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ compute_coordinates()

__inline__ __host__ __device__ double3 grid_gpu_backend::compute_coordinates ( const double *__restrict__ dh_,
const double x,
const double y,
const double z )

Definition at line 250 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ compute_cube_properties()

template<typename T, typename T3, bool orthorhombic_>
__inline__ T grid_gpu_backend::compute_cube_properties ( const T radius,
const T *const __restrict__ dh_,
const T *const __restrict__ dh_inv_,
const T3 & rp,
T3 & roffset,
int3 & cubecenter,
int3 & lb_cube,
int3 & cube_size )

Definition at line 265 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ compute_window_size()

__inline__ __device__ void grid_gpu_backend::compute_window_size ( const int3 grid_size,
const int border_mask,
const int3 & border_width,
int3 & window_size,
int3 & window_shift )

Definition at line 404 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ wrap_grid_index()

__inline__ __device__ int grid_gpu_backend::wrap_grid_index ( const int idx,
const int full_size )

Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry shared by collocate_kernel and integrate_kernel.

Definition at line 436 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ setup_task_cube_center()

template<typename T, typename T3, bool distributed__>
__device__ __inline__ void grid_gpu_backend::setup_task_cube_center ( const kernel_params & dev_,
smem_task_reduced< T, T3 > & task )

Shifts task.cube_center into the local grid's coordinate system and, for distributed grids, derives the border window. Single point of entry shared by collocate_kernel and integrate_kernel so the two cannot drift apart on which axis maps to which.

Definition at line 451 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ decode_pair()

__device__ __inline__ void grid_gpu_backend::decode_pair ( int t,
int & p,
int & q )

Factorized cab <-> cxyz transformations.

  cab_to_cxyz expands (x - a)^la (x - b)^lb around p in each direction.
  It is the product of three steps: move the expansion center of the a
  index from a to p, of the b index from b to p, then multiply the
  polynomials,

    (x - a)^n = sum_k C(n, k) (p - a)^k (x - p)^(n - k)
    cxyz[c]   = prefactor sum_{a' + b' = c} cab_p[b'][a']

  The center moves are triangular, separable in x, y and z and map cab
  onto itself, so they are done in place, one 1D line of cab per
  thread. cxyz_to_cab is the transpose: a gather followed by the
  transposed moves. The expansion coefficients alpha of the product are
  never formed.

Definition at line 486 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ line_index()

__device__ __inline__ int grid_gpu_backend::line_index ( const int dir,
const int m,
const int p,
const int q,
const bool on_a,
const int other,
const int n1 )

Definition at line 496 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ move_center()

template<typename T, bool TRANSPOSE>
__device__ __inline__ void grid_gpu_backend::move_center ( const int la,
const int lb,
const int n1,
const int dir,
const bool on_a,
const T dr,
T * cab )

Moves the expansion center of the a (on_a) or b cartesian index of cab along direction dir, in place:

forward: new[m] = sum_{k >= 0} C(m + k, k) dr^k old[m + k] transpose: new[m] = sum_{k <= m} C(m, k) dr^k old[m - k]

Each thread owns a whole line (all other indices fixed). The forward update is a Taylor shift of the polynomial sum_m old[m] x^m by dr, done in place by repeated synthetic division (Ruffini-Horner): for i = 0 .. len-1, for j = len-1 down to i: c[j] += dr c[j+1] The transpose applies the transposed elementary updates in reverse order: for i = len-1 down to 0, for j = i .. len-1: c[j+1] += dr c[j] No binomial coefficient is needed, len (len + 1) / 2 FMA per line.

Definition at line 524 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ cab_to_cxyz_factorized()

template<typename T>
__device__ __inline__ void grid_gpu_backend::cab_to_cxyz_factorized ( const smem_task< T > & task,
T *__restrict__ cab,
T *__restrict__ cxyz )

Transforms the coefficients C_ab into C_xyz, cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a] (see cxyz_to_cab_factorized). cab is overwritten.

Definition at line 570 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ cxyz_to_cab_factorized()

template<typename T>
__device__ __inline__ void grid_gpu_backend::cxyz_to_cab_factorized ( const smem_task< T > & task,
const T *__restrict__ cxyz,
T *__restrict__ cab )

Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.

cab[b][a] = prefactor sum_c alpha[a][b][c] cxyz[c]

where alpha expands the product of the two cartesian monomials around p, (x - A)^a (x - B)^b = sum_c alpha[a][b][c] (x - P)^c. In each direction, with dA = P - A and dB = P - B,

(x - A)^a = sum_{a' <= a} TA[a][a'] (x - P)^a', TA[a][a'] = C(a, a') dA^(a - a')

and similarly TB for b. Multiplying the two expansions,

alpha[a][b][c] = sum_{a' + b' = c} TA[a][a'] TB[b][b'],

hence

cab[b][a] = sum_{a', b'} TB[b][b'] TA[a][a'] G[b'][a'], G[b'][a'] = prefactor cxyz[a' + b'].

The sum is done in three steps without forming alpha:

  1. gather G from cxyz (a' + b' summed per direction);
  2. apply TA to the a index, one direction at a time, since the 3D TA is the product of the x, y and z 1D matrices: new[m] = sum_{k <= m} C(m, k) dA^k old[m - k] (move_center<T, true>, in place, one 1D line per thread);
  3. apply TB to the b index in the same way.

TA and TB are lower triangular, so both steps map cab onto itself. cab_to_cxyz_factorized is the transpose of the same product: TA^T and TB^T (move_center<T, false>), then the scatter cxyz[c] = prefactor sum_{a' + b' = c} cab[b'][a'].

Definition at line 642 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fill_smem_task_reduced() [2/2]

template<typename T, typename T3>
__device__ __inline__ void grid_gpu_backend::fill_smem_task_reduced ( const kernel_params & dev,
const int task_id,
smem_task_reduced< T, T3 > & task )

Copies a task from global to shared memory.

Definition at line 682 of file grid_gpu_internal_header.h.

Here is the call graph for this function:

◆ fill_smem_task_coef()

template<typename T>
__device__ __inline__ void grid_gpu_backend::fill_smem_task_coef ( const kernel_params & dev,
const int task_id,
smem_task< T > & task )

Copies a task from global to shared memory and does precomputations.

Definition at line 736 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ sphi_work_size()

int grid_gpu_backend::sphi_work_size ( const int lmax,
const int max_nsgf_set )
inline

Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calculate_coefficients_two_steps and compute_hab_two_steps: [max_nsgf_set][ld] and [ld][ld] with ld = ncoset(lmax).

Definition at line 824 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ max_dynamic_smem()

template<typename Kernel>
size_t grid_gpu_backend::max_dynamic_smem ( Kernel kernel,
const size_t budget )
inline

Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memory together must fit in budget bytes. The static part (e.g. shared task structures) is taken from the compiled kernel and rounded up to 16 bytes for the alignment of the dynamic array placed after it.

Definition at line 842 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ allocate_workspace()

template<typename T>
__inline__ __device__ T * grid_gpu_backend::allocate_workspace ( const kernel_params & dev_)

Definition at line 910 of file grid_gpu_internal_header.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ calculate_ymix_ymax_boundaries()

template<typename T, typename T3>
__device__ __inline__ T grid_gpu_backend::calculate_ymix_ymax_boundaries ( smem_task_reduced< T, T3 > & task,
const int z,
int & ymin,
int & ymax )

Definition at line 917 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ calculate_xmin_xmax_boundaries()

template<typename T, typename T3>
__device__ __inline__ void grid_gpu_backend::calculate_xmin_xmax_boundaries ( smem_task_reduced< T, T3 > & task,
const int y,
const T kremain,
int & xmin,
int & xmax )

Definition at line 933 of file grid_gpu_internal_header.h.

Here is the caller graph for this function:

◆ prepare_pab_AB()

template<typename T>
__device__ __inline__ void grid_gpu_backend::prepare_pab_AB ( const orbital a,
const orbital b,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_AB, ie. identity transformation.

Definition at line 28 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_DADB()

template<typename T>
__device__ __inline__ void grid_gpu_backend::prepare_pab_DADB ( const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_DADB.

Definition at line 41 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_ADBmDAB()

template<typename T>
__device__ void grid_gpu_backend::prepare_pab_ADBmDAB ( const int idir,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_ADBmDAB_{X,Y,Z}.

Definition at line 62 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_ARDBmDARB()

template<typename T>
__device__ void grid_gpu_backend::prepare_pab_ARDBmDARB ( const int idir,
const int ir,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_ARDBmDARB_{X,Y,Z}{X,Y,Z}.

Definition at line 83 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_DABpADB()

template<typename T>
__device__ void grid_gpu_backend::prepare_pab_DABpADB ( const int idir,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_DABpADB_{X,Y,Z}.

Author
Ole Schuett

Definition at line 108 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_Di()

template<typename T>
__device__ __inline__ void grid_gpu_backend::prepare_pab_Di ( const int ider,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_{DX,DY,DZ}.

Author
Ole Schuett

Definition at line 131 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ oneterm_dijdij()

template<typename T>
__device__ __inline__ void grid_gpu_backend::oneterm_dijdij ( const int idir,
const T func_a,
const orbital a,
const orbital b,
const T zetb,
const int n,
T * cab )

Helper for grid_prepare_pab_DiDj.

Definition at line 155 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_DiDj()

template<typename T>
__device__ __inline__ void grid_gpu_backend::prepare_pab_DiDj ( const int ider1,
const int ider2,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_{DXDY,DYDZ,DZDX}.

Author
Ole Schuett

Definition at line 185 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ oneterm_diidii()

template<typename T>
__device__ void grid_gpu_backend::oneterm_diidii ( const int idir,
const T func_a,
const orbital a,
const orbital b,
const T zetb,
const int n,
T * cab )

Helper for grid_prepare_pab_Di2.

Author
Ole Schuett

Definition at line 212 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab_Di2()

template<typename T>
__device__ __inline__ void grid_gpu_backend::prepare_pab_Di2 ( const int ider,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Implementation of function GRID_FUNC_{DXDX,DYDY,DZDZ}.

Definition at line 227 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_pab()

template<typename T>
__device__ __inline__ void grid_gpu_backend::prepare_pab ( const enum grid_func func,
const orbital a,
const orbital b,
const T zeta,
const T zetb,
const T pab_val,
const int n,
T * cab )

Transforms a given element of the density matrix according to func.

Parameters
funcTransformation function to apply, one of GRID_FUNC_*.
{a,b}Orbital angular momenta.
zet_{a,b}Gaussian exponents.
pab_valInput matrix element of pab.
nLeading dimensions of output matrix cab.
cabOutput matrix.

Definition at line 255 of file grid_gpu_prepare_pab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ prepare_get_ldiffs()

ldiffs_value grid_gpu_backend::prepare_get_ldiffs ( const enum grid_func func)
inline

Returns difference in angular momentum range for given func.

Author
Ole Schuett

Definition at line 347 of file grid_gpu_prepare_pab.h.

Here is the caller graph for this function:

◆ get_term()

template<typename T>
__device__ __inline__ T grid_gpu_backend::get_term ( const orbital & a,
const orbital & b,
const int n,
const T * cab )

Returns matrix element cab[idx(b)][idx(a)].

Author
Ole Schuett

Definition at line 27 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_force_a_normal()

template<typename T>
__device__ __inline__ T grid_gpu_backend::get_force_a_normal ( const orbital & a,
const orbital & b,
const int i,
const T zeta,
const int n,
const T * cab )

Returns i'th component of force on atom a for compute_tau=false.

Author
Ole Schuett

Definition at line 37 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_force_a()

template<bool compute_tau, typename T>
__device__ __inline__ double grid_gpu_backend::get_force_a ( const orbital & a,
const orbital & b,
const int i,
const T zeta,
const T zetb,
const int n,
const T * cab )

Returns i'th component of force on atom a.

Author
Ole Schuett

Definition at line 51 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_force_b_normal()

template<typename T>
__device__ __inline__ double grid_gpu_backend::get_force_b_normal ( const orbital & a,
const orbital & b,
const int i,
const T zetb,
const T rab[3],
const int n,
const T * cab )

Returns i'th component of force on atom b for compute_tau=false.

Author
Ole Schuett

Definition at line 81 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_force_b()

template<bool compute_tau, typename T>
__device__ __inline__ T grid_gpu_backend::get_force_b ( const orbital & a,
const orbital & b,
const int i,
const T zeta,
const T zetb,
const T rab[3],
const int n,
const T * cab )

Returns i'th component of force on atom b.

Author
Ole Schuett

Definition at line 94 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_virial_a_normal()

template<typename T>
__device__ __inline__ double grid_gpu_backend::get_virial_a_normal ( const orbital & a,
const orbital & b,
const int i,
const int j,
const T zeta,
const int n,
const T * cab )

Returns element i,j of virial on atom a for compute_tau=false.

Author
Ole Schuett

Definition at line 125 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_virial_a()

template<bool compute_tau, typename T>
__device__ __inline__ T grid_gpu_backend::get_virial_a ( const orbital & a,
const orbital & b,
const int i,
const int j,
const T zeta,
const T zetb,
const int n,
const T * cab )

Returns element i,j of virial on atom a.

Author
Ole Schuett

Definition at line 136 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_virial_b_normal()

template<typename T>
__device__ __inline__ double grid_gpu_backend::get_virial_b_normal ( const orbital & a,
const orbital & b,
const int i,
const int j,
const T zetb,
const T rab[3],
const int n,
const T * cab )

Returns element i,j of virial on atom b for compute_tau=false.

Author
Ole Schuett

Definition at line 168 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_virial_b()

template<bool compute_tau, typename T>
__device__ __inline__ double grid_gpu_backend::get_virial_b ( const orbital & a,
const orbital & b,
const int i,
const int j,
const T zeta,
const T zetb,
const T rab[3],
const int n,
const T * cab )

Returns element i,j of virial on atom b.

Author
Ole Schuett

Definition at line 186 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_hab()

template<bool compute_tau, typename T>
__device__ __inline__ T grid_gpu_backend::get_hab ( const orbital & a,
const orbital & b,
const T zeta,
const T zetb,
const int n,
const T * cab )

Returns element i,j of hab matrix.

Author
Ole Schuett

Definition at line 217 of file grid_gpu_process_vab.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ process_get_ldiffs()

ldiffs_value grid_gpu_backend::process_get_ldiffs ( bool calculate_forces,
bool calculate_virial,
bool compute_tau )
inline

Returns difference in angular momentum range for given flags.

Definition at line 241 of file grid_gpu_process_vab.h.

Here is the caller graph for this function:

◆ sep_advance_wrapped_index()

template<int STRIDE>
__device__ __inline__ int grid_gpu_backend::sep_advance_wrapped_index ( const int x2,
const int n )

Definition at line 54 of file grid_gpu_separable.h.

◆ separable_lmax_bucket()

int grid_gpu_backend::separable_lmax_bucket ( const int lp_max)
inline

Definition at line 62 of file grid_gpu_separable.h.

Here is the caller graph for this function:

◆ init_separable_constant_memory()

void grid_gpu_backend::init_separable_constant_memory ( )
static

Initializes the constant memory of integrate_separable_kernel.

Definition at line 87 of file grid_gpu_separable.h.

Here is the caller graph for this function:

◆ build_lattice_to_cartesian_matrix()

std::vector< double > grid_gpu_backend::build_lattice_to_cartesian_matrix ( const double * dh,
const int lmax )
static

Builds the matrix expanding the cartesian monomials x^a y^b z^c in lattice monomials u^i v^j w^k, where x = dh[0] u + dh[3] v + dh[6] w, y = dh[1] u + dh[4] v + dh[7] w, z = dh[2] u + dh[5] v + dh[8] w. Only the blocks a + b + c = i + j + k = l are non zero. They are stored one after the other for l = 0..lmax, block l being row major of size nl x nl, nl = (l + 1)(l + 2) / 2, rows and columns in coset order.

With H[d][e] = dh[3 e + d] and p = (a, b, c), the entries are given by the multinomial expansion of prod_d (sum_e H[d][e] u_e)^p_d,

M[(abc)][(ijk)] = sum_n prod_d p_d! / (n_d0! n_d1! n_d2!) prod_e H[d][e]^n_de

the sum running over the 3x3 non negative integer matrices n with row sums (a, b, c) and column sums (i, j, k). Block l is the symmetric power Sym^l(H) of the linear map in the monomial basis, so that M(H1 H2) = M(H1) M(H2) and M(H)^-1 = M(H^-1).

The factorials are never evaluated. Each row is built by multiplying the polynomial 1 by one linear form at a time,

x_d u^i v^j w^k = H[d][0] u^(i+1) v^j w^k + H[d][1] u^i v^(j+1) w^k

  • H[d][2] u^i v^j w^(k+1)

a times with d = 0, b times with d = 1 and c times with d = 2, the multinomial coefficients being generated by the repeated additions as in Pascal's triangle. Equivalently, for a > 0 (similarly for b, c with rows 1, 2 of H),

M[(abc)][(ijk)] = H[0][0] M[(a-1,b,c)][(i-1,j,k)]

  • H[0][1] M[(a-1,b,c)][(i,j-1,k)]
  • H[0][2] M[(a-1,b,c)][(i,j,k-1)]

Integrate maps lattice moments to cartesian moments with M, collocate maps cartesian coefficients to lattice coefficients with M^T. The matrix is built on the host once per grid level and cached.

Definition at line 150 of file grid_gpu_separable.h.

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_lattice_to_cartesian()

const double * grid_gpu_backend::get_lattice_to_cartesian ( grid_info< double > & grid,
const int lmax )
static

Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the lattice vectors changed or a higher angular momentum is needed.

Definition at line 196 of file grid_gpu_separable.h.

Here is the call graph for this function:
Here is the caller graph for this function:

Variable Documentation

◆ ld

__global__ const int grid_gpu_backend::ld

Definition at line 130 of file grid_gpu_collocate.cu.

◆ max_nsgf_set

__global__ const int const int grid_gpu_backend::max_nsgf_set
Initial value:
{
extern __shared__ T shared_memory[]

Definition at line 130 of file grid_gpu_collocate.cu.

◆ number_of_tasks

const int grid_gpu_backend::number_of_tasks = dev_.num_tasks_per_block_dev[block_index()]

Definition at line 132 of file grid_gpu_collocate.cu.

◆ smem_t

T* grid_gpu_backend::smem_t = &shared_memory[0]

Definition at line 137 of file grid_gpu_collocate.cu.

◆ smem_p

T* grid_gpu_backend::smem_p = smem_t + max_nsgf_set * ld

Definition at line 138 of file grid_gpu_collocate.cu.

◆ tid

const int grid_gpu_backend::tid = thread_global_index()

Definition at line 139 of file grid_gpu_collocate.cu.

◆ nthreads

const int grid_gpu_backend::nthreads = blockDim.x * blockDim.y * blockDim.z

Definition at line 140 of file grid_gpu_collocate.cu.

◆ offset

const int grid_gpu_backend::offset = dev_.sorted_blocks_offset_dev[block_index()]

Definition at line 141 of file grid_gpu_collocate.cu.

◆ smem_cab

T *__restrict__ grid_gpu_backend::smem_cab
Initial value:
= reinterpret_cast<double *>(
__builtin_assume_aligned(allocate_workspace<T>(dev_), 32))
__inline__ __device__ T * allocate_workspace(const kernel_params &dev_)

Definition at line 142 of file grid_gpu_collocate.cu.

◆ mc

__global__ const int grid_gpu_backend::mc

Definition at line 258 of file grid_gpu_collocate.cu.

◆ lp_max

__global__ const int const int grid_gpu_backend::lp_max

Definition at line 258 of file grid_gpu_collocate.cu.

◆ erows

__global__ const int const int const int grid_gpu_backend::erows

Definition at line 258 of file grid_gpu_collocate.cu.

◆ M

__global__ const int const int const int const double* __restrict__ grid_gpu_backend::M
Initial value:
{
constexpr bool orthogonal_ = axis_aligned_

Definition at line 259 of file grid_gpu_collocate.cu.

◆ glb_task

const task_info & grid_gpu_backend::glb_task = dev_.tasks[dev_.first_task + block_index()]

Definition at line 264 of file grid_gpu_collocate.cu.

◆ ls

const int grid_gpu_backend::ls = lp_max + 1

Definition at line 269 of file grid_gpu_collocate.cu.

◆ sep_smem

__shared__ double grid_gpu_backend::sep_smem
extern

Definition at line 964 of file grid_gpu_integrate.cu.

◆ ex

double * grid_gpu_backend::ex = sep_smem

Definition at line 272 of file grid_gpu_collocate.cu.

◆ ey

double* grid_gpu_backend::ey = ex + mc

Definition at line 273 of file grid_gpu_collocate.cu.

◆ D

double* grid_gpu_backend::D = ey + mc

Definition at line 274 of file grid_gpu_collocate.cu.

◆ C

double* grid_gpu_backend::C = D + ls * ls

Definition at line 275 of file grid_gpu_collocate.cu.

◆ E

double* grid_gpu_backend::E = C + ncoset(lp_max)

Definition at line 276 of file grid_gpu_collocate.cu.

◆ task

__shared__ smem_task_reduced< double, double3 > grid_gpu_backend::task

Definition at line 278 of file grid_gpu_collocate.cu.

◆ lp

const int grid_gpu_backend::lp = task.lp

Definition at line 285 of file grid_gpu_collocate.cu.

◆ ncoef

const int grid_gpu_backend::ncoef = ncoset(lp)

Definition at line 286 of file grid_gpu_collocate.cu.

◆ zetp

const double grid_gpu_backend::zetp = task.zetp

Definition at line 287 of file grid_gpu_collocate.cu.

◆ r2max

const double grid_gpu_backend::r2max = task.radius * task.radius

Definition at line 288 of file grid_gpu_collocate.cu.

◆ trim

const bool grid_gpu_backend::trim = orthogonal_ && !task.apply_border_mask

Definition at line 289 of file grid_gpu_collocate.cu.

◆ coef

double * grid_gpu_backend::coef
Initial value:
=
&dev_.buffers_dev.coef[glb_task.coef_offset]
const task_info & glb_task

Definition at line 291 of file grid_gpu_collocate.cu.

◆ npairs

const int grid_gpu_backend::npairs = (lp + 1) * (lp + 2) / 2

Definition at line 325 of file grid_gpu_collocate.cu.

◆ row_group

const int grid_gpu_backend::row_group = tid / GRID_SEP_COLLOCATE_ROW_LANES

Definition at line 326 of file grid_gpu_collocate.cu.

◆ lane

const int grid_gpu_backend::lane = tid % GRID_SEP_COLLOCATE_ROW_LANES

Definition at line 327 of file grid_gpu_collocate.cu.

◆ nrow_groups

int grid_gpu_backend::nrow_groups = 64 / GRID_SEP_COLLOCATE_ROW_LANES
constexpr

Definition at line 328 of file grid_gpu_collocate.cu.

◆ integrate_separable_lp_threshold

int grid_gpu_backend::integrate_separable_lp_threshold = 0
constexpr

Definition at line 39 of file grid_gpu_integrate.cu.

◆ block_reduce_64_smem

size_t grid_gpu_backend::block_reduce_64_smem = 64 * sizeof(double)
constexpr

Definition at line 57 of file grid_gpu_integrate.cu.

◆ smem_h

T * grid_gpu_backend::smem_h = &shared_memory[0]

Definition at line 165 of file grid_gpu_integrate.cu.

◆ smem_u

T* grid_gpu_backend::smem_u = smem_h + ld * ld

Definition at line 166 of file grid_gpu_integrate.cu.

◆ smem_w

T* grid_gpu_backend::smem_w = smem_h + ld * ld

Definition at line 415 of file grid_gpu_integrate.cu.

◆ compute_virial

const bool grid_gpu_backend::compute_virial = (dev_.buffers_dev.virial != nullptr)

Definition at line 419 of file grid_gpu_integrate.cu.

◆ fa

T grid_gpu_backend::fa[3] = {0.0, 0.0, 0.0}

Definition at line 423 of file grid_gpu_integrate.cu.

◆ fb

T grid_gpu_backend::fb[3] = {0.0, 0.0, 0.0}

Definition at line 423 of file grid_gpu_integrate.cu.

◆ virial

T grid_gpu_backend::virial[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}

Definition at line 424 of file grid_gpu_integrate.cu.

◆ task_id

const int grid_gpu_backend::task_id = dev_.task_sorted_by_blocks_dev[offset]

Definition at line 509 of file grid_gpu_integrate.cu.

◆ forces_a

T* grid_gpu_backend::forces_a = &dev_.buffers_dev.forces[3 * glb_task.iatom]

Definition at line 511 of file grid_gpu_integrate.cu.

◆ forces_b

T* grid_gpu_backend::forces_b = &dev_.buffers_dev.forces[3 * glb_task.jatom]

Definition at line 512 of file grid_gpu_integrate.cu.

◆ sum

T* grid_gpu_backend::sum = (T *)shared_memory

Definition at line 514 of file grid_gpu_integrate.cu.

◆ lattice_to_cartesian

__global__ const int const int const double* __restrict__ grid_gpu_backend::lattice_to_cartesian
Initial value:
{
constexpr bool orthogonal_ = axis_aligned_

Definition at line 943 of file grid_gpu_integrate.cu.

◆ py

double* grid_gpu_backend::py = ex + mc

Definition at line 970 of file grid_gpu_integrate.cu.

◆ pz

double* grid_gpu_backend::pz = py + ls * mc

Definition at line 973 of file grid_gpu_integrate.cu.

◆ A

double* grid_gpu_backend::A = pz + ls

Definition at line 974 of file grid_gpu_integrate.cu.

◆ acc

double* grid_gpu_backend::acc = A + mc * ls

Definition at line 975 of file grid_gpu_integrate.cu.

◆ else

grid_gpu_backend::else
Initial value:
{
for (int i = tid; i < ncoef; i += 64) {
const orbital &co = coset_inv[i];
const int l = co.l[0] + co.l[1] + co.l[2];
const int base = (l > 0) ? ncoset(l - 1) : 0;
const int nl = ncoset(l) - base;
const double *__restrict__ m =
lattice_to_cartesian + sep_m_offset[l] + (i - base) * nl;
double val = 0.0;
for (int j = 0; j < nl; j++)
val += m[j] * acc[base + j];
coef[i] = val;
}
}
}
static size_t integrate_separable_smem_size(const int mc, const int lp_max) {
const int ls = lp_max + 1;
return sizeof(double) * (mc + ls * mc + ls + mc * ls + ncoset(lp_max));
}
using integrate_separable_kernel_t = void (*)(const kernel_params, const int,
const int, const double *)
static void const int const int i
__constant__ orbital coset_inv[1330]
__global__ const int mc
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
__constant__ int sep_m_offset[GRID_SEP_MAX_L+1]
__global__ const int const int const double *__restrict__ lattice_to_cartesian
const double *__restrict__ coef
__global__ const int const int lp_max
Parameters of the collocate kernel.
Orbital angular momentum.

Definition at line 1165 of file grid_gpu_integrate.cu.

◆ grid_gpu_max_l

int grid_gpu_backend::grid_gpu_max_l = 18
constexpr

Definition at line 67 of file grid_gpu_internal_header.h.

◆ coset_inv

__constant__ orbital grid_gpu_backend::coset_inv[1330]

Definition at line 69 of file grid_gpu_internal_header.h.

◆ sphi_two_step_max_smem

size_t grid_gpu_backend::sphi_two_step_max_smem = 48 * 1024
constexpr

Definition at line 832 of file grid_gpu_internal_header.h.

◆ separable_max_smem

size_t grid_gpu_backend::separable_max_smem = 48 * 1024
constexpr

Definition at line 74 of file grid_gpu_separable.h.

◆ sep_pair_ab

__constant__ char2 grid_gpu_backend::sep_pair_ab[(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2]

Definition at line 78 of file grid_gpu_separable.h.

◆ sep_m_offset

__constant__ int grid_gpu_backend::sep_m_offset[GRID_SEP_MAX_L+1]

Definition at line 82 of file grid_gpu_separable.h.