![]() |
(git:27c966b)
|
Data Structures | |
| class | gpu_vector |
| class | grid_info |
| struct | task_info |
| Internal representation of a task. More... | |
| struct | kernel_buffers |
| Device buffers used by the collocate and integrate kernels. Which fields are non-null depends on which kernel is launched. More... | |
| struct | kernel_params |
| Parameters of the collocate kernel. More... | |
| class | context_info |
| struct | orbital |
| Orbital angular momentum. More... | |
| struct | ldiffs_value |
| Differences in angular momentum. More... | |
| struct | smem_task |
| data needed for calculating the coefficients, forces, and stress More... | |
| struct | smem_task_reduced |
| data needed for collocate and integrate kernels More... | |
| class | smem_parameters |
Functions | |
| template<typename T, bool IS_FUNC_AB> | |
| __device__ __inline__ void | block_to_cab (const kernel_params ¶ms, const smem_task< T > &task, T *cab) |
| Decontracts the subblock, going from spherical to cartesian harmonics. | |
| template<typename T, bool IS_FUNC_AB> | |
| __global__ | __launch_bounds__ (64) void calculate_coefficients_two_steps(const kernel_params dev_ |
| Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done in two steps,. | |
| if (number_of_tasks==0) return | |
| for (int tk=0;tk< number_of_tasks;tk++) | |
| template<bool distributed__, bool axis_aligned_> | |
| __global__ | __launch_bounds__ (64) void collocate_separable_kernel(const kernel_params dev_ |
| if (glb_task.skip_task) return | |
| fill_smem_task_reduced (dev_, dev_.first_task+block_index(), task) | |
| if (tid==0) | |
| __syncthreads () | |
| if (axis_aligned_) | |
| static size_t | collocate_separable_smem_size (const int mc, const int lp_max, const int erows) |
| static int | collocate_separable_erows (const int mc, const int lp_max, const size_t smem_max) |
| constexpr size_t | align_up_elems (size_t n_elems, size_t elem_alignment) |
| size_t | round_up16 (size_t n) |
| __inline__ __device__ double | sep_row_reduce (double val) |
| template<typename T> | |
| __device__ __inline__ T | block_reduce_64 (T *table, const T val_, const int tid) |
| template<typename T, bool COMPUTE_TAU> | |
| __device__ __inline__ void | accumulate_pair_forces (const smem_task< T > &task, const orbital &a, const orbital &b, const T p, const bool compute_virial, const T *__restrict__ cab, T *fa, T *fb, T *virial) |
| Adds the contribution of the cartesian pair (a, b), weighted by the cartesian density p, to the forces and the virial. | |
| template<typename T, typename T3, bool COMPUTE_TAU> | |
| __global__ | __launch_bounds__ (64) void compute_hab_forces(const kernel_params dev_) |
| Fallback of compute_hab_forces_two_steps when its work arrays do not fit in the shared memory. hab is computed as in compute_hab. The forces and the virial only depend on the cartesian pair (a, b), so they are evaluated once per pair with the cartesian density. | |
| if (compute_virial) | |
| template<typename T, typename T3, bool distributed__, bool orthogonal_> | |
| __global__ | __launch_bounds__ (64) void integrate_kernel(const kernel_params dev_) |
| template<bool distributed__, bool axis_aligned_, int lmax_> | |
| __global__ | __launch_bounds__ (64) void integrate_separable_kernel(const kernel_params dev_ |
| if (glb_task.la_max+dev_.la_max_diff+glb_task.lb_max+dev_.lb_max_diff< dev_.separable_lp_min) return | |
| template<int lmax_> | |
| static integrate_separable_kernel_t | select_integrate_separable (const bool distributed, const bool axis_aligned) |
| Selects the instance of integrate_separable_kernel for one value of lmax_. | |
| static integrate_separable_kernel_t | select_integrate_separable (const int lp_max, const bool distributed, const bool axis_aligned) |
| Selects the instance of integrate_separable_kernel with the smallest bucket of the angular momentum covering lp_max. | |
| __host__ __device__ __inline__ int | ncoset (const int l) |
| Number of Cartesian orbitals up to given angular momentum quantum. | |
| __host__ __device__ __inline__ int | coset (int lx, int ly, int lz) |
| Maps three angular momentum components to a single zero based index. | |
| __device__ __inline__ orbital | up (const int i, const orbital &a) |
| Increase i'th component of given orbital angular momentum. | |
| __inline__ __device__ orbital | down (const int i, const orbital &a) |
| Decrease i'th component of given orbital angular momentum. | |
| __inline__ __device__ int | idx (const orbital a) |
| Return coset index of given orbital angular momentum. | |
| __device__ __inline__ double | power (const double x, const int expo) |
| template<typename T = double> | |
| __device__ __inline__ void | prep_term (const orbital a, const orbital b, const T value, const int n, T *cab) |
| Adds given value to matrix element cab[idx(b)][idx(a)]. | |
| static void | init_constant_memory () |
| Initializes the device's constant memory. | |
| __inline__ __device__ unsigned int | block_index () |
| __inline__ __device__ unsigned int | thread_global_index () |
| __inline__ __host__ __device__ double3 | compute_coordinates (const double *__restrict__ dh_, const double x, const double y, const double z) |
| template<typename T, typename T3, bool orthorhombic_> | |
| __inline__ T | compute_cube_properties (const T radius, const T *const __restrict__ dh_, const T *const __restrict__ dh_inv_, const T3 &rp, T3 &roffset, int3 &cubecenter, int3 &lb_cube, int3 &cube_size) |
| __inline__ __device__ void | compute_window_size (const int3 grid_size, const int border_mask, const int3 &border_width, int3 &window_size, int3 &window_shift) |
| __inline__ __device__ int | wrap_grid_index (const int idx, const int full_size) |
| Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry shared by collocate_kernel and integrate_kernel. | |
| template<typename T, typename T3, bool distributed__> | |
| __device__ __inline__ void | setup_task_cube_center (const kernel_params &dev_, smem_task_reduced< T, T3 > &task) |
| Shifts task.cube_center into the local grid's coordinate system and, for distributed grids, derives the border window. Single point of entry shared by collocate_kernel and integrate_kernel so the two cannot drift apart on which axis maps to which. | |
| __device__ __inline__ void | decode_pair (int t, int &p, int &q) |
| Factorized cab <-> cxyz transformations. | |
| __device__ __inline__ int | line_index (const int dir, const int m, const int p, const int q, const bool on_a, const int other, const int n1) |
| template<typename T, bool TRANSPOSE> | |
| __device__ __inline__ void | move_center (const int la, const int lb, const int n1, const int dir, const bool on_a, const T dr, T *cab) |
| Moves the expansion center of the a (on_a) or b cartesian index of cab along direction dir, in place: | |
| template<typename T> | |
| __device__ __inline__ void | cab_to_cxyz_factorized (const smem_task< T > &task, T *__restrict__ cab, T *__restrict__ cxyz) |
| Transforms the coefficients C_ab into C_xyz, cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a] (see cxyz_to_cab_factorized). cab is overwritten. | |
| template<typename T> | |
| __device__ __inline__ void | cxyz_to_cab_factorized (const smem_task< T > &task, const T *__restrict__ cxyz, T *__restrict__ cab) |
| Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,. | |
| template<typename T, typename T3> | |
| __device__ __inline__ void | fill_smem_task_reduced (const kernel_params &dev, const int task_id, smem_task_reduced< T, T3 > &task) |
| Copies a task from global to shared memory. | |
| template<typename T> | |
| __device__ __inline__ void | fill_smem_task_coef (const kernel_params &dev, const int task_id, smem_task< T > &task) |
| Copies a task from global to shared memory and does precomputations. | |
| int | sphi_work_size (const int lmax, const int max_nsgf_set) |
| Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calculate_coefficients_two_steps and compute_hab_two_steps: [max_nsgf_set][ld] and [ld][ld] with ld = ncoset(lmax). | |
| template<typename Kernel> | |
| size_t | max_dynamic_smem (Kernel kernel, const size_t budget) |
| Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memory together must fit in budget bytes. The static part (e.g. shared task structures) is taken from the compiled kernel and rounded up to 16 bytes for the alignment of the dynamic array placed after it. | |
| template<typename T> | |
| __inline__ __device__ T * | allocate_workspace (const kernel_params &dev_) |
| template<typename T, typename T3> | |
| __device__ __inline__ T | calculate_ymix_ymax_boundaries (smem_task_reduced< T, T3 > &task, const int z, int &ymin, int &ymax) |
| template<typename T, typename T3> | |
| __device__ __inline__ void | calculate_xmin_xmax_boundaries (smem_task_reduced< T, T3 > &task, const int y, const T kremain, int &xmin, int &xmax) |
| template<typename T> | |
| __device__ __inline__ void | prepare_pab_AB (const orbital a, const orbital b, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_AB, ie. identity transformation. | |
| template<typename T> | |
| __device__ __inline__ void | prepare_pab_DADB (const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_DADB. | |
| template<typename T> | |
| __device__ void | prepare_pab_ADBmDAB (const int idir, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_ADBmDAB_{X,Y,Z}. | |
| template<typename T> | |
| __device__ void | prepare_pab_ARDBmDARB (const int idir, const int ir, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_ARDBmDARB_{X,Y,Z}{X,Y,Z}. | |
| template<typename T> | |
| __device__ void | prepare_pab_DABpADB (const int idir, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_DABpADB_{X,Y,Z}. | |
| template<typename T> | |
| __device__ __inline__ void | prepare_pab_Di (const int ider, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_{DX,DY,DZ}. | |
| template<typename T> | |
| __device__ __inline__ void | oneterm_dijdij (const int idir, const T func_a, const orbital a, const orbital b, const T zetb, const int n, T *cab) |
| Helper for grid_prepare_pab_DiDj. | |
| template<typename T> | |
| __device__ __inline__ void | prepare_pab_DiDj (const int ider1, const int ider2, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_{DXDY,DYDZ,DZDX}. | |
| template<typename T> | |
| __device__ void | oneterm_diidii (const int idir, const T func_a, const orbital a, const orbital b, const T zetb, const int n, T *cab) |
| Helper for grid_prepare_pab_Di2. | |
| template<typename T> | |
| __device__ __inline__ void | prepare_pab_Di2 (const int ider, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Implementation of function GRID_FUNC_{DXDX,DYDY,DZDZ}. | |
| template<typename T> | |
| __device__ __inline__ void | prepare_pab (const enum grid_func func, const orbital a, const orbital b, const T zeta, const T zetb, const T pab_val, const int n, T *cab) |
| Transforms a given element of the density matrix according to func. | |
| ldiffs_value | prepare_get_ldiffs (const enum grid_func func) |
| Returns difference in angular momentum range for given func. | |
| template<typename T> | |
| __device__ __inline__ T | get_term (const orbital &a, const orbital &b, const int n, const T *cab) |
| Returns matrix element cab[idx(b)][idx(a)]. | |
| template<typename T> | |
| __device__ __inline__ T | get_force_a_normal (const orbital &a, const orbital &b, const int i, const T zeta, const int n, const T *cab) |
| Returns i'th component of force on atom a for compute_tau=false. | |
| template<bool compute_tau, typename T> | |
| __device__ __inline__ double | get_force_a (const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const int n, const T *cab) |
| Returns i'th component of force on atom a. | |
| template<typename T> | |
| __device__ __inline__ double | get_force_b_normal (const orbital &a, const orbital &b, const int i, const T zetb, const T rab[3], const int n, const T *cab) |
| Returns i'th component of force on atom b for compute_tau=false. | |
| template<bool compute_tau, typename T> | |
| __device__ __inline__ T | get_force_b (const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const T rab[3], const int n, const T *cab) |
| Returns i'th component of force on atom b. | |
| template<typename T> | |
| __device__ __inline__ double | get_virial_a_normal (const orbital &a, const orbital &b, const int i, const int j, const T zeta, const int n, const T *cab) |
| Returns element i,j of virial on atom a for compute_tau=false. | |
| template<bool compute_tau, typename T> | |
| __device__ __inline__ T | get_virial_a (const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const int n, const T *cab) |
| Returns element i,j of virial on atom a. | |
| template<typename T> | |
| __device__ __inline__ double | get_virial_b_normal (const orbital &a, const orbital &b, const int i, const int j, const T zetb, const T rab[3], const int n, const T *cab) |
| Returns element i,j of virial on atom b for compute_tau=false. | |
| template<bool compute_tau, typename T> | |
| __device__ __inline__ double | get_virial_b (const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const T rab[3], const int n, const T *cab) |
| Returns element i,j of virial on atom b. | |
| template<bool compute_tau, typename T> | |
| __device__ __inline__ T | get_hab (const orbital &a, const orbital &b, const T zeta, const T zetb, const int n, const T *cab) |
| Returns element i,j of hab matrix. | |
| ldiffs_value | process_get_ldiffs (bool calculate_forces, bool calculate_virial, bool compute_tau) |
| Returns difference in angular momentum range for given flags. | |
| template<int STRIDE> | |
| __device__ __inline__ int | sep_advance_wrapped_index (const int x2, const int n) |
| int | separable_lmax_bucket (const int lp_max) |
| static void | init_separable_constant_memory () |
| Initializes the constant memory of integrate_separable_kernel. | |
| static std::vector< double > | build_lattice_to_cartesian_matrix (const double *dh, const int lmax) |
| Builds the matrix expanding the cartesian monomials x^a y^b z^c in lattice monomials u^i v^j w^k, where x = dh[0] u + dh[3] v + dh[6] w, y = dh[1] u + dh[4] v + dh[7] w, z = dh[2] u + dh[5] v + dh[8] w. Only the blocks a + b + c = i + j + k = l are non zero. They are stored one after the other for l = 0..lmax, block l being row major of size nl x nl, nl = (l + 1)(l + 2) / 2, rows and columns in coset order. | |
| static const double * | get_lattice_to_cartesian (grid_info< double > &grid, const int lmax) |
| Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the lattice vectors changed or a higher angular momentum is needed. | |
Variables | |
| __global__ const int | ld |
| __global__ const int const int | max_nsgf_set |
| const int | number_of_tasks = dev_.num_tasks_per_block_dev[block_index()] |
| T * | smem_t = &shared_memory[0] |
| T * | smem_p = smem_t + max_nsgf_set * ld |
| const int | tid = thread_global_index() |
| const int | nthreads = blockDim.x * blockDim.y * blockDim.z |
| const int | offset = dev_.sorted_blocks_offset_dev[block_index()] |
| T * | smem_cab |
| __global__ const int | mc |
| __global__ const int const int | lp_max |
| __global__ const int const int const int | erows |
| __global__ const int const int const int const double *__restrict__ | M |
| const task_info & | glb_task = dev_.tasks[dev_.first_task + block_index()] |
| const int | ls = lp_max + 1 |
| __shared__ double | sep_smem [] |
| double * | ex = sep_smem |
| double * | ey = ex + mc |
| double * | D = ey + mc |
| double * | C = D + ls * ls |
| double * | E = C + ncoset(lp_max) |
| __shared__ smem_task_reduced< double, double3 > | task |
| const int | lp = task.lp |
| const int | ncoef = ncoset(lp) |
| const double | zetp = task.zetp |
| const double | r2max = task.radius * task.radius |
| const bool | trim = orthogonal_ && !task.apply_border_mask |
| const double *__restrict__ | coef |
| const int | npairs = (lp + 1) * (lp + 2) / 2 |
| const int | row_group = tid / GRID_SEP_COLLOCATE_ROW_LANES |
| const int | lane = tid % GRID_SEP_COLLOCATE_ROW_LANES |
| constexpr int | nrow_groups = 64 / GRID_SEP_COLLOCATE_ROW_LANES |
| constexpr int | integrate_separable_lp_threshold = 0 |
| constexpr size_t | block_reduce_64_smem = 64 * sizeof(double) |
| T * | smem_h = &shared_memory[0] |
| T * | smem_u = smem_h + ld * ld |
| T * | smem_w = smem_h + ld * ld |
| const bool | compute_virial = (dev_.buffers_dev.virial != nullptr) |
| T | fa [3] = {0.0, 0.0, 0.0} |
| T | fb [3] = {0.0, 0.0, 0.0} |
| T | virial [9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} |
| const int | task_id = dev_.task_sorted_by_blocks_dev[offset] |
| T * | forces_a = &dev_.buffers_dev.forces[3 * glb_task.iatom] |
| T * | forces_b = &dev_.buffers_dev.forces[3 * glb_task.jatom] |
| T * | sum = (T *)shared_memory |
| __global__ const int const int const double *__restrict__ | lattice_to_cartesian |
| double * | py = ex + mc |
| double * | pz = py + ls * mc |
| double * | A = pz + ls |
| double * | acc = A + mc * ls |
| else | |
| constexpr int | grid_gpu_max_l = 18 |
| __constant__ orbital | coset_inv [1330] |
| constexpr size_t | sphi_two_step_max_smem = 48 * 1024 |
| constexpr size_t | separable_max_smem = 48 * 1024 |
| __constant__ char2 | sep_pair_ab [(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2] |
| __constant__ int | sep_m_offset [GRID_SEP_MAX_L+1] |
| __device__ __inline__ void grid_gpu_backend::block_to_cab | ( | const kernel_params & | params, |
| const smem_task< T > & | task, | ||
| T * | cab ) |
Decontracts the subblock, going from spherical to cartesian harmonics.
Definition at line 36 of file grid_gpu_collocate.cu.
| __global__ grid_gpu_backend::__launch_bounds__ | ( | 64 | ) | const |
Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done in two steps,.
Same as compute_hab_forces with the spherical to cartesian transformations done as matrix products. The forces and the virial only depend on the cartesian pair (a, b), so they are contracted with the cartesian density.
Same as compute_hab, with the cartesian to spherical transformation done in two steps,.
T[i][ico] = sum_j pab[i][j] sphia[j][ico] (nsgfb nsgfa ncoa) P[jco][ico] = sum_i sphib[i][jco] T[i][ico] (ncob nsgfb ncoa)
instead of the fused loop of cost ncoa ncob nsgfb (nsgfa + 1). The work arrays T [max_nsgf_set][ld] and P [ld][ld] live in the dynamic shared memory, see sphi_work_size.
H[jco][ico] = get_hab(a, b, cab) (ncob ncoa) U[i][ico] = sum_jco sphib[i][jco] H[jco][ico] (nsgfb ncob ncoa) hab[i][j] += sum_ico U[i][ico] sphia[j][ico] (nsgfb nsgfa ncoa)
instead of evaluating get_hab and both sphi factors in the innermost loop of a nsgfb nsgfa ncob ncoa loop. The work arrays H [ld][ld] and U [max_nsgf_set][ld] live in the dynamic shared memory, see sphi_work_size.
P[jco][ico] = sum_ij sphib[i][jco] pab[i][j] sphia[j][ico]
and evaluated once per pair instead of once per (i, j, a, b):
W[i][ico] = sum_j pab[i][j] sphia[j][ico] (nsgfb nsgfa ncoa) for each (jco, ico): p = sum_i sphib[i][jco] W[i][ico] (nsgfb ncob ncoa) fa, fb, virial += p * get_force / get_virial (a, b) H[jco][ico] = get_hab(a, b) U[i][ico] = sum_jco sphib[i][jco] H[jco][ico] (nsgfb ncob ncoa) hab[i][j] += sum_ico U[i][ico] sphia[j][ico] (nsgfb nsgfa ncoa)
P is never stored. The work arrays H [ld][ld] and W / U [max_nsgf_set][ld] (U overwrites W) live in the dynamic shared memory, see sphi_work_size.
Definition at line 84 of file grid_gpu_collocate.cu.
| grid_gpu_backend::if | ( | number_of_tasks | = =0 | ) |
| grid_gpu_backend::for | ( | ) |
| __global__ grid_gpu_backend::__launch_bounds__ | ( | 64 | ) | const |
Separable (plane streaming) collocate kernel. This is integrate_separable_kernel (grid_gpu_integrate.cu) run backwards.
Axis aligned orthorhombic cell (axis_aligned_ = true):
sum_{abc} C[abc] x^a y^b z^c exp(-zetp r^2) = sum_a [x^a e^{-zetp x^2}] sum_b [y^b e^{-zetp y^2}] sum_c [z^c e^{-zetp z^2}] C[abc]
Other cells (axis_aligned_ = false): the cartesian coefficients are first expressed in lattice monomials u^i v^j w^k (u, v, w lattice coordinates of the point relative to the gaussian center),
C_ijk[(i,j,k)] = sum_{a+b+c=i+j+k} M[(a,b,c)][(i,j,k)] C[(a,b,c)]
with the matrix M of build_lattice_to_cartesian_matrix (the transpose of the integrate transformation). Only the polynomial is separated, the gaussian exp(-zetp r^2) is evaluated per grid point from the r^2 of the sphere test, as in integrate_separable_kernel. Factoring it as
exp(-zetp g00 u^2) exp(-zetp g11 v^2) exp(-zetp g22 w^2) exp(-2 zetp g12 v w) exp(-2 zetp u (g01 v + g02 w))
with the metric g_ij = a_i . a_j is not numerically stable: on strongly sheared cells the diagonal factors underflow while the cross terms overflow, even though their product is finite.
For each z plane of the cube:
D[a][b] = sum_c C[coset(a, b, c)] pz[c] a + b <= lp (step 1) E[y][a] = sum_b D[a][b] py[b][y] (step 2) grid[z][y][x] += sum_a E[y][a] px[a][x] (step 3)
so that a grid point costs lp + 1 FMAs (Horner in x) instead of ncoset(lp). Steps 1 and 2 are O(lp^3) and O(ny lp^2) per plane. The set of grid points visited is the same as for the original collocate kernel: same window masks, same y / x trimming in the orthogonal case, same sphere test.
Dynamic shared memory layout (doubles), mc = max cube size, ls = lp_max + 1: ex[mc], ey[mc], D[ls][ls], C[ncoset(lp_max)], E[erows][ls]
The powers of y are computed on the fly in step 2 and the rows of a plane are processed by chunks of erows rows, so that the shared memory is bounded for any cube size (see collocate_separable_erows), the x coordinates are computed on the fly. ex and ey are only used when axis_aligned_ = true.
| grid_gpu_backend::if | ( | glb_task. | skip_task | ) |
| grid_gpu_backend::fill_smem_task_reduced | ( | dev_ | , |
| dev_.first_task+ | block_index(), | ||
| task | ) |
| grid_gpu_backend::if | ( | tid | = = 0 | ) |
| grid_gpu_backend::__syncthreads | ( | ) |
| grid_gpu_backend::if | ( | axis_aligned_ | ) |
Definition at line 313 of file grid_gpu_collocate.cu.
|
static |
Definition at line 451 of file grid_gpu_collocate.cu.
|
static |
Definition at line 458 of file grid_gpu_collocate.cu.
|
constexpr |
|
inline |
| __inline__ __device__ double grid_gpu_backend::sep_row_reduce | ( | double | val | ) |
Definition at line 41 of file grid_gpu_integrate.cu.
| __device__ __inline__ T grid_gpu_backend::block_reduce_64 | ( | T * | table, |
| const T | val_, | ||
| const int | tid ) |
Definition at line 60 of file grid_gpu_integrate.cu.
| __device__ __inline__ void grid_gpu_backend::accumulate_pair_forces | ( | const smem_task< T > & | task, |
| const orbital & | a, | ||
| const orbital & | b, | ||
| const T | p, | ||
| const bool | compute_virial, | ||
| const T *__restrict__ | cab, | ||
| T * | fa, | ||
| T * | fb, | ||
| T * | virial ) |
Adds the contribution of the cartesian pair (a, b), weighted by the cartesian density p, to the forces and the virial.
Definition at line 236 of file grid_gpu_integrate.cu.
| __global__ grid_gpu_backend::__launch_bounds__ | ( | 64 | ) | const |
Fallback of compute_hab_forces_two_steps when its work arrays do not fit in the shared memory. hab is computed as in compute_hab. The forces and the virial only depend on the cartesian pair (a, b), so they are evaluated once per pair with the cartesian density.
p[jco][ico] = sum_i sphib[i][jco] sum_j pab[i][j] sphia[j][ico]
computed on the fly (nsgfb (nsgfa + 1) FMA per pair), instead of once per (i, j, a, b) in the hab loop.
Definition at line 270 of file grid_gpu_integrate.cu.
| grid_gpu_backend::if | ( | compute_virial | ) |
| __global__ grid_gpu_backend::__launch_bounds__ | ( | 64 | ) | const |
Cuda kernel for calculating the coefficients of a potential (density, etc...) for a given pair of gaussian basis sets
We compute the discretized version of the following integral
$$\int _\infty ^\infty V_{ijk} P^\alpha_iP^beta_j P ^ \gamma _ k Exp(- \eta |r_{ijk} - r_c|^2)$$
where in practice the summation is over a finite domain. The discrete form has this shape
$$ \sum_{ijk < domain} V_{ijk} (x_i - x_c)^\alpha (y_j - y_c)^\beta (z_k - z_c) ^ \gamma Exp(- \eta |r_{ijk} - r_c|^2) $$
where $0 \le \alpha + \beta + \gamma \le lmax$
It is formely the same operation than collocate (from a discrete point of view) but the implementation differ because of technical reasons.
So most of the code is the same except the core of the routine that is specialized to the integration.
Definition at line 561 of file grid_gpu_integrate.cu.
| __global__ grid_gpu_backend::__launch_bounds__ | ( | 64 | ) | const |
Separable (plane streaming) integrate kernel.
Computes the same coefficients as integrate_kernel, but with a cost per grid point linear in lp instead of proportional to ncoset(lp), and reading the cube only once whatever lp is.
It uses the same underlying method than the DGEMM backend which is a tensor contraction. Speedup comes from different areas:
Axis aligned cell (axis_aligned_ = true): r3.x, r3.y, r3.z only depend on x, y and z respectively, so that
x^a y^b z^c exp(-zetp r^2) = [x^a e^{-zetp x^2}] [y^b e^{-zetp y^2}] [z^c e^{-zetp z^2}]
Non orthogonal cell (axis_aligned_ = false): the moments are accumulated in lattice monomials u^i v^j w^k (u, v, w are the lattice coordinates of the point relative to the gaussian center) with the full gaussian evaluated per point, and transformed to cartesian monomials at the end with the matrix built by build_lattice_to_cartesian_matrix. It is the same transformation as grid_transform_coef_jik_to_yxz in the dgemm backend, precomputed once per grid level.
For each z plane of the cube:
A[y][a] = sum_x V[z][y][x] px[a][x] (step A) B[a][b] = sum_y A[y][a] py[b][y] a + b <= lp (step B) acc[coset(a, b, c)] += B[a][b] pz[c] c <= lp - a - b (step C)
B only lives in registers and the reduction over z is a running sum in acc, so the shared memory does not depend on the z extent of the cube. The set of grid points visited is the same as for integrate_kernel (window masks, y / x trimming in the orthogonal case, sphere test).
Shared memory layout: mc = max cube size, ls = lp_max + 1: ex[mc], py[ls][mc], pz[ls], A[mc][ls], acc[ncoset(lp_max)] pz only holds the values of the current plane, ex is only used for an axis aligned cell.
| grid_gpu_backend::if | ( | ) |
|
static |
Selects the instance of integrate_separable_kernel for one value of lmax_.
Definition at line 1196 of file grid_gpu_integrate.cu.
|
static |
Selects the instance of integrate_separable_kernel with the smallest bucket of the angular momentum covering lp_max.
Definition at line 1209 of file grid_gpu_integrate.cu.
| __host__ __device__ __inline__ int grid_gpu_backend::ncoset | ( | const int | l | ) |
Number of Cartesian orbitals up to given angular momentum quantum.
Definition at line 149 of file grid_gpu_internal_header.h.
| __host__ __device__ __inline__ int grid_gpu_backend::coset | ( | int | lx, |
| int | ly, | ||
| int | lz ) |
Maps three angular momentum components to a single zero based index.
Definition at line 161 of file grid_gpu_internal_header.h.
Increase i'th component of given orbital angular momentum.
Definition at line 173 of file grid_gpu_internal_header.h.
Decrease i'th component of given orbital angular momentum.
Definition at line 182 of file grid_gpu_internal_header.h.
| __inline__ __device__ int grid_gpu_backend::idx | ( | const orbital | a | ) |
Return coset index of given orbital angular momentum.
Definition at line 191 of file grid_gpu_internal_header.h.
| __device__ __inline__ double grid_gpu_backend::power | ( | const double | x, |
| const int | expo ) |
Definition at line 195 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::prep_term | ( | const orbital | a, |
| const orbital | b, | ||
| const T | value, | ||
| const int | n, | ||
| T * | cab ) |
Adds given value to matrix element cab[idx(b)][idx(a)].
Definition at line 206 of file grid_gpu_internal_header.h.
|
inlinestatic |
Initializes the device's constant memory.
Definition at line 215 of file grid_gpu_internal_header.h.
| __inline__ __device__ unsigned int grid_gpu_backend::block_index | ( | ) |
Definition at line 240 of file grid_gpu_internal_header.h.
| __inline__ __device__ unsigned int grid_gpu_backend::thread_global_index | ( | ) |
Definition at line 245 of file grid_gpu_internal_header.h.
| __inline__ __host__ __device__ double3 grid_gpu_backend::compute_coordinates | ( | const double *__restrict__ | dh_, |
| const double | x, | ||
| const double | y, | ||
| const double | z ) |
Definition at line 250 of file grid_gpu_internal_header.h.
| __inline__ T grid_gpu_backend::compute_cube_properties | ( | const T | radius, |
| const T *const __restrict__ | dh_, | ||
| const T *const __restrict__ | dh_inv_, | ||
| const T3 & | rp, | ||
| T3 & | roffset, | ||
| int3 & | cubecenter, | ||
| int3 & | lb_cube, | ||
| int3 & | cube_size ) |
Definition at line 265 of file grid_gpu_internal_header.h.
| __inline__ __device__ void grid_gpu_backend::compute_window_size | ( | const int3 | grid_size, |
| const int | border_mask, | ||
| const int3 & | border_width, | ||
| int3 & | window_size, | ||
| int3 & | window_shift ) |
Definition at line 404 of file grid_gpu_internal_header.h.
| __inline__ __device__ int grid_gpu_backend::wrap_grid_index | ( | const int | idx, |
| const int | full_size ) |
Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry shared by collocate_kernel and integrate_kernel.
Definition at line 436 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::setup_task_cube_center | ( | const kernel_params & | dev_, |
| smem_task_reduced< T, T3 > & | task ) |
Shifts task.cube_center into the local grid's coordinate system and, for distributed grids, derives the border window. Single point of entry shared by collocate_kernel and integrate_kernel so the two cannot drift apart on which axis maps to which.
Definition at line 451 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::decode_pair | ( | int | t, |
| int & | p, | ||
| int & | q ) |
Factorized cab <-> cxyz transformations.
cab_to_cxyz expands (x - a)^la (x - b)^lb around p in each direction.
It is the product of three steps: move the expansion center of the a
index from a to p, of the b index from b to p, then multiply the
polynomials,
(x - a)^n = sum_k C(n, k) (p - a)^k (x - p)^(n - k)
cxyz[c] = prefactor sum_{a' + b' = c} cab_p[b'][a']
The center moves are triangular, separable in x, y and z and map cab
onto itself, so they are done in place, one 1D line of cab per
thread. cxyz_to_cab is the transpose: a gather followed by the
transposed moves. The expansion coefficients alpha of the product are
never formed.
Definition at line 486 of file grid_gpu_internal_header.h.
| __device__ __inline__ int grid_gpu_backend::line_index | ( | const int | dir, |
| const int | m, | ||
| const int | p, | ||
| const int | q, | ||
| const bool | on_a, | ||
| const int | other, | ||
| const int | n1 ) |
Definition at line 496 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::move_center | ( | const int | la, |
| const int | lb, | ||
| const int | n1, | ||
| const int | dir, | ||
| const bool | on_a, | ||
| const T | dr, | ||
| T * | cab ) |
Moves the expansion center of the a (on_a) or b cartesian index of cab along direction dir, in place:
forward: new[m] = sum_{k >= 0} C(m + k, k) dr^k old[m + k] transpose: new[m] = sum_{k <= m} C(m, k) dr^k old[m - k]
Each thread owns a whole line (all other indices fixed). The forward update is a Taylor shift of the polynomial sum_m old[m] x^m by dr, done in place by repeated synthetic division (Ruffini-Horner): for i = 0 .. len-1, for j = len-1 down to i: c[j] += dr c[j+1] The transpose applies the transposed elementary updates in reverse order: for i = len-1 down to 0, for j = i .. len-1: c[j+1] += dr c[j] No binomial coefficient is needed, len (len + 1) / 2 FMA per line.
Definition at line 524 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::cab_to_cxyz_factorized | ( | const smem_task< T > & | task, |
| T *__restrict__ | cab, | ||
| T *__restrict__ | cxyz ) |
Transforms the coefficients C_ab into C_xyz, cxyz[c] = prefactor sum_{a, b} alpha[a][b][c] cab[b][a] (see cxyz_to_cab_factorized). cab is overwritten.
Definition at line 570 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::cxyz_to_cab_factorized | ( | const smem_task< T > & | task, |
| const T *__restrict__ | cxyz, | ||
| T *__restrict__ | cab ) |
Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.
cab[b][a] = prefactor sum_c alpha[a][b][c] cxyz[c]
where alpha expands the product of the two cartesian monomials around p, (x - A)^a (x - B)^b = sum_c alpha[a][b][c] (x - P)^c. In each direction, with dA = P - A and dB = P - B,
(x - A)^a = sum_{a' <= a} TA[a][a'] (x - P)^a', TA[a][a'] = C(a, a') dA^(a - a')
and similarly TB for b. Multiplying the two expansions,
alpha[a][b][c] = sum_{a' + b' = c} TA[a][a'] TB[b][b'],
hence
cab[b][a] = sum_{a', b'} TB[b][b'] TA[a][a'] G[b'][a'], G[b'][a'] = prefactor cxyz[a' + b'].
The sum is done in three steps without forming alpha:
TA and TB are lower triangular, so both steps map cab onto itself. cab_to_cxyz_factorized is the transpose of the same product: TA^T and TB^T (move_center<T, false>), then the scatter cxyz[c] = prefactor sum_{a' + b' = c} cab[b'][a'].
Definition at line 642 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::fill_smem_task_reduced | ( | const kernel_params & | dev, |
| const int | task_id, | ||
| smem_task_reduced< T, T3 > & | task ) |
Copies a task from global to shared memory.
Definition at line 682 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::fill_smem_task_coef | ( | const kernel_params & | dev, |
| const int | task_id, | ||
| smem_task< T > & | task ) |
Copies a task from global to shared memory and does precomputations.
Definition at line 736 of file grid_gpu_internal_header.h.
|
inline |
Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calculate_coefficients_two_steps and compute_hab_two_steps: [max_nsgf_set][ld] and [ld][ld] with ld = ncoset(lmax).
Definition at line 824 of file grid_gpu_internal_header.h.
|
inline |
Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memory together must fit in budget bytes. The static part (e.g. shared task structures) is taken from the compiled kernel and rounded up to 16 bytes for the alignment of the dynamic array placed after it.
Definition at line 842 of file grid_gpu_internal_header.h.
| __inline__ __device__ T * grid_gpu_backend::allocate_workspace | ( | const kernel_params & | dev_ | ) |
Definition at line 910 of file grid_gpu_internal_header.h.
| __device__ __inline__ T grid_gpu_backend::calculate_ymix_ymax_boundaries | ( | smem_task_reduced< T, T3 > & | task, |
| const int | z, | ||
| int & | ymin, | ||
| int & | ymax ) |
Definition at line 917 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::calculate_xmin_xmax_boundaries | ( | smem_task_reduced< T, T3 > & | task, |
| const int | y, | ||
| const T | kremain, | ||
| int & | xmin, | ||
| int & | xmax ) |
Definition at line 933 of file grid_gpu_internal_header.h.
| __device__ __inline__ void grid_gpu_backend::prepare_pab_AB | ( | const orbital | a, |
| const orbital | b, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_AB, ie. identity transformation.
Definition at line 28 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ void grid_gpu_backend::prepare_pab_DADB | ( | const orbital | a, |
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_DADB.
Definition at line 41 of file grid_gpu_prepare_pab.h.
| __device__ void grid_gpu_backend::prepare_pab_ADBmDAB | ( | const int | idir, |
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_ADBmDAB_{X,Y,Z}.
Definition at line 62 of file grid_gpu_prepare_pab.h.
| __device__ void grid_gpu_backend::prepare_pab_ARDBmDARB | ( | const int | idir, |
| const int | ir, | ||
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_ARDBmDARB_{X,Y,Z}{X,Y,Z}.
Definition at line 83 of file grid_gpu_prepare_pab.h.
| __device__ void grid_gpu_backend::prepare_pab_DABpADB | ( | const int | idir, |
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_DABpADB_{X,Y,Z}.
Definition at line 108 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ void grid_gpu_backend::prepare_pab_Di | ( | const int | ider, |
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_{DX,DY,DZ}.
Definition at line 131 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ void grid_gpu_backend::oneterm_dijdij | ( | const int | idir, |
| const T | func_a, | ||
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zetb, | ||
| const int | n, | ||
| T * | cab ) |
Helper for grid_prepare_pab_DiDj.
Definition at line 155 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ void grid_gpu_backend::prepare_pab_DiDj | ( | const int | ider1, |
| const int | ider2, | ||
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_{DXDY,DYDZ,DZDX}.
Definition at line 185 of file grid_gpu_prepare_pab.h.
| __device__ void grid_gpu_backend::oneterm_diidii | ( | const int | idir, |
| const T | func_a, | ||
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zetb, | ||
| const int | n, | ||
| T * | cab ) |
Helper for grid_prepare_pab_Di2.
Definition at line 212 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ void grid_gpu_backend::prepare_pab_Di2 | ( | const int | ider, |
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Implementation of function GRID_FUNC_{DXDX,DYDY,DZDZ}.
Definition at line 227 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ void grid_gpu_backend::prepare_pab | ( | const enum grid_func | func, |
| const orbital | a, | ||
| const orbital | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | pab_val, | ||
| const int | n, | ||
| T * | cab ) |
Transforms a given element of the density matrix according to func.
| func | Transformation function to apply, one of GRID_FUNC_*. |
| {a,b} | Orbital angular momenta. |
| zet_{a,b} | Gaussian exponents. |
| pab_val | Input matrix element of pab. |
| n | Leading dimensions of output matrix cab. |
| cab | Output matrix. |
Definition at line 255 of file grid_gpu_prepare_pab.h.
|
inline |
Returns difference in angular momentum range for given func.
Definition at line 347 of file grid_gpu_prepare_pab.h.
| __device__ __inline__ T grid_gpu_backend::get_term | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | n, | ||
| const T * | cab ) |
Returns matrix element cab[idx(b)][idx(a)].
Definition at line 27 of file grid_gpu_process_vab.h.
| __device__ __inline__ T grid_gpu_backend::get_force_a_normal | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const T | zeta, | ||
| const int | n, | ||
| const T * | cab ) |
Returns i'th component of force on atom a for compute_tau=false.
Definition at line 37 of file grid_gpu_process_vab.h.
| __device__ __inline__ double grid_gpu_backend::get_force_a | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const int | n, | ||
| const T * | cab ) |
Returns i'th component of force on atom a.
Definition at line 51 of file grid_gpu_process_vab.h.
| __device__ __inline__ double grid_gpu_backend::get_force_b_normal | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const T | zetb, | ||
| const T | rab[3], | ||
| const int | n, | ||
| const T * | cab ) |
Returns i'th component of force on atom b for compute_tau=false.
Definition at line 81 of file grid_gpu_process_vab.h.
| __device__ __inline__ T grid_gpu_backend::get_force_b | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | rab[3], | ||
| const int | n, | ||
| const T * | cab ) |
Returns i'th component of force on atom b.
Definition at line 94 of file grid_gpu_process_vab.h.
| __device__ __inline__ double grid_gpu_backend::get_virial_a_normal | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const int | j, | ||
| const T | zeta, | ||
| const int | n, | ||
| const T * | cab ) |
Returns element i,j of virial on atom a for compute_tau=false.
Definition at line 125 of file grid_gpu_process_vab.h.
| __device__ __inline__ T grid_gpu_backend::get_virial_a | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const int | j, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const int | n, | ||
| const T * | cab ) |
Returns element i,j of virial on atom a.
Definition at line 136 of file grid_gpu_process_vab.h.
| __device__ __inline__ double grid_gpu_backend::get_virial_b_normal | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const int | j, | ||
| const T | zetb, | ||
| const T | rab[3], | ||
| const int | n, | ||
| const T * | cab ) |
Returns element i,j of virial on atom b for compute_tau=false.
Definition at line 168 of file grid_gpu_process_vab.h.
| __device__ __inline__ double grid_gpu_backend::get_virial_b | ( | const orbital & | a, |
| const orbital & | b, | ||
| const int | i, | ||
| const int | j, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const T | rab[3], | ||
| const int | n, | ||
| const T * | cab ) |
Returns element i,j of virial on atom b.
Definition at line 186 of file grid_gpu_process_vab.h.
| __device__ __inline__ T grid_gpu_backend::get_hab | ( | const orbital & | a, |
| const orbital & | b, | ||
| const T | zeta, | ||
| const T | zetb, | ||
| const int | n, | ||
| const T * | cab ) |
Returns element i,j of hab matrix.
Definition at line 217 of file grid_gpu_process_vab.h.
|
inline |
Returns difference in angular momentum range for given flags.
Definition at line 241 of file grid_gpu_process_vab.h.
| __device__ __inline__ int grid_gpu_backend::sep_advance_wrapped_index | ( | const int | x2, |
| const int | n ) |
Definition at line 54 of file grid_gpu_separable.h.
|
inline |
|
static |
Initializes the constant memory of integrate_separable_kernel.
Definition at line 87 of file grid_gpu_separable.h.
|
static |
Builds the matrix expanding the cartesian monomials x^a y^b z^c in lattice monomials u^i v^j w^k, where x = dh[0] u + dh[3] v + dh[6] w, y = dh[1] u + dh[4] v + dh[7] w, z = dh[2] u + dh[5] v + dh[8] w. Only the blocks a + b + c = i + j + k = l are non zero. They are stored one after the other for l = 0..lmax, block l being row major of size nl x nl, nl = (l + 1)(l + 2) / 2, rows and columns in coset order.
With H[d][e] = dh[3 e + d] and p = (a, b, c), the entries are given by the multinomial expansion of prod_d (sum_e H[d][e] u_e)^p_d,
M[(abc)][(ijk)] = sum_n prod_d p_d! / (n_d0! n_d1! n_d2!) prod_e H[d][e]^n_de
the sum running over the 3x3 non negative integer matrices n with row sums (a, b, c) and column sums (i, j, k). Block l is the symmetric power Sym^l(H) of the linear map in the monomial basis, so that M(H1 H2) = M(H1) M(H2) and M(H)^-1 = M(H^-1).
The factorials are never evaluated. Each row is built by multiplying the polynomial 1 by one linear form at a time,
x_d u^i v^j w^k = H[d][0] u^(i+1) v^j w^k + H[d][1] u^i v^(j+1) w^k
a times with d = 0, b times with d = 1 and c times with d = 2, the multinomial coefficients being generated by the repeated additions as in Pascal's triangle. Equivalently, for a > 0 (similarly for b, c with rows 1, 2 of H),
M[(abc)][(ijk)] = H[0][0] M[(a-1,b,c)][(i-1,j,k)]
Integrate maps lattice moments to cartesian moments with M, collocate maps cartesian coefficients to lattice coefficients with M^T. The matrix is built on the host once per grid level and cached.
Definition at line 150 of file grid_gpu_separable.h.
|
static |
Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the lattice vectors changed or a higher angular momentum is needed.
Definition at line 196 of file grid_gpu_separable.h.
| __global__ const int grid_gpu_backend::ld |
Definition at line 130 of file grid_gpu_collocate.cu.
| __global__ const int const int grid_gpu_backend::max_nsgf_set |
Definition at line 130 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::number_of_tasks = dev_.num_tasks_per_block_dev[block_index()] |
Definition at line 132 of file grid_gpu_collocate.cu.
| T* grid_gpu_backend::smem_t = &shared_memory[0] |
Definition at line 137 of file grid_gpu_collocate.cu.
| T* grid_gpu_backend::smem_p = smem_t + max_nsgf_set * ld |
Definition at line 138 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::tid = thread_global_index() |
Definition at line 139 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::nthreads = blockDim.x * blockDim.y * blockDim.z |
Definition at line 140 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::offset = dev_.sorted_blocks_offset_dev[block_index()] |
Definition at line 141 of file grid_gpu_collocate.cu.
| T *__restrict__ grid_gpu_backend::smem_cab |
Definition at line 142 of file grid_gpu_collocate.cu.
| __global__ const int grid_gpu_backend::mc |
Definition at line 258 of file grid_gpu_collocate.cu.
| __global__ const int const int grid_gpu_backend::lp_max |
Definition at line 258 of file grid_gpu_collocate.cu.
| __global__ const int const int const int grid_gpu_backend::erows |
Definition at line 258 of file grid_gpu_collocate.cu.
| __global__ const int const int const int const double* __restrict__ grid_gpu_backend::M |
Definition at line 259 of file grid_gpu_collocate.cu.
| const task_info & grid_gpu_backend::glb_task = dev_.tasks[dev_.first_task + block_index()] |
Definition at line 264 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::ls = lp_max + 1 |
Definition at line 269 of file grid_gpu_collocate.cu.
|
extern |
Definition at line 964 of file grid_gpu_integrate.cu.
| double * grid_gpu_backend::ex = sep_smem |
Definition at line 272 of file grid_gpu_collocate.cu.
Definition at line 273 of file grid_gpu_collocate.cu.
Definition at line 274 of file grid_gpu_collocate.cu.
Definition at line 275 of file grid_gpu_collocate.cu.
Definition at line 276 of file grid_gpu_collocate.cu.
| __shared__ smem_task_reduced< double, double3 > grid_gpu_backend::task |
Definition at line 278 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::lp = task.lp |
Definition at line 285 of file grid_gpu_collocate.cu.
Definition at line 286 of file grid_gpu_collocate.cu.
| const double grid_gpu_backend::zetp = task.zetp |
Definition at line 287 of file grid_gpu_collocate.cu.
| const double grid_gpu_backend::r2max = task.radius * task.radius |
Definition at line 288 of file grid_gpu_collocate.cu.
| const bool grid_gpu_backend::trim = orthogonal_ && !task.apply_border_mask |
Definition at line 289 of file grid_gpu_collocate.cu.
| double * grid_gpu_backend::coef |
Definition at line 291 of file grid_gpu_collocate.cu.
Definition at line 325 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::row_group = tid / GRID_SEP_COLLOCATE_ROW_LANES |
Definition at line 326 of file grid_gpu_collocate.cu.
| const int grid_gpu_backend::lane = tid % GRID_SEP_COLLOCATE_ROW_LANES |
Definition at line 327 of file grid_gpu_collocate.cu.
|
constexpr |
Definition at line 328 of file grid_gpu_collocate.cu.
|
constexpr |
Definition at line 39 of file grid_gpu_integrate.cu.
|
constexpr |
Definition at line 57 of file grid_gpu_integrate.cu.
| T * grid_gpu_backend::smem_h = &shared_memory[0] |
Definition at line 165 of file grid_gpu_integrate.cu.
Definition at line 166 of file grid_gpu_integrate.cu.
Definition at line 415 of file grid_gpu_integrate.cu.
| const bool grid_gpu_backend::compute_virial = (dev_.buffers_dev.virial != nullptr) |
Definition at line 419 of file grid_gpu_integrate.cu.
| T grid_gpu_backend::fa[3] = {0.0, 0.0, 0.0} |
Definition at line 423 of file grid_gpu_integrate.cu.
| T grid_gpu_backend::fb[3] = {0.0, 0.0, 0.0} |
Definition at line 423 of file grid_gpu_integrate.cu.
| T grid_gpu_backend::virial[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} |
Definition at line 424 of file grid_gpu_integrate.cu.
| const int grid_gpu_backend::task_id = dev_.task_sorted_by_blocks_dev[offset] |
Definition at line 509 of file grid_gpu_integrate.cu.
| T* grid_gpu_backend::forces_a = &dev_.buffers_dev.forces[3 * glb_task.iatom] |
Definition at line 511 of file grid_gpu_integrate.cu.
| T* grid_gpu_backend::forces_b = &dev_.buffers_dev.forces[3 * glb_task.jatom] |
Definition at line 512 of file grid_gpu_integrate.cu.
| T* grid_gpu_backend::sum = (T *)shared_memory |
Definition at line 514 of file grid_gpu_integrate.cu.
| __global__ const int const int const double* __restrict__ grid_gpu_backend::lattice_to_cartesian |
Definition at line 943 of file grid_gpu_integrate.cu.
Definition at line 970 of file grid_gpu_integrate.cu.
Definition at line 973 of file grid_gpu_integrate.cu.
Definition at line 974 of file grid_gpu_integrate.cu.
Definition at line 975 of file grid_gpu_integrate.cu.
| grid_gpu_backend::else |
Definition at line 1165 of file grid_gpu_integrate.cu.
|
constexpr |
Definition at line 67 of file grid_gpu_internal_header.h.
| __constant__ orbital grid_gpu_backend::coset_inv[1330] |
Definition at line 69 of file grid_gpu_internal_header.h.
|
constexpr |
Definition at line 832 of file grid_gpu_internal_header.h.
|
constexpr |
Definition at line 74 of file grid_gpu_separable.h.
| __constant__ char2 grid_gpu_backend::sep_pair_ab[(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2] |
Definition at line 78 of file grid_gpu_separable.h.
| __constant__ int grid_gpu_backend::sep_m_offset[GRID_SEP_MAX_L+1] |
Definition at line 82 of file grid_gpu_separable.h.