35#error "OpenMP should not be used in .cu files to accommodate HIP."
41 return (n_elems + elem_alignment - 1) & ~(elem_alignment - 1);
45context_info::set_kernel_parameters(
const int level,
46 const smem_parameters &smem_params) {
48 params.cab_size_ = smem_params.cab_size();
49 params.first_task = 0;
51 params.la_min_diff = smem_params.ldiffs().la_min_diff;
52 params.lb_min_diff = smem_params.ldiffs().lb_min_diff;
54 params.la_max_diff = smem_params.ldiffs().la_max_diff;
55 params.lb_max_diff = smem_params.ldiffs().lb_max_diff;
61 params.la_min_diff = smem_params.ldiffs().la_min_diff;
62 params.lb_min_diff = smem_params.ldiffs().lb_min_diff;
63 params.la_max_diff = smem_params.ldiffs().la_max_diff;
64 params.lb_max_diff = smem_params.ldiffs().lb_max_diff;
69 params.buffers_dev.grid =
grid_[level].data();
70 memcpy(params.dh_,
grid_[level].dh(), 9 *
sizeof(
double));
71 memcpy(params.dh_inv_,
grid_[level].dh_inv(), 9 *
sizeof(
double));
74 params.grid_full_size_ =
grid_[level].full_size();
75 params.grid_local_size_ =
grid_[level].local_size();
76 params.grid_lower_corner_ =
grid_[level].lower_corner();
77 params.grid_border_width_ =
grid_[level].border_width();
80 params.buffers_dev.coef = this->
coef_dev_.data();
81 params.buffers_dev.hab_block =
hab_block_.data();
82 params.buffers_dev.forces =
forces_.data();
83 params.buffers_dev.virial =
virial_.data();
84 params.buffers_dev.cab = this->
cab_dev_.data();
86 params.sphi_dev = this->
sphi_dev.data();
102 const int lb_max_diff) {
106 std::vector<int> offsets(
nblocks, 0);
108 for (
int b = 0; b <
nblocks; b++) {
109 offsets[b] = (int)total;
116 if (total > (
size_t)INT_MAX) {
117 fprintf(stderr,
"ERROR: the cab workspace of the GPU grid backend is too "
118 "large for 32 bits offsets.\n");
122 cab_dev_.resize(std::max(total, (
size_t)1));
135 const bool ortho,
const int ntasks,
const int nlevels,
const int natoms,
136 const int nkinds,
const int nblocks,
const int *block_offsets,
137 const double *atom_positions,
const int *atom_kinds,
139 const int *iatom_list,
const int *jatom_list,
const int *iset_list,
140 const int *jset_list,
const int *ipgf_list,
const int *jpgf_list,
141 const int *border_mask_list,
const int *block_num_list,
142 const double *radius_list,
const double *rab_list,
const int *npts_global,
143 const int *
npts_local,
const int *shift_local,
const int *border_width,
150 if ((nblocks == 0) || (ntasks == 0)) {
157 if (*ctx_out ==
nullptr) {
171 ctx->
grid_.resize(nlevels);
174 std::vector<double> dh_max(ctx->
nlevels, 0);
176 for (
int level = 0; level < ctx->
nlevels; level++) {
177 ctx->
grid_[level].resize(npts_global + 3 * level,
npts_local + 3 * level,
178 shift_local + 3 * level, border_width + 3 * level);
179 ctx->
grid_[level].is_distributed(
false);
180 ctx->
grid_[level].set_lattice_vectors(&dh[9 * level], &dh_inv[9 * level]);
181 ctx->
grid_[level].check_orthogonality(ortho);
182 ctx->
grid_[level].max_cube_size(0);
183 for (
int i = 0;
i < 9;
i++)
184 dh_max[level] = std::max(dh_max[level], std::abs(dh[9 * level +
i]));
197 std::vector<grid_gpu_backend::task_info> tasks_host(ntasks);
199 size_t coef_size = 0;
202 for (
int i = 0;
i < ntasks;
i++) {
203 const int level = level_list[
i] - 1;
208 const int iatom = iatom_list[
i] - 1;
209 const int jatom = jatom_list[
i] - 1;
210 const int iset = iset_list[
i] - 1;
211 const int jset = jset_list[
i] - 1;
212 const int ipgf = ipgf_list[
i] - 1;
213 const int jpgf = jpgf_list[
i] - 1;
214 const int ikind = atom_kinds[iatom] - 1;
215 const int jkind = atom_kinds[jatom] - 1;
222 tasks_host[
i].level = level;
223 tasks_host[
i].iatom = iatom;
224 tasks_host[
i].jatom = jatom;
225 tasks_host[
i].iset = iset;
226 tasks_host[
i].jset = jset;
227 tasks_host[
i].ipgf = ipgf;
228 tasks_host[
i].jpgf = jpgf;
229 tasks_host[
i].ikind = ikind;
230 tasks_host[
i].jkind = jkind;
231 tasks_host[
i].border_mask = border_mask_list[
i];
232 tasks_host[
i].block_num = block_num_list[
i] - 1;
234 if (border_mask_list[
i]) {
235 ctx->
grid_[level].is_distributed(
true);
238 tasks_host[
i].radius = radius_list[
i];
239 tasks_host[
i].rab[0] = rab_list[3 *
i];
240 tasks_host[
i].rab[1] = rab_list[3 *
i + 1];
241 tasks_host[
i].rab[2] = rab_list[3 *
i + 2];
242 tasks_host[
i].zeta = ibasis->
zet[iset * ibasis->
maxpgf + ipgf];
243 tasks_host[
i].zetb = jbasis->
zet[jset * jbasis->
maxpgf + jpgf];
244 tasks_host[
i].zetp = tasks_host[
i].zeta + tasks_host[
i].zetb;
245 const double f = tasks_host[
i].zetb / tasks_host[
i].zetp;
246 tasks_host[
i].rab2 = 0.0;
247 for (
int d = 0; d < 3; d++) {
248 tasks_host[
i].rab[d] = tasks_host[
i].rab[d];
249 tasks_host[
i].rab2 += tasks_host[
i].rab[d] * tasks_host[
i].rab[d];
250 tasks_host[
i].ra[d] = atom_positions[3 * iatom + d];
251 tasks_host[
i].rb[d] = tasks_host[
i].ra[d] + tasks_host[
i].rab[d];
252 tasks_host[
i].rp[d] = tasks_host[
i].ra[d] + tasks_host[
i].rab[d] * f;
255 tasks_host[
i].skip_task = (2 * tasks_host[
i].radius < dh_max[level]);
256 tasks_host[
i].prefactor = exp(-tasks_host[
i].zeta * f * tasks_host[
i].rab2);
258 tasks_host[
i].off_diag_twice = (iatom == jatom) ? 1.0 : 2.0;
260 const int la_max_basis = ibasis->
lmax[iset];
261 const int lb_max_basis = jbasis->
lmax[jset];
262 const int la_min_basis = ibasis->
lmin[iset];
263 const int lb_min_basis = jbasis->
lmin[jset];
266 tasks_host[
i].la_max = la_max_basis;
267 tasks_host[
i].lb_max = lb_max_basis;
268 tasks_host[
i].la_min = la_min_basis;
269 tasks_host[
i].lb_min = lb_min_basis;
271 lmax_ = std::max(lmax_, tasks_host[
i].la_max);
272 lmax_ = std::max(lmax_, tasks_host[
i].lb_max);
275 tasks_host[
i].first_coseta =
277 tasks_host[
i].first_cosetb =
284 tasks_host[
i].nsgfa = ibasis->
nsgf;
285 tasks_host[
i].nsgfb = jbasis->
nsgf;
288 tasks_host[
i].nsgf_seta = ibasis->
nsgf_set[iset];
289 tasks_host[
i].nsgf_setb = jbasis->
nsgf_set[jset];
292 tasks_host[
i].maxcoa = ibasis->
maxco;
293 tasks_host[
i].maxcob = jbasis->
maxco;
295 tasks_host[
i].sgfa = ibasis->
first_sgf[iset] - 1;
296 tasks_host[
i].sgfb = jbasis->
first_sgf[jset] - 1;
298 tasks_host[
i].block_transposed = (iatom > jatom);
299 tasks_host[
i].subblock_offset =
300 (tasks_host[
i].block_transposed)
301 ? (tasks_host[
i].sgfa * tasks_host[
i].nsgfb + tasks_host[
i].sgfb)
302 : (tasks_host[
i].sgfb * tasks_host[
i].nsgfa + tasks_host[
i].sgfa);
316 tasks_host[
i].lp_max =
317 std::min(tasks_host[
i].lb_max + tasks_host[
i].la_max + 6,
320 tasks_host[
i].coef_offset = 0;
322 tasks_host[
i].coef_offset =
323 tasks_host[
i - 1].coef_offset +
335 tasks_host[
i].apply_border_mask = (tasks_host[
i].border_mask != 0);
337 if (
grid.is_orthogonal() && (tasks_host[
i].border_mask == 0)) {
338 tasks_host[
i].discrete_radius =
340 tasks_host[
i].radius,
grid.dh(),
grid.dh_inv(),
341 make_double3(tasks_host[
i].rp[0], tasks_host[
i].rp[1],
342 tasks_host[
i].rp[2]),
345 tasks_host[
i].cube_center,
346 tasks_host[
i].lb_cube,
347 tasks_host[
i].cube_size);
349 tasks_host[
i].discrete_radius =
351 tasks_host[
i].radius,
grid.dh(),
grid.dh_inv(),
352 make_double3(tasks_host[
i].rp[0], tasks_host[
i].rp[1],
353 tasks_host[
i].rp[2]),
356 tasks_host[
i].cube_center,
357 tasks_host[
i].lb_cube,
358 tasks_host[
i].cube_size);
361 const int3 &cube_size = tasks_host[
i].cube_size;
363 std::max(
grid.max_cube_size(),
364 std::max(cube_size.x, std::max(cube_size.y, cube_size.z))));
377 ctx->
tasks_dev.resize(tasks_host.size());
381 std::vector<std::vector<int>> task_sorted_by_block(nblocks);
382 std::vector<int> sorted_blocks(ntasks, 0);
383 std::vector<int> num_tasks_per_block(nblocks, 0);
384 std::vector<int> sorted_blocks_offset(nblocks, 0);
385 for (
auto &block : task_sorted_by_block)
388 for (
int i = 0;
i < ntasks;
i++) {
389 task_sorted_by_block[block_num_list[
i] - 1].push_back(
i);
390 num_tasks_per_block[block_num_list[
i] - 1]++;
395 for (
int i = 0;
i < (int)task_sorted_by_block.size();
i++) {
396 auto &task_list = task_sorted_by_block[
i];
400 if (!task_list.empty()) {
401 memcpy(&sorted_blocks[offset], &task_list[0],
402 sizeof(
int) * task_list.size());
404 sorted_blocks_offset[
i] = offset;
405 offset += task_list.size();
421 for (
int i = 0;
i < (int)sorted_blocks_offset.size();
i++) {
423 if (
i == (
int)sorted_blocks_offset.size() - 1)
424 num_tasks = ntasks - sorted_blocks_offset[
i];
426 num_tasks = sorted_blocks_offset[
i + 1] - sorted_blocks_offset[
i];
429 assert(num_tasks == num_tasks_per_block[
i]);
431 for (
int tk = 0; tk < num_tasks; tk++) {
433 tasks_host[sorted_blocks[tk + sorted_blocks_offset[
i]]];
435 assert(task.block_num ==
i);
444 for (
auto &block : task_sorted_by_block)
446 task_sorted_by_block.clear();
448 sorted_blocks.clear();
449 sorted_blocks_offset.clear();
458 memset(ctx->
stats, 0, 2 * 20 *
sizeof(
int));
459 for (
int itask = 0; itask < ntasks; itask++) {
460 const int iatom = iatom_list[itask] - 1;
461 const int jatom = jatom_list[itask] - 1;
462 const int ikind = atom_kinds[iatom] - 1;
463 const int jkind = atom_kinds[jatom] - 1;
464 const int iset = iset_list[itask] - 1;
465 const int jset = jset_list[itask] - 1;
466 const int la_max = basis_sets[ikind]->
lmax[iset];
467 const int lb_max = basis_sets[jkind]->
lmax[jset];
468 const int lp = std::min(la_max + lb_max, 19);
469 const bool has_border_mask = (border_mask_list[itask] != 0);
470 ctx->
stats[has_border_mask][lp]++;
516 assert(ctx->
nlevels == nlevels);
520 pab_blocks->
size /
sizeof(
double));
534 for (
int level = 0; level < ctx->
nlevels; level++) {
535 ctx->
grid_[level].associate(grids[level]->host_buffer,
536 grids[level]->device_buffer,
537 grids[level]->size /
sizeof(
double));
543 for (
int level = 0; level < ctx->
nlevels; level++) {
548 for (
int level = 0; level < ctx->
nlevels; level++) {
559 for (
int has_border_mask = 0; has_border_mask <= 1; has_border_mask++) {
560 for (
int lp = 0; lp < 20; lp++) {
561 const int count = ctx->
stats[has_border_mask][lp];
562 if (ctx->
grid_[0].is_orthogonal() && !has_border_mask) {
574 for (
int level = 0; level < ctx->
nlevels; level++) {
592 assert(ctx->
nlevels == nlevels);
598 for (
int level = 0; level < ctx->
nlevels; level++) {
600 ctx->
grid_[level].associate(grids[level]->host_buffer,
601 grids[level]->device_buffer,
602 grids[level]->size /
sizeof(
double));
607 if ((forces !=
nullptr) || (virial !=
nullptr)) {
610 pab_blocks->
size /
sizeof(
double));
617 hab_blocks->
size /
sizeof(
double));
624 if (forces !=
nullptr) {
629 if (virial !=
nullptr) {
638 for (
int level = 0; level < ctx->
nlevels; level++) {
645 for (
int has_border_mask = 0; has_border_mask <= 1; has_border_mask++) {
646 for (
int lp = 0; lp < 20; lp++) {
647 const int count = ctx->
stats[has_border_mask][lp];
648 if (ctx->
grid_[0].is_orthogonal() && !has_border_mask) {
660 for (
int level = 0; level < ctx->
nlevels; level++) {
668 if (forces != NULL) {
671 if (virial != NULL) {
gpu_vector< double * > sphi_dev
std::vector< int > block_lb_max_
gpu_vector< int > block_offsets_dev
void initialize_basis_sets(const grid_basis_set **basis_sets, const int nkinds__)
std::vector< offloadStream_t > level_streams
std::vector< int > block_la_max_
gpu_vector< double > virial_
void synchronize(offloadStream_t &stream)
offloadStream_t main_stream
gpu_vector< int > task_sorted_by_blocks_dev
void compute_hab_coefficients()
std::vector< int > number_of_tasks_per_level_
gpu_vector< double > pab_block_
gpu_vector< double > cab_dev_
gpu_vector< task_info > tasks_dev
gpu_vector< double > coef_dev_
gpu_vector< int > cab_block_offset_dev
void calculate_all_coefficients(const enum grid_func func, int *lp_diff)
gpu_vector< double > forces_
void collocate_one_grid_level(const int level, const enum grid_func func, int *lp_diff)
Launches the Cuda kernel that collocates all tasks of one grid level.
void integrate_one_grid_level(const int level, int *lp_diff)
Launches the Cuda kernel that integrates all tasks of one grid level.
void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff)
Lays out the cab workspace of the coefficient kernels for the angular momentum offsets of the current...
gpu_vector< int > sorted_blocks_offset_dev
std::vector< grid_info< double > > grid_
std::vector< int > first_task_per_level_
gpu_vector< int > num_tasks_per_block_dev_
gpu_vector< double > hab_block_
void copy_associated_host_to_gpu(offloadStream_t &stream__)
void copy_gpu_to_associated_host(offloadStream_t &stream__)
void associate(void *host_ptr__, void *device_ptr__, const size_t size__)
void copy_to_gpu(const T *data__)
void resize(const size_t new_size__)
void copy_from_gpu(T *data__, offloadStream_t &stream__)
void zero(offloadStream_t &stream__)
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i
static void const int const int const int const int const int const double const int const int const int npts_local[3]
void grid_gpu_collocate_task_list(const grid_gpu_task_list *ptr, const enum grid_func func, const int nlevels, const offload_buffer *pab_blocks, offload_buffer **grids)
Collocate all tasks of in given list onto given grids.
void grid_gpu_create_task_list(const bool ortho, const int ntasks, const int nlevels, const int natoms, const int nkinds, const int nblocks, const int *block_offsets, const double *atom_positions, const int *atom_kinds, const grid_basis_set **basis_sets, const int *level_list, const int *iatom_list, const int *jatom_list, const int *iset_list, const int *jset_list, const int *ipgf_list, const int *jpgf_list, const int *border_mask_list, const int *block_num_list, const double *radius_list, const double *rab_list, const int *npts_global, const int *npts_local, const int *shift_local, const int *border_width, const double *dh, const double *dh_inv, grid_gpu_task_list **ptr)
Allocates a task list for the GPU backend. See grid_ctx.h for details.
void grid_gpu_free_task_list(grid_gpu_task_list *ptr)
destroy a context
void grid_gpu_integrate_task_list(const grid_gpu_task_list *ptr, const bool compute_tau, const int nlevels, const offload_buffer *pab_blocks, const offload_buffer **grids, offload_buffer *hab_blocks, double *forces, double *virial)
Integrate all tasks of in given list onto given grids. See grid_ctx.h for details.
void grid_library_counter_add(const int lp, const enum grid_backend backend, const enum grid_library_kernel kernel, const int increment)
Adds given increment to counter specified by lp, backend, and kernel.
constexpr size_t align_up_elems(size_t n_elems, size_t elem_alignment)
__inline__ T compute_cube_properties(const T radius, const T *const __restrict__ dh_, const T *const __restrict__ dh_inv_, const T3 &rp, T3 &roffset, int3 &cubecenter, int3 &lb_cube, int3 &cube_size)
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
constexpr int grid_gpu_max_l
int offload_get_chosen_device(void)
Returns the chosen device.
Internal representation of a basis set.
Internal representation of a buffer.