(git:27c966b)
Loading...
Searching...
No Matches
grid_gpu_context.cu
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7
8/*
9 * Authors :
10 - Mathieu Taillefumier (ETH Zurich / CSCS)
11 - Advanced Micro Devices, Inc.
12 - Ole Schuett
13*/
14
15#include <algorithm>
16#include <cassert>
17#include <cstdio>
18#include <cstdlib>
19#include <cstring>
20#include <iostream>
21
23extern "C" {
27}
28
29#include "grid_gpu_context.h"
31
32#include "grid_gpu_task_list.h"
33
34#if defined(_OMP_H)
35#error "OpenMP should not be used in .cu files to accommodate HIP."
36#endif
37
38namespace grid_gpu_backend {
39
40constexpr size_t align_up_elems(size_t n_elems, size_t elem_alignment) {
41 return (n_elems + elem_alignment - 1) & ~(elem_alignment - 1);
42}
43
44kernel_params
45context_info::set_kernel_parameters(const int level,
46 const smem_parameters &smem_params) {
47 kernel_params params;
48 params.cab_size_ = smem_params.cab_size();
49 params.first_task = 0;
50
51 params.la_min_diff = smem_params.ldiffs().la_min_diff;
52 params.lb_min_diff = smem_params.ldiffs().lb_min_diff;
53
54 params.la_max_diff = smem_params.ldiffs().la_max_diff;
55 params.lb_max_diff = smem_params.ldiffs().lb_max_diff;
56 params.tasks = this->tasks_dev.data();
57 params.task_sorted_by_blocks_dev = task_sorted_by_blocks_dev.data();
58 params.sorted_blocks_offset_dev = sorted_blocks_offset_dev.data();
59 params.num_tasks_per_block_dev = this->num_tasks_per_block_dev_.data();
60 params.block_offsets = this->block_offsets_dev.data();
61 params.la_min_diff = smem_params.ldiffs().la_min_diff;
62 params.lb_min_diff = smem_params.ldiffs().lb_min_diff;
63 params.la_max_diff = smem_params.ldiffs().la_max_diff;
64 params.lb_max_diff = smem_params.ldiffs().lb_max_diff;
65
66 params.buffers_dev.pab_block = pab_block_.data();
67
68 if (level >= 0) {
69 params.buffers_dev.grid = grid_[level].data();
70 memcpy(params.dh_, grid_[level].dh(), 9 * sizeof(double));
71 memcpy(params.dh_inv_, grid_[level].dh_inv(), 9 * sizeof(double));
72 params.first_task = first_task_per_level_[level];
73
74 params.grid_full_size_ = grid_[level].full_size();
75 params.grid_local_size_ = grid_[level].local_size();
76 params.grid_lower_corner_ = grid_[level].lower_corner();
77 params.grid_border_width_ = grid_[level].border_width();
78 }
79
80 params.buffers_dev.coef = this->coef_dev_.data();
81 params.buffers_dev.hab_block = hab_block_.data();
82 params.buffers_dev.forces = forces_.data();
83 params.buffers_dev.virial = virial_.data();
84 params.buffers_dev.cab = this->cab_dev_.data();
85 params.cab_block_offset_dev = this->cab_block_offset_dev.data();
86 params.sphi_dev = this->sphi_dev.data();
87 return params;
88}
89
90/*******************************************************************************
91 * \brief Lays out the cab workspace of the coefficient kernels for the angular
92 * momentum offsets of the current operation: block b gets
93 * ncoset(la_max_b + la_max_diff) * ncoset(lb_max_b + lb_max_diff)
94 * elements, la_max_b / lb_max_b being the largest basis set angular
95 * momenta of its tasks. Must be called before set_kernel_parameters,
96 * when no coefficient kernel is running (the collocate and integrate
97 * entry points synchronize the main stream before returning). The
98 * allocation only grows, the layout is recomputed when the offsets
99 * change.
100 ******************************************************************************/
101void context_info::prepare_cab_workspace(const int la_max_diff,
102 const int lb_max_diff) {
103 if ((la_max_diff == cab_la_max_diff_) && (lb_max_diff == cab_lb_max_diff_))
104 return;
105
106 std::vector<int> offsets(nblocks, 0);
107 size_t total = 0;
108 for (int b = 0; b < nblocks; b++) {
109 offsets[b] = (int)total;
110 if (block_la_max_[b] >= 0)
111 total += align_up_elems(ncoset(block_la_max_[b] + la_max_diff) *
112 ncoset(block_lb_max_[b] + lb_max_diff),
113 4);
114 }
115
116 if (total > (size_t)INT_MAX) {
117 fprintf(stderr, "ERROR: the cab workspace of the GPU grid backend is too "
118 "large for 32 bits offsets.\n");
119 abort();
120 }
121
122 cab_dev_.resize(std::max(total, (size_t)1));
124 cab_block_offset_dev.copy_to_gpu(offsets);
125 cab_la_max_diff_ = la_max_diff;
126 cab_lb_max_diff_ = lb_max_diff;
127}
128}; // namespace grid_gpu_backend
129
130/*******************************************************************************
131 * \brief Allocates a task list for the GPU backend.
132 * See grid_ctx.h for details.
133 ******************************************************************************/
135 const bool ortho, const int ntasks, const int nlevels, const int natoms,
136 const int nkinds, const int nblocks, const int *block_offsets,
137 const double *atom_positions, const int *atom_kinds,
138 const grid_basis_set **basis_sets, const int *level_list,
139 const int *iatom_list, const int *jatom_list, const int *iset_list,
140 const int *jset_list, const int *ipgf_list, const int *jpgf_list,
141 const int *border_mask_list, const int *block_num_list,
142 const double *radius_list, const double *rab_list, const int *npts_global,
143 const int *npts_local, const int *shift_local, const int *border_width,
144 const double *dh, const double *dh_inv, grid_gpu_task_list **ptr) {
145
148
149 // Yes it makes no sense
150 if ((nblocks == 0) || (ntasks == 0)) {
151 *ptr = nullptr;
152 return;
153 }
154
155 // Select GPU device.
156 grid_gpu_backend::context_info *ctx = nullptr;
157 if (*ctx_out == nullptr) {
159 *ctx_out = ctx;
160 } else {
161 ctx = *ctx_out;
162 // verify that the object is the right one
163 ctx->verify_checksum();
164 }
165
166 ctx->ntasks = ntasks;
167 ctx->nlevels = nlevels;
168 ctx->natoms = natoms;
169 ctx->nblocks = nblocks;
170 ctx->nkinds = nkinds;
171 ctx->grid_.resize(nlevels);
172 ctx->set_device();
173
174 std::vector<double> dh_max(ctx->nlevels, 0);
175
176 for (int level = 0; level < ctx->nlevels; level++) {
177 ctx->grid_[level].resize(npts_global + 3 * level, npts_local + 3 * level,
178 shift_local + 3 * level, border_width + 3 * level);
179 ctx->grid_[level].is_distributed(false);
180 ctx->grid_[level].set_lattice_vectors(&dh[9 * level], &dh_inv[9 * level]);
181 ctx->grid_[level].check_orthogonality(ortho);
182 ctx->grid_[level].max_cube_size(0);
183 for (int i = 0; i < 9; i++)
184 dh_max[level] = std::max(dh_max[level], std::abs(dh[9 * level + i]));
185 }
186
187 ctx->block_offsets_dev.resize(nblocks);
188 ctx->block_offsets_dev.copy_to_gpu(block_offsets);
189 ctx->initialize_basis_sets(basis_sets, nkinds);
190
191 ctx->first_task_per_level_.resize(nlevels, 0);
192 ctx->number_of_tasks_per_level_.resize(nlevels, 0);
193
194 memset(ctx->first_task_per_level_.data(), 0, sizeof(int) * nlevels);
195 memset(ctx->number_of_tasks_per_level_.data(), 0, sizeof(int) * nlevels);
196
197 std::vector<grid_gpu_backend::task_info> tasks_host(ntasks);
198
199 size_t coef_size = 0;
200 int lmax_ = 0;
201
202 for (int i = 0; i < ntasks; i++) {
203 const int level = level_list[i] - 1;
204
205 // count the number of task per level
206 ctx->number_of_tasks_per_level_[level]++;
207
208 const int iatom = iatom_list[i] - 1;
209 const int jatom = jatom_list[i] - 1;
210 const int iset = iset_list[i] - 1;
211 const int jset = jset_list[i] - 1;
212 const int ipgf = ipgf_list[i] - 1;
213 const int jpgf = jpgf_list[i] - 1;
214 const int ikind = atom_kinds[iatom] - 1;
215 const int jkind = atom_kinds[jatom] - 1;
216
217 /* set parameters related to atom type orbital etc.... */
218 const grid_basis_set *ibasis = basis_sets[ikind];
219 const grid_basis_set *jbasis = basis_sets[jkind];
220
221 tasks_host[i] = {};
222 tasks_host[i].level = level;
223 tasks_host[i].iatom = iatom;
224 tasks_host[i].jatom = jatom;
225 tasks_host[i].iset = iset;
226 tasks_host[i].jset = jset;
227 tasks_host[i].ipgf = ipgf;
228 tasks_host[i].jpgf = jpgf;
229 tasks_host[i].ikind = ikind;
230 tasks_host[i].jkind = jkind;
231 tasks_host[i].border_mask = border_mask_list[i];
232 tasks_host[i].block_num = block_num_list[i] - 1;
233
234 if (border_mask_list[i]) {
235 ctx->grid_[level].is_distributed(true);
236 }
237 /* parameters for the gaussian */
238 tasks_host[i].radius = radius_list[i];
239 tasks_host[i].rab[0] = rab_list[3 * i];
240 tasks_host[i].rab[1] = rab_list[3 * i + 1];
241 tasks_host[i].rab[2] = rab_list[3 * i + 2];
242 tasks_host[i].zeta = ibasis->zet[iset * ibasis->maxpgf + ipgf];
243 tasks_host[i].zetb = jbasis->zet[jset * jbasis->maxpgf + jpgf];
244 tasks_host[i].zetp = tasks_host[i].zeta + tasks_host[i].zetb;
245 const double f = tasks_host[i].zetb / tasks_host[i].zetp;
246 tasks_host[i].rab2 = 0.0;
247 for (int d = 0; d < 3; d++) {
248 tasks_host[i].rab[d] = tasks_host[i].rab[d];
249 tasks_host[i].rab2 += tasks_host[i].rab[d] * tasks_host[i].rab[d];
250 tasks_host[i].ra[d] = atom_positions[3 * iatom + d];
251 tasks_host[i].rb[d] = tasks_host[i].ra[d] + tasks_host[i].rab[d];
252 tasks_host[i].rp[d] = tasks_host[i].ra[d] + tasks_host[i].rab[d] * f;
253 }
254
255 tasks_host[i].skip_task = (2 * tasks_host[i].radius < dh_max[level]);
256 tasks_host[i].prefactor = exp(-tasks_host[i].zeta * f * tasks_host[i].rab2);
257
258 tasks_host[i].off_diag_twice = (iatom == jatom) ? 1.0 : 2.0;
259 // angular momentum range of basis set
260 const int la_max_basis = ibasis->lmax[iset];
261 const int lb_max_basis = jbasis->lmax[jset];
262 const int la_min_basis = ibasis->lmin[iset];
263 const int lb_min_basis = jbasis->lmin[jset];
264
265 // angular momentum range for the actual collocate/integrate opteration.
266 tasks_host[i].la_max = la_max_basis;
267 tasks_host[i].lb_max = lb_max_basis;
268 tasks_host[i].la_min = la_min_basis;
269 tasks_host[i].lb_min = lb_min_basis;
270
271 lmax_ = std::max(lmax_, tasks_host[i].la_max);
272 lmax_ = std::max(lmax_, tasks_host[i].lb_max);
273
274 // start of decontracted set, ie. pab and hab
275 tasks_host[i].first_coseta =
276 (la_min_basis > 0) ? grid_gpu_backend::ncoset(la_min_basis - 1) : 0;
277 tasks_host[i].first_cosetb =
278 (lb_min_basis > 0) ? grid_gpu_backend::ncoset(lb_min_basis - 1) : 0;
279
280 // size of decontracted set, ie. pab and hab
281 tasks_host[i].ncoseta = grid_gpu_backend::ncoset(la_max_basis);
282 tasks_host[i].ncosetb = grid_gpu_backend::ncoset(lb_max_basis);
283 // size of entire spherical basis
284 tasks_host[i].nsgfa = ibasis->nsgf;
285 tasks_host[i].nsgfb = jbasis->nsgf;
286
287 // size of spherical set
288 tasks_host[i].nsgf_seta = ibasis->nsgf_set[iset];
289 tasks_host[i].nsgf_setb = jbasis->nsgf_set[jset];
290
291 // strides of the sphi transformation matrices
292 tasks_host[i].maxcoa = ibasis->maxco;
293 tasks_host[i].maxcob = jbasis->maxco;
294
295 tasks_host[i].sgfa = ibasis->first_sgf[iset] - 1;
296 tasks_host[i].sgfb = jbasis->first_sgf[jset] - 1;
297
298 tasks_host[i].block_transposed = (iatom > jatom);
299 tasks_host[i].subblock_offset =
300 (tasks_host[i].block_transposed)
301 ? (tasks_host[i].sgfa * tasks_host[i].nsgfb + tasks_host[i].sgfb)
302 : (tasks_host[i].sgfb * tasks_host[i].nsgfa + tasks_host[i].sgfa);
303
304 /* the constant 6 is important here since we do not know ahead of time what
305 * specific operation we will be doing. collocate functions can go up to 4
306 * while integrate can go up to 5 (but put 6 for safety reasons). It is
307 * capped at grid_gpu_max_l, the end of the ncoset table: an operation needs
308 * at most min(la + lb + 5, grid_gpu_max_l) since smem_parameters rejects
309 * anything larger. */
310
311 /* this block is only as temporary scratch for calculating the coefficients.
312 * Doing this avoid a lot of atomic operations that are costly on hardware
313 * that only have partial support of them. For better performance we should
314 * most probably align the offsets as well. it is 256 bytes on Mi100 and
315 * above */
316 tasks_host[i].lp_max =
317 std::min(tasks_host[i].lb_max + tasks_host[i].la_max + 6,
319 if (i == 0) {
320 tasks_host[i].coef_offset = 0;
321 } else {
322 tasks_host[i].coef_offset =
323 tasks_host[i - 1].coef_offset +
325 grid_gpu_backend::ncoset(tasks_host[i - 1].lp_max), 4);
326 }
327
328 // calculate the size such that the coef table is a multiple of 4.
330 grid_gpu_backend::ncoset(tasks_host[i].lp_max), 4);
331
332 auto &grid = ctx->grid_[tasks_host[i].level];
333 // compute the cube properties
334
335 tasks_host[i].apply_border_mask = (tasks_host[i].border_mask != 0);
336
337 if (grid.is_orthogonal() && (tasks_host[i].border_mask == 0)) {
338 tasks_host[i].discrete_radius =
340 tasks_host[i].radius, grid.dh(), grid.dh_inv(),
341 make_double3(tasks_host[i].rp[0], tasks_host[i].rp[1],
342 tasks_host[i].rp[2]), // center of the gaussian
343 tasks_host[i]
344 .roffset, // offset compared to the closest grid point
345 tasks_host[i].cube_center, // center coordinates in grid space
346 tasks_host[i].lb_cube, // lower boundary
347 tasks_host[i].cube_size);
348 } else {
349 tasks_host[i].discrete_radius =
351 tasks_host[i].radius, grid.dh(), grid.dh_inv(),
352 make_double3(tasks_host[i].rp[0], tasks_host[i].rp[1],
353 tasks_host[i].rp[2]), // center of the gaussian
354 tasks_host[i]
355 .roffset, // offset compared to the closest grid point
356 tasks_host[i].cube_center, // center coordinates in grid space
357 tasks_host[i].lb_cube, // lower boundary
358 tasks_host[i].cube_size);
359 }
360
361 const int3 &cube_size = tasks_host[i].cube_size;
362 grid.max_cube_size(
363 std::max(grid.max_cube_size(),
364 std::max(cube_size.x, std::max(cube_size.y, cube_size.z))));
365 }
366
367 // we need to sort the task list although I expect it to be sorted already
368 // it is a exclusive scan actually
369 for (int level = 1; level < (int)ctx->number_of_tasks_per_level_.size();
370 level++) {
371 ctx->first_task_per_level_[level] =
372 ctx->first_task_per_level_[level - 1] +
373 ctx->number_of_tasks_per_level_[level - 1];
374 }
375
376 ctx->tasks_dev.clear();
377 ctx->tasks_dev.resize(tasks_host.size());
378 ctx->tasks_dev.copy_to_gpu(tasks_host);
379
380 /* Sort the blocks */
381 std::vector<std::vector<int>> task_sorted_by_block(nblocks);
382 std::vector<int> sorted_blocks(ntasks, 0);
383 std::vector<int> num_tasks_per_block(nblocks, 0);
384 std::vector<int> sorted_blocks_offset(nblocks, 0);
385 for (auto &block : task_sorted_by_block)
386 block.clear();
387
388 for (int i = 0; i < ntasks; i++) {
389 task_sorted_by_block[block_num_list[i] - 1].push_back(i);
390 num_tasks_per_block[block_num_list[i] - 1]++;
391 }
392
393 int offset = 0;
394 // flatten the task_sorted_by_block and compute the offsets
395 for (int i = 0; i < (int)task_sorted_by_block.size(); i++) {
396 auto &task_list = task_sorted_by_block[i];
397
398 // take care of the case where the blocks are not associated to a given
399 // task. (and also a workaround in the grid_replay.c file)
400 if (!task_list.empty()) {
401 memcpy(&sorted_blocks[offset], &task_list[0],
402 sizeof(int) * task_list.size());
403 }
404 sorted_blocks_offset[i] = offset;
405 offset += task_list.size();
406 }
407
408 // copy the blocks offsets
409 ctx->sorted_blocks_offset_dev.resize(sorted_blocks_offset.size());
410 ctx->sorted_blocks_offset_dev.copy_to_gpu(sorted_blocks_offset);
411
412 // copy the task list sorted by block (not by level) to the gpu
413 ctx->task_sorted_by_blocks_dev.resize(sorted_blocks.size());
414 ctx->task_sorted_by_blocks_dev.copy_to_gpu(sorted_blocks);
415
416 // largest basis set angular momenta of each block. The cab workspace of a
417 // block is sized from them in prepare_cab_workspace, once the angular
418 // momentum offsets of the operation are known.
419 ctx->block_la_max_.assign(nblocks, -1);
420 ctx->block_lb_max_.assign(nblocks, -1);
421 for (int i = 0; i < (int)sorted_blocks_offset.size(); i++) {
422 int num_tasks = 0;
423 if (i == (int)sorted_blocks_offset.size() - 1)
424 num_tasks = ntasks - sorted_blocks_offset[i];
425 else
426 num_tasks = sorted_blocks_offset[i + 1] - sorted_blocks_offset[i];
427
428 // invariants tests since they should be equal.
429 assert(num_tasks == num_tasks_per_block[i]);
430
431 for (int tk = 0; tk < num_tasks; tk++) {
432 const auto &task =
433 tasks_host[sorted_blocks[tk + sorted_blocks_offset[i]]];
434 // check that all tasks point to the same block
435 assert(task.block_num == i);
436 ctx->block_la_max_[i] = std::max(ctx->block_la_max_[i], task.la_max);
437 ctx->block_lb_max_[i] = std::max(ctx->block_lb_max_[i], task.lb_max);
438 }
439 }
440 // the layout of the cab workspace has to be recomputed for this task list
441 ctx->cab_la_max_diff_ = -1;
442 ctx->cab_lb_max_diff_ = -1;
443
444 for (auto &block : task_sorted_by_block)
445 block.clear();
446 task_sorted_by_block.clear();
447
448 sorted_blocks.clear();
449 sorted_blocks_offset.clear();
450
451 ctx->num_tasks_per_block_dev_.resize(num_tasks_per_block.size());
452 ctx->num_tasks_per_block_dev_.copy_to_gpu(num_tasks_per_block);
453
454 // allocate workspace for the coefficients
455 ctx->coef_dev_.resize(coef_size);
456
457 // collect stats
458 memset(ctx->stats, 0, 2 * 20 * sizeof(int));
459 for (int itask = 0; itask < ntasks; itask++) {
460 const int iatom = iatom_list[itask] - 1;
461 const int jatom = jatom_list[itask] - 1;
462 const int ikind = atom_kinds[iatom] - 1;
463 const int jkind = atom_kinds[jatom] - 1;
464 const int iset = iset_list[itask] - 1;
465 const int jset = jset_list[itask] - 1;
466 const int la_max = basis_sets[ikind]->lmax[iset];
467 const int lb_max = basis_sets[jkind]->lmax[jset];
468 const int lp = std::min(la_max + lb_max, 19);
469 const bool has_border_mask = (border_mask_list[itask] != 0);
470 ctx->stats[has_border_mask][lp]++;
471 }
472
473 ctx->create_streams();
474 ctx->compute_checksum();
475
476 // cleanup
477 tasks_host.clear();
478
479 // return newly created or updated context
480 *ctx_out = ctx;
481}
482
483/*******************************************************************************
484 * \brief destroy a context
485 ******************************************************************************/
487
489 // Select GPU device.
490 if (ctx == nullptr)
491 return;
492 ctx->verify_checksum();
493 ctx->set_device();
494 delete ctx;
495}
496
497/*******************************************************************************
498 * \brief Collocate all tasks of in given list onto given grids.
499 ******************************************************************************/
501 const enum grid_func func,
502 const int nlevels,
503 const offload_buffer *pab_blocks,
504 offload_buffer **grids) {
506
507 if (ptr == nullptr)
508 return;
509
510 ctx->verify_checksum();
511
512 if ((ctx->nblocks == 0) || (ctx->ntasks == 0)) {
513 return;
514 }
515
516 assert(ctx->nlevels == nlevels);
517 ctx->set_device();
518
519 ctx->pab_block_.associate(pab_blocks->host_buffer, pab_blocks->device_buffer,
520 pab_blocks->size / sizeof(double));
521
522 /*
523 There are 3 scenario here.
524 - Mi300 : no copy will happen as the two buffers have the same address
525 - Mi250X : an internal copy will happen. We do not need to do anything
526 explicit
527 - no unified memory : Explicit copy will happen
528 */
529 int lp_diff = -1;
531 ctx->coef_dev_.zero(ctx->main_stream);
532 ctx->calculate_all_coefficients(func, &lp_diff);
533
534 for (int level = 0; level < ctx->nlevels; level++) {
535 ctx->grid_[level].associate(grids[level]->host_buffer,
536 grids[level]->device_buffer,
537 grids[level]->size / sizeof(double));
538 ctx->grid_[level].zero(ctx->level_streams[level]);
539 }
540
541 ctx->synchronize(ctx->main_stream);
542
543 for (int level = 0; level < ctx->nlevels; level++) {
544 ctx->collocate_one_grid_level(level, func, &lp_diff);
545 }
546
547 // download result from device to host.
548 for (int level = 0; level < ctx->nlevels; level++) {
549 ctx->grid_[level].copy_to_host(ctx->level_streams[level]);
550 }
551
552 // update counters while we wait for kernels to finish. It is not thread safe
553 // at all since the function grid_library_counter_add has global static
554 // states. We need a much better mechanism than this for instance move this
555 // information one level up and encapsulate it in the context associated to
556 // the library.
557
558 if (lp_diff > -1) {
559 for (int has_border_mask = 0; has_border_mask <= 1; has_border_mask++) {
560 for (int lp = 0; lp < 20; lp++) {
561 const int count = ctx->stats[has_border_mask][lp];
562 if (ctx->grid_[0].is_orthogonal() && !has_border_mask) {
564 GRID_COLLOCATE_ORTHO, count);
565 } else {
568 }
569 }
570 }
571 }
572
573 // need to wait for all streams to finish
574 for (int level = 0; level < ctx->nlevels; level++) {
575 ctx->synchronize(ctx->level_streams[level]);
576 }
577}
578
579/*******************************************************************************
580 * \brief Integrate all tasks of in given list onto given grids.
581 * See grid_ctx.h for details.
582 ******************************************************************************/
584 const grid_gpu_task_list *ptr, const bool compute_tau, const int nlevels,
585 const offload_buffer *pab_blocks, const offload_buffer **grids,
586 offload_buffer *hab_blocks, double *forces, double *virial) {
587
589
590 if (ptr == nullptr)
591 return;
592 assert(ctx->nlevels == nlevels);
593
594 ctx->verify_checksum();
595 // Select GPU device.
596 ctx->set_device();
597
598 for (int level = 0; level < ctx->nlevels; level++) {
599 if (ctx->number_of_tasks_per_level_[level]) {
600 ctx->grid_[level].associate(grids[level]->host_buffer,
601 grids[level]->device_buffer,
602 grids[level]->size / sizeof(double));
603 ctx->grid_[level].copy_to_gpu(ctx->level_streams[level]);
604 }
605 }
606
607 if ((forces != nullptr) || (virial != nullptr)) {
608 ctx->pab_block_.associate(pab_blocks->host_buffer,
609 pab_blocks->device_buffer,
610 pab_blocks->size / sizeof(double));
612 }
613
614 // we do not need to wait for this to start the computations since the matrix
615 // elements are computed after all coefficients are calculated.
616 ctx->hab_block_.associate(hab_blocks->host_buffer, hab_blocks->device_buffer,
617 hab_blocks->size / sizeof(double));
618 ctx->hab_block_.zero(ctx->main_stream);
619
620 ctx->calculate_forces = (forces != nullptr);
621 ctx->calculate_virial = (virial != nullptr);
622 ctx->compute_tau = compute_tau;
623
624 if (forces != nullptr) {
625 ctx->forces_.resize(3 * ctx->natoms);
626 ctx->forces_.zero(ctx->main_stream);
627 }
628
629 if (virial != nullptr) {
630 ctx->virial_.resize(9);
631 ctx->virial_.zero(ctx->main_stream);
632 }
633
634 int lp_diff = -1;
635
636 // we can actually treat the full task list without bothering about the level
637 // at that stage. This can be taken care of inside the kernel.
638 for (int level = 0; level < ctx->nlevels; level++) {
639 // launch kernel, but only after grid has arrived
640 ctx->integrate_one_grid_level(level, &lp_diff);
641 }
642
643 if (lp_diff > -1) {
644 // update counters while we wait for kernels to finish
645 for (int has_border_mask = 0; has_border_mask <= 1; has_border_mask++) {
646 for (int lp = 0; lp < 20; lp++) {
647 const int count = ctx->stats[has_border_mask][lp];
648 if (ctx->grid_[0].is_orthogonal() && !has_border_mask) {
650 GRID_INTEGRATE_ORTHO, count);
651 } else {
654 }
655 }
656 }
657 }
658
659 // need to wait for all streams to finish
660 for (int level = 0; level < ctx->nlevels; level++) {
661 ctx->synchronize(ctx->level_streams[level]);
662 }
663 // computing the hab coefficients does not depend on the number of grids so we
664 // can run these calculations on the main stream
667
668 if (forces != NULL) {
669 ctx->forces_.copy_from_gpu(forces, ctx->main_stream);
670 }
671 if (virial != NULL) {
672 ctx->virial_.copy_from_gpu(virial, ctx->main_stream);
673 }
674
675 ctx->synchronize(ctx->main_stream);
676}
gpu_vector< double * > sphi_dev
void initialize_basis_sets(const grid_basis_set **basis_sets, const int nkinds__)
std::vector< offloadStream_t > level_streams
void synchronize(offloadStream_t &stream)
gpu_vector< int > task_sorted_by_blocks_dev
std::vector< int > number_of_tasks_per_level_
gpu_vector< task_info > tasks_dev
void calculate_all_coefficients(const enum grid_func func, int *lp_diff)
void collocate_one_grid_level(const int level, const enum grid_func func, int *lp_diff)
Launches the Cuda kernel that collocates all tasks of one grid level.
void integrate_one_grid_level(const int level, int *lp_diff)
Launches the Cuda kernel that integrates all tasks of one grid level.
void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff)
Lays out the cab workspace of the coefficient kernels for the angular momentum offsets of the current...
gpu_vector< int > sorted_blocks_offset_dev
std::vector< grid_info< double > > grid_
std::vector< int > first_task_per_level_
gpu_vector< int > num_tasks_per_block_dev_
void copy_associated_host_to_gpu(offloadStream_t &stream__)
void copy_gpu_to_associated_host(offloadStream_t &stream__)
void associate(void *host_ptr__, void *device_ptr__, const size_t size__)
void copy_to_gpu(const T *data__)
void resize(const size_t new_size__)
void copy_from_gpu(T *data__, offloadStream_t &stream__)
void zero(offloadStream_t &stream__)
@ GRID_BACKEND_GPU
grid_func
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i
static void const int const int const int const int const int const double const int const int const int npts_local[3]
void grid_gpu_collocate_task_list(const grid_gpu_task_list *ptr, const enum grid_func func, const int nlevels, const offload_buffer *pab_blocks, offload_buffer **grids)
Collocate all tasks of in given list onto given grids.
void grid_gpu_create_task_list(const bool ortho, const int ntasks, const int nlevels, const int natoms, const int nkinds, const int nblocks, const int *block_offsets, const double *atom_positions, const int *atom_kinds, const grid_basis_set **basis_sets, const int *level_list, const int *iatom_list, const int *jatom_list, const int *iset_list, const int *jset_list, const int *ipgf_list, const int *jpgf_list, const int *border_mask_list, const int *block_num_list, const double *radius_list, const double *rab_list, const int *npts_global, const int *npts_local, const int *shift_local, const int *border_width, const double *dh, const double *dh_inv, grid_gpu_task_list **ptr)
Allocates a task list for the GPU backend. See grid_ctx.h for details.
void grid_gpu_free_task_list(grid_gpu_task_list *ptr)
destroy a context
void grid_gpu_integrate_task_list(const grid_gpu_task_list *ptr, const bool compute_tau, const int nlevels, const offload_buffer *pab_blocks, const offload_buffer **grids, offload_buffer *hab_blocks, double *forces, double *virial)
Integrate all tasks of in given list onto given grids. See grid_ctx.h for details.
void grid_gpu_task_list
void grid_library_counter_add(const int lp, const enum grid_backend backend, const enum grid_library_kernel kernel, const int increment)
Adds given increment to counter specified by lp, backend, and kernel.
@ GRID_INTEGRATE_GENERAL
@ GRID_COLLOCATE_ORTHO
@ GRID_COLLOCATE_GENERAL
@ GRID_INTEGRATE_ORTHO
constexpr size_t align_up_elems(size_t n_elems, size_t elem_alignment)
__inline__ T compute_cube_properties(const T radius, const T *const __restrict__ dh_, const T *const __restrict__ dh_inv_, const T3 &rp, T3 &roffset, int3 &cubecenter, int3 &lb_cube, int3 &cube_size)
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
int offload_get_chosen_device(void)
Returns the chosen device.
Internal representation of a basis set.
Internal representation of a buffer.
double * device_buffer
double * host_buffer