(git:27c966b)
Loading...
Searching...
No Matches
grid_gpu_context.h
Go to the documentation of this file.
1/*----------------------------------------------------------------------------*/
2/* CP2K: A general program to perform molecular dynamics simulations */
3/* Copyright 2000-2026 CP2K developers group <https://cp2k.org> */
4/* */
5/* SPDX-License-Identifier: BSD-3-Clause */
6/*----------------------------------------------------------------------------*/
7
8/*
9 * Authors :
10 - Dr Mathieu Taillefumier (ETH Zurich / CSCS)
11 - Advanced Micro Devices, Inc.
12*/
13
14#ifndef GRID_GPU_CONTEXT_H
15#define GRID_GPU_CONTEXT_H
16
17#ifdef __OFFLOAD_HIP
18#include <hip/hip_runtime_api.h>
19#else
20#include <cuda_runtime.h>
21#endif
22#include <array>
23#include <climits>
24#include <vector>
25
26extern "C" {
29}
30
33
34namespace grid_gpu_backend {
35// a little helper class in the same spirit than std::vector. it must exist
36// somewhere. Maybe possible to get the same thing with std::vector and
37// specific allocator.
38
39inline size_t round_up16(size_t n) { return (n + 15) & ~size_t(15); }
40
41class smem_parameters;
42template <typename T> class gpu_vector {
43 size_t allocated_size_{0};
44 size_t current_size_{0};
45 bool internal_allocation_{false};
46 T *device_ptr_{nullptr};
47 T *host_ptr_{nullptr};
48
49 void move_from(gpu_vector &&other) noexcept {
50 allocated_size_ = other.allocated_size_;
51 current_size_ = other.current_size_;
52 internal_allocation_ = other.internal_allocation_;
53 device_ptr_ = other.device_ptr_;
54 host_ptr_ = other.host_ptr_;
55
56 other.allocated_size_ = 0;
57 other.current_size_ = 0;
58 other.internal_allocation_ = false;
59 other.device_ptr_ = nullptr;
60 other.host_ptr_ = nullptr;
61 }
62
63public:
64 gpu_vector() = default;
65
66 gpu_vector(const gpu_vector &) = delete;
67 gpu_vector &operator=(const gpu_vector &) = delete;
68
69 gpu_vector(gpu_vector &&other) noexcept { move_from(std::move(other)); }
70
71 gpu_vector &operator=(gpu_vector &&other) noexcept {
72 if (this != &other) {
73 reset();
74 move_from(std::move(other));
75 }
76 return *this;
77 }
78
79 // size is the number of elements not the memory size
80 explicit gpu_vector(const size_t size__) {
81 allocated_size_ = (size__ < 16) ? 16 : round_up16(size__);
82 current_size_ = size__;
83 internal_allocation_ = true;
84
85#ifndef __OFFLOAD_UNIFIED_MEMORY
86 offloadMalloc((void **)&device_ptr_, sizeof(T) * allocated_size_);
87#else
88 hipMallocManaged((void **)&device_ptr_, sizeof(T) * allocated_size_);
89#endif
90 assert(device_ptr_ != nullptr);
91 }
92
93 gpu_vector(const size_t size__, void *ptr__) {
94 allocated_size_ = size__;
95 current_size_ = size__;
96 internal_allocation_ = false;
97 device_ptr_ = static_cast<T *>(ptr__);
98 }
100
101 inline size_t size() const { return current_size_; }
102
103 inline void copy_to_gpu(const T *data__) {
104 assert(device_ptr_ != nullptr);
105 assert(data__ != nullptr);
106 offloadMemcpyHtoD(device_ptr_, data__, sizeof(T) * current_size_);
107 }
108
109 inline void copy_to_gpu(const T *data__, offloadStream_t &stream__) {
110 assert(device_ptr_ != nullptr);
111 assert(data__ != nullptr);
112 offloadMemcpyAsyncHtoD(device_ptr_, data__, sizeof(T) * current_size_,
113 stream__);
114 }
115
116 inline void copy_associated_host_to_gpu(offloadStream_t &stream__) {
117 assert(device_ptr_ != nullptr);
118 assert(host_ptr_ != nullptr);
119 // If the second assert fails it means that the object was created without
120 // host buffer. It should not happen in the current scenario
121
122 offloadMemcpyAsyncHtoD(device_ptr_, host_ptr_, sizeof(T) * current_size_,
123 stream__);
124 }
125
126 inline void copy_from_gpu(T *data__, offloadStream_t &stream__) {
127 assert(device_ptr_ != nullptr);
128 assert(data__ != nullptr);
129 offloadMemcpyAsyncDtoH(data__, device_ptr_, sizeof(T) * current_size_,
130 stream__);
131 }
132
133 inline void copy_gpu_to_associated_host(offloadStream_t &stream__) {
134 assert(device_ptr_ != nullptr);
135 assert(host_ptr_ != nullptr);
136 // If the second assert fails it means that the object was created without
137 // host buffer. It should not happen in the current scenario
138
139 offloadMemcpyAsyncDtoH(host_ptr_, device_ptr_, sizeof(T) * current_size_,
140 stream__);
141 }
142
143 inline void zero(offloadStream_t &stream__) {
144 assert(device_ptr_ != nullptr);
145 // zero device grid buffers
146 offloadMemsetAsync(device_ptr_, 0, sizeof(T) * current_size_, stream__);
147 }
148
149 inline void associate(void *host_ptr__, void *device_ptr__,
150 const size_t size__) {
151 assert(host_ptr__ != nullptr);
152 assert(device_ptr__ != nullptr);
153 reset();
154 internal_allocation_ = false;
155 allocated_size_ = size__;
156 current_size_ = size__;
157 device_ptr_ = static_cast<T *>(device_ptr__);
158 host_ptr_ = static_cast<T *>(host_ptr__);
159 }
160
161 inline void zero() {
162 // zero device grid buffers
163 offloadMemset(device_ptr_, 0, sizeof(T) * current_size_);
164 }
165
166 inline void copy_to_gpu(const std::vector<T> &data__) {
167 assert(data__.size() == current_size_);
168 // if it fails it means that the vector on the gpu does not have the right
169 // size. two option then
170 // - resize the gpu vector
171 // - or the cpu vector and gpu vector are not representing the quantity.
172 assert(device_ptr_ != nullptr);
173 offloadMemcpyHtoD(device_ptr_, data__.data(), sizeof(T) * data__.size());
174 }
175
176 inline void resize(const size_t new_size__) {
177 if (!internal_allocation_) {
178 allocated_size_ = 0;
179 device_ptr_ = nullptr;
180 host_ptr_ = nullptr;
181 }
182
183 assert(new_size__ != 0);
184 if (allocated_size_ < new_size__) {
185 if (internal_allocation_ && device_ptr_ != nullptr)
186 offloadFree(device_ptr_);
187 device_ptr_ = nullptr;
188 allocated_size_ = (new_size__ < 16) ? 16 : round_up16(new_size__);
189 offloadMalloc((void **)&device_ptr_, sizeof(T) * allocated_size_);
190 internal_allocation_ = true;
191 host_ptr_ = nullptr;
192 }
193 assert(device_ptr_ != nullptr);
194 current_size_ = new_size__;
195 }
196
197 // does not invalidate the pointer. The memory is still allocated
198 inline void clear() { current_size_ = 0; }
199
200 // reset the class and free memory
201 inline void reset() {
202 if (internal_allocation_ && (device_ptr_ != nullptr)) {
203 offloadFree(device_ptr_);
204 }
205
206 allocated_size_ = 0;
207 current_size_ = 0;
208 device_ptr_ = nullptr;
209 host_ptr_ = nullptr;
210 internal_allocation_ = false;
211 }
212
213 inline T *data() { return device_ptr_; }
214 inline const T *data() const { return device_ptr_; }
215};
216
217template <typename T> class grid_info {
218private:
219 int3 full_size_;
220 int3 local_size_;
221 // origin of the local part of the grid in grid point
222 int3 lower_corner_;
223 int3 border_width_;
224 std::array<T, 9> dh_;
225 std::array<T, 9> dh_inv_;
226 bool orthogonal_{false};
227 // lattice vectors along x, y and z
228 bool axis_aligned_{false};
229 bool is_distributed_{false};
230 // largest cube size (in grid points, all directions) of the tasks of this
231 // level. Used to size the shared memory of the separable kernels.
232 int max_cube_size_{0};
233 gpu_vector<T> grid_;
234 // expansion of the cartesian monomials in lattice monomials, see
235 // build_lattice_to_cartesian_matrix in grid_gpu_separable.h. Valid up to
236 // lattice_to_cartesian_lmax_ (-1 when not computed for the current dh).
237 gpu_vector<T> lattice_to_cartesian_;
238 int lattice_to_cartesian_lmax_{-1};
239
240public:
241 grid_info(const grid_info &) = delete;
242 grid_info &operator=(const grid_info &) = delete;
243
244 grid_info(grid_info &&) noexcept = default;
245 grid_info &operator=(grid_info &&) noexcept = default;
246
248
249 grid_info(const int *full_size__, const int *local_size__,
250 const int *border_width__) {
251 int roffset__[3] = {0, 0, 0};
252 initialize(full_size__, local_size__, roffset__, border_width__);
253 }
254
255 ~grid_info() = default;
256
257 inline void copy_to_gpu(const T *data, offloadStream_t &stream) {
258 assert(data != nullptr);
259 grid_.copy_to_gpu(data, stream);
260 }
261
262 inline void copy_to_gpu(offloadStream_t &stream) {
263 grid_.copy_associated_host_to_gpu(stream);
264 }
265
266 inline void reset() { grid_.reset(); }
267
268 /*
269 * We do not allocate memory as the buffer is always coming from the outside
270 * world. We only initialize the sizes, etc...
271 */
272 inline void resize(const int *full_size__, const int *local_size__,
273 const int *const roffset__,
274 const int *const border_width__) {
275 initialize(full_size__, local_size__, roffset__, border_width__);
276 }
277
278 inline size_t size() const { return grid_.size(); }
279
280 inline void zero(offloadStream_t &stream) { grid_.zero(stream); }
281
282 inline void set_lattice_vectors(const T *dh__, const T *dh_inv__) {
283 for (int i = 0; i < 9; ++i) {
284 dh_[i] = dh__[i];
285 dh_inv_[i] = dh_inv__[i];
286 }
287 axis_aligned_ = (dh_[1] == 0.0) && (dh_[2] == 0.0) && (dh_[3] == 0.0) &&
288 (dh_[5] == 0.0) && (dh_[6] == 0.0) && (dh_[7] == 0.0);
289 lattice_to_cartesian_lmax_ = -1;
290 }
291
292 inline bool is_axis_aligned() const { return axis_aligned_; }
293
294 inline int max_cube_size() const { return max_cube_size_; }
295 inline void max_cube_size(const int size__) { max_cube_size_ = size__; }
296
297 inline gpu_vector<T> &lattice_to_cartesian() { return lattice_to_cartesian_; }
298 inline int lattice_to_cartesian_lmax() const {
299 return lattice_to_cartesian_lmax_;
300 }
301 inline void lattice_to_cartesian_lmax(const int lmax__) {
302 lattice_to_cartesian_lmax_ = lmax__;
303 }
304
305 inline void is_distributed(const bool distributed__) {
306 is_distributed_ = distributed__;
307 }
308
309 /// Check if the lattice vectors form a orthogonal basis
310 // The orthogonal code paths (cube setup, x / y trimming, separable kernels)
311 // assume lattice vectors along x, y and z. A rotated orthogonal cell is
312 // therefore handled as non orthogonal. ortho (cell%orthorhombic on the
313 // fortran side) is only true for a diagonal hmat, up to 1e-14.
314 void check_orthogonality(const bool ortho) {
315 if (ortho)
316 axis_aligned_ = true;
317 orthogonal_ = axis_aligned_;
318 }
319
320 inline void copy_to_host(T *data__, offloadStream_t &stream) {
321 assert(data__ != nullptr);
322 grid_.copy_from_gpu(data__, stream);
323 }
324
325 inline void copy_to_host(offloadStream_t &stream) {
326 grid_.copy_gpu_to_associated_host(stream);
327 }
328
329 inline void associate(void *host_ptr__, void *device_ptr__,
330 const size_t size__) {
331 assert(host_ptr__ != nullptr);
332 assert(device_ptr__ != nullptr);
333 grid_.associate(host_ptr__, device_ptr__, size__);
334 }
335 inline bool is_distributed() { return is_distributed_; }
336
337 inline int3 &full_size() { return full_size_; }
338
339 inline int3 &local_size() { return local_size_; }
340
341 inline int3 &lower_corner() { return lower_corner_; }
342
343 inline int3 &border_width() { return border_width_; }
344
345 inline T *data() { return grid_.data(); }
346 inline const T *data() const { return grid_.data(); }
347
348 inline gpu_vector<T> &grid() { return grid_; }
349 inline const gpu_vector<T> &grid() const { return grid_; }
350
351 inline T *dh() { return dh_.data(); }
352 inline const T *dh() const { return dh_.data(); }
353
354 inline T *dh_inv() { return dh_inv_.data(); }
355 inline const T *dh_inv() const { return dh_inv_.data(); }
356
357 inline bool is_orthogonal() const { return orthogonal_; }
358 inline bool is_distributed() const { return is_distributed_; }
359
360private:
361 void initialize(const int *const full_size__, const int *const local_size__,
362 const int *const roffset__, const int *const border_width__) {
363 // the calling code store things like this cube[z][y][x] (in fortran
364 // cube(x,y,z)).
365
366 full_size_.x = full_size__[0];
367 full_size_.y = full_size__[1];
368 full_size_.z = full_size__[2];
369
370 local_size_.x = local_size__[0];
371 local_size_.y = local_size__[1];
372 local_size_.z = local_size__[2];
373
374 lower_corner_.x = roffset__[0];
375 lower_corner_.y = roffset__[1];
376 lower_corner_.z = roffset__[2];
377
378 is_distributed_ = (full_size_.x != local_size_.x) ||
379 (full_size_.y != local_size_.y) ||
380 (full_size_.z != local_size_.z);
381
382 border_width_.x = border_width__[0];
383 border_width_.y = border_width__[1];
384 border_width_.z = border_width__[2];
385 }
386};
387
388/*******************************************************************************
389 * \brief Internal representation of a task.
390 ******************************************************************************/
421
422/*******************************************************************************
423 * \brief Device buffers used by the collocate and integrate kernels.
424 * Which fields are non-null depends on which kernel is launched.
425 ******************************************************************************/
427 double *pab_block{nullptr};
428 double *grid{nullptr};
429 double *coef{nullptr};
430 double *hab_block{nullptr};
431 double *forces{nullptr};
432 double *virial{nullptr};
433 double *cab{nullptr};
434};
435
436/*******************************************************************************
437 * \brief Parameters of the collocate kernel.
438 ******************************************************************************/
439
442 // max size of cab.
443 int cab_size_{0};
444 int3 grid_full_size_ = make_int3(0, 0, 0);
445 int3 grid_local_size_ = make_int3(0, 0, 0);
446 int3 grid_lower_corner_ = make_int3(0, 0, 0);
447 int3 grid_border_width_ = make_int3(0, 0, 0);
448 double dh_[9];
449 double dh_inv_[9];
451 int *block_offsets{nullptr};
452 char la_min_diff{0};
453 char lb_min_diff{0};
454 char la_max_diff{0};
455 char lb_max_diff{0};
458 double **sphi_dev{nullptr};
459 int ntasks{0};
463 int *cab_block_offset_dev{nullptr};
464 // integrate: tasks with lp >= separable_lp_min are handled by
465 // integrate_separable_kernel, the others by integrate_kernel.
466 int separable_lp_min{INT_MAX};
467};
468
469/* regroup all information about the context. */
471private:
472 int device_id_{-1};
473 int lmax_{0};
474 // largest number of spherical functions of a set over all basis sets. Sizes
475 // the shared memory of calculate_coefficients / compute_hab.
476 int max_nsgf_set_{0};
477 unsigned int checksum_{0};
478
479public:
480 int ntasks{0};
481 int nlevels{0};
482 int natoms{0};
483 int nkinds{0};
484 int nblocks{0};
486 std::vector<double *> sphi;
487 std::vector<offloadStream_t> level_streams;
488 offloadStream_t main_stream;
489 int stats[2][20]; // [has_border_mask][lp]
490 // all these tables are on the gpu. we can resize them copy to them and copy
491 // from them
496 // largest basis set la_max / lb_max of the tasks of each block (-1 for an
497 // empty block), used to size the cab workspace of each block
498 std::vector<int> block_la_max_, block_lb_max_;
499 // (la_max_diff, lb_max_diff) the cab workspace is currently laid out for
507 std::vector<grid_info<double>> grid_;
509 std::vector<int> first_task_per_level_;
510 std::vector<int> sphi_size;
513 bool calculate_forces{false};
514 bool calculate_virial{false};
515 bool compute_tau{false};
516 bool apply_border_mask{false};
517 // Default constructor
518 context_info() = default;
519
520 explicit context_info(int device_id__) {
521 device_id_ = (device_id__ < 0) ? 0 : device_id__;
522 }
523
525
526 // Non-copyable
527 context_info(const context_info &) = delete;
529
530 // Movable
531 context_info(context_info &&other) noexcept
532 : device_id_{other.device_id_}, lmax_{other.lmax_},
533 max_nsgf_set_{other.max_nsgf_set_}, checksum_{other.checksum_},
534 ntasks{other.ntasks}, nlevels{other.nlevels}, natoms{other.natoms},
535 nkinds{other.nkinds}, nblocks{other.nblocks},
536 cab_size_per_block_{other.cab_size_per_block_},
537 sphi{std::move(other.sphi)},
538 level_streams{std::move(other.level_streams)},
539 main_stream{other.main_stream},
540 block_offsets_dev{std::move(other.block_offsets_dev)},
541 cab_block_offset_dev{std::move(other.cab_block_offset_dev)},
542 coef_dev_{std::move(other.coef_dev_)},
543 cab_dev_{std::move(other.cab_dev_)},
544 block_la_max_{std::move(other.block_la_max_)},
545 block_lb_max_{std::move(other.block_lb_max_)},
546 cab_la_max_diff_{other.cab_la_max_diff_},
547 cab_lb_max_diff_{other.cab_lb_max_diff_},
548 pab_block_{std::move(other.pab_block_)},
549 hab_block_{std::move(other.hab_block_)},
550 forces_{std::move(other.forces_)}, virial_{std::move(other.virial_)},
551 tasks_dev{std::move(other.tasks_dev)},
552 num_tasks_per_block_dev_{std::move(other.num_tasks_per_block_dev_)},
553 grid_{std::move(other.grid_)},
554 number_of_tasks_per_level_{std::move(other.number_of_tasks_per_level_)},
555 first_task_per_level_{std::move(other.first_task_per_level_)},
556 sphi_size{std::move(other.sphi_size)},
557 sphi_dev{std::move(other.sphi_dev)},
558 task_sorted_by_blocks_dev{std::move(other.task_sorted_by_blocks_dev)},
559 sorted_blocks_offset_dev{std::move(other.sorted_blocks_offset_dev)},
560 calculate_forces{other.calculate_forces},
561 calculate_virial{other.calculate_virial},
562 compute_tau{other.compute_tau},
563 apply_border_mask{other.apply_border_mask} {
564
565 // copy stats
566 for (int i = 0; i < 2; ++i)
567 for (int j = 0; j < 20; ++j)
568 stats[i][j] = other.stats[i][j];
569
570 // leave other in a safe state (no ownership)
571 other.device_id_ = -1;
572 other.lmax_ = 0;
573 other.max_nsgf_set_ = 0;
574 other.checksum_ = 0;
575 other.ntasks = 0;
576 other.nlevels = 0;
577 other.natoms = 0;
578 other.nkinds = 0;
579 other.nblocks = 0;
580 other.main_stream = {};
581 other.calculate_forces = false;
582 other.calculate_virial = false;
583 other.compute_tau = false;
584 other.apply_border_mask = false;
585 }
586
588 if (this != &other) {
589 clear(); // free current GPU resources and streams
590
591 device_id_ = other.device_id_;
592 lmax_ = other.lmax_;
593 max_nsgf_set_ = other.max_nsgf_set_;
594 checksum_ = other.checksum_;
595 ntasks = other.ntasks;
596 nlevels = other.nlevels;
597 natoms = other.natoms;
598 nkinds = other.nkinds;
599 nblocks = other.nblocks;
600 sphi = std::move(other.sphi);
601 level_streams = std::move(other.level_streams);
602 main_stream = other.main_stream;
603
604 for (int i = 0; i < 2; ++i)
605 for (int j = 0; j < 20; ++j)
606 stats[i][j] = other.stats[i][j];
607
608 block_offsets_dev = std::move(other.block_offsets_dev);
609 cab_block_offset_dev = std::move(other.cab_block_offset_dev);
610 coef_dev_ = std::move(other.coef_dev_);
611 cab_dev_ = std::move(other.cab_dev_);
612 block_la_max_ = std::move(other.block_la_max_);
613 block_lb_max_ = std::move(other.block_lb_max_);
614 cab_la_max_diff_ = other.cab_la_max_diff_;
615 cab_lb_max_diff_ = other.cab_lb_max_diff_;
616 pab_block_ = std::move(other.pab_block_);
617 hab_block_ = std::move(other.hab_block_);
618 forces_ = std::move(other.forces_);
619 virial_ = std::move(other.virial_);
620 tasks_dev = std::move(other.tasks_dev);
621 num_tasks_per_block_dev_ = std::move(other.num_tasks_per_block_dev_);
622 grid_ = std::move(other.grid_);
623 number_of_tasks_per_level_ = std::move(other.number_of_tasks_per_level_);
624 first_task_per_level_ = std::move(other.first_task_per_level_);
625 sphi_size = std::move(other.sphi_size);
626 sphi_dev = std::move(other.sphi_dev);
627 task_sorted_by_blocks_dev = std::move(other.task_sorted_by_blocks_dev);
628 sorted_blocks_offset_dev = std::move(other.sorted_blocks_offset_dev);
629
630 calculate_forces = other.calculate_forces;
631 calculate_virial = other.calculate_virial;
632 compute_tau = other.compute_tau;
633 apply_border_mask = other.apply_border_mask;
634
635 // reset other
636 other.device_id_ = -1;
637 other.lmax_ = 0;
638 other.max_nsgf_set_ = 0;
639 other.checksum_ = 0;
640 other.ntasks = 0;
641 other.nlevels = 0;
642 other.natoms = 0;
643 other.nkinds = 0;
644 other.nblocks = 0;
645 other.main_stream = {};
646 other.calculate_forces = false;
647 other.calculate_virial = false;
648 other.compute_tau = false;
649 other.apply_border_mask = false;
650 }
651 return *this;
652 }
653
654 void clear() {
655 if (device_id_ >= 0) {
656 offload_set_chosen_device(device_id_);
658 }
659
660 tasks_dev.reset();
661 block_offsets_dev.reset();
662 cab_block_offset_dev.reset();
663 coef_dev_.reset();
664 cab_dev_.reset();
665 block_la_max_.clear();
666 block_lb_max_.clear();
667 cab_la_max_diff_ = -1;
668 cab_lb_max_diff_ = -1;
671 sphi_dev.reset();
672 forces_.reset();
673 virial_.reset();
674
675 for (auto &phi : sphi) {
676 if (phi != nullptr)
677 offloadFree(phi);
678 }
679 sphi.clear();
680
681 if (main_stream) {
682 offloadStreamDestroy(main_stream);
683 main_stream = {};
684 }
685
686 for (auto &stream : level_streams) {
687 if (stream)
688 offloadStreamDestroy(stream);
689 }
690 level_streams.clear();
691
692 for (auto &g : grid_) {
693 g.reset();
694 }
695 grid_.clear();
696 }
697
698 int lmax() const { return lmax_; }
699 int max_nsgf_set() const { return max_nsgf_set_; }
700
701 void initialize_basis_sets(const grid_basis_set **basis_sets,
702 const int nkinds__) {
703 nkinds = nkinds__;
704 if (nkinds__ > (int)sphi.size()) {
705 for (auto &phi : sphi)
706 if (phi != nullptr) {
707 offloadFree(phi);
708 }
709
710 sphi_dev.resize(nkinds__);
711
712 sphi.resize(nkinds__, nullptr);
713 sphi_size.clear();
714 sphi_size.resize(nkinds__, 0);
715 sphi_dev.resize(nkinds__);
716 }
717
718 // Upload basis sets to device.
719 for (int i = 0; i < nkinds__; i++) {
720 const auto &basis_set = basis_sets[i];
721 if (sphi_size[i] < basis_set->nsgf * basis_set->maxco) {
722 offloadMalloc((void **)&sphi[i],
723 basis_set->nsgf * basis_set->maxco * sizeof(double));
724 sphi_size[i] = basis_set->nsgf * basis_set->maxco;
725 }
726 // offloadMemset(sphi[i], 0, sizeof(double) * sphi_size[i]);
727 offloadMemcpyHtoD(sphi[i], basis_set->sphi,
728 basis_set->nsgf * basis_set->maxco * sizeof(double));
729 }
730 sphi_dev.copy_to_gpu(sphi);
731 // Find largest angular momentum and set size.
732 lmax_ = 0;
733 max_nsgf_set_ = 0;
734 for (int ikind = 0; ikind < nkinds; ikind++) {
735 for (int iset = 0; iset < basis_sets[ikind]->nset; iset++) {
736 lmax_ = std::max(lmax_, basis_sets[ikind]->lmax[iset]);
737 max_nsgf_set_ =
738 std::max(max_nsgf_set_, basis_sets[ikind]->nsgf_set[iset]);
739 }
740 }
741 }
742
744 // allocate main hip stream
745 offloadStreamCreate(&main_stream);
746
747 // allocate one hip stream per grid level
748 if ((int)level_streams.size() < nlevels) {
749 level_streams.resize(nlevels);
750 for (auto &stream : level_streams) {
751 offloadStreamCreate(&stream);
752 }
753 }
754 }
755
756 void synchronize(offloadStream_t &stream) {
757 offloadStreamSynchronize(stream);
758 }
759
764
765 void collocate_one_grid_level(const int level, const enum grid_func func,
766 int *lp_diff);
767 void integrate_one_grid_level(const int level, int *lp_diff);
769 /* basic checksum computation for simple verification that the object is sane
770 */
771 void compute_checksum() { checksum_ = compute_checksum_(); }
773 if (checksum_ != compute_checksum_()) {
774 fprintf(stderr, "This object does not seem to have the right structure.\n"
775 "A casting went wrong or the object is corrupted\n");
776 abort();
777 }
778 }
779 void calculate_all_coefficients(const enum grid_func func, int *lp_diff);
780 void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff);
781
782private:
783 kernel_params set_kernel_parameters(const int level,
784 const smem_parameters &smem_params);
785 unsigned int compute_checksum_() {
786 return natoms ^ ntasks ^ nlevels ^ nkinds ^ nblocks ^ 0x4F2C5D1A;
787 }
788};
789} // namespace grid_gpu_backend
790#endif
gpu_vector< double * > sphi_dev
context_info & operator=(context_info &&other) noexcept
context_info(context_info &&other) noexcept
context_info & operator=(const context_info &)=delete
void initialize_basis_sets(const grid_basis_set **basis_sets, const int nkinds__)
std::vector< offloadStream_t > level_streams
void synchronize(offloadStream_t &stream)
gpu_vector< int > task_sorted_by_blocks_dev
std::vector< int > number_of_tasks_per_level_
gpu_vector< task_info > tasks_dev
void calculate_all_coefficients(const enum grid_func func, int *lp_diff)
void collocate_one_grid_level(const int level, const enum grid_func func, int *lp_diff)
Launches the Cuda kernel that collocates all tasks of one grid level.
void integrate_one_grid_level(const int level, int *lp_diff)
Launches the Cuda kernel that integrates all tasks of one grid level.
context_info(const context_info &)=delete
void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff)
Lays out the cab workspace of the coefficient kernels for the angular momentum offsets of the current...
gpu_vector< int > sorted_blocks_offset_dev
std::vector< grid_info< double > > grid_
std::vector< int > first_task_per_level_
gpu_vector< int > num_tasks_per_block_dev_
std::vector< double * > sphi
gpu_vector(const gpu_vector &)=delete
void copy_to_gpu(const T *data__, offloadStream_t &stream__)
gpu_vector(const size_t size__)
void copy_associated_host_to_gpu(offloadStream_t &stream__)
gpu_vector & operator=(gpu_vector &&other) noexcept
gpu_vector & operator=(const gpu_vector &)=delete
void copy_to_gpu(const std::vector< T > &data__)
void copy_gpu_to_associated_host(offloadStream_t &stream__)
gpu_vector(gpu_vector &&other) noexcept
void associate(void *host_ptr__, void *device_ptr__, const size_t size__)
void copy_to_gpu(const T *data__)
void resize(const size_t new_size__)
void copy_from_gpu(T *data__, offloadStream_t &stream__)
gpu_vector(const size_t size__, void *ptr__)
void zero(offloadStream_t &stream__)
grid_info(grid_info &&) noexcept=default
grid_info & operator=(const grid_info &)=delete
void set_lattice_vectors(const T *dh__, const T *dh_inv__)
const gpu_vector< T > & grid() const
void lattice_to_cartesian_lmax(const int lmax__)
void copy_to_gpu(offloadStream_t &stream)
void associate(void *host_ptr__, void *device_ptr__, const size_t size__)
gpu_vector< T > & lattice_to_cartesian()
grid_info(const int *full_size__, const int *local_size__, const int *border_width__)
void copy_to_gpu(const T *data, offloadStream_t &stream)
void check_orthogonality(const bool ortho)
Check if the lattice vectors form a orthogonal basis.
void copy_to_host(T *data__, offloadStream_t &stream)
void zero(offloadStream_t &stream)
void max_cube_size(const int size__)
grid_info(const grid_info &)=delete
void copy_to_host(offloadStream_t &stream)
void resize(const int *full_size__, const int *local_size__, const int *const roffset__, const int *const border_width__)
void is_distributed(const bool distributed__)
grid_func
static void const int const int i
size_t round_up16(size_t n)
void offload_activate_chosen_device(void)
Activates the device selected via offload_set_chosen_device().
void offload_set_chosen_device(int device_id)
Selects the chosen device to be used.
Internal representation of a basis set.
Device buffers used by the collocate and integrate kernels. Which fields are non-null depends on whic...
Parameters of the collocate kernel.
Internal representation of a task.