14#ifndef GRID_GPU_CONTEXT_H
15#define GRID_GPU_CONTEXT_H
18#include <hip/hip_runtime_api.h>
20#include <cuda_runtime.h>
39inline size_t round_up16(
size_t n) {
return (n + 15) & ~size_t(15); }
43 size_t allocated_size_{0};
44 size_t current_size_{0};
45 bool internal_allocation_{
false};
46 T *device_ptr_{
nullptr};
47 T *host_ptr_{
nullptr};
50 allocated_size_ = other.allocated_size_;
51 current_size_ = other.current_size_;
52 internal_allocation_ = other.internal_allocation_;
53 device_ptr_ = other.device_ptr_;
54 host_ptr_ = other.host_ptr_;
56 other.allocated_size_ = 0;
57 other.current_size_ = 0;
58 other.internal_allocation_ =
false;
59 other.device_ptr_ =
nullptr;
60 other.host_ptr_ =
nullptr;
74 move_from(std::move(other));
81 allocated_size_ = (size__ < 16) ? 16 :
round_up16(size__);
82 current_size_ = size__;
83 internal_allocation_ =
true;
85#ifndef __OFFLOAD_UNIFIED_MEMORY
86 offloadMalloc((
void **)&device_ptr_,
sizeof(T) * allocated_size_);
88 hipMallocManaged((
void **)&device_ptr_,
sizeof(T) * allocated_size_);
90 assert(device_ptr_ !=
nullptr);
94 allocated_size_ = size__;
95 current_size_ = size__;
96 internal_allocation_ =
false;
97 device_ptr_ =
static_cast<T *
>(ptr__);
101 inline size_t size()
const {
return current_size_; }
104 assert(device_ptr_ !=
nullptr);
105 assert(data__ !=
nullptr);
106 offloadMemcpyHtoD(device_ptr_, data__,
sizeof(T) * current_size_);
109 inline void copy_to_gpu(
const T *data__, offloadStream_t &stream__) {
110 assert(device_ptr_ !=
nullptr);
111 assert(data__ !=
nullptr);
112 offloadMemcpyAsyncHtoD(device_ptr_, data__,
sizeof(T) * current_size_,
117 assert(device_ptr_ !=
nullptr);
118 assert(host_ptr_ !=
nullptr);
122 offloadMemcpyAsyncHtoD(device_ptr_, host_ptr_,
sizeof(T) * current_size_,
127 assert(device_ptr_ !=
nullptr);
128 assert(data__ !=
nullptr);
129 offloadMemcpyAsyncDtoH(data__, device_ptr_,
sizeof(T) * current_size_,
134 assert(device_ptr_ !=
nullptr);
135 assert(host_ptr_ !=
nullptr);
139 offloadMemcpyAsyncDtoH(host_ptr_, device_ptr_,
sizeof(T) * current_size_,
143 inline void zero(offloadStream_t &stream__) {
144 assert(device_ptr_ !=
nullptr);
146 offloadMemsetAsync(device_ptr_, 0,
sizeof(T) * current_size_, stream__);
149 inline void associate(
void *host_ptr__,
void *device_ptr__,
150 const size_t size__) {
151 assert(host_ptr__ !=
nullptr);
152 assert(device_ptr__ !=
nullptr);
154 internal_allocation_ =
false;
155 allocated_size_ = size__;
156 current_size_ = size__;
157 device_ptr_ =
static_cast<T *
>(device_ptr__);
158 host_ptr_ =
static_cast<T *
>(host_ptr__);
163 offloadMemset(device_ptr_, 0,
sizeof(T) * current_size_);
167 assert(data__.size() == current_size_);
172 assert(device_ptr_ !=
nullptr);
173 offloadMemcpyHtoD(device_ptr_, data__.data(),
sizeof(T) * data__.size());
176 inline void resize(
const size_t new_size__) {
177 if (!internal_allocation_) {
179 device_ptr_ =
nullptr;
183 assert(new_size__ != 0);
184 if (allocated_size_ < new_size__) {
185 if (internal_allocation_ && device_ptr_ !=
nullptr)
186 offloadFree(device_ptr_);
187 device_ptr_ =
nullptr;
188 allocated_size_ = (new_size__ < 16) ? 16 :
round_up16(new_size__);
189 offloadMalloc((
void **)&device_ptr_,
sizeof(T) * allocated_size_);
190 internal_allocation_ =
true;
193 assert(device_ptr_ !=
nullptr);
194 current_size_ = new_size__;
198 inline void clear() { current_size_ = 0; }
202 if (internal_allocation_ && (device_ptr_ !=
nullptr)) {
203 offloadFree(device_ptr_);
208 device_ptr_ =
nullptr;
210 internal_allocation_ =
false;
213 inline T *
data() {
return device_ptr_; }
214 inline const T *
data()
const {
return device_ptr_; }
224 std::array<T, 9> dh_;
225 std::array<T, 9> dh_inv_;
226 bool orthogonal_{
false};
228 bool axis_aligned_{
false};
229 bool is_distributed_{
false};
232 int max_cube_size_{0};
238 int lattice_to_cartesian_lmax_{-1};
249 grid_info(
const int *full_size__,
const int *local_size__,
250 const int *border_width__) {
251 int roffset__[3] = {0, 0, 0};
252 initialize(full_size__, local_size__, roffset__, border_width__);
258 assert(
data !=
nullptr);
259 grid_.copy_to_gpu(
data, stream);
263 grid_.copy_associated_host_to_gpu(stream);
266 inline void reset() { grid_.reset(); }
272 inline void resize(
const int *full_size__,
const int *local_size__,
273 const int *
const roffset__,
274 const int *
const border_width__) {
275 initialize(full_size__, local_size__, roffset__, border_width__);
278 inline size_t size()
const {
return grid_.size(); }
280 inline void zero(offloadStream_t &stream) { grid_.zero(stream); }
283 for (
int i = 0;
i < 9; ++
i) {
285 dh_inv_[
i] = dh_inv__[
i];
287 axis_aligned_ = (dh_[1] == 0.0) && (dh_[2] == 0.0) && (dh_[3] == 0.0) &&
288 (dh_[5] == 0.0) && (dh_[6] == 0.0) && (dh_[7] == 0.0);
289 lattice_to_cartesian_lmax_ = -1;
299 return lattice_to_cartesian_lmax_;
302 lattice_to_cartesian_lmax_ = lmax__;
306 is_distributed_ = distributed__;
316 axis_aligned_ =
true;
317 orthogonal_ = axis_aligned_;
321 assert(data__ !=
nullptr);
322 grid_.copy_from_gpu(data__, stream);
326 grid_.copy_gpu_to_associated_host(stream);
329 inline void associate(
void *host_ptr__,
void *device_ptr__,
330 const size_t size__) {
331 assert(host_ptr__ !=
nullptr);
332 assert(device_ptr__ !=
nullptr);
333 grid_.associate(host_ptr__, device_ptr__, size__);
345 inline T *
data() {
return grid_.data(); }
346 inline const T *
data()
const {
return grid_.data(); }
351 inline T *
dh() {
return dh_.data(); }
352 inline const T *
dh()
const {
return dh_.data(); }
354 inline T *
dh_inv() {
return dh_inv_.data(); }
355 inline const T *
dh_inv()
const {
return dh_inv_.data(); }
361 void initialize(
const int *
const full_size__,
const int *
const local_size__,
362 const int *
const roffset__,
const int *
const border_width__) {
366 full_size_.x = full_size__[0];
367 full_size_.y = full_size__[1];
368 full_size_.z = full_size__[2];
370 local_size_.x = local_size__[0];
371 local_size_.y = local_size__[1];
372 local_size_.z = local_size__[2];
374 lower_corner_.x = roffset__[0];
375 lower_corner_.y = roffset__[1];
376 lower_corner_.z = roffset__[2];
378 is_distributed_ = (full_size_.x != local_size_.x) ||
379 (full_size_.y != local_size_.y) ||
380 (full_size_.z != local_size_.z);
382 border_width_.x = border_width__[0];
383 border_width_.y = border_width__[1];
384 border_width_.z = border_width__[2];
476 int max_nsgf_set_{0};
477 unsigned int checksum_{0};
507 std::vector<grid_info<double>>
grid_;
521 device_id_ = (device_id__ < 0) ? 0 : device_id__;
532 : device_id_{other.device_id_}, lmax_{other.lmax_},
533 max_nsgf_set_{other.max_nsgf_set_}, checksum_{other.checksum_},
537 sphi{std::move(other.sphi)},
543 cab_dev_{std::move(other.cab_dev_)},
550 forces_{std::move(other.forces_)},
virial_{std::move(other.virial_)},
553 grid_{std::move(other.grid_)},
557 sphi_dev{std::move(other.sphi_dev)},
566 for (
int i = 0;
i < 2; ++
i)
567 for (
int j = 0; j < 20; ++j)
568 stats[
i][j] = other.stats[
i][j];
571 other.device_id_ = -1;
573 other.max_nsgf_set_ = 0;
580 other.main_stream = {};
581 other.calculate_forces =
false;
582 other.calculate_virial =
false;
583 other.compute_tau =
false;
584 other.apply_border_mask =
false;
588 if (
this != &other) {
591 device_id_ = other.device_id_;
593 max_nsgf_set_ = other.max_nsgf_set_;
594 checksum_ = other.checksum_;
600 sphi = std::move(other.sphi);
604 for (
int i = 0;
i < 2; ++
i)
605 for (
int j = 0; j < 20; ++j)
606 stats[
i][j] = other.stats[
i][j];
611 cab_dev_ = std::move(other.cab_dev_);
618 forces_ = std::move(other.forces_);
619 virial_ = std::move(other.virial_);
622 grid_ = std::move(other.grid_);
626 sphi_dev = std::move(other.sphi_dev);
636 other.device_id_ = -1;
638 other.max_nsgf_set_ = 0;
645 other.main_stream = {};
646 other.calculate_forces =
false;
647 other.calculate_virial =
false;
648 other.compute_tau =
false;
649 other.apply_border_mask =
false;
655 if (device_id_ >= 0) {
675 for (
auto &phi :
sphi) {
688 offloadStreamDestroy(stream);
692 for (
auto &g :
grid_) {
698 int lmax()
const {
return lmax_; }
702 const int nkinds__) {
704 if (nkinds__ > (
int)
sphi.size()) {
705 for (
auto &phi :
sphi)
706 if (phi !=
nullptr) {
712 sphi.resize(nkinds__,
nullptr);
719 for (
int i = 0;
i < nkinds__;
i++) {
720 const auto &basis_set = basis_sets[
i];
721 if (
sphi_size[
i] < basis_set->nsgf * basis_set->maxco) {
722 offloadMalloc((
void **)&
sphi[
i],
723 basis_set->nsgf * basis_set->maxco *
sizeof(
double));
724 sphi_size[
i] = basis_set->nsgf * basis_set->maxco;
727 offloadMemcpyHtoD(
sphi[
i], basis_set->sphi,
728 basis_set->nsgf * basis_set->maxco *
sizeof(
double));
734 for (
int ikind = 0; ikind <
nkinds; ikind++) {
735 for (
int iset = 0; iset < basis_sets[ikind]->
nset; iset++) {
736 lmax_ = std::max(lmax_, basis_sets[ikind]->
lmax[iset]);
738 std::max(max_nsgf_set_, basis_sets[ikind]->nsgf_set[iset]);
751 offloadStreamCreate(&stream);
757 offloadStreamSynchronize(stream);
773 if (checksum_ != compute_checksum_()) {
774 fprintf(stderr,
"This object does not seem to have the right structure.\n"
775 "A casting went wrong or the object is corrupted\n");
785 unsigned int compute_checksum_() {
context_info(int device_id__)
gpu_vector< double * > sphi_dev
context_info & operator=(context_info &&other) noexcept
std::vector< int > block_lb_max_
gpu_vector< int > block_offsets_dev
context_info(context_info &&other) noexcept
context_info & operator=(const context_info &)=delete
void initialize_basis_sets(const grid_basis_set **basis_sets, const int nkinds__)
std::vector< offloadStream_t > level_streams
std::vector< int > block_la_max_
gpu_vector< double > virial_
void synchronize(offloadStream_t &stream)
offloadStream_t main_stream
gpu_vector< int > task_sorted_by_blocks_dev
void compute_hab_coefficients()
std::vector< int > number_of_tasks_per_level_
gpu_vector< double > pab_block_
gpu_vector< double > cab_dev_
gpu_vector< task_info > tasks_dev
gpu_vector< double > coef_dev_
gpu_vector< int > cab_block_offset_dev
void calculate_all_coefficients(const enum grid_func func, int *lp_diff)
gpu_vector< double > forces_
void collocate_one_grid_level(const int level, const enum grid_func func, int *lp_diff)
Launches the Cuda kernel that collocates all tasks of one grid level.
void integrate_one_grid_level(const int level, int *lp_diff)
Launches the Cuda kernel that integrates all tasks of one grid level.
context_info(const context_info &)=delete
void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff)
Lays out the cab workspace of the coefficient kernels for the angular momentum offsets of the current...
gpu_vector< int > sorted_blocks_offset_dev
std::vector< grid_info< double > > grid_
std::vector< int > first_task_per_level_
std::vector< int > sphi_size
gpu_vector< int > num_tasks_per_block_dev_
gpu_vector< double > hab_block_
std::vector< double * > sphi
gpu_vector(const gpu_vector &)=delete
void copy_to_gpu(const T *data__, offloadStream_t &stream__)
gpu_vector(const size_t size__)
void copy_associated_host_to_gpu(offloadStream_t &stream__)
gpu_vector & operator=(gpu_vector &&other) noexcept
gpu_vector & operator=(const gpu_vector &)=delete
void copy_to_gpu(const std::vector< T > &data__)
void copy_gpu_to_associated_host(offloadStream_t &stream__)
gpu_vector(gpu_vector &&other) noexcept
void associate(void *host_ptr__, void *device_ptr__, const size_t size__)
void copy_to_gpu(const T *data__)
void resize(const size_t new_size__)
void copy_from_gpu(T *data__, offloadStream_t &stream__)
gpu_vector(const size_t size__, void *ptr__)
void zero(offloadStream_t &stream__)
grid_info(grid_info &&) noexcept=default
grid_info & operator=(const grid_info &)=delete
void set_lattice_vectors(const T *dh__, const T *dh_inv__)
const gpu_vector< T > & grid() const
bool is_axis_aligned() const
void lattice_to_cartesian_lmax(const int lmax__)
void copy_to_gpu(offloadStream_t &stream)
void associate(void *host_ptr__, void *device_ptr__, const size_t size__)
gpu_vector< T > & lattice_to_cartesian()
grid_info(const int *full_size__, const int *local_size__, const int *border_width__)
int lattice_to_cartesian_lmax() const
void copy_to_gpu(const T *data, offloadStream_t &stream)
void check_orthogonality(const bool ortho)
Check if the lattice vectors form a orthogonal basis.
void copy_to_host(T *data__, offloadStream_t &stream)
bool is_orthogonal() const
int max_cube_size() const
void zero(offloadStream_t &stream)
void max_cube_size(const int size__)
grid_info(const grid_info &)=delete
bool is_distributed() const
void copy_to_host(offloadStream_t &stream)
void resize(const int *full_size__, const int *local_size__, const int *const roffset__, const int *const border_width__)
void is_distributed(const bool distributed__)
static void const int const int i
size_t round_up16(size_t n)
void offload_activate_chosen_device(void)
Activates the device selected via offload_set_chosen_device().
void offload_set_chosen_device(int device_id)
Selects the chosen device to be used.
Internal representation of a basis set.
Device buffers used by the collocate and integrate kernels. Which fields are non-null depends on whic...
Parameters of the collocate kernel.
int * task_sorted_by_blocks_dev
int * num_tasks_per_block_dev
int * cab_block_offset_dev
kernel_buffers buffers_dev
int * sorted_blocks_offset_dev
Internal representation of a task.