30#error "OpenMP should not be used in .cu files to accommodate HIP."
44#if defined(__CUDACC__)
69 val = table[
tid] + table[
tid + 32];
72#if defined(__CUDACC__)
73 val += __shfl_down_sync(0xffffffff, val,
offset);
75 val += __shfl_down(val,
offset);
86template <
typename T,
bool COMPUTE_TAU>
101 __shared__ smem_task<T>
task;
102 const int task_id = dev_.task_sorted_by_blocks_dev[
offset + tk];
103 if (dev_.tasks[
task_id].skip_task)
111 T *__restrict__ coef_ =
reinterpret_cast<T *
>(__builtin_assume_aligned(
112 &dev_.buffers_dev.coef[dev_.tasks[
task_id].coef_offset], 32));
118 for (
int i =
tid / 8;
i <
task.nsgf_setb;
i += 8) {
119 for (
int j =
tid % 8; j <
task.nsgf_seta; j += 8) {
121 for (
int jco =
task.first_cosetb; jco <
task.ncosetb; jco++) {
122 const T sphib =
task.sphib[
i *
task.maxcob + jco];
124 for (
int ico =
task.first_coseta; ico <
task.ncoseta; ico++) {
128 const T sphia_times_sphib =
129 task.sphia[j *
task.maxcoa + ico] * sphib;
130 tmp += hab * sphia_times_sphib;
133 if (
task.block_transposed) {
134 task.hab_block[j *
task.nsgfb +
i] += tmp;
136 task.hab_block[
i *
task.nsgfa + j] += tmp;
156template <
typename T,
bool COMPUTE_TAU>
159 extern __shared__ T shared_memory[];
168 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
175 const int task_id = dev_.task_sorted_by_blocks_dev[
offset + tk];
176 if (dev_.tasks[
task_id].skip_task)
182 T *__restrict__ coef_ =
reinterpret_cast<T *
>(__builtin_assume_aligned(
183 &dev_.buffers_dev.coef[dev_.tasks[
task_id].coef_offset], 32));
189 const int nco_a =
task.ncoseta -
task.first_coseta;
190 const int nco_b =
task.ncosetb -
task.first_cosetb;
193 for (
int k =
tid; k < nco_b * nco_a; k +=
nthreads) {
194 const int jj = k / nco_a;
195 const int jco =
task.first_cosetb + jj;
196 const int ico =
task.first_coseta + (k - jj * nco_a);
205 const int i = k / nco_a;
206 const int ico =
task.first_coseta + (k -
i * nco_a);
208 for (
int jco =
task.first_cosetb; jco <
task.ncosetb; jco++)
216 const int i = k /
task.nsgf_seta;
217 const int j = k -
i *
task.nsgf_seta;
219 for (
int ico =
task.first_coseta; ico <
task.ncoseta; ico++)
221 if (
task.block_transposed) {
222 task.hab_block[j *
task.nsgfb +
i] += val;
224 task.hab_block[
i *
task.nsgfa + j] += val;
234template <
typename T,
bool COMPUTE_TAU>
235__device__ __inline__
void
238 const T *__restrict__ cab, T *
fa, T *
fb, T *
virial) {
239 for (
int d = 0; d < 3; d++) {
246 for (
int d1 = 0; d1 < 3; d1++)
247 for (
int d2 = 0; d2 < 3; d2++)
268template <
typename T,
typename T3,
bool COMPUTE_TAU>
272 extern __shared__ T shared_memory[];
279 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
285 T
fa[3] = {0.0, 0.0, 0.0},
fb[3] = {0.0, 0.0, 0.0};
286 T
virial[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
290 const int task_id = dev_.task_sorted_by_blocks_dev[
offset + tk];
291 if (dev_.tasks[
task_id].skip_task)
295 T *__restrict__ coef_ =
296 &dev_.buffers_dev.coef[dev_.tasks[
task_id].coef_offset];
302 for (
int i =
tid / 8;
i <
task.nsgf_setb;
i += 8) {
303 for (
int j =
tid % 8; j <
task.nsgf_seta; j += 8) {
305 for (
int jco =
task.first_cosetb; jco <
task.ncosetb; jco++) {
306 const T sphib =
task.sphib[
i *
task.maxcob + jco];
308 for (
int ico =
task.first_coseta; ico <
task.ncoseta; ico++) {
312 tmp += hab *
task.sphia[j *
task.maxcoa + ico] * sphib;
315 if (
task.block_transposed) {
316 task.hab_block[j *
task.nsgfb +
i] += tmp;
318 task.hab_block[
i *
task.nsgfa + j] += tmp;
324 const int nco_a =
task.ncoseta -
task.first_coseta;
325 const int nco_b =
task.ncosetb -
task.first_cosetb;
326 for (
int k =
tid; k < nco_b * nco_a; k +=
nthreads) {
327 const int jj = k / nco_a;
328 const int jco =
task.first_cosetb + jj;
329 const int ico =
task.first_coseta + (k - jj * nco_a);
332 for (
int i = 0;
i <
task.nsgf_setb;
i++) {
334 for (
int j = 0; j <
task.nsgf_seta; j++) {
335 const T block_value =
task.block_transposed
338 w += block_value *
task.sphia[j *
task.maxcoa + ico];
340 p +=
task.sphib[
i *
task.maxcob + jco] * w;
356 T *
forces_a = &dev_.buffers_dev.forces[3 * iatom];
357 T *
forces_b = &dev_.buffers_dev.forces[3 * jatom];
359 T *
sum = (T *)shared_memory;
360 if (dev_.buffers_dev.virial !=
nullptr) {
362 for (
int i = 0;
i < 9;
i++) {
366 atomicAdd(dev_.buffers_dev.virial +
i,
virial[
i]);
370 for (
int i = 0;
i < 3;
i++) {
405template <
typename T,
bool COMPUTE_TAU>
408 extern __shared__ T shared_memory[];
414 T *
smem_h = &shared_memory[0];
417 const int nthreads = blockDim.x * blockDim.y * blockDim.z;
423 T
fa[3] = {0.0, 0.0, 0.0},
fb[3] = {0.0, 0.0, 0.0};
424 T
virial[9] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
427 __shared__ smem_task<T>
task;
428 const int task_id = dev_.task_sorted_by_blocks_dev[
offset + tk];
429 if (dev_.tasks[
task_id].skip_task)
436 T *__restrict__ coef_ =
reinterpret_cast<T *
>(__builtin_assume_aligned(
437 &dev_.buffers_dev.coef[dev_.tasks[
task_id].coef_offset], 32));
442 const int nco_a =
task.ncoseta -
task.first_coseta;
443 const int nco_b =
task.ncosetb -
task.first_cosetb;
447 const int i = k / nco_a;
448 const int ico =
task.first_coseta + (k -
i * nco_a);
450 for (
int j = 0; j <
task.nsgf_seta; j++) {
451 const T block_value =
task.block_transposed
454 val += block_value *
task.sphia[j *
task.maxcoa + ico];
461 for (
int k =
tid; k < nco_b * nco_a; k +=
nthreads) {
462 const int jj = k / nco_a;
463 const int jco =
task.first_cosetb + jj;
464 const int ico =
task.first_coseta + (k - jj * nco_a);
469 for (
int i = 0;
i <
task.nsgf_setb;
i++)
482 const int i = k / nco_a;
483 const int ico =
task.first_coseta + (k -
i * nco_a);
485 for (
int jco =
task.first_cosetb; jco <
task.ncosetb; jco++)
493 const int i = k /
task.nsgf_seta;
494 const int j = k -
i *
task.nsgf_seta;
496 for (
int ico =
task.first_coseta; ico <
task.ncoseta; ico++)
498 if (
task.block_transposed) {
499 task.hab_block[j *
task.nsgfb +
i] += val;
501 task.hab_block[
i *
task.nsgfa + j] += val;
514 T *
sum = (T *)shared_memory;
516 for (
int i = 0;
i < 9;
i++) {
519 atomicAdd(dev_.buffers_dev.virial +
i,
virial[
i]);
523 for (
int i = 0;
i < 3;
i++) {
559template <
typename T,
typename T3,
bool distributed__,
bool orthogonal_>
562 const int lbatch = 20;
569 dev_.separable_lp_min)
587 __shared__ T accumulator[20][64];
592 const short int size_loop =
595 for (
int ico = 0; ico < size_loop; ico += lbatch) {
597 for (
int i = 0;
i < lbatch;
i++)
598 accumulator[
i][
tid] = 0.0;
602 for (
int z = threadIdx.z; z <
task.cube_size.z; z += blockDim.z) {
607 if (
task.apply_border_mask) {
609 if ((z2 <
task.window_shift.z) || (z2 >
task.window_size.z)) {
617 int ymax =
task.cube_size.y - 1;
620 if (orthogonal_ && !
task.apply_border_mask) {
624 for (
int y = ymin + threadIdx.y; y <= ymax; y += blockDim.y) {
630 if (
task.apply_border_mask) {
631 if ((y2 <
task.window_shift.y) || (y2 >
task.window_size.y)) {
638 int xmax =
task.cube_size.x - 1;
640 if (orthogonal_ && !
task.apply_border_mask) {
644 for (
int x = xmin + threadIdx.x; x <= xmax; x += blockDim.x) {
650 if (
task.apply_border_mask) {
651 if ((x2 <
task.window_shift.x) || (x2 >
task.window_size.x)) {
662 (y +
task.lb_cube.y +
task.roffset.y),
663 (z +
task.lb_cube.z +
task.roffset.z));
667 const T r3x2 = r3.x * r3.x;
668 const T r3y2 = r3.y * r3.y;
669 const T r3z2 = r3.z * r3.z;
676 if (((
task.radius *
task.radius) <= (r3x2 + r3y2 + r3z2)) &&
677 (!orthogonal_ ||
task.apply_border_mask))
681 if ((!orthogonal_) &&
682 ((
task.radius *
task.radius) <= (r3x2 + r3y2 + r3z2)))
690 const int grid_index =
691 (z2 * dev_.grid_local_size_.y + y2) * dev_.grid_local_size_.x +
693 T grid_value = __ldg(&dev_.buffers_dev.grid[grid_index]);
695 const T r3xy = r3.x * r3.y;
696 const T r3xz = r3.x * r3.z;
697 const T r3yz = r3.y * r3.z;
699 grid_value *= exp(-(r3x2 + r3y2 + r3z2) *
task.zetp);
701 switch (ico / lbatch) {
703 accumulator[0][
tid] += grid_value;
706 accumulator[1][
tid] += grid_value * r3.x;
707 accumulator[2][
tid] += grid_value * r3.y;
708 accumulator[3][
tid] += grid_value * r3.z;
712 accumulator[4][
tid] += grid_value * r3x2;
713 accumulator[5][
tid] += grid_value * r3xy;
714 accumulator[6][
tid] += grid_value * r3xz;
715 accumulator[7][
tid] += grid_value * r3y2;
716 accumulator[8][
tid] += grid_value * r3yz;
717 accumulator[9][
tid] += grid_value * r3z2;
720 T tmp = grid_value * r3x2;
721 accumulator[10][
tid] += tmp * r3.x;
722 accumulator[11][
tid] += tmp * r3.y;
723 accumulator[12][
tid] += tmp * r3.z;
724 tmp = grid_value * r3.x;
725 accumulator[13][
tid] += tmp * r3y2;
726 accumulator[14][
tid] += tmp * r3yz;
727 accumulator[15][
tid] += tmp * r3z2;
728 tmp = grid_value * r3y2;
729 accumulator[16][
tid] += tmp * r3.y;
730 accumulator[17][
tid] += tmp * r3.z;
731 tmp = grid_value * r3z2;
732 accumulator[18][
tid] += tmp * r3.y;
733 accumulator[19][
tid] += tmp * r3.z;
738 T tmp = grid_value * r3x2;
739 accumulator[0][
tid] += tmp * r3x2;
740 accumulator[1][
tid] += tmp * r3xy;
741 accumulator[2][
tid] += tmp * r3xz;
742 accumulator[3][
tid] += tmp * r3y2;
743 accumulator[4][
tid] += tmp * r3yz;
744 accumulator[5][
tid] += tmp * r3z2;
745 tmp = grid_value * r3y2;
746 accumulator[6][
tid] += tmp * r3xy;
747 accumulator[7][
tid] += tmp * r3xz;
748 tmp = grid_value * r3z2;
749 accumulator[8][
tid] += tmp * r3xy;
750 accumulator[9][
tid] += tmp * r3xz;
753 T tmp = grid_value * r3y2;
754 accumulator[10][
tid] += tmp * r3y2;
755 accumulator[11][
tid] += tmp * r3yz;
756 accumulator[12][
tid] += tmp * r3z2;
757 accumulator[13][
tid] += grid_value * r3yz * r3z2;
758 accumulator[14][
tid] += grid_value * r3z2 * r3z2;
761 T tmp = grid_value * r3x2 * r3x2;
762 accumulator[15][
tid] += tmp * r3.x;
763 accumulator[16][
tid] += tmp * r3.y;
764 accumulator[17][
tid] += tmp * r3.z;
765 tmp = grid_value * r3x2;
766 accumulator[18][
tid] += tmp * r3.x * r3y2;
767 accumulator[19][
tid] += tmp * r3xy * r3.z;
771 T tmp = grid_value * r3x2;
772 accumulator[0][
tid] += tmp * r3.x * r3z2;
773 accumulator[1][
tid] += tmp * r3y2 * r3.y;
774 accumulator[2][
tid] += tmp * r3y2 * r3.z;
775 accumulator[3][
tid] += tmp * r3.y * r3z2;
776 accumulator[4][
tid] += tmp * r3z2 * r3.z;
777 tmp = grid_value * r3.x * r3y2;
778 accumulator[5][
tid] += tmp * r3y2;
779 accumulator[6][
tid] += tmp * r3yz;
780 accumulator[7][
tid] += tmp * r3z2;
781 tmp = grid_value * r3.x * r3z2;
782 accumulator[8][
tid] += tmp * r3yz;
783 accumulator[9][
tid] += tmp * r3z2;
784 tmp = grid_value * r3y2 * r3.y;
785 accumulator[10][
tid] += tmp * r3y2;
786 accumulator[11][
tid] += tmp * r3yz;
787 accumulator[12][
tid] += tmp * r3z2;
788 accumulator[13][
tid] += grid_value * r3y2 * r3z2 * r3.z;
789 accumulator[14][
tid] += grid_value * r3.y * r3z2 * r3z2;
790 accumulator[15][
tid] += grid_value * r3z2 * r3z2 * r3.z;
792 tmp = grid_value * r3x2 * r3x2;
793 accumulator[16][
tid] += tmp * r3x2;
794 accumulator[17][
tid] += tmp * r3xy;
795 accumulator[18][
tid] += tmp * r3xz;
796 accumulator[19][
tid] += tmp * r3y2;
800 T tmp = grid_value * r3x2;
801 accumulator[0][
tid] += tmp * r3x2 * r3yz;
802 accumulator[1][
tid] += tmp * r3x2 * r3z2;
803 accumulator[2][
tid] += tmp * r3y2 * r3xy;
804 accumulator[3][
tid] += tmp * r3y2 * r3xz;
805 accumulator[4][
tid] += tmp * r3xy * r3z2;
806 accumulator[5][
tid] += tmp * r3z2 * r3xz;
807 accumulator[6][
tid] += tmp * r3y2 * r3y2;
808 accumulator[7][
tid] += tmp * r3y2 * r3yz;
809 accumulator[8][
tid] += tmp * r3y2 * r3z2;
810 accumulator[9][
tid] += tmp * r3z2 * r3yz;
811 accumulator[10][
tid] += tmp * r3z2 * r3z2;
812 tmp = grid_value * r3y2 * r3y2;
813 accumulator[11][
tid] += tmp * r3xy;
814 accumulator[12][
tid] += tmp * r3xz;
815 accumulator[13][
tid] +=
816 grid_value * r3y2 * r3xy * r3z2;
817 accumulator[14][
tid] +=
818 grid_value * r3y2 * r3z2 * r3xz;
819 accumulator[15][
tid] += grid_value * r3xy * r3z2 * r3z2;
820 accumulator[16][
tid] += grid_value * r3z2 * r3z2 * r3xz;
821 accumulator[17][
tid] += tmp * r3y2;
822 accumulator[18][
tid] += tmp * r3yz;
823 accumulator[19][
tid] += tmp * r3z2;
826 for (
int ic = 0; (ic < lbatch) && ((ic + ico) < length); ic++) {
829 for (
int po = 0; po < (co.l[2] >> 1); po++)
833 for (
int po = 0; po < (co.l[1] >> 1); po++)
837 for (
int po = 0; po < (co.l[0] >> 1); po++)
841 accumulator[ic][
tid] += tmp * grid_value;
849 const int max_i = min(length - ico, lbatch);
858 for (
int i = 0;
i < max_i;
i++) {
860 T val = accumulator[
i][
tid] + accumulator[
i][
tid + 32];
871#if defined(__CUDACC__)
872 val += __shfl_down_sync(0xffffffff, val,
offset);
874 val += __shfl_down(val,
offset);
879 accumulator[
i][0] = val;
881#if defined(__CUDACC__)
886#if !defined(__CUDACC__)
890 if (
tid < min(length - ico, lbatch)) {
891 const size_t coef_offset =
892 dev_.tasks[dev_.first_task +
block_index()].coef_offset;
893 dev_.buffers_dev.coef[coef_offset +
tid + ico] = accumulator[
tid][0];
940template <
bool distributed__,
bool axis_aligned_,
int lmax_>
946 constexpr bool orthogonal_ = axis_aligned_;
950 static_assert(lmax_ <=
GRID_SEP_MAX_L,
"lmax_ larger than GRID_SEP_MAX_L");
958 dev_.separable_lp_min)
988 const bool trim = orthogonal_ && !
task.apply_border_mask;
992 for (
int x =
tid; x <
task.cube_size.x; x += 64) {
993 const double u = x +
task.lb_cube.x +
task.roffset.x;
995 const double r = u * dev_.dh_[0];
996 ex[x] = exp(-
zetp * r * r);
1001 for (
int y =
tid; y <
task.cube_size.y; y += 64) {
1002 const double v = y +
task.lb_cube.y +
task.roffset.y;
1003 const double r = axis_aligned_ ? v * dev_.dh_[4] : v;
1004 double p = axis_aligned_ ? exp(-
zetp * r * r) : 1.0;
1005 for (
int b = 0;
b <=
lp;
b++,
p *= r)
1017 for (
int z = 0; z <
task.cube_size.z; z++) {
1021 if (distributed__ &&
task.apply_border_mask &&
1022 ((z2 <
task.window_shift.z) || (z2 >
task.window_size.z)))
1025 int ymin = 0, ymax =
task.cube_size.y - 1;
1026 double kremain = 0.0;
1032 const double w = z +
task.lb_cube.z +
task.roffset.z;
1033 const double rz = axis_aligned_ ?
w * dev_.dh_[8] :
w;
1041 double p = axis_aligned_ ? exp(-
zetp * rz * rz) : 1.0;
1042 for (
int c = 0;
c <=
lp;
c++,
p *= rz)
1049 for (
int y0 = ymin; y0 <= ymax; y0 +=
nrow_groups) {
1051 const bool row_valid = (y <= ymax);
1052 double part[lmax_ + 1];
1054 for (
int a = 0;
a <= lmax_;
a++)
1059 const bool row_masked =
1060 distributed__ &&
task.apply_border_mask &&
1061 ((y2 <
task.window_shift.y) || (y2 >
task.window_size.y));
1063 if (row_valid && !row_masked) {
1064 int xmin = 0, xmax =
task.cube_size.x - 1;
1068 const double v = y +
task.lb_cube.y +
task.roffset.y;
1069 const double ry = v * dev_.dh_[4];
1070 const double r2yz = ry * ry + rz * rz;
1071 const double *__restrict__ row =
1072 &dev_.buffers_dev.grid[(z2 * dev_.grid_local_size_.y + y2) *
1073 dev_.grid_local_size_.x];
1076 const int nx_full = dev_.grid_full_size_.x;
1081 if (distributed__ &&
task.apply_border_mask &&
1082 ((x2 <
task.window_shift.x) || (x2 >
task.window_size.x)))
1085 const double u = x +
task.lb_cube.x +
task.roffset.x;
1086 const double r = axis_aligned_ ? u * dev_.dh_[0] : u;
1088 if (axis_aligned_) {
1092 r2 = r3.x * r3.x + r3.y * r3.y + r3.z * r3.z;
1096 const bool outside = (
r2max <= r2);
1097 if (distributed__) {
1098 if (outside && (!orthogonal_ ||
task.apply_border_mask))
1100 }
else if (!orthogonal_ && outside) {
1106 double p = __ldg(&row[x2]);
1107 p *= axis_aligned_ ?
ex[x] : exp(-
zetp * r2);
1109 for (
int a = 0;
a <= lmax_;
a++) {
1121 for (
int a = 0;
a <= lmax_;
a++) {
1124 if (row_valid &&
lane == 0)
1125 A[(y - ymin) *
ls + a] = val;
1143 for (
int ip =
tid; ip <
npairs; ip += 64) {
1147 for (
int y = ymin; y <= ymax; y++)
1148 bab +=
A[(y - ymin) *
ls +
a] *
py[
b *
mc + y];
1149 for (
int c = 0;
c <=
lp -
a -
b;
c++)
1161 double *
coef = &dev_.buffers_dev.coef[
glb_task.coef_offset];
1162 if (axis_aligned_) {
1169 const int l = co.l[0] + co.l[1] + co.l[2];
1170 const int base = (l > 0) ?
ncoset(l - 1) : 0;
1171 const int nl =
ncoset(l) - base;
1172 const double *__restrict__ m =
1175 for (
int j = 0; j < nl; j++)
1176 val += m[j] *
acc[base + j];
1182static size_t integrate_separable_smem_size(
const int mc,
const int lp_max) {
1187using integrate_separable_kernel_t = void (*)(
const kernel_params,
const int,
1188 const int,
const double *);
1195static integrate_separable_kernel_t
1198 return axis_aligned ? integrate_separable_kernel<true, true, lmax_>
1199 : integrate_separable_kernel<true, false, lmax_>;
1200 return axis_aligned ? integrate_separable_kernel<false, true, lmax_>
1201 : integrate_separable_kernel<false, false, lmax_>;
1208static integrate_separable_kernel_t
1210 const bool axis_aligned) {
1238 *lp_diff = smem_params.
lp_diff();
1241 kernel_params params = set_kernel_parameters(level, smem_params);
1251 const int max_cube_size =
grid.max_cube_size();
1252 const bool axis_aligned =
grid.is_axis_aligned();
1256 const integrate_separable_kernel_t separable_kernel =
1258 const bool use_separable =
1261 (axis_aligned || !
grid.is_orthogonal()) &&
1262 (integrate_separable_smem_size(max_cube_size,
lp_max) <=
1275 const bool use_integrate_kernel =
1277 const dim3 threads_per_block(4, 4, 4);
1278 if (!use_integrate_kernel) {
1280 }
else if (
grid_[level].is_distributed()) {
1281 if (
grid_[level].is_orthogonal()) {
1282 integrate_kernel<double, double3, true, true>
1286 integrate_kernel<double, double3, true, false>
1291 if (
grid_[level].is_orthogonal()) {
1292 integrate_kernel<double, double3, false, true>
1296 integrate_kernel<double, double3, false, false>
1302 if (use_integrate_kernel)
1312 const size_t smem = integrate_separable_smem_size(max_cube_size,
lp_max);
1314 separable_kernel<<<nblocks_level, 64, smem, level_streams[level]>>>(
1331 kernel_params params = set_kernel_parameters(-1, smem_params);
1337 const dim3 threads_per_block(4, 4, 4);
1343 const size_t smem_two_steps =
1347 const auto two_steps_kernel =
1349 ? (
compute_tau ? compute_hab_forces_two_steps<double, true>
1350 : compute_hab_forces_two_steps<double, false>)
1351 : (
compute_tau ? compute_hab_two_steps<double, true>
1352 : compute_hab_two_steps<double, false>);
1353 if (smem_two_steps <=
1355 two_steps_kernel<<<this->
nblocks, threads_per_block, smem_two_steps,
1362 const auto fused_kernel =
1364 ? (
compute_tau ? compute_hab_forces<double, double3, true>
1365 : compute_hab_forces<double, double3, false>)
1367 : compute_hab<double, false>);
1369 fused_kernel<<<this->
nblocks, threads_per_block, smem_fused,
std::vector< offloadStream_t > level_streams
offloadStream_t main_stream
void compute_hab_coefficients()
std::vector< int > number_of_tasks_per_level_
void integrate_one_grid_level(const int level, int *lp_diff)
Launches the Cuda kernel that integrates all tasks of one grid level.
void prepare_cab_workspace(const int la_max_diff, const int lb_max_diff)
Lays out the cab workspace of the coefficient kernels for the angular momentum offsets of the current...
std::vector< grid_info< double > > grid_
static void const int const int const int const int const int const double const int const int const int int GRID_CONST_WHEN_COLLOCATE double GRID_CONST_WHEN_INTEGRATE double * grid
static void const int const int i
#define GRID_SEP_INTEGRATE_ROW_LANES
fill_smem_task_reduced(dev_, dev_.first_task+block_index(), task)
__device__ __inline__ T block_reduce_64(T *table, const T val_, const int tid)
__device__ __inline__ int sep_advance_wrapped_index(const int x2, const int n)
int separable_lmax_bucket(const int lp_max)
__device__ __inline__ T get_hab(const orbital &a, const orbital &b, const T zeta, const T zetb, const int n, const T *cab)
Returns element i,j of hab matrix.
constexpr size_t sphi_two_step_max_smem
int sphi_work_size(const int lmax, const int max_nsgf_set)
Size (in doubles) of the work arrays of the two step spherical <-> cartesian transformations of calcu...
static integrate_separable_kernel_t select_integrate_separable(const bool distributed, const bool axis_aligned)
Selects the instance of integrate_separable_kernel for one value of lmax_.
__inline__ __host__ __device__ double3 compute_coordinates(const double *__restrict__ dh_, const double x, const double y, const double z)
const task_info & glb_task
static void init_constant_memory()
Initializes the device's constant memory.
__device__ __inline__ T get_force_b(const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const T rab[3], const int n, const T *cab)
Returns i'th component of force on atom b.
__inline__ __device__ unsigned int block_index()
__device__ __inline__ void setup_task_cube_center(const kernel_params &dev_, smem_task_reduced< T, T3 > &task)
Shifts task.cube_center into the local grid's coordinate system and, for distributed grids,...
__device__ __inline__ T calculate_ymix_ymax_boundaries(smem_task_reduced< T, T3 > &task, const int z, int &ymin, int &ymax)
__shared__ double sep_smem[]
__device__ __inline__ void fill_smem_task_coef(const kernel_params &dev, const int task_id, smem_task< T > &task)
Copies a task from global to shared memory and does precomputations.
constexpr int nrow_groups
__inline__ __device__ unsigned int thread_global_index()
constexpr size_t block_reduce_64_smem
constexpr int integrate_separable_lp_threshold
ldiffs_value process_get_ldiffs(bool calculate_forces, bool calculate_virial, bool compute_tau)
Returns difference in angular momentum range for given flags.
__device__ __inline__ T get_virial_a(const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const int n, const T *cab)
Returns element i,j of virial on atom a.
const int number_of_tasks
__device__ __inline__ void accumulate_pair_forces(const smem_task< T > &task, const orbital &a, const orbital &b, const T p, const bool compute_virial, const T *__restrict__ cab, T *fa, T *fb, T *virial)
Adds the contribution of the cartesian pair (a, b), weighted by the cartesian density p,...
static void init_separable_constant_memory()
Initializes the constant memory of integrate_separable_kernel.
__inline__ __device__ int wrap_grid_index(const int idx, const int full_size)
Wraps a cube-local coordinate into the periodic full grid along one axis. Single point of entry share...
__constant__ orbital coset_inv[1330]
size_t max_dynamic_smem(Kernel kernel, const size_t budget)
Largest dynamic shared memory the kernel can be launched with when its static and dynamic shared memo...
__host__ __device__ __inline__ int coset(int lx, int ly, int lz)
Maps three angular momentum components to a single zero based index.
__global__ const int const int max_nsgf_set
static const double * get_lattice_to_cartesian(grid_info< double > &grid, const int lmax)
Returns the lattice to cartesian matrix of a grid level on the device, (re)computing it when the latt...
__global__ __launch_bounds__(64) void calculate_coefficients(const kernel_params dev_)
Same as calculate_coefficients, with the spherical to cartesian transformation of block_to_cab done i...
__inline__ __device__ T * allocate_workspace(const kernel_params &dev_)
__device__ __inline__ double get_force_a(const orbital &a, const orbital &b, const int i, const T zeta, const T zetb, const int n, const T *cab)
Returns i'th component of force on atom a.
if(number_of_tasks==0) return
__host__ __device__ __inline__ int ncoset(const int l)
Number of Cartesian orbitals up to given angular momentum quantum.
__constant__ int sep_m_offset[GRID_SEP_MAX_L+1]
__device__ __inline__ void cxyz_to_cab_factorized(const smem_task< T > &task, const T *__restrict__ cxyz, T *__restrict__ cab)
Transforms the coefficients C_xyz into C_ab, with the 3D multi-indices a, b, c,.
__inline__ __device__ double sep_row_reduce(double val)
__device__ __inline__ void calculate_xmin_xmax_boundaries(smem_task_reduced< T, T3 > &task, const int y, const T kremain, int &xmin, int &xmax)
__global__ const int const int const double *__restrict__ lattice_to_cartesian
__shared__ smem_task_reduced< double, double3 > task
const double *__restrict__ coef
__constant__ char2 sep_pair_ab[(GRID_SEP_MAX_L+1) *(GRID_SEP_MAX_L+2)/2]
__device__ __inline__ double get_virial_b(const orbital &a, const orbital &b, const int i, const int j, const T zeta, const T zetb, const T rab[3], const int n, const T *cab)
Returns element i,j of virial on atom b.
constexpr size_t separable_max_smem
__global__ const int const int lp_max
const bool compute_virial
Parameters of the collocate kernel.
Differences in angular momentum.
Orbital angular momentum.
data needed for collocate and integrate kernels
data needed for calculating the coefficients, forces, and stress
Internal representation of a task.
Orbital angular momentum.