53#include "./base/base_uses.f90"
57 CHARACTER(len=*),
PARAMETER,
PRIVATE :: moduleN =
'kpoint_transform'
58 INTEGER(KIND=int_8),
PARAMETER :: fft_batch_bytes = 32_int_8*1024_int_8**2
59 INTEGER(KIND=int_8),
PARAMETER :: rskp_panel_bytes = 64_int_8*1024_int_8**2
60 CHARACTER(LEN=14),
PARAMETER :: fft_profile_stages(9) = [character(len=14) :: &
61 'initialize',
'geometry',
'buffers',
'grid-fill',
'FFT-X/plan',
'FFT-Y/plan',
'FFT-Z/plan',
'extract',
'release']
62 INTEGER(KIND=int_8),
PARAMETER,
PRIVATE :: rskp_gemm_bytes = 1000_int_8*1024_int_8**2, &
63 dbcsr_batch_bytes = 1000_int_8*1024_int_8**2
79 INTEGER :: first(4) = 1
80 INTEGER,
ALLOCATABLE :: coordinate(:)
81 REAL(kind=
dp),
ALLOCATABLE :: cosine(:, :), sine(:, :)
85 INTEGER :: nentry = 0, ngroup = 0
86 LOGICAL :: symmetric = .false.
87 INTEGER,
ALLOCATABLE,
DIMENSION(:) :: col, col_offset, group_start, image, row, row_offset
88 INTEGER,
ALLOCATABLE,
DIMENSION(:, :) :: cell
89 REAL(kind=
dp),
ALLOCATABLE,
DIMENSION(:) :: symmetry_sign
93 TYPE rskp_backend_plan_type
95 INTEGER,
ALLOCATABLE :: packed_first(:), fold_map(:, :)
96 INTEGER,
ALLOCATABLE :: phase_index(:), phase_cells(:, :), phase_count(:, :), phase_first(:)
97 INTEGER(KIND=int_8) :: max_per_k = 0_int_8
98 REAL(kind=
dp),
ALLOCATABLE :: phases(:, :, :)
100 END TYPE rskp_backend_plan_type
108 TYPE(rskp_backend_plan_type) :: transform = rskp_backend_plan_type()
109 INTEGER :: nrow = 0, ncol = 0, nimage = 0, nvalue = 0
110 LOGICAL :: world_target = .false.
111 INTEGER(KIND=int_8) :: storage = 0_int_8
112 INTEGER,
ALLOCATABLE :: entry(:, :), block_first(:), block_order(:), element_first(:)
113 INTEGER(KIND=int_8),
ALLOCATABLE :: element_offset(:, :)
114 INTEGER,
ALLOCATABLE :: row_size(:), col_size(:), first(:)
115 INTEGER,
ALLOCATABLE :: send_entry(:), send_first(:), scount(:, :), rcount(:, :), &
116 sdisp(:, :), rdisp(:, :), send_element(:)
117 REAL(kind=
dp),
ALLOCATABLE :: send_values(:), recv_values(:)
122 REAL(kind=
dp),
ALLOCATABLE :: packed(:)
123 LOGICAL,
ALLOCATABLE :: active(:, :)
125 COMPLEX(KIND=dp),
ALLOCATABLE :: fft_values(:, :)
127 INTEGER :: matrix_row = 0, mirror_sign = 1
136 INTEGER,
ALLOCATABLE :: kp_dist(:, :)
137 REAL(kind=
dp),
ALLOCATABLE :: work(:)
159 profile_fft, profile_label)
160 REAL(kind=
dp),
INTENT(IN) :: values_rs(:, :, :)
161 INTEGER,
INTENT(IN) :: cells(:, :)
162 REAL(kind=
dp),
INTENT(IN) :: xkp(:, :)
163 INTEGER,
INTENT(IN) :: nkp_grid(3)
164 COMPLEX(KIND=dp),
INTENT(INOUT) :: values_k(:, :, :)
165 INTEGER,
INTENT(IN) :: backend
166 LOGICAL,
INTENT(IN),
OPTIONAL :: accumulate, profile_fft
167 CHARACTER(LEN=*),
INTENT(IN),
OPTIONAL :: profile_label
169 INTEGER :: column, count, first_column, first_row, handle, i, image, kfirst, last, &
170 last_column, matrix_rows, nk, nphase, nrow, ntile, nworkers, odd, offset, omp_limit, &
171 panels(4), pos, row, sense, start, unit_nr, width
172 INTEGER(KIND=int_8) :: cache_bytes, ifirst, nvalues, &
174 LOGICAL :: active(4), add, measure_fft
175 REAL(kind=
dp) :: profile_started, started
176 REAL(kind=
dp),
ALLOCATABLE :: factors(:, :), packed(:), result(:, :)
178 TYPE(rskp_backend_plan_type) :: plan
180 CALL timeset(
'cell_to_k_transform', handle)
181 measure_fft = .false.
182 IF (
PRESENT(profile_fft)) measure_fft = profile_fft .AND. backend ==
rskp_backend_fft
183 IF (measure_fft)
THEN
184 ALLOCATE (fft_profile)
187 cpassert(
SIZE(cells, 1) == 3 .AND.
SIZE(xkp, 1) == 3)
188 cpassert(
SIZE(cells, 2) ==
SIZE(values_rs, 3))
189 cpassert(
SIZE(values_k, 1) ==
SIZE(values_rs, 1))
190 cpassert(
SIZE(values_k, 2) ==
SIZE(values_rs, 2))
192 cpassert(
SIZE(values_k, 3) == nk)
194 IF (
PRESENT(accumulate)) add = accumulate
197 IF (measure_fft) fft_profile%seconds(1) =
m_walltime() - profile_started
198 IF (
SIZE(values_rs, kind=
int_8) == 0 .OR. nk == 0)
THEN
199 CALL timestop(handle)
202 SELECT CASE (backend)
205 require_fft=.true., accumulate=add, profile=fft_profile)
207 nvalues =
SIZE(values_rs, 1, kind=
int_8)*
SIZE(values_rs, 2, kind=
int_8)
208 matrix_rows =
SIZE(values_rs, 1)
211 worker_bytes = rskp_panel_bytes/(2*nworkers)
212 ntile = int(min(nvalues, 4096_int_8, &
213 max(1_int_8, worker_bytes/(16_int_8*(
SIZE(cells, 2, kind=
int_8) + 1)))))
214 CALL cell_to_k_phase_prepare(plan, cells, [1,
SIZE(cells, 2) + 1], &
215 [(image, image=1,
SIZE(cells, 2))], [1, ntile + 1], &
216 xkp, rskp_panel_bytes/2, 0_int_8)
217 cache_bytes = 0_int_8
218 IF (
ALLOCATED(plan%phases)) cache_bytes = 8_int_8*
SIZE(plan%phases, kind=
int_8)
219 worker_bytes = (rskp_panel_bytes - cache_bytes)/nworkers - &
220 8_int_8*(plan%packed_first(2) - 1)
221 width = int(min(int(nk,
int_8), max(1_int_8, worker_bytes/(16_int_8*plan%max_per_k))))
222 nphase = maxval(plan%phase_count(1:2, 1))
223 panels = plan%phase_count([1, 1, 2, 3], 1)
228 ALLOCATE (packed(plan%packed_first(2) - 1), factors(nphase, 2*width), result(ntile, 2*width))
230 DO ifirst = 1, nvalues, ntile
231 nrow = int(min(int(ntile,
int_8), nvalues - ifirst + 1))
232 first_row = int(
modulo(ifirst - 1, int(matrix_rows,
int_8))) + 1
233 first_column = int((ifirst - 1)/int(matrix_rows,
int_8)) + 1
234 last_column = int((ifirst + int(nrow,
int_8) - 2)/int(matrix_rows,
int_8)) + 1
236 DO image = 1,
SIZE(values_rs, 3)
237 pos = plan%fold_map(1, image)
238 odd = abs(plan%fold_map(2, image))
239 sense = sign(1, plan%fold_map(2, image))
244 DO column = first_column, last_column
245 count = min(nrow - offset, matrix_rows - row + 1)
246 packed(pos + offset:pos + offset + count - 1) = &
247 packed(pos + offset:pos + offset + count - 1) + values_rs(row:row + count - 1, column, image)
249 packed(odd + offset:odd + offset + count - 1) = &
250 packed(odd + offset:odd + offset + count - 1) + &
251 REAL(sense,
dp)*values_rs(row:row + count - 1, column, image)
253 offset = offset + count
260 IF (panels(i) > 0) active(i) = any(packed(start:start + ntile*panels(i) - 1) /= 0.0_dp)
261 start = start + ntile*panels(i)
263 DO kfirst = 1, nk, width
264 last = min(kfirst + width - 1, nk)
265 CALL cell_to_k_block_gemm(packed, plan, 1, xkp(:, kfirst:last), kfirst, &
266 factors(:, 1:2*(last - kfirst + 1)), &
267 result(:, 1:2*(last - kfirst + 1)), active)
270 DO column = first_column, last_column
271 count = min(nrow - offset, matrix_rows - row + 1)
272 values_k(row:row + count - 1, column, kfirst:last) = &
273 values_k(row:row + count - 1, column, kfirst:last) + &
274 cmplx(result(offset + 1:offset + count, 1:last - kfirst + 1), &
275 result(offset + 1:offset + count, last - kfirst + 2:2*(last - kfirst + 1)), kind=
dp)
276 offset = offset + count
282 DEALLOCATE (packed, factors, result)
285 cpabort(
'Unknown R-to-K backend')
287 IF (measure_fft)
THEN
292 IF (unit_nr > 0)
THEN
293 WRITE (unit_nr,
'(T2,A)')
'R2K_FFT_PROFILE| Local panels; logger-rank wall seconds, no MPI reduction'
294 IF (
PRESENT(profile_label))
WRITE (unit_nr,
'(T2,A,A)')
'R2K_FFT_PROFILE| operation: ', trim(profile_label)
295 WRITE (unit_nr,
'(T2,A,3I10)')
'R2K_FFT_PROFILE| source shape: ', shape(values_rs)
296 WRITE (unit_nr,
'(T2,A,6I8)')
'R2K_FFT_PROFILE| grid / padded: ', nkp_grid, fft_profile%nfft
297 WRITE (unit_nr,
'(T2,A,2I16,2I8)')
'R2K_FFT_PROFILE| elements/batches/tile min/max: ', &
298 fft_profile%elements, fft_profile%batches, fft_profile%tile_min, fft_profile%tile_max
299 WRITE (unit_nr,
'(T2,A,I8,2L3)')
'R2K_FFT_PROFILE| FFT OpenMP limit / input,output contiguous: ', &
300 omp_limit, is_contiguous(values_rs), is_contiguous(values_k)
301 IF (fft_profile%pair_major)
WRITE (unit_nr,
'(T2,A)') &
302 'R2K_FFT_PROFILE| FFT-X/plan reports full 3D batched FFT (Y/Z entries zero)'
303 WRITE (unit_nr,
'(T2,A)') &
304 'R2K_FFT_PROFILE| grid-fill/extract use size-gated OpenMP; FFT axes include one-shot planning'
305 DO i = 1,
SIZE(fft_profile_stages)
306 WRITE (unit_nr,
'(T2,A,A14,F14.6)')
'R2K_FFT_PROFILE| ', fft_profile_stages(i), fft_profile%seconds(i)
308 WRITE (unit_nr,
'(T2,A,F14.6)')
'R2K_FFT_PROFILE| total ', started
311 CALL timestop(handle)
328 COMPLEX(KIND=dp),
CONTIGUOUS,
INTENT(IN),
TARGET :: values_k(:, :, :)
329 REAL(kind=
dp),
INTENT(IN) :: xkp(:, :)
330 INTEGER,
INTENT(IN) :: nkp_grid(3), cells(:, :)
331 REAL(kind=
dp),
CONTIGUOUS,
INTENT(OUT),
TARGET :: values_rs(:, :, :)
332 INTEGER,
INTENT(IN) :: backend
333 LOGICAL,
INTENT(IN),
OPTIONAL :: profile_fft
335 INTEGER :: handle, i, ncell, nk, omp_limit, unit_nr
336 LOGICAL :: measure_fft
337 REAL(kind=
dp) :: profile_started, started
341 CALL timeset(
'k_to_real_cell_transform', handle)
342 measure_fft = .false.
343 IF (
PRESENT(profile_fft)) measure_fft = profile_fft .AND. backend ==
rskp_backend_fft
344 IF (measure_fft)
THEN
345 ALLOCATE (fft_profile)
349 ncell =
SIZE(cells, 2)
350 cpassert(
SIZE(cells, 1) == 3 .AND.
SIZE(xkp, 1) == 3)
351 cpassert(nk > 0 .AND.
SIZE(values_k, 3) == nk)
352 cpassert(
SIZE(values_rs, 1) ==
SIZE(values_k, 1))
353 cpassert(
SIZE(values_rs, 2) ==
SIZE(values_k, 2))
354 cpassert(
SIZE(values_rs, 3) == ncell)
355 IF (
SIZE(values_rs, kind=
int_8) == 0)
THEN
356 CALL timestop(handle)
362 IF (measure_fft) fft_profile%seconds(2) =
m_walltime() - started
363 CALL k_grid_to_cell_execute(fft_work, values_k, values_real=values_rs, require_fft=.true., profile=fft_profile)
366 IF (measure_fft) fft_profile%seconds(9) =
m_walltime() - started
367 IF (measure_fft)
THEN
372 IF (unit_nr > 0)
THEN
373 WRITE (unit_nr,
'(T2,A)')
'K2R_FFT_PROFILE| Local panels; logger-rank wall seconds, no MPI reduction'
374 WRITE (unit_nr,
'(T2,A,3I10)')
'K2R_FFT_PROFILE| source shape: ', shape(values_k)
375 WRITE (unit_nr,
'(T2,A,6I8)')
'K2R_FFT_PROFILE| grid / padded: ', nkp_grid, fft_profile%nfft
376 WRITE (unit_nr,
'(T2,A,2I16,2I8)')
'K2R_FFT_PROFILE| elements/batches/tile min/max: ', &
377 fft_profile%elements, fft_profile%batches, fft_profile%tile_min, fft_profile%tile_max
378 WRITE (unit_nr,
'(T2,A,I8,2L3)')
'K2R_FFT_PROFILE| FFT OpenMP limit / input,output contiguous: ', &
379 omp_limit, is_contiguous(values_k), is_contiguous(values_rs)
380 WRITE (unit_nr,
'(T2,A)') &
381 'K2R_FFT_PROFILE| grid-fill/extract use size-gated OpenMP; FFT axes include one-shot planning'
382 WRITE (unit_nr,
'(T2,A,L2)')
'K2R_FFT_PROFILE| pair-major FFT input/output: ', &
383 fft_profile%pair_major
384 IF (fft_profile%pair_major)
WRITE (unit_nr,
'(T2,A)') &
385 'K2R_FFT_PROFILE| FFT-X/plan reports full 3D batched FFT (Y/Z entries zero)'
386 DO i = 1,
SIZE(fft_profile_stages)
387 WRITE (unit_nr,
'(T2,A,A14,F14.6)')
'K2R_FFT_PROFILE| ', fft_profile_stages(i), fft_profile%seconds(i)
389 WRITE (unit_nr,
'(T2,A,F14.6)')
'K2R_FFT_PROFILE| total ', started
392 CALL timestop(handle)
408 xkp, cell_to_index, sab_nl, is_complex, rs_sign)
413 INTEGER,
INTENT(IN) :: ispin
414 REAL(kind=
dp),
DIMENSION(3),
INTENT(IN) :: xkp
415 INTEGER,
DIMENSION(:, :, :),
POINTER :: cell_to_index
418 LOGICAL,
INTENT(IN),
OPTIONAL :: is_complex
419 REAL(kind=
dp),
INTENT(IN),
OPTIONAL :: rs_sign
421 CHARACTER(LEN=*),
PARAMETER :: routinen =
'rskp_transform'
423 INTEGER :: handle, iatom, ic, icol, irow, jatom, &
425 INTEGER,
DIMENSION(3) :: cell
426 LOGICAL :: do_symmetric, found, my_complex, &
428 REAL(kind=
dp) :: arg, coskl, fsign, fsym, sinkl
429 REAL(kind=
dp),
DIMENSION(:, :),
POINTER :: cblock, rblock, rsblock
431 DIMENSION(:),
POINTER :: nl_iterator
433 CALL timeset(routinen, handle)
436 IF (
PRESENT(is_complex)) my_complex = is_complex
439 IF (
PRESENT(rs_sign)) fsign = rs_sign
441 wfn_real_only = .true.
442 IF (
PRESENT(cmatrix)) wfn_real_only = .false.
444 nimg =
SIZE(rsmat, 2)
457 IF (do_symmetric .AND. (iatom > jatom))
THEN
463 ic = cell_to_index(cell(1), cell(2), cell(3))
464 IF (ic < 1 .OR. ic > nimg) cycle
466 arg = real(cell(1),
dp)*xkp(1) + real(cell(2),
dp)*xkp(2) + real(cell(3),
dp)*xkp(3)
468 coskl = fsign*fsym*cos(
twopi*arg)
469 sinkl = fsign*sin(
twopi*arg)
471 coskl = fsign*cos(
twopi*arg)
472 sinkl = fsign*fsym*sin(
twopi*arg)
476 block=rsblock, found=found)
477 IF (.NOT. found) cycle
479 IF (wfn_real_only)
THEN
481 block=rblock, found=found)
482 IF (.NOT. found) cycle
483 rblock = rblock + coskl*rsblock
486 block=rblock, found=found)
487 IF (.NOT. found) cycle
489 block=cblock, found=found)
490 IF (.NOT. found) cycle
491 rblock = rblock + coskl*rsblock
492 cblock = cblock + sinkl*rsblock
498 CALL timestop(handle)
523 world, group, inter, backend, xkp, nkp_grid, kp_dist, mirror_sign, scale)
527 INTEGER,
INTENT(IN) :: source_rows(:)
529 POINTER :: source_sab
530 INTEGER,
DIMENSION(:, :, :),
POINTER :: cell_to_index
534 INTEGER,
INTENT(IN) :: backend
535 REAL(kind=
dp),
INTENT(IN) :: xkp(:, :)
536 INTEGER,
INTENT(IN) :: nkp_grid(3)
537 INTEGER,
INTENT(IN),
OPTIONAL :: kp_dist(:, :), mirror_sign
538 COMPLEX(KIND=dp),
INTENT(IN),
OPTIONAL :: scale
540 CHARACTER(LEN=*),
PARAMETER :: routinen =
'rskp_transform_prepare'
542 INTEGER :: first, handle, irow, nk_local
543 INTEGER(KIND=int_8) :: max_backend_bytes
544 INTEGER,
ALLOCATABLE :: eval_kp_dist(:, :)
546 REAL(kind=
dp),
ALLOCATABLE :: phase_xkp(:, :)
549 CALL timeset(routinen, handle)
551 cpassert(
SIZE(source_rows) > 0)
552 cpassert(all(source_rows >= 1) .AND. all(source_rows <=
SIZE(source, 1)))
553 cpassert(
SIZE(xkp, 1) == 3 .AND.
SIZE(xkp, 2) > 0)
554 cpassert(
ASSOCIATED(target_struct) .AND.
ASSOCIATED(world))
555 IF (
PRESENT(group) .EQV.
PRESENT(inter))
THEN
556 IF (
PRESENT(group))
THEN
557 cpassert(
PRESENT(kp_dist))
558 cpassert(
ASSOCIATED(group) .AND.
ASSOCIATED(inter))
562 cpassert(.NOT.
PRESENT(kp_dist))
563 cpassert(
ASSOCIATED(target_struct%para_env, world))
568 cpabort(
"R-to-K group and inter communicators must be supplied together")
570 IF (
PRESENT(kp_dist))
THEN
571 cpassert(
SIZE(kp_dist, 1) == 2)
572 ALLOCATE (eval_kp_dist, source=kp_dist)
574 ALLOCATE (eval_kp_dist(2, 1))
575 eval_kp_dist(:, 1) = [1,
SIZE(xkp, 2)]
577 max_backend_bytes = rskp_gemm_bytes
579 ALLOCATE (prepared%kp_dist, source=eval_kp_dist)
580 ALLOCATE (prepared%values(
SIZE(source_rows)))
581 CALL rskp_route_create(source, source_sab, cell_to_index, target_struct, world, eval_group, eval_inter, backend, &
582 prepared%route, ready)
583 IF (.NOT. ready)
THEN
584 cpabort(
"R-to-K route has an incompatible source or target layout")
590 first = eval_kp_dist(1, eval_inter%mepos + 1)
591 ALLOCATE (phase_xkp, source=xkp(:, first:eval_kp_dist(2, eval_inter%mepos + 1)))
594 DO irow = 1,
SIZE(source_rows)
595 CALL rskp_capture(source, source_rows(irow), prepared%route, prepared%values(irow), mirror_sign, scale)
598 nk_local =
SIZE(xkp, 2)
599 IF (.NOT. prepared%route%world_target)
THEN
600 nk_local = eval_kp_dist(2, eval_inter%mepos + 1) - &
601 eval_kp_dist(1, eval_inter%mepos + 1) + 1
603 DO irow = 1,
SIZE(source_rows)
604 ALLOCATE (prepared%values(irow)%fft_values( &
605 prepared%route%element_first(
SIZE(prepared%route%element_first)) - 1, nk_local))
607 eval_kp_dist, eval_inter, &
608 max_backend_bytes, cache=prepared%values(irow)%fft_values)
609 DEALLOCATE (prepared%values(irow)%packed)
611 ALLOCATE (prepared%work(0))
613 ALLOCATE (prepared%work(2_int_8*prepared%route%transform%max_per_k))
615 CALL timestop(handle)
633 INTEGER,
INTENT(IN) :: source_index, k
634 REAL(kind=
dp),
INTENT(IN) :: xkp(:, :)
635 TYPE(
cp_fm_type),
INTENT(INOUT),
OPTIONAL,
TARGET :: fm_re, fm_im
636 TYPE(
cp_cfm_type),
INTENT(INOUT),
OPTIONAL,
TARGET :: cfm
638 COMPLEX(KIND=dp),
CONTIGUOUS,
POINTER :: batch(:, :, :)
639 INTEGER :: first_local, ncomp
640 LOGICAL :: target_layout_matches
643 cpassert(
ALLOCATED(prepared%values))
644 cpassert(source_index >= 1 .AND. source_index <=
SIZE(prepared%values))
645 cpassert(
PRESENT(fm_re) .NEQV.
PRESENT(cfm))
646 cpassert(.NOT.
PRESENT(fm_im) .OR.
PRESENT(fm_re))
647 cpassert(
ALLOCATED(prepared%work))
648 cpassert(k >= 1 .AND. k <=
SIZE(xkp, 2))
650 IF (.NOT. prepared%route%world_target)
THEN
651 IF (k < prepared%kp_dist(1, prepared%route%inter%mepos + 1) .OR. &
652 k > prepared%kp_dist(2, prepared%route%inter%mepos + 1))
RETURN
653 first_local = k - prepared%kp_dist(1, prepared%route%inter%mepos + 1) + 1
655 IF (
PRESENT(cfm))
THEN
657 cpassert(target_layout_matches)
658 batch(1:
SIZE(cfm%local_data, 1), 1:
SIZE(cfm%local_data, 2), 1:1) => cfm%local_data
660 xkp(:, k:k), first_local, prepared%work, batch=batch)
662 output(1, 1)%matrix => fm_re
664 IF (
PRESENT(fm_im))
THEN
665 output(1, 2)%matrix => fm_im
669 xkp(:, k:k), first_local, prepared%work, output=output(:, 1:ncomp))
682 IF (
ALLOCATED(prepared%values))
THEN
683 DO irow = 1,
SIZE(prepared%values)
686 DEALLOCATE (prepared%values)
689 IF (
ALLOCATED(prepared%kp_dist))
DEALLOCATE (prepared%kp_dist)
690 IF (
ALLOCATED(prepared%work))
DEALLOCATE (prepared%work)
709 TYPE(
dbcsr_p_type),
POINTER :: source(:, :), real_k(:, :), imag_k(:, :)
710 REAL(kind=
dp),
INTENT(IN) :: xkp(:, :)
711 INTEGER,
DIMENSION(:, :, :),
POINTER :: cell_to_index
714 INTEGER,
INTENT(IN) :: nkp_grid(3), backend
716 CHARACTER(LEN=*),
PARAMETER :: routinen =
'rskp_dbcsr_transform_batch'
718 COMPLEX(KIND=dp),
ALLOCATABLE :: fft_output(:, :, :)
719 INTEGER :: col, first, first_k, g, handle, i, ik, ispin, last, max_images, max_nelem, ncol, &
720 nelem, nentry, nfold, ngroup, nimages, nk, nphase, nrow, nspin, odd, offset, panels(4), &
721 pos, row_first, row_last, tile_first, tile_size, tile_width, width
722 INTEGER(KIND=int_8) :: available_bytes, fft_tile_budget, &
723 nfactor, panel_bytes, per_k_bytes, &
725 INTEGER,
ALLOCATABLE :: element_first(:), group_first(:), &
726 order(:), signed_cells(:, :)
727 INTEGER,
DIMENSION(:),
POINTER :: col_size, row_size
728 LOGICAL :: active(4), found, found_im, found_re
729 REAL(kind=
dp),
ALLOCATABLE,
TARGET :: packed(:), work(:)
730 REAL(kind=
dp),
CONTIGUOUS,
POINTER :: factors(:, :), fft_input(:, :, :), &
731 image_re(:, :), result(:, :)
732 REAL(kind=
dp),
POINTER :: iblock(:, :), rblock(:, :), &
733 result_block(:, :), rsblock(:, :)
736 TYPE(rskp_backend_plan_type) :: transform
738 CALL timeset(routinen, handle)
739 nspin =
SIZE(source, 1)
743 cpabort(
"Unknown R-to-K backend")
745 cpassert(nspin > 0 .AND.
SIZE(real_k, 1) == nspin .AND.
SIZE(imag_k, 1) == nspin)
746 cpassert(
SIZE(real_k, 2) == nk .AND.
SIZE(imag_k, 2) == nk)
747 cpassert(
SIZE(source, 2) > 0)
748 cpassert(
ASSOCIATED(sab_nl) .AND.
ASSOCIATED(cell_to_index))
751 CALL dbcsr_set(real_k(ispin, ik)%matrix, 0.0_dp)
752 CALL dbcsr_set(imag_k(ispin, ik)%matrix, 0.0_dp)
758 IF (nentry == 0)
THEN
759 CALL timestop(handle)
762 CALL dbcsr_get_info(source(1, 1)%matrix, row_blk_size=row_size, col_blk_size=col_size)
763 ALLOCATE (group_first(nentry + 1), element_first(nentry + 1), order(nentry), signed_cells(3, nentry))
766 signed_cells(:, i) = nint(plan%symmetry_sign(i))*plan%cell(:, i)
773 DO WHILE (first <= nentry)
775 DO WHILE (last < nentry)
776 IF (plan%row(last + 1) /= plan%row(first) .OR. plan%col(last + 1) /= plan%col(first))
EXIT
780 group_first(ngroup) = first
781 panel_bytes = int(row_size(plan%row(first)),
int_8)*col_size(plan%col(first))*nspin
782 IF (panel_bytes + element_first(ngroup) > huge(nelem))
THEN
783 cpabort(
"R-to-K DBCSR panel offsets exceed the integer range")
785 element_first(ngroup + 1) = element_first(ngroup) + int(panel_bytes)
786 max_nelem = max(max_nelem, int(panel_bytes))
787 max_images = max(max_images, last - first + 1)
790 group_first(ngroup + 1) = nentry + 1
791 panel_bytes = 8_int_8*max_nelem*max_images
792 available_bytes = dbcsr_batch_bytes - panel_bytes
795 available_bytes = max(0_int_8, available_bytes)
796 CALL cell_to_k_phase_prepare(transform, signed_cells, group_first(1:ngroup + 1), order, &
797 element_first(1:ngroup + 1), xkp, available_bytes, 0_int_8)
798 IF (
ALLOCATED(transform%phases))
THEN
799 available_bytes = available_bytes - 8_int_8*
SIZE(transform%phases, kind=
int_8)
801 per_k_bytes = 16_int_8*transform%max_per_k
802 tile_width = int(min(int(nk,
int_8), max(1_int_8, available_bytes/max(1_int_8, per_k_bytes))))
803 ALLOCATE (work(per_k_bytes/8*tile_width))
805 fft_tile_budget = fft_batch_bytes
807 IF (.NOT. fft_work%regular)
THEN
808 cpabort(
"Forced lattice FFT needs a supported regular grid or subset")
811 ALLOCATE (packed(panel_bytes/8))
814 first = group_first(g)
815 last = group_first(g + 1) - 1
816 nimages = last - first + 1
817 nrow = row_size(plan%row(first))
818 ncol = col_size(plan%col(first))
821 panels = transform%phase_count([1, 1, 2, 3], g)
826 image_re(1:nelem*nspin, 1:nfold) => packed(1:int(nelem,
int_8)*int(nspin,
int_8)*int(nfold,
int_8))
832 pos = (transform%fold_map(1, i) - transform%packed_first(g))/(nelem*nspin) + 1
833 IF (transform%fold_map(2, i) /= 0)
THEN
834 odd = (abs(transform%fold_map(2, i)) - transform%packed_first(g))/(nelem*nspin) + 1
840 IF (.NOT. found) cycle
841 cpassert(
SIZE(rsblock, 1) == nrow .AND.
SIZE(rsblock, 2) == ncol)
842 image_re((ispin - 1)*nelem + 1:ispin*nelem, pos) = &
843 image_re((ispin - 1)*nelem + 1:ispin*nelem, pos) + reshape(rsblock, [nelem])
845 image_re((ispin - 1)*nelem + 1:ispin*nelem, odd) = &
846 image_re((ispin - 1)*nelem + 1:ispin*nelem, odd) + &
847 REAL(sign(1, transform%fold_map(2, i)),
dp)*reshape(rsblock, [nelem])
850 IF (.NOT. any(image_re /= 0.0_dp)) cycle
855 active(i) = any(image_re(:, pos:pos + panels(i) - 1) /= 0.0_dp)
856 pos = pos + panels(i)
858 nphase = maxval(transform%phase_count(1:2, g))
859 DO first_k = 1, nk, tile_width
860 width = min(tile_width, nk - first_k + 1)
861 nfactor = 2_int_8*nphase*width
862 factors(1:nphase, 1:2*width) => work(1:nfactor)
863 result(1:nelem*nspin, 1:2*width) => work(nfactor + 1:nfactor + 2_int_8*nelem*nspin*width)
864 CALL cell_to_k_block_gemm(packed(1:nelem*nspin*nfold), transform, g, &
865 xkp(:, first_k:first_k + width - 1), first_k, factors, result, active)
868 CALL dbcsr_get_block_p(real_k(ispin, first_k + ik - 1)%matrix, plan%row(first), plan%col(first), &
871 cpassert(
SIZE(rblock, 1) == nrow .AND.
SIZE(rblock, 2) == ncol)
872 result_block(1:nrow, 1:ncol) => result((ispin - 1)*nelem + 1:ispin*nelem, ik)
873 rblock = result_block
875 CALL dbcsr_get_block_p(imag_k(ispin, first_k + ik - 1)%matrix, plan%row(first), plan%col(first), &
878 cpassert(
SIZE(iblock, 1) == nrow .AND.
SIZE(iblock, 2) == ncol)
879 result_block(1:nrow, 1:ncol) => result((ispin - 1)*nelem + 1:ispin*nelem, width + ik)
880 iblock = result_block
886 per_tile_bytes = 16_int_8*nspin*nk
888 cpabort(
"R-to-K DBCSR FFT tile exceeds the integer range")
890 tile_width = min(nelem, int(max(1_int_8, min(fft_tile_budget/per_tile_bytes, &
892 fft_input(1:nelem, 1:nspin, 1:nimages) => packed(1:int(nelem,
int_8)*int(nspin,
int_8)*int(nimages,
int_8))
893 ALLOCATE (fft_output(tile_width, nspin, nk))
894 DO tile_first = 1, nelem, tile_width
895 tile_size = min(tile_width, nelem - tile_first + 1)
897 signed_cells(:, first:last), xkp, &
898 nkp_grid, fft_output(1:tile_size, :, :), require_fft=.true., work=fft_work, &
902 CALL dbcsr_get_block_p(real_k(ispin, ik)%matrix, plan%row(first), plan%col(first), &
905 cpassert(
SIZE(rblock, 1) == nrow .AND.
SIZE(rblock, 2) == ncol)
907 CALL dbcsr_get_block_p(imag_k(ispin, ik)%matrix, plan%row(first), plan%col(first), &
910 cpassert(
SIZE(iblock, 1) == nrow .AND.
SIZE(iblock, 2) == ncol)
912 IF (.NOT. (found_re .OR. found_im)) cycle
914 DO col = (tile_first - 1)/nrow + 1, (tile_first + tile_size - 2)/nrow + 1
915 offset = (col - 1)*nrow
916 row_first = max(1, tile_first - offset)
917 row_last = min(nrow, tile_first + tile_size - 1 - offset)
918 pos = offset + row_first - tile_first + 1
919 IF (found_re) rblock(row_first:row_last, col) = &
920 REAL(fft_output(pos:pos + row_last - row_first, ispin, ik),
dp)
921 IF (found_im) iblock(row_first:row_last, col) = &
922 aimag(fft_output(pos:pos + row_last - row_first, ispin, ik))
927 DEALLOCATE (fft_output)
931 CALL timestop(handle)
948 INTEGER,
DIMENSION(:, :, :),
POINTER :: cell_to_index
949 INTEGER,
INTENT(IN) :: nimg
950 TYPE(
dbcsr_type),
INTENT(IN),
OPTIONAL :: block_template
951 LOGICAL,
INTENT(IN),
OPTIONAL :: group_entries
953 INTEGER :: i, iatom, icell, icol, igroup, irow, &
955 INTEGER(KIND=int_8),
ALLOCATABLE,
DIMENSION(:) :: key
956 INTEGER,
ALLOCATABLE,
DIMENSION(:) :: order
957 INTEGER,
DIMENSION(3) :: cell
958 INTEGER,
DIMENSION(:),
POINTER :: col_offsets, row_offsets
959 LOGICAL :: do_grouping, do_symmetric, store_offsets
961 DIMENSION(:),
POINTER :: iterator
964 plan%symmetric = do_symmetric
965 do_grouping = .false.
966 IF (
PRESENT(group_entries)) do_grouping = group_entries
967 store_offsets =
PRESENT(block_template)
968 IF (store_offsets)
THEN
969 CALL dbcsr_get_info(block_template, row_blk_offset=row_offsets, col_blk_offset=col_offsets)
974 icell = cell_to_index(cell(1), cell(2), cell(3))
975 IF (icell >= 1 .AND. icell <= nimg) plan%nentry = plan%nentry + 1
979 ALLOCATE (plan%row(plan%nentry), plan%col(plan%nentry), plan%image(plan%nentry), &
980 plan%cell(3, plan%nentry), plan%symmetry_sign(plan%nentry))
981 IF (store_offsets)
THEN
982 ALLOCATE (plan%row_offset(plan%nentry), plan%col_offset(plan%nentry))
989 icell = cell_to_index(cell(1), cell(2), cell(3))
990 IF (icell < 1 .OR. icell > nimg) cycle
994 plan%symmetry_sign(i) = 1.0_dp
995 IF (do_symmetric .AND. iatom > jatom)
THEN
998 plan%symmetry_sign(i) = -1.0_dp
1002 IF (store_offsets)
THEN
1003 plan%row_offset(i) = row_offsets(irow)
1004 plan%col_offset(i) = col_offsets(icol)
1006 plan%image(i) = icell
1007 plan%cell(:, i) = cell
1010 cpassert(i == plan%nentry)
1012 IF (do_grouping .AND. plan%nentry > 0)
THEN
1013 nblock = max(maxval(plan%row), maxval(plan%col))
1014 ALLOCATE (key(plan%nentry), order(plan%nentry))
1015 DO i = 1, plan%nentry
1016 key(i) = int(plan%image(i), kind=
int_8) + int(nimg, kind=
int_8)* &
1017 (int(plan%row(i) - 1, kind=
int_8) + int(nblock, kind=
int_8)* &
1018 int(plan%col(i) - 1, kind=
int_8))
1020 CALL sort(key, plan%nentry, order)
1021 plan%row(:) = plan%row(order)
1022 plan%col(:) = plan%col(order)
1023 IF (store_offsets)
THEN
1024 plan%row_offset(:) = plan%row_offset(order)
1025 plan%col_offset(:) = plan%col_offset(order)
1027 plan%image(:) = plan%image(order)
1028 plan%cell(:, :) = plan%cell(:, order)
1029 plan%symmetry_sign(:) = plan%symmetry_sign(order)
1032 DO i = 2, plan%nentry
1033 IF (key(i) /= key(i - 1)) plan%ngroup = plan%ngroup + 1
1035 ALLOCATE (plan%group_start(plan%ngroup + 1))
1037 plan%group_start(igroup) = 1
1038 DO i = 2, plan%nentry
1039 IF (key(i) /= key(i - 1))
THEN
1041 plan%group_start(igroup) = i
1044 plan%group_start(plan%ngroup + 1) = plan%nentry + 1
1045 DEALLOCATE (key, order)
1060 SUBROUTINE rskp_gemm_contract(nrow, nimages, nk, values_re, phase_re, result_re, beta)
1061 INTEGER,
INTENT(IN) :: nrow, nimages, nk
1062 REAL(kind=
dp),
CONTIGUOUS,
INTENT(IN) :: values_re(:, :), phase_re(:, :)
1063 REAL(kind=
dp),
CONTIGUOUS,
INTENT(INOUT) :: result_re(:, :)
1064 REAL(kind=
dp),
INTENT(IN) :: beta
1066 cpassert(nrow > 0 .AND. nimages > 0 .AND. nk > 0)
1067 cpassert(
SIZE(values_re, 1) >= nrow .AND.
SIZE(values_re, 2) >= nimages)
1068 cpassert(
SIZE(phase_re, 1) >= nimages .AND.
SIZE(phase_re, 2) >= nk)
1069 cpassert(
SIZE(result_re, 1) >= nrow .AND.
SIZE(result_re, 2) >= nk)
1071 CALL dgemm(
'N',
'N', nrow, nk, nimages, 1.0_dp, values_re,
SIZE(values_re, 1), &
1072 phase_re,
SIZE(phase_re, 1), beta, result_re,
SIZE(result_re, 1))
1073 END SUBROUTINE rskp_gemm_contract
1091 SUBROUTINE rskp_route_create(source, source_sab, cell_to_index, matrix_struct, world, group, inter, &
1092 backend, route, ready)
1096 POINTER :: source_sab
1097 INTEGER,
POINTER :: cell_to_index(:, :, :)
1100 INTEGER,
INTENT(IN) :: backend
1102 LOGICAL,
INTENT(OUT) :: ready
1104 CHARACTER(LEN=*),
PARAMETER :: routinen =
'rskp_route_create'
1106 INTEGER :: compatible, dest, handle, i, iblock, &
1107 layout(11), layout_max(11), &
1108 layout_min(11), nblock, ncol, nowner, &
1110 INTEGER(KIND=int_8) :: nr(2), ns(2), storage
1111 INTEGER(KIND=int_8),
ALLOCATABLE :: recv_size(:, :), send_size(:, :)
1112 INTEGER,
ALLOCATABLE :: identities(:, :), owner(:), &
1113 position(:, :), send_metadata(:, :), &
1114 value_count(:), value_start(:)
1115 INTEGER,
POINTER :: col_size(:), row_size(:)
1116 LOGICAL,
ALLOCATABLE :: needed(:)
1118 CALL timeset(routinen, handle)
1120 route%world => world
1121 route%group => group
1122 route%inter => inter
1123 CALL dbcsr_get_info(source(1, 1)%matrix, row_blk_size=row_size, col_blk_size=col_size)
1125 SIZE(source, 2), block_template=source(1, 1)%matrix)
1126 route%source_sab => source_sab
1127 route%transform%backend = backend
1128 route%world_target =
ASSOCIATED(matrix_struct%para_env, world)
1129 nowner = group%num_pe
1130 IF (route%world_target) nowner = world%num_pe
1131 compatible = merge(1, 0, route%world_target .OR. &
1132 ASSOCIATED(matrix_struct%para_env, group))
1133 IF (.NOT. route%world_target)
THEN
1134 layout = [matrix_struct%nrow_global, matrix_struct%ncol_global, &
1135 matrix_struct%nrow_block, matrix_struct%ncol_block, matrix_struct%first_p_pos, &
1136 matrix_struct%context%num_pe, matrix_struct%context%mepos, matrix_struct%local_leading_dimension]
1139 CALL inter%min(layout_min)
1140 CALL inter%max(layout_max)
1141 IF (any(layout_min /= layout_max)) compatible = 0
1143 IF (sum(row_size) /= matrix_struct%nrow_global .OR. &
1144 sum(col_size) /= matrix_struct%ncol_global) compatible = 0
1145 IF (route%traversal%symmetric)
THEN
1146 IF (matrix_struct%nrow_global /= matrix_struct%ncol_global .OR.
SIZE(row_size) /=
SIZE(col_size))
THEN
1148 ELSE IF (any(row_size /= col_size))
THEN
1152 CALL world%min(compatible)
1153 IF (compatible == 0)
THEN
1154 CALL cp_warn(__location__, &
1155 "Prepared R-to-K transform has an incompatible source or target layout")
1156 CALL timestop(handle)
1159 route%nimage =
SIZE(source, 2)
1160 ALLOCATE (route%row_size(
SIZE(row_size)), route%col_size(
SIZE(col_size)))
1161 route%row_size(:) = row_size
1162 route%col_size(:) = col_size
1163 ALLOCATE (identities(2, 0:world%num_pe - 1), owner(0:nowner - 1), &
1164 needed(0:nowner - 1), value_count(0:nowner - 1), value_start(0:nowner - 1))
1165 CALL world%allgather([group%mepos, inter%mepos], identities)
1167 IF (route%world_target)
THEN
1168 DO rank = 0, world%num_pe - 1
1172 DO rank = 0, world%num_pe - 1
1173 IF (identities(2, rank) == 0) owner(identities(1, rank)) = rank
1176 cpassert(all(owner >= 0))
1177 ALLOCATE (send_size(2, 0:world%num_pe - 1), recv_size(2, 0:world%num_pe - 1))
1178 send_size(:, :) = 0_int_8
1179 DO i = 1, route%traversal%nentry
1180 nrow = row_size(route%traversal%row(i))
1181 ncol = col_size(route%traversal%col(i))
1182 CALL rskp_owners(route, matrix_struct, i, needed, value_count)
1183 DO p = 0,
SIZE(owner) - 1
1184 IF (.NOT. needed(p)) cycle
1186 send_size(1, dest) = send_size(1, dest) + 1_int_8
1187 send_size(2, dest) = send_size(2, dest) + int(value_count(p),
int_8)
1190 CALL world%alltoall(send_size, recv_size, 2)
1193 ns = sum(send_size, dim=2)
1194 nr = sum(recv_size, dim=2)
1195 IF (.NOT. route%world_target)
CALL inter%bcast(nr, 0)
1197 storage = 40_int_8*route%traversal%nentry + &
1198 4_int_8*(
SIZE(row_size) +
SIZE(col_size) + matrix_struct%nrow_global + matrix_struct%ncol_global) + &
1199 48_int_8*ns(1) + merge(0_int_8, 8_int_8*ns(2), world%num_pe == 1) + 68_int_8*nr(1) + &
1200 128_int_8*world%num_pe + 16_int_8*
SIZE(owner) + 8_int_8
1202 IF (
SIZE(owner) == 1) storage = storage + 4_int_8*(nr(1) + 1_int_8)
1205 storage = storage + 52_int_8*(nr(1) + 1_int_8) + 16_int_8*nr(2)
1206 IF (world%num_pe > 1) storage = storage + 8_int_8*nr(2)
1207 IF (
SIZE(owner) > 1) storage = storage + 4_int_8*ns(2)
1208 route%storage = storage
1210 IF (maxval(ns) >= int(huge(0),
int_8) .OR. maxval(nr) >= int(huge(0),
int_8) .OR. &
1211 max(ns(1), nr(1)) > int(huge(0),
int_8)/8_int_8)
THEN
1212 CALL cp_abort(__location__,
"R-to-K route counts exceed the integer range")
1216 route%nvalue = int(nr(2))
1217 ALLOCATE (route%scount(2, 0:world%num_pe - 1), route%rcount(2, 0:world%num_pe - 1), &
1218 route%sdisp(2, 0:world%num_pe - 1), route%rdisp(2, 0:world%num_pe - 1))
1219 route%scount(:, :) = int(send_size)
1220 route%rcount(:, :) = int(recv_size)
1221 IF (
SIZE(owner) == 1 .AND. .NOT. route%world_target)
CALL inter%bcast(route%rcount, 0)
1222 route%sdisp(:, 0) = 0
1223 route%rdisp(:, 0) = 0
1224 DO rank = 1, world%num_pe - 1
1225 route%sdisp(:, rank) = route%sdisp(:, rank - 1) + route%scount(:, rank - 1)
1226 route%rdisp(:, rank) = route%rdisp(:, rank - 1) + route%rcount(:, rank - 1)
1228 ALLOCATE (route%send_entry(int(ns(1))), route%send_first(int(ns(1)) + 1), &
1229 route%send_values(merge(0, int(ns(2)), world%num_pe == 1)), send_metadata(8, int(ns(1))), &
1230 position(2, 0:world%num_pe - 1), route%entry(8, nblock), &
1231 route%first(nblock + 1))
1232 IF (
SIZE(owner) > 1)
ALLOCATE (route%send_element(int(ns(2))))
1233 IF (world%num_pe > 1)
ALLOCATE (route%recv_values(route%nvalue))
1234 position(:, :) = route%sdisp
1235 DO i = 1, route%traversal%nentry
1236 nrow = row_size(route%traversal%row(i))
1237 ncol = col_size(route%traversal%col(i))
1238 IF (
ALLOCATED(route%send_element))
THEN
1239 value_start(:) = position(2, owner)
1240 CALL rskp_owners(route, matrix_struct, i, needed, value_count, &
1241 value_start, route%send_element)
1243 CALL rskp_owners(route, matrix_struct, i, needed, value_count)
1245 DO p = 0,
SIZE(owner) - 1
1246 IF (.NOT. needed(p)) cycle
1248 iblock = position(1, dest) + 1
1249 pos = position(2, dest) + 1
1250 route%send_entry(iblock) = i
1251 route%send_first(iblock) = pos
1252 send_metadata(1:4, iblock) = [route%traversal%row_offset(i), route%traversal%col_offset(i), nrow, ncol]
1253 send_metadata(5:7, iblock) = nint(route%traversal%symmetry_sign(i))*route%traversal%cell(:, i)
1254 send_metadata(8, iblock) = merge(1, 0, route%traversal%symmetric .AND. &
1255 route%traversal%row(i) /= route%traversal%col(i))
1256 position(:, dest) = position(:, dest) + [1, value_count(p)]
1259 route%send_first(int(ns(1)) + 1) = int(ns(2)) + 1
1260 IF (
SIZE(owner) == 1 .AND. world%num_pe == 1)
THEN
1261 CALL world%allgatherv(send_metadata, route%entry, 8*route%rcount(1, :), 8*route%rdisp(1, :))
1262 ELSE IF (.NOT. route%world_target .AND.
SIZE(owner) == 1)
THEN
1263 CALL world%allgatherv(send_metadata, route%entry, 8*route%rcount(1, :), 8*route%rdisp(1, :))
1265 CALL world%alltoall(send_metadata, 8*route%scount(1, :), 8*route%sdisp(1, :), &
1266 route%entry, 8*route%rcount(1, :), 8*route%rdisp(1, :))
1267 IF (.NOT. route%world_target)
CALL inter%bcast(route%entry, 0)
1269 route%matrix_struct => matrix_struct
1271 CALL rskp_tile_map(route)
1273 CALL timestop(handle)
1282 SUBROUTINE rskp_tile_map(route)
1285 INTEGER :: c0, col, element, first, g, i, image, m, &
1286 nc, ngroup, nr, overlap, pos, r0, row
1287 INTEGER(KIND=int_8),
ALLOCATABLE :: key(:)
1288 INTEGER,
ALLOCATABLE :: element_count(:)
1289 LOGICAL :: direct, mirror
1290 INTEGER :: rows(0:route%matrix_struct%nrow_global), cols(0:route%matrix_struct%ncol_global)
1294 DO i = 1,
SIZE(route%matrix_struct%row_indices)
1295 r0 = route%matrix_struct%row_indices(i)
1296 IF (r0 >= 1 .AND. r0 <= ubound(rows, 1)) rows(r0) = 1
1298 DO i = 1,
SIZE(route%matrix_struct%col_indices)
1299 c0 = route%matrix_struct%col_indices(i)
1300 IF (c0 >= 1 .AND. c0 <= ubound(cols, 1)) cols(c0) = 1
1302 DO i = 1, ubound(rows, 1)
1303 rows(i) = rows(i) + rows(i - 1)
1305 DO i = 1, ubound(cols, 1)
1306 cols(i) = cols(i) + cols(i - 1)
1308 route%nrow = rows(ubound(rows, 1))
1309 route%ncol = cols(ubound(cols, 1))
1310 ALLOCATE (route%block_order(
SIZE(route%entry, 2)), element_count(
SIZE(route%entry, 2)))
1313 ALLOCATE (key(
SIZE(route%entry, 2)))
1315 key(i) = int(route%entry(1, i),
int_8) + &
1316 int(route%matrix_struct%nrow_global,
int_8)*(route%entry(2, i) - 1)
1318 IF (
SIZE(key) > 0)
CALL sort(key,
SIZE(key), route%block_order)
1320 IF (
SIZE(key) > 0) ngroup = 1 + count(key(2:) /= key(:
SIZE(key) - 1))
1321 ALLOCATE (route%block_first(ngroup + 1))
1322 route%block_first(1) = 1
1325 IF (key(i) == key(i - 1)) cycle
1327 route%block_first(g) = i
1329 route%block_first(ngroup + 1) =
SIZE(key) + 1
1331 ALLOCATE (route%element_first(ngroup + 1))
1332 route%element_first(1) = 1
1335 first = route%block_order(route%block_first(g))
1336 r0 = route%entry(1, first)
1337 c0 = route%entry(2, first)
1338 nr = route%entry(3, first)
1339 nc = route%entry(4, first)
1340 m = (rows(r0 + nr - 1) - rows(r0 - 1))*(cols(c0 + nc - 1) - cols(c0 - 1))
1341 IF (route%entry(8, first) /= 0)
THEN
1344 overlap = count(rows(r0:r0 + nr - 1) > rows(r0 - 1:r0 + nr - 2) .AND. &
1345 cols(r0:r0 + nr - 1) > cols(r0 - 1:r0 + nr - 2))* &
1346 count(rows(c0:c0 + nc - 1) > rows(c0 - 1:c0 + nc - 2) .AND. &
1347 cols(c0:c0 + nc - 1) > cols(c0 - 1:c0 + nc - 2))
1348 m = m + (rows(c0 + nc - 1) - rows(c0 - 1))*(cols(r0 + nr - 1) - cols(r0 - 1)) - overlap
1350 route%element_first(g + 1) = route%element_first(g) + m
1351 DO i = route%block_first(g), route%block_first(g + 1) - 1
1352 image = route%block_order(i)
1353 element_count(image) = m
1358 DO i = 1,
SIZE(element_count)
1359 route%first(i + 1) = route%first(i) + element_count(i)
1361 cpassert(pos == route%nvalue + 1)
1362 cpassert(route%first(
SIZE(element_count) + 1) == pos)
1363 ALLOCATE (route%element_offset(2, route%element_first(ngroup + 1) - 1))
1364 route%element_offset(:, :) = 0_int_8
1366 first = route%block_order(route%block_first(g))
1367 r0 = route%entry(1, first)
1368 c0 = route%entry(2, first)
1369 nr = route%entry(3, first)
1370 nc = route%entry(4, first)
1371 element = route%element_first(g)
1372 DO col = c0, c0 + nc - 1
1373 DO row = r0, r0 + nr - 1
1374 direct = rows(row) > rows(row - 1) .AND. cols(col) > cols(col - 1)
1376 IF (route%entry(8, first) /= 0)
THEN
1377 mirror = rows(col) > rows(col - 1) .AND. cols(row) > cols(row - 1)
1379 IF (.NOT. (direct .OR. mirror)) cycle
1380 IF (direct) route%element_offset(1, element) = &
1381 int(rows(row),
int_8) + &
1382 int(cols(col) - 1,
int_8)*route%matrix_struct%local_leading_dimension
1383 IF (mirror) route%element_offset(2, element) = &
1384 int(rows(col),
int_8) + &
1385 int(cols(row) - 1,
int_8)*route%matrix_struct%local_leading_dimension
1386 element = element + 1
1389 cpassert(element == route%element_first(g + 1))
1392 ALLOCATE (route%transform%packed_first(ngroup + 1), route%transform%fold_map(2,
SIZE(route%entry, 2)), source=0)
1395 route%transform%packed_first(g) = pos
1396 m = route%element_first(g + 1) - route%element_first(g)
1397 DO i = route%block_first(g), route%block_first(g + 1) - 1
1398 route%transform%fold_map(1, route%block_order(i)) = pos
1402 route%transform%packed_first(ngroup + 1) = pos
1404 END SUBROUTINE rskp_tile_map
1416 SUBROUTINE rskp_owners(route, fms, i, needed, value_count, value_start, value_index)
1419 TYPE(cp_fm_struct_type),
INTENT(IN) :: fms
1420 INTEGER,
INTENT(IN) :: i
1421 LOGICAL,
INTENT(OUT) :: needed(0:)
1422 INTEGER,
INTENT(OUT) :: value_count(0:)
1423 INTEGER,
INTENT(IN),
OPTIONAL :: value_start(0:)
1424 INTEGER,
INTENT(INOUT),
OPTIONAL :: value_index(:)
1426 INTEGER :: c0, col, direct, mirror, nc, nr, r0, row
1427 LOGICAL :: mirror_block
1429 r0 = route%traversal%row_offset(i)
1430 c0 = route%traversal%col_offset(i)
1431 nr = route%row_size(route%traversal%row(i))
1432 nc = route%col_size(route%traversal%col(i))
1433 mirror_block = route%traversal%symmetric .AND. &
1434 route%traversal%row(i) /= route%traversal%col(i)
1435 cpassert(
PRESENT(value_start) .EQV.
PRESENT(value_index))
1437 IF (
SIZE(needed) == 1)
THEN
1439 value_count(0) = nr*nc
1446 direct = fms%context%blacs2mpi(fms%g2p_row(r0 + row), fms%g2p_col(c0 + col))
1448 IF (mirror_block)
THEN
1449 mirror = fms%context%blacs2mpi(fms%g2p_row(c0 + col), fms%g2p_col(r0 + row))
1451 value_count(direct) = value_count(direct) + 1
1452 IF (
PRESENT(value_index))
THEN
1453 value_index(value_start(direct) + value_count(direct)) = row + col*nr + 1
1455 IF (mirror == direct) cycle
1456 value_count(mirror) = value_count(mirror) + 1
1457 IF (
PRESENT(value_index))
THEN
1458 value_index(value_start(mirror) + value_count(mirror)) = row + col*nr + 1
1462 needed(:) = value_count > 0
1463 END SUBROUTINE rskp_owners
1476 TYPE(dbcsr_p_type),
POINTER :: source(:, :)
1477 INTEGER,
INTENT(IN) :: matrix_row
1478 TYPE(neighbor_list_set_p_type),
DIMENSION(:), &
1479 POINTER :: source_sab
1480 LOGICAL :: compatible
1483 INTEGER,
POINTER :: col_size(:), row_size(:)
1485 compatible = .false.
1486 IF (.NOT.
ASSOCIATED(route%source_sab, source_sab))
RETURN
1487 IF (
SIZE(source, 2) /= route%nimage)
RETURN
1488 IF (matrix_row < 1 .OR. matrix_row >
SIZE(source, 1))
RETURN
1489 DO image = 1,
SIZE(source, 2)
1490 CALL dbcsr_get_info(source(matrix_row, image)%matrix, row_blk_size=row_size, col_blk_size=col_size)
1491 IF (
SIZE(row_size) /=
SIZE(route%row_size) .OR.
SIZE(col_size) /=
SIZE(route%col_size))
RETURN
1492 IF (any(row_size /= route%row_size) .OR. any(col_size /= route%col_size))
RETURN
1509 SUBROUTINE rskp_capture(source, matrix_row, route, snapshot, mirror_sign, scale)
1511 TYPE(dbcsr_p_type),
POINTER :: source(:, :)
1512 INTEGER,
INTENT(IN) :: matrix_row
1515 INTEGER,
INTENT(IN),
OPTIONAL :: mirror_sign
1516 COMPLEX(KIND=dp),
INTENT(IN),
OPTIONAL :: scale
1518 CHARACTER(LEN=*),
PARAMETER :: routinen =
'rskp_capture'
1520 INTEGER :: col, element, g, handle, i, iblock, m, &
1521 ncol, nrow, odd, panels(4), pos, row
1523 REAL(kind=dp) :: sense
1524 REAL(kind=dp),
CONTIGUOUS,
POINTER :: received(:)
1525 REAL(kind=dp),
POINTER :: block(:, :)
1526 TYPE(mp_para_env_type),
POINTER :: inter, world
1528 CALL timeset(routinen, handle)
1529 world => route%world
1530 inter => route%inter
1531 m = route%transform%packed_first(
SIZE(route%transform%packed_first)) - 1
1532 cpassert(m <= route%nvalue)
1533 ALLOCATE (snapshot%packed(m), source=0.0_dp)
1534 snapshot%source => source
1535 snapshot%matrix_row = matrix_row
1536 IF (
PRESENT(mirror_sign)) snapshot%mirror_sign = mirror_sign
1537 cpassert(abs(snapshot%mirror_sign) == 1)
1538 IF (
PRESENT(scale)) snapshot%scale = scale
1539 IF (world%num_pe == 1)
THEN
1541 DO iblock = 1,
SIZE(route%entry, 2)
1542 i = route%send_entry(iblock)
1543 CALL dbcsr_get_readonly_block_p(source(matrix_row, route%traversal%image(i))%matrix, &
1544 route%traversal%row(i), route%traversal%col(i), block, found)
1545 IF (.NOT. found) cycle
1546 pos = route%transform%fold_map(1, iblock)
1547 odd = abs(route%transform%fold_map(2, iblock))
1548 sense = real(sign(1, route%transform%fold_map(2, iblock)), dp)
1549 DO col = 1,
SIZE(block, 2)
1550 DO row = 1,
SIZE(block, 1)
1551 snapshot%packed(pos) = snapshot%packed(pos) + block(row, col)
1553 snapshot%packed(odd) = snapshot%packed(odd) + sense*block(row, col)
1561 received => route%recv_values
1562 route%send_values(:) = 0.0_dp
1565 DO iblock = 1,
SIZE(route%send_entry)
1566 i = route%send_entry(iblock)
1567 CALL dbcsr_get_readonly_block_p(source(matrix_row, route%traversal%image(i))%matrix, &
1568 route%traversal%row(i), route%traversal%col(i), block, found)
1569 IF (.NOT. found) cycle
1570 nrow = route%row_size(route%traversal%row(i))
1571 ncol = route%col_size(route%traversal%col(i))
1572 cpassert(
SIZE(block, 1) == nrow .AND.
SIZE(block, 2) == ncol)
1573 pos = route%send_first(iblock)
1574 IF (
ALLOCATED(route%send_element))
THEN
1575 DO element = pos, route%send_first(iblock + 1) - 1
1576 row =
modulo(route%send_element(element) - 1, nrow) + 1
1577 col = (route%send_element(element) - 1)/nrow + 1
1578 route%send_values(element) = block(row, col)
1584 route%send_values(pos) = block(row, col)
1590 IF (world%num_pe == 1 .OR. &
1591 (.NOT. route%world_target .AND. route%group%num_pe == 1))
THEN
1592 CALL world%allgatherv(route%send_values, received, route%rcount(2, :), route%rdisp(2, :))
1594 CALL world%alltoall(route%send_values, route%scount(2, :), route%sdisp(2, :), &
1595 received, route%rcount(2, :), route%rdisp(2, :))
1596 IF (.NOT. route%world_target)
CALL inter%bcast(received, 0)
1599 DO iblock = 1,
SIZE(route%entry, 2)
1600 pos = route%transform%fold_map(1, iblock)
1601 odd = abs(route%transform%fold_map(2, iblock))
1602 m = route%first(iblock + 1) - route%first(iblock)
1603 snapshot%packed(pos:pos + m - 1) = snapshot%packed(pos:pos + m - 1) + &
1604 received(route%first(iblock):route%first(iblock + 1) - 1)
1606 sense = real(sign(1, route%transform%fold_map(2, iblock)), dp)
1607 snapshot%packed(odd:odd + m - 1) = snapshot%packed(odd:odd + m - 1) + &
1608 sense*received(route%first(iblock):route%first(iblock + 1) - 1)
1612 IF (route%transform%backend == rskp_backend_gemm)
THEN
1613 ALLOCATE (snapshot%active(4,
SIZE(route%transform%phase_count, 2)))
1614 DO g = 1,
SIZE(snapshot%active, 2)
1615 m = route%element_first(g + 1) - route%element_first(g)
1616 panels = route%transform%phase_count([1, 1, 2, 3], g)
1617 pos = route%transform%packed_first(g)
1619 snapshot%active(i, g) = any(snapshot%packed(pos:pos + m*panels(i) - 1) /= 0.0_dp)
1620 pos = pos + m*panels(i)
1624 CALL timestop(handle)
1636 IF (
ALLOCATED(snapshot%packed))
DEALLOCATE (snapshot%packed)
1637 IF (
ALLOCATED(snapshot%active))
DEALLOCATE (snapshot%active)
1638 IF (
ALLOCATED(snapshot%fft_values))
DEALLOCATE (snapshot%fft_values)
1639 NULLIFY (snapshot%source)
1640 snapshot%matrix_row = 0
1641 snapshot%mirror_sign = 1
1642 snapshot%scale = z_one
1653 CALL cell_to_k_grid_release(route%transform%fft_work)
1654 IF (
ASSOCIATED(route%matrix_struct))
CALL cp_fm_struct_release(route%matrix_struct)
1674 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
1675 INTEGER,
INTENT(IN) :: nkp_grid(3), kp_dist(:, :)
1676 TYPE(mp_para_env_type),
INTENT(IN) :: inter
1677 INTEGER(KIND=int_8),
INTENT(IN) :: max_bytes
1678 COMPLEX(KIND=dp),
CONTIGUOUS,
INTENT(OUT), &
1679 OPTIONAL :: cache(:, :)
1680 TYPE(cp_fm_p_type),
INTENT(IN),
OPTIONAL :: output(:, :)
1682 COMPLEX(KIND=dp),
ALLOCATABLE,
TARGET :: received(:), transformed(:, :, :)
1683 COMPLEX(KIND=dp),
CONTIGUOUS,
POINTER :: canonical(:, :), send(:)
1684 INTEGER :: element, first, group, i, igroup, ik, &
1685 last, my_peer, nelem, nimages, nk, &
1686 nlocal, npeers, ntile, nwave, offset, &
1688 INTEGER(KIND=int_8) :: available, fixed_bytes, ngrid, &
1689 per_element, workspace_bytes
1690 INTEGER,
ALLOCATABLE :: ranges(:, :), rcount(:), rdisp(:), &
1692 REAL(kind=dp),
CONTIGUOUS,
POINTER :: values(:, :, :)
1694 cpassert(route%transform%backend == rskp_backend_fft)
1695 cpassert(
PRESENT(cache) .NEQV.
PRESENT(output))
1696 cpassert(route%transform%fft_work%regular)
1697 ngrid = product(int(route%transform%fft_work%nfft, int_8))
1699 IF (
PRESENT(cache))
THEN
1700 cpassert(
SIZE(cache, 1) == route%element_first(
SIZE(route%element_first)) - 1)
1701 nlocal =
SIZE(cache, 2)
1704 nlocal =
SIZE(output, 1)
1706 DO i = 1,
SIZE(output, 2)
1707 output(ik, i)%matrix%local_data(:, :) = 0.0_dp
1713 IF (.NOT. route%world_target)
THEN
1714 npeers = inter%num_pe
1715 my_peer = inter%mepos
1717 ALLOCATE (ranges(2, npeers))
1718 ranges(:, 1) = [1, nk]
1719 IF (.NOT. route%world_target) ranges(:, :) = kp_dist
1720 cpassert(nlocal == ranges(2, my_peer + 1) - ranges(1, my_peer + 1) + 1)
1721 available = max_bytes
1722 IF (.NOT. route%world_target)
CALL inter%min(available)
1723 ALLOCATE (scount(npeers), sdisp(npeers), rcount(npeers), rdisp(npeers))
1724 DO igroup = 1,
SIZE(route%block_first) - 1
1725 first = route%block_first(igroup)
1726 last = route%block_first(igroup + 1) - 1
1727 nelem = route%element_first(igroup + 1) - route%element_first(igroup)
1728 nimages = last - first + 1
1729 values(1:nelem, 1:1, 1:nimages) => &
1730 snapshot%packed(route%transform%packed_first(igroup):route%transform%packed_first(igroup + 1) - 1)
1731 IF (.NOT. any(values /= 0.0_dp)) cycle
1733 fixed_bytes = 16_int_8*nk + 28_int_8*
SIZE(route%entry, 2) + 16_int_8*npeers
1734 per_element = 16_int_8*nk + 48_int_8*ngrid + &
1735 16_int_8*npeers*maxval(ranges(2, :) - ranges(1, :) + 1)
1736 width = int(max(0_int_8, min((int(nelem, int_8) + int(npeers, int_8) - 1_int_8)/int(npeers, int_8), &
1737 max(1_int_8, fft_batch_bytes/(48_int_8*ngrid)), &
1738 max(1_int_8, (available - fixed_bytes)/per_element), int(huge(0), int_8)/nk, &
1739 int(huge(0), int_8)/(int(npeers, int_8)* &
1740 maxval(ranges(2, :) - ranges(1, :) + 1)))))
1741 IF (width == 0)
THEN
1742 cpabort(
"Lattice FFT tile exceeds the integer MPI count range")
1744 DO element = 1, nelem, width*npeers
1745 start = element + my_peer*width
1746 ntile = max(0, min(width, nelem - start + 1))
1747 nwave = min(width*npeers, nelem - element + 1)
1748 ALLOCATE (transformed(ntile, 1, nk), received(nwave*nlocal))
1750 workspace_bytes = available - fixed_bytes - 16_int_8*(
SIZE(transformed, kind=int_8) + &
1751 SIZE(received, kind=int_8))
1752 CALL cell_to_k_grid_fft(values(start:start + ntile - 1, :, :), &
1753 route%entry(5:7, route%block_order(first:last)), &
1754 xkp, nkp_grid, transformed, &
1755 require_fft=.true., work=route%transform%fft_work, cell_first=first, &
1756 max_workspace_bytes=workspace_bytes)
1758 send(1:
SIZE(transformed)) => transformed
1759 scount(:) = ntile*(ranges(2, :) - ranges(1, :) + 1)
1760 sdisp(:) = ntile*(ranges(1, :) - 1)
1762 DO group = 1, npeers
1763 rcount(group) = max(0, min(width, nelem - element - (group - 1)*width + 1))*nlocal
1764 rdisp(group) = offset
1765 offset = offset + rcount(group)
1767 IF (npeers > 1)
THEN
1768 CALL inter%alltoall(send, scount, sdisp, received, rcount, rdisp)
1772 DO group = 1, npeers
1773 ntile = rcount(group)/max(1, nlocal)
1774 IF (ntile == 0) cycle
1775 start = route%element_first(igroup) + element + (group - 1)*width - 1
1776 canonical(1:ntile, 1:nlocal) => received(rdisp(group) + 1:rdisp(group) + rcount(group))
1777 IF (
PRESENT(cache))
THEN
1778 cache(start:start + ntile - 1, :) = canonical
1780 CALL cell_to_k_scatter(canonical, start, route, snapshot%mirror_sign, snapshot%scale, output=output)
1783 DEALLOCATE (transformed, received)
1799 SUBROUTINE cell_to_k_scatter(values, first_element, route, mirror_sign, scale, batch, output)
1800 COMPLEX(KIND=dp),
INTENT(IN) :: values(:, :)
1801 INTEGER,
INTENT(IN) :: first_element
1803 INTEGER,
INTENT(IN) :: mirror_sign
1804 COMPLEX(KIND=dp),
INTENT(IN) :: scale
1805 COMPLEX(KIND=dp),
CONTIGUOUS,
INTENT(INOUT), &
1806 OPTIONAL,
TARGET :: batch(:, :, :)
1807 TYPE(cp_fm_p_type),
INTENT(IN),
OPTIONAL :: output(:, :)
1809 COMPLEX(KIND=dp) :: value
1810 COMPLEX(KIND=dp),
CONTIGUOUS,
POINTER :: dense(:, :)
1811 INTEGER :: element, i, ic, k
1812 INTEGER(KIND=int_8) :: dest
1813 REAL(kind=dp),
CONTIGUOUS,
POINTER :: real_values(:)
1815 cpassert(
PRESENT(batch) .NEQV.
PRESENT(output))
1816 IF (
PRESENT(batch))
THEN
1817 cpassert(
SIZE(batch, 3) ==
SIZE(values, 2))
1818 dense(1:
SIZE(batch, 1, kind=int_8)*
SIZE(batch, 2, kind=int_8), 1:
SIZE(batch, 3)) => batch
1819 DO k = 1,
SIZE(values, 2)
1821 DO i = 1,
SIZE(values, 1)
1822 element = first_element + i - 1
1823 dest = route%element_offset(1, element)
1824 IF (dest > 0) dense(dest, k) = scale*values(i, k)
1825 dest = route%element_offset(2, element)
1826 value = real(mirror_sign, dp)*conjg(values(i, k))
1827 IF (dest > 0) dense(dest, k) = scale*
value
1831 cpassert(
SIZE(output, 1) ==
SIZE(values, 2))
1832 cpassert(
SIZE(output, 2) >= 1 .AND.
SIZE(output, 2) <= 2)
1833 DO k = 1,
SIZE(values, 2)
1834 DO ic = 1,
SIZE(output, 2)
1835 real_values(1:
SIZE(output(k, ic)%matrix%local_data, kind=int_8)) => output(k, ic)%matrix%local_data
1837 DO i = 1,
SIZE(values, 1)
1838 element = first_element + i - 1
1839 dest = route%element_offset(1, element)
1840 value = scale*values(i, k)
1841 IF (dest > 0) real_values(dest) = merge(real(
value, dp), aimag(
value), ic == 1)
1842 dest = route%element_offset(2, element)
1843 value = scale*(real(mirror_sign, dp)*conjg(values(i, k)))
1844 IF (dest > 0) real_values(dest) = merge(real(
value, dp), aimag(
value), ic == 1)
1849 END SUBROUTINE cell_to_k_scatter
1864 SUBROUTINE cell_to_k_phase_prepare(plan, translations, group_first, order, element_first, xkp, max_bytes, output_bytes)
1865 TYPE(rskp_backend_plan_type),
INTENT(INOUT) :: plan
1866 INTEGER,
INTENT(IN) :: translations(:, :), group_first(:), &
1867 order(:), element_first(:)
1868 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
1869 INTEGER(KIND=int_8),
INTENT(IN) :: max_bytes, output_bytes
1871 COMPLEX(KIND=dp) :: phase
1872 INTEGER :: c, first, g, i, iblock, ik, j, m, n, &
1873 nfold, nk, npair, nr, nsingle, pos
1874 INTEGER(KIND=int_8) :: cache_bytes, minimum_bytes
1875 INTEGER,
ALLOCATABLE :: cells(:, :), classes(:), &
1876 global_class(:), local_class(:), &
1878 LOGICAL,
ALLOCATABLE :: members(:, :)
1879 REAL(kind=dp) :: arg
1881 CALL rskp_image_classes(translations, classes, plan%phase_cells)
1882 n =
SIZE(group_first) - 1
1883 ALLOCATE (plan%phase_count(3, n), plan%phase_first(n + 1), plan%packed_first(n + 1), &
1884 plan%phase_index(
SIZE(classes)), plan%fold_map(2,
SIZE(classes)))
1886 plan%phase_first(1) = 1
1887 plan%packed_first(1) = 1
1888 plan%max_per_k = 0_int_8
1890 first = group_first(g)
1891 CALL rskp_image_classes(translations(:, order(first:group_first(g + 1) - 1)), &
1893 ALLOCATE (members(2,
SIZE(cells, 2)), slots(
SIZE(cells, 2)), global_class(
SIZE(cells, 2)))
1895 DO i = 1,
SIZE(local_class)
1898 members(merge(1, 2, c > 0), abs(c)) = .true.
1899 iblock = order(first + i - 1)
1900 global_class(abs(c)) = abs(classes(iblock))
1902 npair = count(all(members, dim=1))
1903 nsingle =
SIZE(cells, 2) - npair
1904 plan%phase_count(:, g) = [npair, nsingle, merge(1, 0, any(local_class == 0))]
1907 DO c = 1,
SIZE(cells, 2)
1908 IF (all(members(:, c)))
THEN
1911 plan%phase_index(plan%phase_first(g) + i - 1) = global_class(c)
1914 slots(c) = 2*npair + j
1915 plan%phase_index(plan%phase_first(g) + npair + j - 1) = merge(1, -1, members(1, c))*global_class(c)
1918 m = element_first(g + 1) - element_first(g)
1919 nfold = 2*npair + nsingle + plan%phase_count(3, g)
1920 plan%phase_first(g + 1) = plan%phase_first(g) + npair + nsingle
1921 IF (int(plan%packed_first(g), int_8) + int(m, int_8)*nfold > huge(m))
THEN
1922 cpabort(
"R-to-K folded panel offsets exceed the integer range")
1924 plan%packed_first(g + 1) = plan%packed_first(g) + m*nfold
1925 plan%max_per_k = max(plan%max_per_k, int(m, int_8) + max(npair, nsingle))
1926 DO i = 1,
SIZE(local_class)
1927 iblock = order(first + i - 1)
1930 IF (c /= 0) pos = slots(abs(c))
1931 plan%fold_map(1, iblock) = plan%packed_first(g) + (pos - 1)*m
1933 IF (all(members(:, abs(c))))
THEN
1934 plan%fold_map(2, iblock) = sign(plan%packed_first(g) + (npair + pos - 1)*m, c)
1937 DEALLOCATE (members, slots, global_class)
1940 nr =
SIZE(plan%phase_cells, 2)
1942 cache_bytes = 16_int_8*nr*nk
1943 minimum_bytes = 16_int_8*plan%max_per_k + output_bytes
1944 IF (cache_bytes + minimum_bytes > max_bytes)
RETURN
1945 ALLOCATE (plan%phases(nr, nk, 2))
1949 arg = twopi*dot_product(real(plan%phase_cells(:, i), dp), xkp(:, ik))
1950 phase = exp(cmplx(0.0_dp, arg, kind=dp))
1951 plan%phases(i, ik, 1) = real(phase, dp)
1952 plan%phases(i, ik, 2) = aimag(phase)
1955 END SUBROUTINE cell_to_k_phase_prepare
1968 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
1969 INTEGER,
INTENT(IN) :: nkp_grid(3)
1970 INTEGER(KIND=int_8),
INTENT(IN) :: max_bytes
1972 SELECT CASE (route%transform%backend)
1973 CASE (rskp_backend_gemm)
1974 CALL cell_to_k_phase_prepare(route%transform, route%entry(5:7, :), route%block_first, route%block_order, &
1975 route%element_first, xkp, max_bytes, &
1976 32_int_8*route%matrix_struct%local_leading_dimension*max(1, route%ncol))
1977 CASE (rskp_backend_fft)
1978 CALL cell_to_k_grid_prepare(route%transform%fft_work, route%entry(5:7, route%block_order), xkp, nkp_grid)
1979 IF (.NOT. route%transform%fft_work%regular)
THEN
1980 cpabort(
"Forced lattice FFT needs a supported regular grid or subset")
1983 cpabort(
"Unknown R-to-K backend")
1998 SUBROUTINE cell_to_k_block_gemm(packed, plan, igroup, xkp, first_k, factors, RESULT, active)
1999 REAL(kind=dp),
CONTIGUOUS,
INTENT(IN),
TARGET :: packed(:)
2000 TYPE(rskp_backend_plan_type),
INTENT(IN) :: plan
2001 INTEGER,
INTENT(IN) :: igroup
2002 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
2003 INTEGER,
INTENT(IN) :: first_k
2004 REAL(kind=dp),
CONTIGUOUS,
INTENT(OUT) :: factors(:, :), result(:, :)
2005 LOGICAL,
INTENT(IN) :: active(4)
2007 INTEGER :: first, i, ik, nactive, nelem, nk, npair, &
2008 nsingle, pass, phase, start
2009 LOGICAL :: use_cosine, use_sine
2010 REAL(kind=dp) :: arg, beta(2)
2011 REAL(kind=dp),
CONTIGUOUS,
POINTER :: values(:, :), values_im(:, :)
2013 nelem =
SIZE(result, 1)
2015 cpassert(
SIZE(result, 2) == nk .OR.
SIZE(result, 2) == 2*nk)
2016 npair = plan%phase_count(1, igroup)
2017 nsingle = plan%phase_count(2, igroup)
2018 first = plan%phase_first(igroup)
2020 IF (
ALLOCATED(plan%phases))
THEN
2021 cpassert(first_k >= 1 .AND. first_k + nk - 1 <=
SIZE(plan%phases, 2))
2025 nactive = merge(npair, nsingle, pass == 1)
2026 IF (nactive == 0) cycle
2027 use_cosine = merge(active(1), active(3), pass == 1)
2028 use_sine =
SIZE(result, 2) == 2*nk .AND. merge(active(2), active(3), pass == 1)
2029 IF (.NOT. (use_cosine .OR. use_sine))
THEN
2030 start = start + nelem*nactive*merge(2, 1, pass == 1)
2031 first = first + nactive
2036 phase = plan%phase_index(first + i - 1)
2037 IF (
ALLOCATED(plan%phases))
THEN
2038 IF (use_cosine) factors(i, ik) = plan%phases(abs(phase), first_k + ik - 1, 1)
2040 factors(i, nk + ik) = real(sign(1, phase), dp)*plan%phases(abs(phase), first_k + ik - 1, 2)
2043 arg = twopi*dot_product(real(plan%phase_cells(:, abs(phase)), dp), xkp(:, ik))
2044 IF (use_cosine) factors(i, ik) = cos(arg)
2045 IF (use_sine) factors(i, nk + ik) = real(sign(1, phase), dp)*sin(arg)
2049 values(1:nelem, 1:nactive) => packed(start:start + nelem*nactive - 1)
2051 start = start + nelem*npair
2053 CALL rskp_gemm_contract(nelem, npair, nk, values, factors(:, 1:nk), result(:, 1:nk), 0.0_dp)
2056 IF (
SIZE(result, 2) == 2*nk .AND. active(2))
THEN
2057 values_im(1:nelem, 1:npair) => packed(start:start + nelem*npair - 1)
2058 CALL rskp_gemm_contract(nelem, npair, nk, values_im, factors(:, nk + 1:2*nk), &
2059 result(:, nk + 1:2*nk), 0.0_dp)
2062 start = start + nelem*npair
2064 IF (
SIZE(result, 2) == 2*nk .AND. beta(1) == beta(2))
THEN
2065 CALL rskp_gemm_contract(nelem, nsingle, 2*nk, values, factors, result, beta(1))
2067 CALL rskp_gemm_contract(nelem, nsingle, nk, values, factors(:, 1:nk), result(:, 1:nk), beta(1))
2068 IF (
SIZE(result, 2) == 2*nk)
THEN
2069 CALL rskp_gemm_contract(nelem, nsingle, nk, values, factors(:, nk + 1:2*nk), &
2070 result(:, nk + 1:2*nk), beta(2))
2074 start = start + nelem*nsingle
2076 first = first + nactive
2078 IF (beta(1) == 0.0_dp) result(:, 1:nk) = 0.0_dp
2079 IF (
SIZE(result, 2) == 2*nk .AND. beta(2) == 0.0_dp) result(:, nk + 1:2*nk) = 0.0_dp
2082 result(:, ik) = result(:, ik) + packed(start:start + nelem - 1)
2085 END SUBROUTINE cell_to_k_block_gemm
2099 SUBROUTINE rskp_batch_size(route, nk, ncomp, nworkers, nmatrices, max_bytes, width, work_size)
2101 INTEGER,
INTENT(IN) :: nk, ncomp
2102 INTEGER,
INTENT(INOUT) :: nworkers
2103 INTEGER,
INTENT(IN) :: nmatrices
2104 INTEGER(KIND=int_8),
INTENT(IN) :: max_bytes
2105 INTEGER,
INTENT(OUT) :: width
2106 INTEGER(KIND=int_8),
INTENT(OUT) :: work_size
2109 INTEGER(KIND=int_8) :: available_bytes, output_bytes, &
2112 cpassert(nworkers > 0 .AND. nk > 0 .AND. nmatrices >= 0)
2113 available_bytes = max_bytes
2114 IF (
ALLOCATED(route%transform%phases))
THEN
2115 available_bytes = available_bytes - 8_int_8*
SIZE(route%transform%phases, kind=int_8)
2117 output_bytes = 16_int_8*nmatrices*route%matrix_struct%local_leading_dimension*max(1, route%ncol)
2119 IF (nmatrices == 0) nc = ncomp
2120 per_k_bytes = max(1_int_8, 8_int_8*nc*route%transform%max_per_k + output_bytes)
2121 nworkers = min(nworkers, nk)
2122 nworkers = min(nworkers, int(max(1_int_8, available_bytes/per_k_bytes)))
2123 width = int(min(int(nk, int_8), max(1_int_8, available_bytes/(nworkers*per_k_bytes))))
2124 width = min(width, (nk + nworkers - 1)/nworkers)
2125 work_size = int(nc, int_8)*width*route%transform%max_per_k
2138 SUBROUTINE cell_to_k_gemm_batch(snapshot, route, xkp, first_k, work, batch, output)
2141 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
2142 INTEGER,
INTENT(IN) :: first_k
2143 REAL(kind=dp),
CONTIGUOUS,
INTENT(INOUT),
TARGET :: work(:)
2144 COMPLEX(KIND=dp),
CONTIGUOUS,
INTENT(OUT), &
2145 OPTIONAL,
TARGET :: batch(:, :, :)
2146 TYPE(cp_fm_p_type),
INTENT(IN),
OPTIONAL :: output(:, :)
2148 COMPLEX(KIND=dp) :: value
2149 COMPLEX(KIND=dp),
CONTIGUOUS,
POINTER :: complex_values(:, :)
2150 INTEGER :: element, first, ic, igroup, j, ld, nc, &
2151 nelem, nphase, width
2152 INTEGER(KIND=int_8) :: dest, nfactor
2153 REAL(kind=dp),
CONTIGUOUS,
POINTER :: factors(:, :), real_values(:), &
2156 cpassert(route%transform%backend == rskp_backend_gemm)
2157 cpassert(
PRESENT(batch) .NEQV.
PRESENT(output))
2158 width =
SIZE(xkp, 2)
2159 ld = route%matrix_struct%local_leading_dimension
2161 IF (
PRESENT(batch))
THEN
2162 cpassert(
SIZE(batch, 1) == route%matrix_struct%local_leading_dimension)
2163 cpassert(
SIZE(batch, 2) >= route%ncol .AND.
SIZE(batch, 3) == width)
2165 complex_values(1:
SIZE(batch, 1, kind=int_8)*
SIZE(batch, 2), 1:width) => batch
2167 nc =
SIZE(output, 2)
2168 cpassert(
SIZE(output, 1) == width .AND. (nc == 1 .OR. nc == 2))
2171 cpassert(
SIZE(output(j, ic)%matrix%local_data, 1) == ld)
2172 output(j, ic)%matrix%local_data = 0.0_dp
2175 IF (aimag(snapshot%scale) /= 0.0_dp) nc = 2
2177 DO igroup = 1,
SIZE(route%transform%phase_count, 2)
2178 first = route%element_first(igroup)
2179 nelem = route%element_first(igroup + 1) - first
2180 IF (nelem == 0) cycle
2181 IF (.NOT. any(snapshot%active(:, igroup))) cycle
2182 nphase = maxval(route%transform%phase_count(1:2, igroup))
2183 nfactor = int(nc, int_8)*nphase*width
2184 cpassert(
SIZE(work, kind=int_8) >= nfactor + int(nc, int_8)*nelem*width)
2185 factors(1:nphase, 1:nc*width) => work(1:nfactor)
2186 result(1:nelem, 1:nc*width) => work(nfactor + 1:nfactor + int(nc, int_8)*nelem*width)
2187 CALL cell_to_k_block_gemm(snapshot%packed(route%transform%packed_first(igroup): &
2188 route%transform%packed_first(igroup + 1) - 1), &
2189 route%transform, igroup, xkp, first_k, factors, result, snapshot%active(:, igroup))
2191 IF (
PRESENT(batch))
THEN
2193 DO element = first, route%element_first(igroup + 1) - 1
2194 value = cmplx(result(element - first + 1, j), result(element - first + 1, width + j), kind=dp)
2195 dest = route%element_offset(1, element)
2196 IF (dest > 0) complex_values(dest, j) = snapshot%scale*
value
2197 dest = route%element_offset(2, element)
2198 IF (dest > 0) complex_values(dest, j) = snapshot%scale*real(snapshot%mirror_sign, dp)*conjg(
value)
2201 DO ic = 1,
SIZE(output, 2)
2202 real_values(1:
SIZE(output(j, ic)%matrix%local_data, kind=int_8)) => output(j, ic)%matrix%local_data
2204 DO element = first, route%element_first(igroup + 1) - 1
2205 dest = route%element_offset(1, element)
2206 IF (aimag(snapshot%scale) == 0.0_dp)
THEN
2207 IF (dest > 0) real_values(dest) = real(snapshot%scale, dp)* &
2208 result(element - first + 1, j + (ic - 1)*width)
2210 value = snapshot%scale*cmplx(result(element - first + 1, j), &
2211 result(element - first + 1, width + j), dp)
2212 IF (dest > 0) real_values(dest) = merge(real(
value, dp), aimag(
value), ic == 1)
2214 dest = route%element_offset(2, element)
2215 IF (aimag(snapshot%scale) == 0.0_dp)
THEN
2216 IF (dest > 0) real_values(dest) = real(snapshot%scale, dp)*real(snapshot%mirror_sign, dp)* &
2217 merge(1.0_dp, -1.0_dp, ic == 1)* &
2218 result(element - first + 1, j + (ic - 1)*width)
2220 value = snapshot%scale*real(snapshot%mirror_sign, dp)* &
2221 cmplx(result(element - first + 1, j), -result(element - first + 1, width + j), dp)
2222 IF (dest > 0) real_values(dest) = merge(real(
value, dp), aimag(
value), ic == 1)
2229 END SUBROUTINE cell_to_k_gemm_batch
2246 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
2247 INTEGER,
INTENT(IN) :: first_local
2248 REAL(kind=dp),
CONTIGUOUS,
INTENT(INOUT),
TARGET :: work(:)
2249 COMPLEX(KIND=dp),
CONTIGUOUS,
INTENT(OUT), &
2250 OPTIONAL :: batch(:, :, :)
2251 TYPE(cp_fm_p_type),
INTENT(IN),
OPTIONAL :: output(:, :)
2253 INTEGER :: ic, j, width
2254 LOGICAL :: compatible
2256 cpassert(
PRESENT(batch) .NEQV.
PRESENT(output))
2257 width =
SIZE(xkp, 2)
2258 IF (
PRESENT(output))
THEN
2259 cpassert(
SIZE(output, 1) == width)
2260 cpassert(
SIZE(output, 2) >= 1 .AND.
SIZE(output, 2) <= 2)
2262 DO ic = 1,
SIZE(output, 2)
2263 compatible = cp_fm_struct_equivalent(output(j, ic)%matrix%matrix_struct, route%matrix_struct)
2264 cpassert(compatible)
2268 SELECT CASE (route%transform%backend)
2269 CASE (rskp_backend_gemm)
2270 CALL cell_to_k_gemm_batch(snapshot, route, xkp, first_local, work, batch=batch, output=output)
2271 CASE (rskp_backend_fft)
2272 cpassert(
ALLOCATED(snapshot%fft_values))
2273 cpassert(first_local >= 1)
2274 cpassert(first_local + width - 1 <=
SIZE(snapshot%fft_values, 2))
2275 IF (
PRESENT(batch))
THEN
2276 cpassert(
SIZE(batch, 1) == route%matrix_struct%local_leading_dimension)
2277 cpassert(
SIZE(batch, 2) >= route%ncol)
2278 cpassert(
SIZE(batch, 3) == width)
2282 DO ic = 1,
SIZE(output, 2)
2283 output(j, ic)%matrix%local_data(:, :) = 0.0_dp
2287 CALL cell_to_k_scatter(snapshot%fft_values(:, first_local:first_local + width - 1), 1, route, &
2288 snapshot%mirror_sign, snapshot%scale, batch=batch, output=output)
2290 cpabort(
"Unknown R-to-K backend")
2305 INTEGER,
INTENT(IN) :: cells(:, :)
2306 REAL(kind=dp),
INTENT(IN) :: xkp(:, :)
2307 INTEGER,
INTENT(IN) :: nworkers
2309 COMPLEX(KIND=dp) :: phase
2310 INTEGER :: axis, i, k, n, nk, pos
2311 INTEGER,
ALLOCATABLE :: coordinates(:, :), count(:), &
2312 direction(:), order(:), values(:)
2313 REAL(kind=dp) :: arg
2315 cpassert(
SIZE(cells, 1) == 3 .AND.
SIZE(xkp, 1) == 3)
2316 cpassert(nworkers > 0)
2317 n =
SIZE(cells, 2) + 1
2319 ALLOCATE (coordinates(3, n), count(3), values(n), order(n))
2322 values(2:n) = abs(cells(axis, :))
2323 CALL sort(values, n, order)
2327 IF (values(i) == values(i - 1)) cycle
2329 count(axis) = count(axis) + 1
2330 coordinates(axis, count(axis)) = values(i)
2332 table%first(axis + 1) = table%first(axis) + count(axis)
2335 ALLOCATE (table%coordinate(n), direction(n), table%cosine(nk, n), table%sine(nk, n))
2337 pos = table%first(axis)
2338 table%coordinate(pos:pos + count(axis) - 1) = coordinates(axis, 1:count(axis))
2339 direction(pos:pos + count(axis) - 1) = axis
2346 IF (table%coordinate(i) == 0)
THEN
2347 table%cosine(k, i) = 1.0_dp
2348 table%sine(k, i) = 0.0_dp
2350 arg = twopi*real(table%coordinate(i), dp)*xkp(direction(i), k)
2351 phase = exp(cmplx(0.0_dp, arg, kind=dp))
2352 table%cosine(k, i) = real(phase, dp)
2353 table%sine(k, i) = aimag(phase)
2382 block_shape, block_offset, shift, time_reversal, trans, &
2383 phase_table, sums, gemm_ctx)
2384 TYPE(cp_fm_p_type),
INTENT(IN) :: source(:, :, :)
2385 INTEGER,
INTENT(IN) :: tiles(:, :, :, :, :), ikp_index(:)
2386 REAL(kind=dp),
INTENT(IN) :: xkp(:, :), weight(:)
2387 INTEGER,
INTENT(IN) :: images(:, :), source_pair(2), &
2388 block_shape(2), block_offset(2), &
2390 LOGICAL,
INTENT(IN) :: time_reversal, trans
2392 REAL(kind=dp),
ALLOCATABLE,
INTENT(INOUT) :: sums(:, :, :)
2393 TYPE(local_gemm_ctxt_type),
INTENT(INOUT) :: gemm_ctx(:)
2395 INTEGER(KIND=int_8),
PARAMETER :: max_elements = 1048576_int_8
2397 INTEGER :: a, actual_workers, axis, b, batch, column, first, i, ic, ifirst, ii, ikp, ilast, &
2398 image, ispin, ix, iy, iz, j, jj, k, last, nc, nclass, nelement, nimage, nrow, nsine, &
2399 nspin, nworkers, pos, thread
2400 INTEGER,
ALLOCATABLE :: cells(:, :), image_class(:), &
2401 phase_index(:, :), sine_index(:)
2402 LOGICAL :: contributes, has_zero, identity_gauge, &
2403 real_only, zero_shift
2404 REAL(kind=dp) :: cosine_xy, sine, sine_xy, sx, sy, sz
2405 REAL(kind=dp),
ALLOCATABLE :: beta(:), gauge(:), gauge_factors(:, :, :), packed(:, :), &
2406 phases(:, :), reduced(:, :, :, :), zero_sum(:, :, :)
2407 TYPE(cp_fm_type),
POINTER :: matrix
2409 cpassert(
SIZE(gemm_ctx) > 0)
2410 nc =
SIZE(source, 2)
2411 nspin =
SIZE(source, 3)
2413 cpassert(nc == 1 .OR. nc == 2)
2414 cpassert(
SIZE(xkp, 2) ==
SIZE(ikp_index) .AND.
SIZE(weight) ==
SIZE(ikp_index))
2415 cpassert(
SIZE(phase_table%cosine, 1) ==
SIZE(ikp_index))
2418 nelement = block_shape(1)*block_shape(2)
2419 nimage =
SIZE(images, 2)
2420 IF (
ALLOCATED(sums))
THEN
2421 IF (any(shape(sums) /= [nelement, nimage, nspin]))
DEALLOCATE (sums)
2423 IF (.NOT.
ALLOCATED(sums))
ALLOCATE (sums(nelement, nimage, nspin))
2424 IF (nelement == 0 .OR. nimage == 0 .OR.
SIZE(ikp_index) == 0)
THEN
2428 CALL rskp_image_classes(images(1:3, :), image_class, cells)
2429 nclass =
SIZE(cells, 2)
2430 has_zero = any(image_class == 0)
2432 ALLOCATE (sine_index(nclass), source=0)
2433 IF (.NOT. real_only)
THEN
2434 DO image = 1, nimage
2435 i = abs(image_class(image))
2436 IF (i > 0 .AND. images(5, image) /= 0) sine_index(i) = 1
2441 IF (sine_index(i) == 0) cycle
2443 sine_index(i) = nsine
2445 ALLOCATE (phase_index(3, nclass))
2446 DO image = 1, nclass
2448 first = phase_table%first(axis)
2449 last = phase_table%first(axis + 1) - 1
2450 pos = locate(phase_table%coordinate(first:last), abs(cells(axis, image)))
2452 pos = pos + first - 1
2453 cpassert(phase_table%coordinate(pos) == abs(cells(axis, image)))
2454 phase_index(axis, image) = pos
2457 zero_shift = all(shift == 0)
2458 identity_gauge = zero_shift .AND. .NOT. time_reversal
2459 nworkers = min(
SIZE(gemm_ctx),
SIZE(ikp_index))
2462 batch = min(512, 1 + (
SIZE(ikp_index) - 1)/nworkers)
2463 batch = max(1, min(batch, int(max_elements/(int(nc, int_8)*int(nelement, int_8) + &
2464 int(nclass + nsine, int_8)))))
2465 nworkers = min(nworkers, 1 + (
SIZE(ikp_index) - 1)/batch)
2467 ALLOCATE (reduced(nelement, nclass + nsine, nspin, nworkers))
2468 IF (has_zero)
ALLOCATE (zero_sum(nelement, nspin, nworkers), source=0.0_dp)
2482 ALLOCATE (beta(nspin), source=0.0_dp)
2485 IF (.NOT. identity_gauge .AND. nc == 2)
ALLOCATE (gauge(nelement))
2486 IF (.NOT. identity_gauge)
ALLOCATE (gauge_factors(2, 2, batch))
2488 DO first = 1,
SIZE(ikp_index), batch
2489 last = min(first + batch - 1,
SIZE(ikp_index))
2490 DO image = 1, nclass
2491 ix = phase_index(1, image)
2492 iy = phase_index(2, image)
2493 iz = phase_index(3, image)
2494 sx = real(sign(1, cells(1, image)), dp)
2495 sy = real(sign(1, cells(2, image)), dp)
2496 sz = real(sign(1, cells(3, image)), dp)
2499 cosine_xy = phase_table%cosine(k, ix)*phase_table%cosine(k, iy) - &
2500 sx*sy*phase_table%sine(k, ix)*phase_table%sine(k, iy)
2501 sine_xy = sx*phase_table%sine(k, ix)*phase_table%cosine(k, iy) + &
2502 sy*phase_table%cosine(k, ix)*phase_table%sine(k, iy)
2503 phases(k - first + 1, image) = weight(k)*( &
2504 cosine_xy*phase_table%cosine(k, iz) - sine_xy*sz*phase_table%sine(k, iz))
2505 IF (sine_index(image) > 0)
THEN
2506 phases(k - first + 1, nclass + sine_index(image)) = weight(k)*( &
2507 sine_xy*phase_table%cosine(k, iz) + cosine_xy*sz*phase_table%sine(k, iz))
2511 IF (.NOT. identity_gauge)
THEN
2513 IF (zero_shift)
THEN
2514 gauge_factors(:, :, k - first + 1) = 0.0_dp
2515 gauge_factors(1, 1, k - first + 1) = 1.0_dp
2516 gauge_factors(2, 2, k - first + 1) = merge(-1.0_dp, 1.0_dp, time_reversal)
2519 gauge_factors(:, :, k - first + 1))
2525 contributes = .false.
2529 ifirst = tiles(1, a, ikp, ic, ispin)
2530 ilast = tiles(2, a, ikp, ic, ispin)
2531 IF (ifirst > ilast .OR. tiles(3, b, ikp, ic, ispin) > tiles(4, b, ikp, ic, ispin)) cycle
2532 contributes = .true.
2533 column = k - first + 1 + (ic - 1)*batch
2534 matrix => source(ikp, ic, ispin)%matrix
2535 nrow = ilast - ifirst + 1
2536 DO j = tiles(3, b, ikp, ic, ispin), tiles(4, b, ikp, ic, ispin)
2537 jj = matrix%matrix_struct%col_indices(j) - block_offset(2)
2538 ii = matrix%matrix_struct%row_indices(ifirst) - block_offset(1) + 1 + jj*block_shape(1)
2539 IF (matrix%matrix_struct%row_indices(ilast) - &
2540 matrix%matrix_struct%row_indices(ifirst) == nrow - 1)
THEN
2541 packed(ii:ii + nrow - 1, column) = matrix%local_data(ifirst:ilast, j)
2543 DO i = ifirst, ilast
2544 ii = matrix%matrix_struct%row_indices(i) - block_offset(1) + 1 + jj*block_shape(1)
2545 packed(ii, column) = matrix%local_data(i, j)
2550 column = k - first + 1
2551 IF (.NOT. identity_gauge)
THEN
2553 packed(:, column) = gauge_factors(1, 1, column)*packed(:, column)
2555 gauge(:) = gauge_factors(1, 1, column)*packed(:, column) + &
2556 gauge_factors(1, 2, column)*packed(:, batch + column)
2557 packed(:, batch + column) = gauge_factors(2, 1, column)*packed(:, column) + &
2558 gauge_factors(2, 2, column)*packed(:, batch + column)
2559 packed(:, column) = gauge
2562 IF (has_zero) zero_sum(:, ispin, thread) = zero_sum(:, ispin, thread) + weight(k)*packed(:, column)
2564 IF (.NOT. contributes .OR. nclass == 0) cycle
2565 CALL gemm_ctx(thread)%gemm(
'N',
'N', nelement, nclass, last - first + 1, 1.0_dp, &
2566 packed, nelement, phases, batch, beta(ispin), reduced(:, :, ispin, thread), nelement)
2568 CALL gemm_ctx(thread)%gemm(
'N',
'N', nelement, nsine, last - first + 1, 1.0_dp, &
2569 packed(:, batch + 1:), nelement, phases(:, nclass + 1:), batch, &
2570 beta(ispin), reduced(:, nclass + 1:, ispin, thread), nelement)
2572 beta(ispin) = 1.0_dp
2578 IF (beta(ispin) == 0.0_dp) reduced(:, :, ispin, thread) = 0.0_dp
2580 DEALLOCATE (beta, packed, phases)
2581 IF (
ALLOCATED(gauge))
DEALLOCATE (gauge)
2582 IF (
ALLOCATED(gauge_factors))
DEALLOCATE (gauge_factors)
2585 DO thread = 2, actual_workers
2586 reduced(:, :, :, 1) = reduced(:, :, :, 1) + reduced(:, :, :, thread)
2587 IF (has_zero) zero_sum(:, :, 1) = zero_sum(:, :, 1) + zero_sum(:, :, thread)
2590 DO image = 1, nimage
2591 i = abs(image_class(image))
2593 sums(:, image, ispin) = real(images(4, image), dp)*zero_sum(:, ispin, 1)
2595 sums(:, image, ispin) = real(images(4, image), dp)*reduced(:, i, ispin, 1)
2596 IF (sine_index(i) == 0) cycle
2597 sine = real(images(5, image)*sign(1, image_class(image)), dp)*merge(-1.0_dp, 1.0_dp, trans)
2598 sums(:, image, ispin) = sums(:, image, ispin) + sine*reduced(:, nclass + sine_index(i), ispin, 1)
2610 SUBROUTINE rskp_image_classes(cells, classes, representatives)
2611 INTEGER,
INTENT(IN) :: cells(:, :)
2612 INTEGER,
ALLOCATABLE,
INTENT(OUT) :: classes(:), representatives(:, :)
2614 INTEGER :: d, i, n, nclass
2615 INTEGER,
ALLOCATABLE :: first(:), keys(:, :), order(:), sense(:)
2618 ALLOCATE (keys(3, n), order(n), sense(n), first(n), classes(n))
2622 IF (cells(d, i) == 0) cycle
2623 sense(i) = sign(1, cells(d, i))
2626 keys(:, i) = sense(i)*cells(:, i)
2628 IF (n > 0)
CALL sort(keys, 1, n, 1, 3, order)
2631 IF (all(keys(:, i) == 0))
THEN
2632 classes(order(i)) = 0
2635 IF (nclass == 0)
THEN
2638 ELSE IF (any(keys(:, i) /= keys(:, first(nclass))))
THEN
2642 classes(order(i)) = sense(order(i))*nclass
2644 ALLOCATE (representatives(3, nclass))
2645 representatives(:, :) = keys(:, first(1:nclass))
2646 END SUBROUTINE rskp_image_classes
2657 INTEGER,
INTENT(IN) :: shift(3)
2658 REAL(kind=dp),
INTENT(IN) :: xkp(3)
2659 LOGICAL,
INTENT(IN) :: time_reversal, real_only
2660 REAL(kind=dp),
INTENT(OUT) :: phase(2, 2)
2662 REAL(kind=dp) :: arg, cosine, sine
2664 arg = twopi*(real(shift(1), dp)*xkp(1) + real(shift(2), dp)*xkp(2) + real(shift(3), dp)*xkp(3))
2667 IF (real_only .AND. abs(sine) > 1.e-12_dp)
THEN
2668 CALL cp_abort(__location__,
"Real k-point wavefunctions cannot represent symmetry phases")
2670 phase(:, 1) = [cosine, -sine]
2671 phase(:, 2) = merge(-1.0_dp, 1.0_dp, time_reversal)*[sine, cosine]
2682 REAL(kind=dp),
ALLOCATABLE,
DIMENSION(:, :), &
2683 INTENT(INOUT) :: work
2684 INTEGER,
INTENT(IN) :: nrow, ncol
2686 IF (
ALLOCATED(work))
THEN
2687 IF (
SIZE(work, 1) == nrow .AND.
SIZE(work, 2) == ncol)
RETURN
2690 ALLOCATE (work(nrow, ncol))
static GRID_HOST_DEVICE int modulo(int a, int m)
Equivalent of Fortran's MODULO, which always return a positive number. https://gcc....
static void dgemm(const char transa, const char transb, const int m, const int n, const int k, const double alpha, const double *a, const int lda, const double *b, const int ldb, const double beta, double *c, const int ldc)
Convenient wrapper to hide Fortran nature of dgemm_, swapping a and b.
Represents a complex full matrix distributed on many processors.
subroutine, public dbcsr_get_readonly_block_p(matrix, row, col, block, found, row_size, col_size)
Like dbcsr_get_block_p() but with matrix being INTENT(IN). When invoking this routine,...
subroutine, public dbcsr_get_block_p(matrix, row, col, block, found, row_size, col_size)
...
subroutine, public dbcsr_get_info(matrix, nblkrows_total, nblkcols_total, nfullrows_total, nfullcols_total, nblkrows_local, nblkcols_local, nfullrows_local, nfullcols_local, my_prow, my_pcol, local_rows, local_cols, proc_row_dist, proc_col_dist, row_blk_size, col_blk_size, row_blk_offset, col_blk_offset, distribution, name, matrix_type, group)
...
subroutine, public dbcsr_set(matrix, alpha)
...
represent the structure of a full matrix
logical function, public cp_fm_struct_equivalent(fmstruct1, fmstruct2)
returns true if the two matrix structures are equivalent, false otherwise.
subroutine, public cp_fm_struct_retain(fmstruct)
retains a full matrix structure
subroutine, public cp_fm_struct_release(fmstruct)
releases a full matrix structure
represent a full matrix distributed on many processors
various routines to log and control the output. The idea is that decisions about where to log should ...
integer function, public cp_logger_get_default_io_unit(logger)
returns the unit nr for the ionode (-1 on all other processors) skips as well checks if the procs cal...
Defines the basic variable types.
integer, parameter, public int_8
integer, parameter, public dp
Sparse direct and regular-grid lattice Fourier transforms.
subroutine, public cell_to_k_grid_prepare(work, index_to_cell, xkp, nkp_grid, deriv_direction, hmat)
Prepare fixed forward lattice maps; the caller keeps geometry and k points unchanged.
subroutine, public k_grid_to_cell_prepare(work, xkp, nkp_grid, index_to_cell, weights, allow_incomplete)
Prepare one inverse lattice transform for repeated matrix batches.
subroutine, public cell_to_k_grid_fft(values_rs, index_to_cell, xkp, nkp_grid, values_k, used_fft, deriv_direction, hmat, selected_kpoints, require_fft, work, cell_first, max_workspace_bytes, accumulate, profile)
Transform a batch of real matrices from image cells to every supplied k point.
subroutine, public k_grid_to_cell_release(work)
Release call-scoped inverse lattice maps and FFT buffers.
subroutine, public cell_to_k_grid_release(work)
Release forward lattice maps and numerical buffers.
subroutine, public k_grid_to_cell_execute(work, values_k, values_rs, used_fft, require_fft, allow_direct_fallback, values_real, profile, max_workspace_bytes)
Execute a matrix batch using the prepared inverse lattice transform. Call outside OpenMP worker regio...
Machine interface based on Fortran 2003 and POSIX.
real(kind=dp) function, public m_walltime()
returns time from a real-time clock, protected against rolling early/easily
Definition of mathematical constants and functions.
complex(kind=dp), parameter, public z_one
real(kind=dp), parameter, public twopi
complex(kind=dp), parameter, public z_zero
Interface to the message passing library MPI.
Define the neighbor list data types and the corresponding functionality.
subroutine, public neighbor_list_iterator_create(iterator_set, nl, search, nthread)
Neighbor list iterator functions.
subroutine, public neighbor_list_iterator_release(iterator_set)
...
subroutine, public get_neighbor_list_set_p(neighbor_list_sets, nlist, symmetric)
Return the components of the first neighbor list set.
integer function, public neighbor_list_iterate(iterator_set, mepos)
...
subroutine, public get_iterator_info(iterator_set, mepos, ikind, jkind, nkind, ilist, nlist, inode, nnode, iatom, jatom, r, cell)
...
All kind of helpful little routines.
pure integer function, public locate(array, x)
Purpose: Given an array array(1:n), and given a value x, a value x_index is returned which is the ind...
Represent a complex full matrix.
keeps the information about the structure of a full matrix
just to build arrays of pointers to matrices
stores all the informations relevant to an mpi environment