(git:ec04593)
Loading...
Searching...
No Matches
kpoint_transform.F
Go to the documentation of this file.
1!--------------------------------------------------------------------------------------------------!
2! CP2K: A general program to perform molecular dynamics simulations !
3! Copyright 2000-2026 CP2K developers group <https://cp2k.org> !
4! !
5! SPDX-License-Identifier: GPL-2.0-or-later !
6!--------------------------------------------------------------------------------------------------!
7
8! **************************************************************************************************
9!> \brief Prepared real-space/reciprocal-space transforms and distributed AO adapters.
10!> Routes fix geometry and ownership. Captured source rows own numerical
11!> values; cached evaluation reads them without communication or SCF state.
12! **************************************************************************************************
14 USE cp_cfm_types, ONLY: cp_cfm_type
19 dbcsr_set,&
25 USE cp_fm_types, ONLY: cp_fm_p_type,&
30 USE kinds, ONLY: dp,&
31 int_8
32 USE kpoint_lattice_fft, ONLY: &
37 USE machine, ONLY: m_walltime
38 USE mathconstants, ONLY: twopi,&
39 z_one,&
40 z_zero
49 USE util, ONLY: locate,&
50 sort
51
52!$ USE omp_lib, ONLY: omp_get_max_threads, omp_get_thread_num, omp_get_num_threads
53#include "./base/base_uses.f90"
54
55 IMPLICIT NONE
56 PRIVATE
57 CHARACTER(len=*), PARAMETER, PRIVATE :: moduleN = 'kpoint_transform'
58 INTEGER(KIND=int_8), PARAMETER :: fft_batch_bytes = 32_int_8*1024_int_8**2
59 INTEGER(KIND=int_8), PARAMETER :: rskp_panel_bytes = 64_int_8*1024_int_8**2
60 CHARACTER(LEN=14), PARAMETER :: fft_profile_stages(9) = [character(len=14) :: &
61 'initialize', 'geometry', 'buffers', 'grid-fill', 'FFT-X/plan', 'FFT-Y/plan', 'FFT-Z/plan', 'extract', 'release']
62 INTEGER(KIND=int_8), PARAMETER, PRIVATE :: rskp_gemm_bytes = 1000_int_8*1024_int_8**2, &
63 dbcsr_batch_bytes = 1000_int_8*1024_int_8**2
72 PUBLIC :: rskp_transform
76
77 ! Unweighted one-dimensional phases for one symmetry group's local k domain.
79 INTEGER :: first(4) = 1
80 INTEGER, ALLOCATABLE :: coordinate(:)
81 REAL(kind=dp), ALLOCATABLE :: cosine(:, :), sine(:, :)
83
85 INTEGER :: nentry = 0, ngroup = 0
86 LOGICAL :: symmetric = .false.
87 INTEGER, ALLOCATABLE, DIMENSION(:) :: col, col_offset, group_start, image, row, row_offset
88 INTEGER, ALLOCATABLE, DIMENSION(:, :) :: cell
89 REAL(kind=dp), ALLOCATABLE, DIMENSION(:) :: symmetry_sign
91
92 ! Numerical plans borrow no FM, DBCSR or SCF objects.
93 TYPE rskp_backend_plan_type
94 INTEGER :: backend = rskp_backend_gemm
95 INTEGER, ALLOCATABLE :: packed_first(:), fold_map(:, :)
96 INTEGER, ALLOCATABLE :: phase_index(:), phase_cells(:, :), phase_count(:, :), phase_first(:)
97 INTEGER(KIND=int_8) :: max_per_k = 0_int_8
98 REAL(kind=dp), ALLOCATABLE :: phases(:, :, :)
100 END TYPE rskp_backend_plan_type
101
102 ! Stable canonical AO route and destination mapping, shared by both backends.
104 TYPE(cp_fm_struct_type), POINTER :: matrix_struct => null()
105 TYPE(mp_para_env_type), POINTER :: world => null(), group => null(), inter => null()
107 TYPE(neighbor_list_set_p_type), POINTER :: source_sab(:) => null()
108 TYPE(rskp_backend_plan_type) :: transform = rskp_backend_plan_type()
109 INTEGER :: nrow = 0, ncol = 0, nimage = 0, nvalue = 0
110 LOGICAL :: world_target = .false.
111 INTEGER(KIND=int_8) :: storage = 0_int_8
112 INTEGER, ALLOCATABLE :: entry(:, :), block_first(:), block_order(:), element_first(:)
113 INTEGER(KIND=int_8), ALLOCATABLE :: element_offset(:, :)
114 INTEGER, ALLOCATABLE :: row_size(:), col_size(:), first(:)
115 INTEGER, ALLOCATABLE :: send_entry(:), send_first(:), scount(:, :), rcount(:, :), &
116 sdisp(:, :), rdisp(:, :), send_element(:)
117 REAL(kind=dp), ALLOCATABLE :: send_values(:), recv_values(:)
118 END TYPE rskp_route_type
119
120 ! One source row: backend payload plus the source identity used by adapters.
122 REAL(kind=dp), ALLOCATABLE :: packed(:)
123 LOGICAL, ALLOCATABLE :: active(:, :)
124 ! Unscaled canonical elements by local k; mirrors and FM padding are not stored.
125 COMPLEX(KIND=dp), ALLOCATABLE :: fft_values(:, :)
126 TYPE(dbcsr_p_type), POINTER :: source(:, :) => null()
127 INTEGER :: matrix_row = 0, mirror_sign = 1
128 COMPLEX(KIND=dp) :: scale = z_one
129 END TYPE rskp_values_type
130
131 ! Slot-free owner for captured source rows sharing one target-local R-to-K map.
133 PRIVATE
134 TYPE(rskp_route_type) :: route = rskp_route_type()
135 TYPE(rskp_values_type), ALLOCATABLE :: values(:)
136 INTEGER, ALLOCATABLE :: kp_dist(:, :)
137 REAL(kind=dp), ALLOCATABLE :: work(:)
139
140CONTAINS
141
142! **************************************************************************************************
143!> \brief Transform local real matrix panels with the selected RSKP backend.
144!> GEMM reuses the prepared parity folding and block contraction. FFT reuses one geometry
145!> across bounded matrix tiles. Both accept rectangular panels and regular-grid subsets.
146!> Input and output may be strided; only the internal GEMM panels are contiguous.
147!> Call outside OpenMP worker regions; this service owns its numerical workspace.
148!> \param values_rs real matrices, with one image per third-dimension column
149!> \param cells lattice translations of the images
150!> \param xkp requested reciprocal coordinates, in output order
151!> \param nkp_grid mesh dimensions for FFT
152!> \param values_k complex transformed matrices
153!> \param backend GEMM or FFT
154!> \param accumulate add this image panel to an existing output
155!> \param profile_fft print local FFT stage times on the logger rank, without MPI collectives
156!> \param profile_label identify the caller operation in the optional FFT profile
157! **************************************************************************************************
158 SUBROUTINE cell_to_k_transform(values_rs, cells, xkp, nkp_grid, values_k, backend, accumulate, &
159 profile_fft, profile_label)
160 REAL(kind=dp), INTENT(IN) :: values_rs(:, :, :)
161 INTEGER, INTENT(IN) :: cells(:, :)
162 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
163 INTEGER, INTENT(IN) :: nkp_grid(3)
164 COMPLEX(KIND=dp), INTENT(INOUT) :: values_k(:, :, :)
165 INTEGER, INTENT(IN) :: backend
166 LOGICAL, INTENT(IN), OPTIONAL :: accumulate, profile_fft
167 CHARACTER(LEN=*), INTENT(IN), OPTIONAL :: profile_label
168
169 INTEGER :: column, count, first_column, first_row, handle, i, image, kfirst, last, &
170 last_column, matrix_rows, nk, nphase, nrow, ntile, nworkers, odd, offset, omp_limit, &
171 panels(4), pos, row, sense, start, unit_nr, width
172 INTEGER(KIND=int_8) :: cache_bytes, ifirst, nvalues, &
173 worker_bytes
174 LOGICAL :: active(4), add, measure_fft
175 REAL(kind=dp) :: profile_started, started
176 REAL(kind=dp), ALLOCATABLE :: factors(:, :), packed(:), result(:, :)
177 TYPE(lattice_fft_profile_type), ALLOCATABLE :: fft_profile
178 TYPE(rskp_backend_plan_type) :: plan
179
180 CALL timeset('cell_to_k_transform', handle)
181 measure_fft = .false.
182 IF (PRESENT(profile_fft)) measure_fft = profile_fft .AND. backend == rskp_backend_fft
183 IF (measure_fft) THEN
184 ALLOCATE (fft_profile)
185 profile_started = m_walltime()
186 END IF
187 cpassert(SIZE(cells, 1) == 3 .AND. SIZE(xkp, 1) == 3)
188 cpassert(SIZE(cells, 2) == SIZE(values_rs, 3))
189 cpassert(SIZE(values_k, 1) == SIZE(values_rs, 1))
190 cpassert(SIZE(values_k, 2) == SIZE(values_rs, 2))
191 nk = SIZE(xkp, 2)
192 cpassert(SIZE(values_k, 3) == nk)
193 add = .false.
194 IF (PRESENT(accumulate)) add = accumulate
195 ! A nonempty FFT overwrites its output; GEMM and empty image sums need zeros.
196 IF (.NOT. add .AND. (backend /= rskp_backend_fft .OR. SIZE(values_rs, kind=int_8) == 0)) values_k = z_zero
197 IF (measure_fft) fft_profile%seconds(1) = m_walltime() - profile_started
198 IF (SIZE(values_rs, kind=int_8) == 0 .OR. nk == 0) THEN
199 CALL timestop(handle)
200 RETURN
201 END IF
202 SELECT CASE (backend)
203 CASE (rskp_backend_fft)
204 CALL cell_to_k_grid_fft(values_rs, cells, xkp, nkp_grid, values_k, &
205 require_fft=.true., accumulate=add, profile=fft_profile)
206 CASE (rskp_backend_gemm)
207 nvalues = SIZE(values_rs, 1, kind=int_8)*SIZE(values_rs, 2, kind=int_8)
208 matrix_rows = SIZE(values_rs, 1)
209 nworkers = 1
210!$ nworkers = omp_get_max_threads()
211 worker_bytes = rskp_panel_bytes/(2*nworkers)
212 ntile = int(min(nvalues, 4096_int_8, &
213 max(1_int_8, worker_bytes/(16_int_8*(SIZE(cells, 2, kind=int_8) + 1)))))
214 CALL cell_to_k_phase_prepare(plan, cells, [1, SIZE(cells, 2) + 1], &
215 [(image, image=1, SIZE(cells, 2))], [1, ntile + 1], &
216 xkp, rskp_panel_bytes/2, 0_int_8)
217 cache_bytes = 0_int_8
218 IF (ALLOCATED(plan%phases)) cache_bytes = 8_int_8*SIZE(plan%phases, kind=int_8)
219 worker_bytes = (rskp_panel_bytes - cache_bytes)/nworkers - &
220 8_int_8*(plan%packed_first(2) - 1)
221 width = int(min(int(nk, int_8), max(1_int_8, worker_bytes/(16_int_8*plan%max_per_k))))
222 nphase = maxval(plan%phase_count(1:2, 1))
223 panels = plan%phase_count([1, 1, 2, 3], 1)
224!$OMP PARALLEL DEFAULT(NONE) &
225!$OMP SHARED(values_rs, values_k, matrix_rows, nvalues, xkp, plan, ntile, nphase, panels, nk, width) &
226!$OMP PRIVATE(packed, factors, result, ifirst, last, nrow, image, pos, odd, sense, start, active, i, kfirst) &
227!$OMP PRIVATE(first_row, first_column, last_column, column, row, count, offset)
228 ALLOCATE (packed(plan%packed_first(2) - 1), factors(nphase, 2*width), result(ntile, 2*width))
229!$OMP DO SCHEDULE(STATIC)
230 DO ifirst = 1, nvalues, ntile
231 nrow = int(min(int(ntile, int_8), nvalues - ifirst + 1))
232 first_row = int(modulo(ifirst - 1, int(matrix_rows, int_8))) + 1
233 first_column = int((ifirst - 1)/int(matrix_rows, int_8)) + 1
234 last_column = int((ifirst + int(nrow, int_8) - 2)/int(matrix_rows, int_8)) + 1
235 packed(:) = 0.0_dp
236 DO image = 1, SIZE(values_rs, 3)
237 pos = plan%fold_map(1, image)
238 odd = abs(plan%fold_map(2, image))
239 sense = sign(1, plan%fold_map(2, image))
240 ! Split the flattened tile into column-local row spans. Array descriptors
241 ! retain the caller's strides, including a partial first-dimension chunk.
242 offset = 0
243 row = first_row
244 DO column = first_column, last_column
245 count = min(nrow - offset, matrix_rows - row + 1)
246 packed(pos + offset:pos + offset + count - 1) = &
247 packed(pos + offset:pos + offset + count - 1) + values_rs(row:row + count - 1, column, image)
248 IF (odd > 0) THEN
249 packed(odd + offset:odd + offset + count - 1) = &
250 packed(odd + offset:odd + offset + count - 1) + &
251 REAL(sense, dp)*values_rs(row:row + count - 1, column, image)
252 END IF
253 offset = offset + count
254 row = 1
255 END DO
256 END DO
257 active = .false.
258 start = 1
259 DO i = 1, 4
260 IF (panels(i) > 0) active(i) = any(packed(start:start + ntile*panels(i) - 1) /= 0.0_dp)
261 start = start + ntile*panels(i)
262 END DO
263 DO kfirst = 1, nk, width
264 last = min(kfirst + width - 1, nk)
265 CALL cell_to_k_block_gemm(packed, plan, 1, xkp(:, kfirst:last), kfirst, &
266 factors(:, 1:2*(last - kfirst + 1)), &
267 result(:, 1:2*(last - kfirst + 1)), active)
268 offset = 0
269 row = first_row
270 DO column = first_column, last_column
271 count = min(nrow - offset, matrix_rows - row + 1)
272 values_k(row:row + count - 1, column, kfirst:last) = &
273 values_k(row:row + count - 1, column, kfirst:last) + &
274 cmplx(result(offset + 1:offset + count, 1:last - kfirst + 1), &
275 result(offset + 1:offset + count, last - kfirst + 2:2*(last - kfirst + 1)), kind=dp)
276 offset = offset + count
277 row = 1
278 END DO
279 END DO
280 END DO
281!$OMP END DO
282 DEALLOCATE (packed, factors, result)
283!$OMP END PARALLEL
284 CASE DEFAULT
285 cpabort('Unknown R-to-K backend')
286 END SELECT
287 IF (measure_fft) THEN
288 started = m_walltime() - profile_started
290 omp_limit = 1
291!$ omp_limit = omp_get_max_threads()
292 IF (unit_nr > 0) THEN
293 WRITE (unit_nr, '(T2,A)') 'R2K_FFT_PROFILE| Local panels; logger-rank wall seconds, no MPI reduction'
294 IF (PRESENT(profile_label)) WRITE (unit_nr, '(T2,A,A)') 'R2K_FFT_PROFILE| operation: ', trim(profile_label)
295 WRITE (unit_nr, '(T2,A,3I10)') 'R2K_FFT_PROFILE| source shape: ', shape(values_rs)
296 WRITE (unit_nr, '(T2,A,6I8)') 'R2K_FFT_PROFILE| grid / padded: ', nkp_grid, fft_profile%nfft
297 WRITE (unit_nr, '(T2,A,2I16,2I8)') 'R2K_FFT_PROFILE| elements/batches/tile min/max: ', &
298 fft_profile%elements, fft_profile%batches, fft_profile%tile_min, fft_profile%tile_max
299 WRITE (unit_nr, '(T2,A,I8,2L3)') 'R2K_FFT_PROFILE| FFT OpenMP limit / input,output contiguous: ', &
300 omp_limit, is_contiguous(values_rs), is_contiguous(values_k)
301 IF (fft_profile%pair_major) WRITE (unit_nr, '(T2,A)') &
302 'R2K_FFT_PROFILE| FFT-X/plan reports full 3D batched FFT (Y/Z entries zero)'
303 WRITE (unit_nr, '(T2,A)') &
304 'R2K_FFT_PROFILE| grid-fill/extract use size-gated OpenMP; FFT axes include one-shot planning'
305 DO i = 1, SIZE(fft_profile_stages)
306 WRITE (unit_nr, '(T2,A,A14,F14.6)') 'R2K_FFT_PROFILE| ', fft_profile_stages(i), fft_profile%seconds(i)
307 END DO
308 WRITE (unit_nr, '(T2,A,F14.6)') 'R2K_FFT_PROFILE| total ', started
309 END IF
310 END IF
311 CALL timestop(handle)
312 END SUBROUTINE cell_to_k_transform
313
314! **************************************************************************************************
315!> \brief Real part of the inverse lattice transform, normalized by the number of k points.
316!> Local rectangular panels use shared FFT execution or real GEMM contractions with
317!> cosine and sine weights. Matrix and cell tiles bound temporary storage.
318!> Call outside OpenMP worker regions; this service owns its numerical workspace.
319!> \param values_k complex reciprocal-space matrices
320!> \param xkp reciprocal coordinates
321!> \param nkp_grid mesh dimensions for FFT
322!> \param cells requested lattice translations
323!> \param values_rs real part of the reconstructed matrices
324!> \param backend FFT backend (required)
325!> \param profile_fft print local FFT stage times on the logger rank, without MPI collectives
326! **************************************************************************************************
327 SUBROUTINE k_to_real_cell_transform(values_k, xkp, nkp_grid, cells, values_rs, backend, profile_fft)
328 COMPLEX(KIND=dp), CONTIGUOUS, INTENT(IN), TARGET :: values_k(:, :, :)
329 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
330 INTEGER, INTENT(IN) :: nkp_grid(3), cells(:, :)
331 REAL(kind=dp), CONTIGUOUS, INTENT(OUT), TARGET :: values_rs(:, :, :)
332 INTEGER, INTENT(IN) :: backend
333 LOGICAL, INTENT(IN), OPTIONAL :: profile_fft
334
335 INTEGER :: handle, i, ncell, nk, omp_limit, unit_nr
336 LOGICAL :: measure_fft
337 REAL(kind=dp) :: profile_started, started
338 TYPE(k_grid_to_cell_work_type) :: fft_work
339 TYPE(lattice_fft_profile_type), ALLOCATABLE :: fft_profile
340
341 CALL timeset('k_to_real_cell_transform', handle)
342 measure_fft = .false.
343 IF (PRESENT(profile_fft)) measure_fft = profile_fft .AND. backend == rskp_backend_fft
344 IF (measure_fft) THEN
345 ALLOCATE (fft_profile)
346 profile_started = m_walltime()
347 END IF
348 nk = SIZE(xkp, 2)
349 ncell = SIZE(cells, 2)
350 cpassert(SIZE(cells, 1) == 3 .AND. SIZE(xkp, 1) == 3)
351 cpassert(nk > 0 .AND. SIZE(values_k, 3) == nk)
352 cpassert(SIZE(values_rs, 1) == SIZE(values_k, 1))
353 cpassert(SIZE(values_rs, 2) == SIZE(values_k, 2))
354 cpassert(SIZE(values_rs, 3) == ncell)
355 IF (SIZE(values_rs, kind=int_8) == 0) THEN
356 CALL timestop(handle)
357 RETURN
358 END IF
359 cpassert(backend == rskp_backend_fft)
360 IF (measure_fft) started = m_walltime()
361 CALL k_grid_to_cell_prepare(fft_work, xkp, nkp_grid, cells)
362 IF (measure_fft) fft_profile%seconds(2) = m_walltime() - started
363 CALL k_grid_to_cell_execute(fft_work, values_k, values_real=values_rs, require_fft=.true., profile=fft_profile)
364 IF (measure_fft) started = m_walltime()
365 CALL k_grid_to_cell_release(fft_work)
366 IF (measure_fft) fft_profile%seconds(9) = m_walltime() - started
367 IF (measure_fft) THEN
368 started = m_walltime() - profile_started
370 omp_limit = 1
371!$ omp_limit = omp_get_max_threads()
372 IF (unit_nr > 0) THEN
373 WRITE (unit_nr, '(T2,A)') 'K2R_FFT_PROFILE| Local panels; logger-rank wall seconds, no MPI reduction'
374 WRITE (unit_nr, '(T2,A,3I10)') 'K2R_FFT_PROFILE| source shape: ', shape(values_k)
375 WRITE (unit_nr, '(T2,A,6I8)') 'K2R_FFT_PROFILE| grid / padded: ', nkp_grid, fft_profile%nfft
376 WRITE (unit_nr, '(T2,A,2I16,2I8)') 'K2R_FFT_PROFILE| elements/batches/tile min/max: ', &
377 fft_profile%elements, fft_profile%batches, fft_profile%tile_min, fft_profile%tile_max
378 WRITE (unit_nr, '(T2,A,I8,2L3)') 'K2R_FFT_PROFILE| FFT OpenMP limit / input,output contiguous: ', &
379 omp_limit, is_contiguous(values_k), is_contiguous(values_rs)
380 WRITE (unit_nr, '(T2,A)') &
381 'K2R_FFT_PROFILE| grid-fill/extract use size-gated OpenMP; FFT axes include one-shot planning'
382 WRITE (unit_nr, '(T2,A,L2)') 'K2R_FFT_PROFILE| pair-major FFT input/output: ', &
383 fft_profile%pair_major
384 IF (fft_profile%pair_major) WRITE (unit_nr, '(T2,A)') &
385 'K2R_FFT_PROFILE| FFT-X/plan reports full 3D batched FFT (Y/Z entries zero)'
386 DO i = 1, SIZE(fft_profile_stages)
387 WRITE (unit_nr, '(T2,A,A14,F14.6)') 'K2R_FFT_PROFILE| ', fft_profile_stages(i), fft_profile%seconds(i)
388 END DO
389 WRITE (unit_nr, '(T2,A,F14.6)') 'K2R_FFT_PROFILE| total ', started
390 END IF
391 END IF
392 CALL timestop(handle)
393 END SUBROUTINE k_to_real_cell_transform
394
395! **************************************************************************************************
396!> \brief Transformation of real space matrices to a kpoint
397!> \param rmatrix Real part of kpoint matrix
398!> \param cmatrix Complex part of kpoint matrix (optional)
399!> \param rsmat Real space matrices
400!> \param ispin Spin index
401!> \param xkp Kpoint coordinates
402!> \param cell_to_index mapping of cell indices to RS index
403!> \param sab_nl Defining neighbor list
404!> \param is_complex Matrix to be transformed is imaginary
405!> \param rs_sign Matrix to be transformed is csaled by rs_sign
406! **************************************************************************************************
407 SUBROUTINE rskp_transform(rmatrix, cmatrix, rsmat, ispin, &
408 xkp, cell_to_index, sab_nl, is_complex, rs_sign)
409
410 TYPE(dbcsr_type) :: rmatrix
411 TYPE(dbcsr_type), OPTIONAL :: cmatrix
412 TYPE(dbcsr_p_type), DIMENSION(:, :), POINTER :: rsmat
413 INTEGER, INTENT(IN) :: ispin
414 REAL(kind=dp), DIMENSION(3), INTENT(IN) :: xkp
415 INTEGER, DIMENSION(:, :, :), POINTER :: cell_to_index
416 TYPE(neighbor_list_set_p_type), DIMENSION(:), &
417 POINTER :: sab_nl
418 LOGICAL, INTENT(IN), OPTIONAL :: is_complex
419 REAL(kind=dp), INTENT(IN), OPTIONAL :: rs_sign
420
421 CHARACTER(LEN=*), PARAMETER :: routinen = 'rskp_transform'
422
423 INTEGER :: handle, iatom, ic, icol, irow, jatom, &
424 nimg
425 INTEGER, DIMENSION(3) :: cell
426 LOGICAL :: do_symmetric, found, my_complex, &
427 wfn_real_only
428 REAL(kind=dp) :: arg, coskl, fsign, fsym, sinkl
429 REAL(kind=dp), DIMENSION(:, :), POINTER :: cblock, rblock, rsblock
431 DIMENSION(:), POINTER :: nl_iterator
432
433 CALL timeset(routinen, handle)
434
435 my_complex = .false.
436 IF (PRESENT(is_complex)) my_complex = is_complex
437
438 fsign = 1.0_dp
439 IF (PRESENT(rs_sign)) fsign = rs_sign
440
441 wfn_real_only = .true.
442 IF (PRESENT(cmatrix)) wfn_real_only = .false.
443
444 nimg = SIZE(rsmat, 2)
445
446 CALL get_neighbor_list_set_p(neighbor_list_sets=sab_nl, symmetric=do_symmetric)
447
448 CALL neighbor_list_iterator_create(nl_iterator, sab_nl)
449 DO WHILE (neighbor_list_iterate(nl_iterator) == 0)
450 CALL get_iterator_info(nl_iterator, iatom=iatom, jatom=jatom, cell=cell)
451
452 ! fsym = +- 1 is due to real space matrices being non-symmetric (although in a symmtric type)
453 ! with the link S_mu^0,nu^b = S_nu^0,mu^-b, and the KP matrices beeing Hermitian
454 fsym = 1.0_dp
455 irow = iatom
456 icol = jatom
457 IF (do_symmetric .AND. (iatom > jatom)) THEN
458 irow = jatom
459 icol = iatom
460 fsym = -1.0_dp
461 END IF
462
463 ic = cell_to_index(cell(1), cell(2), cell(3))
464 IF (ic < 1 .OR. ic > nimg) cycle
465
466 arg = real(cell(1), dp)*xkp(1) + real(cell(2), dp)*xkp(2) + real(cell(3), dp)*xkp(3)
467 IF (my_complex) THEN
468 coskl = fsign*fsym*cos(twopi*arg)
469 sinkl = fsign*sin(twopi*arg)
470 ELSE
471 coskl = fsign*cos(twopi*arg)
472 sinkl = fsign*fsym*sin(twopi*arg)
473 END IF
474
475 CALL dbcsr_get_block_p(matrix=rsmat(ispin, ic)%matrix, row=irow, col=icol, &
476 block=rsblock, found=found)
477 IF (.NOT. found) cycle
478
479 IF (wfn_real_only) THEN
480 CALL dbcsr_get_block_p(matrix=rmatrix, row=irow, col=icol, &
481 block=rblock, found=found)
482 IF (.NOT. found) cycle
483 rblock = rblock + coskl*rsblock
484 ELSE
485 CALL dbcsr_get_block_p(matrix=rmatrix, row=irow, col=icol, &
486 block=rblock, found=found)
487 IF (.NOT. found) cycle
488 CALL dbcsr_get_block_p(matrix=cmatrix, row=irow, col=icol, &
489 block=cblock, found=found)
490 IF (.NOT. found) cycle
491 rblock = rblock + coskl*rsblock
492 cblock = cblock + sinkl*rsblock
493 END IF
494
495 END DO
496 CALL neighbor_list_iterator_release(nl_iterator)
497
498 CALL timestop(handle)
499
500 END SUBROUTINE rskp_transform
501
502! **************************************************************************************************
503!> \brief Prepare source rows for a generic group- or world-distributed target.
504!> GEMM retains sparse snapshots and sizes k tiles against a workspace target.
505!> Own route, snapshots, singleton scratch and any FFT cache without SCF slots.
506!> \param prepared transform handle to initialize
507!> \param source real-space source rows and images
508!> \param source_rows source rows sharing the route/backend plan
509!> \param source_sab neighbor-list geometry of source
510!> \param cell_to_index translation-to-image map
511!> \param target_struct output matrix layout
512!> \param world source-matrix communicator
513!> \param group optional target-group communicator; omit for a world target
514!> \param inter optional communicator joining equivalent group ranks; omit for a world target
515!> \param backend selected GEMM or FFT backend
516!> \param xkp full global k-point coordinates
517!> \param nkp_grid regular-grid dimensions
518!> \param kp_dist optional global k ranges by inter-group rank; omit for a world target
519!> \param mirror_sign transpose parity (+1 Hermitian, -1 anti-Hermitian)
520!> \param scale complex prefactor applied to direct and mirrored contributions
521! **************************************************************************************************
522 SUBROUTINE rskp_transform_prepare(prepared, source, source_rows, source_sab, cell_to_index, target_struct, &
523 world, group, inter, backend, xkp, nkp_grid, kp_dist, mirror_sign, scale)
524
525 TYPE(rskp_prepared_transform_type), INTENT(INOUT) :: prepared
526 TYPE(dbcsr_p_type), POINTER :: source(:, :)
527 INTEGER, INTENT(IN) :: source_rows(:)
528 TYPE(neighbor_list_set_p_type), DIMENSION(:), &
529 POINTER :: source_sab
530 INTEGER, DIMENSION(:, :, :), POINTER :: cell_to_index
531 TYPE(cp_fm_struct_type), POINTER :: target_struct
532 TYPE(mp_para_env_type), POINTER :: world
533 TYPE(mp_para_env_type), OPTIONAL, POINTER :: group, inter
534 INTEGER, INTENT(IN) :: backend
535 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
536 INTEGER, INTENT(IN) :: nkp_grid(3)
537 INTEGER, INTENT(IN), OPTIONAL :: kp_dist(:, :), mirror_sign
538 COMPLEX(KIND=dp), INTENT(IN), OPTIONAL :: scale
539
540 CHARACTER(LEN=*), PARAMETER :: routinen = 'rskp_transform_prepare'
541
542 INTEGER :: first, handle, irow, nk_local
543 INTEGER(KIND=int_8) :: max_backend_bytes
544 INTEGER, ALLOCATABLE :: eval_kp_dist(:, :)
545 LOGICAL :: ready
546 REAL(kind=dp), ALLOCATABLE :: phase_xkp(:, :)
547 TYPE(mp_para_env_type), POINTER :: eval_group, eval_inter
548
549 CALL timeset(routinen, handle)
550 CALL rskp_transform_release(prepared)
551 cpassert(SIZE(source_rows) > 0)
552 cpassert(all(source_rows >= 1) .AND. all(source_rows <= SIZE(source, 1)))
553 cpassert(SIZE(xkp, 1) == 3 .AND. SIZE(xkp, 2) > 0)
554 cpassert(ASSOCIATED(target_struct) .AND. ASSOCIATED(world))
555 IF (PRESENT(group) .EQV. PRESENT(inter)) THEN
556 IF (PRESENT(group)) THEN
557 cpassert(PRESENT(kp_dist))
558 cpassert(ASSOCIATED(group) .AND. ASSOCIATED(inter))
559 eval_group => group
560 eval_inter => inter
561 ELSE
562 cpassert(.NOT. PRESENT(kp_dist))
563 cpassert(ASSOCIATED(target_struct%para_env, world))
564 eval_group => world
565 eval_inter => world
566 END IF
567 ELSE
568 cpabort("R-to-K group and inter communicators must be supplied together")
569 END IF
570 IF (PRESENT(kp_dist)) THEN
571 cpassert(SIZE(kp_dist, 1) == 2)
572 ALLOCATE (eval_kp_dist, source=kp_dist)
573 ELSE
574 ALLOCATE (eval_kp_dist(2, 1))
575 eval_kp_dist(:, 1) = [1, SIZE(xkp, 2)]
576 END IF
577 max_backend_bytes = rskp_gemm_bytes
578 IF (backend == rskp_backend_fft) max_backend_bytes = rskp_panel_bytes
579 ALLOCATE (prepared%kp_dist, source=eval_kp_dist)
580 ALLOCATE (prepared%values(SIZE(source_rows)))
581 CALL rskp_route_create(source, source_sab, cell_to_index, target_struct, world, eval_group, eval_inter, backend, &
582 prepared%route, ready)
583 IF (.NOT. ready) THEN
584 cpabort("R-to-K route has an incompatible source or target layout")
585 END IF
586
587 IF (backend == rskp_backend_fft .OR. prepared%route%world_target) THEN
588 CALL cell_to_k_backend_prepare(prepared%route, xkp, nkp_grid, max_backend_bytes)
589 ELSE
590 first = eval_kp_dist(1, eval_inter%mepos + 1)
591 ALLOCATE (phase_xkp, source=xkp(:, first:eval_kp_dist(2, eval_inter%mepos + 1)))
592 CALL cell_to_k_backend_prepare(prepared%route, phase_xkp, nkp_grid, max_backend_bytes)
593 END IF
594 DO irow = 1, SIZE(source_rows)
595 CALL rskp_capture(source, source_rows(irow), prepared%route, prepared%values(irow), mirror_sign, scale)
596 END DO
597 IF (backend == rskp_backend_fft) THEN
598 nk_local = SIZE(xkp, 2)
599 IF (.NOT. prepared%route%world_target) THEN
600 nk_local = eval_kp_dist(2, eval_inter%mepos + 1) - &
601 eval_kp_dist(1, eval_inter%mepos + 1) + 1
602 END IF
603 DO irow = 1, SIZE(source_rows)
604 ALLOCATE (prepared%values(irow)%fft_values( &
605 prepared%route%element_first(SIZE(prepared%route%element_first)) - 1, nk_local))
606 CALL cell_to_k_sparse_fft(prepared%values(irow), prepared%route, xkp, nkp_grid, &
607 eval_kp_dist, eval_inter, &
608 max_backend_bytes, cache=prepared%values(irow)%fft_values)
609 DEALLOCATE (prepared%values(irow)%packed)
610 END DO
611 ALLOCATE (prepared%work(0))
612 ELSE
613 ALLOCATE (prepared%work(2_int_8*prepared%route%transform%max_per_k))
614 END IF
615 CALL timestop(handle)
616 END SUBROUTINE rskp_transform_prepare
617
618! **************************************************************************************************
619!> \brief Evaluate one k point directly into an FM, split-FM or CFM target.
620!> The handle owns mutable scratch; concurrent evaluations need separate handles.
621!> Evaluation uses the batch backend with width one and no communication.
622!> Groups that do not own the requested global k point leave outputs unchanged.
623!> \param prepared initialized transform handle
624!> \param source_index one-based source row index supplied at preparation
625!> \param k global k-point index
626!> \param xkp full global k-point coordinates
627!> \param fm_re real output matrix for split-FM storage
628!> \param fm_im optional imaginary output matrix for split-FM storage
629!> \param cfm optional complex full-matrix output
630! **************************************************************************************************
631 SUBROUTINE rskp_transform_evaluate(prepared, source_index, k, xkp, fm_re, fm_im, cfm)
632 TYPE(rskp_prepared_transform_type), INTENT(INOUT) :: prepared
633 INTEGER, INTENT(IN) :: source_index, k
634 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
635 TYPE(cp_fm_type), INTENT(INOUT), OPTIONAL, TARGET :: fm_re, fm_im
636 TYPE(cp_cfm_type), INTENT(INOUT), OPTIONAL, TARGET :: cfm
637
638 COMPLEX(KIND=dp), CONTIGUOUS, POINTER :: batch(:, :, :)
639 INTEGER :: first_local, ncomp
640 LOGICAL :: target_layout_matches
641 TYPE(cp_fm_p_type) :: output(1, 2)
642
643 cpassert(ALLOCATED(prepared%values))
644 cpassert(source_index >= 1 .AND. source_index <= SIZE(prepared%values))
645 cpassert(PRESENT(fm_re) .NEQV. PRESENT(cfm))
646 cpassert(.NOT. PRESENT(fm_im) .OR. PRESENT(fm_re))
647 cpassert(ALLOCATED(prepared%work))
648 cpassert(k >= 1 .AND. k <= SIZE(xkp, 2))
649 first_local = k
650 IF (.NOT. prepared%route%world_target) THEN
651 IF (k < prepared%kp_dist(1, prepared%route%inter%mepos + 1) .OR. &
652 k > prepared%kp_dist(2, prepared%route%inter%mepos + 1)) RETURN
653 first_local = k - prepared%kp_dist(1, prepared%route%inter%mepos + 1) + 1
654 END IF
655 IF (PRESENT(cfm)) THEN
656 target_layout_matches = cp_fm_struct_equivalent(cfm%matrix_struct, prepared%route%matrix_struct)
657 cpassert(target_layout_matches)
658 batch(1:SIZE(cfm%local_data, 1), 1:SIZE(cfm%local_data, 2), 1:1) => cfm%local_data
659 CALL cell_to_k_backend_evaluate_batch(prepared%values(source_index), prepared%route, &
660 xkp(:, k:k), first_local, prepared%work, batch=batch)
661 ELSE
662 output(1, 1)%matrix => fm_re
663 ncomp = 1
664 IF (PRESENT(fm_im)) THEN
665 output(1, 2)%matrix => fm_im
666 ncomp = 2
667 END IF
668 CALL cell_to_k_backend_evaluate_batch(prepared%values(source_index), prepared%route, &
669 xkp(:, k:k), first_local, prepared%work, output=output(:, 1:ncomp))
670 END IF
671 END SUBROUTINE rskp_transform_evaluate
672
673! **************************************************************************************************
674!> \brief Release the route, snapshot and cached backend state of a prepared transform.
675!> \param prepared transform handle to release
676! **************************************************************************************************
677 SUBROUTINE rskp_transform_release(prepared)
678 TYPE(rskp_prepared_transform_type), INTENT(INOUT) :: prepared
679
680 INTEGER :: irow
681
682 IF (ALLOCATED(prepared%values)) THEN
683 DO irow = 1, SIZE(prepared%values)
684 CALL rskp_values_release(prepared%values(irow))
685 END DO
686 DEALLOCATE (prepared%values)
687 END IF
688 CALL rskp_route_release(prepared%route)
689 IF (ALLOCATED(prepared%kp_dist)) DEALLOCATE (prepared%kp_dist)
690 IF (ALLOCATED(prepared%work)) DEALLOCATE (prepared%work)
691 END SUBROUTINE rskp_transform_release
692
693! **************************************************************************************************
694!> \brief Transform all real-space spin rows to a full k-grid in distributed DBCSR matrices.
695!> The real and imaginary output blocks retain their caller-provided sparsity and
696!> distribution. GEMM contracts all images of an AO block together; FFT transforms
697!> element tiles while reusing one prepared lattice geometry.
698!> \param source real-space image matrices, by spin and image
699!> \param real_k symmetric real output matrices, by spin and k point
700!> \param imag_k antisymmetric imaginary output matrices, by spin and k point
701!> \param xkp full reciprocal grid
702!> \param cell_to_index translation-to-image map
703!> \param sab_nl neighbor-list geometry of source
704!> \param nkp_grid dimensions of the regular reciprocal grid
705!> \param backend selected bulk R-to-K backend
706! **************************************************************************************************
707 SUBROUTINE rskp_dbcsr_transform_batch(source, real_k, imag_k, xkp, cell_to_index, sab_nl, nkp_grid, backend)
708
709 TYPE(dbcsr_p_type), POINTER :: source(:, :), real_k(:, :), imag_k(:, :)
710 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
711 INTEGER, DIMENSION(:, :, :), POINTER :: cell_to_index
712 TYPE(neighbor_list_set_p_type), DIMENSION(:), &
713 POINTER :: sab_nl
714 INTEGER, INTENT(IN) :: nkp_grid(3), backend
715
716 CHARACTER(LEN=*), PARAMETER :: routinen = 'rskp_dbcsr_transform_batch'
717
718 COMPLEX(KIND=dp), ALLOCATABLE :: fft_output(:, :, :)
719 INTEGER :: col, first, first_k, g, handle, i, ik, ispin, last, max_images, max_nelem, ncol, &
720 nelem, nentry, nfold, ngroup, nimages, nk, nphase, nrow, nspin, odd, offset, panels(4), &
721 pos, row_first, row_last, tile_first, tile_size, tile_width, width
722 INTEGER(KIND=int_8) :: available_bytes, fft_tile_budget, &
723 nfactor, panel_bytes, per_k_bytes, &
724 per_tile_bytes
725 INTEGER, ALLOCATABLE :: element_first(:), group_first(:), &
726 order(:), signed_cells(:, :)
727 INTEGER, DIMENSION(:), POINTER :: col_size, row_size
728 LOGICAL :: active(4), found, found_im, found_re
729 REAL(kind=dp), ALLOCATABLE, TARGET :: packed(:), work(:)
730 REAL(kind=dp), CONTIGUOUS, POINTER :: factors(:, :), fft_input(:, :, :), &
731 image_re(:, :), result(:, :)
732 REAL(kind=dp), POINTER :: iblock(:, :), rblock(:, :), &
733 result_block(:, :), rsblock(:, :)
734 TYPE(cell_to_k_grid_work_type) :: fft_work
735 TYPE(kp_transform_plan_type) :: plan
736 TYPE(rskp_backend_plan_type) :: transform
737
738 CALL timeset(routinen, handle)
739 nspin = SIZE(source, 1)
740 nk = SIZE(xkp, 2)
741 cpassert(nk > 0)
742 IF (backend /= rskp_backend_gemm .AND. backend /= rskp_backend_fft) THEN
743 cpabort("Unknown R-to-K backend")
744 END IF
745 cpassert(nspin > 0 .AND. SIZE(real_k, 1) == nspin .AND. SIZE(imag_k, 1) == nspin)
746 cpassert(SIZE(real_k, 2) == nk .AND. SIZE(imag_k, 2) == nk)
747 cpassert(SIZE(source, 2) > 0)
748 cpassert(ASSOCIATED(sab_nl) .AND. ASSOCIATED(cell_to_index))
749 DO ik = 1, nk
750 DO ispin = 1, nspin
751 CALL dbcsr_set(real_k(ispin, ik)%matrix, 0.0_dp)
752 CALL dbcsr_set(imag_k(ispin, ik)%matrix, 0.0_dp)
753 END DO
754 END DO
755
756 CALL kp_transform_plan_create(plan, sab_nl, cell_to_index, SIZE(source, 2), group_entries=.true.)
757 nentry = plan%nentry
758 IF (nentry == 0) THEN
759 CALL timestop(handle)
760 RETURN
761 END IF
762 CALL dbcsr_get_info(source(1, 1)%matrix, row_blk_size=row_size, col_blk_size=col_size)
763 ALLOCATE (group_first(nentry + 1), element_first(nentry + 1), order(nentry), signed_cells(3, nentry))
764 DO i = 1, nentry
765 order(i) = i
766 signed_cells(:, i) = nint(plan%symmetry_sign(i))*plan%cell(:, i)
767 END DO
768 max_images = 0
769 max_nelem = 0
770 ngroup = 0
771 element_first(1) = 1
772 first = 1
773 DO WHILE (first <= nentry)
774 last = first
775 DO WHILE (last < nentry)
776 IF (plan%row(last + 1) /= plan%row(first) .OR. plan%col(last + 1) /= plan%col(first)) EXIT
777 last = last + 1
778 END DO
779 ngroup = ngroup + 1
780 group_first(ngroup) = first
781 panel_bytes = int(row_size(plan%row(first)), int_8)*col_size(plan%col(first))*nspin
782 IF (panel_bytes + element_first(ngroup) > huge(nelem)) THEN
783 cpabort("R-to-K DBCSR panel offsets exceed the integer range")
784 END IF
785 element_first(ngroup + 1) = element_first(ngroup) + int(panel_bytes)
786 max_nelem = max(max_nelem, int(panel_bytes))
787 max_images = max(max_images, last - first + 1)
788 first = last + 1
789 END DO
790 group_first(ngroup + 1) = nentry + 1
791 panel_bytes = 8_int_8*max_nelem*max_images
792 available_bytes = dbcsr_batch_bytes - panel_bytes
793 IF (backend == rskp_backend_gemm) THEN
794 ! The raw image panel is retained; the budget sizes optional phases and k tiles.
795 available_bytes = max(0_int_8, available_bytes)
796 CALL cell_to_k_phase_prepare(transform, signed_cells, group_first(1:ngroup + 1), order, &
797 element_first(1:ngroup + 1), xkp, available_bytes, 0_int_8)
798 IF (ALLOCATED(transform%phases)) THEN
799 available_bytes = available_bytes - 8_int_8*SIZE(transform%phases, kind=int_8)
800 END IF
801 per_k_bytes = 16_int_8*transform%max_per_k
802 tile_width = int(min(int(nk, int_8), max(1_int_8, available_bytes/max(1_int_8, per_k_bytes))))
803 ALLOCATE (work(per_k_bytes/8*tile_width))
804 ELSE
805 fft_tile_budget = fft_batch_bytes
806 CALL cell_to_k_grid_prepare(fft_work, signed_cells, xkp, nkp_grid)
807 IF (.NOT. fft_work%regular) THEN
808 cpabort("Forced lattice FFT needs a supported regular grid or subset")
809 END IF
810 END IF
811 ALLOCATE (packed(panel_bytes/8))
812
813 DO g = 1, ngroup
814 first = group_first(g)
815 last = group_first(g + 1) - 1
816 nimages = last - first + 1
817 nrow = row_size(plan%row(first))
818 ncol = col_size(plan%col(first))
819 nelem = nrow*ncol
820 IF (backend == rskp_backend_gemm) THEN
821 panels = transform%phase_count([1, 1, 2, 3], g)
822 nfold = sum(panels)
823 ELSE
824 nfold = nimages
825 END IF
826 image_re(1:nelem*nspin, 1:nfold) => packed(1:int(nelem, int_8)*int(nspin, int_8)*int(nfold, int_8))
827 image_re = 0.0_dp
828 DO i = first, last
829 pos = i - first + 1
830 odd = 0
831 IF (backend == rskp_backend_gemm) THEN
832 pos = (transform%fold_map(1, i) - transform%packed_first(g))/(nelem*nspin) + 1
833 IF (transform%fold_map(2, i) /= 0) THEN
834 odd = (abs(transform%fold_map(2, i)) - transform%packed_first(g))/(nelem*nspin) + 1
835 END IF
836 END IF
837 DO ispin = 1, nspin
838 CALL dbcsr_get_readonly_block_p(source(ispin, plan%image(i))%matrix, plan%row(first), plan%col(first), &
839 rsblock, found)
840 IF (.NOT. found) cycle
841 cpassert(SIZE(rsblock, 1) == nrow .AND. SIZE(rsblock, 2) == ncol)
842 image_re((ispin - 1)*nelem + 1:ispin*nelem, pos) = &
843 image_re((ispin - 1)*nelem + 1:ispin*nelem, pos) + reshape(rsblock, [nelem])
844 IF (odd == 0) cycle
845 image_re((ispin - 1)*nelem + 1:ispin*nelem, odd) = &
846 image_re((ispin - 1)*nelem + 1:ispin*nelem, odd) + &
847 REAL(sign(1, transform%fold_map(2, i)), dp)*reshape(rsblock, [nelem])
848 END DO
849 END DO
850 IF (.NOT. any(image_re /= 0.0_dp)) cycle
851
852 IF (backend == rskp_backend_gemm) THEN
853 pos = 1
854 DO i = 1, 4
855 active(i) = any(image_re(:, pos:pos + panels(i) - 1) /= 0.0_dp)
856 pos = pos + panels(i)
857 END DO
858 nphase = maxval(transform%phase_count(1:2, g))
859 DO first_k = 1, nk, tile_width
860 width = min(tile_width, nk - first_k + 1)
861 nfactor = 2_int_8*nphase*width
862 factors(1:nphase, 1:2*width) => work(1:nfactor)
863 result(1:nelem*nspin, 1:2*width) => work(nfactor + 1:nfactor + 2_int_8*nelem*nspin*width)
864 CALL cell_to_k_block_gemm(packed(1:nelem*nspin*nfold), transform, g, &
865 xkp(:, first_k:first_k + width - 1), first_k, factors, result, active)
866 DO ik = 1, width
867 DO ispin = 1, nspin
868 CALL dbcsr_get_block_p(real_k(ispin, first_k + ik - 1)%matrix, plan%row(first), plan%col(first), &
869 rblock, found)
870 IF (found) THEN
871 cpassert(SIZE(rblock, 1) == nrow .AND. SIZE(rblock, 2) == ncol)
872 result_block(1:nrow, 1:ncol) => result((ispin - 1)*nelem + 1:ispin*nelem, ik)
873 rblock = result_block
874 END IF
875 CALL dbcsr_get_block_p(imag_k(ispin, first_k + ik - 1)%matrix, plan%row(first), plan%col(first), &
876 iblock, found)
877 IF (found) THEN
878 cpassert(SIZE(iblock, 1) == nrow .AND. SIZE(iblock, 2) == ncol)
879 result_block(1:nrow, 1:ncol) => result((ispin - 1)*nelem + 1:ispin*nelem, width + ik)
880 iblock = result_block
881 END IF
882 END DO
883 END DO
884 END DO
885 ELSE
886 per_tile_bytes = 16_int_8*nspin*nk
887 IF (int(nspin, int_8)*int(nk, int_8) > int(huge(0), int_8)) THEN
888 cpabort("R-to-K DBCSR FFT tile exceeds the integer range")
889 END IF
890 tile_width = min(nelem, int(max(1_int_8, min(fft_tile_budget/per_tile_bytes, &
891 int(huge(0), int_8)/(int(nspin, int_8)*int(nk, int_8))))))
892 fft_input(1:nelem, 1:nspin, 1:nimages) => packed(1:int(nelem, int_8)*int(nspin, int_8)*int(nimages, int_8))
893 ALLOCATE (fft_output(tile_width, nspin, nk))
894 DO tile_first = 1, nelem, tile_width
895 tile_size = min(tile_width, nelem - tile_first + 1)
896 CALL cell_to_k_grid_fft(fft_input(tile_first:tile_first + tile_size - 1, :, :), &
897 signed_cells(:, first:last), xkp, &
898 nkp_grid, fft_output(1:tile_size, :, :), require_fft=.true., work=fft_work, &
899 cell_first=first)
900 DO ik = 1, nk
901 DO ispin = 1, nspin
902 CALL dbcsr_get_block_p(real_k(ispin, ik)%matrix, plan%row(first), plan%col(first), &
903 rblock, found_re)
904 IF (found_re) THEN
905 cpassert(SIZE(rblock, 1) == nrow .AND. SIZE(rblock, 2) == ncol)
906 END IF
907 CALL dbcsr_get_block_p(imag_k(ispin, ik)%matrix, plan%row(first), plan%col(first), &
908 iblock, found_im)
909 IF (found_im) THEN
910 cpassert(SIZE(iblock, 1) == nrow .AND. SIZE(iblock, 2) == ncol)
911 END IF
912 IF (.NOT. (found_re .OR. found_im)) cycle
913 ! Write each column segment directly; FFT tiles may split a column.
914 DO col = (tile_first - 1)/nrow + 1, (tile_first + tile_size - 2)/nrow + 1
915 offset = (col - 1)*nrow
916 row_first = max(1, tile_first - offset)
917 row_last = min(nrow, tile_first + tile_size - 1 - offset)
918 pos = offset + row_first - tile_first + 1
919 IF (found_re) rblock(row_first:row_last, col) = &
920 REAL(fft_output(pos:pos + row_last - row_first, ispin, ik), dp)
921 IF (found_im) iblock(row_first:row_last, col) = &
922 aimag(fft_output(pos:pos + row_last - row_first, ispin, ik))
923 END DO
924 END DO
925 END DO
926 END DO
927 DEALLOCATE (fft_output)
928 END IF
929 END DO
930 CALL cell_to_k_grid_release(fft_work)
931 CALL timestop(handle)
932 END SUBROUTINE rskp_dbcsr_transform_batch
933
934! **************************************************************************************************
935!> \brief Build the immutable neighbor-list traversal shared by R-to-K and K-to-R transforms.
936!> \param plan ...
937!> \param sab_nl ...
938!> \param cell_to_index ...
939!> \param nimg ...
940!> \param block_template ...
941!> \param group_entries ...
942! **************************************************************************************************
943 SUBROUTINE kp_transform_plan_create(plan, sab_nl, cell_to_index, nimg, block_template, group_entries)
944
945 TYPE(kp_transform_plan_type), INTENT(OUT) :: plan
946 TYPE(neighbor_list_set_p_type), DIMENSION(:), &
947 POINTER :: sab_nl
948 INTEGER, DIMENSION(:, :, :), POINTER :: cell_to_index
949 INTEGER, INTENT(IN) :: nimg
950 TYPE(dbcsr_type), INTENT(IN), OPTIONAL :: block_template
951 LOGICAL, INTENT(IN), OPTIONAL :: group_entries
952
953 INTEGER :: i, iatom, icell, icol, igroup, irow, &
954 jatom, nblock
955 INTEGER(KIND=int_8), ALLOCATABLE, DIMENSION(:) :: key
956 INTEGER, ALLOCATABLE, DIMENSION(:) :: order
957 INTEGER, DIMENSION(3) :: cell
958 INTEGER, DIMENSION(:), POINTER :: col_offsets, row_offsets
959 LOGICAL :: do_grouping, do_symmetric, store_offsets
961 DIMENSION(:), POINTER :: iterator
962
963 CALL get_neighbor_list_set_p(neighbor_list_sets=sab_nl, symmetric=do_symmetric)
964 plan%symmetric = do_symmetric
965 do_grouping = .false.
966 IF (PRESENT(group_entries)) do_grouping = group_entries
967 store_offsets = PRESENT(block_template)
968 IF (store_offsets) THEN
969 CALL dbcsr_get_info(block_template, row_blk_offset=row_offsets, col_blk_offset=col_offsets)
970 END IF
971 CALL neighbor_list_iterator_create(iterator, sab_nl)
972 DO WHILE (neighbor_list_iterate(iterator) == 0)
973 CALL get_iterator_info(iterator, cell=cell)
974 icell = cell_to_index(cell(1), cell(2), cell(3))
975 IF (icell >= 1 .AND. icell <= nimg) plan%nentry = plan%nentry + 1
976 END DO
977 CALL neighbor_list_iterator_release(iterator)
978
979 ALLOCATE (plan%row(plan%nentry), plan%col(plan%nentry), plan%image(plan%nentry), &
980 plan%cell(3, plan%nentry), plan%symmetry_sign(plan%nentry))
981 IF (store_offsets) THEN
982 ALLOCATE (plan%row_offset(plan%nentry), plan%col_offset(plan%nentry))
983 END IF
984
985 i = 0
986 CALL neighbor_list_iterator_create(iterator, sab_nl)
987 DO WHILE (neighbor_list_iterate(iterator) == 0)
988 CALL get_iterator_info(iterator, iatom=iatom, jatom=jatom, cell=cell)
989 icell = cell_to_index(cell(1), cell(2), cell(3))
990 IF (icell < 1 .OR. icell > nimg) cycle
991 i = i + 1
992 irow = iatom
993 icol = jatom
994 plan%symmetry_sign(i) = 1.0_dp
995 IF (do_symmetric .AND. iatom > jatom) THEN
996 irow = jatom
997 icol = iatom
998 plan%symmetry_sign(i) = -1.0_dp
999 END IF
1000 plan%row(i) = irow
1001 plan%col(i) = icol
1002 IF (store_offsets) THEN
1003 plan%row_offset(i) = row_offsets(irow)
1004 plan%col_offset(i) = col_offsets(icol)
1005 END IF
1006 plan%image(i) = icell
1007 plan%cell(:, i) = cell
1008 END DO
1009 CALL neighbor_list_iterator_release(iterator)
1010 cpassert(i == plan%nentry)
1011
1012 IF (do_grouping .AND. plan%nentry > 0) THEN
1013 nblock = max(maxval(plan%row), maxval(plan%col))
1014 ALLOCATE (key(plan%nentry), order(plan%nentry))
1015 DO i = 1, plan%nentry
1016 key(i) = int(plan%image(i), kind=int_8) + int(nimg, kind=int_8)* &
1017 (int(plan%row(i) - 1, kind=int_8) + int(nblock, kind=int_8)* &
1018 int(plan%col(i) - 1, kind=int_8))
1019 END DO
1020 CALL sort(key, plan%nentry, order)
1021 plan%row(:) = plan%row(order)
1022 plan%col(:) = plan%col(order)
1023 IF (store_offsets) THEN
1024 plan%row_offset(:) = plan%row_offset(order)
1025 plan%col_offset(:) = plan%col_offset(order)
1026 END IF
1027 plan%image(:) = plan%image(order)
1028 plan%cell(:, :) = plan%cell(:, order)
1029 plan%symmetry_sign(:) = plan%symmetry_sign(order)
1030
1031 plan%ngroup = 1
1032 DO i = 2, plan%nentry
1033 IF (key(i) /= key(i - 1)) plan%ngroup = plan%ngroup + 1
1034 END DO
1035 ALLOCATE (plan%group_start(plan%ngroup + 1))
1036 igroup = 1
1037 plan%group_start(igroup) = 1
1038 DO i = 2, plan%nentry
1039 IF (key(i) /= key(i - 1)) THEN
1040 igroup = igroup + 1
1041 plan%group_start(igroup) = i
1042 END IF
1043 END DO
1044 plan%group_start(plan%ngroup + 1) = plan%nentry + 1
1045 DEALLOCATE (key, order)
1046 END IF
1047
1048 END SUBROUTINE kp_transform_plan_create
1049
1050! **************************************************************************************************
1051!> \brief Contract one real image panel with real phase factors.
1052!> \param nrow number of AO elements in the panel
1053!> \param nimages number of real-space images in the panel
1054!> \param nk number of k-point columns
1055!> \param values_re image values contributing to the real output
1056!> \param phase_re real phase factors
1057!> \param result_re real output panel
1058!> \param beta output accumulation factor
1059! **************************************************************************************************
1060 SUBROUTINE rskp_gemm_contract(nrow, nimages, nk, values_re, phase_re, result_re, beta)
1061 INTEGER, INTENT(IN) :: nrow, nimages, nk
1062 REAL(kind=dp), CONTIGUOUS, INTENT(IN) :: values_re(:, :), phase_re(:, :)
1063 REAL(kind=dp), CONTIGUOUS, INTENT(INOUT) :: result_re(:, :)
1064 REAL(kind=dp), INTENT(IN) :: beta
1065
1066 cpassert(nrow > 0 .AND. nimages > 0 .AND. nk > 0)
1067 cpassert(SIZE(values_re, 1) >= nrow .AND. SIZE(values_re, 2) >= nimages)
1068 cpassert(SIZE(phase_re, 1) >= nimages .AND. SIZE(phase_re, 2) >= nk)
1069 cpassert(SIZE(result_re, 1) >= nrow .AND. SIZE(result_re, 2) >= nk)
1070
1071 CALL dgemm('N', 'N', nrow, nk, nimages, 1.0_dp, values_re, SIZE(values_re, 1), &
1072 phase_re, SIZE(phase_re, 1), beta, result_re, SIZE(result_re, 1))
1073 END SUBROUTINE rskp_gemm_contract
1074
1075! **************************************************************************************************
1076!> \brief Cache routing to the FM tiles of one representative group.
1077!> Equivalent ranks in the other groups receive only their tile's blocks.
1078!> Both backends receive the union of locally owned direct/mirrored AO elements.
1079!> Missing source blocks are packed as zero on a fixed geometry.
1080!> \param source operator layout template
1081!> \param source_sab neighbor lists belonging to source
1082!> \param cell_to_index map from lattice translations to source image indices
1083!> \param matrix_struct target group or world FM layout
1084!> \param world communicator distributing the source DBCSR matrices
1085!> \param group communicator of group-local FM targets
1086!> \param inter communicator joining equivalent FM ranks across groups
1087!> \param backend GEMM or FFT
1088!> \param route context-owned structural route
1089!> \param ready false collectively if source or target layout is incompatible
1090! **************************************************************************************************
1091 SUBROUTINE rskp_route_create(source, source_sab, cell_to_index, matrix_struct, world, group, inter, &
1092 backend, route, ready)
1093
1094 TYPE(dbcsr_p_type), POINTER :: source(:, :)
1095 TYPE(neighbor_list_set_p_type), DIMENSION(:), &
1096 POINTER :: source_sab
1097 INTEGER, POINTER :: cell_to_index(:, :, :)
1098 TYPE(cp_fm_struct_type), POINTER :: matrix_struct
1099 TYPE(mp_para_env_type), POINTER :: world, group, inter
1100 INTEGER, INTENT(IN) :: backend
1101 TYPE(rskp_route_type), INTENT(OUT) :: route
1102 LOGICAL, INTENT(OUT) :: ready
1103
1104 CHARACTER(LEN=*), PARAMETER :: routinen = 'rskp_route_create'
1105
1106 INTEGER :: compatible, dest, handle, i, iblock, &
1107 layout(11), layout_max(11), &
1108 layout_min(11), nblock, ncol, nowner, &
1109 nrow, p, pos, rank
1110 INTEGER(KIND=int_8) :: nr(2), ns(2), storage
1111 INTEGER(KIND=int_8), ALLOCATABLE :: recv_size(:, :), send_size(:, :)
1112 INTEGER, ALLOCATABLE :: identities(:, :), owner(:), &
1113 position(:, :), send_metadata(:, :), &
1114 value_count(:), value_start(:)
1115 INTEGER, POINTER :: col_size(:), row_size(:)
1116 LOGICAL, ALLOCATABLE :: needed(:)
1117
1118 CALL timeset(routinen, handle)
1119 ready = .false.
1120 route%world => world
1121 route%group => group
1122 route%inter => inter
1123 CALL dbcsr_get_info(source(1, 1)%matrix, row_blk_size=row_size, col_blk_size=col_size)
1124 CALL kp_transform_plan_create(route%traversal, source_sab, cell_to_index, &
1125 SIZE(source, 2), block_template=source(1, 1)%matrix)
1126 route%source_sab => source_sab
1127 route%transform%backend = backend
1128 route%world_target = ASSOCIATED(matrix_struct%para_env, world)
1129 nowner = group%num_pe
1130 IF (route%world_target) nowner = world%num_pe
1131 compatible = merge(1, 0, route%world_target .OR. &
1132 ASSOCIATED(matrix_struct%para_env, group))
1133 IF (.NOT. route%world_target) THEN
1134 layout = [matrix_struct%nrow_global, matrix_struct%ncol_global, &
1135 matrix_struct%nrow_block, matrix_struct%ncol_block, matrix_struct%first_p_pos, &
1136 matrix_struct%context%num_pe, matrix_struct%context%mepos, matrix_struct%local_leading_dimension]
1137 layout_min = layout
1138 layout_max = layout
1139 CALL inter%min(layout_min)
1140 CALL inter%max(layout_max)
1141 IF (any(layout_min /= layout_max)) compatible = 0
1142 END IF
1143 IF (sum(row_size) /= matrix_struct%nrow_global .OR. &
1144 sum(col_size) /= matrix_struct%ncol_global) compatible = 0
1145 IF (route%traversal%symmetric) THEN
1146 IF (matrix_struct%nrow_global /= matrix_struct%ncol_global .OR. SIZE(row_size) /= SIZE(col_size)) THEN
1147 compatible = 0
1148 ELSE IF (any(row_size /= col_size)) THEN
1149 compatible = 0
1150 END IF
1151 END IF
1152 CALL world%min(compatible)
1153 IF (compatible == 0) THEN
1154 CALL cp_warn(__location__, &
1155 "Prepared R-to-K transform has an incompatible source or target layout")
1156 CALL timestop(handle)
1157 RETURN
1158 END IF
1159 route%nimage = SIZE(source, 2)
1160 ALLOCATE (route%row_size(SIZE(row_size)), route%col_size(SIZE(col_size)))
1161 route%row_size(:) = row_size
1162 route%col_size(:) = col_size
1163 ALLOCATE (identities(2, 0:world%num_pe - 1), owner(0:nowner - 1), &
1164 needed(0:nowner - 1), value_count(0:nowner - 1), value_start(0:nowner - 1))
1165 CALL world%allgather([group%mepos, inter%mepos], identities)
1166 owner(:) = -1
1167 IF (route%world_target) THEN
1168 DO rank = 0, world%num_pe - 1
1169 owner(rank) = rank
1170 END DO
1171 ELSE
1172 DO rank = 0, world%num_pe - 1
1173 IF (identities(2, rank) == 0) owner(identities(1, rank)) = rank
1174 END DO
1175 END IF
1176 cpassert(all(owner >= 0))
1177 ALLOCATE (send_size(2, 0:world%num_pe - 1), recv_size(2, 0:world%num_pe - 1))
1178 send_size(:, :) = 0_int_8
1179 DO i = 1, route%traversal%nentry
1180 nrow = row_size(route%traversal%row(i))
1181 ncol = col_size(route%traversal%col(i))
1182 CALL rskp_owners(route, matrix_struct, i, needed, value_count)
1183 DO p = 0, SIZE(owner) - 1
1184 IF (.NOT. needed(p)) cycle
1185 dest = owner(p)
1186 send_size(1, dest) = send_size(1, dest) + 1_int_8
1187 send_size(2, dest) = send_size(2, dest) + int(value_count(p), int_8)
1188 END DO
1189 END DO
1190 CALL world%alltoall(send_size, recv_size, 2)
1191 ! Only inter-rank zero receives from the world. The other groups hold
1192 ! equivalent FM tiles, as required by kpoint_env_initialize.
1193 ns = sum(send_size, dim=2)
1194 nr = sum(recv_size, dim=2)
1195 IF (.NOT. route%world_target) CALL inter%bcast(nr, 0)
1196 ! Conservative routing/capture allocation estimate for diagnostics.
1197 storage = 40_int_8*route%traversal%nentry + &
1198 4_int_8*(SIZE(row_size) + SIZE(col_size) + matrix_struct%nrow_global + matrix_struct%ncol_global) + &
1199 48_int_8*ns(1) + merge(0_int_8, 8_int_8*ns(2), world%num_pe == 1) + 68_int_8*nr(1) + &
1200 128_int_8*world%num_pe + 16_int_8*SIZE(owner) + 8_int_8
1201 ! Singleton groups also retain the AO-block group boundaries.
1202 IF (SIZE(owner) == 1) storage = storage + 4_int_8*(nr(1) + 1_int_8)
1203 ! Group permutations, packed offsets, direct/mirror element maps and
1204 ! sorting scratch. The cropped receive buffer is shared by all slots.
1205 storage = storage + 52_int_8*(nr(1) + 1_int_8) + 16_int_8*nr(2)
1206 IF (world%num_pe > 1) storage = storage + 8_int_8*nr(2)
1207 IF (SIZE(owner) > 1) storage = storage + 4_int_8*ns(2)
1208 route%storage = storage
1209 ! Values, offsets and eight-integer metadata records use default INTEGER MPI counts.
1210 IF (maxval(ns) >= int(huge(0), int_8) .OR. maxval(nr) >= int(huge(0), int_8) .OR. &
1211 max(ns(1), nr(1)) > int(huge(0), int_8)/8_int_8) THEN
1212 CALL cp_abort(__location__, "R-to-K route counts exceed the integer range")
1213 END IF
1214 ! Route counts are checked explicitly before conversion to default INTEGER.
1215 nblock = int(nr(1))
1216 route%nvalue = int(nr(2))
1217 ALLOCATE (route%scount(2, 0:world%num_pe - 1), route%rcount(2, 0:world%num_pe - 1), &
1218 route%sdisp(2, 0:world%num_pe - 1), route%rdisp(2, 0:world%num_pe - 1))
1219 route%scount(:, :) = int(send_size)
1220 route%rcount(:, :) = int(recv_size)
1221 IF (SIZE(owner) == 1 .AND. .NOT. route%world_target) CALL inter%bcast(route%rcount, 0)
1222 route%sdisp(:, 0) = 0
1223 route%rdisp(:, 0) = 0
1224 DO rank = 1, world%num_pe - 1
1225 route%sdisp(:, rank) = route%sdisp(:, rank - 1) + route%scount(:, rank - 1)
1226 route%rdisp(:, rank) = route%rdisp(:, rank - 1) + route%rcount(:, rank - 1)
1227 END DO
1228 ALLOCATE (route%send_entry(int(ns(1))), route%send_first(int(ns(1)) + 1), &
1229 route%send_values(merge(0, int(ns(2)), world%num_pe == 1)), send_metadata(8, int(ns(1))), &
1230 position(2, 0:world%num_pe - 1), route%entry(8, nblock), &
1231 route%first(nblock + 1))
1232 IF (SIZE(owner) > 1) ALLOCATE (route%send_element(int(ns(2))))
1233 IF (world%num_pe > 1) ALLOCATE (route%recv_values(route%nvalue))
1234 position(:, :) = route%sdisp
1235 DO i = 1, route%traversal%nentry
1236 nrow = row_size(route%traversal%row(i))
1237 ncol = col_size(route%traversal%col(i))
1238 IF (ALLOCATED(route%send_element)) THEN
1239 value_start(:) = position(2, owner)
1240 CALL rskp_owners(route, matrix_struct, i, needed, value_count, &
1241 value_start, route%send_element)
1242 ELSE
1243 CALL rskp_owners(route, matrix_struct, i, needed, value_count)
1244 END IF
1245 DO p = 0, SIZE(owner) - 1
1246 IF (.NOT. needed(p)) cycle
1247 dest = owner(p)
1248 iblock = position(1, dest) + 1
1249 pos = position(2, dest) + 1
1250 route%send_entry(iblock) = i
1251 route%send_first(iblock) = pos
1252 send_metadata(1:4, iblock) = [route%traversal%row_offset(i), route%traversal%col_offset(i), nrow, ncol]
1253 send_metadata(5:7, iblock) = nint(route%traversal%symmetry_sign(i))*route%traversal%cell(:, i)
1254 send_metadata(8, iblock) = merge(1, 0, route%traversal%symmetric .AND. &
1255 route%traversal%row(i) /= route%traversal%col(i))
1256 position(:, dest) = position(:, dest) + [1, value_count(p)]
1257 END DO
1258 END DO
1259 route%send_first(int(ns(1)) + 1) = int(ns(2)) + 1
1260 IF (SIZE(owner) == 1 .AND. world%num_pe == 1) THEN
1261 CALL world%allgatherv(send_metadata, route%entry, 8*route%rcount(1, :), 8*route%rdisp(1, :))
1262 ELSE IF (.NOT. route%world_target .AND. SIZE(owner) == 1) THEN
1263 CALL world%allgatherv(send_metadata, route%entry, 8*route%rcount(1, :), 8*route%rdisp(1, :))
1264 ELSE
1265 CALL world%alltoall(send_metadata, 8*route%scount(1, :), 8*route%sdisp(1, :), &
1266 route%entry, 8*route%rcount(1, :), 8*route%rdisp(1, :))
1267 IF (.NOT. route%world_target) CALL inter%bcast(route%entry, 0)
1268 END IF
1269 route%matrix_struct => matrix_struct
1270 CALL cp_fm_struct_retain(route%matrix_struct)
1271 CALL rskp_tile_map(route)
1272 ready = .true.
1273 CALL timestop(handle)
1274
1275 END SUBROUTINE rskp_route_create
1276
1277! **************************************************************************************************
1278!> \brief Map routed blocks to local FM ownership and stable atom-pair groups.
1279!> FM indices are ordered globally; dummy indices on empty tiles lie outside the matrix.
1280!> \param route fixed routing and target distribution
1281! **************************************************************************************************
1282 SUBROUTINE rskp_tile_map(route)
1283 TYPE(rskp_route_type), INTENT(INOUT) :: route
1284
1285 INTEGER :: c0, col, element, first, g, i, image, m, &
1286 nc, ngroup, nr, overlap, pos, r0, row
1287 INTEGER(KIND=int_8), ALLOCATABLE :: key(:)
1288 INTEGER, ALLOCATABLE :: element_count(:)
1289 LOGICAL :: direct, mirror
1290 INTEGER :: rows(0:route%matrix_struct%nrow_global), cols(0:route%matrix_struct%ncol_global)
1291
1292 rows(:) = 0
1293 cols(:) = 0
1294 DO i = 1, SIZE(route%matrix_struct%row_indices)
1295 r0 = route%matrix_struct%row_indices(i)
1296 IF (r0 >= 1 .AND. r0 <= ubound(rows, 1)) rows(r0) = 1
1297 END DO
1298 DO i = 1, SIZE(route%matrix_struct%col_indices)
1299 c0 = route%matrix_struct%col_indices(i)
1300 IF (c0 >= 1 .AND. c0 <= ubound(cols, 1)) cols(c0) = 1
1301 END DO
1302 DO i = 1, ubound(rows, 1)
1303 rows(i) = rows(i) + rows(i - 1)
1304 END DO
1305 DO i = 1, ubound(cols, 1)
1306 cols(i) = cols(i) + cols(i - 1)
1307 END DO
1308 route%nrow = rows(ubound(rows, 1))
1309 route%ncol = cols(ubound(cols, 1))
1310 ALLOCATE (route%block_order(SIZE(route%entry, 2)), element_count(SIZE(route%entry, 2)))
1311 block
1312 ! Source-rank reception need not keep a pair contiguous on distributed FM layouts.
1313 ALLOCATE (key(SIZE(route%entry, 2)))
1314 DO i = 1, SIZE(key)
1315 key(i) = int(route%entry(1, i), int_8) + &
1316 int(route%matrix_struct%nrow_global, int_8)*(route%entry(2, i) - 1)
1317 END DO
1318 IF (SIZE(key) > 0) CALL sort(key, SIZE(key), route%block_order)
1319 ngroup = 0
1320 IF (SIZE(key) > 0) ngroup = 1 + count(key(2:) /= key(:SIZE(key) - 1))
1321 ALLOCATE (route%block_first(ngroup + 1))
1322 route%block_first(1) = 1
1323 g = 1
1324 DO i = 2, SIZE(key)
1325 IF (key(i) == key(i - 1)) cycle
1326 g = g + 1
1327 route%block_first(g) = i
1328 END DO
1329 route%block_first(ngroup + 1) = SIZE(key) + 1
1330 END block
1331 ALLOCATE (route%element_first(ngroup + 1))
1332 route%element_first(1) = 1
1333 pos = 1
1334 DO g = 1, ngroup
1335 first = route%block_order(route%block_first(g))
1336 r0 = route%entry(1, first)
1337 c0 = route%entry(2, first)
1338 nr = route%entry(3, first)
1339 nc = route%entry(4, first)
1340 m = (rows(r0 + nr - 1) - rows(r0 - 1))*(cols(c0 + nc - 1) - cols(c0 - 1))
1341 IF (route%entry(8, first) /= 0) THEN
1342 ! Subtract the intersection: a rank owning both orientations
1343 ! needs one input value and one GEMM row for that element.
1344 overlap = count(rows(r0:r0 + nr - 1) > rows(r0 - 1:r0 + nr - 2) .AND. &
1345 cols(r0:r0 + nr - 1) > cols(r0 - 1:r0 + nr - 2))* &
1346 count(rows(c0:c0 + nc - 1) > rows(c0 - 1:c0 + nc - 2) .AND. &
1347 cols(c0:c0 + nc - 1) > cols(c0 - 1:c0 + nc - 2))
1348 m = m + (rows(c0 + nc - 1) - rows(c0 - 1))*(cols(r0 + nr - 1) - cols(r0 - 1)) - overlap
1349 END IF
1350 route%element_first(g + 1) = route%element_first(g) + m
1351 DO i = route%block_first(g), route%block_first(g + 1) - 1
1352 image = route%block_order(i)
1353 element_count(image) = m
1354 pos = pos + m
1355 END DO
1356 END DO
1357 route%first(1) = 1
1358 DO i = 1, SIZE(element_count)
1359 route%first(i + 1) = route%first(i) + element_count(i)
1360 END DO
1361 cpassert(pos == route%nvalue + 1)
1362 cpassert(route%first(SIZE(element_count) + 1) == pos)
1363 ALLOCATE (route%element_offset(2, route%element_first(ngroup + 1) - 1))
1364 route%element_offset(:, :) = 0_int_8
1365 DO g = 1, ngroup
1366 first = route%block_order(route%block_first(g))
1367 r0 = route%entry(1, first)
1368 c0 = route%entry(2, first)
1369 nr = route%entry(3, first)
1370 nc = route%entry(4, first)
1371 element = route%element_first(g)
1372 DO col = c0, c0 + nc - 1
1373 DO row = r0, r0 + nr - 1
1374 direct = rows(row) > rows(row - 1) .AND. cols(col) > cols(col - 1)
1375 mirror = .false.
1376 IF (route%entry(8, first) /= 0) THEN
1377 mirror = rows(col) > rows(col - 1) .AND. cols(row) > cols(row - 1)
1378 END IF
1379 IF (.NOT. (direct .OR. mirror)) cycle
1380 IF (direct) route%element_offset(1, element) = &
1381 int(rows(row), int_8) + &
1382 int(cols(col) - 1, int_8)*route%matrix_struct%local_leading_dimension
1383 IF (mirror) route%element_offset(2, element) = &
1384 int(rows(col), int_8) + &
1385 int(cols(row) - 1, int_8)*route%matrix_struct%local_leading_dimension
1386 element = element + 1
1387 END DO
1388 END DO
1389 cpassert(element == route%element_first(g + 1))
1390 END DO
1391 IF (route%transform%backend == rskp_backend_fft) THEN
1392 ALLOCATE (route%transform%packed_first(ngroup + 1), route%transform%fold_map(2, SIZE(route%entry, 2)), source=0)
1393 pos = 1
1394 DO g = 1, ngroup
1395 route%transform%packed_first(g) = pos
1396 m = route%element_first(g + 1) - route%element_first(g)
1397 DO i = route%block_first(g), route%block_first(g + 1) - 1
1398 route%transform%fold_map(1, route%block_order(i)) = pos
1399 pos = pos + m
1400 END DO
1401 END DO
1402 route%transform%packed_first(ngroup + 1) = pos
1403 END IF
1404 END SUBROUTINE rskp_tile_map
1405
1406! **************************************************************************************************
1407!> \brief Find group-local FM owners and optionally pack their cropped source indices.
1408!> \param route source AO block layout and neighbor traversal
1409!> \param fms target FM layout within each group
1410!> \param i traversal entry
1411!> \param needed recipient group-local ranks, without duplicate mirror sends
1412!> \param value_count cropped value counts by target rank
1413!> \param value_start optional offsets into the value-index buffer
1414!> \param value_index optional column-major source indices, packed by recipient
1415! **************************************************************************************************
1416 SUBROUTINE rskp_owners(route, fms, i, needed, value_count, value_start, value_index)
1417
1418 TYPE(rskp_route_type), INTENT(IN) :: route
1419 TYPE(cp_fm_struct_type), INTENT(IN) :: fms
1420 INTEGER, INTENT(IN) :: i
1421 LOGICAL, INTENT(OUT) :: needed(0:)
1422 INTEGER, INTENT(OUT) :: value_count(0:)
1423 INTEGER, INTENT(IN), OPTIONAL :: value_start(0:)
1424 INTEGER, INTENT(INOUT), OPTIONAL :: value_index(:)
1425
1426 INTEGER :: c0, col, direct, mirror, nc, nr, r0, row
1427 LOGICAL :: mirror_block
1428
1429 r0 = route%traversal%row_offset(i)
1430 c0 = route%traversal%col_offset(i)
1431 nr = route%row_size(route%traversal%row(i))
1432 nc = route%col_size(route%traversal%col(i))
1433 mirror_block = route%traversal%symmetric .AND. &
1434 route%traversal%row(i) /= route%traversal%col(i)
1435 cpassert(PRESENT(value_start) .EQV. PRESENT(value_index))
1436 value_count(:) = 0
1437 IF (SIZE(needed) == 1) THEN
1438 needed(:) = .true.
1439 value_count(0) = nr*nc
1440 RETURN
1441 END IF
1442 ! Send each canonical source element at most once to a recipient,
1443 ! even when that rank owns both its direct and mirrored destinations.
1444 DO col = 0, nc - 1
1445 DO row = 0, nr - 1
1446 direct = fms%context%blacs2mpi(fms%g2p_row(r0 + row), fms%g2p_col(c0 + col))
1447 mirror = direct
1448 IF (mirror_block) THEN
1449 mirror = fms%context%blacs2mpi(fms%g2p_row(c0 + col), fms%g2p_col(r0 + row))
1450 END IF
1451 value_count(direct) = value_count(direct) + 1
1452 IF (PRESENT(value_index)) THEN
1453 value_index(value_start(direct) + value_count(direct)) = row + col*nr + 1
1454 END IF
1455 IF (mirror == direct) cycle
1456 value_count(mirror) = value_count(mirror) + 1
1457 IF (PRESENT(value_index)) THEN
1458 value_index(value_start(mirror) + value_count(mirror)) = row + col*nr + 1
1459 END IF
1460 END DO
1461 END DO
1462 needed(:) = value_count > 0
1463 END SUBROUTINE rskp_owners
1464
1465! **************************************************************************************************
1466!> \brief Check the source AO layout, not its values or sparse block presence.
1467!> \param route fixed context route
1468!> \param source current operator images
1469!> \param matrix_row row to capture
1470!> \param source_sab source neighbor list, required to match the route's geometry
1471!> \return whether the row has the route's image count and AO layout
1472! **************************************************************************************************
1473 FUNCTION rskp_compatible(route, source, matrix_row, source_sab) RESULT(compatible)
1474
1475 TYPE(rskp_route_type), INTENT(IN) :: route
1476 TYPE(dbcsr_p_type), POINTER :: source(:, :)
1477 INTEGER, INTENT(IN) :: matrix_row
1478 TYPE(neighbor_list_set_p_type), DIMENSION(:), &
1479 POINTER :: source_sab
1480 LOGICAL :: compatible
1481
1482 INTEGER :: image
1483 INTEGER, POINTER :: col_size(:), row_size(:)
1484
1485 compatible = .false.
1486 IF (.NOT. ASSOCIATED(route%source_sab, source_sab)) RETURN
1487 IF (SIZE(source, 2) /= route%nimage) RETURN
1488 IF (matrix_row < 1 .OR. matrix_row > SIZE(source, 1)) RETURN
1489 DO image = 1, SIZE(source, 2)
1490 CALL dbcsr_get_info(source(matrix_row, image)%matrix, row_blk_size=row_size, col_blk_size=col_size)
1491 IF (SIZE(row_size) /= SIZE(route%row_size) .OR. SIZE(col_size) /= SIZE(route%col_size)) RETURN
1492 IF (any(row_size /= route%row_size) .OR. any(col_size /= route%col_size)) RETURN
1493 END DO
1494 compatible = .true.
1495
1496 END FUNCTION rskp_compatible
1497
1498! **************************************************************************************************
1499!> \brief Capture current values using cached routing, with no per-k communication.
1500!> GEMM/FFT snapshots own packed local AO-element/image matrices. A shared
1501!> cropped receive buffer is reused across slots and operator refreshes.
1502!> \param source current real-space operator images
1503!> \param matrix_row row to snapshot
1504!> \param route context-owned metadata, counts and reusable packing buffer
1505!> \param snapshot immutable tile-local values
1506!> \param mirror_sign transpose parity of the unscaled operator (+1 or -1)
1507!> \param scale complex prefactor, applied after forming direct and mirrored values
1508! **************************************************************************************************
1509 SUBROUTINE rskp_capture(source, matrix_row, route, snapshot, mirror_sign, scale)
1510
1511 TYPE(dbcsr_p_type), POINTER :: source(:, :)
1512 INTEGER, INTENT(IN) :: matrix_row
1513 TYPE(rskp_route_type), INTENT(INOUT), TARGET :: route
1514 TYPE(rskp_values_type), INTENT(OUT), TARGET :: snapshot
1515 INTEGER, INTENT(IN), OPTIONAL :: mirror_sign
1516 COMPLEX(KIND=dp), INTENT(IN), OPTIONAL :: scale
1517
1518 CHARACTER(LEN=*), PARAMETER :: routinen = 'rskp_capture'
1519
1520 INTEGER :: col, element, g, handle, i, iblock, m, &
1521 ncol, nrow, odd, panels(4), pos, row
1522 LOGICAL :: found
1523 REAL(kind=dp) :: sense
1524 REAL(kind=dp), CONTIGUOUS, POINTER :: received(:)
1525 REAL(kind=dp), POINTER :: block(:, :)
1526 TYPE(mp_para_env_type), POINTER :: inter, world
1527
1528 CALL timeset(routinen, handle)
1529 world => route%world
1530 inter => route%inter
1531 m = route%transform%packed_first(SIZE(route%transform%packed_first)) - 1
1532 cpassert(m <= route%nvalue)
1533 ALLOCATE (snapshot%packed(m), source=0.0_dp)
1534 snapshot%source => source
1535 snapshot%matrix_row = matrix_row
1536 IF (PRESENT(mirror_sign)) snapshot%mirror_sign = mirror_sign
1537 cpassert(abs(snapshot%mirror_sign) == 1)
1538 IF (PRESENT(scale)) snapshot%scale = scale
1539 IF (world%num_pe == 1) THEN
1540 ! Add each image to its fixed raw-FFT or parity-GEMM panels.
1541 DO iblock = 1, SIZE(route%entry, 2)
1542 i = route%send_entry(iblock)
1543 CALL dbcsr_get_readonly_block_p(source(matrix_row, route%traversal%image(i))%matrix, &
1544 route%traversal%row(i), route%traversal%col(i), block, found)
1545 IF (.NOT. found) cycle
1546 pos = route%transform%fold_map(1, iblock)
1547 odd = abs(route%transform%fold_map(2, iblock))
1548 sense = real(sign(1, route%transform%fold_map(2, iblock)), dp)
1549 DO col = 1, SIZE(block, 2)
1550 DO row = 1, SIZE(block, 1)
1551 snapshot%packed(pos) = snapshot%packed(pos) + block(row, col)
1552 IF (odd > 0) THEN
1553 snapshot%packed(odd) = snapshot%packed(odd) + sense*block(row, col)
1554 odd = odd + 1
1555 END IF
1556 pos = pos + 1
1557 END DO
1558 END DO
1559 END DO
1560 ELSE
1561 received => route%recv_values
1562 route%send_values(:) = 0.0_dp
1563!$OMP PARALLEL DO DEFAULT(NONE) SHARED(source, matrix_row, route) &
1564!$OMP PRIVATE(iblock, i, block, found, nrow, ncol, pos, row, col, element) SCHEDULE(STATIC)
1565 DO iblock = 1, SIZE(route%send_entry)
1566 i = route%send_entry(iblock)
1567 CALL dbcsr_get_readonly_block_p(source(matrix_row, route%traversal%image(i))%matrix, &
1568 route%traversal%row(i), route%traversal%col(i), block, found)
1569 IF (.NOT. found) cycle
1570 nrow = route%row_size(route%traversal%row(i))
1571 ncol = route%col_size(route%traversal%col(i))
1572 cpassert(SIZE(block, 1) == nrow .AND. SIZE(block, 2) == ncol)
1573 pos = route%send_first(iblock)
1574 IF (ALLOCATED(route%send_element)) THEN
1575 DO element = pos, route%send_first(iblock + 1) - 1
1576 row = modulo(route%send_element(element) - 1, nrow) + 1
1577 col = (route%send_element(element) - 1)/nrow + 1
1578 route%send_values(element) = block(row, col)
1579 END DO
1580 cycle
1581 END IF
1582 DO col = 1, ncol
1583 DO row = 1, nrow
1584 route%send_values(pos) = block(row, col)
1585 pos = pos + 1
1586 END DO
1587 END DO
1588 END DO
1589!$OMP END PARALLEL DO
1590 IF (world%num_pe == 1 .OR. &
1591 (.NOT. route%world_target .AND. route%group%num_pe == 1)) THEN
1592 CALL world%allgatherv(route%send_values, received, route%rcount(2, :), route%rdisp(2, :))
1593 ELSE
1594 CALL world%alltoall(route%send_values, route%scount(2, :), route%sdisp(2, :), &
1595 received, route%rcount(2, :), route%rdisp(2, :))
1596 IF (.NOT. route%world_target) CALL inter%bcast(received, 0)
1597 END IF
1598 block
1599 DO iblock = 1, SIZE(route%entry, 2)
1600 pos = route%transform%fold_map(1, iblock)
1601 odd = abs(route%transform%fold_map(2, iblock))
1602 m = route%first(iblock + 1) - route%first(iblock)
1603 snapshot%packed(pos:pos + m - 1) = snapshot%packed(pos:pos + m - 1) + &
1604 received(route%first(iblock):route%first(iblock + 1) - 1)
1605 IF (odd == 0) cycle
1606 sense = real(sign(1, route%transform%fold_map(2, iblock)), dp)
1607 snapshot%packed(odd:odd + m - 1) = snapshot%packed(odd:odd + m - 1) + &
1608 sense*received(route%first(iblock):route%first(iblock + 1) - 1)
1609 END DO
1610 END block
1611 END IF
1612 IF (route%transform%backend == rskp_backend_gemm) THEN
1613 ALLOCATE (snapshot%active(4, SIZE(route%transform%phase_count, 2)))
1614 DO g = 1, SIZE(snapshot%active, 2)
1615 m = route%element_first(g + 1) - route%element_first(g)
1616 panels = route%transform%phase_count([1, 1, 2, 3], g)
1617 pos = route%transform%packed_first(g)
1618 DO i = 1, 4
1619 snapshot%active(i, g) = any(snapshot%packed(pos:pos + m*panels(i) - 1) /= 0.0_dp)
1620 pos = pos + m*panels(i)
1621 END DO
1622 END DO
1623 END IF
1624 CALL timestop(handle)
1625
1626 END SUBROUTINE rskp_capture
1627
1628! **************************************************************************************************
1629!> \brief Release one captured row without invalidating other slots or routing.
1630!> \param snapshot row to clear
1631! **************************************************************************************************
1632 SUBROUTINE rskp_values_release(snapshot)
1633
1634 TYPE(rskp_values_type), INTENT(INOUT) :: snapshot
1635
1636 IF (ALLOCATED(snapshot%packed)) DEALLOCATE (snapshot%packed)
1637 IF (ALLOCATED(snapshot%active)) DEALLOCATE (snapshot%active)
1638 IF (ALLOCATED(snapshot%fft_values)) DEALLOCATE (snapshot%fft_values)
1639 NULLIFY (snapshot%source)
1640 snapshot%matrix_row = 0
1641 snapshot%mirror_sign = 1
1642 snapshot%scale = z_one
1643
1644 END SUBROUTINE rskp_values_release
1645
1646! **************************************************************************************************
1647!> \brief Release backend-owned geometry and FFT work buffers from a route.
1648!> \param route route whose backend plan is no longer needed
1649! **************************************************************************************************
1650 SUBROUTINE rskp_route_release(route)
1651 TYPE(rskp_route_type), INTENT(INOUT) :: route
1652
1653 CALL cell_to_k_grid_release(route%transform%fft_work)
1654 IF (ASSOCIATED(route%matrix_struct)) CALL cp_fm_struct_release(route%matrix_struct)
1655 route = rskp_route_type()
1656 END SUBROUTINE rskp_route_release
1657
1658! **************************************************************************************************
1659!> \brief FFT captured raw-element panels and retain the destination group's k points.
1660!> Reuse the sparse route and the dense layout consumed by the batched solvers.
1661!> \param snapshot immutable packed raw-image panels
1662!> \param route fixed element/image layout and equivalent target tiles across groups
1663!> \param xkp full reciprocal grid
1664!> \param nkp_grid reciprocal-grid dimensions
1665!> \param kp_dist global k-point range of each group
1666!> \param inter communicator joining equivalent FM ranks across groups
1667!> \param max_bytes soft workspace target, excluding retained inputs and destinations
1668!> \param cache optional unscaled canonical elements by local k point
1669!> \param output optional existing split-FM destinations, instead of cache
1670! **************************************************************************************************
1671 SUBROUTINE cell_to_k_sparse_fft(snapshot, route, xkp, nkp_grid, kp_dist, inter, max_bytes, cache, output)
1672 TYPE(rskp_values_type), INTENT(IN), TARGET :: snapshot
1673 TYPE(rskp_route_type), INTENT(INOUT) :: route
1674 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
1675 INTEGER, INTENT(IN) :: nkp_grid(3), kp_dist(:, :)
1676 TYPE(mp_para_env_type), INTENT(IN) :: inter
1677 INTEGER(KIND=int_8), INTENT(IN) :: max_bytes
1678 COMPLEX(KIND=dp), CONTIGUOUS, INTENT(OUT), &
1679 OPTIONAL :: cache(:, :)
1680 TYPE(cp_fm_p_type), INTENT(IN), OPTIONAL :: output(:, :)
1681
1682 COMPLEX(KIND=dp), ALLOCATABLE, TARGET :: received(:), transformed(:, :, :)
1683 COMPLEX(KIND=dp), CONTIGUOUS, POINTER :: canonical(:, :), send(:)
1684 INTEGER :: element, first, group, i, igroup, ik, &
1685 last, my_peer, nelem, nimages, nk, &
1686 nlocal, npeers, ntile, nwave, offset, &
1687 start, width
1688 INTEGER(KIND=int_8) :: available, fixed_bytes, ngrid, &
1689 per_element, workspace_bytes
1690 INTEGER, ALLOCATABLE :: ranges(:, :), rcount(:), rdisp(:), &
1691 scount(:), sdisp(:)
1692 REAL(kind=dp), CONTIGUOUS, POINTER :: values(:, :, :)
1693
1694 cpassert(route%transform%backend == rskp_backend_fft)
1695 cpassert(PRESENT(cache) .NEQV. PRESENT(output))
1696 cpassert(route%transform%fft_work%regular)
1697 ngrid = product(int(route%transform%fft_work%nfft, int_8))
1698 nk = SIZE(xkp, 2)
1699 IF (PRESENT(cache)) THEN
1700 cpassert(SIZE(cache, 1) == route%element_first(SIZE(route%element_first)) - 1)
1701 nlocal = SIZE(cache, 2)
1702 cache = z_zero
1703 ELSE
1704 nlocal = SIZE(output, 1)
1705 DO ik = 1, nlocal
1706 DO i = 1, SIZE(output, 2)
1707 output(ik, i)%matrix%local_data(:, :) = 0.0_dp
1708 END DO
1709 END DO
1710 END IF
1711 npeers = 1
1712 my_peer = 0
1713 IF (.NOT. route%world_target) THEN
1714 npeers = inter%num_pe
1715 my_peer = inter%mepos
1716 END IF
1717 ALLOCATE (ranges(2, npeers))
1718 ranges(:, 1) = [1, nk]
1719 IF (.NOT. route%world_target) ranges(:, :) = kp_dist
1720 cpassert(nlocal == ranges(2, my_peer + 1) - ranges(1, my_peer + 1) + 1)
1721 available = max_bytes
1722 IF (.NOT. route%world_target) CALL inter%min(available)
1723 ALLOCATE (scount(npeers), sdisp(npeers), rcount(npeers), rdisp(npeers))
1724 DO igroup = 1, SIZE(route%block_first) - 1
1725 first = route%block_first(igroup)
1726 last = route%block_first(igroup + 1) - 1
1727 nelem = route%element_first(igroup + 1) - route%element_first(igroup)
1728 nimages = last - first + 1
1729 values(1:nelem, 1:1, 1:nimages) => &
1730 snapshot%packed(route%transform%packed_first(igroup):route%transform%packed_first(igroup + 1) - 1)
1731 IF (.NOT. any(values /= 0.0_dp)) cycle
1732 ! Each group transforms different AO elements; exchange only the owned k ranges.
1733 fixed_bytes = 16_int_8*nk + 28_int_8*SIZE(route%entry, 2) + 16_int_8*npeers
1734 per_element = 16_int_8*nk + 48_int_8*ngrid + &
1735 16_int_8*npeers*maxval(ranges(2, :) - ranges(1, :) + 1)
1736 width = int(max(0_int_8, min((int(nelem, int_8) + int(npeers, int_8) - 1_int_8)/int(npeers, int_8), &
1737 max(1_int_8, fft_batch_bytes/(48_int_8*ngrid)), &
1738 max(1_int_8, (available - fixed_bytes)/per_element), int(huge(0), int_8)/nk, &
1739 int(huge(0), int_8)/(int(npeers, int_8)* &
1740 maxval(ranges(2, :) - ranges(1, :) + 1)))))
1741 IF (width == 0) THEN
1742 cpabort("Lattice FFT tile exceeds the integer MPI count range")
1743 END IF
1744 DO element = 1, nelem, width*npeers
1745 start = element + my_peer*width
1746 ntile = max(0, min(width, nelem - start + 1))
1747 nwave = min(width*npeers, nelem - element + 1)
1748 ALLOCATE (transformed(ntile, 1, nk), received(nwave*nlocal))
1749 IF (ntile > 0) THEN
1750 workspace_bytes = available - fixed_bytes - 16_int_8*(SIZE(transformed, kind=int_8) + &
1751 SIZE(received, kind=int_8))
1752 CALL cell_to_k_grid_fft(values(start:start + ntile - 1, :, :), &
1753 route%entry(5:7, route%block_order(first:last)), &
1754 xkp, nkp_grid, transformed, &
1755 require_fft=.true., work=route%transform%fft_work, cell_first=first, &
1756 max_workspace_bytes=workspace_bytes)
1757 END IF
1758 send(1:SIZE(transformed)) => transformed
1759 scount(:) = ntile*(ranges(2, :) - ranges(1, :) + 1)
1760 sdisp(:) = ntile*(ranges(1, :) - 1)
1761 offset = 0
1762 DO group = 1, npeers
1763 rcount(group) = max(0, min(width, nelem - element - (group - 1)*width + 1))*nlocal
1764 rdisp(group) = offset
1765 offset = offset + rcount(group)
1766 END DO
1767 IF (npeers > 1) THEN
1768 CALL inter%alltoall(send, scount, sdisp, received, rcount, rdisp)
1769 ELSE
1770 received(:) = send
1771 END IF
1772 DO group = 1, npeers
1773 ntile = rcount(group)/max(1, nlocal)
1774 IF (ntile == 0) cycle
1775 start = route%element_first(igroup) + element + (group - 1)*width - 1
1776 canonical(1:ntile, 1:nlocal) => received(rdisp(group) + 1:rdisp(group) + rcount(group))
1777 IF (PRESENT(cache)) THEN
1778 cache(start:start + ntile - 1, :) = canonical
1779 ELSE
1780 CALL cell_to_k_scatter(canonical, start, route, snapshot%mirror_sign, snapshot%scale, output=output)
1781 END IF
1782 END DO
1783 DEALLOCATE (transformed, received)
1784 END DO
1785 END DO
1786 END SUBROUTINE cell_to_k_sparse_fft
1787
1788! **************************************************************************************************
1789!> \brief Scatter unscaled canonical element tiles into caller-owned dense destinations.
1790!> The caller clears the destinations once. Mirror parity applies before complex scaling.
1791!> \param values canonical elements by local k point
1792!> \param first_element first canonical element in the route
1793!> \param route immutable destination offsets
1794!> \param mirror_sign transpose parity
1795!> \param scale complex output prefactor
1796!> \param batch optional complex matrix batch
1797!> \param output optional split-FM destinations instead of batch
1798! **************************************************************************************************
1799 SUBROUTINE cell_to_k_scatter(values, first_element, route, mirror_sign, scale, batch, output)
1800 COMPLEX(KIND=dp), INTENT(IN) :: values(:, :)
1801 INTEGER, INTENT(IN) :: first_element
1802 TYPE(rskp_route_type), INTENT(IN) :: route
1803 INTEGER, INTENT(IN) :: mirror_sign
1804 COMPLEX(KIND=dp), INTENT(IN) :: scale
1805 COMPLEX(KIND=dp), CONTIGUOUS, INTENT(INOUT), &
1806 OPTIONAL, TARGET :: batch(:, :, :)
1807 TYPE(cp_fm_p_type), INTENT(IN), OPTIONAL :: output(:, :)
1808
1809 COMPLEX(KIND=dp) :: value
1810 COMPLEX(KIND=dp), CONTIGUOUS, POINTER :: dense(:, :)
1811 INTEGER :: element, i, ic, k
1812 INTEGER(KIND=int_8) :: dest
1813 REAL(kind=dp), CONTIGUOUS, POINTER :: real_values(:)
1814
1815 cpassert(PRESENT(batch) .NEQV. PRESENT(output))
1816 IF (PRESENT(batch)) THEN
1817 cpassert(SIZE(batch, 3) == SIZE(values, 2))
1818 dense(1:SIZE(batch, 1, kind=int_8)*SIZE(batch, 2, kind=int_8), 1:SIZE(batch, 3)) => batch
1819 DO k = 1, SIZE(values, 2)
1820!$OMP SIMD PRIVATE(element,dest,value)
1821 DO i = 1, SIZE(values, 1)
1822 element = first_element + i - 1
1823 dest = route%element_offset(1, element)
1824 IF (dest > 0) dense(dest, k) = scale*values(i, k)
1825 dest = route%element_offset(2, element)
1826 value = real(mirror_sign, dp)*conjg(values(i, k))
1827 IF (dest > 0) dense(dest, k) = scale*value
1828 END DO
1829 END DO
1830 ELSE
1831 cpassert(SIZE(output, 1) == SIZE(values, 2))
1832 cpassert(SIZE(output, 2) >= 1 .AND. SIZE(output, 2) <= 2)
1833 DO k = 1, SIZE(values, 2)
1834 DO ic = 1, SIZE(output, 2)
1835 real_values(1:SIZE(output(k, ic)%matrix%local_data, kind=int_8)) => output(k, ic)%matrix%local_data
1836!$OMP SIMD PRIVATE(element,dest,value)
1837 DO i = 1, SIZE(values, 1)
1838 element = first_element + i - 1
1839 dest = route%element_offset(1, element)
1840 value = scale*values(i, k)
1841 IF (dest > 0) real_values(dest) = merge(real(value, dp), aimag(value), ic == 1)
1842 dest = route%element_offset(2, element)
1843 value = scale*(real(mirror_sign, dp)*conjg(values(i, k)))
1844 IF (dest > 0) real_values(dest) = merge(real(value, dp), aimag(value), ic == 1)
1845 END DO
1846 END DO
1847 END DO
1848 END IF
1849 END SUBROUTINE cell_to_k_scatter
1850
1851! **************************************************************************************************
1852!> \brief Fix parity panels per canonical atom pair and cache their shared phases.
1853!> Reserve one complex worker and caller-specified retained output storage.
1854!> Oversized tables use the same contraction kernel with tile-local phases.
1855!> \param plan geometry-only dictionary and optional immutable phase table
1856!> \param translations signed image translations
1857!> \param group_first boundaries of canonical AO groups in order
1858!> \param order image permutation grouped by canonical AO pair
1859!> \param element_first cumulative row counts of the packed group panels
1860!> \param xkp fixed group-local reciprocal coordinates
1861!> \param max_bytes shared-phase and worker/output-workspace budget
1862!> \param output_bytes workspace reserved for retained output tiles
1863! **************************************************************************************************
1864 SUBROUTINE cell_to_k_phase_prepare(plan, translations, group_first, order, element_first, xkp, max_bytes, output_bytes)
1865 TYPE(rskp_backend_plan_type), INTENT(INOUT) :: plan
1866 INTEGER, INTENT(IN) :: translations(:, :), group_first(:), &
1867 order(:), element_first(:)
1868 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
1869 INTEGER(KIND=int_8), INTENT(IN) :: max_bytes, output_bytes
1870
1871 COMPLEX(KIND=dp) :: phase
1872 INTEGER :: c, first, g, i, iblock, ik, j, m, n, &
1873 nfold, nk, npair, nr, nsingle, pos
1874 INTEGER(KIND=int_8) :: cache_bytes, minimum_bytes
1875 INTEGER, ALLOCATABLE :: cells(:, :), classes(:), &
1876 global_class(:), local_class(:), &
1877 slots(:)
1878 LOGICAL, ALLOCATABLE :: members(:, :)
1879 REAL(kind=dp) :: arg
1880
1881 CALL rskp_image_classes(translations, classes, plan%phase_cells)
1882 n = SIZE(group_first) - 1
1883 ALLOCATE (plan%phase_count(3, n), plan%phase_first(n + 1), plan%packed_first(n + 1), &
1884 plan%phase_index(SIZE(classes)), plan%fold_map(2, SIZE(classes)))
1885 plan%fold_map = 0
1886 plan%phase_first(1) = 1
1887 plan%packed_first(1) = 1
1888 plan%max_per_k = 0_int_8
1889 DO g = 1, n
1890 first = group_first(g)
1891 CALL rskp_image_classes(translations(:, order(first:group_first(g + 1) - 1)), &
1892 local_class, cells)
1893 ALLOCATE (members(2, SIZE(cells, 2)), slots(SIZE(cells, 2)), global_class(SIZE(cells, 2)))
1894 members = .false.
1895 DO i = 1, SIZE(local_class)
1896 c = local_class(i)
1897 IF (c == 0) cycle
1898 members(merge(1, 2, c > 0), abs(c)) = .true.
1899 iblock = order(first + i - 1)
1900 global_class(abs(c)) = abs(classes(iblock))
1901 END DO
1902 npair = count(all(members, dim=1))
1903 nsingle = SIZE(cells, 2) - npair
1904 plan%phase_count(:, g) = [npair, nsingle, merge(1, 0, any(local_class == 0))]
1905 i = 0
1906 j = 0
1907 DO c = 1, SIZE(cells, 2)
1908 IF (all(members(:, c))) THEN
1909 i = i + 1
1910 slots(c) = i
1911 plan%phase_index(plan%phase_first(g) + i - 1) = global_class(c)
1912 ELSE
1913 j = j + 1
1914 slots(c) = 2*npair + j
1915 plan%phase_index(plan%phase_first(g) + npair + j - 1) = merge(1, -1, members(1, c))*global_class(c)
1916 END IF
1917 END DO
1918 m = element_first(g + 1) - element_first(g)
1919 nfold = 2*npair + nsingle + plan%phase_count(3, g)
1920 plan%phase_first(g + 1) = plan%phase_first(g) + npair + nsingle
1921 IF (int(plan%packed_first(g), int_8) + int(m, int_8)*nfold > huge(m)) THEN
1922 cpabort("R-to-K folded panel offsets exceed the integer range")
1923 END IF
1924 plan%packed_first(g + 1) = plan%packed_first(g) + m*nfold
1925 plan%max_per_k = max(plan%max_per_k, int(m, int_8) + max(npair, nsingle))
1926 DO i = 1, SIZE(local_class)
1927 iblock = order(first + i - 1)
1928 c = local_class(i)
1929 pos = nfold
1930 IF (c /= 0) pos = slots(abs(c))
1931 plan%fold_map(1, iblock) = plan%packed_first(g) + (pos - 1)*m
1932 IF (c == 0) cycle
1933 IF (all(members(:, abs(c)))) THEN
1934 plan%fold_map(2, iblock) = sign(plan%packed_first(g) + (npair + pos - 1)*m, c)
1935 END IF
1936 END DO
1937 DEALLOCATE (members, slots, global_class)
1938 END DO
1939
1940 nr = SIZE(plan%phase_cells, 2)
1941 nk = SIZE(xkp, 2)
1942 cache_bytes = 16_int_8*nr*nk
1943 minimum_bytes = 16_int_8*plan%max_per_k + output_bytes
1944 IF (cache_bytes + minimum_bytes > max_bytes) RETURN
1945 ALLOCATE (plan%phases(nr, nk, 2))
1946 DO ik = 1, nk
1947!$OMP SIMD PRIVATE(arg,phase)
1948 DO i = 1, nr
1949 arg = twopi*dot_product(real(plan%phase_cells(:, i), dp), xkp(:, ik))
1950 phase = exp(cmplx(0.0_dp, arg, kind=dp))
1951 plan%phases(i, ik, 1) = real(phase, dp)
1952 plan%phases(i, ik, 2) = aimag(phase)
1953 END DO
1954 END DO
1955 END SUBROUTINE cell_to_k_phase_prepare
1956
1957! **************************************************************************************************
1958!> \brief Prepare the selected R-to-K backend plan once for a fixed route.
1959!> Both backends retain their geometry with the route; the snapshots then
1960!> provide only values and use the common evaluate interface.
1961!> \param route fixed sparse AO route and backend workspace
1962!> \param xkp reciprocal coordinates for the route's k-point domain
1963!> \param nkp_grid regular-grid dimensions required by the FFT backend
1964!> \param max_bytes phase-cache budget for the GEMM backend
1965! **************************************************************************************************
1966 SUBROUTINE cell_to_k_backend_prepare(route, xkp, nkp_grid, max_bytes)
1967 TYPE(rskp_route_type), INTENT(INOUT) :: route
1968 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
1969 INTEGER, INTENT(IN) :: nkp_grid(3)
1970 INTEGER(KIND=int_8), INTENT(IN) :: max_bytes
1971
1972 SELECT CASE (route%transform%backend)
1973 CASE (rskp_backend_gemm)
1974 CALL cell_to_k_phase_prepare(route%transform, route%entry(5:7, :), route%block_first, route%block_order, &
1975 route%element_first, xkp, max_bytes, &
1976 32_int_8*route%matrix_struct%local_leading_dimension*max(1, route%ncol))
1977 CASE (rskp_backend_fft)
1978 CALL cell_to_k_grid_prepare(route%transform%fft_work, route%entry(5:7, route%block_order), xkp, nkp_grid)
1979 IF (.NOT. route%transform%fft_work%regular) THEN
1980 cpabort("Forced lattice FFT needs a supported regular grid or subset")
1981 END IF
1982 CASE DEFAULT
1983 cpabort("Unknown R-to-K backend")
1984 END SELECT
1985 END SUBROUTINE cell_to_k_backend_prepare
1986
1987! **************************************************************************************************
1988!> \brief Contract parity-folded pairs and concatenate phases for unpaired translations.
1989!> \param packed immutable parity panels of one AO group
1990!> \param plan phase dictionary shared by the FM and DBCSR adapters
1991!> \param igroup canonical atom-pair group
1992!> \param xkp k coordinates for this tile
1993!> \param first_k first index in the route-local phase table
1994!> \param factors cosine columns followed by optional sine columns
1995!> \param RESULT real columns followed by optional imaginary columns
1996!> \param active nonzero even, odd, single and origin panels, determined at capture
1997! **************************************************************************************************
1998 SUBROUTINE cell_to_k_block_gemm(packed, plan, igroup, xkp, first_k, factors, RESULT, active)
1999 REAL(kind=dp), CONTIGUOUS, INTENT(IN), TARGET :: packed(:)
2000 TYPE(rskp_backend_plan_type), INTENT(IN) :: plan
2001 INTEGER, INTENT(IN) :: igroup
2002 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
2003 INTEGER, INTENT(IN) :: first_k
2004 REAL(kind=dp), CONTIGUOUS, INTENT(OUT) :: factors(:, :), result(:, :)
2005 LOGICAL, INTENT(IN) :: active(4)
2006
2007 INTEGER :: first, i, ik, nactive, nelem, nk, npair, &
2008 nsingle, pass, phase, start
2009 LOGICAL :: use_cosine, use_sine
2010 REAL(kind=dp) :: arg, beta(2)
2011 REAL(kind=dp), CONTIGUOUS, POINTER :: values(:, :), values_im(:, :)
2012
2013 nelem = SIZE(result, 1)
2014 nk = SIZE(xkp, 2)
2015 cpassert(SIZE(result, 2) == nk .OR. SIZE(result, 2) == 2*nk)
2016 npair = plan%phase_count(1, igroup)
2017 nsingle = plan%phase_count(2, igroup)
2018 first = plan%phase_first(igroup)
2019 start = 1
2020 IF (ALLOCATED(plan%phases)) THEN
2021 cpassert(first_k >= 1 .AND. first_k + nk - 1 <= SIZE(plan%phases, 2))
2022 END IF
2023 beta = 0.0_dp
2024 DO pass = 1, 2
2025 nactive = merge(npair, nsingle, pass == 1)
2026 IF (nactive == 0) cycle
2027 use_cosine = merge(active(1), active(3), pass == 1)
2028 use_sine = SIZE(result, 2) == 2*nk .AND. merge(active(2), active(3), pass == 1)
2029 IF (.NOT. (use_cosine .OR. use_sine)) THEN
2030 start = start + nelem*nactive*merge(2, 1, pass == 1)
2031 first = first + nactive
2032 cycle
2033 END IF
2034 DO ik = 1, nk
2035 DO i = 1, nactive
2036 phase = plan%phase_index(first + i - 1)
2037 IF (ALLOCATED(plan%phases)) THEN
2038 IF (use_cosine) factors(i, ik) = plan%phases(abs(phase), first_k + ik - 1, 1)
2039 IF (use_sine) THEN
2040 factors(i, nk + ik) = real(sign(1, phase), dp)*plan%phases(abs(phase), first_k + ik - 1, 2)
2041 END IF
2042 ELSE
2043 arg = twopi*dot_product(real(plan%phase_cells(:, abs(phase)), dp), xkp(:, ik))
2044 IF (use_cosine) factors(i, ik) = cos(arg)
2045 IF (use_sine) factors(i, nk + ik) = real(sign(1, phase), dp)*sin(arg)
2046 END IF
2047 END DO
2048 END DO
2049 values(1:nelem, 1:nactive) => packed(start:start + nelem*nactive - 1)
2050 IF (pass == 1) THEN
2051 start = start + nelem*npair
2052 IF (active(1)) THEN
2053 CALL rskp_gemm_contract(nelem, npair, nk, values, factors(:, 1:nk), result(:, 1:nk), 0.0_dp)
2054 beta(1) = 1.0_dp
2055 END IF
2056 IF (SIZE(result, 2) == 2*nk .AND. active(2)) THEN
2057 values_im(1:nelem, 1:npair) => packed(start:start + nelem*npair - 1)
2058 CALL rskp_gemm_contract(nelem, npair, nk, values_im, factors(:, nk + 1:2*nk), &
2059 result(:, nk + 1:2*nk), 0.0_dp)
2060 beta(2) = 1.0_dp
2061 END IF
2062 start = start + nelem*npair
2063 ELSE
2064 IF (SIZE(result, 2) == 2*nk .AND. beta(1) == beta(2)) THEN
2065 CALL rskp_gemm_contract(nelem, nsingle, 2*nk, values, factors, result, beta(1))
2066 ELSE
2067 CALL rskp_gemm_contract(nelem, nsingle, nk, values, factors(:, 1:nk), result(:, 1:nk), beta(1))
2068 IF (SIZE(result, 2) == 2*nk) THEN
2069 CALL rskp_gemm_contract(nelem, nsingle, nk, values, factors(:, nk + 1:2*nk), &
2070 result(:, nk + 1:2*nk), beta(2))
2071 END IF
2072 END IF
2073 beta = 1.0_dp
2074 start = start + nelem*nsingle
2075 END IF
2076 first = first + nactive
2077 END DO
2078 IF (beta(1) == 0.0_dp) result(:, 1:nk) = 0.0_dp
2079 IF (SIZE(result, 2) == 2*nk .AND. beta(2) == 0.0_dp) result(:, nk + 1:2*nk) = 0.0_dp
2080 IF (active(4)) THEN
2081 DO ik = 1, nk
2082 result(:, ik) = result(:, ik) + packed(start:start + nelem - 1)
2083 END DO
2084 END IF
2085 END SUBROUTINE cell_to_k_block_gemm
2086
2087! **************************************************************************************************
2088!> \brief Bound worker scratch and output tiles for either prepared backend.
2089!> Packed input belongs to the snapshot and is shared by every k tile.
2090!> \param route borrowed local AO-element layout
2091!> \param nk local k-point count
2092!> \param ncomp contraction components for split-FM outputs (two for a complex source scale)
2093!> \param nworkers requested workers, reduced to fit the workspace
2094!> \param nmatrices retained complex operator tiles; zero writes existing split-FM outputs
2095!> \param max_bytes shared-phase, worker-scratch and output-tile budget
2096!> \param width k points per worker
2097!> \param work_size real scratch elements per worker
2098! **************************************************************************************************
2099 SUBROUTINE rskp_batch_size(route, nk, ncomp, nworkers, nmatrices, max_bytes, width, work_size)
2100 TYPE(rskp_route_type), INTENT(IN) :: route
2101 INTEGER, INTENT(IN) :: nk, ncomp
2102 INTEGER, INTENT(INOUT) :: nworkers
2103 INTEGER, INTENT(IN) :: nmatrices
2104 INTEGER(KIND=int_8), INTENT(IN) :: max_bytes
2105 INTEGER, INTENT(OUT) :: width
2106 INTEGER(KIND=int_8), INTENT(OUT) :: work_size
2107
2108 INTEGER :: nc
2109 INTEGER(KIND=int_8) :: available_bytes, output_bytes, &
2110 per_k_bytes
2111
2112 cpassert(nworkers > 0 .AND. nk > 0 .AND. nmatrices >= 0)
2113 available_bytes = max_bytes
2114 IF (ALLOCATED(route%transform%phases)) THEN
2115 available_bytes = available_bytes - 8_int_8*SIZE(route%transform%phases, kind=int_8)
2116 END IF
2117 output_bytes = 16_int_8*nmatrices*route%matrix_struct%local_leading_dimension*max(1, route%ncol)
2118 nc = 2
2119 IF (nmatrices == 0) nc = ncomp
2120 per_k_bytes = max(1_int_8, 8_int_8*nc*route%transform%max_per_k + output_bytes)
2121 nworkers = min(nworkers, nk)
2122 nworkers = min(nworkers, int(max(1_int_8, available_bytes/per_k_bytes)))
2123 width = int(min(int(nk, int_8), max(1_int_8, available_bytes/(nworkers*per_k_bytes))))
2124 width = min(width, (nk + nworkers - 1)/nworkers)
2125 work_size = int(nc, int_8)*width*route%transform%max_per_k
2126 END SUBROUTINE rskp_batch_size
2127
2128! **************************************************************************************************
2129!> \brief Assemble a local complex operator tile directly from packed sparse inputs.
2130!> \param snapshot immutable BLAS-layout input values
2131!> \param route borrowed local AO-element layout
2132!> \param xkp coordinates for this k tile
2133!> \param first_k first index in the route-local phase table
2134!> \param work caller-owned real phase and result scratch
2135!> \param batch optional complex output (row, column, k point), including padding
2136!> \param output optional existing split-FM destinations (k point, component)
2137! **************************************************************************************************
2138 SUBROUTINE cell_to_k_gemm_batch(snapshot, route, xkp, first_k, work, batch, output)
2139 TYPE(rskp_values_type), INTENT(IN), TARGET :: snapshot
2140 TYPE(rskp_route_type), INTENT(IN) :: route
2141 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
2142 INTEGER, INTENT(IN) :: first_k
2143 REAL(kind=dp), CONTIGUOUS, INTENT(INOUT), TARGET :: work(:)
2144 COMPLEX(KIND=dp), CONTIGUOUS, INTENT(OUT), &
2145 OPTIONAL, TARGET :: batch(:, :, :)
2146 TYPE(cp_fm_p_type), INTENT(IN), OPTIONAL :: output(:, :)
2147
2148 COMPLEX(KIND=dp) :: value
2149 COMPLEX(KIND=dp), CONTIGUOUS, POINTER :: complex_values(:, :)
2150 INTEGER :: element, first, ic, igroup, j, ld, nc, &
2151 nelem, nphase, width
2152 INTEGER(KIND=int_8) :: dest, nfactor
2153 REAL(kind=dp), CONTIGUOUS, POINTER :: factors(:, :), real_values(:), &
2154 result(:, :)
2155
2156 cpassert(route%transform%backend == rskp_backend_gemm)
2157 cpassert(PRESENT(batch) .NEQV. PRESENT(output))
2158 width = SIZE(xkp, 2)
2159 ld = route%matrix_struct%local_leading_dimension
2160 nc = 2
2161 IF (PRESENT(batch)) THEN
2162 cpassert(SIZE(batch, 1) == route%matrix_struct%local_leading_dimension)
2163 cpassert(SIZE(batch, 2) >= route%ncol .AND. SIZE(batch, 3) == width)
2164 batch = z_zero
2165 complex_values(1:SIZE(batch, 1, kind=int_8)*SIZE(batch, 2), 1:width) => batch
2166 ELSE
2167 nc = SIZE(output, 2)
2168 cpassert(SIZE(output, 1) == width .AND. (nc == 1 .OR. nc == 2))
2169 DO j = 1, width
2170 DO ic = 1, nc
2171 cpassert(SIZE(output(j, ic)%matrix%local_data, 1) == ld)
2172 output(j, ic)%matrix%local_data = 0.0_dp
2173 END DO
2174 END DO
2175 IF (aimag(snapshot%scale) /= 0.0_dp) nc = 2
2176 END IF
2177 DO igroup = 1, SIZE(route%transform%phase_count, 2)
2178 first = route%element_first(igroup)
2179 nelem = route%element_first(igroup + 1) - first
2180 IF (nelem == 0) cycle
2181 IF (.NOT. any(snapshot%active(:, igroup))) cycle
2182 nphase = maxval(route%transform%phase_count(1:2, igroup))
2183 nfactor = int(nc, int_8)*nphase*width
2184 cpassert(SIZE(work, kind=int_8) >= nfactor + int(nc, int_8)*nelem*width)
2185 factors(1:nphase, 1:nc*width) => work(1:nfactor)
2186 result(1:nelem, 1:nc*width) => work(nfactor + 1:nfactor + int(nc, int_8)*nelem*width)
2187 CALL cell_to_k_block_gemm(snapshot%packed(route%transform%packed_first(igroup): &
2188 route%transform%packed_first(igroup + 1) - 1), &
2189 route%transform, igroup, xkp, first_k, factors, result, snapshot%active(:, igroup))
2190 DO j = 1, width
2191 IF (PRESENT(batch)) THEN
2192!$OMP SIMD PRIVATE(dest,value)
2193 DO element = first, route%element_first(igroup + 1) - 1
2194 value = cmplx(result(element - first + 1, j), result(element - first + 1, width + j), kind=dp)
2195 dest = route%element_offset(1, element)
2196 IF (dest > 0) complex_values(dest, j) = snapshot%scale*value
2197 dest = route%element_offset(2, element)
2198 IF (dest > 0) complex_values(dest, j) = snapshot%scale*real(snapshot%mirror_sign, dp)*conjg(value)
2199 END DO
2200 ELSE
2201 DO ic = 1, SIZE(output, 2)
2202 real_values(1:SIZE(output(j, ic)%matrix%local_data, kind=int_8)) => output(j, ic)%matrix%local_data
2203!$OMP SIMD PRIVATE(dest,value)
2204 DO element = first, route%element_first(igroup + 1) - 1
2205 dest = route%element_offset(1, element)
2206 IF (aimag(snapshot%scale) == 0.0_dp) THEN
2207 IF (dest > 0) real_values(dest) = real(snapshot%scale, dp)* &
2208 result(element - first + 1, j + (ic - 1)*width)
2209 ELSE
2210 value = snapshot%scale*cmplx(result(element - first + 1, j), &
2211 result(element - first + 1, width + j), dp)
2212 IF (dest > 0) real_values(dest) = merge(real(value, dp), aimag(value), ic == 1)
2213 END IF
2214 dest = route%element_offset(2, element)
2215 IF (aimag(snapshot%scale) == 0.0_dp) THEN
2216 IF (dest > 0) real_values(dest) = real(snapshot%scale, dp)*real(snapshot%mirror_sign, dp)* &
2217 merge(1.0_dp, -1.0_dp, ic == 1)* &
2218 result(element - first + 1, j + (ic - 1)*width)
2219 ELSE
2220 value = snapshot%scale*real(snapshot%mirror_sign, dp)* &
2221 cmplx(result(element - first + 1, j), -result(element - first + 1, width + j), dp)
2222 IF (dest > 0) real_values(dest) = merge(real(value, dp), aimag(value), ic == 1)
2223 END IF
2224 END DO
2225 END DO
2226 END IF
2227 END DO
2228 END DO
2229 END SUBROUTINE cell_to_k_gemm_batch
2230
2231! **************************************************************************************************
2232!> \brief Evaluate a k-point tile through the selected prepared R-to-K backend.
2233!> GEMM contracts the retained real-space panel; FFT returns a slice of
2234!> the canonical reciprocal-space snapshot materialized during capture.
2235!> \param snapshot backend input or prepared reciprocal-space values
2236!> \param route fixed sparse layout and backend plan
2237!> \param xkp reciprocal coordinates for this k tile
2238!> \param first_local first k index in the route-local domain
2239!> \param work caller-owned GEMM scratch; unused by the FFT backend
2240!> \param batch complex output tile (row, column, k point)
2241!> \param output optional split-FM output tile (k point, component), instead of batch
2242! **************************************************************************************************
2243 SUBROUTINE cell_to_k_backend_evaluate_batch(snapshot, route, xkp, first_local, work, batch, output)
2244 TYPE(rskp_values_type), INTENT(IN), TARGET :: snapshot
2245 TYPE(rskp_route_type), INTENT(IN) :: route
2246 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
2247 INTEGER, INTENT(IN) :: first_local
2248 REAL(kind=dp), CONTIGUOUS, INTENT(INOUT), TARGET :: work(:)
2249 COMPLEX(KIND=dp), CONTIGUOUS, INTENT(OUT), &
2250 OPTIONAL :: batch(:, :, :)
2251 TYPE(cp_fm_p_type), INTENT(IN), OPTIONAL :: output(:, :)
2252
2253 INTEGER :: ic, j, width
2254 LOGICAL :: compatible
2255
2256 cpassert(PRESENT(batch) .NEQV. PRESENT(output))
2257 width = SIZE(xkp, 2)
2258 IF (PRESENT(output)) THEN
2259 cpassert(SIZE(output, 1) == width)
2260 cpassert(SIZE(output, 2) >= 1 .AND. SIZE(output, 2) <= 2)
2261 DO j = 1, width
2262 DO ic = 1, SIZE(output, 2)
2263 compatible = cp_fm_struct_equivalent(output(j, ic)%matrix%matrix_struct, route%matrix_struct)
2264 cpassert(compatible)
2265 END DO
2266 END DO
2267 END IF
2268 SELECT CASE (route%transform%backend)
2269 CASE (rskp_backend_gemm)
2270 CALL cell_to_k_gemm_batch(snapshot, route, xkp, first_local, work, batch=batch, output=output)
2271 CASE (rskp_backend_fft)
2272 cpassert(ALLOCATED(snapshot%fft_values))
2273 cpassert(first_local >= 1)
2274 cpassert(first_local + width - 1 <= SIZE(snapshot%fft_values, 2))
2275 IF (PRESENT(batch)) THEN
2276 cpassert(SIZE(batch, 1) == route%matrix_struct%local_leading_dimension)
2277 cpassert(SIZE(batch, 2) >= route%ncol)
2278 cpassert(SIZE(batch, 3) == width)
2279 batch = z_zero
2280 ELSE
2281 DO j = 1, width
2282 DO ic = 1, SIZE(output, 2)
2283 output(j, ic)%matrix%local_data(:, :) = 0.0_dp
2284 END DO
2285 END DO
2286 END IF
2287 CALL cell_to_k_scatter(snapshot%fft_values(:, first_local:first_local + width - 1), 1, route, &
2288 snapshot%mirror_sign, snapshot%scale, batch=batch, output=output)
2289 CASE DEFAULT
2290 cpabort("Unknown R-to-K backend")
2291 END SELECT
2293
2294! **************************************************************************************************
2295!> \brief Tabulate distinct absolute translation coordinates for a group's local k points.
2296!> All AO pairs share these unweighted phases. Zero coordinates are exact;
2297!> negative coordinates use the same cosine and the opposite sine.
2298!> \param table owned axis-coordinate dictionary and phases
2299!> \param cells globally available image translations
2300!> \param xkp this symmetry group's local contribution coordinates
2301!> \param nworkers maximum number of OpenMP workers
2302! **************************************************************************************************
2303 SUBROUTINE k_grid_to_cell_phase_prepare(table, cells, xkp, nworkers)
2304 TYPE(k_grid_to_cell_phase_type), INTENT(OUT) :: table
2305 INTEGER, INTENT(IN) :: cells(:, :)
2306 REAL(kind=dp), INTENT(IN) :: xkp(:, :)
2307 INTEGER, INTENT(IN) :: nworkers
2308
2309 COMPLEX(KIND=dp) :: phase
2310 INTEGER :: axis, i, k, n, nk, pos
2311 INTEGER, ALLOCATABLE :: coordinates(:, :), count(:), &
2312 direction(:), order(:), values(:)
2313 REAL(kind=dp) :: arg
2314
2315 cpassert(SIZE(cells, 1) == 3 .AND. SIZE(xkp, 1) == 3)
2316 cpassert(nworkers > 0)
2317 n = SIZE(cells, 2) + 1
2318 nk = SIZE(xkp, 2)
2319 ALLOCATE (coordinates(3, n), count(3), values(n), order(n))
2320 DO axis = 1, 3
2321 values(1) = 0
2322 values(2:n) = abs(cells(axis, :))
2323 CALL sort(values, n, order)
2324 count(axis) = 0
2325 DO i = 1, n
2326 IF (i > 1) THEN
2327 IF (values(i) == values(i - 1)) cycle
2328 END IF
2329 count(axis) = count(axis) + 1
2330 coordinates(axis, count(axis)) = values(i)
2331 END DO
2332 table%first(axis + 1) = table%first(axis) + count(axis)
2333 END DO
2334 n = sum(count)
2335 ALLOCATE (table%coordinate(n), direction(n), table%cosine(nk, n), table%sine(nk, n))
2336 DO axis = 1, 3
2337 pos = table%first(axis)
2338 table%coordinate(pos:pos + count(axis) - 1) = coordinates(axis, 1:count(axis))
2339 direction(pos:pos + count(axis) - 1) = axis
2340 END DO
2341!$OMP PARALLEL DEFAULT(NONE) NUM_THREADS(MIN(nworkers,MAX(1,nk))) IF(MIN(nworkers,MAX(1,nk)) > 1) &
2342!$OMP SHARED(table,xkp,direction,n,nk) PRIVATE(i,k,arg,phase)
2343!$OMP DO COLLAPSE(2) SCHEDULE(STATIC)
2344 DO i = 1, n
2345 DO k = 1, nk
2346 IF (table%coordinate(i) == 0) THEN
2347 table%cosine(k, i) = 1.0_dp
2348 table%sine(k, i) = 0.0_dp
2349 ELSE
2350 arg = twopi*real(table%coordinate(i), dp)*xkp(direction(i), k)
2351 phase = exp(cmplx(0.0_dp, arg, kind=dp))
2352 table%cosine(k, i) = real(phase, dp)
2353 table%sine(k, i) = aimag(phase)
2354 END IF
2355 END DO
2356 END DO
2357!$OMP END DO NOWAIT
2358!$OMP END PARALLEL
2359 END SUBROUTINE k_grid_to_cell_phase_prepare
2360
2361! **************************************************************************************************
2362!> \brief Weighted K-to-R contraction of one AO block, before spatial rotations and MPI.
2363!> The caller supplies the atom mapping and image multiplicities. Phases include
2364!> the gauge shift, time reversal and conjugation of a transposed source block.
2365!> \param source borrowed local matrices by k point, real/imaginary component and spin
2366!> \param tiles local FM intervals (first/last row and column) by atom, k, component and spin
2367!> \param ikp_index local source indices for this symmetry contribution
2368!> \param xkp reciprocal coordinates for these contributions
2369!> \param weight original k-point weights, including symmetry multiplicity
2370!> \param images translations, real multiplicities and imaginary sign sums
2371!> \param source_pair source AO atom pair
2372!> \param block_shape source AO row and column sizes
2373!> \param block_offset global first row and column of the source block
2374!> \param shift column-minus-row gauge shift
2375!> \param time_reversal conjugate the source before applying its gauge phase
2376!> \param trans conjugate the imaginary contribution on transposition
2377!> \param phase_table group-local unweighted one-dimensional phases
2378!> \param sums allocated flattened source blocks by image and spin
2379!> \param gemm_ctx worker-owned host GEMM contexts
2380! **************************************************************************************************
2381 SUBROUTINE k_grid_to_cell_contract(source, tiles, ikp_index, xkp, weight, images, source_pair, &
2382 block_shape, block_offset, shift, time_reversal, trans, &
2383 phase_table, sums, gemm_ctx)
2384 TYPE(cp_fm_p_type), INTENT(IN) :: source(:, :, :)
2385 INTEGER, INTENT(IN) :: tiles(:, :, :, :, :), ikp_index(:)
2386 REAL(kind=dp), INTENT(IN) :: xkp(:, :), weight(:)
2387 INTEGER, INTENT(IN) :: images(:, :), source_pair(2), &
2388 block_shape(2), block_offset(2), &
2389 shift(3)
2390 LOGICAL, INTENT(IN) :: time_reversal, trans
2391 TYPE(k_grid_to_cell_phase_type), INTENT(IN) :: phase_table
2392 REAL(kind=dp), ALLOCATABLE, INTENT(INOUT) :: sums(:, :, :)
2393 TYPE(local_gemm_ctxt_type), INTENT(INOUT) :: gemm_ctx(:)
2394
2395 INTEGER(KIND=int_8), PARAMETER :: max_elements = 1048576_int_8
2396
2397 INTEGER :: a, actual_workers, axis, b, batch, column, first, i, ic, ifirst, ii, ikp, ilast, &
2398 image, ispin, ix, iy, iz, j, jj, k, last, nc, nclass, nelement, nimage, nrow, nsine, &
2399 nspin, nworkers, pos, thread
2400 INTEGER, ALLOCATABLE :: cells(:, :), image_class(:), &
2401 phase_index(:, :), sine_index(:)
2402 LOGICAL :: contributes, has_zero, identity_gauge, &
2403 real_only, zero_shift
2404 REAL(kind=dp) :: cosine_xy, sine, sine_xy, sx, sy, sz
2405 REAL(kind=dp), ALLOCATABLE :: beta(:), gauge(:), gauge_factors(:, :, :), packed(:, :), &
2406 phases(:, :), reduced(:, :, :, :), zero_sum(:, :, :)
2407 TYPE(cp_fm_type), POINTER :: matrix
2408
2409 cpassert(SIZE(gemm_ctx) > 0)
2410 nc = SIZE(source, 2)
2411 nspin = SIZE(source, 3)
2412 real_only = nc == 1
2413 cpassert(nc == 1 .OR. nc == 2)
2414 cpassert(SIZE(xkp, 2) == SIZE(ikp_index) .AND. SIZE(weight) == SIZE(ikp_index))
2415 cpassert(SIZE(phase_table%cosine, 1) == SIZE(ikp_index))
2416 a = source_pair(1)
2417 b = source_pair(2)
2418 nelement = block_shape(1)*block_shape(2)
2419 nimage = SIZE(images, 2)
2420 IF (ALLOCATED(sums)) THEN
2421 IF (any(shape(sums) /= [nelement, nimage, nspin])) DEALLOCATE (sums)
2422 END IF
2423 IF (.NOT. ALLOCATED(sums)) ALLOCATE (sums(nelement, nimage, nspin))
2424 IF (nelement == 0 .OR. nimage == 0 .OR. SIZE(ikp_index) == 0) THEN
2425 sums = 0.0_dp
2426 RETURN
2427 END IF
2428 CALL rskp_image_classes(images(1:3, :), image_class, cells)
2429 nclass = SIZE(cells, 2)
2430 has_zero = any(image_class == 0)
2431 ! Only classes whose output multiplicities use the imaginary source need sine.
2432 ALLOCATE (sine_index(nclass), source=0)
2433 IF (.NOT. real_only) THEN
2434 DO image = 1, nimage
2435 i = abs(image_class(image))
2436 IF (i > 0 .AND. images(5, image) /= 0) sine_index(i) = 1
2437 END DO
2438 END IF
2439 nsine = 0
2440 DO i = 1, nclass
2441 IF (sine_index(i) == 0) cycle
2442 nsine = nsine + 1
2443 sine_index(i) = nsine
2444 END DO
2445 ALLOCATE (phase_index(3, nclass))
2446 DO image = 1, nclass
2447 DO axis = 1, 3
2448 first = phase_table%first(axis)
2449 last = phase_table%first(axis + 1) - 1
2450 pos = locate(phase_table%coordinate(first:last), abs(cells(axis, image)))
2451 cpassert(pos > 0)
2452 pos = pos + first - 1
2453 cpassert(phase_table%coordinate(pos) == abs(cells(axis, image)))
2454 phase_index(axis, image) = pos
2455 END DO
2456 END DO
2457 zero_shift = all(shift == 0)
2458 identity_gauge = zero_shift .AND. .NOT. time_reversal
2459 nworkers = min(SIZE(gemm_ctx), SIZE(ikp_index))
2460 ! Split K for parallelism and an 8 MiB packed/phase scratch target per worker.
2461 ! N retains the complete canonical image set; no image-based task limit applies.
2462 batch = min(512, 1 + (SIZE(ikp_index) - 1)/nworkers)
2463 batch = max(1, min(batch, int(max_elements/(int(nc, int_8)*int(nelement, int_8) + &
2464 int(nclass + nsine, int_8)))))
2465 nworkers = min(nworkers, 1 + (SIZE(ikp_index) - 1)/batch)
2466 actual_workers = 1
2467 ALLOCATE (reduced(nelement, nclass + nsine, nspin, nworkers))
2468 IF (has_zero) ALLOCATE (zero_sum(nelement, nspin, nworkers), source=0.0_dp)
2469!$OMP PARALLEL DEFAULT(NONE) NUM_THREADS(nworkers) IF(nworkers > 1) &
2470!$OMP SHARED(source,tiles,ikp_index,xkp,weight,A,b,block_shape,block_offset,shift,time_reversal, &
2471!$OMP nelement,nspin,nclass,nsine,nc,batch,cells,sine_index,identity_gauge,real_only,has_zero, &
2472!$OMP phase_table,phase_index,zero_shift, &
2473!$OMP reduced,zero_sum,gemm_ctx,actual_workers) &
2474!$OMP PRIVATE(thread,packed,phases,beta,gauge,gauge_factors,first,last,image,k,ispin, &
2475!$OMP contributes,ikp,ic,ifirst,ilast,column,matrix,nrow,j,jj,ii,i, &
2476!$OMP ix,iy,iz,sx,sy,sz,cosine_xy,sine_xy)
2477 thread = 1
2478!$ thread = omp_get_thread_num() + 1
2479!$OMP MASTER
2480!$ actual_workers = omp_get_num_threads()
2481!$OMP END MASTER
2482 ALLOCATE (beta(nspin), source=0.0_dp)
2483 CALL rskp_work_matrix(packed, nelement, nc*batch)
2484 CALL rskp_work_matrix(phases, batch, nclass + nsine)
2485 IF (.NOT. identity_gauge .AND. nc == 2) ALLOCATE (gauge(nelement))
2486 IF (.NOT. identity_gauge) ALLOCATE (gauge_factors(2, 2, batch))
2487!$OMP DO SCHEDULE(STATIC)
2488 DO first = 1, SIZE(ikp_index), batch
2489 last = min(first + batch - 1, SIZE(ikp_index))
2490 DO image = 1, nclass
2491 ix = phase_index(1, image)
2492 iy = phase_index(2, image)
2493 iz = phase_index(3, image)
2494 sx = real(sign(1, cells(1, image)), dp)
2495 sy = real(sign(1, cells(2, image)), dp)
2496 sz = real(sign(1, cells(3, image)), dp)
2497!$OMP SIMD PRIVATE(cosine_xy,sine_xy)
2498 DO k = first, last
2499 cosine_xy = phase_table%cosine(k, ix)*phase_table%cosine(k, iy) - &
2500 sx*sy*phase_table%sine(k, ix)*phase_table%sine(k, iy)
2501 sine_xy = sx*phase_table%sine(k, ix)*phase_table%cosine(k, iy) + &
2502 sy*phase_table%cosine(k, ix)*phase_table%sine(k, iy)
2503 phases(k - first + 1, image) = weight(k)*( &
2504 cosine_xy*phase_table%cosine(k, iz) - sine_xy*sz*phase_table%sine(k, iz))
2505 IF (sine_index(image) > 0) THEN
2506 phases(k - first + 1, nclass + sine_index(image)) = weight(k)*( &
2507 sine_xy*phase_table%cosine(k, iz) + cosine_xy*sz*phase_table%sine(k, iz))
2508 END IF
2509 END DO
2510 END DO
2511 IF (.NOT. identity_gauge) THEN
2512 DO k = first, last
2513 IF (zero_shift) THEN
2514 gauge_factors(:, :, k - first + 1) = 0.0_dp
2515 gauge_factors(1, 1, k - first + 1) = 1.0_dp
2516 gauge_factors(2, 2, k - first + 1) = merge(-1.0_dp, 1.0_dp, time_reversal)
2517 ELSE
2518 CALL rskp_phase_factors(shift, xkp(:, k), time_reversal, real_only, &
2519 gauge_factors(:, :, k - first + 1))
2520 END IF
2521 END DO
2522 END IF
2523 DO ispin = 1, nspin
2524 packed = 0.0_dp
2525 contributes = .false.
2526 DO k = first, last
2527 ikp = ikp_index(k)
2528 DO ic = 1, nc
2529 ifirst = tiles(1, a, ikp, ic, ispin)
2530 ilast = tiles(2, a, ikp, ic, ispin)
2531 IF (ifirst > ilast .OR. tiles(3, b, ikp, ic, ispin) > tiles(4, b, ikp, ic, ispin)) cycle
2532 contributes = .true.
2533 column = k - first + 1 + (ic - 1)*batch
2534 matrix => source(ikp, ic, ispin)%matrix
2535 nrow = ilast - ifirst + 1
2536 DO j = tiles(3, b, ikp, ic, ispin), tiles(4, b, ikp, ic, ispin)
2537 jj = matrix%matrix_struct%col_indices(j) - block_offset(2)
2538 ii = matrix%matrix_struct%row_indices(ifirst) - block_offset(1) + 1 + jj*block_shape(1)
2539 IF (matrix%matrix_struct%row_indices(ilast) - &
2540 matrix%matrix_struct%row_indices(ifirst) == nrow - 1) THEN
2541 packed(ii:ii + nrow - 1, column) = matrix%local_data(ifirst:ilast, j)
2542 ELSE
2543 DO i = ifirst, ilast
2544 ii = matrix%matrix_struct%row_indices(i) - block_offset(1) + 1 + jj*block_shape(1)
2545 packed(ii, column) = matrix%local_data(i, j)
2546 END DO
2547 END IF
2548 END DO
2549 END DO
2550 column = k - first + 1
2551 IF (.NOT. identity_gauge) THEN
2552 IF (nc == 1) THEN
2553 packed(:, column) = gauge_factors(1, 1, column)*packed(:, column)
2554 ELSE
2555 gauge(:) = gauge_factors(1, 1, column)*packed(:, column) + &
2556 gauge_factors(1, 2, column)*packed(:, batch + column)
2557 packed(:, batch + column) = gauge_factors(2, 1, column)*packed(:, column) + &
2558 gauge_factors(2, 2, column)*packed(:, batch + column)
2559 packed(:, column) = gauge
2560 END IF
2561 END IF
2562 IF (has_zero) zero_sum(:, ispin, thread) = zero_sum(:, ispin, thread) + weight(k)*packed(:, column)
2563 END DO
2564 IF (.NOT. contributes .OR. nclass == 0) cycle
2565 CALL gemm_ctx(thread)%gemm('N', 'N', nelement, nclass, last - first + 1, 1.0_dp, &
2566 packed, nelement, phases, batch, beta(ispin), reduced(:, :, ispin, thread), nelement)
2567 IF (nsine > 0) THEN
2568 CALL gemm_ctx(thread)%gemm('N', 'N', nelement, nsine, last - first + 1, 1.0_dp, &
2569 packed(:, batch + 1:), nelement, phases(:, nclass + 1:), batch, &
2570 beta(ispin), reduced(:, nclass + 1:, ispin, thread), nelement)
2571 END IF
2572 beta(ispin) = 1.0_dp
2573 END DO
2574 END DO
2575!$OMP END DO
2576 ! An idle worker, or a spin with no locally owned source elements, contributes zero.
2577 DO ispin = 1, nspin
2578 IF (beta(ispin) == 0.0_dp) reduced(:, :, ispin, thread) = 0.0_dp
2579 END DO
2580 DEALLOCATE (beta, packed, phases)
2581 IF (ALLOCATED(gauge)) DEALLOCATE (gauge)
2582 IF (ALLOCATED(gauge_factors)) DEALLOCATE (gauge_factors)
2583!$OMP END PARALLEL
2584 ! Merge local k contributions before expanding multiplicities and rotating AO blocks.
2585 DO thread = 2, actual_workers
2586 reduced(:, :, :, 1) = reduced(:, :, :, 1) + reduced(:, :, :, thread)
2587 IF (has_zero) zero_sum(:, :, 1) = zero_sum(:, :, 1) + zero_sum(:, :, thread)
2588 END DO
2589 DO ispin = 1, nspin
2590 DO image = 1, nimage
2591 i = abs(image_class(image))
2592 IF (i == 0) THEN
2593 sums(:, image, ispin) = real(images(4, image), dp)*zero_sum(:, ispin, 1)
2594 ELSE
2595 sums(:, image, ispin) = real(images(4, image), dp)*reduced(:, i, ispin, 1)
2596 IF (sine_index(i) == 0) cycle
2597 sine = real(images(5, image)*sign(1, image_class(image)), dp)*merge(-1.0_dp, 1.0_dp, trans)
2598 sums(:, image, ispin) = sums(:, image, ispin) + sine*reduced(:, nclass + sine_index(i), ispin, 1)
2599 END IF
2600 END DO
2601 END DO
2602 END SUBROUTINE k_grid_to_cell_contract
2603
2604! **************************************************************************************************
2605!> \brief Classify translations by +/-R, without assuming operator or k-point symmetry.
2606!> \param cells integer translations
2607!> \param classes signed class for each input; zero denotes the origin
2608!> \param representatives canonical nonzero translations, one per class
2609! **************************************************************************************************
2610 SUBROUTINE rskp_image_classes(cells, classes, representatives)
2611 INTEGER, INTENT(IN) :: cells(:, :)
2612 INTEGER, ALLOCATABLE, INTENT(OUT) :: classes(:), representatives(:, :)
2613
2614 INTEGER :: d, i, n, nclass
2615 INTEGER, ALLOCATABLE :: first(:), keys(:, :), order(:), sense(:)
2616
2617 n = SIZE(cells, 2)
2618 ALLOCATE (keys(3, n), order(n), sense(n), first(n), classes(n))
2619 DO i = 1, n
2620 sense(i) = 1
2621 DO d = 1, 3
2622 IF (cells(d, i) == 0) cycle
2623 sense(i) = sign(1, cells(d, i))
2624 EXIT
2625 END DO
2626 keys(:, i) = sense(i)*cells(:, i)
2627 END DO
2628 IF (n > 0) CALL sort(keys, 1, n, 1, 3, order)
2629 nclass = 0
2630 DO i = 1, n
2631 IF (all(keys(:, i) == 0)) THEN
2632 classes(order(i)) = 0
2633 cycle
2634 END IF
2635 IF (nclass == 0) THEN
2636 nclass = 1
2637 first(nclass) = i
2638 ELSE IF (any(keys(:, i) /= keys(:, first(nclass)))) THEN
2639 nclass = nclass + 1
2640 first(nclass) = i
2641 END IF
2642 classes(order(i)) = sense(order(i))*nclass
2643 END DO
2644 ALLOCATE (representatives(3, nclass))
2645 representatives(:, :) = keys(:, first(1:nclass))
2646 END SUBROUTINE rskp_image_classes
2647
2648! **************************************************************************************************
2649!> \brief Real two-component representation of the Bloch phase and time reversal.
2650!> \param shift column-minus-row cell shift with the selected phase direction
2651!> \param xkp target k-point coordinates
2652!> \param time_reversal conjugate the source before applying the phase
2653!> \param real_only reject phases not representable by real wavefunctions
2654!> \param phase coefficients mapping source [Re,Im] to transformed [Re,Im]
2655! **************************************************************************************************
2656 SUBROUTINE rskp_phase_factors(shift, xkp, time_reversal, real_only, phase)
2657 INTEGER, INTENT(IN) :: shift(3)
2658 REAL(kind=dp), INTENT(IN) :: xkp(3)
2659 LOGICAL, INTENT(IN) :: time_reversal, real_only
2660 REAL(kind=dp), INTENT(OUT) :: phase(2, 2)
2661
2662 REAL(kind=dp) :: arg, cosine, sine
2663
2664 arg = twopi*(real(shift(1), dp)*xkp(1) + real(shift(2), dp)*xkp(2) + real(shift(3), dp)*xkp(3))
2665 cosine = cos(arg)
2666 sine = sin(arg)
2667 IF (real_only .AND. abs(sine) > 1.e-12_dp) THEN
2668 CALL cp_abort(__location__, "Real k-point wavefunctions cannot represent symmetry phases")
2669 END IF
2670 phase(:, 1) = [cosine, -sine]
2671 phase(:, 2) = merge(-1.0_dp, 1.0_dp, time_reversal)*[sine, cosine]
2672 END SUBROUTINE rskp_phase_factors
2673
2674! **************************************************************************************************
2675!> \brief Allocate a dense work matrix with the requested shape
2676!> \param work dense work matrix
2677!> \param nrow number of rows
2678!> \param ncol number of columns
2679! **************************************************************************************************
2680 SUBROUTINE rskp_work_matrix(work, nrow, ncol)
2681
2682 REAL(kind=dp), ALLOCATABLE, DIMENSION(:, :), &
2683 INTENT(INOUT) :: work
2684 INTEGER, INTENT(IN) :: nrow, ncol
2685
2686 IF (ALLOCATED(work)) THEN
2687 IF (SIZE(work, 1) == nrow .AND. SIZE(work, 2) == ncol) RETURN
2688 DEALLOCATE (work)
2689 END IF
2690 ALLOCATE (work(nrow, ncol))
2691
2692 END SUBROUTINE rskp_work_matrix
2693
2694END MODULE kpoint_transform
static GRID_HOST_DEVICE int modulo(int a, int m)
Equivalent of Fortran's MODULO, which always return a positive number. https://gcc....
static void dgemm(const char transa, const char transb, const int m, const int n, const int k, const double alpha, const double *a, const int lda, const double *b, const int ldb, const double beta, double *c, const int ldc)
Convenient wrapper to hide Fortran nature of dgemm_, swapping a and b.
Represents a complex full matrix distributed on many processors.
subroutine, public dbcsr_get_readonly_block_p(matrix, row, col, block, found, row_size, col_size)
Like dbcsr_get_block_p() but with matrix being INTENT(IN). When invoking this routine,...
subroutine, public dbcsr_get_block_p(matrix, row, col, block, found, row_size, col_size)
...
subroutine, public dbcsr_get_info(matrix, nblkrows_total, nblkcols_total, nfullrows_total, nfullcols_total, nblkrows_local, nblkcols_local, nfullrows_local, nfullcols_local, my_prow, my_pcol, local_rows, local_cols, proc_row_dist, proc_col_dist, row_blk_size, col_blk_size, row_blk_offset, col_blk_offset, distribution, name, matrix_type, group)
...
subroutine, public dbcsr_set(matrix, alpha)
...
represent the structure of a full matrix
logical function, public cp_fm_struct_equivalent(fmstruct1, fmstruct2)
returns true if the two matrix structures are equivalent, false otherwise.
subroutine, public cp_fm_struct_retain(fmstruct)
retains a full matrix structure
subroutine, public cp_fm_struct_release(fmstruct)
releases a full matrix structure
represent a full matrix distributed on many processors
Definition cp_fm_types.F:15
various routines to log and control the output. The idea is that decisions about where to log should ...
integer function, public cp_logger_get_default_io_unit(logger)
returns the unit nr for the ionode (-1 on all other processors) skips as well checks if the procs cal...
collects all constants needed in input so that they can be used without circular dependencies
integer, parameter, public rskp_backend_fft
integer, parameter, public rskp_backend_gemm
Defines the basic variable types.
Definition kinds.F:23
integer, parameter, public int_8
Definition kinds.F:54
integer, parameter, public dp
Definition kinds.F:34
Sparse direct and regular-grid lattice Fourier transforms.
subroutine, public cell_to_k_grid_prepare(work, index_to_cell, xkp, nkp_grid, deriv_direction, hmat)
Prepare fixed forward lattice maps; the caller keeps geometry and k points unchanged.
subroutine, public k_grid_to_cell_prepare(work, xkp, nkp_grid, index_to_cell, weights, allow_incomplete)
Prepare one inverse lattice transform for repeated matrix batches.
subroutine, public cell_to_k_grid_fft(values_rs, index_to_cell, xkp, nkp_grid, values_k, used_fft, deriv_direction, hmat, selected_kpoints, require_fft, work, cell_first, max_workspace_bytes, accumulate, profile)
Transform a batch of real matrices from image cells to every supplied k point.
subroutine, public k_grid_to_cell_release(work)
Release call-scoped inverse lattice maps and FFT buffers.
subroutine, public cell_to_k_grid_release(work)
Release forward lattice maps and numerical buffers.
subroutine, public k_grid_to_cell_execute(work, values_k, values_rs, used_fft, require_fft, allow_direct_fallback, values_real, profile, max_workspace_bytes)
Execute a matrix batch using the prepared inverse lattice transform. Call outside OpenMP worker regio...
Prepared real-space/reciprocal-space transforms and distributed AO adapters. Routes fix geometry and ...
subroutine, public rskp_transform_prepare(prepared, source, source_rows, source_sab, cell_to_index, target_struct, world, group, inter, backend, xkp, nkp_grid, kp_dist, mirror_sign, scale)
Prepare source rows for a generic group- or world-distributed target. GEMM retains sparse snapshots a...
subroutine, public cell_to_k_transform(values_rs, cells, xkp, nkp_grid, values_k, backend, accumulate, profile_fft, profile_label)
Transform local real matrix panels with the selected RSKP backend. GEMM reuses the prepared parity fo...
subroutine, public rskp_dbcsr_transform_batch(source, real_k, imag_k, xkp, cell_to_index, sab_nl, nkp_grid, backend)
Transform all real-space spin rows to a full k-grid in distributed DBCSR matrices....
subroutine, public k_grid_to_cell_phase_prepare(table, cells, xkp, nworkers)
Tabulate distinct absolute translation coordinates for a group's local k points. All AO pairs share t...
subroutine, public cell_to_k_sparse_fft(snapshot, route, xkp, nkp_grid, kp_dist, inter, max_bytes, cache, output)
FFT captured raw-element panels and retain the destination group's k points. Reuse the sparse route a...
subroutine, public rskp_batch_size(route, nk, ncomp, nworkers, nmatrices, max_bytes, width, work_size)
Bound worker scratch and output tiles for either prepared backend. Packed input belongs to the snapsh...
subroutine, public rskp_capture(source, matrix_row, route, snapshot, mirror_sign, scale)
Capture current values using cached routing, with no per-k communication. GEMM/FFT snapshots own pack...
subroutine, public k_to_real_cell_transform(values_k, xkp, nkp_grid, cells, values_rs, backend, profile_fft)
Real part of the inverse lattice transform, normalized by the number of k points. Local rectangular p...
subroutine, public rskp_transform_release(prepared)
Release the route, snapshot and cached backend state of a prepared transform.
subroutine, public k_grid_to_cell_contract(source, tiles, ikp_index, xkp, weight, images, source_pair, block_shape, block_offset, shift, time_reversal, trans, phase_table, sums, gemm_ctx)
Weighted K-to-R contraction of one AO block, before spatial rotations and MPI. The caller supplies th...
subroutine, public cell_to_k_backend_prepare(route, xkp, nkp_grid, max_bytes)
Prepare the selected R-to-K backend plan once for a fixed route. Both backends retain their geometry ...
logical function, public rskp_compatible(route, source, matrix_row, source_sab)
Check the source AO layout, not its values or sparse block presence.
subroutine, public kp_transform_plan_create(plan, sab_nl, cell_to_index, nimg, block_template, group_entries)
Build the immutable neighbor-list traversal shared by R-to-K and K-to-R transforms.
subroutine, public rskp_transform_evaluate(prepared, source_index, k, xkp, fm_re, fm_im, cfm)
Evaluate one k point directly into an FM, split-FM or CFM target. The handle owns mutable scratch; co...
subroutine, public rskp_route_release(route)
Release backend-owned geometry and FFT work buffers from a route.
subroutine, public cell_to_k_backend_evaluate_batch(snapshot, route, xkp, first_local, work, batch, output)
Evaluate a k-point tile through the selected prepared R-to-K backend. GEMM contracts the retained rea...
subroutine, public rskp_route_create(source, source_sab, cell_to_index, matrix_struct, world, group, inter, backend, route, ready)
Cache routing to the FM tiles of one representative group. Equivalent ranks in the other groups recei...
subroutine, public rskp_values_release(snapshot)
Release one captured row without invalidating other slots or routing.
subroutine, public rskp_phase_factors(shift, xkp, time_reversal, real_only, phase)
Real two-component representation of the Bloch phase and time reversal.
subroutine, public rskp_work_matrix(work, nrow, ncol)
Allocate a dense work matrix with the requested shape.
subroutine, public rskp_transform(rmatrix, cmatrix, rsmat, ispin, xkp, cell_to_index, sab_nl, is_complex, rs_sign)
Transformation of real space matrices to a kpoint.
Machine interface based on Fortran 2003 and POSIX.
Definition machine.F:17
real(kind=dp) function, public m_walltime()
returns time from a real-time clock, protected against rolling early/easily
Definition machine.F:144
Definition of mathematical constants and functions.
complex(kind=dp), parameter, public z_one
real(kind=dp), parameter, public twopi
complex(kind=dp), parameter, public z_zero
Interface to the message passing library MPI.
Define the neighbor list data types and the corresponding functionality.
subroutine, public neighbor_list_iterator_create(iterator_set, nl, search, nthread)
Neighbor list iterator functions.
subroutine, public neighbor_list_iterator_release(iterator_set)
...
subroutine, public get_neighbor_list_set_p(neighbor_list_sets, nlist, symmetric)
Return the components of the first neighbor list set.
integer function, public neighbor_list_iterate(iterator_set, mepos)
...
subroutine, public get_iterator_info(iterator_set, mepos, ikind, jkind, nkind, ilist, nlist, inode, nnode, iatom, jatom, r, cell)
...
All kind of helpful little routines.
Definition util.F:14
pure integer function, public locate(array, x)
Purpose: Given an array array(1:n), and given a value x, a value x_index is returned which is the ind...
Definition util.F:61
Represent a complex full matrix.
keeps the information about the structure of a full matrix
just to build arrays of pointers to matrices
represent a full matrix
stores all the informations relevant to an mpi environment