From d65bbd8aa6ca282ee116b978bc97b7f4b9d27e87 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 20 Feb 2026 22:21:22 -0500 Subject: [PATCH 1/8] Fix WP_MOK hardcoded to 8 bytes, use storage_size for precision portability MPI file offsets assume 8-byte reals. Single-precision builds would read from wrong offsets. Use storage_size(0._wp)/8 instead. Co-Authored-By: Claude Opus 4.6 --- src/post_process/m_data_input.f90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index a3e3f22886..6b87c585df 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -133,7 +133,7 @@ impure subroutine s_setup_mpi_io_params(data_size, m_MOK, n_MOK, p_MOK, WP_MOK, m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -177,7 +177,7 @@ impure subroutine s_read_ib_data_files(file_loc_base, t_step) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) save_index = t_step/t_step_save ! get the number of saves done to this point data_size = (m + 1)*(n + 1)*(p + 1) @@ -517,7 +517,7 @@ impure subroutine s_read_parallel_conservative_data(t_step, m_MOK, n_MOK, p_MOK, m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) From c260aadfe1bbcaee66cd97c434e2a1529495e849 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 21 Feb 2026 00:31:25 -0500 Subject: [PATCH 2/8] Extend storage_size fix to simulation and pre_process MPI-IO The same int(8._wp, MPI_OFFSET_KIND) pattern that was fixed in post_process/m_data_input.f90 was present in 7 additional locations across simulation/m_data_output.fpp, simulation/m_start_up.fpp, pre_process/m_data_output.fpp, and pre_process/m_start_up.fpp. All hardcoded 8-byte strides are replaced with storage_size(0._wp)/8 so MPI file offsets are correct in single-precision builds. Co-Authored-By: Claude Sonnet 4.6 --- src/pre_process/m_data_output.fpp | 4 ++-- src/pre_process/m_start_up.fpp | 2 +- src/simulation/m_data_output.fpp | 6 +++--- src/simulation/m_start_up.fpp | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pre_process/m_data_output.fpp b/src/pre_process/m_data_output.fpp index bac4fdc038..8a34f7b17a 100644 --- a/src/pre_process/m_data_output.fpp +++ b/src/pre_process/m_data_output.fpp @@ -549,7 +549,7 @@ contains m_MOK = int(m_glb_save, MPI_OFFSET_KIND) n_MOK = int(n_glb_save, MPI_OFFSET_KIND) p_MOK = int(p_glb_save, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -615,7 +615,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) diff --git a/src/pre_process/m_start_up.fpp b/src/pre_process/m_start_up.fpp index 3b45547c29..dbd6ce211d 100644 --- a/src/pre_process/m_start_up.fpp +++ b/src/pre_process/m_start_up.fpp @@ -660,7 +660,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) diff --git a/src/simulation/m_data_output.fpp b/src/simulation/m_data_output.fpp index 5051d5e15d..d239f61ee6 100644 --- a/src/simulation/m_data_output.fpp +++ b/src/simulation/m_data_output.fpp @@ -895,7 +895,7 @@ contains m_MOK = int(m_glb_save + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb_save + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb_save + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -963,7 +963,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(alt_sys, MPI_OFFSET_KIND) @@ -1087,7 +1087,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) write (file_loc, '(A)') 'ib.dat' diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index d8d0c87417..793ee2a25c 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -652,7 +652,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(8._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) From bbfef040c3a5aeb8205d0912efd6223d03b4b640 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 24 Feb 2026 15:13:30 -0500 Subject: [PATCH 3/8] Fix IB marker stride using WP size instead of integer size IB markers are MPI_INTEGER (4 bytes), but WP_MOK was computed as storage_size(0._wp)/8 = 8 bytes in double precision, causing a factor-of-2 displacement error when writing/reading ib.dat via MPI-IO. Fix by using storage_size(0)/8 (default integer byte size) at the two IB-specific MPI-IO displacement sites: - src/simulation/m_data_output.fpp (s_write_parallel_ib_data) - src/post_process/m_data_input.f90 (s_read_ib_data_files) Also fix the Lustre restart read path in src/simulation/m_start_up.fpp which had WP_MOK hardcoded to 4 instead of storage_size(0._wp)/8. Co-Authored-By: Claude Sonnet 4.6 --- src/post_process/m_data_input.f90 | 2 +- src/simulation/m_data_output.fpp | 2 +- src/simulation/m_start_up.fpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index 6b87c585df..498a78c0a4 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -177,7 +177,7 @@ impure subroutine s_read_ib_data_files(file_loc_base, t_step) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0)/8, MPI_OFFSET_KIND) save_index = t_step/t_step_save ! get the number of saves done to this point data_size = (m + 1)*(n + 1)*(p + 1) diff --git a/src/simulation/m_data_output.fpp b/src/simulation/m_data_output.fpp index d239f61ee6..bf076951c4 100644 --- a/src/simulation/m_data_output.fpp +++ b/src/simulation/m_data_output.fpp @@ -1087,7 +1087,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) write (file_loc, '(A)') 'ib.dat' diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 793ee2a25c..3d77255b41 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -575,7 +575,7 @@ contains m_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) n_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) p_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) - WP_MOK = int(4._wp, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) From b047d7f794f1eaeb0c4f8ceb431ed78904210bfc Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Tue, 24 Feb 2026 16:16:40 -0500 Subject: [PATCH 4/8] Fix WP_MOK to use stp instead of wp for main field data I/O MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MPI-IO displacement (WP_MOK) must match the storage precision of the data being read/written. Main field variables are written with mpi_io_p (which tracks stp), not mpi_p (which tracks wp). In mixed-precision mode stp = half (2 bytes) while wp = double (8 bytes), so using storage_size(0._wp) gives a 4× wrong displacement. Change all main-data WP_MOK assignments from storage_size(0._wp) to storage_size(0._stp) across all three I/O paths: - src/simulation/m_data_output.fpp (write, two call sites) - src/simulation/m_start_up.fpp (restart read, two call sites) - src/post_process/m_data_input.f90 (post-process read, two call sites) IB marker I/O (MPI_INTEGER, 4 bytes) was already fixed to use storage_size(0)/8 in the previous commit and is not changed here. Co-Authored-By: Claude Sonnet 4.6 --- src/post_process/m_data_input.f90 | 4 ++-- src/simulation/m_data_output.fpp | 4 ++-- src/simulation/m_start_up.fpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index 498a78c0a4..933f4274fe 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -133,7 +133,7 @@ impure subroutine s_setup_mpi_io_params(data_size, m_MOK, n_MOK, p_MOK, WP_MOK, m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -517,7 +517,7 @@ impure subroutine s_read_parallel_conservative_data(t_step, m_MOK, n_MOK, p_MOK, m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) diff --git a/src/simulation/m_data_output.fpp b/src/simulation/m_data_output.fpp index bf076951c4..b1b3c7a1a4 100644 --- a/src/simulation/m_data_output.fpp +++ b/src/simulation/m_data_output.fpp @@ -895,7 +895,7 @@ contains m_MOK = int(m_glb_save + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb_save + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb_save + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -963,7 +963,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(alt_sys, MPI_OFFSET_KIND) diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 3d77255b41..3faf27b27a 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -575,7 +575,7 @@ contains m_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) n_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) p_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -652,7 +652,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) From 2f6dd049eb07162ed5ce032d1ab83ba87df82439 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 25 Feb 2026 23:26:00 -0500 Subject: [PATCH 5/8] Fix typo in m_start_up.fpp: equilirium -> equilibrium Co-Authored-By: Claude Opus 4.6 --- src/pre_process/m_start_up.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pre_process/m_start_up.fpp b/src/pre_process/m_start_up.fpp index dbd6ce211d..1e7528c27e 100644 --- a/src/pre_process/m_start_up.fpp +++ b/src/pre_process/m_start_up.fpp @@ -798,7 +798,7 @@ contains if (relax) then if (proc_rank == 0) then print *, 'initial condition might have been altered due to enforcement of & -& pTg-equilirium (relax = "T" activated)' +& pTg-equilibrium (relax = "T" activated)' end if call s_infinite_relaxation_k(q_cons_vf) From 21e1e5dd6fdec84858330509c53948d4879737c7 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 26 Feb 2026 00:22:54 -0500 Subject: [PATCH 6/8] Fix wp -> stp in pre_process I/O and remove dead Lustre block Pre-process MPI I/O strides should use storage precision (stp) not working precision (wp) since field arrays use stp. Also removes the dead file_per_process sequential-read path that contained known copy-paste bugs (n_MOK/p_MOK both using m_glb_read). Co-Authored-By: Claude Opus 4.6 --- src/pre_process/m_data_output.fpp | 4 +- src/pre_process/m_start_up.fpp | 2 +- src/simulation/m_start_up.fpp | 209 +++++++++--------------------- 3 files changed, 61 insertions(+), 154 deletions(-) diff --git a/src/pre_process/m_data_output.fpp b/src/pre_process/m_data_output.fpp index 8a34f7b17a..bf07702bf8 100644 --- a/src/pre_process/m_data_output.fpp +++ b/src/pre_process/m_data_output.fpp @@ -549,7 +549,7 @@ contains m_MOK = int(m_glb_save, MPI_OFFSET_KIND) n_MOK = int(n_glb_save, MPI_OFFSET_KIND) p_MOK = int(p_glb_save, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) @@ -615,7 +615,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) diff --git a/src/pre_process/m_start_up.fpp b/src/pre_process/m_start_up.fpp index 1e7528c27e..42725bc3dd 100644 --- a/src/pre_process/m_start_up.fpp +++ b/src/pre_process/m_start_up.fpp @@ -660,7 +660,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) str_MOK = int(name_len, MPI_OFFSET_KIND) NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 3faf27b27a..1aa1435938 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -532,134 +532,55 @@ contains end if end if - if (file_per_process) then - if (cfl_dt) then - call s_int_to_str(n_start, t_step_start_string) - write (file_loc, '(I0,A1,I7.7,A)') n_start, '_', proc_rank, '.dat' - else - call s_int_to_str(t_step_start, t_step_start_string) - write (file_loc, '(I0,A1,I7.7,A)') t_step_start, '_', proc_rank, '.dat' - end if - file_loc = trim(case_dir)//'/restart_data/lustre_'//trim(t_step_start_string)//trim(mpiiofs)//trim(file_loc) - inquire (FILE=trim(file_loc), EXIST=file_exist) - - if (file_exist) then - call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) - - ! Initialize MPI data I/O - if (down_sample) then - call s_initialize_mpi_data_ds(q_cons_vf) - else - if (ib) then - call s_initialize_mpi_data(q_cons_vf, ib_markers) - else - call s_initialize_mpi_data(q_cons_vf) - end if - end if - - if (down_sample) then - ! Size of local arrays - data_size = (m_ds + 3)*(n_ds + 3)*(p_ds + 3) - m_glb_read = m_glb_ds + 1 - n_glb_read = n_glb_ds + 1 - p_glb_read = p_glb_ds + 1 - else - ! Size of local arrays - data_size = (m + 1)*(n + 1)*(p + 1) - m_glb_read = m_glb + 1 - n_glb_read = n_glb + 1 - p_glb_read = p_glb + 1 - end if - - ! Resize some integers so MPI can read even the biggest file - m_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) - n_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) - p_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) - MOK = int(1._wp, MPI_OFFSET_KIND) - str_MOK = int(name_len, MPI_OFFSET_KIND) - NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) - - ! Read the data for each variable - if (bubbles_euler .or. elasticity) then - do i = 1, sys_size!adv_idx%end - var_MOK = int(i, MPI_OFFSET_KIND) - - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - !Read pb and mv for non-polytropic qbmm - if (qbmm .and. .not. polytropic) then - do i = sys_size + 1, sys_size + 2*nb*nnode - var_MOK = int(i, MPI_OFFSET_KIND) - - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - end if - else - if (down_sample) then - do i = 1, sys_size - var_MOK = int(i, MPI_OFFSET_KIND) - - call MPI_FILE_READ(ifile, q_cons_temp(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - else - do i = 1, sys_size - var_MOK = int(i, MPI_OFFSET_KIND) - - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - end if - end if - - call s_mpi_barrier() - - call MPI_FILE_CLOSE(ifile, ierr) - - else - call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.') - end if + ! Open the file to read conservative variables + if (cfl_dt) then + write (file_loc, '(I0,A)') n_start, '.dat' else - ! Open the file to read conservative variables - if (cfl_dt) then - write (file_loc, '(I0,A)') n_start, '.dat' - else - write (file_loc, '(I0,A)') t_step_start, '.dat' - end if - file_loc = trim(case_dir)//'/restart_data'//trim(mpiiofs)//trim(file_loc) - inquire (FILE=trim(file_loc), EXIST=file_exist) + write (file_loc, '(I0,A)') t_step_start, '.dat' + end if + file_loc = trim(case_dir)//'/restart_data'//trim(mpiiofs)//trim(file_loc) + inquire (FILE=trim(file_loc), EXIST=file_exist) - if (file_exist) then - call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) + if (file_exist) then + call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) - ! Initialize MPI data I/O + ! Initialize MPI data I/O - if (ib) then - call s_initialize_mpi_data(q_cons_vf, ib_markers) - else + if (ib) then + call s_initialize_mpi_data(q_cons_vf, ib_markers) + else - call s_initialize_mpi_data(q_cons_vf) + call s_initialize_mpi_data(q_cons_vf) - end if + end if - ! Size of local arrays - data_size = (m + 1)*(n + 1)*(p + 1) - - ! Resize some integers so MPI can read even the biggest file - m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) - n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) - p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) - MOK = int(1._wp, MPI_OFFSET_KIND) - str_MOK = int(name_len, MPI_OFFSET_KIND) - NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) - - ! Read the data for each variable - if (bubbles_euler .or. elasticity) then - do i = 1, sys_size !adv_idx%end + ! Size of local arrays + data_size = (m + 1)*(n + 1)*(p + 1) + + ! Resize some integers so MPI can read even the biggest file + m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) + n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) + p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) + MOK = int(1._wp, MPI_OFFSET_KIND) + str_MOK = int(name_len, MPI_OFFSET_KIND) + NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) + + ! Read the data for each variable + if (bubbles_euler .or. elasticity) then + do i = 1, sys_size !adv_idx%end + var_MOK = int(i, MPI_OFFSET_KIND) + ! Initial displacement to skip at beginning of file + disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) + + call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & + 'native', mpi_info_int, ierr) + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + !Read pb and mv for non-polytropic qbmm + if (qbmm .and. .not. polytropic) then + do i = sys_size + 1, sys_size + 2*nb*nnode var_MOK = int(i, MPI_OFFSET_KIND) ! Initial displacement to skip at beginning of file disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) @@ -669,41 +590,27 @@ contains call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & mpi_io_p, status, ierr) end do - !Read pb and mv for non-polytropic qbmm - if (qbmm .and. .not. polytropic) then - do i = sys_size + 1, sys_size + 2*nb*nnode - var_MOK = int(i, MPI_OFFSET_KIND) - ! Initial displacement to skip at beginning of file - disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) - - call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & - 'native', mpi_info_int, ierr) - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - end if - else - do i = 1, sys_size - var_MOK = int(i, MPI_OFFSET_KIND) - - ! Initial displacement to skip at beginning of file - disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) - - call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & - 'native', mpi_info_int, ierr) - call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do end if + else + do i = 1, sys_size + var_MOK = int(i, MPI_OFFSET_KIND) - call s_mpi_barrier() - - call MPI_FILE_CLOSE(ifile, ierr) + ! Initial displacement to skip at beginning of file + disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) - else - call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.') + call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & + 'native', mpi_info_int, ierr) + call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do end if + call s_mpi_barrier() + + call MPI_FILE_CLOSE(ifile, ierr) + + else + call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.') end if deallocate (x_cb_glb, y_cb_glb, z_cb_glb) From 23d9466d3033e1047ef0e509e11b316ebc1dc6e6 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 26 Feb 2026 00:50:34 -0500 Subject: [PATCH 7/8] Restore file_per_process Lustre sequential-read path This code path is still used in practice on HPC clusters even though it is not exercised by the test suite. Reverts the removal while keeping the wp -> stp precision fix in both branches. Co-Authored-By: Claude Opus 4.6 --- src/simulation/m_start_up.fpp | 209 ++++++++++++++++++++++++---------- 1 file changed, 151 insertions(+), 58 deletions(-) diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 1aa1435938..3faf27b27a 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -532,55 +532,134 @@ contains end if end if - ! Open the file to read conservative variables - if (cfl_dt) then - write (file_loc, '(I0,A)') n_start, '.dat' - else - write (file_loc, '(I0,A)') t_step_start, '.dat' - end if - file_loc = trim(case_dir)//'/restart_data'//trim(mpiiofs)//trim(file_loc) - inquire (FILE=trim(file_loc), EXIST=file_exist) + if (file_per_process) then + if (cfl_dt) then + call s_int_to_str(n_start, t_step_start_string) + write (file_loc, '(I0,A1,I7.7,A)') n_start, '_', proc_rank, '.dat' + else + call s_int_to_str(t_step_start, t_step_start_string) + write (file_loc, '(I0,A1,I7.7,A)') t_step_start, '_', proc_rank, '.dat' + end if + file_loc = trim(case_dir)//'/restart_data/lustre_'//trim(t_step_start_string)//trim(mpiiofs)//trim(file_loc) + inquire (FILE=trim(file_loc), EXIST=file_exist) - if (file_exist) then - call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) + if (file_exist) then + call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) - ! Initialize MPI data I/O + ! Initialize MPI data I/O + if (down_sample) then + call s_initialize_mpi_data_ds(q_cons_vf) + else + if (ib) then + call s_initialize_mpi_data(q_cons_vf, ib_markers) + else + call s_initialize_mpi_data(q_cons_vf) + end if + end if - if (ib) then - call s_initialize_mpi_data(q_cons_vf, ib_markers) - else + if (down_sample) then + ! Size of local arrays + data_size = (m_ds + 3)*(n_ds + 3)*(p_ds + 3) + m_glb_read = m_glb_ds + 1 + n_glb_read = n_glb_ds + 1 + p_glb_read = p_glb_ds + 1 + else + ! Size of local arrays + data_size = (m + 1)*(n + 1)*(p + 1) + m_glb_read = m_glb + 1 + n_glb_read = n_glb + 1 + p_glb_read = p_glb + 1 + end if - call s_initialize_mpi_data(q_cons_vf) + ! Resize some integers so MPI can read even the biggest file + m_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) + n_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) + p_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) + MOK = int(1._wp, MPI_OFFSET_KIND) + str_MOK = int(name_len, MPI_OFFSET_KIND) + NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) + + ! Read the data for each variable + if (bubbles_euler .or. elasticity) then + do i = 1, sys_size!adv_idx%end + var_MOK = int(i, MPI_OFFSET_KIND) + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + !Read pb and mv for non-polytropic qbmm + if (qbmm .and. .not. polytropic) then + do i = sys_size + 1, sys_size + 2*nb*nnode + var_MOK = int(i, MPI_OFFSET_KIND) + + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + end if + else + if (down_sample) then + do i = 1, sys_size + var_MOK = int(i, MPI_OFFSET_KIND) + + call MPI_FILE_READ(ifile, q_cons_temp(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + else + do i = 1, sys_size + var_MOK = int(i, MPI_OFFSET_KIND) + + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + end if + end if + + call s_mpi_barrier() + + call MPI_FILE_CLOSE(ifile, ierr) + + else + call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.') end if + else + ! Open the file to read conservative variables + if (cfl_dt) then + write (file_loc, '(I0,A)') n_start, '.dat' + else + write (file_loc, '(I0,A)') t_step_start, '.dat' + end if + file_loc = trim(case_dir)//'/restart_data'//trim(mpiiofs)//trim(file_loc) + inquire (FILE=trim(file_loc), EXIST=file_exist) - ! Size of local arrays - data_size = (m + 1)*(n + 1)*(p + 1) - - ! Resize some integers so MPI can read even the biggest file - m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) - n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) - p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) - MOK = int(1._wp, MPI_OFFSET_KIND) - str_MOK = int(name_len, MPI_OFFSET_KIND) - NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) - - ! Read the data for each variable - if (bubbles_euler .or. elasticity) then - do i = 1, sys_size !adv_idx%end - var_MOK = int(i, MPI_OFFSET_KIND) - ! Initial displacement to skip at beginning of file - disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) - - call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & - 'native', mpi_info_int, ierr) - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - !Read pb and mv for non-polytropic qbmm - if (qbmm .and. .not. polytropic) then - do i = sys_size + 1, sys_size + 2*nb*nnode + if (file_exist) then + call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr) + + ! Initialize MPI data I/O + + if (ib) then + call s_initialize_mpi_data(q_cons_vf, ib_markers) + else + + call s_initialize_mpi_data(q_cons_vf) + + end if + + ! Size of local arrays + data_size = (m + 1)*(n + 1)*(p + 1) + + ! Resize some integers so MPI can read even the biggest file + m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) + n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) + p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) + MOK = int(1._wp, MPI_OFFSET_KIND) + str_MOK = int(name_len, MPI_OFFSET_KIND) + NVARS_MOK = int(sys_size, MPI_OFFSET_KIND) + + ! Read the data for each variable + if (bubbles_euler .or. elasticity) then + do i = 1, sys_size !adv_idx%end var_MOK = int(i, MPI_OFFSET_KIND) ! Initial displacement to skip at beginning of file disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) @@ -590,27 +669,41 @@ contains call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & mpi_io_p, status, ierr) end do - end if - else - do i = 1, sys_size - var_MOK = int(i, MPI_OFFSET_KIND) + !Read pb and mv for non-polytropic qbmm + if (qbmm .and. .not. polytropic) then + do i = sys_size + 1, sys_size + 2*nb*nnode + var_MOK = int(i, MPI_OFFSET_KIND) + ! Initial displacement to skip at beginning of file + disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) + + call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & + 'native', mpi_info_int, ierr) + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + end if + else + do i = 1, sys_size + var_MOK = int(i, MPI_OFFSET_KIND) - ! Initial displacement to skip at beginning of file - disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) + ! Initial displacement to skip at beginning of file + disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1) - call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & - 'native', mpi_info_int, ierr) - call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & - mpi_io_p, status, ierr) - end do - end if + call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & + 'native', mpi_info_int, ierr) + call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + mpi_io_p, status, ierr) + end do + end if - call s_mpi_barrier() + call s_mpi_barrier() - call MPI_FILE_CLOSE(ifile, ierr) + call MPI_FILE_CLOSE(ifile, ierr) + + else + call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.') + end if - else - call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.') end if deallocate (x_cb_glb, y_cb_glb, z_cb_glb) From 1cdea6b0b9d036859e796b8ef9a7fc56211e797c Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 26 Feb 2026 00:56:52 -0500 Subject: [PATCH 8/8] Fix IB data WP_MOK to use stp instead of default integer The IB read/write displacement formula uses WP_MOK to skip past floating-point field data. Using storage_size(0) gave the default integer size (4 bytes), not the field storage precision size, causing incorrect byte offsets in double-precision builds. Co-Authored-By: Claude Opus 4.6 --- src/post_process/m_data_input.f90 | 2 +- src/simulation/m_data_output.fpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index 933f4274fe..e1dc941682 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -177,7 +177,7 @@ impure subroutine s_read_ib_data_files(file_loc_base, t_step) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) save_index = t_step/t_step_save ! get the number of saves done to this point data_size = (m + 1)*(n + 1)*(p + 1) diff --git a/src/simulation/m_data_output.fpp b/src/simulation/m_data_output.fpp index b1b3c7a1a4..5760e33089 100644 --- a/src/simulation/m_data_output.fpp +++ b/src/simulation/m_data_output.fpp @@ -1087,7 +1087,7 @@ contains m_MOK = int(m_glb + 1, MPI_OFFSET_KIND) n_MOK = int(n_glb + 1, MPI_OFFSET_KIND) p_MOK = int(p_glb + 1, MPI_OFFSET_KIND) - WP_MOK = int(storage_size(0)/8, MPI_OFFSET_KIND) + WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND) MOK = int(1._wp, MPI_OFFSET_KIND) write (file_loc, '(A)') 'ib.dat'