Move Frontier to CCE 21.0.2 / ROCm 7.2.0

src/common/include/omp_macros.fpp

@@ -26,7 +26,24 @@
             #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
                 #:set default_val = 'defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) '
             #:elif MFC_COMPILER == CCE_COMPILER_ID
-                #:set default_val = 'defaultmap(tofrom:aggregate) defaultmap(present:allocatable) defaultmap(present:pointer) '
+                #! Emit no defaultmap at all on CCE. MEASURED on 21.0.2: the presence of
+                #! ANY defaultmap clause makes a resident (declare target + enter data)
+                #! array read as ZERO inside the target region. A 60-line reproducer with
+                #! a pointer component, an allocatable component and a bare module array
+                #! reads all three correctly with a bare directive and gets 0 from all
+                #! three with defaultmap added -- and each of tofrom:aggregate,
+                #! present:allocatable and present:pointer reproduces it on its own.
+                #! This is what made every IBM case fail under -homp: ib_markers read as
+                #! empty, so the ghost-point count came back 0.
+                #! Related: present:aggregate was tried and reverted. It aborts with
+                #! "find_in_present_table failed for 'length(:)'" at m_ib_patches.fpp:145.
+                #! That is NOT an MFC residency bug -- `length` is real(wp) :: length(3),
+                #! explicitly listed in that loop's private() clause, so nothing should
+                #! ever look for it in the present table. Measured on a 20-line standalone:
+                #! an explicitly-private local array passes with no defaultmap and aborts
+                #! the same way with defaultmap(present:aggregate). Same defect as above,
+                #! overriding `private` instead of `map` or present-table residency.
+                #:set default_val = ''
             #:elif MFC_COMPILER == AMD_COMPILER_ID
                 #:set default_val = ''
             #:else
@@ -179,7 +196,12 @@
     #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
         #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
     #:elif MFC_COMPILER == CCE_COMPILER_ID
-        #:set omp_start_directive = '!$omp target teams distribute parallel do defaultmap(firstprivate:scalar) '
+        #! No defaultmap(firstprivate:scalar) here: it only restates the OpenMP 5.0
+        #! default for scalars in a target region, and CCE 21 lets it override an
+        #! explicit map(to:) on a scalar. A scalar that is atomically incremented to
+        #! hand out array slots then becomes per-thread, so every thread gets the
+        #! same index -- this broke every immersed-boundary case under -homp.
+        #:set omp_start_directive = '!$omp target teams distribute parallel do '
     #:elif MFC_COMPILER == AMD_COMPILER_ID
         #:set omp_start_directive = '!$omp target teams distribute parallel do '
     #:else

src/common/include/parallel_macros.fpp

@@ -63,51 +63,26 @@
                                       & extraAccArgs=extraAccArgs)
     #:set omp_directive = OMP_ROUTINE(function_name=function_name, nohost=nohost, extraOmpArgs=extraOmpArgs)
 
-    #:if cray_noinline == True
+    #! The caller asks for a Cray inlining decision per routine, so honour it on every
+    #! Cray build — CPU and GPU alike. The !DIR$ line is a Cray-only directive, and it
+    #! accompanies the offload directive rather than replacing it. Note the named
+    #! never-inline directive is INLINENEVER; !DIR$ NOINLINE is a separate, nameless,
+    #! call-site directive that ftn rejects when given a routine name (ftn-790).
+    #:if cray_inline or cray_noinline
         #:if not isinstance(function_name, str)
-            #:stop "When using cray_noinline, function name must be given and given as a string"
+            #:stop "When using cray_inline/cray_noinline, function name must be given and given as a string"
         #:endif
-        #:set cray_noinline_directive = ('!DIR$ NOINLINE ' + function_name).strip('\n')
+        #:set cray_keyword = 'INLINEALWAYS' if cray_inline else 'INLINENEVER'
+        #:set cray_directive = ('!DIR$ ' + cray_keyword + ' ' + function_name).strip('\n')
 #ifdef _CRAYFTN
-#if MFC_OpenACC
-        $:acc_directive
-#elif MFC_OpenMP
-        $:omp_directive
-#else
-        $:cray_noinline_directive
-#endif
-        #! On non-Cray CPU builds (no _CRAYFTN, no MFC_OpenACC, no MFC_OpenMP), nothing is
-        #! emitted — intentional, since !DIR$ NOINLINE is a Cray-specific directive.
-#elif MFC_OpenACC
-        $:acc_directive
-#elif MFC_OpenMP
-        $:omp_directive
-#endif
-    #:elif cray_inline == True
-        #:if not isinstance(function_name, str)
-            #:stop "When inlining for Cray Compiler, function name must be given and given as a string"
-        #:endif
-        #:set cray_directive = ('!DIR$ INLINEALWAYS ' + function_name).strip('\n')
-#ifdef _CRAYFTN
-#if MFC_OpenACC
-        $:acc_directive
-#elif MFC_OpenMP
-        $:omp_directive
-#else
         $:cray_directive
 #endif
-#elif MFC_OpenACC
-        $:acc_directive
-#elif MFC_OpenMP
-        $:omp_directive
-#endif
-    #:else
+    #:endif
 #if MFC_OpenACC
-        $:acc_directive
+    $:acc_directive
 #elif MFC_OpenMP
-        $:omp_directive
+    $:omp_directive
 #endif
-    #:endif
 #:enddef
 
 ! Declare device-resident data

src/common/m_variables_conversion.fpp

@@ -94,11 +94,16 @@ contains
 
         ! Chemistry
         real(wp), dimension(1:num_species), intent(in) :: rhoYks
-        real(wp), dimension(1:num_species)             :: Y_rs
-        real(wp)                                       :: E_e
-        real(wp)                                       :: e_Per_Kg, Pdyn_Per_Kg
-        real(wp)                                       :: T_guess
-        integer                                        :: s  !< Generic loop iterator
+        !> Lower bound 0 is a deliberate, unused pad element: it keeps Y_rs(1) off private frame offset 0. Cray ftn builds a flat
+        !! pointer to a private object as zext(offset) with no aperture, so the flat->private cast back (select(p == null, ~0, trunc
+        !! p)) turns offset 0 into 0xFFFFFFFF and the scratch store lands 4 GiB out of bounds. Faults every chemistry case on CCE
+        !! 21/gfx90a. Do not tighten to 1:num_species without re-checking that the device image is free of `s_cselect_b32 sN, 0,
+        !! -1`.
+        real(wp), dimension(0:num_species) :: Y_rs
+        real(wp)                           :: E_e
+        real(wp)                           :: e_Per_Kg, Pdyn_Per_Kg
+        real(wp)                           :: T_guess
+        integer                            :: s  !< Generic loop iterator
         #:if not chemistry
             ! Depending on model_eqns and bubbles_euler, the appropriate procedure for computing pressure is targeted by the
             ! procedure pointer
@@ -134,14 +139,14 @@ contains
             end if
         #:else
             ! Reacting mixture pressure from temperature and species
-            Y_rs(:) = rhoYks(:)/rho
+            Y_rs(1:num_species) = rhoYks(:)/rho
             e_Per_Kg = energy/rho
             Pdyn_Per_Kg = dyn_p/rho
 
             T_guess = T
 
-            call get_temperature(e_Per_Kg - Pdyn_Per_Kg, T_guess, Y_rs, .true., T)
-            call get_pressure(rho, T, Y_rs, pres)
+            call get_temperature(e_Per_Kg - Pdyn_Per_Kg, T_guess, Y_rs(1:num_species), .true., T)
+            call get_pressure(rho, T, Y_rs(1:num_species), pres)
         #:endif
 
     end subroutine s_compute_pressure

src/simulation/m_ibm.fpp

@@ -144,7 +144,7 @@ contains
 
         type(scalar_field), dimension(sys_size), intent(inout) :: q_cons_vf  !< Primitive Variables
         type(scalar_field), dimension(sys_size), intent(inout) :: q_prim_vf  !< Primitive Variables
-        real(stp), dimension(idwbuff(1)%beg:,idwbuff(2)%beg:,idwbuff(3)%beg:,1:,1:), optional, intent(inout) :: pb_in, mv_in
+        real(stp), dimension(idwbuff(1)%beg:,idwbuff(2)%beg:,idwbuff(3)%beg:,1:,1:), intent(inout) :: pb_in, mv_in
         integer :: i, j, k, l, q, r                                          !< Iterator variables
         integer :: patch_id, patch_id_temp                                   !< Patch ID of ghost point
         real(wp) :: rho, gamma, pi_inf, dyn_pres                             !< Mixture variables

src/simulation/m_riemann_solver_lf.fpp

@@ -505,7 +505,13 @@ contains
                         idx_right_phys(1) = j
                         idx_right_phys(2) = k
                         idx_right_phys(3) = l
-                        idx_right_phys(norm_dir) = idx_right_phys(norm_dir) + 1
+                        if (norm_dir == 1) then
+                            idx_right_phys(1) = j + 1
+                        else if (norm_dir == 2) then
+                            idx_right_phys(2) = k + 1
+                        else
+                            idx_right_phys(3) = l + 1
+                        end if
 
                         if (norm_dir == 1) then
                             $:GPU_LOOP(parallelism='[seq]')

src/simulation/m_riemann_state.fpp

@@ -613,7 +613,13 @@ contains
                 do j = ix%beg, ix%end
                     ! Determine indices for the 'right' state for averaging across the interface
                     idx_rp = [j, k, l]
-                    idx_rp(norm_dir) = idx_rp(norm_dir) + 1
+                    if (norm_dir == 1) then
+                        idx_rp(1) = j + 1
+                    else if (norm_dir == 2) then
+                        idx_rp(2) = k + 1
+                    else
+                        idx_rp(3) = l + 1
+                    end if
 
                     ! Average velocities and their derivatives at the interface For cylindrical: x-dir ~ axial (z_cyl), y-dir ~
                     ! radial (r_cyl), z-dir ~ azimuthal (theta_cyl)
@@ -828,7 +834,13 @@ contains
                     idx_right_phys(1) = j_loop
                     idx_right_phys(2) = k_loop
                     idx_right_phys(3) = l_loop
-                    idx_right_phys(norm_dir) = idx_right_phys(norm_dir) + 1
+                    if (norm_dir == 1) then
+                        idx_right_phys(1) = j_loop + 1
+                    else if (norm_dir == 2) then
+                        idx_right_phys(2) = k_loop + 1
+                    else
+                        idx_right_phys(3) = l_loop + 1
+                    end if
 
                     vel_grad_avg = 0.0_wp
                     do vel_comp_idx = 1, num_dims

src/simulation/m_time_steppers.fpp

@@ -550,11 +550,10 @@ contains
                 end if
 
                 ! update the ghost fluid properties point values based on IB state
-                if (qbmm .and. .not. polytropic) then
-                    call s_ibm_correct_state(q_cons_ts(1)%vf, q_prim_vf, pb_ts(1)%sf, mv_ts(1)%sf)
-                else
-                    call s_ibm_correct_state(q_cons_ts(1)%vf, q_prim_vf)
-                end if
+                ! pb_ts/mv_ts are always allocated (zero-sized when unused), so pass
+                ! them unconditionally: an absent optional array reaches the OpenACC
+                ! data environment as a null dope vector, which CCE 21 dereferences.
+                call s_ibm_correct_state(q_cons_ts(1)%vf, q_prim_vf, pb_ts(1)%sf, mv_ts(1)%sf)
             end if
 
             ! Grind: minimum wall-clock time of a full RK stage (compute + halo H2D/D2H +

toolchain/mfc/build.py

@@ -367,9 +367,21 @@ def get_install_binpath(self, case: Case) -> str:
         return os.sep.join([self.get_install_dirpath(case), "bin", self.name])
 
     def is_configured(self, case: Case) -> bool:
-        # We assume that if the CMakeCache.txt file exists, then the target is
-        # configured. (this isn't perfect, but it's good enough for now)
-        return os.path.isfile(os.sep.join([self.get_staging_dirpath(case), "CMakeCache.txt"]))
+        # CMake writes CMakeCache.txt before it generates the build system, so the
+        # cache alone does not mean the target is ready to build: a configure that
+        # was interrupted leaves the cache behind with no Makefile, and skipping
+        # configure on that basis fails later with "No rule to make target".
+        # Require the generator's build file too.
+        staging_dirpath = self.get_staging_dirpath(case)
+        if not os.path.isfile(os.sep.join([staging_dirpath, "CMakeCache.txt"])):
+            return False
+
+        return any(os.path.isfile(os.sep.join([staging_dirpath, f])) for f in ("build.ninja", "Makefile"))
+
+    def is_installed(self, case: Case) -> bool:
+        # CMake writes install_manifest.txt only after a successful install, so
+        # unlike CMakeCache.txt it is not left behind by a build that crashed.
+        return os.path.isfile(os.sep.join([self.get_staging_dirpath(case), "install_manifest.txt"]))
 
     def get_configuration_txt(self, case: Case) -> typing.Optional[dict]:
         if not self.is_configured(case):
@@ -593,10 +605,12 @@ def __build_target(target: typing.Union[MFCTarget, str], case: input.MFCInputFil
 
     history.add(target.name)
 
-    # Dependencies are pinned to fixed versions. If already configured
-    # (built & installed by a prior --deps-only step), skip entirely
-    # to avoid re-entering the superbuild (which may access the network).
-    if target.isDependency and target.is_configured(case):
+    # Dependencies are pinned to fixed versions, so skip one that a prior
+    # --deps-only step already installed, avoiding a re-entry into the superbuild
+    # (which may access the network). Gate on the install, not on the configure:
+    # a dependency build that crashed leaves CMakeCache.txt behind, and skipping
+    # on that would point CMAKE_PREFIX_PATH at an empty install tree forever.
+    if target.isDependency and target.is_installed(case):
         return
 
     for dep in target.requires.compute():

toolchain/modules

@@ -49,9 +49,29 @@ pifx-cpu MPICC=mpiicc MPICXX=mpiicpc MPIFC=mpiifort
 pifx-cpu I_MPI_CC=icx I_MPI_CXX=icpx I_MPI_F90=ifx I_MPI_F77=ifx
 
 f     OLCF Frontier
-f-all cpe/25.03 rocm/6.3.1
+# cpe/26.03 defaults to cce/21.0.0; pin 21.0.2. rocprofiler-compute is not listed:
+# its modulefile reloads darshan-runtime, which cpe/26.03 deactivates, and rocm/7
+# already provides rocprof, rocprof-compute, and rocprofv3.
+f-all cpe/26.03 cce/21.0.2 rocm/7.2.0
 f-all cray-fftw cray-hdf5 python cmake
-f-gpu python craype-accel-amd-gfx90a rocprofiler-compute/3.0.0
+f-gpu craype-accel-amd-gfx90a
+# Two CCE 21 device-codegen workarounds. Keep the value quoted: modules.sh runs
+# eval "export $line", so an unquoted multi-word value exports only the first
+# -plugin-opt and reports the rest to stderr, which is routinely discarded.
+#
+# -mattr=-mai-insts: CCE 21 aborts linking simulation in the AMDGPU
+#   "Rewrite AGPR-Copy-MFMA" pass (SlotIndexes.h "Attempt to compare reserved
+#   index") on the s_compute_bubble_EE_source kernel. Dropping the MAI subtarget
+#   feature makes the pass bail out. Not codegen-neutral -- it costs the AGPR half
+#   of the unified register file, so pressure moves into scratch.
+#
+# -disable-promote-alloca-to-vector: AMDGPUPromoteAllocaToVector silently DISCARDS
+#   a dynamically-indexed store into a 1-based local array. For arr(v) CCE 21
+#   canonicalizes the element index as v + 0x3FFFFFFF, which is harmless in
+#   wrapping address arithmetic but not once the array is registerized and the
+#   index becomes an equality comparison. Wrong answers, no diagnostic. The three
+#   sites we found are also fixed in source; this covers the ones we have not.
+f-gpu CRAY_CCE_LLD_ARGS="-plugin-opt=-mattr=-mai-insts -plugin-opt=-disable-promote-alloca-to-vector"
 
 famd    OLCF Frontier AMD
 famd-all python cmake