Frontier: move to CCE 21.0.2 / ROCm 7.0.2

src/common/include/omp_macros.fpp

@@ -26,7 +26,10 @@
             #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
                 #:set default_val = 'defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) '
             #:elif MFC_COMPILER == CCE_COMPILER_ID
-                #:set default_val = 'defaultmap(tofrom:aggregate) defaultmap(present:allocatable) defaultmap(present:pointer) '
+                #! CCE 21: ANY defaultmap clause makes an already-resident array read as
+                #! zero inside the target region, so a resident array silently appears
+                #! empty. Emit nothing, as is already done for AMD above.
+                #:set default_val = ''
             #:elif MFC_COMPILER == AMD_COMPILER_ID
                 #:set default_val = ''
             #:else
@@ -179,7 +182,9 @@
     #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
         #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
     #:elif MFC_COMPILER == CCE_COMPILER_ID
-        #:set omp_start_directive = '!$omp target teams distribute parallel do defaultmap(firstprivate:scalar) '
+        #! CCE 21: defaultmap(firstprivate:scalar) overrides an explicit map(to:) on a
+        #! scalar, so an atomic-updated counter reads back its initial value.
+        #:set omp_start_directive = '!$omp target teams distribute parallel do '
     #:elif MFC_COMPILER == AMD_COMPILER_ID
         #:set omp_start_directive = '!$omp target teams distribute parallel do '
     #:else

src/common/include/parallel_macros.fpp

@@ -67,20 +67,25 @@
         #:if not isinstance(function_name, str)
             #:stop "When using cray_noinline, function name must be given and given as a string"
         #:endif
-        #:set cray_noinline_directive = ('!DIR$ NOINLINE ' + function_name).strip('\n')
+        #! INLINENEVER is the Cray spelling; !DIR$ NOINLINE draws ftn-790 "Unknown or
+        #! unsupported compiler directive" and is silently ignored.
+        #:set cray_noinline_directive = ('!DIR$ INLINENEVER ' + function_name).strip('\n')
 #ifdef _CRAYFTN
-#if MFC_OpenACC
+        #! !DIR$ INLINENEVER is independent of the offload directive: on a Cray GPU build the
+        #! routine needs BOTH its acc/omp device-registration directive and the inlining
+        #! control. Emitting only one of them silently dropped every cray_noinline request
+        #! on Cray GPU builds.
+#if defined(MFC_OpenACC)
         $:acc_directive
-#elif MFC_OpenMP
+#elif defined(MFC_OpenMP)
         $:omp_directive
-#else
-        $:cray_noinline_directive
 #endif
+        $:cray_noinline_directive
         #! On non-Cray CPU builds (no _CRAYFTN, no MFC_OpenACC, no MFC_OpenMP), nothing is
-        #! emitted — intentional, since !DIR$ NOINLINE is a Cray-specific directive.
-#elif MFC_OpenACC
+        #! emitted — intentional, since !DIR$ INLINENEVER is a Cray-specific directive.
+#elif defined(MFC_OpenACC)
         $:acc_directive
-#elif MFC_OpenMP
+#elif defined(MFC_OpenMP)
         $:omp_directive
 #endif
     #:elif cray_inline == True
@@ -89,16 +94,17 @@
         #:endif
         #:set cray_directive = ('!DIR$ INLINEALWAYS ' + function_name).strip('\n')
 #ifdef _CRAYFTN
-#if MFC_OpenACC
+        #! Same as cray_noinline above: the offload directive and !DIR$ INLINEALWAYS are
+        #! independent, and a Cray GPU build needs both.
+#if defined(MFC_OpenACC)
         $:acc_directive
-#elif MFC_OpenMP
+#elif defined(MFC_OpenMP)
         $:omp_directive
-#else
-        $:cray_directive
 #endif
-#elif MFC_OpenACC
+        $:cray_directive
+#elif defined(MFC_OpenACC)
         $:acc_directive
-#elif MFC_OpenMP
+#elif defined(MFC_OpenMP)
         $:omp_directive
 #endif
     #:else

src/common/m_variables_conversion.fpp

@@ -94,11 +94,16 @@ contains
 
         ! Chemistry
         real(wp), dimension(1:num_species), intent(in) :: rhoYks
-        real(wp), dimension(1:num_species)             :: Y_rs
-        real(wp)                                       :: E_e
-        real(wp)                                       :: e_Per_Kg, Pdyn_Per_Kg
-        real(wp)                                       :: T_guess
-        integer                                        :: s  !< Generic loop iterator
+        !> Lower bound 0 is a deliberate, unused pad element: it keeps Y_rs(1) off private frame offset 0. Cray ftn builds a flat
+        !! pointer to a private object as zext(offset) with no aperture, so the flat->private cast back (select(p == null, ~0, trunc
+        !! p)) turns offset 0 into 0xFFFFFFFF and the scratch store lands 4 GiB out of bounds. Faults every chemistry case on CCE
+        !! 21/gfx90a. Do not tighten to 1:num_species without re-checking that the device image is free of `s_cselect_b32 sN, 0,
+        !! -1`.
+        real(wp), dimension(0:num_species) :: Y_rs
+        real(wp)                           :: E_e
+        real(wp)                           :: e_Per_Kg, Pdyn_Per_Kg
+        real(wp)                           :: T_guess
+        integer                            :: s  !< Generic loop iterator
         #:if not chemistry
             ! Depending on model_eqns and bubbles_euler, the appropriate procedure for computing pressure is targeted by the
             ! procedure pointer
@@ -134,14 +139,14 @@ contains
             end if
         #:else
             ! Reacting mixture pressure from temperature and species
-            Y_rs(:) = rhoYks(:)/rho
+            Y_rs(1:num_species) = rhoYks(:)/rho
             e_Per_Kg = energy/rho
             Pdyn_Per_Kg = dyn_p/rho
 
             T_guess = T
 
-            call get_temperature(e_Per_Kg - Pdyn_Per_Kg, T_guess, Y_rs, .true., T)
-            call get_pressure(rho, T, Y_rs, pres)
+            call get_temperature(e_Per_Kg - Pdyn_Per_Kg, T_guess, Y_rs(1:num_species), .true., T)
+            call get_pressure(rho, T, Y_rs(1:num_species), pres)
         #:endif
 
     end subroutine s_compute_pressure

src/simulation/m_bubbles_EL.fpp

@@ -633,7 +633,7 @@ contains
         $:GPU_PARALLEL_LOOP(private='[k, myalpha_rho, myalpha, Re, cell, myVapFlux, preterm1, term2, paux, pint, Romega, &
                             & term1_fac, myR_m, mygamma_m, myPb, myMass_n, myMass_v, myR, myV, myBeta_c, myBeta_t, myR0, myPbdot, &
                             & myMvdot, myPinf, aux1, aux2, myCson, myRho, gamma, pi_inf, qv, dmalf, dmntait, dmBtait, &
-                            & dm_bub_adv_src, dm_divu, adap_dt_stop, myPos, myVel]', copy='[adap_dt_stop_sum]',copyin='[stage]')
+                            & dm_bub_adv_src, dm_divu, adap_dt_stop, myPos, myVel]', copy='[adap_dt_stop_sum]',copyin='[stage]',)
         do k = 1, n_el_bubs_loc
             ! Keller-Miksis model