Merge branch 'develop' into feature/mathomp4/v11-cleanup-setup

Author: mathomp4

DynCore_GridCompMod.F90

@@ -1046,6 +1046,15 @@ Subroutine SetServices ( gc, rc )
          VLOCATION  = MAPL_VLocationEdge,               RC=STATUS  )
      VERIFY_(STATUS)
 
+    call MAPL_AddExportSpec ( gc,                                  &
+         SHORT_NAME = 'MZ',                                       &
+         LONG_NAME  = 'vertical_mass_flux',                        &
+         UNITS      = 'kg m-2 s-1',                                &
+         PRECISION  = ESMF_KIND_R4,                                &
+         DIMS       = MAPL_DimsHorzVert,                           &
+         VLOCATION  = MAPL_VLocationEdge,               RC=STATUS  )
+     VERIFY_(STATUS)
+
     call MAPL_AddExportSpec ( gc,                                  &
          SHORT_NAME = 'PV',                                        &
          LONG_NAME  = 'ertels_isentropic_potential_vorticity',     &
@@ -4533,6 +4542,7 @@ subroutine Run(gc, import, export, clock, rc)
 ! Compute and return the vertical mass flux
       call getVerticalMassFlux(mfxxyz, mfyxyz, mfzxyz, dt)
       call FILLOUT3r8 (export, 'MFZ' , mfzxyz , rc=status); VERIFY_(STATUS)
+      call FILLOUT3 (export, 'MZ' , mfzxyz , rc=status); VERIFY_(STATUS)
 
       call FILLOUT3 (export, 'U'      , ur      , rc=status); VERIFY_(STATUS)
       call FILLOUT3 (export, 'V'      , vr      , rc=status); VERIFY_(STATUS)

scripts/fv3_setup

@@ -290,49 +290,30 @@ ASKPROC:
 
 if ( $SITE == 'NCCS' ) then
 
-   set BUILT_ON_SLES15 = @BUILT_ON_SLES15@
-
-   if ("$BUILT_ON_SLES15" == "TRUE") then
-      echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
-      echo "   ${C2}mil  (Milan)${CN} (default)"
-      echo " "
-      set MODEL = `echo $<`
-      set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
-      if ( .$MODEL == .) then
-         set MODEL = 'mil'
-      endif
-
-      if( $MODEL != 'mil' ) goto ASKPROC
-
-      if ($MODEL == 'mil') then
-         # We save a couple processes for the kernel
-         set NCPUS_PER_NODE = 126
-      endif
-   else
-      echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
-      echo "   ${C2}sky  (Skylake)${CN}"
-      echo "   ${C2}cas  (Cascade Lake) (default)${CN}"
-      echo " "
-      set MODEL = `echo $<`
-      set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
-      if ( .$MODEL == .) then
-         set MODEL = 'cas'
-      endif
+   echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
+   echo "   ${C2}mil  (Milan)${CN} (default)"
+   echo "   ${C2}cas  (Cascade Lake)${CN}"
+   echo " "
+   set MODEL = `echo $<`
+   set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
+   if ( .$MODEL == .) then
+      set MODEL = 'mil'
+   endif
 
-      if( $MODEL != 'sky'  & \
-          $MODEL != 'cas' ) goto ASKPROC
-
-      if ($MODEL == 'sky') then
-         set NCPUS_PER_NODE = 40
-      else if ($MODEL == 'cas') then
-         # NCCS currently recommends that users do not run with
-         # 48 cores per node on SCU16 due to OS issues and
-         # recommends that CPU-intensive works run with 46 or less
-         # cores. As 45 is a multiple of 3, it's the best value
-         # that doesn't waste too much
-         #set NCPUS_PER_NODE = 48
-         set NCPUS_PER_NODE = 45
-      endif
+   if( $MODEL != 'mil' & \
+         $MODEL != 'cas' ) goto ASKPROC
+
+   if ($MODEL == 'mil') then
+      # We save a couple processes for the kernel
+      set NCPUS_PER_NODE = 126
+   else if ($MODEL == 'cas') then
+      # NCCS currently recommends that users do not run with
+      # 48 cores per node on SCU16 due to OS issues and
+      # recommends that CPU-intensive works run with 46 or less
+      # cores. As 45 is a multiple of 3, it's the best value
+      # that doesn't waste too much
+      #set NCPUS_PER_NODE = 48
+      set NCPUS_PER_NODE = 45
    endif
 
 else if ( $SITE == 'NAS' ) then
@@ -804,51 +785,13 @@ EOF
 if ( $SITE == 'NCCS' ) then
 
 cat >> $EXPDIR/SETENV.commands << EOF
-setenv I_MPI_SHM_HEAP_VSIZE 512
-setenv PSM2_MEMORY large
-EOF
-
-# Testing at NCCS showed these caused a crash at higher resolutions at
-# restart read
-if ( $USING_SINGULARITY == FALSE ) then
-
-cat >> $EXPDIR/SETENV.commands << EOF
-setenv I_MPI_EXTRA_FILESYSTEM 1
-setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs
-EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
 
-endif # if NOT Singularity
-
-# Testing by Bill Putman found these to be
-# useful flags with Intel MPI on SLES15 on the
-# Milan nodes.
-# Note 1: Testing by NCCS shows the PSM3 provider
-# runs on the Infiniband fabric. Tests show it runs
-# up to C720.
-# Note 2: When the Cascade Lakes are moved to
-# SLES15, these will need to be Milan-only flags
-# as Intel MPI will probably work just fine with
-# Intel chips.
-if ("$BUILT_ON_SLES15" == "TRUE") then
-cat >> $EXPDIR/SETENV.commands << EOF
-setenv I_MPI_FALLBACK 0
-setenv I_MPI_FABRICS ofi
+setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
-setenv I_MPI_ADJUST_SCATTER 2
-setenv I_MPI_ADJUST_SCATTERV 2
-setenv I_MPI_ADJUST_GATHER 2
-setenv I_MPI_ADJUST_GATHERV 3
-setenv I_MPI_ADJUST_ALLGATHER 3
-setenv I_MPI_ADJUST_ALLGATHERV 3
-setenv I_MPI_ADJUST_ALLREDUCE 12
-setenv I_MPI_ADJUST_REDUCE 10
-setenv I_MPI_ADJUST_BCAST 11
-setenv I_MPI_ADJUST_REDUCE_SCATTER 4
-setenv I_MPI_ADJUST_BARRIER 9
 EOF
 
-endif # if SLES15
-
 endif # if NCCS
 
 endif # if mpi