Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 26 additions & 83 deletions scripts/fv3_setup
Original file line number Diff line number Diff line change
Expand Up @@ -290,49 +290,30 @@ ASKPROC:

if ( $SITE == 'NCCS' ) then

set BUILT_ON_SLES15 = @BUILT_ON_SLES15@

if ("$BUILT_ON_SLES15" == "TRUE") then
echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
echo " ${C2}mil (Milan)${CN} (default)"
echo " "
set MODEL = `echo $<`
set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
if ( .$MODEL == .) then
set MODEL = 'mil'
endif

if( $MODEL != 'mil' ) goto ASKPROC

if ($MODEL == 'mil') then
# We save a couple processes for the kernel
set NCPUS_PER_NODE = 126
endif
else
echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
echo " ${C2}sky (Skylake)${CN}"
echo " ${C2}cas (Cascade Lake) (default)${CN}"
echo " "
set MODEL = `echo $<`
set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
if ( .$MODEL == .) then
set MODEL = 'cas'
endif
echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
echo " ${C2}mil (Milan)${CN} (default)"
echo " ${C2}cas (Cascade Lake)${CN}"
echo " "
set MODEL = `echo $<`
set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
if ( .$MODEL == .) then
set MODEL = 'mil'
endif

if( $MODEL != 'sky' & \
$MODEL != 'cas' ) goto ASKPROC

if ($MODEL == 'sky') then
set NCPUS_PER_NODE = 40
else if ($MODEL == 'cas') then
# NCCS currently recommends that users do not run with
# 48 cores per node on SCU16 due to OS issues and
# recommends that CPU-intensive works run with 46 or less
# cores. As 45 is a multiple of 3, it's the best value
# that doesn't waste too much
#set NCPUS_PER_NODE = 48
set NCPUS_PER_NODE = 45
endif
if( $MODEL != 'mil' & \
$MODEL != 'cas' ) goto ASKPROC

if ($MODEL == 'mil') then
# We save a couple processes for the kernel
set NCPUS_PER_NODE = 126
else if ($MODEL == 'cas') then
# NCCS currently recommends that users do not run with
# 48 cores per node on SCU16 due to OS issues and
# recommends that CPU-intensive works run with 46 or less
# cores. As 45 is a multiple of 3, it's the best value
# that doesn't waste too much
#set NCPUS_PER_NODE = 48
set NCPUS_PER_NODE = 45
endif

else if ( $SITE == 'NAS' ) then
Expand Down Expand Up @@ -811,51 +792,13 @@ EOF
if ( $SITE == 'NCCS' ) then

cat >> $EXPDIR/SETENV.commands << EOF
setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF

# Testing at NCCS showed these caused a crash at higher resolutions at
# restart read
if ( $USING_SINGULARITY == FALSE ) then

cat >> $EXPDIR/SETENV.commands << EOF
setenv I_MPI_EXTRA_FILESYSTEM 1
setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs
EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $EXPDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS

endif # if mpi
Expand Down
Loading