Merge pull request #771 from GEOS-ESM/feature/v11-intelmpi-nas

Author: mathomp4

.github/workflows/push-to-develop.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           filter: blob:none

.github/workflows/spack-ci.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           filter: blob:none

.github/workflows/validate_yaml_files.yml

@@ -17,69 +17,21 @@ concurrency:
 
 jobs:
   build_gcm:
-    name: Build GEOSgcm
-    if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')"
-    runs-on: ubuntu-24.04
-    container:
-      image: gmao/ubuntu24-geos-env:v7.33.0-intelmpi_2021.13-ifort_2021.13
-      # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
-      # It seems like we might not need secrets on GitHub Actions which is good for forked
-      # pull requests
-      #credentials:
-        #username: ${{ secrets.DOCKERHUB_USERNAME }}
-        #password: ${{ secrets.DOCKERHUB_TOKEN }}
+    strategy:
+      matrix:
+        compiler: [ifort, gfortran-14, gfortran-15]
+        build-type: [Debug]
+    uses: GEOS-ESM/CI-workflows/.github/workflows/geosgcm_build_tests.yml@project/geosgcm
+    with:
+      compiler: ${{ matrix.compiler }}
+      cmake-build-type: ${{ matrix.build-type }}
+      fixture-repo: GEOS-ESM/GEOSgcm
+
+  spack_build:
+    uses: GEOS-ESM/CI-workflows/.github/workflows/spack_gcc_build.yml@project/geosgcm
+    secrets:
+      BUILDCACHE_USERNAME: ${{ secrets.BUILDCACHE_USERNAME }}
+      BUILDCACHE_TOKEN: ${{ secrets.BUILDCACHE_TOKEN }}
+    with:
+      fixture-repo: GEOS-ESM/GEOSgcm
 
-    env:
-      OMPI_ALLOW_RUN_AS_ROOT: 1
-      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-      OMPI_MCA_btl_vader_single_copy_mechanism: none
-
-    steps:
-      # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449
-      - name: Delete huge unnecessary tools folder
-        run: rm -rf /opt/hostedtoolcache
-
-      - name: Checkout GCM
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-          filter: blob:none
-          repository: GEOS-ESM/GEOSgcm
-
-      - name: Set all directories as git safe
-        run: |
-          git config --global --add safe.directory '*'
-
-      - name: Versions etc.
-        run: |
-          ifort --version
-          mpirun --version
-          echo $BASEDIR
-
-      - name: Mepo clone external repos
-        run: |
-          mepo clone --partial blobless
-          mepo status
-
-      - name: Mepo develop usual suspects
-        run: |
-          mepo develop GEOSgcm_GridComp GEOSgcm_App GMAO_Shared GEOS_Util
-          mepo status
-
-      - name: Debug PR branch
-        run: echo "PR is coming from ${{ github.event.pull_request.head.ref }}"
-
-      - name: Update other branches
-        if: ${{ github.event.pull_request.head.ref != 'main' && github.event.pull_request.head.ref != 'develop' }}
-        run: |
-          mepo checkout-if-exists ${GITHUB_HEAD_REF}
-          mepo status
-
-      - name: CMake
-        run: |
-          cmake -B build -S . --install-prefix=${pwd}/install -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' -DUSE_F2PY=OFF
-
-      - name: Build
-        run: |
-          cmake --build build -j 4
-          cmake --install build

.github/workflows/workflow.yml

@@ -2258,16 +2258,12 @@ cat > $HOMDIR/SETENV.commands << EOF
 #setenv I_MPI_DEBUG 9
 EOF
 
-# These are options determined to be useful at NCCS
+# These are options determined to be useful at NCCS and NAS
 # Not setting generally as they are more fabric/cluster
-# specific compared to the above adjustments
-if ( $SITE == 'NCCS' ) then
-
-# Some flags we know work on SLES15 and Milan (see below)
-# For safety, we keep the old SLES12 flags for that system
-#
-# NOTE: When Cascade Lake is on SLES15, the following flags
-# might need to be Milan only
+# specific compared to the above adjustments. We protect
+# like this because these settings do *not* work on systems like
+# CI runners
+if ( $SITE == 'NCCS' | $SITE == 'NAS' ) then
 
 cat >> $HOMDIR/SETENV.commands << EOF
 setenv I_MPI_ADJUST_ALLREDUCE 12
@@ -2278,9 +2274,18 @@ setenv I_MPI_OFI_PROVIDER psm3
 
 # This has been found to help with congestion
 setenv FI_PSM3_CONN_TIMEOUT 120
+
+# NOTE: If you are seeing oddities with Intel MPI
+# and a -perhost/-ppn use, you might need to
+# uncomment the below line
+#
+#setenv I_MPI_JOB_RESPECT_PROCESS_PLACEMENT disable
+#
+# This is usually needed when running interactively
+# as SLURM/PBS in batch mode is usually fine with process placement
 EOF
 
-endif # if NCCS
+endif # if NCCS or NAS
 
 endif # if mpi
 

gcm_setup

@@ -2288,16 +2288,12 @@ cat > $HOMDIR/SETENV.commands << EOF
 #setenv I_MPI_DEBUG 9
 EOF
 
-# These are options determined to be useful at NCCS
+# These are options determined to be useful at NCCS and NAS
 # Not setting generally as they are more fabric/cluster
-# specific compared to the above adjustments
-if ( $SITE == 'NCCS' ) then
-
-# Some flags we know work on SLES15 and Milan (see below)
-# For safety, we keep the old SLES12 flags for that system
-#
-# NOTE: When Cascade Lake is on SLES15, the following flags
-# might need to be Milan only
+# specific compared to the above adjustments. We protect
+# like this because these settings do *not* work on systems like
+# CI runners
+if ( $SITE == 'NCCS' | $SITE == 'NAS' ) then
 
 cat >> $HOMDIR/SETENV.commands << EOF
 setenv I_MPI_ADJUST_ALLREDUCE 12
@@ -2308,9 +2304,18 @@ setenv I_MPI_OFI_PROVIDER psm3
 
 # This has been found to help with congestion
 setenv FI_PSM3_CONN_TIMEOUT 120
+
+# NOTE: If you are seeing oddities with Intel MPI
+# and a -perhost/-ppn use, you might need to
+# uncomment the below line
+#
+#setenv I_MPI_JOB_RESPECT_PROCESS_PLACEMENT disable
+#
+# This is usually needed when running interactively
+# as SLURM/PBS in batch mode is usually fine with process placement
 EOF
 
-endif # if NCCS
+endif # if NCCS or NAS
 
 endif # if mpi
 

geoschemchem_setup

@@ -2460,16 +2460,12 @@ cat > $HOMDIR/SETENV.commands << EOF
 #setenv I_MPI_DEBUG 9
 EOF
 
-# These are options determined to be useful at NCCS
+# These are options determined to be useful at NCCS and NAS
 # Not setting generally as they are more fabric/cluster
-# specific compared to the above adjustments
-if ( $SITE == 'NCCS' ) then
-
-# Some flags we know work on SLES15 and Milan (see below)
-# For safety, we keep the old SLES12 flags for that system
-#
-# NOTE: When Cascade Lake is on SLES15, the following flags
-# might need to be Milan only
+# specific compared to the above adjustments. We protect
+# like this because these settings do *not* work on systems like
+# CI runners
+if ( $SITE == 'NCCS' | $SITE == 'NAS' ) then
 
 cat >> $HOMDIR/SETENV.commands << EOF
 setenv I_MPI_ADJUST_ALLREDUCE 12
@@ -2480,9 +2476,18 @@ setenv I_MPI_OFI_PROVIDER psm3
 
 # This has been found to help with congestion
 setenv FI_PSM3_CONN_TIMEOUT 120
+
+# NOTE: If you are seeing oddities with Intel MPI
+# and a -perhost/-ppn use, you might need to
+# uncomment the below line
+#
+#setenv I_MPI_JOB_RESPECT_PROCESS_PLACEMENT disable
+#
+# This is usually needed when running interactively
+# as SLURM/PBS in batch mode is usually fine with process placement
 EOF
 
-endif # if NCCS
+endif # if NCCS or NAS
 
 endif # if mpi
 

gmichem_setup

@@ -2273,16 +2273,12 @@ cat > $HOMDIR/SETENV.commands << EOF
 #setenv I_MPI_DEBUG 9
 EOF
 
-# These are options determined to be useful at NCCS
+# These are options determined to be useful at NCCS and NAS
 # Not setting generally as they are more fabric/cluster
-# specific compared to the above adjustments
-if ( $SITE == 'NCCS' ) then
-
-# Some flags we know work on SLES15 and Milan (see below)
-# For safety, we keep the old SLES12 flags for that system
-#
-# NOTE: When Cascade Lake is on SLES15, the following flags
-# might need to be Milan only
+# specific compared to the above adjustments. We protect
+# like this because these settings do *not* work on systems like
+# CI runners
+if ( $SITE == 'NCCS' | $SITE == 'NAS' ) then
 
 cat >> $HOMDIR/SETENV.commands << EOF
 setenv I_MPI_ADJUST_ALLREDUCE 12
@@ -2293,9 +2289,18 @@ setenv I_MPI_OFI_PROVIDER psm3
 
 # This has been found to help with congestion
 setenv FI_PSM3_CONN_TIMEOUT 120
+
+# NOTE: If you are seeing oddities with Intel MPI
+# and a -perhost/-ppn use, you might need to
+# uncomment the below line
+#
+#setenv I_MPI_JOB_RESPECT_PROCESS_PLACEMENT disable
+#
+# This is usually needed when running interactively
+# as SLURM/PBS in batch mode is usually fine with process placement
 EOF
 
-endif # if NCCS
+endif # if NCCS or NAS
 
 endif # if mpi