Test bot PR155 with software-layer PR237

.github/workflows/test_eessi.yml

@@ -40,4 +40,33 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+
+        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
+              echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+              # note, check_missing_installations.sh exits 1 if a package was
+              # missing, which is intepreted as false (exit code based, not
+              # boolean logic), hence when the script exits 0 if no package was
+              # missing it is interpreted as true, thus the test did not capture
+              # the missing package
+              if ./check_missing_installations.sh; then
+                echo "did NOT capture missing package; test FAILED"
+                exit 1
+              else
+                echo "captured missing package; test PASSED"
+                exit 0
+              fi

EESSI-pilot-install-software.sh

@@ -206,6 +206,9 @@ fi
 
 echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."
 
+echo "check rate limits before first -[-include-easyblock]-from-pr"
+./check_rate_limits.sh ALL
+
 # install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up,
 # see https://github.com/EESSI/software-layer/issues/123
 # and https://github.com/easybuilders/easybuild-easyblocks/pull/2557
@@ -415,6 +418,10 @@ $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
 # use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
 # which includes a patch to fix installation on POWER
 $EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
+
+echo "check rate limits after last -[-include-easyblock]-from-pr"
+./check_rate_limits.sh ALL
+
 # ignore failing FlexiBLAS tests when building on POWER;
 # some tests are failing due to a segmentation fault due to "invalid memory reference",
 # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
@@ -430,6 +437,7 @@ $EB SciPy-bundle-2021.05-foss-2021a.eb --robot
 check_exit_code $? "${ok_msg}" "${fail_msg}"
 
 ### add packages here
+$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot
 
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"

bot/build.sh

@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+#
+# script to build the EESSI software layer. Intended use is that it is called
+# by a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files referenced in job.cfg
+
+# stop as soon as something fails
+set -e
+
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
+
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
+echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'"
+
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
+echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
+
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
+echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
+echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
+    export SINGULARITY_CACHEDIR
+fi
+
+echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+# replace any env variable in ${LOCAL_TMP} with its
+#   current value (e.g., a value that is local to the job)
+STORAGE=$(envsubst <<< ${LOCAL_TMP})
+echo "'${STORAGE}'"
+
+# make sure ${STORAGE} exists
+mkdir -p ${STORAGE}
+
+# make sure the base tmp storage is unique
+JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
+# obtain list of modules to be loaded
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
+echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
+
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(cfg_get_value "repository" "container")
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+# load modules if LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/build.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/build.sh: no modules to be loaded"
+fi
+
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
+
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# next script (eessi_container.sh) makes use of it via sourcing init scripts
+# (e.g., init/eessi_defaults or init/minimal_eessi_env)
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
+# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
+# "source init/eessi_defaults" via sourcing init/minimal_eessi_env
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
+echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
+
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
+echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to build step
+declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
+BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
+
+# prepare arguments to install_software_layer.sh (specific to build step)
+GENERIC_OPT=
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+    GENERIC_OPT="--generic"
+fi
+
+# create tmp file for output of build step
+build_outerr=$(mktemp build.outerr.XXXX)
+
+echo "check rate limits before launching build step"
+./check_rate_limits.sh ALL
+
+echo "Executing command to build software:"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
+echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
+                     -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
+
+echo "check rate limits after running build step"
+./check_rate_limits.sh ALL
+
+# prepare directory to store tarball of tmp for tarball step
+TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step
+mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR}
+
+# create tmp file for output of tarball step
+tar_outerr=$(mktemp tar.outerr.XXXX)
+
+# prepare arguments to eessi_container.sh specific to tarball step
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}")
+
+# determine temporary directory to resume from
+BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
+TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
+
+timestamp=$(date +%s)
+# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
+source init/eessi_defaults
+export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+
+# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
+# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
+# TODO should we make this a configurable parameter of eessi_container.sh using
+# /tmp as default?
+TMP_IN_CONTAINER=/tmp
+echo "Executing command to create tarball:"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
+echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
+                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+
+exit 0

check_missing_installations.sh

@@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
 ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+# the above assesses the installed software for each easyconfig provided in
+# the easystack file and then print messages such as
+# `No missing modules!`
+# or
+# `2 out of 3 required modules missing:`
+# depending on the result of the assessment. Hence, we need to check if the
+# output does not contain any line with ` required modules missing:`
+
+grep " required modules missing:" ${eb_missing_out} > /dev/null
+
+# we need to process the result (from finding `No missing modules` to NOT finding
+# ` required modules missing:` and no other error happened)
+#
+# if grep returns 1 (` required modules missing:` was NOT found), we set
+# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
+[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}"

check_rate_limits.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/bash
+ALL=0
+if [[ $# -gt 0 ]]; then
+  ALL=1
+fi
+
+if [[ ! -z ${GITHUB_TOKEN} ]]; then
+  echo "checking limits for GITHUB_TOKEN"
+else
+  echo "checking limits for unauthorized requests"
+fi
+
+echo "now..: $(date)"
+
+if [[ ! -z ${GITHUB_TOKEN} ]]; then
+  echo "reset: $(date -d @$(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "Authorization: Bearer ${GITHUB_TOKEN}"  -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.reset))"
+  echo "left : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "Authorization: Bearer ${GITHUB_TOKEN}"  -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.remaining)"
+  [[ "${ALL}" -eq "1" ]] && echo "all  : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "Authorization: Bearer ${GITHUB_TOKEN}"  -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq)"
+else
+  echo "reset: $(date -d @$(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.reset))"
+  echo "left : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq --jsonargs .rate.remaining)"
+  [[ "${ALL}" -eq "1" ]] && echo "all  : $(curl --silent -L   -H "Accept: application/vnd.github+json"   -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/rate_limit | jq)"
+fi

eessi-2021.12.yml

@@ -67,3 +67,7 @@ software:
          versions: 
            '3.9.1.1': 
              versionsuffix: -dmpar
+  CaDiCaL:
+    toolchains:
+      GCC-9.3.0:
+        versions: ['1.3.0']

eessi_container.sh

@@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 #    https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
 
 display_help() {
-  echo "usage: $0 [OPTIONS] [SCRIPT]"
+  echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]"
   echo " OPTIONS:"
   echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG   - image file or URL defining the container to use"
@@ -77,10 +77,10 @@ display_help() {
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
-  echo "                           MODE==run (run a script) [default: shell]"
+  echo "                           MODE==run (run a script or command) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
   echo "                           repository to use [default: EESSI-pilot via"
-  echo "                          container configuration]"
+  echo "                           default container, see --container]"
   echo "  -u | --resume DIR/TGZ  - resume a previous run from a directory or tarball,"
   echo "                           where DIR points to a previously used tmp directory"
   echo "                           (check for output 'Using DIR as tmp ...' of a previous"
@@ -98,7 +98,9 @@ display_help() {
   echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
-  echo " If value for --mode is 'run', the SCRIPT provided is executed."
+  echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If"
+  echo " arguments to the script/command start with '-' or '--', use the flag terminator"
+  echo " '--' to let eessi_container.sh stop parsing arguments."
 }
 
 # set defaults for command line arguments
@@ -173,6 +175,11 @@ while [[ $# -gt 0 ]]; do
       export https_proxy=${HTTPS_PROXY}
       shift 2
       ;;
+    --)
+      shift
+      POSITIONAL_ARGS+=("$@") # save positional args
+      break
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -185,7 +192,6 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
-
 if [[ ${LIST_REPOS} -eq 1 ]]; then
     echo "Listing available repositories with format 'name [source]':"
     echo "    EESSI-pilot [default]"
@@ -285,7 +291,7 @@ else
     [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-  echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
+  echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')."
 fi
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
@@ -572,7 +578,7 @@ if [[ ! -z ${SAVE} ]]; then
     TGZ=${SAVE}
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
-  echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)