From 447bb2d0da7f6b41a149344eb055ed16ca188663 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 21:59:41 +0100 Subject: [PATCH 01/57] added a function to obtain full path to a tool --- utils.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/utils.sh b/utils.sh index 5d8455bb68..cd7f758a0d 100644 --- a/utils.sh +++ b/utils.sh @@ -31,3 +31,35 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which ${tool_name} 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} From e2c24cf757b912aef167364aa34d0abf723b7b03 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 22:01:13 +0100 Subject: [PATCH 02/57] partial bot/build.sh for eessi_container.sh --- bot/build.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 bot/build.sh diff --git a/bot/build.sh b/bot/build.sh new file mode 100755 index 0000000000..90753db8a8 --- /dev/null +++ b/bot/build.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# script to build the EESSI software layer. Intended use is that it is called +# by a (batch) job running on a compute node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files references in job.cfg +# - the tool 'yq' for working with json files is available via the PATH or +# the environment variable BOT_YQ (see https://github.com/mikefarah/yq) + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# source utils.sh +source utils.sh + +# check setup / define key variables +# get path for 'yq' (if not found, an empty string is returned) +YQ=$(get_path_for_tool "yq" "BOT_YQ") +exit_code=$? +if [[ ${exit_code} -ne 0 ]]; then + fatal_error "could not find path to 'yq'; exiting" +else + echo_green "found yq (${YQ})" +fi + +# check if './cfg/job.cfg' exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "obtaining configuration settings from '${JOB_CFG_FILE}'" + +LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE}) +echo "LOCAL_TMP='${LOCAL_TMP}'" +echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +storage=$(envsubst <<< ${LOCAL_TMP}) +echo "'${storage}'" + + +# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND +LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE}) +echo "LOAD_MODULES='${LOAD_MODULES}'" + +export SINGULARITY_HOME="$(pwd):/eessi_bot_job" +export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +if [[ ${storage} != /tmp* ]] ; +then + export SINGULARITY_BIND="${storage}:/tmp" +fi +echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" + +# load modules LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/build.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/build.sh: no modules to be loaded" +fi + From a71f8e4f79d53f5be848d5f7064ce3569c4e0477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:37:55 +0100 Subject: [PATCH 03/57] added function to determine IPv4 address --- utils.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils.sh b/utils.sh index cd7f758a0d..db533adc12 100644 --- a/utils.sh +++ b/utils.sh @@ -63,3 +63,12 @@ function get_path_for_tool { fi fi } + +function get_ipv4_address { + hname=$1 + hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +} From b5bf008e4136b7877e06ca0602b6fd6aa49128c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:39:53 +0100 Subject: [PATCH 04/57] various updates for bot/build.sh and eessi_container.sh --- bot/build.sh | 69 ++++++++++++++++++++++---- eessi_container.sh | 117 ++++++++++++++++++++++++--------------------- 2 files changed, 121 insertions(+), 65 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 90753db8a8..e7cc7983bb 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -42,30 +42,37 @@ if [[ ! -r "${JOB_CFG_FILE}" ]]; then fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" -LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE}) +# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy +HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) +echo "HTTP_PROXY='${HTTP_PROXY}'" + +LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" -echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> " +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) -storage=$(envsubst <<< ${LOCAL_TMP}) -echo "'${storage}'" - +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" -# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND -LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE}) +# obtain list of modules to be loaded +LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) echo "LOAD_MODULES='${LOAD_MODULES}'" +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE}) export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -if [[ ${storage} != /tmp* ]] ; +if [[ ${STORAGE} != /tmp* ]] ; then - export SINGULARITY_BIND="${storage}:/tmp" + export SINGULARITY_BIND="${STORAGE}:/tmp" fi echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" -# load modules LOAD_MODULES is not empty +# load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') do @@ -76,3 +83,45 @@ else echo "bot/build.sh: no modules to be loaded" fi +# determine repository to be used from entry .repository in cfg/job.cfg +REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE}) + +# determine architecture to be used from entry .architecture in cfg/job.cfg +# default: leave empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) + +source init/minimal_eessi_env + +# TODO +# - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh +# - add handling of http(s)_proxy to eessi_container.sh, in there needs the +# CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust +# way to determine the IP address of a proxy) +# - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying +# files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg') + +# prepare options and directories for calling eessi_container.sh +mkdir -p previous_tmp +run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX) +CONTAINER_OPT= +if [[ ! -z ${CONTAINER} ]]; then + CONTAINER_OPT="--container ${CONTAINER}" +fi +HTTP_PROXY_OPT= +if [[ ! -z ${HTTP_PROXY} ]]; then + HTTP_PROXY_OPT="--container ${HTTP_PROXY}" +fi +REPOSITORY_OPT= +if [[ ! -z ${REPOSITORY} ]]; then + REPOSITORY_OPT="--repository ${REPOSITORY}" +fi +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --save $(pwd)/previous_tmp \ + --storage ${STORAGE} \ + ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr) diff --git a/eessi_container.sh b/eessi_container.sh index 57b003368b..0016c8c9f0 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings @@ -59,43 +59,43 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" display_help() { echo "usage: $0 [OPTIONS] [SCRIPT]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -h | --help - display this usage information [default: false]" - echo " -g | --storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" - echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," - echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp ...' of a previous" - echo " run) and TGZ is the path to a tarball which is" - echo " unpacked the tmp dir stored on the local storage space" - echo " (see option --storage above) [default: not set]" - echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" - echo " directory DIR or provided with the fixed full path TGZ" - echo " when a directory is provided, the format of the" - echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" - echo " [default: not set]" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --info - display configuration information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]; uses env var \$http_proxy if set" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]; uses env var \$https_proxy if set" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo echo " FEATURES/OPTIONS to be implemented:" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" - echo " -i | --info - display configured repositories [default: false]" - #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - #echo " [default: not set]" - #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - #echo " [default: not set]" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" } # set defaults for command line arguments ACCESS="ro" -CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= @@ -103,8 +103,8 @@ MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -#HTTP_PROXY= -#HTTPS_PROXY= +HTTP_PROXY=${http_proxy:-} +HTTPS_PROXY=${https_proxy:-} RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; -# -x|--http-proxy) -# HTTP_PROXY="$2" -# export http_proxy=${HTTP_PROXY} -# shift 2 -# ;; -# -y|--https-proxy) -# HTTPS_PROXY="$2" -# export https_proxy=${HTTPS_PROXY} -# shift 2 -# ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -281,26 +281,26 @@ fi # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR='${EESSI_TMPDIR}'" # configure Singularity export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB='${EESSI_CVMFS_VAR_LIB}'" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN='${EESSI_CVMFS_VAR_RUN}'" # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" + [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) @@ -308,9 +308,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" -# set up repository config (always create cfg dir and populate it with info when +# set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) -mkdir -p ${EESSI_TMPDIR}/cfg +mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${base_dir}/init/eessi_defaults @@ -367,13 +367,13 @@ else # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" @@ -397,8 +397,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -427,7 +427,14 @@ if [[ -z ${SINGULARITY_BIND} ]]; then else export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" fi -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" + +# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) + export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" From b5c07eeb99e77191d818e64788b0d419031cf526 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 27 Jan 2023 20:30:58 +0100 Subject: [PATCH 05/57] provide CVMFS_HTTP_PROXY if necessary --- bot/build.sh | 3 ++- eessi_container.sh | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index e7cc7983bb..a4f54cba3c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -95,7 +95,8 @@ source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# - add handling of http(s)_proxy to eessi_container.sh, in there needs the +# TODO ensure that the bot makes use of that. +# - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) # - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying diff --git a/eessi_container.sh b/eessi_container.sh index 0016c8c9f0..c2b4988162 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -19,8 +19,8 @@ # 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize host storage/tmp from previous run if provided -# 6. run container +# 5. run container +# 6. save tmp (if requested) # -. initial settings & exit codes base_dir=$(dirname $(realpath $0)) @@ -276,7 +276,7 @@ fi # |-overlay-upper # |-overlay-work # |-home -# |-cfg +# |-repos_cfg # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} @@ -358,9 +358,9 @@ else cfg_init_file_map "${config_map}" [[ ${INFO} -eq 1 ]] && cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg if [[ ! -r ${config_bundle} ]]; then fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi @@ -381,6 +381,26 @@ else source ${base_dir}/init/eessi_defaults fi +# if http_proxy is not empty, we assume that the machine accesses internet +# via a proxy. then we need to add CVMFS_HTTP_PROXY to +# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# MOUNT if it was not yet in BIND_PATHS) +if [[ ! -z ${http_proxy} ]]; then + # TODO tolerate other formats for proxy URLs, for now assume format is + # http://SOME_HOSTNAME:SOME_PORT + PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// + PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port + HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} + HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ + "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + >> ${EESSI_TMPDIR}/repos_cfg/default.local + cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS + if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + fi +fi # 4. set up vars and dirs specific to a scenario @@ -417,10 +437,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize host storage/tmp from previous run if provided - - -# 6. run container +# 5. run container # final settings if [[ -z ${SINGULARITY_BIND} ]]; then export SINGULARITY_BIND="${BIND_PATHS}" @@ -442,7 +459,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ # provided to the script singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} -# 7. save tmp if requested (arg -s|--save) +# 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then # Note, for now we don't try to be smart and record in any way the OS and # ARCH which might have been used internally, eg, when software packages From 4ff37218793559a6e3a8ea9cef58198e33b7dcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:41:36 +0100 Subject: [PATCH 06/57] add functions to derive hostname and port number from proxy URL --- utils.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/utils.sh b/utils.sh index db533adc12..fb4eff1e3f 100644 --- a/utils.sh +++ b/utils.sh @@ -64,6 +64,30 @@ function get_path_for_tool { fi } +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + function get_ipv4_address { hname=$1 hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) From 7a58fbff63712d81dc7dd3206ae2d35535c58b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:43:27 +0100 Subject: [PATCH 07/57] add function to determine config sections --- cfg_files.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index 885ebd0877..ffd29db50a 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -47,9 +47,7 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) - #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space @@ -69,6 +67,20 @@ function cfg_print { done } +function cfg_sections { + declare -A sections + for key in "${!cfg_repos[@]}" + do + # extract section from the associative key + section=$(echo $key | cut -f1 -d ' ') + sections[${section}]=1 + done + for repo in "${!sections[@]}" + do + echo "${repo}" + done +} + function cfg_get_value { section=$1 key=$2 From a5ea7d8aea2c92380839ecbf290c28d278c43835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:44:06 +0100 Subject: [PATCH 08/57] add parameter to list repositories, fix bugs fixed bugs - determining IP address for proxy host was wrong - handling of positional parameters for executing commands with arguments was wrong --- eessi_container.sh | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index c2b4988162..d8d7cb2bdb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -66,6 +66,7 @@ display_help() { echo " -i | --info - display configuration information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" @@ -99,13 +100,13 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= +LIST_REPOS=0 MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= HTTP_PROXY=${http_proxy:-} HTTPS_PROXY=${https_proxy:-} -RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -135,6 +136,10 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; + -l|--list-repos) + LIST_REPOS=1 + shift 1 + ;; -m|--mode) MODE="$2" shift 2 @@ -173,6 +178,17 @@ done set -- "${POSITIONAL_ARGS[@]}" +if [[ ${LIST_REPOS} -eq 1 ]]; then + echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':" + echo " EESSI-pilot [default]" + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id}" + done <<< "${sections}" + exit 0 +fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported @@ -214,8 +230,6 @@ fi if [[ "${MODE}" == "run" ]]; then if [[ $# -eq 0 ]]; then fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}" - else - RUN_SCRIPT_AND_ARGS=$@ fi fi @@ -387,17 +401,20 @@ fi # MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is - # http://SOME_HOSTNAME:SOME_PORT - PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// - PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port - HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} - HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) - echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ - "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + # http://SOME_HOSTNAME:SOME_PORT/ + [[ ${INFO} -eq 1 ]] && echo "http_proxy='${http_proxy}'" + PROXY_HOST=$(get_host_from_url ${http_proxy}) + [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" + PROXY_PORT=$(get_port_from_url ${http_proxy}) + [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi @@ -454,10 +471,10 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From 6ed3d85ff5f11bca37f838c533320733c3ddaf08 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 14:27:26 +0100 Subject: [PATCH 09/57] fix bash pattern matching operator --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index c2b4988162..0cab47e8fb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -397,7 +397,7 @@ if [[ ! -z ${http_proxy} ]]; then >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi From fd5cc3d339bffd7db248c43be909d195db8a64bd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 14:29:25 +0100 Subject: [PATCH 10/57] various updates to bot/build.sh - added handling of HTTPS_PROXY - moved CONTAINER setting from site_config to repository section (different repositories could require different containers) - added handling of CPU_TARGET and EESSI_SOFTWARE_SUBDIR_OVERRIDE - updated TODOs - added setting of EESSI_REPOS_CFG_FILE_OVERRIDE - bug fixes --- bot/build.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a4f54cba3c..3d06477a15 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -46,6 +46,10 @@ echo "obtaining configuration settings from '${JOB_CFG_FILE}'" HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) echo "HTTP_PROXY='${HTTP_PROXY}'" +# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy +HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE}) +echo "HTTPS_PROXY='${HTTPS_PROXY}'" + LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided @@ -61,7 +65,7 @@ LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) echo "LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND -CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE}) +CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE}) export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} @@ -89,13 +93,19 @@ EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos. # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used -EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) +if [[ ! -z "${CPU_TARGET}" ]]; then + EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} +else + EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) +fi source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# TODO ensure that the bot makes use of that. +# TODO ensure that the bot makes use of that. (currently sets env var +# CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also +# add it to cfg/job.cfg - .architecture.software_subdir) # - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) @@ -111,18 +121,25 @@ if [[ ! -z ${CONTAINER} ]]; then fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then - HTTP_PROXY_OPT="--container ${HTTP_PROXY}" + HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" +fi +HTTPS_PROXY_OPT= +if [[ ! -z ${HTTPS_PROXY} ]]; then + HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg +export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ --info \ --mode run \ ${REPOSITORY_OPT} \ --save $(pwd)/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr) + ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr} From b2eab080cba313c72e2c72f91429242b1bf04990 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:10:40 +0100 Subject: [PATCH 11/57] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE --- bot/build.sh | 8 ++++---- eessi_container.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 3d06477a15..7de3594d41 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -89,7 +89,7 @@ fi # determine repository to be used from entry .repository in cfg/job.cfg REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE}) # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used @@ -131,8 +131,8 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg -export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg +# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg +export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -142,4 +142,4 @@ export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg ${REPOSITORY_OPT} \ --save $(pwd)/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr} + ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr} diff --git a/eessi_container.sh b/eessi_container.sh index d8d7cb2bdb..4ca494e136 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -47,8 +47,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" # repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" # 0. parse args From 7e200a185782727ad61f7c1e78f4b52228300a4d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:56:41 +0100 Subject: [PATCH 12/57] fix whitespace issues --- cfg_files.sh | 1 - eessi_container.sh | 2 +- init/eessi_defaults | 1 - utils.sh | 4 ++-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index ffd29db50a..57ea2f7c03 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -165,4 +165,3 @@ function cfg_print_map { echo "${index} --> ${cfg_file_map[${index}]}" done } - diff --git a/eessi_container.sh b/eessi_container.sh index 4ca494e136..3a9fac2955 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -407,7 +407,7 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" PROXY_PORT=$(get_port_from_url ${http_proxy}) [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" - HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local diff --git a/init/eessi_defaults b/init/eessi_defaults index 1b5ce07fb1..f482cbc269 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -10,4 +10,3 @@ export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" - diff --git a/utils.sh b/utils.sh index fb4eff1e3f..d0da95e87f 100644 --- a/utils.sh +++ b/utils.sh @@ -73,7 +73,7 @@ function get_host_from_url { else echo "" return 1 - fi + fi } function get_port_from_url { @@ -85,7 +85,7 @@ function get_port_from_url { else echo "" return 1 - fi + fi } function get_ipv4_address { From 57b6f27191ffce26c05065b1ce2a7e0841309e4d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 17:48:52 +0100 Subject: [PATCH 13/57] fix handling of HTTP(S)_PROXY settings --- bot/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 7de3594d41..d82048e86f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -43,11 +43,13 @@ fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy -HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) +HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE}) +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} echo "HTTP_PROXY='${HTTP_PROXY}'" # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy -HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE}) +HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE}) +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} echo "HTTPS_PROXY='${HTTPS_PROXY}'" LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) From fce504fb466be005f1d417e96e47f442c8f29c52 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 11:54:47 +0100 Subject: [PATCH 14/57] include changes made to test PR for building with the bot --- bot/build.sh | 87 ++++++++++++++++++++++++++++++++------ eessi_container.sh | 50 +++++++++++++++------- run_in_compat_layer_env.sh | 15 ++++++- 3 files changed, 121 insertions(+), 31 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index d82048e86f..8565511bfa 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -72,12 +72,6 @@ export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -if [[ ${STORAGE} != /tmp* ]] ; -then - export SINGULARITY_BIND="${STORAGE}:/tmp" -fi -echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" - # load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') @@ -91,7 +85,20 @@ fi # determine repository to be used from entry .repository in cfg/job.cfg REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} + +# determine pilot version to be used from .repository.repo_version in cfg/job.cfg +# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# next script (eessi_container.sh) makes use of it via sourcing init scripts +# (e.g., init/eessi_defaults or init/minimal_eessi_env) +export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE}) + +# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg +# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down +# "source init/eessi_defaults" via sourcing init/minimal_eessi_env +export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE}) + # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used @@ -100,8 +107,11 @@ if [[ ! -z "${CPU_TARGET}" ]]; then else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi +export EESSI_SOFTWARE_SUBDIR_OVERRIDE -source init/minimal_eessi_env +# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) +EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE}) +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh @@ -112,11 +122,9 @@ source init/minimal_eessi_env # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) # - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying -# files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg') +# files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') # prepare options and directories for calling eessi_container.sh -mkdir -p previous_tmp -run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX) CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" @@ -133,8 +141,21 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +mkdir -p previous_tmp +build_outerr=$(mktemp build.outerr.XXXX) +echo "Executing command to build software:" +echo "./eessi_container.sh --access rw" +echo " ${CONTAINER_OPT}" +echo " ${HTTP_PROXY_OPT}" +echo " ${HTTPS_PROXY_OPT}" +echo " --info" +echo " --mode run" +echo " ${REPOSITORY_OPT}" +echo " --save ${PWD}/previous_tmp" +echo " --storage ${STORAGE}" +echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg -export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg +export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -142,6 +163,44 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg --info \ --mode run \ ${REPOSITORY_OPT} \ - --save $(pwd)/previous_tmp \ + --save ${PWD}/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr} + ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} + +# determine temporary directory to resume from +BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") + +tar_outerr=$(mktemp tar.outerr.XXXX) +timestamp=$(date +%s) +# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +source init/eessi_defaults +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) + +# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be +# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh +# TODO should we make this a configurable parameter of eessi_container.sh using +# /tmp as default? +TMP_IN_CONTAINER=/tmp +echo "Executing command to create tarball:" +echo "./eessi_container.sh --access rw" +echo " ${CONTAINER_OPT}" +echo " ${HTTP_PROXY_OPT}" +echo " ${HTTPS_PROXY_OPT}" +echo " --info" +echo " --mode run" +echo " ${REPOSITORY_OPT}" +echo " --resume ${BUILD_TMPDIR}" +echo " --save ${PWD}/previous_tmp" +echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --resume ${BUILD_TMPDIR} \ + --save ${PWD}/previous_tmp \ + ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + +exit 0 diff --git a/eessi_container.sh b/eessi_container.sh index 3a9fac2955..1eb630a096 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -46,9 +46,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" +# directory for tmp used inside container +export TMP_IN_CONTAINER=/tmp + # repository cfg file, default name (default location: $PWD) # can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" +# other repository cfg files in directory, default location: $PWD +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" # 0. parse args @@ -193,7 +199,7 @@ fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then - fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi # TODO (arg -c|--container) check container (is it a file or URL & access those) @@ -205,11 +211,14 @@ fi # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then - fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then + fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +fi # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -243,6 +252,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then # skip creating a new tmp directory, just set environment variables echo "Resuming from previous run using temporary storage at ${RESUME}" EESSI_HOST_STORAGE=${RESUME} + echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" else # we need a tmp location (and possibly init it with ${RESUME} if it was not # a directory @@ -319,7 +329,9 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +# provide a '/tmp' inside the container +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" + [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # set up repository config (always create directory repos_cfg and populate it with info when @@ -375,13 +387,22 @@ else # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, # define BIND mounts and override repo name and version # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - if [[ ! -r ${config_bundle} ]]; then - fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} + else + config_bundle_path=${config_bundle} + fi + + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" @@ -430,12 +451,8 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -444,8 +461,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" export EESSI_PILOT_WRITABLE_OVERLAY @@ -470,11 +487,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +# if INFO is set to 1 (arg --info), add argument '-q' +RUN_QUIET=${INFO:--q} echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then @@ -493,6 +512,7 @@ if [[ ! -z ${SAVE} ]]; then fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "RESUME_FROM_TGZ ${TGZ}" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index e4850d677d..c70077bf15 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -3,7 +3,6 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10" if [ -z $EESSI_PILOT_VERSION ]; then echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 exit 1 @@ -18,6 +17,18 @@ INPUT=$(echo "$@") if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then + INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${http_proxy} ]; then + INPUT="export http_proxy=${http_proxy}; ${INPUT}" +fi +if [ ! -z ${https_proxy} ]; then + INPUT="export https_proxy=${https_proxy}; ${INPUT}" +fi -echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 712c40e567842da1c50b8c196fad0c9c5d30a69a Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 22:54:43 +0100 Subject: [PATCH 15/57] improvements to resuming job environment cherry-picked via 23e773ce0d8c6504e1440629a4178318265b73d2 Note only applied part for bot/build.sh --- bot/build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8565511bfa..2c15b904f3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -mkdir -p previous_tmp +mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh --access rw" @@ -151,7 +151,7 @@ echo " ${HTTPS_PROXY_OPT}" echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" -echo " --save ${PWD}/previous_tmp" +echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg @@ -163,7 +163,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg --info \ --mode run \ ${REPOSITORY_OPT} \ - --save ${PWD}/previous_tmp \ + --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} @@ -190,7 +190,7 @@ echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" -echo " --save ${PWD}/previous_tmp" +echo " --save ${PWD}/previous_tmp/tarball_step" echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ @@ -200,7 +200,7 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT --mode run \ ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ - --save ${PWD}/previous_tmp \ + --save ${PWD}/previous_tmp/tarball_step \ ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From 16d98572d4148049a3a9f6a4258467eaf785d9f4 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:00:42 +0100 Subject: [PATCH 16/57] improved handling of container cache + explicit pull of image cherry-picked via bfb1b29103dc309a12573f3ec9c247d432f08f73 Note, only applied part for bot/build.sh --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 2c15b904f3..06503e59a0 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -56,6 +56,12 @@ LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided +SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE}) +echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR +fi + echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) From 9e8ca62cab90b013c6fb1b55180e37592aea02b5 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:02:35 +0100 Subject: [PATCH 17/57] changed --info -> --verbose --- bot/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 06503e59a0..77aa2d9077 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -154,7 +154,7 @@ echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" echo " ${HTTPS_PROXY_OPT}" -echo " --info" +echo " --verbose" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" @@ -166,7 +166,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ ${HTTPS_PROXY_OPT} \ - --info \ + --verbose \ --mode run \ ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ @@ -192,7 +192,7 @@ echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" echo " ${HTTPS_PROXY_OPT}" -echo " --info" +echo " --verbose" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" @@ -202,7 +202,7 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ ${HTTPS_PROXY_OPT} \ - --info \ + --verbose \ --mode run \ ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ From c0350a9da92f6632c9ba52a57a5a28e03cb9c040 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:14:38 +0100 Subject: [PATCH 18/57] improved messages for resume info and adjusted parsing in bot/build.sh --- bot/build.sh | 2 +- eessi_container.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 77aa2d9077..f9d8c9cede 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -174,7 +174,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from -BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") +BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) diff --git a/eessi_container.sh b/eessi_container.sh index f1c755938c..060b003f33 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -284,7 +284,7 @@ else [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." + echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')." fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} @@ -565,7 +565,7 @@ if [[ ! -z ${SAVE} ]]; then TGZ=${SAVE} fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . - echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From ef9a552aa1b0a96a7b13c4319b501e0a79459766 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:25:27 +0100 Subject: [PATCH 19/57] delete one of two tmp storage tarballs created by bot/build.sh --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index f9d8c9cede..5926f005d5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -209,4 +209,10 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT --save ${PWD}/previous_tmp/tarball_step \ ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +# if two tarballs have been generated, only keep the one from tarball step +NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l) +if [[ ${NUM_TARBALLS} -eq 2 ]]; then + rm -f previous_tmp/build_step/*.tgz +fi + exit 0 From 7cc65750c50bae0a2ea3f0cb9d96137fb21aa452 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 02:55:37 +0100 Subject: [PATCH 20/57] support cmd line arg flag terminator --- eessi_container.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index b1f8922d61..b9553a7ea1 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -173,6 +173,11 @@ while [[ $# -gt 0 ]]; do export https_proxy=${HTTPS_PROXY} shift 2 ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; From 0def4e91ce9d52281e0e8fc3d30c96f095c75bd5 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 22:07:02 +0100 Subject: [PATCH 21/57] just print contents of cfg/job.cfg --- bot/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 5926f005d5..906f1bfd7c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -23,6 +23,10 @@ # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +echo "bot/build.sh: Showing job.cfg from software-layer side" +cat cfg/job.cfg +exit 0 + # source utils.sh source utils.sh From ef608d15f20bea94bae116b7ece71a21586c7e3c Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 23:55:44 +0100 Subject: [PATCH 22/57] switching from json (YQ) to ini (cfg_files.sh) --- bot/build.sh | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 906f1bfd7c..c621b9524f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -17,50 +17,39 @@ # - the working directory contains a directory 'cfg' where the main config # file 'job.cfg' has been deposited # - the directory may contain any additional files references in job.cfg -# - the tool 'yq' for working with json files is available via the PATH or -# the environment variable BOT_YQ (see https://github.com/mikefarah/yq) # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" echo "bot/build.sh: Showing job.cfg from software-layer side" cat cfg/job.cfg -exit 0 - -# source utils.sh -source utils.sh -# check setup / define key variables -# get path for 'yq' (if not found, an empty string is returned) -YQ=$(get_path_for_tool "yq" "BOT_YQ") -exit_code=$? -if [[ ${exit_code} -ne 0 ]]; then - fatal_error "could not find path to 'yq'; exiting" -else - echo_green "found yq (${YQ})" -fi +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh # check if './cfg/job.cfg' exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy -HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE}) +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} echo "HTTP_PROXY='${HTTP_PROXY}'" # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy -HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE}) +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} echo "HTTPS_PROXY='${HTTPS_PROXY}'" -LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided -SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE}) +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR @@ -73,11 +62,11 @@ STORAGE=$(envsubst <<< ${LOCAL_TMP}) echo "'${STORAGE}'" # obtain list of modules to be loaded -LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") echo "LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND -CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE}) +CONTAINER=$(cfg_get_value "repository" "container") export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} @@ -94,20 +83,20 @@ else fi # determine repository to be used from entry .repository in cfg/job.cfg -REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} # determine pilot version to be used from .repository.repo_version in cfg/job.cfg # here, just set & export EESSI_PILOT_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE}) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") # determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE}) +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") # determine architecture to be used from entry .architecture in cfg/job.cfg @@ -115,12 +104,12 @@ export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG if [[ ! -z "${CPU_TARGET}" ]]; then EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} else - EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) + EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") fi export EESSI_SOFTWARE_SUBDIR_OVERRIDE # get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) -EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE}) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # TODO From 429d5a03ad0da75e8e8c52fb8897b341d1c70511 Mon Sep 17 00:00:00 2001 From: trz42 Date: Thu, 23 Feb 2023 23:56:28 +0100 Subject: [PATCH 23/57] add --generic arg when running install script --- bot/build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index c621b9524f..387a9c95c9 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -140,6 +140,11 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +GENERIC_OPT= +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then + GENERIC_OPT="--generic" +fi + mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" @@ -152,7 +157,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ @@ -164,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ - ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} + ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) From bd31faa6b6e4e3945447118b6121667d01b4af7d Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:17:57 +0100 Subject: [PATCH 24/57] fix error in regex --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 387a9c95c9..860f888a46 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi GENERIC_OPT= -if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then GENERIC_OPT="--generic" fi From 1bafb7889fa8f49b183e048d63a8fbdabee041f1 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:43:23 +0100 Subject: [PATCH 25/57] use command separator -- --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 860f888a46..cdd59d6081 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -157,7 +157,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ @@ -169,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ - ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} + -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) From c9041e34604ec881e6a5aeb96fbf31da91bf63a1 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 00:29:25 +0100 Subject: [PATCH 26/57] try fixing arg issue --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index cdd59d6081..f56f456dcf 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -161,7 +161,7 @@ echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ + "${CONTAINER_OPT}" \ ${HTTP_PROXY_OPT} \ ${HTTPS_PROXY_OPT} \ --verbose \ From 9d46a22892a4462f1b87c52c679d90cfed6256bb Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 00:41:36 +0100 Subject: [PATCH 27/57] put build args into an array --- bot/build.sh | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index f56f456dcf..8720c137a5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -124,21 +124,30 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') # prepare options and directories for calling eessi_container.sh +declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--access" "rw") +BUILD_STEP_ARGS+=("--mode" "run") +BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step") +BUILD_STEP_ARGS+=("--storage" "${STORAGE}") CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" + BUILD_STEP_ARGS+=("--container" "${CONTAINER}") fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" + BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") fi HTTPS_PROXY_OPT= if [[ ! -z ${HTTPS_PROXY} ]]; then HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" + BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" + BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}") fi GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then @@ -148,27 +157,13 @@ fi mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" -echo "./eessi_container.sh --access rw" -echo " ${CONTAINER_OPT}" -echo " ${HTTP_PROXY_OPT}" -echo " ${HTTPS_PROXY_OPT}" +echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}" echo " --verbose" -echo " --mode run" -echo " ${REPOSITORY_OPT}" -echo " --save ${PWD}/previous_tmp/build_step" -echo " --storage ${STORAGE}" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg -./eessi_container.sh --access rw \ - "${CONTAINER_OPT}" \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ +./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \ --verbose \ - --mode run \ - ${REPOSITORY_OPT} \ - --save ${PWD}/previous_tmp/build_step \ - --storage ${STORAGE} \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from From ff5045f0940730951c25cfb5260148655fbf933d Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 01:19:25 +0100 Subject: [PATCH 28/57] use array to define args for build and tarball step --- bot/build.sh | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8720c137a5..bb1f4319a1 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -125,29 +125,39 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # prepare options and directories for calling eessi_container.sh declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--verbose") BUILD_STEP_ARGS+=("--access" "rw") BUILD_STEP_ARGS+=("--mode" "run") BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--verbose") +TARBALL_STEP_ARGS+=("--access" "rw") +TARBALL_STEP_ARGS+=("--mode" "run") +TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step") CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" BUILD_STEP_ARGS+=("--container" "${CONTAINER}") + TARBALL_STEP_ARGS+=("--container" "${CONTAINER}") fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") + TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") fi HTTPS_PROXY_OPT= if [[ ! -z ${HTTPS_PROXY} ]]; then HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") + TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}") + TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}") fi GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then @@ -158,16 +168,15 @@ mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}" -echo " --verbose" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \ - --verbose \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) +TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) @@ -181,26 +190,10 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "./eessi_container.sh --access rw" -echo " ${CONTAINER_OPT}" -echo " ${HTTP_PROXY_OPT}" -echo " ${HTTPS_PROXY_OPT}" -echo " --verbose" -echo " --mode run" -echo " ${REPOSITORY_OPT}" -echo " --resume ${BUILD_TMPDIR}" -echo " --save ${PWD}/previous_tmp/tarball_step" -echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ - --verbose \ - --mode run \ - ${REPOSITORY_OPT} \ - --resume ${BUILD_TMPDIR} \ - --save ${PWD}/previous_tmp/tarball_step \ - ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \ + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} # if two tarballs have been generated, only keep the one from tarball step NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l) From d08bd211ed6c23c5b917b01bd01ef75a5cb7b1b9 Mon Sep 17 00:00:00 2001 From: trz42 Date: Tue, 7 Mar 2023 14:43:52 +0100 Subject: [PATCH 29/57] addressed comments and changes requested by first review --- bot/build.sh | 172 ++++++++++++++++++++++++--------------------------- 1 file changed, 82 insertions(+), 90 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index bb1f4319a1..004e2d4881 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -16,41 +16,44 @@ # pull request (OR by some other means) # - the working directory contains a directory 'cfg' where the main config # file 'job.cfg' has been deposited -# - the directory may contain any additional files references in job.cfg +# - the directory may contain any additional files referenced in job.cfg -# defaults -export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" - -echo "bot/build.sh: Showing job.cfg from software-layer side" -cat cfg/job.cfg +# stop as soon as something fails +set -e # source utils.sh and cfg_files.sh source scripts/utils.sh source scripts/cfg_files.sh -# check if './cfg/job.cfg' exists +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" fi -echo "obtaining configuration settings from '${JOB_CFG_FILE}'" +echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" cfg_load ${JOB_CFG_FILE} -# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} -echo "HTTP_PROXY='${HTTP_PROXY}'" +echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'" -# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} -echo "HTTPS_PROXY='${HTTPS_PROXY}'" +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") -echo "LOCAL_TMP='${LOCAL_TMP}'" +echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") -echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR fi @@ -61,14 +64,21 @@ echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " STORAGE=$(envsubst <<< ${LOCAL_TMP}) echo "'${STORAGE}'" +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + # obtain list of modules to be loaded LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") -echo "LOAD_MODULES='${LOAD_MODULES}'" +echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") -export SINGULARITY_HOME="$(pwd):/eessi_bot_job" -export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" +export SINGULARITY_HOME="${pwd}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty @@ -82,103 +92,91 @@ else echo "bot/build.sh: no modules to be loaded" fi -# determine repository to be used from entry .repository in cfg/job.cfg +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} REPOSITORY=$(cfg_get_value "repository" "repo_id") EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in cfg/job.cfg +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} # here, just set & export EESSI_PILOT_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" -# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") - - -# determine architecture to be used from entry .architecture in cfg/job.cfg -# default: leave empty to let downstream script(s) determine subdir to be used -if [[ ! -z "${CPU_TARGET}" ]]; then - EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} -else - EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") -fi +echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" -# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} - -# TODO -# - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# TODO ensure that the bot makes use of that. (currently sets env var -# CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also -# add it to cfg/job.cfg - .architecture.software_subdir) -# - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the -# CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust -# way to determine the IP address of a proxy) -# - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying -# files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') - -# prepare options and directories for calling eessi_container.sh +echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to build step declare -a BUILD_STEP_ARGS=() -BUILD_STEP_ARGS+=("--verbose") -BUILD_STEP_ARGS+=("--access" "rw") -BUILD_STEP_ARGS+=("--mode" "run") -BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step") +BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") -declare -a TARBALL_STEP_ARGS=() -TARBALL_STEP_ARGS+=("--verbose") -TARBALL_STEP_ARGS+=("--access" "rw") -TARBALL_STEP_ARGS+=("--mode" "run") -TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step") -CONTAINER_OPT= -if [[ ! -z ${CONTAINER} ]]; then - CONTAINER_OPT="--container ${CONTAINER}" - BUILD_STEP_ARGS+=("--container" "${CONTAINER}") - TARBALL_STEP_ARGS+=("--container" "${CONTAINER}") -fi -HTTP_PROXY_OPT= -if [[ ! -z ${HTTP_PROXY} ]]; then - HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" - BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") - TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") -fi -HTTPS_PROXY_OPT= -if [[ ! -z ${HTTPS_PROXY} ]]; then - HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" - BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") - TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") -fi -REPOSITORY_OPT= -if [[ ! -z ${REPOSITORY} ]]; then - REPOSITORY_OPT="--repository ${REPOSITORY}" - BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}") - TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}") -fi + +# prepare arguments to install_software_layer.sh (specific to build step) GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then GENERIC_OPT="--generic" fi -mkdir -p previous_tmp/{build_step,tarball_step} +# create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) + echo "Executing command to build software:" -echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" -# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg -export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg -./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \ +./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} +# prepare directory to store tarball of tmp for tarball step +TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step +mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR} + +# create tmp file for output of tarball step +tar_outerr=$(mktemp tar.outerr.XXXX) + +# prepare arguments to eessi_container.sh specific to tarball step +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") + # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") -tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) # to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now source init/eessi_defaults @@ -190,15 +188,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \ +./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} -# if two tarballs have been generated, only keep the one from tarball step -NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l) -if [[ ${NUM_TARBALLS} -eq 2 ]]; then - rm -f previous_tmp/build_step/*.tgz -fi - exit 0 From f59bfdededf7b56bc95dc64f19021a3770d8ac61 Mon Sep 17 00:00:00 2001 From: trz42 Date: Tue, 7 Mar 2023 15:33:24 +0100 Subject: [PATCH 30/57] fix check for missing installations --- check_missing_installations.sh | 20 ++++++++++++++++++-- scripts/utils.sh | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e4c75aebd8..45f276dee1 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" + +# the above assesses the installed software for each easyconfig provided in +# the easystack file and then print messages such as +# `No missing modules!` +# or +# `2 out of 3 required modules missing:` +# depending on the result of the assessment. Hence, we need to check if the +# output does not contain any line with ` required modules missing:` + +grep " required modules missing:" ${eb_missing_out} > /dev/null + +# we need to process the result (from finding `No missing modules` to NOT finding +# ` required modules missing:` and no other error happened) +# +# if grep returns 1 (` required modules missing:` was NOT found), we set +# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 +[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" diff --git a/scripts/utils.sh b/scripts/utils.sh index d0da95e87f..1f46ae69b4 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -28,7 +28,7 @@ function check_exit_code { if [[ $ec -eq 0 ]]; then echo_green "${ok_msg}" else - fatal_error "${fail_msg}" + echo_red "${fail_msg}" fi } From 7bb7c1de90b2bfb47a11124702d6dd968531267a Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:07:04 +0100 Subject: [PATCH 31/57] add test with missing package --- .github/workflows/test_eessi.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 92b1f71cad..1735bb77ed 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -40,4 +40,13 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" + echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml + ./check_missing_installations.sh + [[ $? -eq 0 ]] && exit 1 || exit 0 From 68a068b64ea5af8fad5a69ff812a5942cacd7167 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:25:04 +0100 Subject: [PATCH 32/57] separated test cases + updated evaluation of test result --- .github/workflows/test_eessi.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1735bb77ed..073edaf4e8 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -42,11 +42,26 @@ jobs: env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + + - name: Test check_missing_installations.sh with missing package + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml - ./check_missing_installations.sh - [[ $? -eq 0 ]] && exit 1 || exit 0 + if ./check_missing_installations.sh; then + echo "captured missing package; test PASSED" + exit 0 + else + echo "did NOT capture missing package; test FAILED" + exit 1 + fi From 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:30:17 +0100 Subject: [PATCH 33/57] fix small bug in env var name --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 073edaf4e8..f3bf2e7ab1 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -38,7 +38,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh @@ -50,7 +50,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 094e82df3b0de5a195d99fa914674024aa2e8145 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:35:29 +0100 Subject: [PATCH 34/57] set software subdir override before sourcing init script --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index f3bf2e7ab1..d0b530e2e4 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -33,24 +33,24 @@ jobs: - name: Test check_missing_installations.sh script run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 60cf7e2715b7231810479a3fd91d20499c9b7670 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:42:01 +0100 Subject: [PATCH 35/57] Revert "set software subdir override before sourcing init script" This reverts commit 094e82df3b0de5a195d99fa914674024aa2e8145. --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index d0b530e2e4..f3bf2e7ab1 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -33,24 +33,24 @@ jobs: - name: Test check_missing_installations.sh script run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 6f471520b63b0cee6b9d94818e7f42938dbc1bf8 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:42:51 +0100 Subject: [PATCH 36/57] Revert "fix small bug in env var name" This reverts commit 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb. --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index f3bf2e7ab1..073edaf4e8 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -38,7 +38,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh @@ -50,7 +50,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From a15676ddac70b5cb18d82625a9a048d59199a41b Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:48:13 +0100 Subject: [PATCH 37/57] tweaking output of test --- .github/workflows/test_eessi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 073edaf4e8..23b30c0063 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -43,7 +43,7 @@ jobs: echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - - name: Test check_missing_installations.sh with missing package + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild @@ -52,12 +52,12 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml if ./check_missing_installations.sh; then echo "captured missing package; test PASSED" exit 0 From 54d2a21f04f7e10ee8577c674be7b2d411da5026 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 13:31:25 +0100 Subject: [PATCH 38/57] improved usage information --- eessi_container.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index b9553a7ea1..48c4653ba9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash display_help() { - echo "usage: $0 [OPTIONS] [SCRIPT]" + echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" @@ -77,7 +77,7 @@ display_help() { echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" + echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use [default: EESSI-pilot via" echo " default container, see --container]" @@ -98,7 +98,9 @@ display_help() { echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]; uses env var \$https_proxy if set" echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." + echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" + echo " arguments to the script/command start with '-' or '--', use the flag terminator" + echo " '--' to let eessi_container.sh stop parsing arguments." } # set defaults for command line arguments From 99231e84eade092ae45fb5fa725d752eb04edf92 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 00:07:09 +0100 Subject: [PATCH 39/57] if exit_code is not zero, use fatal_error (as it was) --- scripts/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/utils.sh b/scripts/utils.sh index 1f46ae69b4..d0da95e87f 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -28,7 +28,7 @@ function check_exit_code { if [[ $ec -eq 0 ]]; then echo_green "${ok_msg}" else - echo_red "${fail_msg}" + fatal_error "${fail_msg}" fi } From b94bd111a3dcf422e7e78fa58bdbf52a74415fb2 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 00:52:43 +0100 Subject: [PATCH 40/57] fix bug in test logic --- .github/workflows/test_eessi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 23b30c0063..ce58fc3a97 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -59,9 +59,9 @@ jobs: echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml if ./check_missing_installations.sh; then - echo "captured missing package; test PASSED" - exit 0 - else echo "did NOT capture missing package; test FAILED" exit 1 + else + echo "captured missing package; test PASSED" + exit 0 fi From 5e4bea9b71a37a09359def990fd12a1c01a5bf1c Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 01:03:20 +0100 Subject: [PATCH 41/57] added explanation for test cases (exit code vs boolean logic) --- .github/workflows/test_eessi.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index ce58fc3a97..04195dd619 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -58,6 +58,11 @@ jobs: echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package if ./check_missing_installations.sh; then echo "did NOT capture missing package; test FAILED" exit 1 From 64d1866638288e315de91a41ad37a40fdaf53736 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 01:33:50 +0100 Subject: [PATCH 42/57] fix bug, env var PWD is uppercase --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 004e2d4881..20334501ed 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -77,8 +77,8 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") -export SINGULARITY_HOME="${pwd}:/eessi_bot_job" -export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir" +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty From ff78c62df541075ecfb58c37e8a14bc4a0e0d80f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 15 Mar 2023 07:04:02 +0100 Subject: [PATCH 43/57] make sure that CPU arch specific directories are used as container cache dir --- bot/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 20334501ed..c8def2cdd3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -27,6 +27,7 @@ source scripts/cfg_files.sh # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then @@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} export SINGULARITY_CACHEDIR fi From 22661e66deb8e31ba053aae3125ca30ca8411f73 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 18:58:28 +0100 Subject: [PATCH 44/57] restore PATHs only after last run of pip installed eb --- EESSI-pilot-install-software.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 2830754b29..c05ccf8ab9 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -165,10 +165,6 @@ else eb --install-latest-eb-release &> ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" - # restore origin $PATH and $PYTHONPATH values - export PATH=${ORIG_PATH} - export PYTHONPATH=${ORIG_PYTHONPATH} - eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null if [[ $? -eq 0 ]]; then ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!" @@ -177,6 +173,10 @@ else check_exit_code $? "${ok_msg}" "${fail_msg}" fi + # restore origin $PATH and $PYTHONPATH values + export PATH=${ORIG_PATH} + export PYTHONPATH=${ORIG_PYTHONPATH} + module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> EasyBuild module installed!" From 092cb49cf76735df48110fe86a1e9bb4e2c571c0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 12:42:05 +0100 Subject: [PATCH 45/57] attempt to fix issue #225 --- create_tarball.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 56ac8ab7ad..08dc4f4d7e 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -38,6 +38,7 @@ cd ${overlay_upper_dir}/versions/ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..." files_list=${tmpdir}/files.list.txt +module_files_list=${tmpdir}/module_files.list.txt if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), @@ -49,12 +50,26 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + # module files and symlinks + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \ + | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ + >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then # installation directories - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + for package_version in $(cat ${module_files_list}); do + echo "handling ${package_version}" + ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + | grep -v '/\.wh\.' >> ${files_list} + done fi +echo "wrote file list to ${files_list}" +cat ${files_list} +echo "wrote module file list to ${module_files_list}" +cat ${module_files_list} + topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tgz} from ${topdir}..." From 7a750d508d00821cd5b90e38b1471b6a2433a77e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 19:17:02 +0100 Subject: [PATCH 46/57] add comments, small improvements --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 08dc4f4d7e..9381407d10 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -51,13 +51,12 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \ + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then - # installation directories - # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + # installation directories but only those for which module files were created for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ @@ -65,6 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then done fi +# add a bit debug output echo "wrote file list to ${files_list}" cat ${files_list} echo "wrote module file list to ${module_files_list}" From 6299c9b4a84a5ced2afde69e9c668c4de9752331 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Mar 2023 19:36:00 +0100 Subject: [PATCH 47/57] add configuration file for production build-and-deploy bot in AWS CitC cluster --- bot/bot-eessi-aws-citc.cfg | 151 +++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 bot/bot-eessi-aws-citc.cfg diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg new file mode 100644 index 0000000000..5b3ad34612 --- /dev/null +++ b/bot/bot-eessi-aws-citc.cfg @@ -0,0 +1,151 @@ +# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 + +[github] +# replace '123456' with the ID of your GitHub App +app_id = 281041 + +# a short (!) name for your app instance that can be used for example +# when adding/updating a comment to a PR +# (!) a short yet descriptive name is preferred because it appears in +# comments to the PR +# for example, the name could include the name of the cluster the bot +# runs on and the username which runs the bot +# NOTE avoid putting an actual username here as it will be visible on +# potentially publicly accessible GitHub pages. +app_name = eessi-bot-citc-aws + +# replace '12345678' with the ID of the installation of your GitHub App +# (can be derived by creating an event and then checking for the list +# of sent events and its payload either via the Smee channel's web page +# or via the Advanced section of your GitHub App on github.com) +installation_id = 33078935 + +# path to the private key that was generated when the GitHub App was registered +private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem + + +[buildenv] +# name of the job script used for building an EESSI stack +build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm + +# The container_cachedir may be used to reuse downloaded container image files +# across jobs. Thus, jobs can more quickly launch containers. +container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir + +# it may happen that we need to customize some CVMFS configuration +# the value of cvmfs_customizations is a dictionary which maps a file +# name to an entry that needs to be added to that file +cvmfs_customizations = {} + +# if compute nodes have no internet connection, we need to set http(s)_proxy +# or commands such as pip3 cannot download software from package repositories +# for example, the temporary EasyBuild is installed via pip3 first +# http_proxy = http://PROXY_DNS:3128/ +# https_proxy = http://PROXY_DNS:3128/ + +# directory under which the bot prepares directories per job +# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR +jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# configure environment +# list of comma-separated modules to be loaded by build_job_script +# useful/needed if some tool is not provided as system-wide package +# (read by bot and handed over to build_job_script via parameter +# --load-modules) +load_modules = + +# PATH to temporary directory on build node ... ends up being used for +# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI +# escaping variables with '\' delays expansion to the start of the +# build_job_script; this can be used for referencing environment +# variables that are only set inside a Slurm job +local_tmp = /tmp/$USER/EESSI + +# parameters to be added to all job submissions +# NOTE do not quote parameter string. Quotes are retained when reading in config and +# then the whole 'string' is recognised as a single parameter. +# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs +# to be initialised as if it is for a login shell. +# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed +slurm_params = --hold --time=24:0:0 + +# full path to the job submission command +submit_command = /usr/bin/sbatch + +# which GH account has the permission to trigger the build (by setting +# the label 'bot:build' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the build +# value can be a space delimited list of GH accounts +build_permission = boegel trz42 bedroge + +[architecturetargets] +# defines both for which architectures the bot will build +# and what submission parameters shall be used +# medium instances (8 cores, 16GB RAM) +#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} +# larger instances (16 cores, 32GB RAM) +arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} + +[repo_targets] +# defines for which repository a arch_target should be build for +# +# only building for repository EESSI-pilot +repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} + +# points to definition of repositories (default EESSI-pilot defined by build container) +repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles + +# configuration for event handler which receives events from a GitHub repository. +[event_handler] +# path to the log file to log messages for event handler +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log + + +[job_manager] +# path to the log file to log messages for job manager +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log + +# directory where job manager stores information about jobs to be tracked +# e.g. as symbolic link JOBID -> directory to job +job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# full path to the job status checking command +poll_command = /usr/bin/squeue + +# polling interval in seconds +poll_interval = 60 + +# full path to the command for manipulating existing jobs +scontrol_command = /usr/bin/scontrol + +[deploycfg] +# script for uploading built software packages +tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging + +# URL to S3/minio bucket +# if attribute is set, bucket_base will be constructed as follows +# bucket_base=${endpoint_url}/${bucket_name} +# otherwise, bucket_base will be constructed as follows +# bucket_base=https://${bucket_name}.s3.amazonaws.com +# - The former variant is used for non AWS S3 services, eg, minio, or when +# the bucket name is not provided in the hostname (see latter case). +# - The latter variant is used for AWS S3 services. +#endpoint_url = URL_TO_S3_SERVER + +# bucket name +bucket_name = eessi-staging + +# upload policy: defines what policy is used for uploading built artefacts +# to an S3 bucket +# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) +# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) +# only upload the latest built artefact +# 'once' : only once upload any built artefact for the build target +# 'none' : do not upload any built artefacts +upload_policy = once + +# which GH account has the permission to trigger the deployment (by setting +# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the deployment +# value can be a space delimited list of GH accounts +deploy_permission = boegel trz42 bedroge From 69f662795dca88559f4885e5fa80a16c0da655fe Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 22:13:21 +0100 Subject: [PATCH 48/57] improve handling of potentially non existing list files --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 9381407d10..fd04a195cb 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -55,7 +55,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ >> ${module_files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then +if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" @@ -66,9 +66,9 @@ fi # add a bit debug output echo "wrote file list to ${files_list}" -cat ${files_list} +[ -r ${files_list} ] && cat ${files_list} echo "wrote module file list to ${module_files_list}" -cat ${module_files_list} +[ -r ${module_files_list} ] && cat ${module_files_list} topdir=${cvmfs_repo}/versions/ From a4c8e6471bad5ce80df906c5126f5a02a06c55a0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 11 Apr 2023 09:35:44 +0200 Subject: [PATCH 49/57] removing comment to address review comment --- check_missing_installations.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 45f276dee1..926f475903 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -38,9 +38,6 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim grep " required modules missing:" ${eb_missing_out} > /dev/null -# we need to process the result (from finding `No missing modules` to NOT finding -# ` required modules missing:` and no other error happened) -# # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 [[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 From 4d132273569f6d40e503bdd48b5b3aca88f7c907 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 11 Apr 2023 12:34:39 +0200 Subject: [PATCH 50/57] update determining directories to tar + adding a note on assumptions made --- create_tarball.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index fd04a195cb..b6c72b341d 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -52,11 +52,16 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ - | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ + | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created + # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' + # using EasyBuild's standard module naming scheme) match the name of the + # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). + # If either side changes (module naming scheme or naming of software + # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ From 49d41153d1a4bac24ebb77eaf84420ed42400c23 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 27 Apr 2023 14:20:01 +0200 Subject: [PATCH 51/57] Make sure installations are read only This should help us avoid the issue we saw with `.pyc` files --- configure_easybuild | 1 + 1 file changed, 1 insertion(+) diff --git a/configure_easybuild b/configure_easybuild index 19b2d7454b..245553f342 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -13,6 +13,7 @@ export EASYBUILD_ZIP_LOGS=bzip2 export EASYBUILD_RPATH=1 export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH +export EASYBUILD_READ_ONLY_INSTALLDIR=1 # assume that eb_hooks.py is located in same directory as this script (configure_easybuild) TOPDIR=$(dirname $(realpath $BASH_SOURCE)) From 8620428abb6fae254e62609536fc1afef6ccb6ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 May 2023 14:08:25 +0200 Subject: [PATCH 52/57] add replacement init script for removed 2021.06 pilot version --- versions/2021.06/init/Magic_Castle/bash | 3 +++ versions/2021.06/init/bash | 3 +++ .../2021.06/init/print_deprecation_warning.sh | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 versions/2021.06/init/Magic_Castle/bash create mode 100644 versions/2021.06/init/bash create mode 100755 versions/2021.06/init/print_deprecation_warning.sh diff --git a/versions/2021.06/init/Magic_Castle/bash b/versions/2021.06/init/Magic_Castle/bash new file mode 100644 index 0000000000..5f149c817f --- /dev/null +++ b/versions/2021.06/init/Magic_Castle/bash @@ -0,0 +1,3 @@ +/cvmfs/pilot.eessi-hpc.org/versions/2021.06/init/print_deprecation_warning.sh + +source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/Magic_Castle/bash diff --git a/versions/2021.06/init/bash b/versions/2021.06/init/bash new file mode 100644 index 0000000000..82a078849a --- /dev/null +++ b/versions/2021.06/init/bash @@ -0,0 +1,3 @@ +/cvmfs/pilot.eessi-hpc.org/versions/2021.06/init/print_deprecation_warning.sh + +source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/bash diff --git a/versions/2021.06/init/print_deprecation_warning.sh b/versions/2021.06/init/print_deprecation_warning.sh new file mode 100755 index 0000000000..b721ed2f71 --- /dev/null +++ b/versions/2021.06/init/print_deprecation_warning.sh @@ -0,0 +1,19 @@ +#!/bin/bash +function echo_yellow_stderr() { + echo -e "\e[33m${1}\e[0m" >&2 +} + +echo_yellow_stderr +echo_yellow_stderr "WARNING: Version 2021.06 of the EESSI pilot repository has been removed since 16 May 2023." +echo_yellow_stderr +echo_yellow_stderr "Version 2021.12 of the EESSI pilot repository can be used as a drop-in replacement, " +echo_yellow_stderr "so we have prepared your environment to use that instead." +echo_yellow_stderr +echo_yellow_stderr "In the future, please run" +echo_yellow_stderr +echo_yellow_stderr " source /cvmfs/pilot.eessi-hpc.org/latest/init/bash" +echo_yellow_stderr +echo_yellow_stderr "to prepare your start using the EESSI pilot repository." +echo_yellow_stderr +echo_yellow_stderr "See also https://eessi.github.io/docs/using_eessi/setting_up_environment ." +echo_yellow_stderr From fc3573b8a975cc94c50d89ca4aa066cd17993ca1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 May 2023 09:16:13 +0200 Subject: [PATCH 53/57] don't print warning when unknown targets are found (fixes #247) --- init/eessi_software_subdir_for_host.py | 2 -- init/test.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/init/eessi_software_subdir_for_host.py b/init/eessi_software_subdir_for_host.py index b09b4711e4..58e9cfd2e6 100755 --- a/init/eessi_software_subdir_for_host.py +++ b/init/eessi_software_subdir_for_host.py @@ -101,8 +101,6 @@ def find_best_target(eessi_prefix): continue if uarch in KNOWN_CPU_UARCHS: target_uarchs.append(KNOWN_CPU_UARCHS[uarch]) - else: - warning('Ignoring unknown target "%s"' % uarch) host_uarch = KNOWN_CPU_UARCHS[host_cpu_name] compat_target_uarchs = sorted([x for x in target_uarchs if x <= host_uarch]) diff --git a/init/test.py b/init/test.py index 0ed09abd6f..f10be5e66e 100644 --- a/init/test.py +++ b/init/test.py @@ -60,12 +60,12 @@ def broadwell_host_triple(): prep_tmpdir(tmpdir, ['x86_64/intel/ivybridge']) assert find_best_target(tmpdir) == 'x86_64/intel/ivybridge' - # unknown targets don't cause trouble (only warning) + # unknown targets don't cause trouble prep_tmpdir(tmpdir, ['x86_64/intel/no_such_intel_cpu']) assert find_best_target(tmpdir) == 'x86_64/intel/ivybridge' captured = capsys.readouterr() assert captured.out == '' - assert captured.err == 'WARNING: Ignoring unknown target "no_such_intel_cpu"\n' + assert captured.err == '' # older targets have to no impact on best target (sandybridge < ivybridge) prep_tmpdir(tmpdir, ['x86_64/intel/sandybridge']) From e688ec8de4b6684296de00f31eba1578cbc0d72c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 May 2023 10:01:24 +0200 Subject: [PATCH 54/57] avoid that versions/2021.06/init is included in init tarball --- create_directory_tarballs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh index be282463eb..70e666f871 100755 --- a/create_directory_tarballs.sh +++ b/create_directory_tarballs.sh @@ -26,7 +26,7 @@ fi tartmp=$(mktemp -t -d init.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-init-$(date +%s).tar.gz" -curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */init/ +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/init/' source "${tartmp}/${version}/init/minimal_eessi_env" if [ "${EESSI_PILOT_VERSION}" != "${version}" ] then @@ -42,7 +42,7 @@ echo_green "Done! Created tarball ${tarname}." tartmp=$(mktemp -t -d scripts.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-scripts-$(date +%s).tar.gz" -curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/' +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/scripts/' tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" From 7554604828ab97c6026ccaa6194213fb492329b3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 27 May 2023 17:27:58 +0200 Subject: [PATCH 55/57] implement post-prepare hook for GCCcore to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' --- eb_hooks.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index df7742f999..9b07ed3b8e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -3,8 +3,11 @@ import os import re +from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option +from easybuild.tools.filetools import copy_file, which +from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -89,6 +92,25 @@ def pre_prepare_hook(self, *args, **kwargs): mpi_family, rpath_override_dirs) +def gcc_postprepare(self, *args, **kwargs): + """ + Post-configure hook for GCCcore: + - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' + """ + if self.name == 'GCCcore': + config_guess = obtain_config_guess() + system_type, _ = run_cmd(config_guess, log_all=True) + cmd_prefix = '%s-' % system_type.strip() + for cmd in ('ld', 'ld.gold', 'ld.bfd'): + wrapper = which(cmd) + self.log.info("Path to %s wrapper: %s" % (cmd, wrapper)) + wrapper_dir = os.path.dirname(wrapper) + prefix_wrapper = os.path.join(wrapper_dir, cmd_prefix + cmd) + copy_file(wrapper, prefix_wrapper) + self.log.info("Path to %s wrapper with '%s' prefix: %s" % (cmd, cmd_prefix, which(prefix_wrapper))) + else: + raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!") + def post_prepare_hook(self, *args, **kwargs): """Main post-prepare hook: trigger custom functions.""" @@ -98,6 +120,9 @@ def post_prepare_hook(self, *args, **kwargs): print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR)) delattr(self, EESSI_RPATH_OVERRIDE_ATTR) + if self.name in POST_PREPARE_HOOKS: + POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) + def cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" @@ -187,6 +212,10 @@ def wrf_preconfigure(self, *args, **kwargs): 'UCX': ucx_eprefix, } +POST_PREPARE_HOOKS = { + 'GCCcore': gcc_postprepare, +} + PRE_CONFIGURE_HOOKS = { 'libfabric': libfabric_disable_psm3_x86_64_generic, 'MetaBAT': metabat_preconfigure, From c2ec231652a5fe12c0e90e3d9576d6e36ae7fbfa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 27 May 2023 19:34:35 +0200 Subject: [PATCH 56/57] patch copied wrapper script to make sure it's actually working (CMD needs to be set to 'ld' for 'x86_64-pc-linux-gnu-ld', because EasyBuild's rpath_args.py only takes into account 'ld' when determining whether or not to use -Wl,-rpath or just -rpath --- eb_hooks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9b07ed3b8e..777d583c7b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -6,7 +6,7 @@ from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option -from easybuild.tools.filetools import copy_file, which +from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -108,6 +108,17 @@ def gcc_postprepare(self, *args, **kwargs): prefix_wrapper = os.path.join(wrapper_dir, cmd_prefix + cmd) copy_file(wrapper, prefix_wrapper) self.log.info("Path to %s wrapper with '%s' prefix: %s" % (cmd, cmd_prefix, which(prefix_wrapper))) + + # we need to tweak the copied wrapper script, so that: + regex_subs = [ + # - CMD in the script is set to the command name without prefix, because EasyBuild's rpath_args.py + # script that is used by the wrapper script only checks for 'ld', 'ld.gold', etc. + # when checking whether or not to use -Wl + ('^CMD=.*', 'CMD=%s' % cmd), + # - the path to the correct actual binary is logged and called + ('/%s ' % cmd, '/%s ' % (cmd_prefix + cmd)), + ] + apply_regex_substitutions(prefix_wrapper, regex_subs) else: raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!") From cd233a8a168b477b7a31ecd4c0d5190d906d5d6c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 30 May 2023 12:03:14 +0200 Subject: [PATCH 57/57] testing bot PR 172 --- EESSI-pilot-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c05ccf8ab9..b7a4c536e6 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -6,6 +6,8 @@ # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +echo "test run for bot PR 172" + display_help() { echo "usage: $0 [OPTIONS]" echo " -g | --generic - instructs script to build for generic architecture target"