Skip to content

reorganisation of constants and validation of config settings #266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Neves-P merged 5 commits into from
May 6, 2024
Merged

reorganisation of constants and validation of config settings #266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
13a9a18
reorganisation of constants and validation of config settings
truib Apr 22, 2024
933e0fa
return result when validating config settings and show result on console
truib Apr 23, 2024
68b6011
fix hound issues
truib Apr 28, 2024
1ceab6f
attempt to clarify comments about repository settings
truib May 6, 2024
d3b211c
clarifying constants used for job result files
truib May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions connections/github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ def get_token():

global _token
cfg = config.read_config()
github_cfg = cfg['github']
app_id = github_cfg.get('app_id')
installation_id = github_cfg.get('installation_id')
private_key_path = github_cfg.get('private_key')
github_cfg = cfg[config.SECTION_GITHUB]
app_id = github_cfg.get(config.GITHUB_SETTING_APP_ID)
installation_id = github_cfg.get(config.GITHUB_SETTING_INSTALLATION_ID)
private_key_path = github_cfg.get(config.GITHUB_SETTING_PRIVATE_KEY)
private_key = ''

with open(private_key_path, 'r') as private_key_file:
Expand Down
92 changes: 68 additions & 24 deletions eessi_bot_event_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,8 @@

# Local application imports (anything from EESSI/eessi-bot-software-layer)
from connections import github
import tasks.build as build
from tasks.build import check_build_permission, get_architecture_targets, get_repo_cfg, \
request_bot_build_issue_comments, submit_build_jobs
import tasks.deploy as deploy
from tasks.deploy import deploy_built_artefacts
from tools import config
from tools.args import event_handler_parse
Expand All @@ -39,11 +37,59 @@
from tools.pr_comments import create_comment


APP_NAME = "app_name"
BOT_CONTROL = "bot_control"
COMMAND_RESPONSE_FMT = "command_response_fmt"
GITHUB = "github"
REPO_TARGET_MAP = "repo_target_map"
REQUIRED_CONFIG = {
config.SECTION_ARCHITECTURETARGETS: [
config.ARCHITECTURETARGETS_SETTING_ARCH_TARGET_MAP], # required
config.SECTION_BOT_CONTROL: [
config.BOT_CONTROL_SETTING_COMMAND_PERMISSION, # required
config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT], # required
config.SECTION_BUILDENV: [
config.BUILDENV_SETTING_BUILD_JOB_SCRIPT, # required
config.BUILDENV_SETTING_BUILD_LOGS_DIR, # optional+recommended
config.BUILDENV_SETTING_BUILD_PERMISSION, # optional+recommended
config.BUILDENV_SETTING_CONTAINER_CACHEDIR, # optional+recommended
# config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS, # optional
# config.BUILDENV_SETTING_HTTPS_PROXY, # optional
# config.BUILDENV_SETTING_HTTP_PROXY, # optional
config.BUILDENV_SETTING_JOBS_BASE_DIR, # required
# config.BUILDENV_SETTING_LOAD_MODULES, # optional
config.BUILDENV_SETTING_LOCAL_TMP, # required
config.BUILDENV_SETTING_NO_BUILD_PERMISSION_COMMENT, # required
config.BUILDENV_SETTING_SHARED_FS_PATH, # optional+recommended
# config.BUILDENV_SETTING_SLURM_PARAMS, # optional
config.BUILDENV_SETTING_SUBMIT_COMMAND], # required
config.SECTION_DEPLOYCFG: [
config.DEPLOYCFG_SETTING_ARTEFACT_PREFIX, # (required)
config.DEPLOYCFG_SETTING_ARTEFACT_UPLOAD_SCRIPT, # required
config.DEPLOYCFG_SETTING_BUCKET_NAME, # required
config.DEPLOYCFG_SETTING_DEPLOY_PERMISSION, # optional+recommended
# config.DEPLOYCFG_SETTING_ENDPOINT_URL, # optional
config.DEPLOYCFG_SETTING_METADATA_PREFIX, # (required)
config.DEPLOYCFG_SETTING_NO_DEPLOY_PERMISSION_COMMENT, # required
config.DEPLOYCFG_SETTING_UPLOAD_POLICY], # required
config.SECTION_DOWNLOAD_PR_COMMENTS: [
config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_TIP, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_TIP, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE, # required
config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP], # required
config.SECTION_EVENT_HANDLER: [
config.EVENT_HANDLER_SETTING_LOG_PATH], # required
config.SECTION_GITHUB: [
config.GITHUB_SETTING_APP_ID, # required
config.GITHUB_SETTING_APP_NAME, # required
config.GITHUB_SETTING_INSTALLATION_ID, # required
config.GITHUB_SETTING_PRIVATE_KEY], # required
config.SECTION_REPO_TARGETS: [
config.REPO_TARGETS_SETTING_REPO_TARGET_MAP, # required
config.REPO_TARGETS_SETTING_REPOS_CFG_DIR], # required
config.SECTION_SUBMITTED_JOB_COMMENTS: [
config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT, # required
config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE] # required
}


class EESSIBotSoftwareLayer(PyGHee):
Expand All @@ -62,8 +108,8 @@ def __init__(self, *args, **kwargs):
super(EESSIBotSoftwareLayer, self).__init__(*args, **kwargs)

self.cfg = config.read_config()
event_handler_cfg = self.cfg['event_handler']
self.logfile = event_handler_cfg.get('log_path')
event_handler_cfg = self.cfg[config.SECTION_EVENT_HANDLER]
self.logfile = event_handler_cfg.get(config.EVENT_HANDLER_SETTING_LOG_PATH)

def log(self, msg, *args):
"""
Expand Down Expand Up @@ -110,8 +156,8 @@ def handle_issue_comment_event(self, event_info, log_file=None):
# log level is set to debug
self.log(f"Comment in {issue_url} (owned by @{owner}) {action} by @{sender}")

app_name = self.cfg[GITHUB][APP_NAME]
command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
command_response_fmt = self.cfg[config.SECTION_BOT_CONTROL][config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT]

# currently, only commands in new comments are supported
# - commands have the syntax 'bot: COMMAND [ARGS*]'
Expand Down Expand Up @@ -301,8 +347,8 @@ def handle_pull_request_labeled_event(self, event_info, pr):
request_body = event_info['raw_request_body']
repo_name = request_body['repository']['full_name']
pr_number = request_body['pull_request']['number']
app_name = self.cfg[GITHUB][APP_NAME]
command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
command_response_fmt = self.cfg[config.SECTION_BOT_CONTROL][config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT]
comment_body = command_response_fmt.format(
app_name=app_name,
comment_response=msg,
Expand Down Expand Up @@ -330,7 +376,7 @@ def handle_pull_request_opened_event(self, event_info, pr):
PyGithub, not the github from the internal connections module)
"""
self.log("PR opened: waiting for label bot:build")
app_name = self.cfg[GITHUB][APP_NAME]
app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
# TODO check if PR already has a comment with arch targets and
# repositories
arch_map = get_architecture_targets(self.cfg)
Expand All @@ -343,7 +389,8 @@ def handle_pull_request_opened_event(self, event_info, pr):
comment += f"{', '.join([f'`{arch}`' for arch in architectures])}"
else:
comment += "none"
repositories = list(set([repo_id for repo_ids in repo_cfg[REPO_TARGET_MAP].values() for repo_id in repo_ids]))
repositories = list(set([repo_id for repo_ids in repo_cfg[config.REPO_TARGETS_SETTING_REPO_TARGET_MAP].values()
for repo_id in repo_ids]))
comment += "\n- repositories: "
if len(repositories) > 0:
comment += f"{', '.join([f'`{repo_id}`' for repo_id in repositories])}"
Expand Down Expand Up @@ -547,9 +594,7 @@ def start(self, app, port=3000):
print(port_info)
self.log(port_info)

event_handler_cfg = self.cfg['event_handler']
my_logfile = event_handler_cfg.get('log_path')
log_file_info = "logging in to %s" % my_logfile
log_file_info = "logging in to %s" % self.logfile
print(log_file_info)
self.log(log_file_info)
waitress.serve(app, listen='*:%s' % port)
Expand All @@ -563,13 +608,12 @@ def main():
"""
opts = event_handler_parse()

required_config = {
build.SUBMITTED_JOB_COMMENTS: [build.INITIAL_COMMENT, build.AWAITS_RELEASE],
build.BUILDENV: [build.NO_BUILD_PERMISSION_COMMENT],
deploy.DEPLOYCFG: [deploy.NO_DEPLOY_PERMISSION_COMMENT]
}
# config is read and checked for settings to raise an exception early when the event_handler starts.
config.check_required_cfg_settings(required_config)
if config.check_required_cfg_settings(REQUIRED_CONFIG):
print("Configuration check: PASSED")
else:
print("Configuration check: FAILED")
sys.exit(1)
github.connect()

if opts.file:
Expand Down
73 changes: 42 additions & 31 deletions eessi_bot_job_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,21 +48,27 @@
from tools.pr_comments import get_submitted_job_comment, update_comment


AWAITS_LAUNCH = "awaits_launch"
FINISHED_JOB_COMMENTS = "finished_job_comments"
JOB_RESULT_COMMENT_DESCRIPTION = "comment_description"
JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt"
JOB_TEST_COMMENT_DESCRIPTION = "comment_description"
JOB_TEST_UNKNOWN_FMT = "job_test_unknown_fmt"
NEW_JOB_COMMENTS = "new_job_comments"
RUNNING_JOB = "running_job"
RUNNING_JOB_COMMENTS = "running_job_comments"

# settings that are required in 'app.cfg'
REQUIRED_CONFIG = {
FINISHED_JOB_COMMENTS: [JOB_RESULT_UNKNOWN_FMT, JOB_TEST_UNKNOWN_FMT],
NEW_JOB_COMMENTS: [AWAITS_LAUNCH],
RUNNING_JOB_COMMENTS: [RUNNING_JOB]
}
config.SECTION_FINISHED_JOB_COMMENTS: [
config.FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT, # required
config.FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT], # required
config.SECTION_GITHUB: [
config.GITHUB_SETTING_APP_ID, # required
# config.GITHUB_SETTING_APP_NAME, # unused
config.GITHUB_SETTING_INSTALLATION_ID, # required
config.GITHUB_SETTING_PRIVATE_KEY], # required
config.SECTION_JOB_MANAGER: [
config.JOB_MANAGER_SETTING_LOG_PATH, # required
config.JOB_MANAGER_SETTING_JOB_IDS_DIR, # required
config.JOB_MANAGER_SETTING_POLL_COMMAND, # required
config.JOB_MANAGER_SETTING_POLL_INTERVAL, # optional+recommended
config.JOB_MANAGER_SETTING_SCONTROL_COMMAND], # required
config.SECTION_NEW_JOB_COMMENTS: [
config.NEW_JOB_COMMENTS_SETTING_AWAITS_LAUNCH], # required
config.SECTION_RUNNING_JOB_COMMENTS: [
config.RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB] # required
}


class EESSIBotSoftwareLayerJobManager:
Expand All @@ -77,8 +83,8 @@ def __init__(self):
configuration to set the path to the logfile.
"""
cfg = config.read_config()
job_manager_cfg = cfg['job_manager']
self.logfile = job_manager_cfg.get('log_path')
job_manager_cfg = cfg[config.SECTION_JOB_MANAGER]
self.logfile = job_manager_cfg.get(config.JOB_MANAGER_SETTING_LOG_PATH)

def get_current_jobs(self):
"""
Expand Down Expand Up @@ -340,10 +346,10 @@ def process_new_job(self, new_job):

# update status table if we found a comment
if "comment_id" in new_job:
new_job_comments_cfg = config.read_config()[NEW_JOB_COMMENTS]
new_job_comments_cfg = config.read_config()[config.SECTION_NEW_JOB_COMMENTS]
dt = datetime.now(timezone.utc)
update = "\n|%s|released|" % dt.strftime("%b %d %X %Z %Y")
update += f"{new_job_comments_cfg[AWAITS_LAUNCH]}|"
update += f"{new_job_comments_cfg[config.NEW_JOB_COMMENTS_SETTING_AWAITS_LAUNCH]}|"
update_comment(new_job["comment_id"], pr, update)
else:
log(
Expand Down Expand Up @@ -412,8 +418,9 @@ def process_running_jobs(self, running_job):

if "comment_id" in running_job:
dt = datetime.now(timezone.utc)
running_job_comments_cfg = config.read_config()[RUNNING_JOB_COMMENTS]
running_msg = running_job_comments_cfg[RUNNING_JOB].format(job_id=running_job['jobid'])
running_job_comments_cfg = config.read_config()[config.SECTION_RUNNING_JOB_COMMENTS]
running_msg_fmt = running_job_comments_cfg[config.RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB]
running_msg = running_msg_fmt.format(job_id=running_job['jobid'])
if "comment_body" in running_job and running_msg in running_job["comment_body"]:
log("Not updating comment, '%s' already found" % running_msg)
else:
Expand Down Expand Up @@ -476,7 +483,7 @@ def process_finished_job(self, finished_job):
# status = {SUCCESS,FAILURE,UNKNOWN}

# obtain format templates from app.cfg
finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS]
finished_job_comments_cfg = config.read_config()[config.SECTION_FINISHED_JOB_COMMENTS]

# check if _bot_jobJOBID.result exits
job_result_file = f"_bot_job{job_id}.result"
Expand All @@ -485,13 +492,13 @@ def process_finished_job(self, finished_job):
job_metadata.JOB_RESULT_SECTION,
self.logfile)

job_result_unknown_fmt = finished_job_comments_cfg[JOB_RESULT_UNKNOWN_FMT]
job_result_unknown_fmt = finished_job_comments_cfg[config.FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT]
# set fallback comment_description in case no result file was found
# (job_metadata.get_section_from_file returned None)
comment_description = job_result_unknown_fmt.format(filename=job_result_file)
if job_results:
# get preformatted comment_description or use previously set default for unknown
comment_description = job_results.get(JOB_RESULT_COMMENT_DESCRIPTION, comment_description)
comment_description = job_results.get(job_metadata.JOB_RESULT_COMMENT_DESCRIPTION, comment_description)

# report to log
log(f"{fn}(): finished job {job_id}\n"
Expand All @@ -514,13 +521,13 @@ def process_finished_job(self, finished_job):
job_metadata.JOB_TEST_SECTION,
self.logfile)

job_test_unknown_fmt = finished_job_comments_cfg[JOB_TEST_UNKNOWN_FMT]
job_test_unknown_fmt = finished_job_comments_cfg[config.FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT]
# set fallback comment_description in case no test file was found
# (job_metadata.get_section_from_file returned None)
comment_description = job_test_unknown_fmt.format(filename=job_test_file)
if job_tests:
# get preformatted comment_description or use previously set default for unknown
comment_description = job_tests.get(JOB_TEST_COMMENT_DESCRIPTION, comment_description)
comment_description = job_tests.get(job_metadata.JOB_TEST_COMMENT_DESCRIPTION, comment_description)

# report to log
log(f"{fn}(): finished job {job_id}, test suite result\n"
Expand Down Expand Up @@ -570,7 +577,11 @@ def main():

# config is read and checked for settings to raise an exception early when
# the job_manager runs
config.check_required_cfg_settings(REQUIRED_CONFIG)
if config.check_required_cfg_settings(REQUIRED_CONFIG):
print("Configuration check: PASSED")
else:
print("Configuration check: FAILED")
sys.exit(1)
github.connect()

job_manager = EESSIBotSoftwareLayerJobManager()
Expand Down Expand Up @@ -611,16 +622,16 @@ def main():
job_manager.scontrol_command = ""
if max_iter != 0:
cfg = config.read_config()
job_mgr = cfg["job_manager"]
job_manager.job_ids_dir = job_mgr.get("job_ids_dir")
job_mgr = cfg[config.SECTION_JOB_MANAGER]
job_manager.job_ids_dir = job_mgr.get(config.JOB_MANAGER_SETTING_JOB_IDS_DIR)
job_manager.submitted_jobs_dir = os.path.join(
job_manager.job_ids_dir, "submitted"
)
job_manager.poll_command = job_mgr.get("poll_command") or False
poll_interval = int(job_mgr.get("poll_interval") or 0)
job_manager.poll_command = job_mgr.get(config.JOB_MANAGER_SETTING_POLL_COMMAND) or False
poll_interval = int(job_mgr.get(config.JOB_MANAGER_SETTING_POLL_INTERVAL) or 0)
if poll_interval <= 0:
poll_interval = 60
job_manager.scontrol_command = job_mgr.get("scontrol_command") or False
job_manager.scontrol_command = job_mgr.get(config.JOB_MANAGER_SETTING_SCONTROL_COMMAND) or False
os.makedirs(job_manager.submitted_jobs_dir, exist_ok=True)

# max_iter
Expand Down
Loading