diff --git a/README.md b/README.md index 8af58a9..feb9d38 100644 --- a/README.md +++ b/README.md @@ -645,17 +645,24 @@ scontrol_command = /usr/bin/scontrol #### `[submitted_job_comments]` section The `[submitted_job_comments]` section specifies templates for messages about newly submitted jobs. -``` -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` -``` -`initial_comment` is used to create a comment to a PR when a new job has been created. - ``` awaits_release = job id `{job_id}` awaits release by job manager ``` `awaits_release` is used to provide a status update of a job (shown as a row in the job's status table). +``` +initial_comment = New job on instance `{app_name}` for architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}` +``` +`initial_comment` is used to create a comment to a PR when a new job has been +created. Note, the part '{accelerator_spec}' is only filled-in by the bot if the +argument 'accelerator' to the `bot: build` command has been used. +``` +with_accelerator =  and accelerator `{accelerator}` +``` +`with_accelerator` is used to provide information about the accelerator the job +should build for if and only if the argument `accelerator:X/Y` has been provided. + #### `[new_job_comments]` section The `[new_job_comments]` section sets templates for messages about jobs whose `hold` flag was released. diff --git a/app.cfg.example b/app.cfg.example index 25abf91..152de2b 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -243,8 +243,9 @@ scontrol_command = /usr/bin/scontrol # are removed, the output (in PR comments) will lack important # information. [submitted_job_comments] -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` awaits_release = job id `{job_id}` awaits release by job manager +initial_comment = New job on instance `{app_name}` for CPU micro-architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}` +with_accelerator =  and accelerator `{accelerator}` [new_job_comments] diff --git a/tasks/build.py b/tasks/build.py index a967ef7..d70103e 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -33,6 +33,7 @@ # Local application imports (anything from EESSI/eessi-bot-software-layer) from connections import github from tools import config, cvmfs_repository, job_metadata, pr_comments, run_cmd +import tools.filter as tools_filter # defaults (used if not specified via, eg, 'app.cfg') @@ -46,7 +47,7 @@ _ERROR_NONE = "none" -Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id')) +Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id', 'accelerator')) # global repo_cfg repo_cfg = {} @@ -474,6 +475,17 @@ def prepare_jobs(pr, cfg, event_info, action_filter): # call to just before download_pr year_month, pr_id, run_dir = create_pr_dir(pr, cfg, event_info) + # determine accelerator from action_filter argument + accelerators = action_filter.get_filter_by_component(tools_filter.FILTER_COMPONENT_ACCEL) + if len(accelerators) == 1: + accelerator = accelerators[0] + elif len(accelerators) > 1: + log(f"{fn}(): found more than one ({len(accelerators)}) accelerator requirement") + accelerator = None + else: + log(f"{fn}(): found no accelerator requirement") + accelerator = None + jobs = [] for arch, slurm_opt in arch_map.items(): arch_dir = arch.replace('/', '_') @@ -501,6 +513,15 @@ def prepare_jobs(pr, cfg, event_info, action_filter): continue else: log(f"{fn}(): context DOES satisfy filter(s), going on with job") + # we reached this point when the filter matched (otherwise we + # 'continue' with the next repository) + # for each match of the filter we create a specific job directory + # however, matching CPU architectures works differently to handling + # accelerators; multiple CPU architectures defined in arch_target_map + # can match the (CPU) architecture component of a filter; in + # contrast, the value of the accelerator filter is just passed down + # to scripts in bot/ directory of the pull request (see function + # prepare_job_cfg and creation of Job tuple below) job_dir = os.path.join(run_dir, arch_dir, repo_id) os.makedirs(job_dir, exist_ok=True) log(f"{fn}(): job_dir '{job_dir}'") @@ -513,11 +534,14 @@ def prepare_jobs(pr, cfg, event_info, action_filter): # prepare job configuration file 'job.cfg' in directory /cfg cpu_target = '/'.join(arch.split('/')[1:]) os_type = arch.split('/')[0] - log(f"{fn}(): arch = '{arch}' => cpu_target = '{cpu_target}' , os_type = '{os_type}'") - prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type) + + log(f"{fn}(): arch = '{arch}' => cpu_target = '{cpu_target}' , os_type = '{os_type}'" + f", accelerator = '{accelerator}'") + + prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type, accelerator) # enlist jobs to proceed - job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id) + job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id, accelerator) jobs.append(job) log(f"{fn}(): {len(jobs)} jobs to proceed after applying white list") @@ -527,7 +551,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter): return jobs -def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, os_type): +def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, os_type, accelerator): """ Set up job configuration file 'job.cfg' in directory /cfg @@ -538,6 +562,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, repo_id (string): identifier of the repository to build for software_subdir (string): software subdirectory to build for (e.g., 'x86_64/generic') os_type (string): type of the os (e.g., 'linux') + accelerator (string): defines accelerator to build for (e.g., 'nvidia/cc80') Returns: None (implicitly) @@ -563,6 +588,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, # [architecture] # software_subdir = software_subdir # os_type = os_type + # accelerator = accelerator job_cfg = configparser.ConfigParser() job_cfg[job_metadata.JOB_CFG_SITE_CONFIG_SECTION] = {} build_env_to_job_cfg_keys = { @@ -607,6 +633,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, job_cfg[job_cfg_arch_section] = {} job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_SOFTWARE_SUBDIR] = software_subdir job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_OS_TYPE] = os_type + job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_ACCELERATOR] = accelerator if accelerator else '' # copy contents of directory containing repository configuration to directory # containing job configuration/metadata @@ -726,11 +753,18 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink): # obtain arch from job.arch_target which has the format OS/ARCH arch_name = '-'.join(job.arch_target.split('/')[1:]) + submitted_job_comments_cfg = config.read_config()[config.SECTION_SUBMITTED_JOB_COMMENTS] + + # set string for accelerator if job.accelerator is defined/set (e.g., not None) + accelerator_spec_str = '' + if job.accelerator: + accelerator_spec = f"{submitted_job_comments_cfg[config.SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR]}" + accelerator_spec_str = accelerator_spec.format(accelerator=job.accelerator) + # get current date and time dt = datetime.now(timezone.utc) # construct initial job comment - submitted_job_comments_cfg = config.read_config()[config.SECTION_SUBMITTED_JOB_COMMENTS] job_comment = (f"{submitted_job_comments_cfg[config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT]}" f"\n|date|job status|comment|\n" f"|----------|----------|------------------------|\n" @@ -741,7 +775,8 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink): arch_name=arch_name, symlink=symlink, repo_id=job.repo_id, - job_id=job_id) + job_id=job_id, + accelerator_spec=accelerator_spec_str) # create comment to pull request repo_name = pr.base.repo.full_name diff --git a/tests/test_app.cfg b/tests/test_app.cfg index fd91ed8..43e11bf 100644 --- a/tests/test_app.cfg +++ b/tests/test_app.cfg @@ -17,8 +17,9 @@ # variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it [submitted_job_comments] -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` awaits_release = job id `{job_id}` awaits release by job manager +initial_comment = New job on instance `{app_name}` for CPU micro-architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}` +with_accelerator =  and accelerator `{accelerator}` [new_job_comments] awaits_lauch = job awaits launch by Slurm scheduler diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 6f79fc7..f432d76 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -282,7 +282,7 @@ def test_create_pr_comment_succeeds(mocked_github, tmpdir): print("CREATING PR COMMENT") ym = datetime.today().strftime('%Y.%m') pr_number = 1 - job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number) + job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number, "fpga/magic") job_id = "123" app_name = "pytest" @@ -311,7 +311,7 @@ def test_create_pr_comment_succeeds_none(mocked_github, tmpdir): print("CREATING PR COMMENT") ym = datetime.today().strftime('%Y.%m') pr_number = 1 - job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number) + job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number, "fpga/magic") job_id = "123" app_name = "pytest" @@ -336,7 +336,7 @@ def test_create_pr_comment_raises_once_then_succeeds(mocked_github, tmpdir): print("CREATING PR COMMENT") ym = datetime.today().strftime('%Y.%m') pr_number = 1 - job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number) + job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number, "fpga/magic") job_id = "123" app_name = "pytest" @@ -361,7 +361,7 @@ def test_create_pr_comment_always_raises(mocked_github, tmpdir): print("CREATING PR COMMENT") ym = datetime.today().strftime('%Y.%m') pr_number = 1 - job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number) + job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number, "fpga/magic") job_id = "123" app_name = "pytest" @@ -387,7 +387,7 @@ def test_create_pr_comment_three_raises(mocked_github, tmpdir): print("CREATING PR COMMENT") ym = datetime.today().strftime('%Y.%m') pr_number = 1 - job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number) + job = Job(tmpdir, "test/architecture", "EESSI", "--speed-up", ym, pr_number, "fpga/magic") job_id = "123" app_name = "pytest" @@ -409,7 +409,7 @@ def test_create_read_metadata_file(mocked_github, tmpdir): # create some test data ym = datetime.today().strftime('%Y.%m') pr_number = 999 - job = Job(tmpdir, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number) + job = Job(tmpdir, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number, "fpga/magic") job_id = "123" @@ -440,14 +440,14 @@ def test_create_read_metadata_file(mocked_github, tmpdir): # use directory that does not exist dir_does_not_exist = os.path.join(tmpdir, "dir_does_not_exist") - job2 = Job(dir_does_not_exist, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number) + job2 = Job(dir_does_not_exist, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number, "fpga/magic") job_id2 = "222" with pytest.raises(FileNotFoundError): create_metadata_file(job2, job_id2, pr_comment) # use directory without write permission dir_without_write_perm = os.path.join("/") - job3 = Job(dir_without_write_perm, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number) + job3 = Job(dir_without_write_perm, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number, "fpga/magic") job_id3 = "333" with pytest.raises(OSError): create_metadata_file(job3, job_id3, pr_comment) @@ -457,7 +457,7 @@ def test_create_read_metadata_file(mocked_github, tmpdir): # use undefined values for parameters # job_id = None - job4 = Job(tmpdir, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number) + job4 = Job(tmpdir, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number, "fpga/magic") job_id4 = None create_metadata_file(job4, job_id4, pr_comment) @@ -472,7 +472,7 @@ def test_create_read_metadata_file(mocked_github, tmpdir): # use undefined values for parameters # job.working_dir = None - job5 = Job(None, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number) + job5 = Job(None, "test/architecture", "EESSI", "--speed_up_job", ym, pr_number, "fpga/magic") job_id5 = "555" with pytest.raises(TypeError): create_metadata_file(job5, job_id5, pr_comment) diff --git a/tools/config.py b/tools/config.py index da8a287..ff641eb 100644 --- a/tools/config.py +++ b/tools/config.py @@ -105,8 +105,9 @@ RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB = 'running_job' SECTION_SUBMITTED_JOB_COMMENTS = 'submitted_job_comments' -SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT = 'initial_comment' SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE = 'awaits_release' +SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT = 'initial_comment' +SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR = 'with_accelerator' SECTION_CLEAN_UP = 'clean_up' CLEAN_UP_SETTING_TRASH_BIN_ROOT_DIR = 'trash_bin_dir' diff --git a/tools/filter.py b/tools/filter.py index 721cf3d..8b14f5e 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -39,6 +39,7 @@ COMPONENT_UNKNOWN = "unknown component={component} in {component}:{pattern}" FILTER_EMPTY_PATTERN = "pattern in filter string '{filter_string}' is empty" FILTER_FORMAT_ERROR = "filter string '{filter_string}' does not conform to format 'component:pattern'" +UNKNOWN_COMPONENT_CONST = "unknown component constant {component}" Filter = namedtuple('Filter', ('component', 'pattern')) @@ -175,6 +176,26 @@ def add_filter_from_string(self, filter_string): raise EESSIBotActionFilterError(msg) self.add_filter(_filter_split[0], _filter_split[1]) + def get_filter_by_component(self, component): + """ + Returns filter pattern for component. + + Args: + component (string): one of FILTER_COMPONENTS + + Returns: + (list): list of pattern for filters whose component matches argument + """ + if component not in FILTER_COMPONENTS: + msg = UNKNOWN_COMPONENT_CONST.format(component=component) + raise EESSIBotActionFilterError(msg) + + pattern = [] + for _filter in self.action_filters: + if component == _filter.component: + pattern.append(_filter.pattern) + return pattern + def remove_filter(self, component, pattern): """ Removes all elements matching the filter given by (component, pattern) diff --git a/tools/job_metadata.py b/tools/job_metadata.py index 286f71c..d400019 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -33,6 +33,7 @@ JOB_CFG_ARCHITECTURE_SECTION = "architecture" JOB_CFG_ARCHITECTURE_OS_TYPE = "os_type" JOB_CFG_ARCHITECTURE_SOFTWARE_SUBDIR = "software_subdir" +JOB_CFG_ARCHITECTURE_ACCELERATOR = "accelerator" JOB_CFG_REPOSITORY_SECTION = "repository" JOB_CFG_REPOSITORY_CONTAINER = "container"