From bb323478d6cd80b98f0b7ea2faf57a22f9500651 Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Mon, 3 Oct 2022 14:50:10 -0500 Subject: [PATCH 1/7] merging: Extract Config subclass from MergeUpstreamConfig - Make a subclass so that common code for the MergeStagingToProductionConfig can also use it. - Isolate common functions create_from_dlproject() and asyaml() - Move the YAML key to a ClassVar so that each subclass has its own key. --- tasks/merging.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tasks/merging.py b/tasks/merging.py index 7d39c2cc7624e..0c34f6e355e3e 100644 --- a/tasks/merging.py +++ b/tasks/merging.py @@ -10,6 +10,7 @@ import shutil import tempfile import textwrap +import typing from enum import Enum, auto from typing import Optional @@ -88,40 +89,56 @@ def url(self) -> str: return f'git@{self.host}:{self.fork}/conan-center-index.git' -@dataclasses.dataclass -class MergeUpstreamConfig: - """Configuration for the merge-upstream task.""" - cci: ConanCenterIndexConfig = dataclasses.field(default_factory=ConanCenterIndexConfig) - """Configuration for Conan Center Index""" - upstream: UpstreamConfig = dataclasses.field(default_factory=UpstreamConfig) - """Configuration for the Datalogics upstream""" - pull_request: PullRequestConfig = dataclasses.field(default_factory=PullRequestConfig) +class Config: + """Base class for Config dataclasses that read from dlproject.yaml""" + yaml_key = None class ConfigurationError(Exception): """Configuration error when reading data.""" + @classmethod + def _check_attributes(cls): + if cls.yaml_key is None: + raise NotImplementedError(f"Class {cls.__name__} must define 'yaml_key' as a 'ClassVar[str]' \n" + ' which indicates the key for the config in dlproject.yaml.') + @classmethod def create_from_dlproject(cls): - """Create a MergeUpstreamConfig with defaults updated from dlproject.yaml""" + """Create an instance of cls with defaults updated from dlproject.yaml""" + cls._check_attributes() with open('dlproject.yaml', encoding='utf-8') as dlproject_file: dlproject = yaml.safe_load(dlproject_file) - config_data = dlproject.get('merge_upstream', {}) + config_data = dlproject.get(cls.yaml_key, {}) try: - return dacite.from_dict(data_class=MergeUpstreamConfig, + return dacite.from_dict(data_class=cls, data=config_data, config=dacite.Config(strict=True)) except dacite.DaciteError as exception: raise cls.ConfigurationError( - f'Error reading merge_upstream from dlproject.yaml: {exception}') from exception + f'Error reading {cls.yaml_key} from dlproject.yaml: {exception}') from exception def asyaml(self): """Return a string containing the yaml for this dataclass, in canonical form.""" + self._check_attributes() # sort_keys=False to preserve the ordering that's in the dataclasses # dict objects preserve order since Python 3.7 return yaml.dump(dataclasses.asdict(self), sort_keys=False, indent=4) +@dataclasses.dataclass +class MergeUpstreamConfig(Config): + """Configuration for the merge-upstream task.""" + cci: ConanCenterIndexConfig = dataclasses.field(default_factory=ConanCenterIndexConfig) + """Configuration for Conan Center Index""" + upstream: UpstreamConfig = dataclasses.field(default_factory=UpstreamConfig) + """Configuration for the Datalogics upstream""" + pull_request: PullRequestConfig = dataclasses.field(default_factory=PullRequestConfig) + """Configuration for the pull request""" + yaml_key: typing.ClassVar[str] = 'merge_upstream' + """Key for this configuration in dlproject.yaml.""" + + @dataclasses.dataclass class GitFileStatus: """A Git status""" From 55c0ad8b859d597867e698d579e2bb07401b17ff Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Mon, 3 Oct 2022 14:56:24 -0500 Subject: [PATCH 2/7] Add merge-staging-to-production task --- dlproject.yaml | 6 ++++++ tasks/__init__.py | 1 + tasks/merging.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/dlproject.yaml b/dlproject.yaml index 95526ae3d2af4..bd3501ad9160f 100644 --- a/dlproject.yaml +++ b/dlproject.yaml @@ -647,3 +647,9 @@ merge_upstream: # assignee: null # labels: # - from-conan-io +merge_staging_to_production: + # Defaults: + # host: octocat.dlogics.com + # organization: datalogics + # staging_branch: develop + # production_branch: master diff --git a/tasks/__init__.py b/tasks/__init__.py index 27ba7ebb45e07..d5b306b9e83e4 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -136,6 +136,7 @@ def upload_one_package_name(ctx, package_name, remote, upload=True): tasks = [] tasks.extend([v for v in locals().values() if isinstance(v, Task)]) tasks.append(merging.merge_upstream) +tasks.append(merging.merge_staging_to_production) conan_tasks = Collection() conan_tasks.add_task(conan.install_config) diff --git a/tasks/merging.py b/tasks/merging.py index 0c34f6e355e3e..7b550d1ee062d 100644 --- a/tasks/merging.py +++ b/tasks/merging.py @@ -139,6 +139,26 @@ class MergeUpstreamConfig(Config): """Key for this configuration in dlproject.yaml.""" +@dataclasses.dataclass +class MergeStagingToProductionConfig(Config): + """Configuration describing parameters for production merges in the upstream repo. (usually Datalogics)""" + host: str = 'octocat.dlogics.com' + """Host for the Datalogics upstream""" + organization: str = 'datalogics' + """Name of the upstream organization""" + staging_branch: str = 'develop' + """Name of the staging branch""" + production_branch: str = 'master' + """Name of the production branch""" + yaml_key: typing.ClassVar[str] = 'merge_staging_to_production' + """Key for this configuration in dlproject.yaml.""" + + @property + def url(self) -> str: + """The URL for the upstream Git repository.""" + return f'git@{self.host}:{self.organization}/conan-center-index.git' + + @dataclasses.dataclass class GitFileStatus: """A Git status""" @@ -178,6 +198,24 @@ def merge_upstream(ctx): _write_status_file(MergeStatus.PULL_REQUEST) +@Task +def merge_staging_to_production(ctx): + """Merge the staging branch to the production branch""" + config = MergeStagingToProductionConfig.create_from_dlproject() + logger.info('merge-staging-to-production configuration:\n%s', config.asyaml()) + with _preserving_branch_and_commit(ctx): + logger.info('Check out production branch...') + ctx.run(f'git fetch {config.url} {config.production_branch}') + ctx.run('git checkout --detach FETCH_HEAD') + + logger.info('Merge staging branch...') + ctx.run(f'git fetch {config.url} {config.staging_branch}') + ctx.run(f'git merge --no-ff --no-edit --no-verify --into-name {config.production_branch} FETCH_HEAD') + + logger.info('Push merged production branch...') + ctx.run(f'git push {config.url} HEAD:refs/heads/{config.production_branch}') + + def _remove_status_file(): try: os.remove(MERGE_UPSTREAM_STATUS) From 669cf4e808eae96bb1a9390799b3185e5708b2bc Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Mon, 3 Oct 2022 16:49:43 -0500 Subject: [PATCH 3/7] merging: Factor writing status file and counting revisions --- tasks/merging.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tasks/merging.py b/tasks/merging.py index 7b550d1ee062d..6fc2c434b1c95 100644 --- a/tasks/merging.py +++ b/tasks/merging.py @@ -183,19 +183,19 @@ def merge_upstream(ctx): logger.info('merge-upstream configuration:\n%s', config.asyaml()) # if anything fails past this point, the missing status file will also abort the Jenkins run. - _remove_status_file() + _remove_status_file(MERGE_UPSTREAM_STATUS) # Nested context handlers; see https://docs.python.org/3.10/reference/compound_stmts.html#the-with-statement with _preserving_branch_and_commit(ctx), _merge_remote(ctx, config): # Try to merge from CCI try: - _write_status_file(_merge_and_push(ctx, config)) + _write_status_file(_merge_and_push(ctx, config), to_file=MERGE_UPSTREAM_STATUS) except MergeHadConflicts: try: pr_body = _form_pr_body(ctx, config) finally: ctx.run('git merge --abort') _create_pull_request(ctx, config, pr_body) - _write_status_file(MergeStatus.PULL_REQUEST) + _write_status_file(MergeStatus.PULL_REQUEST, to_file=MERGE_UPSTREAM_STATUS) @Task @@ -216,18 +216,18 @@ def merge_staging_to_production(ctx): ctx.run(f'git push {config.url} HEAD:refs/heads/{config.production_branch}') -def _remove_status_file(): +def _remove_status_file(filename): try: - os.remove(MERGE_UPSTREAM_STATUS) + os.remove(filename) except FileNotFoundError: pass -def _write_status_file(merge_status): +def _write_status_file(merge_status, to_file): """Write the merge status to the status file.""" - logger.info('Write status %s to file %s', merge_status.name, MERGE_UPSTREAM_STATUS) - with open(MERGE_UPSTREAM_STATUS, 'w', encoding='utf-8') as merge_upstream_status: - merge_upstream_status.write(merge_status.name) + logger.info('Write status %s to file %s', merge_status.name, to_file) + with open(to_file, 'w', encoding='utf-8') as status: + status.write(merge_status.name) @contextlib.contextmanager @@ -295,6 +295,13 @@ def _branch_exists(ctx, branch): return result.ok +def _count_revs(ctx, commit): + """Count the revisions in the given commit, which can be a range like branch..HEAD, or + other commit expression.""" + count_revs_result = ctx.run(f'git rev-list {commit} --count', hide='stdout', pty=False) + return int(count_revs_result.stdout) + + def _merge_and_push(ctx, config): """Attempt to merge upstream branch and push it to the local repo.""" logger.info('Check out local %s branch...', config.upstream.branch) @@ -368,10 +375,8 @@ def _retrieve_merge_conflicts(ctx): def _maybe_push(ctx, config): """Check to see if a push is necessary by counting the number of revisions that differ between current head and the push destination. Push if necessary""" - count_revs_result = ctx.run( - f'git rev-list {config.upstream.remote_name}/{config.upstream.branch}..HEAD --count', - hide='stdout', pty=False) - if int(count_revs_result.stdout) == 0: + + if _count_revs(ctx, f'{config.upstream.remote_name}/{config.upstream.branch}..HEAD') == 0: logger.info('Repo is already up to date') return MergeStatus.UP_TO_DATE logger.info('Push to local repo...') From e506f4f7ed8caf5e79471877afaad3a9b44544b2 Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Mon, 3 Oct 2022 16:52:45 -0500 Subject: [PATCH 4/7] Add merge-staging-to-production to Jenkins - Add a parameter MERGE_STAGING_TO_PRODUCTION - Return the status from the Invoke task to Jenkins. - Skip the rest of the run if there were no further changes. --- .gitignore | 1 + Jenkinsfile | 23 +++++++++++++++++++++++ tasks/merging.py | 7 +++++++ 3 files changed, 31 insertions(+) diff --git a/.gitignore b/.gitignore index 55864f394a9f7..cf4c384a2baec 100644 --- a/.gitignore +++ b/.gitignore @@ -456,3 +456,4 @@ requirements.txt # outputs from invoke tasks /.merge-upstream-status +/.merge-staging-to-production-status diff --git a/Jenkinsfile b/Jenkinsfile index 45f72d5ac33c4..8ae9c79089ea4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,8 @@ pipeline { description: 'Force build of all tools, and their requirements. By default, Conan will download the tool and test it if it\'s already built' booleanParam name: 'MERGE_UPSTREAM', defaultValue: false, description: 'If building develop branch, merge changes from upstream, i.e., conan-io/conan-center-index' + booleanParam name: 'MERGE_STAGING_TO_PRODUCTION', defaultValue: false, + description: 'If building master branch, merge changes from the develop branch' } options{ buildDiscarder logRotator(artifactDaysToKeepStr: '4', artifactNumToKeepStr: '10', daysToKeepStr: '7', numToKeepStr: '10') @@ -209,6 +211,27 @@ pipeline { } } } + stage('Merge staging to production') { + when { + expression { + // Merge upstream on master-prefixed branches if forced by parameter + env.BRANCH_NAME =~ 'master' && params.MERGE_STAGING_TO_PRODUCTION + } + } + steps { + script { + sh """ + . ${ENV_LOC['noarch']}/bin/activate + invoke merge-staging-to-production + """ + def merge_staging_to_production_status = readFile(file: '.merge-staging-to-production-status') + echo "merge-staging-to-production status is ${merge_staging_to_production_status}" + // If the status of the merge is MERGED, then don't do anything + // else; Jenkins will notice the branch changed and re-run. + skipBuilding = merge_staging_to_production_status == 'MERGED' + } + } + } stage('Upload new or changed recipes') { when { allOf { diff --git a/tasks/merging.py b/tasks/merging.py index 6fc2c434b1c95..18d62397a4480 100644 --- a/tasks/merging.py +++ b/tasks/merging.py @@ -20,6 +20,7 @@ # Name of a status file MERGE_UPSTREAM_STATUS = '.merge-upstream-status' +MERGE_STAGING_TO_PRODUCTION_STATUS = '.merge-staging-to-production-status' logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -204,16 +205,22 @@ def merge_staging_to_production(ctx): config = MergeStagingToProductionConfig.create_from_dlproject() logger.info('merge-staging-to-production configuration:\n%s', config.asyaml()) with _preserving_branch_and_commit(ctx): + _remove_status_file(MERGE_STAGING_TO_PRODUCTION_STATUS) logger.info('Check out production branch...') ctx.run(f'git fetch {config.url} {config.production_branch}') ctx.run('git checkout --detach FETCH_HEAD') logger.info('Merge staging branch...') ctx.run(f'git fetch {config.url} {config.staging_branch}') + if _count_revs(ctx, 'HEAD..FETCH_HEAD') == 0: + logger.info('%s is up to date.', config.production_branch) + _write_status_file(MergeStatus.UP_TO_DATE, to_file=MERGE_STAGING_TO_PRODUCTION_STATUS) + return ctx.run(f'git merge --no-ff --no-edit --no-verify --into-name {config.production_branch} FETCH_HEAD') logger.info('Push merged production branch...') ctx.run(f'git push {config.url} HEAD:refs/heads/{config.production_branch}') + _write_status_file(MergeStatus.MERGED, to_file=MERGE_STAGING_TO_PRODUCTION_STATUS) def _remove_status_file(filename): From c667f742af9368a16a808df6827b43146cc93ec9 Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Tue, 4 Oct 2022 11:33:02 -0500 Subject: [PATCH 5/7] dlproject.yaml: Use common build profiles from conan-config - Use the common build profiles from conan-config for the host profile for all the tools builders. - Don't use build profiles (profile_build) yet, because not all the recipes in Conan Center Index support them yet. To support build and host profiles, a recipe has to use the new Conan tools and environments. --- dlproject.yaml | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/dlproject.yaml b/dlproject.yaml index bd3501ad9160f..54e67c0e0a1f8 100644 --- a/dlproject.yaml +++ b/dlproject.yaml @@ -178,7 +178,7 @@ config: build_folder: build-release description: macOS Release profile_host: - - apple-clang-13.0-intel + - build-profile-macos-intel DebugTool: &macOSDebugTool include: - ReleaseTool @@ -230,7 +230,7 @@ config: build_folder: build-release description: macOS Release profile_host: - - apple-clang-13.0-arm + - build-profile-macos-arm DebugTool: include: - ReleaseTool @@ -292,7 +292,7 @@ config: build_folder: build-release-tool description: RedHat Release Tool profile_host: - - devtoolset-7 + - build-profile-linux-intel DebugTool: &redhatDebugTool include: ReleaseTool build_folder: build-debug-tool @@ -350,22 +350,18 @@ config: description: RedHat Debug settings: - build_type=Debug - ToolCommon: - env: - # This is necessary to get SWIG to link on ARM; for some reason, it's not automatic - - swig:LDFLAGS=-ldl ReleaseTool: build_folder: build-release-tool description: RedHat Release - include: - - Release - - ToolCommon + profile_host: + - build-profile-linux-arm DebugTool: build_folder: build-debug-tool description: RedHat Debug include: - - Debug - - ToolCommon + - ReleaseTool + settings: + - build_type=Debug prebuilt_tools: - cmake/[>=3.23.0] - doxygen/1.9.1 @@ -420,7 +416,7 @@ config: ReleaseTool: build_folder: build-release-tool description: Windows Release - profile_host: visual-studio-16 + profile_host: build-profile-windows-intel DebugTool: build_folder: build-debug-tool description: Windows Debug Tool @@ -493,12 +489,11 @@ config: <<: *aixCommon build_folder: build-release-tool description: AIX Release - profile_host: aix-xlc16-ppc + profile_host: build-profile-aix-ppc ReleaseToolGCC: build_folder: build-release-tool description: AIX Release Tool with GCC - include: - - Release + profile_host: build-profile-aix-ppc-gcc DebugTool: build_folder: build-debug-tool description: AIX Debug Tool @@ -577,11 +572,7 @@ config: ReleaseTool: build_folder: build-release-tool description: Sparc Release Tool - include: - - Release - env: - # Override to not contain -std=c99, which breaks m4 by turning off the 'asm' keyword - - CFLAGS=-pthread -m64 + profile_host: build-profile-solaris-sparc DebugTool: build_folder: build-debug-tool description: Sparc Debug Tool @@ -592,11 +583,7 @@ config: ReleaseTool32: build_folder: build-release-tool-32 description: Sparc Release 32 Tool 32 - include: - - Release32 - env: - # Override to not contain -std=c99, which breaks m4 by turning off the 'asm' keyword - - CFLAGS="-pthread -m32" + profile_host: build-profile-solaris-sparc-32 DebugTool32: build_folder: build-debug-tool-32 description: Sparc Debug 32 Tool From 5d656f7750c454819faf2bc62df64fd870e58242 Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Wed, 5 Oct 2022 11:42:04 -0500 Subject: [PATCH 6/7] merging: asyaml() dumps configs with their main key --- tasks/merging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/merging.py b/tasks/merging.py index 18d62397a4480..f6324918ee524 100644 --- a/tasks/merging.py +++ b/tasks/merging.py @@ -124,7 +124,7 @@ def asyaml(self): self._check_attributes() # sort_keys=False to preserve the ordering that's in the dataclasses # dict objects preserve order since Python 3.7 - return yaml.dump(dataclasses.asdict(self), sort_keys=False, indent=4) + return yaml.dump({self.yaml_key: dataclasses.asdict(self)}, sort_keys=False, indent=4) @dataclasses.dataclass From d90a8520d761ac944fb24fc58a77c663ebb79362 Mon Sep 17 00:00:00 2001 From: "Kevin A. Mitchell" Date: Wed, 5 Oct 2022 16:52:31 -0500 Subject: [PATCH 7/7] test_tools: When installing msys2 from Conan, update --- tests/test_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index c669b4b95abba..38fa0890bdd4c 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -103,8 +103,8 @@ def msys_env(release_tool_config, tmpdir_factory, upload_to): if platform.system() == 'Windows': msys2_dir = tmpdir_factory.mktemp('msys2_install') install_json = msys2_dir / 'install.json' - args = ['conan', 'install', 'msys2/cci.latest@', '-if', msys2_dir, '-g', 'json', '--build', 'missing', - '-j', install_json] + args = ['conan', 'install', '--update', 'msys2/cci.latest@', '-if', msys2_dir, '-g', 'json', + '--build', 'missing', '-j', install_json] args.extend(release_tool_config.install_options()) subprocess.run(args, check=True)