diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c7c8b9244..2e862966d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -71,7 +71,7 @@ jobs: if [ "$EVENTNAME" == "pull_request" ]; then - CHANGES=$(doit util_list_changed_dirs_with_last_commit | tail -n -1) + CHANGES=$(doit util_list_changed_dirs_with_last_commit --exclude-website-metadata --exclude-test-data | tail -n -1) CHANGEDPROJECTS=$(echo $CHANGES | jq -c -r '.changed') REMOVEDPROJECTS=$(echo $CHANGES | jq -c -r '.removed') diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ee8902718..9e29f2641 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -142,7 +142,7 @@ jobs: git config user.name "github-actions" # git fetch and git diff - CHANGES=$(doit util_list_changed_dirs_with_last_commit | tail -n -1) + CHANGES=$(doit util_list_changed_dirs_with_last_commit --exclude-website-metadata --exclude-deployments-metadata --exclude-test-data | tail -n -1) CHANGEDPROJECTS=$(echo $CHANGES | jq -c -r '.changed') REMOVEDPROJECTS=$(echo $CHANGES | jq -c -r '.removed') diff --git a/.github/workflows/pr_flow.yml b/.github/workflows/pr_flow.yml index 55b6dff72..4c89e2709 100644 --- a/.github/workflows/pr_flow.yml +++ b/.github/workflows/pr_flow.yml @@ -24,6 +24,12 @@ permissions: # To allow the docs workflow to submit a comment using thollander/actions-comment-pull-request pull-requests: write +# Project file changes: +# - Always validate it +# - Website only metadata -> Skip test/build +# - Deployment metadata (examples_config.deployments, commands) -> We should redeploy, but skip test/build/doc +# - Docs is special, it's just to pull the `evaluated` branch, and this is only useful to get the built notebooks + jobs: setup: runs-on: ubuntu-latest @@ -34,6 +40,7 @@ jobs: outputs: changedprojects: ${{ steps.set-list.outputs.changedprojects }} removedprojects: ${{ steps.set-list.outputs.removedprojects }} + changedprojectsfile: ${{ steps.set-list.outputs.changedprojectsfile }} steps: - uses: actions/checkout@v3 with: @@ -48,22 +55,34 @@ jobs: - name: infer project list id: set-list run: | - CHANGES=$(doit util_list_changed_dirs_with_main | tail -n -1) - + CHANGES=$(doit util_list_changed_dirs_with_main --exclude-website-metadata --exclude-deployments-metadata | tail -n -1) CHANGEDPROJECTS=$(echo $CHANGES | jq -c -r '.changed') REMOVEDPROJECTS=$(echo $CHANGES | jq -c -r '.removed') - echo "Changed projects: $CHANGEDPROJECTS" - echo "Removed projects: $REMOVEDPROJECTS" - if [ "$CHANGEDPROJECTS" != "[]" -a "$REMOVEDPROJECTS" != "[]" ]; then echo "No support for removing and updating projects together" echo "Open a PR that just remove project" exit 1 fi + echo "Changed projects: $CHANGEDPROJECTS" + echo "Removed projects: $REMOVEDPROJECTS" + echo "CHANGEDPROJECTS=$CHANGEDPROJECTS" >> $GITHUB_OUTPUT echo "REMOVEDPROJECTS=$REMOVEDPROJECTS" >> $GITHUB_OUTPUT + # There's logic to skip testing/building when only some metadata in the + # project file changes. In these cases, we still want to validate the project file. + - name: validate project files + run: | + CHANGESPROJECTSFILE=$(doit util_list_changed_dirs_with_main --only-project-file | tail -n -1) + CHANGEDPROJECTSFILE=$(echo $CHANGESPROJECTSFILE | jq -c -r '.changed') + echo "Changed projects (project file only): $CHANGEDPROJECTSFILE" + if [ "$CHANGEDPROJECTSFILE" != "[]" ]; then + items=$(echo $CHANGEDPROJECTSFILE | jq -c -r '.[]') + for item in ${items[@]}; do + doit validate_project_file:$item + done + fi test: needs: setup uses: ./.github/workflows/test.yml diff --git a/dodo.py b/dodo.py index 3022660e3..24cd0177b 100644 --- a/dodo.py +++ b/dodo.py @@ -41,19 +41,18 @@ 'template', ] -PROJECT_CONFIG_IGNORE_KEYS = [ - # Website only +PROJECT_CONFIG_IGNORES_KEYS_WEBSITE = [ 'description', 'examples_config.created', 'examples_config.maintainers', 'examples_config.labels', 'examples_config.title', 'examples_config.categories', +] - # TODO: Ideally, we could ignore these two to test/build the project - # but consider them to re-deploy a project. - # 'examples_config.deployments', - # 'commands', +PROJECT_CONFIG_IGNORES_KEYS_DEPLOYMENTS = [ + 'examples_config.deployments', + 'commands', ] CATNAME_TO_CAT_MAP = { @@ -190,6 +189,34 @@ 'default': 'all' } +exclude_website_metadata_param = { + 'name': 'exclude_website_metadata', + 'long': 'exclude-website-metadata', + 'type': bool, + 'default': False, +} + +exclude_deployments_metadata_param = { + 'name': 'exclude_deployments_metadata', + 'long': 'exclude-deployments-metadata', + 'type': bool, + 'default': False, +} + +only_project_file_param = { + 'name': 'only_project_file', + 'long': 'only-project-file', + 'type': bool, + 'default': False, +} + +exclude_test_data_param = { + 'name': 'exclude_test_data', + 'long': 'exclude-test-data', + 'type': bool, + 'default': False, +} + ##### Exceptions #### class ExamplesError(Exception): @@ -342,7 +369,7 @@ def remove_nested_key(mapping, ref): return mapping -def yaml_file_changed(file_path, git_cmd_tmpl, ignore_keys=PROJECT_CONFIG_IGNORE_KEYS): +def yaml_file_changed(file_path, git_cmd_tmpl, ignored_keys: list): """ Check whether the content of a yaml file changed between the current branch and some git reference, optionally ignoring some keys. @@ -359,13 +386,13 @@ def yaml_file_changed(file_path, git_cmd_tmpl, ignore_keys=PROJECT_CONFIG_IGNORE ) previous_yaml = safe_load(previous_version.stdout.decode()) - current_yaml = remove_ignored_keys(current_yaml, ignore_keys) - previous_yaml = remove_ignored_keys(previous_yaml, ignore_keys) + current_yaml = remove_ignored_keys(current_yaml, ignored_keys) + previous_yaml = remove_ignored_keys(previous_yaml, ignored_keys) return current_yaml != previous_yaml -def print_changes_in_dir(paths: list[str], project_file_changed): +def print_changes_in_dir(paths: list[str], project_file_changed_cb, only_project_file=False, exclude_test_data=False): """Dumps as JSON a dict of the changed projects and removed projects. New projects are in the changed list. @@ -375,16 +402,27 @@ def print_changes_in_dir(paths: list[str], project_file_changed): changed_dirs = [] removed_dirs = [] for path in paths: + if path.name != 'anaconda-project.yml' and only_project_file: + continue root = path.parts[0] # empty suffix is a hint for a directory, useful to catch when # a non-project file has been removed if pathlib.Path(root).is_file() or pathlib.Path(root).suffix != '': continue + if not exclude_test_data and root == 'test_data': + try: + test_data_dir = path.parts[1] + except IndexError: + print(f'Unhandled path when printing the changes {path}') + continue + if test_data_dir in all_projects: + changed_dirs.append(test_data_dir) + continue if root in DEFAULT_EXCLUDE: continue if root in all_projects: if path.name == 'anaconda-project.yml': - if project_file_changed(path): + if project_file_changed_cb(path): changed_dirs.append(root) else: changed_dirs.append(root) @@ -1069,7 +1107,12 @@ def task_util_list_changed_dirs_with_main(): """ Print the projects that changed compared to main """ - def list_changed_dirs_with_main(): + def list_changed_dirs_with_main( + exclude_website_metadata, + exclude_deployments_metadata, + only_project_file, + exclude_test_data, + ): subprocess.run( ['git', 'fetch', 'origin', 'main'] ) @@ -1079,25 +1122,69 @@ def list_changed_dirs_with_main(): ) files = result.stdout.decode().splitlines() tmpl = 'git show $(git merge-base origin/main HEAD):{file_path}' - print_changes_in_dir(files, functools.partial(yaml_file_changed, git_cmd_tmpl=tmpl)) + ignored_keys = [] + if not only_project_file: + if exclude_website_metadata: + ignored_keys.extend(PROJECT_CONFIG_IGNORES_KEYS_WEBSITE) + if exclude_deployments_metadata: + ignored_keys.extend(PROJECT_CONFIG_IGNORES_KEYS_DEPLOYMENTS) + func = functools.partial( + yaml_file_changed, + git_cmd_tmpl=tmpl, + ignored_keys=ignored_keys, + ) + print_changes_in_dir(files, func, only_project_file, exclude_test_data) - return {'actions': [list_changed_dirs_with_main]} + return { + 'actions': [list_changed_dirs_with_main], + 'params': [ + exclude_website_metadata_param, + exclude_deployments_metadata_param, + only_project_file_param, + exclude_test_data_param, + ], + } def task_util_list_changed_dirs_with_last_commit(): """ Print the projects that changed compared to the last commit. """ - def list_changed_dirs_with_main(): + def list_changed_dirs_with_last_commit( + exclude_website_metadata, + exclude_deployments_metadata, + only_project_file, + exclude_test_data, + ): result = subprocess.run( - ['git', 'diff', '--merge-base', '--name-only', 'origin/main'], + ['git', 'diff', 'HEAD^', 'HEAD', '--name-only'], stdout=subprocess.PIPE ) files = result.stdout.decode().splitlines() tmpl = 'git show HEAD^:{file_path}' - print_changes_in_dir(files, functools.partial(yaml_file_changed, git_cmd_tmpl=tmpl)) + ignored_keys = [] + if not only_project_file: + if exclude_website_metadata: + ignored_keys.extend(PROJECT_CONFIG_IGNORES_KEYS_WEBSITE) + if exclude_deployments_metadata: + ignored_keys.extend(PROJECT_CONFIG_IGNORES_KEYS_DEPLOYMENTS) + func = functools.partial( + yaml_file_changed, + git_cmd_tmpl=tmpl, + ignored_keys=ignored_keys, + ) + print_changes_in_dir(files, func, only_project_file, exclude_test_data) + - return {'actions': [list_changed_dirs_with_main]} + return { + 'actions': [list_changed_dirs_with_last_commit], + 'params': [ + exclude_website_metadata_param, + exclude_deployments_metadata_param, + only_project_file_param, + exclude_test_data_param, + ], + } def task_util_list_comma_separated_projects():