Run all checks #154

Workflow file for this run

	name: Run all checks

	on:
	pull_request:
	branches:
	- main
	workflow_dispatch:
	inputs:
	ref:
	description: 'The git ref to build the package for'
	required: false
	default: ''
	type: string
	use_lkg:
	description: 'Whether to use the last known good versions of dependencies'
	required: false
	default: True
	type: boolean
	# nightly
	schedule:
	- cron: '0 0 * * *'

	# Only run once per PR, canceling any previous runs
	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	# Precompute the ref if the workflow was triggered by a workflow dispatch rather than copying this logic repeatedly
	env:
	ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref \|\| null }}
	# we want to use the LKG if that is explicitly requested, or if we're in a PR, but not a nightly run
	# the final `\|\| ''` is because env vars are always converted to strings and the string 'false' is truthy (!!)
	# (see https://github.com/orgs/community/discussions/25645)
	use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) \|\| github.event_name == 'pull_request' \|\| ''}}

	jobs:
	eval:
	name: Evaluate changes
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	name: Checkout repository
	with:
	ref: ${{ env.ref }}
	fetch-depth: 2

	# We want to enforce the following rules for PRs:
	# * if all modifications are to README.md
	# no testing is needed
	# * if there are modifications to docs/* or to any code
	# then docs need to be built to verify consistency
	# * if there are modifications to notebooks/* or to any code
	# then notebooks need to be run to verify consistency
	# * for any code changes (or changes to metadata files)
	# linting and testing should be run
	# For a PR build, HEAD will be the merge commit, and we want to diff against the base branch,
	# which will be the first parent: HEAD^
	# (For non-PR changes, we will always perform all CI tasks)
	# Note that GitHub Actions provides path filters, but they operate at the workflow level, not the job level
	- run: \|
	if ($env:GITHUB_EVENT_NAME -eq 'pull_request') {
	$editedFiles = git diff HEAD^ --name-only
	$editedFiles # echo edited files to enable easier debugging
	$codeChanges = $false
	$docChanges = $false
	$nbChanges = $false
	$changeType = "none"
	foreach ($file in $editedFiles) {
	switch -Wildcard ($file) {
	"README.md" { Continue }
	".gitignore" { Continue }
	"econml/_version.py" { Continue }
	"prototypes/*" { Continue }
	"images/*" { Continue }
	"doc/*" { $docChanges = $true; Continue }
	"notebooks/*" { $nbChanges = $true; Continue }
	default { $codeChanges = $true; Continue }
	}
	}
	}
	echo "buildDocs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($docChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT
	echo "buildNbs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($nbChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT
	echo "testCode=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or $codeChanges)" >> $env:GITHUB_OUTPUT
	shell: pwsh
	name: Determine type of code change
	id: eval
	outputs:
	buildDocs: ${{ steps.eval.outputs.buildDocs }}
	buildNbs: ${{ steps.eval.outputs.buildNbs }}
	testCode: ${{ steps.eval.outputs.testCode }}

	lint:
	name: Lint code
	needs: [eval]
	if: ${{ needs.eval.outputs.testCode == 'True' }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	name: Checkout repository
	with:
	ref: ${{ env.ref }}
	- uses: actions/setup-python@v4
	name: Setup Python
	with:
	python-version: '3.9'
	- run: python -m pip install --upgrade pip && pip install --upgrade setuptools
	name: Ensure latest pip and setuptools
	- run: 'pip install pycodestyle && pycodestyle econml'

	notebooks:
	name: Run notebooks
	needs: [eval]
	if: ${{ needs.eval.outputs.buildNbs == 'True' }}
	runs-on: ubuntu-latest
	strategy:
	matrix:
	kind: [except-customer-scenarios, customer-scenarios]
	include:
	- kind: "except-customer-scenarios"
	extras: "[tf,plt]"
	pattern: "(?!CustomerScenarios)"
	install_graphviz: true
	version: '3.8' # no supported version of tensorflow for 3.9
	- kind: "customer-scenarios"
	extras: "[plt,dowhy]"
	pattern: "CustomerScenarios"
	version: '3.9'
	install_graphviz: false
	fail-fast: false
	steps:
	- uses: actions/checkout@v3
	name: Checkout repository
	with:
	ref: ${{ env.ref }}
	- uses: actions/setup-python@v4
	name: Setup Python
	with:
	python-version: ${{ matrix.version }}
	- run: python -m pip install --upgrade pip && pip install --upgrade setuptools
	name: Ensure latest pip and setuptools
	- run: sudo apt-get -yq install graphviz
	name: Install graphviz
	if: ${{ matrix.install_graphviz }}
	# Add verbose flag to pip installation if in debug mode
	- run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg-notebook.txt' }}
	name: Install econml
	# Install notebook requirements (if not already done as part of lkg)
	- run: pip install jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm
	name: Install notebook requirements
	if: ${{ !env.use_lkg }}
	# NOTE: if you change the requirements file name here, the pattern to parse it must also be updated in generate_lkg.py
	- run: pip freeze --exclude-editable > notebooks-${{ matrix.version }}-${{ matrix.kind }}-requirements.txt
	name: Save installed packages
	- uses: actions/upload-artifact@v3
	name: Upload installed packages
	with:
	name: requirements
	path: notebooks-${{ matrix.version }}-${{ matrix.kind }}-requirements.txt
	- run: pip install pytest pytest-runner coverage
	name: Install pytest

	- run: python setup.py pytest
	name: Run notebook tests
	id: run_tests
	env:
	PYTEST_ADDOPTS: '-m "notebook"'
	NOTEBOOK_DIR_PATTERN: ${{ matrix.pattern }}
	COVERAGE_PROCESS_START: 'setup.cfg'
	- run: mv .coverage .coverage.${{ matrix.kind }}
	# Run whether or not the tests passed, but only if they ran at all
	if: success() \|\| failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
	name: Make coverage filename unique
	- uses: actions/upload-artifact@v3
	name: Upload coverage report
	if: success() \|\| failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
	with:
	name: coverage
	path: .coverage.${{ matrix.kind }}

	tests:
	name: "Run tests"
	needs: [eval]
	if: ${{ needs.eval.outputs.testCode == 'True' }}
	strategy:
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	python-version: ['3.7', '3.8', '3.9', '3.10']
	kind: [serial, other, dml, main, treatment]
	exclude:
	# Serial tests fail randomly on mac sometimes, so we don't run them there
	- os: macos-latest
	kind: serial
	# Python 3.7 is broken on the mac runner image, see https://github.com/actions/runner-images/issues/7764
	- os: macos-latest
	python-version: '3.7'
	# Assign the correct package and testing options for each kind of test
	include:
	- kind: serial
	opts: '-m "serial" -n 1'
	extras: "[tf,plt]"
	- kind: other
	opts: '-m "cate_api" -n auto'
	extras: "[tf,plt]"
	- kind: dml
	opts: '-m "dml"'
	extras: "[tf,plt]"
	- kind: main
	opts: '-m "not (notebook or automl or dml or serial or cate_api or treatment_featurization)" -n 2'
	extras: "[tf,plt,dowhy]"
	- kind: treatment
	opts: '-m "treatment_featurization" -n auto'
	extras: "[tf,plt]"
	fail-fast: false
	runs-on: ${{ matrix.os }}
	steps:
	- uses: actions/checkout@v3
	name: Checkout repository
	with:
	ref: ${{ env.ref }}
	- uses: actions/setup-python@v4
	name: Setup Python
	with:
	python-version: ${{ matrix.python-version }}
	- run: python -m pip install --upgrade pip && pip install --upgrade setuptools
	name: Ensure latest pip and setuptools
	# Add verbose flag to pip installation if in debug mode
	- run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg.txt' }}
	name: Install econml
	# NOTE: if you change the requirements file name here, the pattern to parse it must also be updated in generate_lkg.py
	- run: pip freeze --exclude-editable > tests-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}-requirements.txt
	name: Save installed packages
	- uses: actions/upload-artifact@v3
	name: Upload installed packages
	with:
	name: requirements
	path: tests-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}-requirements.txt
	- run: pip install pytest pytest-runner coverage
	name: Install pytest
	- run: python setup.py pytest
	name: Run tests
	id: run_tests
	env:
	PYTEST_ADDOPTS: ${{ matrix.opts }}
	COVERAGE_PROCESS_START: 'setup.cfg'
	- run: mv .coverage .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}
	# Run whether or not the tests passed, but only if they ran at all
	if: success() \|\| failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
	name: Make coverage filename unique
	- uses: actions/upload-artifact@v3
	name: Upload coverage report
	if: success() \|\| failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
	with:
	name: coverage
	path: .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}

	coverage-report:
	name: "Coverage report"
	needs: [tests, notebooks]
	if: success() \|\| failure()
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	name: Checkout repository
	with:
	ref: ${{ env.ref }}
	- uses: actions/download-artifact@v3
	name: Get coverage reports
	with:
	name: coverage
	path: coverage
	- uses: actions/setup-python@v4
	name: Setup Python
	with:
	python-version: '3.8'
	- run: pip install coverage
	name: Install coverage
	- run: coverage combine coverage/
	name: Combine coverage reports
	- run: coverage report -m --format=markdown > $GITHUB_STEP_SUMMARY
	name: Generate coverage report
	- run: coverage html
	name: Generate coverage html --fail-under=86
	- uses: actions/upload-artifact@v3
	name: Upload coverage report
	with:
	name: coverage
	path: htmlcov

	build:
	name: Build package
	needs: [eval]
	if: ${{ needs.eval.outputs.testCode == 'True' }}
	uses: ./.github/workflows/publish-package.yml
	with:
	publish: false
	repository: testpypi
	# don't have access to env context here for some reason
	ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref \|\| null }}
	# can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic
	use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) \|\| github.event_name == 'pull_request' }}

	docs:
	name: Build documentation
	needs: [eval]
	if: ${{ needs.eval.outputs.buildDocs == 'True' }}
	uses: ./.github/workflows/publish-documentation.yml
	with:
	publish: false
	environment: test
	# don't have access to env context here for some reason
	ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref \|\| null }}
	# can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic
	use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) \|\| github.event_name == 'pull_request' }}


	update_lkg:
	name: Update LKG
	needs: [tests, notebooks]
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	name: Checkout repository
	with:
	ref: ${{ env.ref }}
	- uses: actions/download-artifact@v3
	name: Get requirements files
	with:
	name: requirements
	path: requirements
	- uses: actions/setup-python@v4
	name: Setup Python
	with:
	python-version: '3.11'
	- run: python .github/workflows/generate_lkg.py requirements .
	name: Generate LKG files
	- run: git diff --exit-code lkg.txt
	name: Check for LKG changes
	id: lkg_diff
	continue-on-error: true
	- run: git diff --exit-code lkg-notebook.txt
	name: Check for notebook LKG changes
	id: lkg_notebook_diff
	continue-on-error: true
	# Create a PR if lkg_diff or lkg_notebook_diff is not empty
	- uses: actions/github-script@v6
	name: Create PR
	if: ${{ steps.lkg_diff.outcome == 'failure' \|\| steps.lkg_notebook_diff.outcome == 'failure' }}
	with:
	script: \|
	const fs = require('fs');
	const title = 'Update LKG';
	const body = 'This PR was automatically generated by the [CI workflow](../actions/workflows/ci.yml).';

	// Get sha for existing tree
	const treeSha = await github.rest.git.getRef({
	owner: context.repo.owner,
	repo: context.repo.repo,
	ref: context.ref,
	}).then((response) => {
	return github.rest.git.getCommit({
	owner: context.repo.owner,
	repo: context.repo.repo,
	commit_sha: response.data.object.sha,
	});
	}).then((response) => {
	return response.data.tree.sha;
	});

	// Add lkg.txt and lkg-notebook.txt to a tree, put it into a commit, add it to a new branch and generate a PR
	await github.rest.git.createTree({
	owner: context.repo.owner,
	repo: context.repo.repo,
	tree: [
	{
	path: 'lkg.txt',
	mode: '100644',
	type: 'blob',
	content: fs.readFileSync('lkg.txt', 'utf8'),
	},
	{
	path: 'lkg-notebook.txt',
	mode: '100644',
	type: 'blob',
	content: fs.readFileSync('lkg-notebook.txt', 'utf8'),
	},
	],
	base_tree: treeSha,
	}).then((response) => {
	return github.rest.git.createCommit({
	owner: context.repo.owner,
	repo: context.repo.repo,
	message: "Update LKG\n\nSigned-off-by: GitHub Actions <noreply@github.com>",
	tree: response.data.sha,
	parents: [context.sha],
	author: {
	name: 'GitHub Actions',
	email: 'noreply@github.com',
	},
	});
	}).then((response) => {
	return github.rest.git.createRef({
	owner: context.repo.owner,
	repo: context.repo.repo,
	ref: `refs/heads/update-lkg-${response.data.sha.slice(0, 7)}`,
	sha: response.data.sha,
	});
	}).then((response) => {
	return github.rest.pulls.create({
	owner: context.repo.owner,
	repo: context.repo.repo,
	title: title,
	body: body,
	head: `update-lkg-${response.data.sha.slice(0, 7)}`,
	base: context.ref,
	});
	});


	verify:
	name: Verify CI checks
	needs: [lint, notebooks, tests, build, docs]
	if: always()
	runs-on: ubuntu-latest
	steps:
	- run: exit 1
	name: At least one check failed or was cancelled
	if: ${{ contains(needs..result, 'failure') \|\| contains(needs..result, 'cancelled') }}
	- run: exit 0
	name: All checks passed
	if: ${{ !(contains(needs..result, 'failure') \|\| contains(needs..result, 'cancelled')) }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Run all checks #154

Workflow file

Run all checks #154

Jobs

Run details

Workflow file for this run