-
Notifications
You must be signed in to change notification settings - Fork 8.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Code coverage setup on CI #49003
Code coverage setup on CI #49003
Changes from 14 commits
34f245a
d6e5b3a
f0f44e2
8a42910
e6000be
90a9773
92178a8
bf72793
3ef67c3
6096f05
f016352
4395436
08537b6
a37279c
16ac05e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,112 @@ | ||
#!/bin/groovy | ||
|
||
library 'kibana-pipeline-library' | ||
kibanaLibrary.load() // load from the Jenkins instance | ||
|
||
stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit | ||
timeout(time: 180, unit: 'MINUTES') { | ||
timestamps { | ||
ansiColor('xterm') { | ||
catchError { | ||
withEnv([ | ||
'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc. | ||
]) { | ||
parallel([ | ||
'kibana-intake-agent': { | ||
withEnv([ | ||
'NODE_ENV=test' // Needed for jest tests only | ||
]) { | ||
kibanaPipeline.legacyJobRunner('kibana-intake')() | ||
} | ||
}, | ||
'x-pack-intake-agent': { | ||
withEnv([ | ||
'NODE_ENV=test' // Needed for jest tests only | ||
]) { | ||
kibanaPipeline.legacyJobRunner('x-pack-intake')() | ||
} | ||
}, | ||
'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ | ||
'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), | ||
'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), | ||
'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), | ||
'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), | ||
'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), | ||
'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), | ||
'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), | ||
'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), | ||
'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), | ||
'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), | ||
'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), | ||
'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), | ||
]), | ||
'kibana-xpack-agent-1': kibanaPipeline.withWorkers('kibana-xpack-tests-1', { kibanaPipeline.buildXpack() }, [ | ||
'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), | ||
'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), | ||
]), | ||
'kibana-xpack-agent-2': kibanaPipeline.withWorkers('kibana-xpack-tests-2', { kibanaPipeline.buildXpack() }, [ | ||
'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), | ||
'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), | ||
]), | ||
|
||
'kibana-xpack-agent-3': kibanaPipeline.withWorkers('kibana-xpack-tests-3', { kibanaPipeline.buildXpack() }, [ | ||
'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), | ||
'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), | ||
'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), | ||
'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), | ||
'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), | ||
'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), | ||
]), | ||
]) | ||
kibanaPipeline.jobRunner('tests-l', false) { | ||
kibanaPipeline.downloadCoverageArtifacts() | ||
kibanaPipeline.bash( | ||
''' | ||
# bootstrap from x-pack folder | ||
source src/dev/ci_setup/setup_env.sh | ||
cd x-pack | ||
yarn kbn bootstrap --prefer-offline | ||
cd .. | ||
# extract archives | ||
mkdir -p /tmp/extracted_coverage | ||
echo extracting intakes | ||
tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage | ||
tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage | ||
echo extracting kibana-oss-tests | ||
tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage | ||
echo extracting kibana-xpack-tests | ||
for i in {1..3}; do | ||
tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests-${i}/kibana-coverage.tar.gz -C /tmp/extracted_coverage | ||
done | ||
# replace path in json files to have valid html report | ||
pwd=$(pwd) | ||
du -sh /tmp/extracted_coverage/target/kibana-coverage/ | ||
echo replacing path in json files | ||
for i in {1..9}; do | ||
sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json & | ||
done | ||
wait | ||
# merge oss & x-pack reports | ||
echo merging coverage reports | ||
yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary | ||
yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary | ||
echo copy mocha reports | ||
mkdir -p target/kibana-coverage/mocha-combined | ||
cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined | ||
''', | ||
"run `yarn kbn bootstrap && merge coverage`" | ||
) | ||
sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*' | ||
kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz') | ||
sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*' | ||
kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz') | ||
sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*' | ||
kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz') | ||
} | ||
} | ||
} | ||
kibanaPipeline.sendMail() | ||
} | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,12 +17,23 @@ | |
* under the License. | ||
*/ | ||
|
||
require('../src/setup_node_env'); | ||
require('@kbn/test').runTestsCli([ | ||
// eslint-disable-next-line no-restricted-syntax | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would all of these changes feel better as a separate file (e.g. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It was in a separate file initially, but I talked to Spencer and made it this way. @spalger is ok or better in an own file? |
||
const alwaysImportedTests = [ | ||
require.resolve('../test/functional/config.js'), | ||
require.resolve('../test/api_integration/config.js'), | ||
require.resolve('../test/plugin_functional/config.js'), | ||
require.resolve('../test/interpreter_functional/config.ts'), | ||
require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'), | ||
]; | ||
// eslint-disable-next-line no-restricted-syntax | ||
const onlyNotInCoverageTests = [ | ||
require.resolve('../test/api_integration/config.js'), | ||
require.resolve('../test/interpreter_functional/config.ts'), | ||
require.resolve('../test/examples/config.js'), | ||
]; | ||
|
||
require('../src/setup_node_env'); | ||
require('@kbn/test').runTestsCli([ | ||
// eslint-disable-next-line no-restricted-syntax | ||
...alwaysImportedTests, | ||
// eslint-disable-next-line no-restricted-syntax | ||
...(!!process.env.CODE_COVERAGE ? [] : onlyNotInCoverageTests), | ||
]); |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -193,7 +193,7 @@ export default () => | |||
.default('localhost'), | ||||
watchPrebuild: Joi.boolean().default(false), | ||||
watchProxyTimeout: Joi.number().default(10 * 60000), | ||||
useBundleCache: Joi.boolean().default(Joi.ref('$prod')), | ||||
useBundleCache: Joi.boolean().default(!!process.env.CODE_COVERAGE ? true : Joi.ref('$prod')), | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not sure that the schema is the best place to put logic like this... Actually, do you need to set this at all? It's always set to true here: Line 41 in 08537b6
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The problem is somehow without setting |
||||
sourceMaps: Joi.when('$prod', { | ||||
is: true, | ||||
then: Joi.boolean().valid(false), | ||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,33 +4,48 @@ set -e | |
|
||
export TEST_BROWSER_HEADLESS=1 | ||
|
||
echo " -> Running mocha tests" | ||
cd "$XPACK_DIR" | ||
checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser | ||
echo "" | ||
echo "" | ||
|
||
echo " -> Running jest tests" | ||
cd "$XPACK_DIR" | ||
checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose | ||
echo "" | ||
echo "" | ||
|
||
echo " -> Running SIEM cyclic dependency test" | ||
cd "$XPACK_DIR" | ||
checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps | ||
echo "" | ||
echo "" | ||
|
||
# FAILING: https://github.com/elastic/kibana/issues/44250 | ||
# echo " -> Running jest contracts tests" | ||
# cd "$XPACK_DIR" | ||
# SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose | ||
# echo "" | ||
# echo "" | ||
|
||
# echo " -> Running jest integration tests" | ||
# cd "$XPACK_DIR" | ||
# node scripts/jest_integration --ci --verbose | ||
# echo "" | ||
# echo "" | ||
if [[ -z "$CODE_COVERAGE" ]] ; then | ||
echo " -> Running mocha tests" | ||
cd "$XPACK_DIR" | ||
checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser | ||
echo "" | ||
echo "" | ||
|
||
echo " -> Running jest tests" | ||
cd "$XPACK_DIR" | ||
checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose | ||
echo "" | ||
echo "" | ||
|
||
echo " -> Running SIEM cyclic dependency test" | ||
cd "$XPACK_DIR" | ||
checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps | ||
echo "" | ||
echo "" | ||
|
||
# FAILING: https://github.com/elastic/kibana/issues/44250 | ||
# echo " -> Running jest contracts tests" | ||
# cd "$XPACK_DIR" | ||
# SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose | ||
# echo "" | ||
# echo "" | ||
|
||
# echo " -> Running jest integration tests" | ||
# cd "$XPACK_DIR" | ||
# node scripts/jest_integration --ci --verbose | ||
# echo "" | ||
# echo "" | ||
else | ||
echo " -> Running jest tests with coverage" | ||
cd "$XPACK_DIR" | ||
# build runtime for canvas | ||
echo "NODE_ENV=$NODE_ENV" | ||
node ./legacy/plugins/canvas/scripts/shareable_runtime | ||
node scripts/jest --ci --verbose --coverage | ||
# rename file in order to be unique one | ||
test -f ../target/kibana-coverage/jest/coverage-final.json \ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This Also, you still might have the problem I mentioned before: If There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Well, if any test will fail So exiting before renaming is fine, build should be red and next step (merge) is not starting. If the tests pass, json file is expected to be generated and I rename it because the json with the same name will be generated for oss jest tests. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I just asked @LeeDr and he feels we should not push the coverage data if there's a failure as that will skew the coverage numbers. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. All good, just wanted to make sure the file not getting renamed wasn't going to cause a downstream issue! |
||
&& mv ../target/kibana-coverage/jest/coverage-final.json \ | ||
../target/kibana-coverage/jest/xpack-coverage-final.json | ||
echo "" | ||
echo "" | ||
fi |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@dmlemeshko
'tests-l'
==== some large instance for testing?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, the default one (PRs) has not enough space for coverage processing (25+ GB)