ci(jenkins): use e2e in the CI

.ci/end2end.groovy

@@ -12,8 +12,7 @@ pipeline {
   environment {
     BASE_DIR = 'src/github.com/elastic/kibana'
     HOME = "${env.WORKSPACE}"
-    APM_ITS = 'apm-integration-testing'
-    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/e2e'
+    E2E_DIR = 'x-pack/legacy/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
   }
   options {
@@ -43,32 +42,6 @@ pipeline {
             env.APM_UPDATED = isGitRegionMatch(patterns: regexps)
           }
         }
-        dir("${APM_ITS}"){
-          git changelog: false,
-              credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
-              poll: false,
-              url: "git@github.com:elastic/${APM_ITS}.git"
-        }
-      }
-    }
-    stage('Start services') {
-      options { skipDefaultCheckout() }
-      when {
-        anyOf {
-          expression { return params.FORCE }
-          expression { return env.APM_UPDATED != "false" }
-        }
-      }
-      steps {
-        notifyStatus('Starting services', 'PENDING')
-        dir("${APM_ITS}"){
-          sh './scripts/compose.py start master --no-kibana'
-        }
-      }
-      post {
-        unsuccessful {
-          notifyStatus('Environmental issue', 'FAILURE')
-        }
       }
     }
     stage('Prepare Kibana') {
@@ -85,7 +58,7 @@ pipeline {
       steps {
         notifyStatus('Preparing kibana', 'PENDING')
         dir("${BASE_DIR}"){
-          sh script: "${CYPRESS_DIR}/ci/prepare-kibana.sh"
+          sh "${E2E_DIR}/ci/prepare-kibana.sh"
         }
       }
       post {
@@ -105,24 +78,27 @@ pipeline {
       steps{
         notifyStatus('Running smoke tests', 'PENDING')
         dir("${BASE_DIR}"){
-          sh '''
-            jobs -l
-            docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) ${CYPRESS_DIR}/ci
-            docker run --rm -t --user "$(id -u):$(id -g)" \
-                    -v `pwd`:/app --network="host" \
-                    --name cypress cypress'''
+          // As long as the kibana in dev mode is not reliable we need to run the tests
+          // a few times.
+          // TODO: APM-UI to provide the kibana run command that works at first.
+          sh "${E2E_DIR}/ci/run-e2e.sh || true"
+          retry(5) {
+            sleep 30
+            sh "${E2E_DIR}/ci/rerun-e2e.sh"
+          }
         }
       }
       post {
         always {
-          dir("${BASE_DIR}"){
-            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/**/screenshots/**,${CYPRESS_DIR}/**/videos/**,${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
-            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
-          }
-          dir("${APM_ITS}"){
-            sh 'docker-compose logs > apm-its.log || true'
-            sh 'docker-compose down -v || true'
-            archiveArtifacts(allowEmptyArchive: false, artifacts: 'apm-its.log')
+          dir("${BASE_DIR}/${E2E_DIR}"){
+            archiveArtifacts(allowEmptyArchive: false, artifacts: 'cypress/screenshots/**,cypress/videos/**,cypress/test-results/*e2e-tests.xml')
+            junit(allowEmptyResults: true, testResults: 'cypress/test-results/*e2e-tests.xml')
+            dir('tmp/apm-integration-testing'){
+              sh 'docker-compose logs > apm-its-docker.log || true'
+              sh 'docker-compose down -v || true'
+              archiveArtifacts(allowEmptyArchive: true, artifacts: 'apm-its-docker.log')
+            }
+            archiveArtifacts(allowEmptyArchive: true, artifacts: 'tmp/*.log')
           }
         }
         unsuccessful {
@@ -137,7 +113,7 @@ pipeline {
   post {
     always {
       dir("${BASE_DIR}"){
-        archiveArtifacts(allowEmptyArchive: true, artifacts: "${CYPRESS_DIR}/ingest-data.log,kibana.log")
+        archiveArtifacts(allowEmptyArchive: true, artifacts: "${E2E_DIR}/kibana.log")
       }
     }
   }

x-pack/legacy/plugins/apm/e2e/README.md

@@ -6,21 +6,4 @@
 x-pack/legacy/plugins/apm/e2e/run-e2e.sh
 ```
 
-_Starts Kibana, APM Server, Elasticsearch (with sample data) and runs the tests_
-
-## Reproducing CI builds
-
-> This process is very slow compared to the local development described above. Consider that the CI must install and configure the build tools and create a Docker image for the project to run tests in a consistent manner.
-
-The Jenkins CI uses a shell script to prepare Kibana:
-
-```shell
-# Prepare and run Kibana locally
-$ x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh
-# Build Docker image for Kibana
-$ docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) x-pack/legacy/plugins/apm/e2e/ci
-# Run Docker image
-$ docker run --rm -t --user "$(id -u):$(id -g)" \
-    -v `pwd`:/app --network="host" \
-    --name cypress cypress
-```
+_Starts APM Server, Elasticsearch (with sample data) and runs the tests_

x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh

@@ -1,21 +1,13 @@
 #!/usr/bin/env bash
-set -e
+set -ex
 
-E2E_DIR="x-pack/legacy/plugins/apm/e2e"
-
-echo "1/3 Install dependencies ..."
+E2E_DIR=x-pack/legacy/plugins/apm/e2e
+echo "1/2 Install dependencies ..."
 # shellcheck disable=SC1091
 source src/dev/ci_setup/setup_env.sh true
-yarn kbn bootstrap
-
-echo "2/3 Ingest test data ..."
-pushd ${E2E_DIR}
-yarn install
-curl --silent https://storage.googleapis.com/apm-ui-e2e-static-data/events.json --output ingest-data/events.json
-node ingest-data/replay.js --server-url http://localhost:8201 --secret-token abcd --events ./events.json > ingest-data.log
+yarn kbn clean && yarn kbn bootstrap
 
-echo "3/3 Start Kibana ..."
-popd
+echo "2/2 Start Kibana ..."
 ## Might help to avoid FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
 export NODE_OPTIONS="--max-old-space-size=4096"
-nohup node scripts/kibana --config "${E2E_DIR}/ci/kibana.e2e.yml" --no-base-path --optimize.watch=false> kibana.log 2>&1 &
+nohup node ./scripts/kibana --no-base-path --no-watch --dev --no-dev-config --config ${E2E_DIR}/ci/kibana.e2e.yml > ${E2E_DIR}/kibana.log 2>&1 &

x-pack/legacy/plugins/apm/e2e/ci/rerun-e2e.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+##
+## This is a wrapper to configure the environment with the right tools in the CI
+## and run the cypress steps.
+##
+## NOTE: it's required to run run-e2e.sh previously. This is the wrapper to help with
+##       the rerun of the e2e.
+##
+
+E2E_DIR="${0%/*}/.."
+# shellcheck disable=SC1091
+source src/dev/ci_setup/setup_env.sh true
+set -ex
+cd "${E2E_DIR}"
+rm cypress/test-results/*.* || true
+rm -rf cypress/screenshots/* || true
+yarn cypress run --config pageLoadTimeout=100000,watchForFileChanges=true

x-pack/legacy/plugins/apm/e2e/ci/run-e2e.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+##
+## This is a wrapper to configure the environment with the right tools in the CI
+## and run the e2e steps.
+##
+
+E2E_DIR="${0%/*}/.."
+# shellcheck disable=SC1091
+source src/dev/ci_setup/setup_env.sh true
+set -ex
+"${E2E_DIR}"/run-e2e.sh

x-pack/legacy/plugins/apm/e2e/cypress.json

@@ -1,7 +1,7 @@
 {
   "nodeVersion": "system",
   "baseUrl": "http://localhost:5701",
-  "video": false,
+  "video": true,
   "trashAssetsBeforeRuns": false,
   "fileServerFolder": "../",
   "fixturesFolder": "./cypress/fixtures",

x-pack/legacy/plugins/apm/e2e/cypress/support/step_definitions/apm.ts

@@ -7,14 +7,17 @@
 import { Given, When, Then } from 'cypress-cucumber-preprocessor/steps';
 import { loginAndWaitForPage } from '../../integration/helpers';
 
+/** The default time in ms to wait for a Cypress command to complete */
+export const DEFAULT_TIMEOUT = 60 * 1000;
+
 Given(`a user browses the APM UI application`, () => {
   // open service overview page
   loginAndWaitForPage(`/app/apm#/services`);
 });
 
 When(`the user inspects the opbeans-node service`, () => {
   // click opbeans-node service
-  cy.get(':contains(opbeans-node)')
+  cy.get(':contains(opbeans-node)', { timeout: DEFAULT_TIMEOUT })
     .last()
     .click({ force: true });
 });

x-pack/legacy/plugins/apm/e2e/run-e2e.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # variables
 KIBANA_PORT=5701
@@ -65,6 +65,8 @@ ${APM_IT_DIR}/scripts/compose.py start master \
     --elasticsearch-port $ELASTICSEARCH_PORT \
     --apm-server-port=$APM_SERVER_PORT \
     --elasticsearch-heap 4g \
+    --apm-server-opt queue.mem.events=8192 \
+    --apm-server-opt output.elasticsearch.bulk_max_size=4096 \
     &> ${TMP_DIR}/apm-it.log
 
 # Stop if apm-integration-testing failed to start correctly
@@ -98,7 +100,7 @@ curl --silent --user admin:changeme -XDELETE "localhost:${ELASTICSEARCH_PORT}/.a
 curl --silent --user admin:changeme -XDELETE "localhost:${ELASTICSEARCH_PORT}/apm*" > /dev/null
 
 # Ingest data into APM Server
-node ingest-data/replay.js --server-url http://localhost:$APM_SERVER_PORT --events ${TMP_DIR}/events.json 2> ${TMP_DIR}/ingest-data.log
+node ingest-data/replay.js --server-url http://localhost:$APM_SERVER_PORT --events ${TMP_DIR}/events.json 2>> ${TMP_DIR}/ingest-data.log
 
 # Stop if not all events were ingested correctly
 if [ $? -ne 0 ]; then
@@ -111,8 +113,7 @@ fi
 ##################################################
 echo "\n${bold}Waiting for Kibana to start...${normal}"
 echo "Note: you need to start Kibana manually. Find the instructions at the top."
-yarn wait-on -i 500 -w 500 http://localhost:$KIBANA_PORT > /dev/null
-
+yarn wait-on -i 500 -w 500 http://localhost:${KIBANA_PORT}/status > /dev/null
 echo "\n✅ Setup completed successfully. Running tests...\n"
 
 #
@@ -129,4 +130,3 @@ ${bold}If you want to run the test interactively, run:${normal}
 
 yarn cypress open --config pageLoadTimeout=100000,watchForFileChanges=true
 "
-