Skip to content

Commit

Permalink
ocp4/e2e: Link test failure with sub-tests
Browse files Browse the repository at this point in the history
Currently, there is a "context" object which contains a lot of the
information the tests need to run. This used to include the test
framework helper instance which one would normally use to output logs or
mark the test as failed.

Unfortunately, this only took into account the main test helper
instance, and so it wasn't entirely clear in what part of the test did
the failure happen. This removes this limitation by passing in the
helper instance along with every function that requires it. This way, we
can pass the sub-test metadata, which will show on what sub-test did the
failure happen.

Note that this PR also increases the manual remediation timeout...
MachineConfigs take a long time to apply unfortunately...

Signed-off-by: Juan Antonio Osorio Robles <jaosorior@redhat.com>
  • Loading branch information
JAORMX committed Jan 27, 2021
1 parent a215c82 commit e5c671d
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 110 deletions.
48 changes: 24 additions & 24 deletions tests/ocp4e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,28 +8,28 @@ import (
func TestE2e(t *testing.T) {
ctx := newE2EContext(t)
t.Run("Parameter setup and validation", func(t *testing.T) {
ctx.assertRootdir()
ctx.assertProfile()
ctx.assertContentImage()
ctx.assertKubeClient()
ctx.assertRootdir(t)
ctx.assertProfile(t)
ctx.assertContentImage(t)
ctx.assertKubeClient(t)
})

t.Run("Operator setup", func(t *testing.T) {
ctx.ensureNamespaceExistsAndSet()
ctx.ensureNamespaceExistsAndSet(t)
if ctx.installOperator {
ctx.ensureCatalogSourceExists()
ctx.ensureOperatorGroupExists()
ctx.ensureSubscriptionExists()
ctx.waitForOperatorToBeReady()
ctx.ensureCatalogSourceExists(t)
ctx.ensureOperatorGroupExists(t)
ctx.ensureSubscriptionExists(t)
ctx.waitForOperatorToBeReady(t)
} else {
t.Logf("Skipping operator install as requested")
}
ctx.resetClientMappings()
})

t.Run("Prereqs setup", func(t *testing.T) {
ctx.ensureTestProfileBundle()
ctx.ensureTestSettings()
ctx.ensureTestProfileBundle(t)
ctx.ensureTestSettings(t)
})

// Remediations
Expand All @@ -53,32 +53,32 @@ func TestE2e(t *testing.T) {

t.Run("Run first compliance scan", func(t *testing.T) {
// Create suite and auto-apply remediations
suite = ctx.createBindingForProfile()
ctx.waitForComplianceSuite(suite)
numberOfRemediations = ctx.getRemediationsForSuite(suite)
numberOfFailuresInit = ctx.getFailuresForSuite(suite)
numberOfCheckResultsInit, manualRemediations = ctx.verifyCheckResultsForSuite(suite, false)
numberOfInvalidResults = ctx.getInvalidResultsFromSuite(suite)
suite = ctx.createBindingForProfile(t)
ctx.waitForComplianceSuite(t, suite)
numberOfRemediations = ctx.getRemediationsForSuite(t, suite)
numberOfFailuresInit = ctx.getFailuresForSuite(t, suite)
numberOfCheckResultsInit, manualRemediations = ctx.verifyCheckResultsForSuite(t, suite, false)
numberOfInvalidResults = ctx.getInvalidResultsFromSuite(t, suite)
})

if numberOfRemediations > 0 || len(manualRemediations) > 0 {
if len(manualRemediations) > 0 {
t.Run("Apply manual remediations", func(t *testing.T) {
ctx.applyManualRemediations(manualRemediations)
ctx.applyManualRemediations(t, manualRemediations)
})
}
t.Run("Wait for Remediations to apply", func(t *testing.T) {
// Lets wait for the MachineConfigs to start applying
time.Sleep(30 * time.Second)
ctx.waitForMachinePoolUpdate("master")
ctx.waitForMachinePoolUpdate("worker")
ctx.waitForMachinePoolUpdate(t, "master")
ctx.waitForMachinePoolUpdate(t, "worker")
})

t.Run("Run second compliance scan", func(t *testing.T) {
ctx.doRescan(suite)
ctx.waitForComplianceSuite(suite)
numberOfFailuresEnd = ctx.getFailuresForSuite(suite)
numberOfCheckResultsEnd, _ = ctx.verifyCheckResultsForSuite(suite, true)
ctx.doRescan(t, suite)
ctx.waitForComplianceSuite(t, suite)
numberOfFailuresEnd = ctx.getFailuresForSuite(t, suite)
numberOfCheckResultsEnd, _ = ctx.verifyCheckResultsForSuite(t, suite, true)
})

t.Run("We should have the same number of check results in each scan", func(t *testing.T) {
Expand Down
Loading

0 comments on commit e5c671d

Please sign in to comment.