Fix integration tests broken after #3005 (#3128)

* Fix integration tests broken after #3005

Signed-off-by: Kim Christensen <kimworking@gmail.com>


---------

Signed-off-by: Kim Christensen <kimworking@gmail.com>

pkg/build/dockerfile-generator_test.go

@@ -305,7 +305,7 @@ COPY mybin /cnab/app/
 	require.NoError(t, err)
 
 	// Verify that we logged the dockerfile contents
-	tests.RequireOutputContains(t, c.TestContext.GetError(), string(gotDockerfile), "expected the dockerfile to be printed to the logs")
+	tests.RequireOutputContains(t, c.TestContext.GetOutput(), string(gotDockerfile), "expected the dockerfile to be printed to the logs")
 	assert.Equal(t, customFrom, string(gotDockerfile))
 }
 

pkg/portercontext/context.go

@@ -241,7 +241,7 @@ func (c *Context) makeConsoleLogger() zapcore.Core {
 	}
 	stdoutEnabler := zap.LevelEnablerFunc(isInformational)
 	stderrEnabler := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
-		return !isInformational(lvl)
+		return !isInformational(lvl) && lvl >= zapcore.ErrorLevel
 	})
 
 	return zapcore.NewTee(

tests/integration/build_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"get.porter.sh/porter/pkg"
@@ -133,9 +134,11 @@ func TestRebuild(t *testing.T) {
 	bumpBundle()
 
 	// Explain the bundle, with --autobuild-disabled. It should work since the bundle has been built
-	explainJson, output := test.RequirePorter("explain", "--autobuild-disabled", "-o=json")
+	explainJson, output := test.RequirePorter("explain", "--autobuild-disabled", "-o=json", "--verbosity=warn")
 	tests.RequireOutputContains(t, output, "WARNING: The bundle is out-of-date. Skipping autobuild because --autobuild-disabled was specified")
 	require.NotContains(t, output, "Building bundle ===>")
+	// todo(kichristensen): in the future this should be improved
+	explainJson = strings.ReplaceAll(explainJson, "WARNING: The bundle is out-of-date. Skipping autobuild because --autobuild-disabled was specified", "")
 	var explainResult map[string]interface{}
 	err = json.Unmarshal([]byte(explainJson), &explainResult)
 	require.NoError(t, err, "could not marshal explain output as json")

tests/integration/invoke_test.go

@@ -52,7 +52,7 @@ func TestInvokeCustomAction(t *testing.T) {
 
 	// Check that debug output from the porter runtime was printed by the bundle and porter collected it
 	// This checks that the PORTER_DEBUG parameter is being properly passed to a bundle when run with porter invoke --debug
-	gotStderr := p.TestConfig.TestContext.GetError()
+	gotStderr := p.TestConfig.TestContext.GetOutput()
 	tests.RequireOutputContains(t, gotStderr, runtimeDebugOutputCheck, "expected debug output from the porter runtime to be output by the bundle")
 
 	// Verify that the custom action was recorded properly

tests/integration/migration_test.go

@@ -34,46 +34,46 @@ func TestMigration(t *testing.T) {
 	_, output := test.RequirePorter("storage", "migrate", "--old-home", oldHome, "--old-account=src", "--namespace", destNamespace)
 
 	// Verify that the installations were migrated to the specified namespace
-	output, _ = test.RequirePorter("list", "--namespace", destNamespace, "--output=json")
+	output, _ = test.RequirePorter("list", "--namespace", destNamespace, "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "installations-list-output.json"), output)
 
 	// Verify that all the previous runs were migrated
-	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "hello1", "--output=json")
+	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "hello1", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "runs-list-hello1-output.json"), output)
 
-	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "hello-llama", "--output=json")
+	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "hello-llama", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "runs-list-hello-llama-output.json"), output)
 
-	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "creds-tutorial", "--output=json")
+	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "creds-tutorial", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "runs-list-creds-tutorial-output.json"), output)
 
-	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "sensitive-data", "--output=json")
+	output, _ = test.RequirePorter("installation", "runs", "list", "--namespace", destNamespace, "sensitive-data", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "runs-list-sensitive-data-output.json"), output)
 
 	// Verify that outputs were migrated, all the installations except sensitive-data only have logs (which aren't printed by installation outputs list)
 	// Show the logs from installing hello1
-	output, _ = test.RequirePorter("installation", "logs", "show", "--namespace", destNamespace, "-r=01G1VJGY43HT3KZN82DS6DDPWK")
+	output, _ = test.RequirePorter("installation", "logs", "show", "--namespace", destNamespace, "-r=01G1VJGY43HT3KZN82DS6DDPWK", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "logs-install-hello1.txt"), output)
 
 	// Show the logs from the last run of hello-llama
-	output, _ = test.RequirePorter("installation", "logs", "show", "--namespace", destNamespace, "-i=hello-llama")
+	output, _ = test.RequirePorter("installation", "logs", "show", "--namespace", destNamespace, "-i=hello-llama", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "logs-hello-llama.txt"), output)
 
 	// Show the outputs of the sensitive-data bundle
-	output, _ = test.RequirePorter("installation", "outputs", "list", "--namespace", destNamespace, "sensitive-data", "--output=json")
+	output, _ = test.RequirePorter("installation", "outputs", "list", "--namespace", destNamespace, "sensitive-data", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "outputs-list-sensitive-data-output.json"), output)
 
 	// Dump out the migrated installations and make sure that all the fields are set correctly
-	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "hello1", "--output=json")
+	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "hello1", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "installation-show-hello1-output.json"), output)
 
-	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "hello-llama", "--output=json")
+	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "hello-llama", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "installation-show-hello-llama-output.json"), output)
 
-	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "creds-tutorial", "--output=json")
+	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "creds-tutorial", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "installation-show-creds-tutorial-output.json"), output)
 
-	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "sensitive-data", "--output=json")
+	output, _ = test.RequirePorter("installation", "show", "--namespace", destNamespace, "sensitive-data", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "installation-show-sensitive-data-output.json"), output)
 
 	// Verify that the sensitive-data installation stored sensitive values in the secret store
@@ -86,18 +86,18 @@ func TestMigration(t *testing.T) {
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "secrets/01G6K8CZ08T78WXTJYHR0NTYBS-password"), string(secretParam))
 
 	// Verify that the parameter sets were migrated to the specified namespace
-	output, _ = test.RequirePorter("parameters", "list", "--namespace", destNamespace, "--output=json")
+	output, _ = test.RequirePorter("parameters", "list", "--namespace", destNamespace, "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "parameters-list-output.json"), output)
 
 	// Dump out the migrated parameter sets and make sure that all the fields are set correctly
-	output, _ = test.RequirePorter("parameters", "show", "--namespace", destNamespace, "hello-llama", "--output=json")
+	output, _ = test.RequirePorter("parameters", "show", "--namespace", destNamespace, "hello-llama", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "parameters-show-hello-llama-output.json"), output)
 
 	// Verify that the credential sets were migrated to the specified namespace
-	output, _ = test.RequirePorter("credentials", "list", "--namespace", destNamespace, "--output=json")
+	output, _ = test.RequirePorter("credentials", "list", "--namespace", destNamespace, "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "credentials-list-output.json"), output)
 
 	// Dump out the migrated credential sets and make sure that all the fields are set correctly
-	output, _ = test.RequirePorter("credentials", "show", "--namespace", destNamespace, "credentials-tutorial", "--output=json")
+	output, _ = test.RequirePorter("credentials", "show", "--namespace", destNamespace, "credentials-tutorial", "--output=json", "--verbosity=info")
 	testhelpers.CompareGoldenFile(t, filepath.Join(testdata, "credentials-show-credentials-tutorial-output.json"), output)
 }

tests/integration/publish_test.go

@@ -23,17 +23,17 @@ func TestPublish(t *testing.T) {
 	test.RequirePorter("create")
 
 	// Try to publish with autobuild disabled, it should fail
-	_, _, err = test.RunPorter("publish", "--autobuild-disabled")
-	require.ErrorContains(t, err, "Skipping autobuild because --autobuild-disabled was specified")
+	_, output, _ := test.RunPorter("publish", "--autobuild-disabled")
+	tests.RequireOutputContains(t, output, "Skipping autobuild because --autobuild-disabled was specified")
 
 	// Try again with autobuild disabled via a config setting instead of a flag
 	// This is a regression test for https://github.com/getporter/porter/issues/2735
 	test.EditYaml(path.Join(test.PorterHomeDir, "config.yaml"), func(yq *yaml.Editor) error {
 		return yq.SetValue("autobuild-disabled", "true")
 	})
-	_, output, err := test.RunPorter("publish")
+	_, output, _ = test.RunPorter("publish")
 	fmt.Println(output)
-	require.ErrorContains(t, err, "Skipping autobuild because --autobuild-disabled was specified")
+	tests.RequireOutputContains(t, output, "Skipping autobuild because --autobuild-disabled was specified")
 
 	// Build with version override
 	test.RequirePorter("build", "--version=0.0.0")

tests/smoke/hello_test.go

@@ -29,7 +29,7 @@ func TestHelloBundle(t *testing.T) {
 	// This also does a quick regression test to validate that we can change the verbosity to a different value than what is in the config file and have it respected
 	_, output := test.RequirePorter("plugins", "install", "-f", "plugins.yaml", "--verbosity=info")
 	require.Contains(t, output, "installed azure plugin", "expected to see plugin successfully installed")
-	require.Containsf(t, output, "Downloading https://cdn.porter.sh/plugins/atom.xml", "Debug information should be printed because verbosity is set to info")
+	require.NotContainsf(t, output, "Downloading https://cdn.porter.sh/plugins/atom.xml", "Debug information should not be printed because verbosity is set to info")
 
 	// Run a stateless action before we install and make sure nothing is persisted
 	_, output = test.RequirePorter("invoke", testdata.MyBuns, "--action=dry-run", "--reference", testdata.MyBunsRef, "-c=mybuns")