Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tweaks to get tests to work with private registry #1870

Merged
merged 5 commits into from
Sep 17, 2019
Merged

tweaks to get tests to work with private registry #1870

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4c14a75
tweaks to get tests to work with private registry
Sep 11, 2019
02c4138
use errors apis
Sep 12, 2019
55c4fc0
shift some stuff around and add some debugging
Sep 13, 2019
45b0732
review tweaks
Sep 14, 2019
23619ef
minor nits
Sep 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 52 additions & 3 deletions test/common/creation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (

corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
messagingv1alpha1 "knative.dev/eventing/pkg/apis/messaging/v1alpha1"
sourcesv1alpha1 "knative.dev/eventing/pkg/apis/sources/v1alpha1"
"knative.dev/eventing/test/base"
Expand Down Expand Up @@ -220,12 +222,58 @@ func (client *Client) CreateServiceAccountOrFail(saName string) {
client.T.Fatalf("Failed to create service account %q: %v", saName, err)
}
client.Tracker.Add(coreAPIGroup, coreAPIVersion, "serviceaccounts", namespace, saName)

// If the "default" Namespace has a secret called
// "kn-eventing-test-pull-secret" then use that as the ImagePullSecret
// on the new ServiceAccount we just created.
// This is needed for cases where the images are in a private registry.

// Get the Interfaces we need to access the resources in the cluster
defSecI := client.Kube.Kube.CoreV1().Secrets("default")
nsSAI := client.Kube.Kube.CoreV1().ServiceAccounts(namespace)
nsSecI := client.Kube.Kube.CoreV1().Secrets(namespace)

testSecret, _ := defSecI.Get(TestPullSecretName, metav1.GetOptions{})

// Check again. I've seen cases where it lies and if we need it
// then the test will fail w/o it, so check again just to be sure.
if testSecret == nil {
testSecret, _ = defSecI.Get(TestPullSecretName, metav1.GetOptions{})
}

if testSecret != nil {
// Found the secret, so now make a copy in our new namespace, but only
// if it doesn't already exist
var err error

// If it already exists in this NS then just use it, otherwise create
newSecret, _ := nsSecI.Get(TestPullSecretName, metav1.GetOptions{})
if newSecret == nil {
newSecret, err = nsSecI.Create(
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: TestPullSecretName,
},
Data: testSecret.Data,
Type: testSecret.Type,
})
if err != nil {
client.T.Fatalf("Error copying the secret: %s", err)
}
}

_, err = nsSAI.Patch(saName, types.StrategicMergePatchType,
[]byte(`{"imagePullSecrets":[{"name":"`+TestPullSecretName+`"}]}`))
if err != nil {
client.T.Fatalf("Patch failed on ServiceAccount: %s", err)
}
}
}

// CreateClusterRoleOrFail creates the given ClusterRole or fail the test if there is an error.
func (client *Client) CreateClusterRoleOrFail(cr *rbacv1.ClusterRole) {
crs := client.Kube.Kube.RbacV1().ClusterRoles()
if _, err := crs.Create(cr); err != nil {
if _, err := crs.Create(cr); err != nil && !errors.IsAlreadyExists(err) {
client.T.Fatalf("Failed to create cluster role %q: %v", cr.Name, err)
}
client.Tracker.Add(rbacAPIGroup, rbacAPIVersion, "clusterroles", "", cr.Name)
Expand All @@ -236,7 +284,8 @@ func (client *Client) CreateRoleBindingOrFail(saName, crName, rbName, rbNamespac
saNamespace := client.Namespace
rb := resources.RoleBinding(saName, saNamespace, crName, rbName, rbNamespace)
rbs := client.Kube.Kube.RbacV1().RoleBindings(rbNamespace)
if _, err := rbs.Create(rb); err != nil {

if _, err := rbs.Create(rb); err != nil && !errors.IsAlreadyExists(err) {
client.T.Fatalf("Failed to create role binding %q: %v", rbName, err)
}
client.Tracker.Add(rbacAPIGroup, rbacAPIVersion, "rolebindings", rbNamespace, rb.GetName())
Expand All @@ -247,7 +296,7 @@ func (client *Client) CreateClusterRoleBindingOrFail(saName, crName, crbName str
saNamespace := client.Namespace
crb := resources.ClusterRoleBinding(saName, saNamespace, crName, crbName)
crbs := client.Kube.Kube.RbacV1().ClusterRoleBindings()
if _, err := crbs.Create(crb); err != nil {
if _, err := crbs.Create(crb); err != nil && !errors.IsAlreadyExists(err) {
client.T.Fatalf("Failed to create cluster role binding %q: %v", crbName, err)
}
client.Tracker.Add(rbacAPIGroup, rbacAPIVersion, "clusterrolebindings", "", crb.GetName())
Expand Down
51 changes: 51 additions & 0 deletions test/common/test_runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
)

var TestPullSecretName = "kn-eventing-test-pull-secret"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
var TestPullSecretName = "kn-eventing-test-pull-secret"
const TestPullSecretName = "kn-eventing-test-pull-secret"

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed


// ChannelTestRunner is used to run tests against channels.
type ChannelTestRunner struct {
ChannelFeatureMap map[string][]Feature
Expand Down Expand Up @@ -133,6 +135,55 @@ func CreateNamespaceIfNeeded(t *testing.T, client *Client, namespace string) {
if err != nil {
t.Fatalf("The default ServiceAccount was not created for the Namespace: %s", namespace)
}

// If the "default" Namespace has a secret called
// "kn-eventing-test-pull-secret" then use that as the ImagePullSecret
// on the "default" ServiceAccount in this new Namespace.
// This is needed for cases where the images are in a private registry.

// Get the Interfaces we need to access the resources in the cluster
defSecI := client.Kube.Kube.CoreV1().Secrets("default")
nsSAI := client.Kube.Kube.CoreV1().ServiceAccounts(namespace)
nsSecI := client.Kube.Kube.CoreV1().Secrets(namespace)

testSecret, _ := defSecI.Get(TestPullSecretName, metav1.GetOptions{})

// Check again. I've seen cases where it lies and if we need it
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this seems odd, and is there a reason why you don't check the err returned? I'm not sure what you mean by "It" lies, are you saying above line returns nil, but the one below returns true?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No I mean that the apiserver will return nil+err even though the secret does exist. I can't explain why or how, but I ran into it often enough that I decided to just ask again to be sure.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@vagababov Have you seen something like this before? Seems b0rk3n?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really. What is the actual error you're getting? If I were to guess you got either network problem error (connection refused, dial timeout) or overload error from API server.

Which leads me to the main problem I see here: "never ignore errors".

Also, I'd highly recommend to switch to informers.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the err that's returned?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me change the code to show the error (if I even get one) to see what's up.. But to add more intrigue, I've also seen cases like this....

I had this code:

		if testSecret != nil {
			// Found the secret, so now make a copy in our new namespace
			newSecret, err := nsSecI.Create(
				&corev1.Secret{
					ObjectMeta: metav1.ObjectMeta{
						Name: testSecret.ObjectMeta.Name,
					},
					Data: testSecret.Data,
					Type: testSecret.Type,
				})
			if err != nil {
				t.Fatalf("TestSetup: Error copying the secret: %s", err)
			}

Notice I would use the "name" from "testSecret", which should never be empty. Yet I would get this error:

test_runner.go:82: namespace is : "test-default-broker-with-many-deprecated-triggers" test_runner.go:168: 
TestSetup: Error copying the secret: Secret "" is invalid: metadata.name: Required value: name or
generateName is required

I never saw this on my local testing, or IKS, only using Prow.

// then the test will fail w/o it, so check again just to be sure.
if testSecret == nil {
testSecret, _ = defSecI.Get(TestPullSecretName, metav1.GetOptions{})
}

if testSecret != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function seems similar in description, yet it's different and I'm trying to understand why? :) Can this be hoisted into a separate function and reused instead of having two versions of it.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

they are similar but different. This one only deals with the "default" SA and knows it can blindly copy the secret. Th other one deals with a new SA (so not "default") and needs to check to see if the secret already exists so it doesn't try to duplicate it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They both have the check and re-check (which I must say is extremely worrying and seems extremely flaky and a bug in the k8s if that's the case).
I think having a method that deals with copying the secret into the namespace would be good to hoist out, then rejiggering the ServiceAccount secret could be the only difference between the two if I understand correctly?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reverse the flow to be more go-style

if tS == nil { fail/return/break}
// normal code goes here unindented

// Found the secret, so now make a copy in our new namespace
newSecret, err := nsSecI.Create(
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: TestPullSecretName,
},
Data: testSecret.Data,
Type: testSecret.Type,
})
if err != nil {
t.Fatalf("TestSetup: Error copying the secret: %s", err)
}

// Now add it to the "default" ServiceAccount as a Pull Secret
newSecretRef := corev1.LocalObjectReference{
Name: TestPullSecretName,
}
sa, err := nsSAI.Get("default", metav1.GetOptions{})
if err != nil {
t.Fatalf("TestSetup: Error getting ServiceAccount: %s", err)
}

sa.ImagePullSecrets = append(sa.ImagePullSecrets, newSecretRef)
if _, err = nsSAI.Update(sa); err != nil {
t.Fatalf("TestSetup: Error adding Secret to ServiceAccount: %s", err)
}
t.Logf("Copied ImagePullSecret(%s) into namespace: %s",
newSecret.ObjectMeta.Name, namespace)
}
}
}

Expand Down