[generators] Expose m3cluster env in pod spec (#197)

Allows users to reference `${M3CLUSTER_ENVIRONMENT}` in custom configmaps to reduce boilerplate. Also fixes a bug in container environment creation that would previously overwrite default environment variables.
m3db · Feb 17, 2020 · aa31b29 · aa31b29
1 parent 23428d7
commit aa31b29
Show file tree

Hide file tree

Showing 5 changed files with 126 additions and 73 deletions.
diff --git a/go.mod b/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/briandowns/spinner v1.8.0 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/cirocosta/grafana-sync v0.0.0-20181123215626-6cbb4a9501c1
+	github.com/d4l3k/messagediff v1.2.1
 	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/fortytw2/leaktest v1.3.0 // indirect

diff --git a/go.sum b/go.sum
@@ -64,6 +64,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
+github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

diff --git a/pkg/k8sops/m3db/generators.go b/pkg/k8sops/m3db/generators.go
@@ -188,7 +188,7 @@ func GenerateStatefulSet(
 
 	if cluster.Spec.EnvVars != nil && len(cluster.Spec.EnvVars) > 0 {
 		cluster := cluster.DeepCopy()
-		m3dbContainer.Env = cluster.Spec.EnvVars
+		m3dbContainer.Env = append(m3dbContainer.Env, cluster.Spec.EnvVars...)
 	}
 
 	return statefulSet, nil

diff --git a/pkg/k8sops/m3db/generators_test.go b/pkg/k8sops/m3db/generators_test.go
@@ -38,6 +38,7 @@ import (
 	"k8s.io/utils/pointer"
 
 	crdutils "github.com/ant31/crd-validation/pkg"
+	"github.com/d4l3k/messagediff"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -193,6 +194,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 										},
 									},
 								},
+								{
+									Name:  "M3CLUSTER_ENVIRONMENT",
+									Value: "foo/m3db-cluster",
+								},
 							},
 							Ports: generateContainerPorts(fixture),
 							VolumeMounts: []v1.VolumeMount{
@@ -294,7 +299,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err := GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Reset spec and fixture, test custom config map
 	ss = baseSS.DeepCopy()
@@ -304,7 +312,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Reset spec and fixture, test custom volume claims
 	ss = baseSS.DeepCopy()
@@ -321,7 +332,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Reset spec and fixture, test per-isogroup storageclasses
 	ss = baseSS.DeepCopy()
@@ -332,7 +346,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Ensure changing another isogroup doesn't affect this statefulset
 	ss = baseSS.DeepCopy()
@@ -342,7 +359,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Test empty tolerations
 	ss = baseSS.DeepCopy()
@@ -353,7 +373,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Make sure nil security context adds one with SYS_RESOURCE
 	ss = baseSS.DeepCopy()
@@ -368,11 +391,26 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 
 	// Test custom env vars
 	ss = baseSS.DeepCopy()
 	ss.Spec.Template.Spec.Containers[0].Env = []v1.EnvVar{
+		{
+			Name: "NAMESPACE",
+			ValueFrom: &v1.EnvVarSource{
+				FieldRef: &v1.ObjectFieldSelector{
+					FieldPath: "metadata.namespace",
+				},
+			},
+		},
+		{
+			Name:  "M3CLUSTER_ENVIRONMENT",
+			Value: "foo/m3db-cluster",
+		},
 		{
 			Name:  "test",
 			Value: "testval",
@@ -409,7 +447,10 @@ func TestGenerateStatefulSet(t *testing.T) {
 	newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount)
 	assert.NoError(t, err)
 	assert.NotNil(t, newSS)
-	assert.Equal(t, ss, newSS)
+	if !assert.Equal(t, ss, newSS) {
+		diff, _ := messagediff.PrettyDiff(ss, newSS)
+		t.Log(diff)
+	}
 }
 
 func TestGenerateM3DBService(t *testing.T) {

diff --git a/pkg/k8sops/m3db/statefulset.go b/pkg/k8sops/m3db/statefulset.go
@@ -25,6 +25,7 @@ import (
 	"fmt"
 
 	myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1"
+	"github.com/m3db/m3db-operator/pkg/k8sops"
 	"github.com/m3db/m3db-operator/pkg/k8sops/annotations"
 	"github.com/m3db/m3db-operator/pkg/k8sops/labels"
 	"github.com/m3db/m3db-operator/pkg/k8sops/podidentity"
@@ -104,79 +105,87 @@ func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBClust
 		}
 	}
 
-	return &appsv1.StatefulSet{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        ssName,
-			Labels:      objLabels,
-			Annotations: objAnnotations,
+	m3dbContainer := v1.Container{
+		Name:            ssName,
+		SecurityContext: specSecurityCtx,
+		ReadinessProbe:  probeReady,
+		LivenessProbe:   probeHealth,
+		Command: []string{
+			"m3dbnode",
+		},
+		Args: []string{
+			"-f",
+			_configurationFileLocation,
+		},
+		Image:           image,
+		ImagePullPolicy: "Always",
+		Env: []v1.EnvVar{
+			{
+				Name: "NAMESPACE",
+				ValueFrom: &v1.EnvVarSource{
+					FieldRef: &v1.ObjectFieldSelector{
+						FieldPath: "metadata.namespace",
+					},
+				},
+			},
+			{
+				Name:  "M3CLUSTER_ENVIRONMENT",
+				Value: k8sops.DefaultM3ClusterEnvironmentName(cluster),
+			},
+		},
+		Ports: nil,
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      _dataVolumeName,
+				MountPath: _dataDirectory,
+			},
+			{
+				Name:      "cache",
+				MountPath: "/var/lib/m3kv/",
+			},
+			generateDownwardAPIVolumeMount(),
 		},
-		Spec: appsv1.StatefulSetSpec{
-			ServiceName: HeadlessServiceName(clusterName),
-			Selector: &metav1.LabelSelector{
-				MatchLabels: objLabels,
+	}
+
+	stsSpec := appsv1.StatefulSetSpec{
+		ServiceName: HeadlessServiceName(clusterName),
+		Selector: &metav1.LabelSelector{
+			MatchLabels: objLabels,
+		},
+		Replicas: &ic,
+		Template: v1.PodTemplateSpec{
+			ObjectMeta: metav1.ObjectMeta{
+				Labels:      objLabels,
+				Annotations: objAnnotations,
 			},
-			Replicas: &ic,
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels:      objLabels,
-					Annotations: objAnnotations,
+			Spec: v1.PodSpec{
+				PriorityClassName: cluster.Spec.PriorityClassName,
+				SecurityContext:   cluster.Spec.PodSecurityContext,
+				ImagePullSecrets:  cluster.Spec.ImagePullSecrets,
+				Containers: []v1.Container{
+					m3dbContainer,
 				},
-				Spec: v1.PodSpec{
-					PriorityClassName: cluster.Spec.PriorityClassName,
-					SecurityContext:   cluster.Spec.PodSecurityContext,
-					ImagePullSecrets:  cluster.Spec.ImagePullSecrets,
-					Containers: []v1.Container{
-						{
-							Name:            ssName,
-							SecurityContext: specSecurityCtx,
-							ReadinessProbe:  probeReady,
-							LivenessProbe:   probeHealth,
-							Command: []string{
-								"m3dbnode",
-							},
-							Args: []string{
-								"-f",
-								_configurationFileLocation,
-							},
-							Image:           image,
-							ImagePullPolicy: "Always",
-							Env: []v1.EnvVar{
-								{
-									Name: "NAMESPACE",
-									ValueFrom: &v1.EnvVarSource{
-										FieldRef: &v1.ObjectFieldSelector{
-											FieldPath: "metadata.namespace",
-										},
-									},
-								},
-							},
-							Ports: nil,
-							VolumeMounts: []v1.VolumeMount{
-								{
-									Name:      _dataVolumeName,
-									MountPath: _dataDirectory,
-								},
-								{
-									Name:      "cache",
-									MountPath: "/var/lib/m3kv/",
-								},
-								generateDownwardAPIVolumeMount(),
-							},
-						},
-					},
-					Volumes: []v1.Volume{
-						{
-							Name: "cache",
-							VolumeSource: v1.VolumeSource{
-								EmptyDir: &v1.EmptyDirVolumeSource{},
-							},
+				Volumes: []v1.Volume{
+					{
+						Name: "cache",
+						VolumeSource: v1.VolumeSource{
+							EmptyDir: &v1.EmptyDirVolumeSource{},
 						},
-						generateDownwardAPIVolume(),
 					},
+					generateDownwardAPIVolume(),
 				},
 			},
 		},
 	}
+
+	return &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        ssName,
+			Labels:      objLabels,
+			Annotations: objAnnotations,
+		},
+		Spec: stsSpec,
+	}
 }
 
 func generateDownwardAPIVolume() v1.Volume {