Allow the ILM policy name to be configurable (#2971)

* Allow the ILM Policy name to be configurable - Add an option --ilm-policy-name to esmapping-generator; - Set the mapping's option in the rollover script (using an env var ES_ILM_POLICY_NAME). The policy's name defaults to jaeger-ilm-policy (both in the mapping generator and in the rollover script), so no breaking changes are added. Signed-off-by: Ricardo Ribeiro <j.ribeiro.fafe@gmail.com> * Add integration test for ILM Policy Name Signed-off-by: Ricardo Ribeiro <j.ribeiro.fafe@gmail.com> Co-authored-by: Albert <26584478+albertteoh@users.noreply.github.com>
jaegertracing · May 4, 2021 · 12bba8c · 12bba8c
1 parent e13f0fe
commit 12bba8c
Show file tree

Hide file tree

Showing 12 changed files with 121 additions and 74 deletions.
diff --git a/cmd/esmapping-generator/app/flags.go b/cmd/esmapping-generator/app/flags.go
@@ -20,21 +20,23 @@ import (
 
 // Options represent configurable parameters for jaeger-esmapping-generator
 type Options struct {
-	Mapping     string
-	EsVersion   uint
-	Shards      int64
-	Replicas    int64
-	IndexPrefix string
-	UseILM      string // using string as util is being used in python and using bool leads to type issues.
+	Mapping       string
+	EsVersion     uint
+	Shards        int64
+	Replicas      int64
+	IndexPrefix   string
+	UseILM        string // using string as util is being used in python and using bool leads to type issues.
+	ILMPolicyName string
 }
 
 const (
-	mappingFlag     = "mapping"
-	esVersionFlag   = "es-version"
-	shardsFlag      = "shards"
-	replicasFlag    = "replicas"
-	indexPrefixFlag = "index-prefix"
-	useILMFlag      = "use-ilm"
+	mappingFlag       = "mapping"
+	esVersionFlag     = "es-version"
+	shardsFlag        = "shards"
+	replicasFlag      = "replicas"
+	indexPrefixFlag   = "index-prefix"
+	useILMFlag        = "use-ilm"
+	ilmPolicyNameFlag = "ilm-policy-name"
 )
 
 // AddFlags adds flags for esmapping-generator main program
@@ -69,6 +71,11 @@ func (o *Options) AddFlags(command *cobra.Command) {
 		useILMFlag,
 		"false",
 		"Set to true to use ILM for managing lifecycle of jaeger indices")
+	command.Flags().StringVar(
+		&o.ILMPolicyName,
+		ilmPolicyNameFlag,
+		"jaeger-ilm-policy",
+		"The name of the ILM policy to use if ILM is active")
 
 	// mark mapping flag as mandatory
 	command.MarkFlagRequired(mappingFlag)

diff --git a/cmd/esmapping-generator/app/flags_test.go b/cmd/esmapping-generator/app/flags_test.go
@@ -33,6 +33,7 @@ func TestOptionsWithDefaultFlags(t *testing.T) {
 	assert.Equal(t, int64(1), o.Replicas)
 	assert.Equal(t, "", o.IndexPrefix)
 	assert.Equal(t, "false", o.UseILM)
+	assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName)
 }
 
 func TestOptionsWithFlags(t *testing.T) {
@@ -47,6 +48,7 @@ func TestOptionsWithFlags(t *testing.T) {
 		"--replicas=1",
 		"--index-prefix=test",
 		"--use-ilm=true",
+		"--ilm-policy-name=jaeger-test-policy",
 	})
 	require.NoError(t, err)
 	assert.Equal(t, "jaeger-span", o.Mapping)
@@ -55,4 +57,5 @@ func TestOptionsWithFlags(t *testing.T) {
 	assert.Equal(t, int64(1), o.Replicas)
 	assert.Equal(t, "test", o.IndexPrefix)
 	assert.Equal(t, "true", o.UseILM)
+	assert.Equal(t, "jaeger-test-policy", o.ILMPolicyName)
 }
diff --git a/cmd/esmapping-generator/app/renderer/render.go b/cmd/esmapping-generator/app/renderer/render.go
@@ -42,6 +42,7 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e
 		EsVersion:       opt.EsVersion,
 		IndexPrefix:     opt.IndexPrefix,
 		UseILM:          enableILM,
+		ILMPolicyName:   opt.ILMPolicyName,
 	}
 	return mappingBuilder.GetMapping(opt.Mapping)
 }

diff --git a/cmd/esmapping-generator/app/renderer/render_test.go b/cmd/esmapping-generator/app/renderer/render_test.go
@@ -50,23 +50,23 @@ func Test_getMappingAsString(t *testing.T) {
 		wantErr error
 	}{
 		{
-			name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true"},
+			name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"},
 			want: "ES version 7",
 		},
 		{
-			name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false"},
+			name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false", ILMPolicyName: "jaeger-test-policy"},
 			want: "ES version 6",
 		},
 		{
-			name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false"},
+			name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false", ILMPolicyName: "jaeger-test-policy"},
 			wantErr: errors.New("parse error"),
 		},
 		{
-			name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true"},
+			name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"},
 			wantErr: errors.New("parse error"),
 		},
 		{
-			name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo"},
+			name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo", ILMPolicyName: "jaeger-test-policy"},
 			wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
 		},
 	}

diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py
@@ -18,6 +18,7 @@
 UNIT_COUNT = 2
 SHARDS = 5
 REPLICAS = 1
+ILM_POLICY_NAME = 'jaeger-ilm-policy'
 
 
 def main():
@@ -39,6 +40,7 @@ def main():
         print('ES_TLS_CERT ... Path to TLS certificate file.')
         print('ES_TLS_KEY ... Path to TLS key file.')
         print('ES_USE_ILM .. Use ILM to manage jaeger indices.')
+        print('ES_ILM_POLICY_NAME .. The name of the ILM policy to use if ILM is active.')
         print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
         print(
             'ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.')
@@ -83,13 +85,14 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t
         replicas = os.getenv('REPLICAS', REPLICAS)
         esVersion = get_version(client)
         use_ilm = str2bool(os.getenv("ES_USE_ILM", 'false'))
+        ilm_policy_name = os.getenv('ES_ILM_POLICY_NAME', ILM_POLICY_NAME)
         if esVersion == 7:
             if use_ilm:
-                check_if_ilm_policy_exists("jaeger-ilm-policy")
+                check_if_ilm_policy_exists(ilm_policy_name)
         else:
             if use_ilm:
                 sys.exit("ILM is supported only for ES version 7+")
-        create_index_template(fix_mapping(template_name, esVersion, shards, replicas, prefix.rstrip("-"), use_ilm),
+        create_index_template(fix_mapping(template_name, esVersion, shards, replicas, prefix.rstrip("-"), use_ilm, ilm_policy_name),
                               prefix + template_name)
 
         index = index_to_rollover + '-000001'
@@ -194,10 +197,11 @@ def str2bool(v):
     return v.lower() in ('true', '1')
 
 
-def fix_mapping(template_name, esVersion, shards, replicas, indexPrefix, use_ilm):
+def fix_mapping(template_name, esVersion, shards, replicas, indexPrefix, use_ilm, ilm_policy_name):
     output = subprocess.Popen(['esmapping-generator', '--mapping', template_name, '--es-version', str(esVersion),
                                '--shards', str(shards), '--replicas',
-                               str(replicas), '--index-prefix', indexPrefix, '--use-ilm', str(use_ilm)],
+                               str(replicas), '--index-prefix', indexPrefix,
+                               '--use-ilm', str(use_ilm), '--ilm-policy-name', ilm_policy_name],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT)
     mapping, stderr = output.communicate()

diff --git a/plugin/storage/es/mappings/fixtures/jaeger-service-7.json b/plugin/storage/es/mappings/fixtures/jaeger-service-7.json
@@ -9,7 +9,7 @@
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true
     ,"lifecycle": {
-        "name": "jaeger-ilm-policy",
+        "name": "jaeger-test-policy",
         "rollover_alias": "test-jaeger-service-write"
     }
   },

diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span-7.json b/plugin/storage/es/mappings/fixtures/jaeger-span-7.json
@@ -9,7 +9,7 @@
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true
     ,"lifecycle": {
-      "name": "jaeger-ilm-policy",
+      "name": "jaeger-test-policy",
       "rollover_alias": "test-jaeger-span-write"
     }
   },

diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json
@@ -12,7 +12,7 @@
     "index.requests.cache.enable":true
   {{- if .UseILM }}
     ,"lifecycle": {
-        "name": "jaeger-ilm-policy",
+        "name": "{{ .ILMPolicyName }}",
         "rollover_alias": "{{ .IndexPrefix }}jaeger-service-write"
     }
   {{- end }}

diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json
@@ -12,7 +12,7 @@
     "index.requests.cache.enable":true
     {{- if .UseILM }}
     ,"lifecycle": {
-      "name": "jaeger-ilm-policy",
+      "name": "{{ .ILMPolicyName }}",
       "rollover_alias": "{{ .IndexPrefix }}jaeger-span-write"
     }
     {{- end }}

diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go
@@ -34,6 +34,7 @@ type MappingBuilder struct {
 	EsVersion       uint
 	IndexPrefix     string
 	UseILM          bool
+	ILMPolicyName   string
 }
 
 // GetMapping returns the rendered mapping based on elasticsearch version

diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go
@@ -54,6 +54,7 @@ func TestMappingBuilder_GetMapping(t *testing.T) {
 				EsVersion:       tt.esVersion,
 				IndexPrefix:     "test-",
 				UseILM:          true,
+				ILMPolicyName:   "jaeger-test-policy",
 			}
 			got, err := mb.GetMapping(tt.mapping)
 			require.NoError(t, err)
@@ -142,6 +143,7 @@ func TestMappingBuilder_fixMapping(t *testing.T) {
 				EsVersion:       7,
 				IndexPrefix:     "test",
 				UseILM:          true,
+				ILMPolicyName:   "jaeger-test-policy",
 			}
 			_, err := mappingBuilder.fixMapping("test")
 			if test.err != "" {
@@ -156,11 +158,12 @@ func TestMappingBuilder_fixMapping(t *testing.T) {
 
 func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 	type args struct {
-		shards      int64
-		replicas    int64
-		esVersion   uint
-		indexPrefix string
-		useILM      bool
+		shards        int64
+		replicas      int64
+		esVersion     uint
+		indexPrefix   string
+		useILM        bool
+		ilmPolicyName string
 	}
 	tests := []struct {
 		name                       string
@@ -171,11 +174,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 		{
 			name: "ES Version 7",
 			args: args{
-				shards:      3,
-				replicas:    3,
-				esVersion:   7,
-				indexPrefix: "test",
-				useILM:      true,
+				shards:        3,
+				replicas:      3,
+				esVersion:     7,
+				indexPrefix:   "test",
+				useILM:        true,
+				ilmPolicyName: "jaeger-test-policy",
 			},
 			mockNewTextTemplateBuilder: func() es.TemplateBuilder {
 				tb := mocks.TemplateBuilder{}
@@ -189,11 +193,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 		{
 			name: "ES Version 7 Service Error",
 			args: args{
-				shards:      3,
-				replicas:    3,
-				esVersion:   7,
-				indexPrefix: "test",
-				useILM:      true,
+				shards:        3,
+				replicas:      3,
+				esVersion:     7,
+				indexPrefix:   "test",
+				useILM:        true,
+				ilmPolicyName: "jaeger-test-policy",
 			},
 			mockNewTextTemplateBuilder: func() es.TemplateBuilder {
 				tb := mocks.TemplateBuilder{}
@@ -209,11 +214,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 		{
 			name: "ES Version < 7",
 			args: args{
-				shards:      3,
-				replicas:    3,
-				esVersion:   6,
-				indexPrefix: "test",
-				useILM:      true,
+				shards:        3,
+				replicas:      3,
+				esVersion:     6,
+				indexPrefix:   "test",
+				useILM:        true,
+				ilmPolicyName: "jaeger-test-policy",
 			},
 			mockNewTextTemplateBuilder: func() es.TemplateBuilder {
 				tb := mocks.TemplateBuilder{}
@@ -227,11 +233,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 		{
 			name: "ES Version < 7 Service Error",
 			args: args{
-				shards:      3,
-				replicas:    3,
-				esVersion:   6,
-				indexPrefix: "test",
-				useILM:      true,
+				shards:        3,
+				replicas:      3,
+				esVersion:     6,
+				indexPrefix:   "test",
+				useILM:        true,
+				ilmPolicyName: "jaeger-test-policy",
 			},
 			mockNewTextTemplateBuilder: func() es.TemplateBuilder {
 				tb := mocks.TemplateBuilder{}
@@ -246,11 +253,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 		{
 			name: "ES Version < 7 Span Error",
 			args: args{
-				shards:      3,
-				replicas:    3,
-				esVersion:   6,
-				indexPrefix: "test",
-				useILM:      true,
+				shards:        3,
+				replicas:      3,
+				esVersion:     6,
+				indexPrefix:   "test",
+				useILM:        true,
+				ilmPolicyName: "jaeger-test-policy",
 			},
 			mockNewTextTemplateBuilder: func() es.TemplateBuilder {
 				tb := mocks.TemplateBuilder{}
@@ -264,11 +272,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 		{
 			name: "ES Version  7 Span Error",
 			args: args{
-				shards:      3,
-				replicas:    3,
-				esVersion:   7,
-				indexPrefix: "test",
-				useILM:      true,
+				shards:        3,
+				replicas:      3,
+				esVersion:     7,
+				indexPrefix:   "test",
+				useILM:        true,
+				ilmPolicyName: "jaeger-test-policy",
 			},
 			mockNewTextTemplateBuilder: func() es.TemplateBuilder {
 				tb := mocks.TemplateBuilder{}
@@ -289,6 +298,7 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) {
 				EsVersion:       test.args.esVersion,
 				IndexPrefix:     test.args.indexPrefix,
 				UseILM:          test.args.useILM,
+				ILMPolicyName:   test.args.ilmPolicyName,
 			}
 			_, _, err := mappingBuilder.GetSpanServiceMappings()
 			if test.err != "" {