diff --git a/Makefile b/Makefile index 8507c3b013d..5ceb601fa0f 100644 --- a/Makefile +++ b/Makefile @@ -225,9 +225,13 @@ build-tracegen: build-anonymizer: $(GOBUILD) -o ./cmd/anonymizer/anonymizer-$(GOOS)-$(GOARCH) ./cmd/anonymizer/main.go -.PHONY: build-templateloader -build-templateloader: - GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/templateloader ./cmd/templateloader/main.go +.PHONY: build-templatizer +build-templatizer: + $(GOBUILD) -o ./plugin/storage/es/templatizer-$(GOOS)-$(GOARCH) ./cmd/templatizer/main.go + +.PHONY: build-templatizer-linux +build-templatizer-linux: + GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/templatizer ./cmd/templatizer/main.go .PHONY: docker-hotrod docker-hotrod: @@ -348,7 +352,7 @@ docker-images-cassandra: @echo "Finished building jaeger-cassandra-schema ==============" .PHONY: docker-images-elastic -docker-images-elastic: build-templateloader +docker-images-elastic: build-templatizer-linux docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover @echo "Finished building jaeger-es-indices-clean ==============" diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go index d87a8441fa8..90462a2f2b5 100644 --- a/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go +++ b/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go @@ -20,6 +20,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter/exporterhelper" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" "github.com/jaegertracing/jaeger/plugin/storage/es" ) @@ -31,8 +32,8 @@ func newExporter(ctx context.Context, config *Config, params component.ExporterC return nil, err } if config.Primary.IsCreateIndexTemplates() { - spanMapping, serviceMapping, e := es.GetSpanServiceMappings(esCfg.GetNumShards(), esCfg.GetNumReplicas(), uint(w.esClientVersion()), esCfg.GetIndexPrefix(), esCfg.GetUseILM()) - if e != nil { + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, esCfg.GetNumShards(), esCfg.GetNumReplicas(), uint(w.esClientVersion()), esCfg.GetIndexPrefix(), esCfg.GetUseILM()) + if err != nil { return nil, err } if err = w.CreateTemplates(ctx, spanMapping, serviceMapping); err != nil { diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go index 22a29e65062..1c0dd285261 100644 --- a/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go +++ b/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go @@ -32,6 +32,7 @@ import ( "github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/esclient" "github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/reader/es/esdependencyreader" "github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/reader/es/esspanreader" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es" @@ -104,7 +105,7 @@ func (s *IntegrationTest) initSpanstore(allTagsAsFields bool) error { return err } esVersion := uint(w.esClientVersion()) - spanMapping, serviceMapping, err := es.GetSpanServiceMappings(numShards, numReplicas, esVersion, "", false) + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, numShards, numReplicas, esVersion, "", false) if err != nil { return err } diff --git a/cmd/templateloader/main.go b/cmd/templateloader/main.go deleted file mode 100644 index a331dda2e9d..00000000000 --- a/cmd/templateloader/main.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2020 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "strconv" - - "go.uber.org/zap" - - esTemplate "github.com/jaegertracing/jaeger/pkg/es" - "github.com/jaegertracing/jaeger/plugin/storage/es" -) - -var logger, _ = zap.NewDevelopment() -var fixMappingFunc = es.FixMapping -var loadMappingFunc = es.LoadMapping -var mapping string -var err error - -func main() { - - mappingName := flag.String("mapping", "", "Pass name of the mapping to parse - should be either jaeger-service or jaeger-span") - esVersion := flag.Int64("esVersion", 7, "Version of ES") - shards := flag.Int64("shards", 3, "Number of shards") - replicas := flag.Int64("replicas", 3, "Number of replicas") - esPrefix := flag.String("esPrefix", "", "Prefix for ES indices") - useILM := flag.String("useILM", "false", "Set to true to enable ILM") - flag.Parse() - if !isValidOption(*mappingName) { - logger.Fatal("please pass either 'jaeger-service' or 'jaeger-span' as argument") - } - parsedMapping, err := getMappingAsString(*mappingName, *esPrefix, *useILM, *esVersion, *shards, *replicas) - if err != nil { - logger.Fatal(err.Error()) - } - print(parsedMapping) - -} - -func getMappingAsString(mappingName, esPrefix, useILM string, esVersion, shards, replicas int64) (string, error) { - - if esVersion == 7 { - enableILM, err := strconv.ParseBool(useILM) - if err != nil { - return "", err - } - mapping, err = fixMappingFunc(esTemplate.TextTemplateBuilder{}, loadMappingFunc("/"+mappingName+"-7.json"), shards, replicas, esPrefix, enableILM) - if err != nil { - return "", err - } - } else { - mapping, err = fixMappingFunc(esTemplate.TextTemplateBuilder{}, loadMappingFunc("/"+mappingName+".json"), shards, replicas, "", false) - if err != nil { - return "", err - } - } - return mapping, nil - -} - -func isValidOption(val string) bool { - allowedValues := []string{"jaeger-span", "jaeger-service"} - for _, value := range allowedValues { - if val == value { - return true - } - } - return false -} diff --git a/cmd/templatizer/app/flags.go b/cmd/templatizer/app/flags.go new file mode 100644 index 00000000000..3f3efe39d8d --- /dev/null +++ b/cmd/templatizer/app/flags.go @@ -0,0 +1,81 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "github.com/spf13/cobra" +) + +// Options represent configurable parameters for jaeger-templatizer +type Options struct { + Mapping string + EsVersion int64 + Shards int64 + Replicas int64 + EsPrefix string + UseILM string // using string as util is being used in python and using bool leads to type issues. +} + +const ( + mappingFlag = "mapping" + esVersionFlag = "esVersion" + shardsFlag = "shards" + replicasFlag = "replicas" + esPrefixFlag = "esPrefix" + useILMFlag = "useILM" +) + +// AddFlags adds flags for templatizer main program +func (o *Options) AddFlags(command *cobra.Command) { + command.Flags().StringVarP( + &o.Mapping, + mappingFlag, + "m", + "", + "Pass either jaeger-span or jaeger-service") + command.Flags().Int64VarP( + &o.EsVersion, + esVersionFlag, + "v", + 7, + "the major Elasticsearch version") + command.Flags().Int64VarP( + &o.Shards, + shardsFlag, + "s", + 5, + "the number of shards per index in Elasticsearch") + command.Flags().Int64VarP( + &o.Replicas, + replicasFlag, + "r", + 1, + "the number of replicas per index in Elasticsearch") + command.Flags().StringVarP( + &o.EsPrefix, + esPrefixFlag, + "e", + "", + "specifies index prefix") + command.Flags().StringVarP( + &o.UseILM, + useILMFlag, + "u", + "false", + "set to true to use ILM for managing lifecycle of jaeger indices") + + // mark mapping flag as mandatory + command.MarkFlagRequired(mappingFlag) +} diff --git a/cmd/templatizer/app/flags_test.go b/cmd/templatizer/app/flags_test.go new file mode 100644 index 00000000000..2d3149a3c6b --- /dev/null +++ b/cmd/templatizer/app/flags_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func TestOptionsWithDefaultFlags(t *testing.T) { + o := Options{} + c := cobra.Command{} + o.AddFlags(&c) + + assert.Equal(t, "", o.Mapping) + assert.Equal(t, int64(7), o.EsVersion) + assert.Equal(t, int64(5), o.Shards) + assert.Equal(t, int64(1), o.Replicas) + assert.Equal(t, "", o.EsPrefix) + assert.Equal(t, "false", o.UseILM) +} + +func TestOptionsWithFlags(t *testing.T) { + o := Options{} + c := cobra.Command{} + + o.AddFlags(&c) + c.ParseFlags([]string{ + "--mapping=jaeger-span", + "--esVersion=6", + "--shards=5", + "--replicas=1", + "--esPrefix=test", + "--useILM=true", + }) + + assert.Equal(t, "jaeger-span", o.Mapping) + assert.Equal(t, int64(6), o.EsVersion) + assert.Equal(t, int64(5), o.Shards) + assert.Equal(t, int64(1), o.Replicas) + assert.Equal(t, "test", o.EsPrefix) + assert.Equal(t, "true", o.UseILM) +} diff --git a/cmd/templatizer/app/renderer/render.go b/cmd/templatizer/app/renderer/render.go new file mode 100644 index 00000000000..61b1aa0f9c7 --- /dev/null +++ b/cmd/templatizer/app/renderer/render.go @@ -0,0 +1,59 @@ +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package renderer + +import ( + "strconv" + + "github.com/jaegertracing/jaeger/cmd/templatizer/app" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/plugin/storage/es" +) + +var fixMappingFunc = es.FixMapping +var loadMappingFunc = es.LoadMapping +var mapping string +var err error + +func GetMappingAsString(opt *app.Options) (string, error) { + + if opt.EsVersion == 7 { + enableILM, err := strconv.ParseBool(opt.UseILM) + if err != nil { + return "", err + } + mapping, err = fixMappingFunc(esTemplate.TextTemplateBuilder{}, loadMappingFunc("/"+opt.Mapping+"-7.json"), opt.Shards, opt.Replicas, opt.EsPrefix, enableILM) + if err != nil { + return "", err + } + } else { + mapping, err = fixMappingFunc(esTemplate.TextTemplateBuilder{}, loadMappingFunc("/"+opt.Mapping+".json"), opt.Shards, opt.Replicas, "", false) + if err != nil { + return "", err + } + } + return mapping, nil + +} + +func IsValidOption(val string) bool { + allowedValues := []string{"jaeger-span", "jaeger-service"} + for _, value := range allowedValues { + if val == value { + return true + } + } + return false +} diff --git a/cmd/templateloader/main_test.go b/cmd/templatizer/app/renderer/render_test.go similarity index 76% rename from cmd/templateloader/main_test.go rename to cmd/templatizer/app/renderer/render_test.go index af20de8f208..272717c005d 100644 --- a/cmd/templateloader/main_test.go +++ b/cmd/templatizer/app/renderer/render_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Jaeger Authors. +// Copyright (c) 2021 The Jaeger Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package renderer import ( "errors" "testing" + "github.com/jaegertracing/jaeger/cmd/templatizer/app" esTemplate "github.com/jaegertracing/jaeger/pkg/es" ) @@ -32,7 +33,7 @@ func TestIsValidOption(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if got := isValidOption(test.arg); got != test.expectedValue { + if got := IsValidOption(test.arg); got != test.expectedValue { t.Errorf("isValidOption() = %v, want %v", got, test.expectedValue) } }) @@ -40,42 +41,35 @@ func TestIsValidOption(t *testing.T) { } func Test_getMappingAsString(t *testing.T) { - type args struct { - mappingName string - esPrefix string - useILM string - esVersion int64 - shards int64 - replicas int64 - } + tests := []struct { name string - args args + args app.Options fixMappingFunc func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) want string wantErr bool }{ - {name: "ES version 7", args: args{"jaeger-span", "test", "true", 7, 5, 5}, + {name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"}, fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { return "ES version 7", nil }, want: "ES version 7", wantErr: false, }, - {name: "ES version 6", args: args{"jaeger-service", "test", "false", 6, 5, 5}, + {name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"}, fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { return "ES version 6", nil }, want: "ES version 6", wantErr: false, }, - {name: "Parse Error version 6", args: args{"jaeger-service", "test", "false", 6, 5, 5}, + {name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"}, fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { return "", errors.New("parse error") }, want: "", wantErr: true, - }, {name: "Parse Error version 7", args: args{"jaeger-service", "test", "false", 7, 5, 5}, + }, {name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"}, fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { return "", errors.New("parse error") }, @@ -95,7 +89,7 @@ func Test_getMappingAsString(t *testing.T) { fixMappingFunc = tt.fixMappingFunc loadMappingFunc = func(string) string { return "test" } - got, err := getMappingAsString(tt.args.mappingName, tt.args.esPrefix, tt.args.useILM, tt.args.esVersion, tt.args.shards, tt.args.replicas) + got, err := GetMappingAsString(&tt.args) if (err != nil) != tt.wantErr { t.Errorf("getMappingAsString() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/cmd/templatizer/main.go b/cmd/templatizer/main.go new file mode 100644 index 00000000000..d9bcdb2535d --- /dev/null +++ b/cmd/templatizer/main.go @@ -0,0 +1,61 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "github.com/jaegertracing/jaeger/pkg/version" + "github.com/spf13/cobra" + "os" + + "go.uber.org/zap" + + "github.com/jaegertracing/jaeger/cmd/templatizer/app" + "github.com/jaegertracing/jaeger/cmd/templatizer/app/renderer" +) + +var logger, _ = zap.NewDevelopment() + +func main() { + var options = app.Options{} + var command = &cobra.Command{ + Use: "jaeger-templatizer", + Short: "Jaeger templatizer prints rendered mappings as string", + Long: `Jaeger templatizer renders passed templates with provided values and prints rendered output to stdout`, + Run: func(cmd *cobra.Command, args []string) { + + if !renderer.IsValidOption(options.Mapping) { + logger.Fatal("please pass either 'jaeger-service' or 'jaeger-span' as argument") + } + + parsedMapping, err := renderer.GetMappingAsString(&options) + if err != nil { + logger.Fatal(err.Error()) + } + print(parsedMapping) + }, + } + + options.AddFlags(command) + + command.AddCommand(version.Command()) + + if err := command.Execute(); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + +} diff --git a/pkg/es/textTemplate.go b/pkg/es/textTemplate.go index 4e9c835add4..304b8b63476 100644 --- a/pkg/es/textTemplate.go +++ b/pkg/es/textTemplate.go @@ -19,12 +19,13 @@ import ( "text/template" ) -// TemplateApplier is an abstraction to support mocking text/template +// TemplateApplier applies a parsed template to input data that maps to the template's variables. type TemplateApplier interface { Execute(wr io.Writer, data interface{}) error } -// TemplateBuilder is an abstraction to support mocking text/template +// TemplateBuilder parses a given string and returns TemplateApplier +// TemplateBuilder is an abstraction to support mocking template/text type TemplateBuilder interface { Parse(text string) (TemplateApplier, error) } @@ -36,8 +37,3 @@ type TextTemplateBuilder struct{} func (t TextTemplateBuilder) Parse(mapping string) (TemplateApplier, error) { return template.New("mapping").Parse(mapping) } - -// NewTextTemplateBuilder returns a TextTemplateBuilder -func NewTextTemplateBuilder() TemplateBuilder { - return TextTemplateBuilder{} -} diff --git a/pkg/es/textTemplate_test.go b/pkg/es/textTemplate_test.go index edd9950ac2a..29488841654 100644 --- a/pkg/es/textTemplate_test.go +++ b/pkg/es/textTemplate_test.go @@ -34,8 +34,3 @@ func TestParse(t *testing.T) { assert.Equal(t, str, writer.String()) } - -func TestNewTextTemplateBuilder(t *testing.T) { - textBuilder := NewTextTemplateBuilder() - assert.IsType(t, TextTemplateBuilder{}, textBuilder) -} diff --git a/plugin/storage/es/Dockerfile.rollover b/plugin/storage/es/Dockerfile.rollover index c4d0bb6df4e..eac9401a531 100644 --- a/plugin/storage/es/Dockerfile.rollover +++ b/plugin/storage/es/Dockerfile.rollover @@ -3,9 +3,9 @@ FROM python:3-alpine3.11 # Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494 RUN pip install urllib3==1.24.3 -RUN pip install elasticsearch elasticsearch-curator pathlib2 Jinja2 +RUN pip install elasticsearch elasticsearch-curator COPY ./mappings/* /mappings/ COPY esRollover.py /es-rollover/ -COPY templateloader /usr/bin/ +COPY templatizer /usr/bin/ ENTRYPOINT ["python3", "/es-rollover/esRollover.py"] diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py index 89c36549d14..cb63110895d 100755 --- a/plugin/storage/es/esRollover.py +++ b/plugin/storage/es/esRollover.py @@ -195,9 +195,9 @@ def str2bool(v): def fix_mapping(template_name, esVersion, shards, replicas, esprefix, use_ilm): - output = subprocess.Popen(['templateloader', '-mapping', template_name, '-esVersion', str(esVersion), - '-shards', str(shards), '-replicas', - str(replicas), '-esPrefix', esprefix, '-useILM', str(use_ilm)], + output = subprocess.Popen(['templatizer', '--mapping', template_name, '--esVersion', str(esVersion), + '--shards', str(shards), '--replicas', + str(replicas), '--esPrefix', esprefix, '--useILM', str(use_ilm)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) mapping, stderr = output.communicate() diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index c54c944f001..79060eae94c 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -39,8 +39,6 @@ const ( archiveNamespace = "es-archive" ) -var newTextTemplateBuilder = es.NewTextTemplateBuilder - // Factory implements storage.Factory for Elasticsearch backend. type Factory struct { Options *Options @@ -168,7 +166,7 @@ func createSpanWriter( return nil, err } - spanMapping, serviceMapping, err := GetSpanServiceMappings(cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion(), cfg.GetIndexPrefix(), cfg.GetUseILM()) + spanMapping, serviceMapping, err := GetSpanServiceMappings(es.TextTemplateBuilder{}, cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion(), cfg.GetIndexPrefix(), cfg.GetUseILM()) if err != nil { return nil, err } @@ -194,23 +192,23 @@ func createSpanWriter( } // GetSpanServiceMappings returns span and service mappings -func GetSpanServiceMappings(shards, replicas int64, esVersion uint, esPrefix string, useILM bool) (string, string, error) { +func GetSpanServiceMappings(tb es.TemplateBuilder, shards, replicas int64, esVersion uint, esPrefix string, useILM bool) (string, string, error) { if esVersion == 7 { - spanMapping, err := FixMapping(newTextTemplateBuilder(), LoadMapping("/jaeger-span-7.json"), shards, replicas, esPrefix, useILM) + spanMapping, err := FixMapping(tb, LoadMapping("/jaeger-span-7.json"), shards, replicas, esPrefix, useILM) if err != nil { return "", "", err } - serviceMapping, err := FixMapping(newTextTemplateBuilder(), LoadMapping("/jaeger-service-7.json"), shards, replicas, esPrefix, useILM) + serviceMapping, err := FixMapping(tb, LoadMapping("/jaeger-service-7.json"), shards, replicas, esPrefix, useILM) if err != nil { return "", "", err } return spanMapping, serviceMapping, nil } - spanMapping, err := FixMapping(newTextTemplateBuilder(), LoadMapping("/jaeger-span.json"), shards, replicas, "", false) + spanMapping, err := FixMapping(tb, LoadMapping("/jaeger-span.json"), shards, replicas, "", false) if err != nil { return "", "", err } - serviceMapping, err := FixMapping(newTextTemplateBuilder(), LoadMapping("/jaeger-service.json"), shards, replicas, "", false) + serviceMapping, err := FixMapping(tb, LoadMapping("/jaeger-service.json"), shards, replicas, "", false) if err != nil { return "", "", err } @@ -218,11 +216,11 @@ func GetSpanServiceMappings(shards, replicas int64, esVersion uint, esPrefix str } // GetDependenciesMappings returns dependencies mappings -func GetDependenciesMappings(shards, replicas int64, esVersion uint) (string, error) { +func GetDependenciesMappings(tb es.TemplateBuilder, shards, replicas int64, esVersion uint) (string, error) { if esVersion == 7 { - return FixMapping(newTextTemplateBuilder(), LoadMapping("/jaeger-dependencies-7.json"), shards, replicas, "", false) + return FixMapping(tb, LoadMapping("/jaeger-dependencies-7.json"), shards, replicas, "", false) } - return FixMapping(newTextTemplateBuilder(), LoadMapping("/jaeger-dependencies.json"), shards, replicas, "", false) + return FixMapping(tb, LoadMapping("/jaeger-dependencies.json"), shards, replicas, "", false) } // LoadMapping returns index mappings from go assets as strings @@ -235,11 +233,11 @@ func LoadMapping(name string) string { func FixMapping(tb es.TemplateBuilder, mapping string, shards, replicas int64, esPrefix string, useILM bool) (string, error) { tmpl, err := tb.Parse(mapping) - writer := new(bytes.Buffer) - if err != nil { return "", err } + writer := new(bytes.Buffer) + if esPrefix != "" { esPrefix += "-" } diff --git a/plugin/storage/es/factory_test.go b/plugin/storage/es/factory_test.go index 450629faed4..834a90a8968 100644 --- a/plugin/storage/es/factory_test.go +++ b/plugin/storage/es/factory_test.go @@ -45,7 +45,7 @@ type mockClientBuilder struct { createTemplateError error } -var mockTextTemplateBuilder = func() es.TemplateBuilder { +var mockErrTextTemplateBuilder = func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) @@ -98,16 +98,6 @@ func TestElasticsearchFactory(t *testing.T) { _, err = f.CreateArchiveSpanWriter() assert.NoError(t, err) - - oldTextTemplateBuilder := newTextTemplateBuilder - defer func() { - newTextTemplateBuilder = oldTextTemplateBuilder - }() - - newTextTemplateBuilder = mockTextTemplateBuilder - _, err = f.CreateSpanWriter() - assert.EqualError(t, err, "template load error") - assert.NoError(t, f.Close()) } @@ -375,7 +365,8 @@ func TestGetSpanServiceMappings(t *testing.T) { mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() tb.On("Parse", mock.Anything).Return(&ta, nil) return &tb }, @@ -412,7 +403,8 @@ func TestGetSpanServiceMappings(t *testing.T) { mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() tb.On("Parse", mock.Anything).Return(&ta, nil) return &tb }, @@ -421,14 +413,7 @@ func TestGetSpanServiceMappings(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - - oldTextTemplateBuilder := newTextTemplateBuilder - defer func() { - newTextTemplateBuilder = oldTextTemplateBuilder - }() - - newTextTemplateBuilder = test.mockNewTextTemplateBuilder - _, _, err := GetSpanServiceMappings(test.args.shards, test.args.replicas, + _, _, err := GetSpanServiceMappings(test.mockNewTextTemplateBuilder(), test.args.shards, test.args.replicas, test.args.esVersion, test.args.esPrefix, test.args.useILM) if test.err != "" { @@ -441,14 +426,9 @@ func TestGetSpanServiceMappings(t *testing.T) { } func TestGetDependenciesMappings(t *testing.T) { - oldTextTemplateBuilder := newTextTemplateBuilder - defer func() { - newTextTemplateBuilder = oldTextTemplateBuilder - }() - newTextTemplateBuilder = mockTextTemplateBuilder - _, err := GetDependenciesMappings(5, 5, 7) + _, err := GetDependenciesMappings(mockErrTextTemplateBuilder(), 5, 5, 7) assert.EqualError(t, err, "template load error") - _, err = GetDependenciesMappings(5, 5, 6) + _, err = GetDependenciesMappings(mockErrTextTemplateBuilder(), 5, 5, 6) assert.EqualError(t, err, "template load error") } diff --git a/plugin/storage/es/templatizer b/plugin/storage/es/templatizer new file mode 100755 index 00000000000..b391f493511 Binary files /dev/null and b/plugin/storage/es/templatizer differ diff --git a/plugin/storage/es/templatizer-darwin-amd64 b/plugin/storage/es/templatizer-darwin-amd64 new file mode 100755 index 00000000000..799616e455a Binary files /dev/null and b/plugin/storage/es/templatizer-darwin-amd64 differ diff --git a/plugin/storage/integration/elasticsearch_test.go b/plugin/storage/integration/elasticsearch_test.go index ede3eb3ba58..ac93d2659dd 100644 --- a/plugin/storage/integration/elasticsearch_test.go +++ b/plugin/storage/integration/elasticsearch_test.go @@ -31,6 +31,7 @@ import ( "go.uber.org/zap" "github.com/jaegertracing/jaeger/model" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" eswrapper "github.com/jaegertracing/jaeger/pkg/es/wrapper" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es" @@ -107,7 +108,7 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro return err } client := eswrapper.WrapESClient(s.client, bp, esVersion) - spanMapping, serviceMapping, err := es.GetSpanServiceMappings(5, 1, client.GetVersion(), indexPrefix, false) + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion(), indexPrefix, false) if err != nil { return err } @@ -137,7 +138,7 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro MaxDocCount: defaultMaxDocCount, }) dependencyStore := dependencystore.NewDependencyStore(client, s.logger, indexPrefix, indexDateLayout, defaultMaxDocCount) - depMapping, err := es.GetDependenciesMappings(5, 1, client.GetVersion()) + depMapping, err := es.GetDependenciesMappings(esTemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion()) if err != nil { return err }