Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Introduce dynamic index patterns #10450

Closed
wants to merge 6 commits into from
Closed

[WIP] Introduce dynamic index patterns #10450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6a5a327
Introduce dynamic index patterns
ruflin Jan 31, 2019
590728e
make it possible to load fields from internal field storage
ruflin Jan 31, 2019
aa01503
add changelog follow up
ruflin Jan 31, 2019
4195e6b
add pr id
ruflin Jan 31, 2019
b8bdb62
remove loader interface as only 1 loader left
ruflin Jan 31, 2019
50a35ba
more cleanup, remove some capabilites
ruflin Jan 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG-developer.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.

==== Breaking changes

- Remove support for loading dashboards into Elastic Stack version 5.x. {pull}10451[10451]
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Taken over for 10451 as it was missing there.


==== Bugfixes

==== Added
Expand Down
92 changes: 0 additions & 92 deletions dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go
View file
Open in desktop

This file was deleted.

4 changes: 2 additions & 2 deletions libbeat/cmd/instance/beat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -705,8 +705,8 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
}

if b.Config.Dashboards.Enabled() {
err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""),
b.Config.Kibana, b.Config.Dashboards, nil)
err := dashboards.ImportDashboards(ctx, b.Info, paths.Resolve(paths.Home, ""),
b.Config.Kibana, b.Config.Dashboards, nil, b.Config.Migration.Enabled(), b.Fields)
if err != nil {
return errw.Wrap(err, "Error importing Kibana dashboards")
}
Expand Down
31 changes: 25 additions & 6 deletions libbeat/dashboards/dashboards.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,13 @@ import (
"context"
"errors"
"fmt"
"log"
"path/filepath"

"github.com/elastic/beats/libbeat/beat"

"github.com/elastic/beats/libbeat/kibana"

errw "github.com/pkg/errors"

"github.com/elastic/beats/libbeat/common"
Expand All @@ -31,17 +36,17 @@ import (
// ImportDashboards tries to import the kibana dashboards.
func ImportDashboards(
ctx context.Context,
beatName, hostname, homePath string,
beatInfo beat.Info, homePath string,
kibanaConfig, dashboardsConfig *common.Config,
msgOutputter MessageOutputter,
msgOutputter MessageOutputter, migration bool, fields []byte,
) error {
if dashboardsConfig == nil || !dashboardsConfig.Enabled() {
return nil
}

// unpack dashboard config
dashConfig := defaultConfig
dashConfig.Beat = beatName
dashConfig.Beat = beatInfo.Beat
dashConfig.Dir = filepath.Join(homePath, defaultDirectory)
err := dashboardsConfig.Unpack(&dashConfig)
if err != nil {
Expand All @@ -57,14 +62,28 @@ func ImportDashboards(
return errors.New("kibana configuration missing for loading dashboards.")
}

return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter)
// Generate index pattern
version, _ := common.NewVersion("7.0.0") // TODO: dynamic version
//indexP, _ := esConfig.String("index", 0)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@urso I think this also ties into your current work around indexing changes. Where should the config which index pattern should be used go for Kibana setup part?

// TODO: What should we do about the index pattern. Kind of strange that ES configs are needed here.
indexPattern, err := kibana.NewGenerator(beatInfo.Beat+"-*", beatInfo.Beat, fields, dashConfig.Dir, beatInfo.Version, *version, migration)
if err != nil {
log.Fatal(err)
}

pattern, err := indexPattern.Generate()
if err != nil {
log.Fatalf("ERROR: %s", err)
}

return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, kibanaConfig, &dashConfig, msgOutputter, pattern)

}

func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config,
dashboardsConfig *Config, msgOutputter MessageOutputter) error {
dashboardsConfig *Config, msgOutputter MessageOutputter, fields common.MapStr) error {

kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter)
kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter, fields)
if err != nil {
return fmt.Errorf("fail to create the Kibana loader: %v", err)
}
Expand Down
16 changes: 4 additions & 12 deletions libbeat/dashboards/importer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,10 @@ type MessageOutputter func(msg string, a ...interface{})
type Importer struct {
cfg *Config
version common.Version

loader Loader
}

type Loader interface {
ImportIndex(file string) error
ImportDashboard(file string) error
statusMsg(msg string, a ...interface{})
Close() error
loader *KibanaLoader
}

func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, error) {
func NewImporter(version common.Version, cfg *Config, loader *KibanaLoader) (*Importer, error) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported function NewImporter should have comment or be unexported


// Current max version is 7
if version.Major > 6 {
Expand Down Expand Up @@ -106,7 +98,7 @@ func (imp Importer) ImportFile(fileType string, file string) error {
if fileType == "dashboard" {
return imp.loader.ImportDashboard(file)
} else if fileType == "index-pattern" {
return imp.loader.ImportIndex(file)
return imp.loader.ImportIndex()
}
return fmt.Errorf("Unexpected file type %s", fileType)
}
Expand Down Expand Up @@ -312,7 +304,7 @@ func (imp Importer) ImportKibanaDir(dir string) error {

check := []string{}
if !imp.cfg.OnlyDashboards {
check = append(check, "index-pattern")
imp.loader.ImportIndex()
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This breaks current behaviour in that it makes it impossible to load an index pattern from a file and requires the one from Beats.

}
wantDashboards := false
if !imp.cfg.OnlyIndex {
Expand Down
21 changes: 5 additions & 16 deletions libbeat/dashboards/kibana_loader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ type KibanaLoader struct {
version common.Version
hostname string
msgOutputter MessageOutputter
fields common.MapStr
}

func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) {
func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter, fields common.MapStr) (*KibanaLoader, error) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported function NewKibanaLoader should have comment or be unexported


if cfg == nil || !cfg.Enabled() {
return nil, fmt.Errorf("Kibana is not configured or enabled")
Expand All @@ -57,6 +58,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *
version: client.GetVersion(),
hostname: hostname,
msgOutputter: msgOutputter,
fields: fields,
}

version := client.GetVersion()
Expand All @@ -81,24 +83,11 @@ func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, r
return client, nil
}

func (loader KibanaLoader) ImportIndex(file string) error {
func (loader KibanaLoader) ImportIndex() error {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported method KibanaLoader.ImportIndex should have comment or be unexported

params := url.Values{}
params.Set("force", "true") //overwrite the existing dashboards

// read json file
reader, err := ioutil.ReadFile(file)
if err != nil {
return fmt.Errorf("fail to read index-pattern from file %s: %v", file, err)
}

var indexContent common.MapStr
err = json.Unmarshal(reader, &indexContent)
if err != nil {
return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err)
}

indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent)

indexContent := ReplaceIndexInIndexPattern(loader.config.Index, loader.fields)
return loader.client.ImportJSON(importAPI, params, indexContent)
}

Expand Down
17 changes: 12 additions & 5 deletions libbeat/kibana/fields_transformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ type fieldsTransformer struct {
transformedFieldFormatMap common.MapStr
version *common.Version
keys map[string]int
migration bool
}

func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) {
func newFieldsTransformer(version *common.Version, fields common.Fields, migration bool) (*fieldsTransformer, error) {
if version == nil {
return nil, errors.New("Version must be given")
}
Expand All @@ -44,6 +45,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field
transformedFields: []common.MapStr{},
transformedFieldFormatMap: common.MapStr{},
keys: map[string]int{},
migration: migration,
}, nil
}

Expand Down Expand Up @@ -90,6 +92,11 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
if t.version.LessThan(v640) {
continue
}

// Skip migration alias
if f.MigrationAlias && !t.migration {
continue
}
if ff := t.fields.GetField(f.AliasPath); ff != nil {
// copy the field, keep
path := f.Path
Expand Down Expand Up @@ -117,10 +124,10 @@ func (t *fieldsTransformer) update(target *common.MapStr, override common.Field)
field, _ := transformField(t.version, override)
if override.Type == "" || (*target)["type"] == field["type"] {
target.Update(field)
if !override.Overwrite {
// compatible duplication
return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true', %+v, %+v", override.Path, override, field)
}
//if !override.Overwrite {
// // compatible duplication
// return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true', %+v, %+v", override.Path, override, field)
//}
return nil
}
// incompatible duplication
Expand Down
Loading