Fix index template loading when the new index is selected (elastic#22682

)

This PR fixes the `setup.template.type` setting. When `index` were selected, Beats based the generated index on the legacy format, not on the component. This prevented Beats from loading the index.

`setup.template.type=index` was broken. Now it is fixed.

```
./filebeat setup --index-management -E setup.template.type=index
```

Closes elastic#22482

(cherry picked from commit 569268d)

CHANGELOG.next.asciidoc

@@ -122,6 +122,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916]
 - Fix metrics hints builder to avoid wrong container metadata usage when port is not exposed {pull}18979[18979]
 - Server-side TLS config now validates certificate and key are both specified {pull}19584[19584]
+- Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682]
 
 *Auditbeat*
 

libbeat/template/template.go

@@ -232,9 +232,8 @@ func (t *Template) loadMinimalComponent() common.MapStr {
 }
 
 func (t *Template) loadMinimalIndex() common.MapStr {
-	m := t.loadMinimalLegacy()
+	m := t.loadMinimalComponent()
 	m["priority"] = t.priority
-	delete(m, "order")
 	return m
 }
 
@@ -301,9 +300,10 @@ func (t *Template) generateComponent(properties common.MapStr) common.MapStr {
 }
 
 func (t *Template) generateIndex(properties common.MapStr) common.MapStr {
-	tmpl := t.generateLegacy(properties)
+	tmpl := t.generateComponent(properties)
 	tmpl["priority"] = t.priority
-	delete(tmpl, "order")
+	keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern())
+	tmpl[keyPattern] = patterns
 	return tmpl
 }
 

libbeat/tests/system/idxmgmt.py

@@ -56,11 +56,28 @@ def assert_index_template_not_loaded(self, template):
         with pytest.raises(NotFoundError):
             self._client.transport.perform_request('GET', '/_template/' + template)
 
-    def assert_index_template_loaded(self, template):
+    def assert_legacy_index_template_loaded(self, template):
         resp = self._client.transport.perform_request('GET', '/_template/' + template)
         assert template in resp
         assert "lifecycle" not in resp[template]["settings"]["index"]
 
+    def assert_index_template_loaded(self, template):
+        resp = self._client.transport.perform_request('GET', '/_index_template/' + template)
+        found = False
+        for index_template in resp['index_templates']:
+            if index_template['name'] == template:
+                found = True
+        assert found
+
+    def assert_component_template_loaded(self, template):
+        resp = self._client.transport.perform_request('GET', '/_component_template/' + template)
+        found = False
+        print(resp)
+        for index_template in resp['component_templates']:
+            if index_template['name'] == template:
+                found = True
+        assert found
+
     def assert_ilm_template_loaded(self, template, policy, alias):
         resp = self._client.transport.perform_request('GET', '/_template/' + template)
         assert resp[template]["settings"]["index"]["lifecycle"]["name"] == policy

libbeat/tests/system/test_cmd_setup_index_management.py

@@ -110,7 +110,7 @@ def test_setup_ilm_disabled(self):
                                               "-E", "setup.ilm.enabled=false"])
 
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.index_name)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
         self.idxmgmt.assert_alias_not_created(self.alias_name)
         self.idxmgmt.assert_policy_not_created(self.policy_name)
 
@@ -242,7 +242,7 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self):
                                               "-E", "setup.template.pattern=" + self.custom_template + "*"])
 
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.custom_template)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.custom_template)
         self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"])
         self.idxmgmt.assert_alias_not_created(self.alias_name)
         self.idxmgmt.assert_policy_not_created(self.policy_name)
@@ -261,7 +261,7 @@ def test_setup_template_with_opts(self):
                                               "-E", "setup.template.settings.index.number_of_shards=2"])
 
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.index_name)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
 
         # check that settings are overwritten
         resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
@@ -284,7 +284,7 @@ def test_setup_overwrite_template_on_ilm_policy_created(self):
                                               "-E", "setup.template.name=" + self.custom_alias,
                                               "-E", "setup.template.pattern=" + self.custom_alias + "*"])
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.custom_alias)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias)
         self.idxmgmt.assert_policy_not_created(self.policy_name)
 
         # ensure ilm policy is created, triggering overwriting existing template

libbeat/tests/system/test_ilm.py

@@ -68,7 +68,7 @@ def test_ilm_disabled(self):
         self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
         proc.check_kill_and_wait()
 
-        self.idxmgmt.assert_index_template_loaded(self.index_name)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
         self.idxmgmt.assert_alias_not_created(self.alias_name)
         self.idxmgmt.assert_policy_not_created(self.policy_name)
 
@@ -234,7 +234,7 @@ def test_setup_ilm_disabled(self):
                                               "-E", "setup.ilm.enabled=false"])
 
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.index_name)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
         self.idxmgmt.assert_alias_not_created(self.alias_name)
         self.idxmgmt.assert_policy_not_created(self.policy_name)
 

libbeat/tests/system/test_template.py

@@ -267,7 +267,7 @@ def test_setup_template_with_opts(self):
                                               "-E", "setup.template.settings.index.number_of_shards=2"])
 
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.index_name)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
 
         # check that settings are overwritten
         resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
@@ -305,7 +305,7 @@ def test_template_created_on_ilm_policy_created(self):
                                               "-E", "setup.template.name=" + self.custom_alias,
                                               "-E", "setup.template.pattern=" + self.custom_alias + "*"])
         assert exit_code == 0
-        self.idxmgmt.assert_index_template_loaded(self.custom_alias)
+        self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias)
         self.idxmgmt.assert_policy_not_created(self.policy_name)
 
         # ensure ilm policy is created, triggering overwriting existing template
@@ -322,6 +322,34 @@ def test_template_created_on_ilm_policy_created(self):
         index = resp[self.custom_alias]["settings"]["index"]
         assert index["number_of_shards"] == "2", index["number_of_shards"]
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @pytest.mark.tag('integration')
+    def test_setup_template_index(self):
+        """
+        Test template setup of new index templates
+        """
+        self.render_config()
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.setupCmd,
+                                              "-E", "setup.template.type=index"])
+
+        assert exit_code == 0
+        self.idxmgmt.assert_index_template_loaded(self.index_name)
+
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @pytest.mark.tag('integration')
+    def test_setup_template_component(self):
+        """
+        Test template setup of component index templates
+        """
+        self.render_config()
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.setupCmd,
+                                              "-E", "setup.template.type=component"])
+
+        assert exit_code == 0
+        self.idxmgmt.assert_component_template_loaded(self.index_name)
+
 
 class TestCommandExportTemplate(BaseTest):
     """