Migrate obs-infraobs-integrations to package-spec v3 #2

packages/golang/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.1.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8171
 - version: "1.0.0"
   changes:
     - description: Make Golang GA.

packages/golang/manifest.yml

@@ -1,14 +1,16 @@
-format_version: 2.0.0
+format_version: "3.0.0"
 name: golang
 title: Golang
-version: "1.0.0"
+version: "1.1.0"
 description: This Elastic integration collects metrics from Golang applications.
 type: integration
 categories:
   - observability
 conditions:
-  kibana.version: ^8.7.1
-  elastic.subscription: basic
+  kibana:
+    version: ^8.7.1
+  elastic:
+    subscription: basic
 screenshots:
   - src: /img/golang-heap-dashboard.png
     title: Golang Heap dashboard
@@ -96,3 +98,4 @@ policy_templates:
               #    -----END CERTIFICATE-----
 owner:
   github: elastic/obs-infraobs-integrations
+  type: elastic

packages/golang/validation.yml

@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - SVR00002

packages/hadoop/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.3.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8171
 - version: "1.2.1"
   changes:
     - description: Remove forwarded tag from metrics data streams.

packages/hadoop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json

@@ -5,11 +5,15 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
                 "module": "httpjson",
                 "original": "{\"allocatedMB\":-1,\"allocatedVCores\":-1,\"amContainerLogs\":\"http://localhost:8042/node/containerlogs/container_1646029193591_0003_01_000001/bhagyaraj\",\"amHostHttpAddress\":\"localhost:8042\",\"amNodeLabelExpression\":\"\",\"amRPCAddress\":\"localhost:38279\",\"applicationTags\":\"\",\"applicationType\":\"MAPREDUCE\",\"clusterId\":1646029193591,\"clusterUsagePercentage\":0,\"diagnostics\":\"\",\"elapsedTime\":35114,\"finalStatus\":\"SUCCEEDED\",\"finishedTime\":1646030395258,\"id\":\"application_1646029193591_0003\",\"launchTime\":1646030360297,\"logAggregationStatus\":\"DISABLED\",\"masterNodeId\":\"localhost:45960\",\"memorySeconds\":103467,\"name\":\"wordcount\",\"numAMContainerPreempted\":0,\"numNonAMContainerPreempted\":0,\"preemptedMemorySeconds\":0,\"preemptedResourceMB\":0,\"preemptedResourceSecondsMap\":{},\"preemptedResourceVCores\":0,\"preemptedVcoreSeconds\":0,\"priority\":0,\"progress\":100,\"queue\":\"default\",\"queueUsagePercentage\":0,\"reservedMB\":-1,\"reservedVCores\":-1,\"resourceSecondsMap\":{\"entry\":{\"key\":\"vcores\",\"value\":\"57\"}},\"runningContainers\":-1,\"startedTime\":1646030360144,\"state\":\"FINISHED\",\"timeouts\":{\"timeout\":[{\"expiryTime\":\"UNLIMITED\",\"remainingTimeInSeconds\":-1,\"type\":\"LIFETIME\"}]},\"trackingUI\":\"History\",\"trackingUrl\":\"http://localhost:8088/proxy/application_1646029193591_0003/\",\"unmanagedApplication\":false,\"user\":\"bhagyaraj\",\"vcoreSeconds\":57}",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "application": {
@@ -38,11 +42,15 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
                 "module": "httpjson",
                 "original": "{\"allocatedMB\":-1,\"allocatedVCores\":-1,\"amContainerLogs\":\"http://localhost:8042/node/containerlogs/container_1646029193591_0001_02_000001/bhagyaraj\",\"amHostHttpAddress\":\"localhost:8042\",\"amNodeLabelExpression\":\"\",\"applicationTags\":\"\",\"applicationType\":\"MAPREDUCE\",\"clusterId\":1646029193591,\"clusterUsagePercentage\":0,\"diagnostics\":\"Application application_1646029193591_0001 failed 2 times due to AM Container for appattempt_1646029193591_0001_000002 exited with  exitCode: 1\\\\nFailing this attempt.Diagnostics: [2022-02-28 12:03:43.814]Exception from container-launch.\\\\nContainer id: container_1646029193591_0001_02_000001\\\\nExit code: 1\\\\n\\\\n[2022-02-28 12:03:43.822]Container exited with a non-zero exit code 1. Error file: prelaunch.err.\\\\nLast 4096 bytes of prelaunch.err :\\\\nLast 4096 bytes of stderr :\\\\nError: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster\\\\nCaused by: java.lang.ClassNotFoundException: org.apache.hadoop.mapreduce.v2.app.MRAppMaster\\\\n\\\\nPlease check whether your \\\\u003cHADOOP_HOME\\\\u003e/etc/hadoop/mapred-site.xml contains the below configuration:\\\\n\\\\u003cproperty\\\\u003e\\\\n  \\\\u003cname\\\\u003eyarn.app.mapreduce.am.env\\\\u003c/name\\\\u003e\\\\n  \\\\u003cvalue\\\\u003eHADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}\\\\u003c/value\\\\u003e\\\\n\\\\u003c/property\\\\u003e\\\\n\\\\u003cproperty\\\\u003e\\\\n  \\\\u003cname\\\\u003emapreduce.map.env\\\\u003c/name\\\\u003e\\\\n  \\\\u003cvalue\\\\u003eHADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}\\\\u003c/value\\\\u003e\\\\n\\\\u003c/property\\\\u003e\\\\n\\\\u003cproperty\\\\u003e\\\\n  \\\\u003cname\\\\u003emapreduce.reduce.env\\\\u003c/name\\\\u003e\\\\n  \\\\u003cvalue\\\\u003eHADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}\\\\u003c/value\\\\u003e\\\\n\\\\u003c/property\\\\u003e\\\\n\\\\n[2022-02-28 12:03:43.823]Container exited with a non-zero exit code 1. Error file: prelaunch.err.\\\\nLast 4096 bytes of prelaunch.err :\\\\nLast 4096 bytes of stderr :\\\\nError: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster\\\\nCaused by: java.lang.ClassNotFoundException: org.apache.hadoop.mapreduce.v2.app.MRAppMaster\\\\n\\\\nPlease check whether your \\\\u003cHADOOP_HOME\\\\u003e/etc/hadoop/mapred-site.xml contains the below configuration:\\\\n\\\\u003cproperty\\\\u003e\\\\n  \\\\u003cname\\\\u003eyarn.app.mapreduce.am.env\\\\u003c/name\\\\u003e\\\\n  \\\\u003cvalue\\\\u003eHADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}\\\\u003c/value\\\\u003e\\\\n\\\\u003c/property\\\\u003e\\\\n\\\\u003cproperty\\\\u003e\\\\n  \\\\u003cname\\\\u003emapreduce.map.env\\\\u003c/name\\\\u003e\\\\n  \\\\u003cvalue\\\\u003eHADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}\\\\u003c/value\\\\u003e\\\\n\\\\u003c/property\\\\u003e\\\\n\\\\u003cproperty\\\\u003e\\\\n  \\\\u003cname\\\\u003emapreduce.reduce.env\\\\u003c/name\\\\u003e\\\\n  \\\\u003cvalue\\\\u003eHADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}\\\\u003c/value\\\\u003e\\\\n\\\\u003c/property\\\\u003e\\\\n\\\\nFor more detailed output, check the application tracking page: http://localhost:8088/cluster/app/application_1646029193591_0001 Then click on links to logs of each attempt.\\\\n. Failing the application.\",\"elapsedTime\":9925,\"finalStatus\":\"FAILED\",\"finishedTime\":1646030023835,\"id\":\"application_1646029193591_0001\",\"launchTime\":1646030016899,\"logAggregationStatus\":\"DISABLED\",\"masterNodeId\":\"localhost:45960\",\"memorySeconds\":15708,\"name\":\"wordcount\",\"numAMContainerPreempted\":0,\"numNonAMContainerPreempted\":0,\"preemptedMemorySeconds\":0,\"preemptedResourceMB\":0,\"preemptedResourceSecondsMap\":{},\"preemptedResourceVCores\":0,\"preemptedVcoreSeconds\":0,\"priority\":0,\"progress\":0,\"queue\":\"default\",\"queueUsagePercentage\":0,\"reservedMB\":-1,\"reservedVCores\":-1,\"resourceSecondsMap\":{\"entry\":{\"key\":\"vcores\",\"value\":\"7\"}},\"runningContainers\":-1,\"startedTime\":1646030013910,\"state\":\"FAILED\",\"timeouts\":{\"timeout\":[{\"expiryTime\":\"UNLIMITED\",\"remainingTimeInSeconds\":-1,\"type\":\"LIFETIME\"}]},\"trackingUI\":\"History\",\"trackingUrl\":\"http://localhost:8088/cluster/app/application_1646029193591_0001\",\"unmanagedApplication\":false,\"user\":\"bhagyaraj\",\"vcoreSeconds\":7}",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "application": {

packages/hadoop/data_stream/application/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,5 @@
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"
 fields:
   tags:
     - preserve_original_event

packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
       value: '8.5.1'
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric
@@ -15,7 +15,7 @@ processors:
       value: httpjson
   - set:
       field: event.category
-      value: database
+      value: [database]
   - rename:
       field: message
       target_field: event.original

packages/hadoop/data_stream/application/fields/beats.yml

@@ -1,6 +1,3 @@
 - name: input.type
   description: Type of Filebeat input.
   type: keyword
-- name: tags
-  type: keyword
-  description: User defined tags

packages/hadoop/data_stream/application/fields/ecs.yml

@@ -14,3 +14,4 @@
   name: host.ip
 - external: ecs
   name: tags
+  description: User defined tags.

packages/hadoop/data_stream/application/sample_event.json

@@ -22,13 +22,17 @@
     },
     "event": {
         "agent_id_status": "verified",
-        "category": "database",
+        "category": [
+            "database"
+        ],
         "created": "2023-02-02T12:03:41.178Z",
         "dataset": "hadoop.application",
         "ingested": "2023-02-02T12:03:42Z",
         "kind": "metric",
         "module": "httpjson",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "hadoop": {
         "application": {

packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json

@@ -6,9 +6,13 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "cluster": {
@@ -34,9 +38,13 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "cluster": {

packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml

@@ -6,13 +6,13 @@ processors:
       value: '8.5.1'
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric
   - set:
       field: event.category
-      value: database
+      value: [database]
   - rename:
       field: http
       target_field: hadoop

packages/hadoop/data_stream/cluster/fields/ecs.yml

@@ -42,4 +42,4 @@
   dimension: true
 - external: ecs
   name: host.name
-  dimension: true
+  dimension: true

packages/hadoop/data_stream/cluster/manifest.yml

@@ -45,5 +45,6 @@ streams:
         show_user: false
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
 elasticsearch:
   index_mode: "time_series"

packages/hadoop/data_stream/cluster/sample_event.json

@@ -22,13 +22,17 @@
     },
     "event": {
         "agent_id_status": "verified",
-        "category": "database",
+        "category": [
+            "database"
+        ],
         "dataset": "hadoop.cluster",
         "duration": 50350559,
         "ingested": "2022-04-04T17:22:25Z",
         "kind": "metric",
         "module": "http",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "hadoop": {
         "cluster": {

packages/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json

@@ -6,9 +6,13 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "datanode": {
@@ -42,9 +46,13 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "datanode": {

packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml

@@ -6,13 +6,13 @@ processors:
       value: '8.5.1'
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric
   - set:
       field: event.category
-      value: database
+      value: [database]
   - rename:
       field: http
       target_field: hadoop

packages/hadoop/data_stream/datanode/fields/ecs.yml

@@ -42,4 +42,4 @@
   dimension: true
 - external: ecs
   name: host.name
-  dimension: true
+  dimension: true

packages/hadoop/data_stream/datanode/manifest.yml

@@ -45,5 +45,6 @@ streams:
         show_user: false
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
 elasticsearch:
   index_mode: "time_series"

packages/hadoop/data_stream/datanode/sample_event.json

@@ -22,13 +22,17 @@
     },
     "event": {
         "agent_id_status": "verified",
-        "category": "database",
+        "category": [
+            "database"
+        ],
         "dataset": "hadoop.datanode",
         "duration": 241651987,
         "ingested": "2023-02-02T12:05:05Z",
         "kind": "metric",
         "module": "http",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "hadoop": {
         "datanode": {

packages/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json

@@ -6,9 +6,13 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "namenode": {

packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml

@@ -6,13 +6,13 @@ processors:
       value: '8.5.1'
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric
   - set:
       field: event.category
-      value: database
+      value: [database]
   - rename:
       field: http
       target_field: hadoop

packages/hadoop/data_stream/namenode/fields/ecs.yml

@@ -42,4 +42,4 @@
   dimension: true
 - external: ecs
   name: host.name
-  dimension: true
+  dimension: true

packages/hadoop/data_stream/namenode/fields/fields.yml

@@ -1,6 +1,5 @@
 - name: hadoop.namenode
   type: group
-  release: beta
   fields:
     - name: blocks
       type: group

packages/hadoop/data_stream/namenode/manifest.yml

@@ -45,5 +45,6 @@ streams:
         show_user: false
         description: >
           Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
 elasticsearch:
   index_mode: "time_series"

packages/hadoop/data_stream/namenode/sample_event.json

@@ -22,13 +22,17 @@
     },
     "event": {
         "agent_id_status": "verified",
-        "category": "database",
+        "category": [
+            "database"
+        ],
         "dataset": "hadoop.namenode",
         "duration": 341259289,
         "ingested": "2022-03-28T11:36:09Z",
         "kind": "metric",
         "module": "http",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "hadoop": {
         "namenode": {

packages/hadoop/data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json

@@ -6,9 +6,13 @@
                 "version": "8.5.1"
             },
             "event": {
-                "category": "database",
+                "category": [
+                    "database"
+                ],
                 "kind": "metric",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "hadoop": {
                 "node_manager": {