feat(#3453): Add additional schema options to StaticMetadataEnricher (#…

…3454)

* feat(#3453): Add additional schema options to StaticMetadataEnricher

* Fix test

* Minor layout fix

* Fix migration

streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java

@@ -63,7 +63,7 @@ protected DataSeries convertResult(QueryResult.Series series,
 
     });
 
-    return new DataSeries(values.size(), resultingValues, columns, series.getTags());
+    return new DataSeries(resultingValues.size(), resultingValues, columns, series.getTags());
   }
 
   protected SpQueryResult postQuery(QueryResult queryResult,

streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/TransformationExtensionModuleExport.java

@@ -23,6 +23,7 @@
 import org.apache.streampipes.extensions.api.migration.IModelMigrator;
 import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
 import org.apache.streampipes.processors.transformation.jvm.migrations.StaticMetadataEnrichmentProcessorMigrationV1;
+import org.apache.streampipes.processors.transformation.jvm.migrations.StaticMetadataEnrichmentProcessorMigrationV2;
 import org.apache.streampipes.processors.transformation.jvm.processor.array.count.CountArrayProcessor;
 import org.apache.streampipes.processors.transformation.jvm.processor.array.split.SplitArrayProcessor;
 import org.apache.streampipes.processors.transformation.jvm.processor.booloperator.counter.BooleanCounterProcessor;
@@ -94,6 +95,8 @@ public List<IStreamPipesPipelineElement<?>> pipelineElements() {
 
   @Override
   public List<IModelMigrator<?, ?>> migrators() {
-    return List.of(new StaticMetadataEnrichmentProcessorMigrationV1());
+    return List.of(
+        new StaticMetadataEnrichmentProcessorMigrationV1(),
+        new StaticMetadataEnrichmentProcessorMigrationV2());
   }
 }

streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/migrations/StaticMetadataEnrichmentProcessorMigrationV2.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.transformation.jvm.migrations;
+
+import org.apache.streampipes.extensions.api.extractor.IDataProcessorParameterExtractor;
+import org.apache.streampipes.extensions.api.migration.IDataProcessorMigrator;
+import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix;
+import org.apache.streampipes.model.graph.DataProcessorInvocation;
+import org.apache.streampipes.model.migration.MigrationResult;
+import org.apache.streampipes.model.migration.ModelMigratorConfig;
+import org.apache.streampipes.model.staticproperty.CollectionStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
+import org.apache.streampipes.processors.transformation.jvm.processor.staticmetadata.StaticMetaDataEnrichmentProcessor;
+import org.apache.streampipes.sdk.StaticProperties;
+import org.apache.streampipes.sdk.helpers.Labels;
+
+import java.util.List;
+
+import static org.apache.streampipes.processors.transformation.jvm.processor.staticmetadata.StaticMetaDataEnrichmentProcessor.STATIC_METADATA_INPUT_DESCRIPTION;
+import static org.apache.streampipes.processors.transformation.jvm.processor.staticmetadata.StaticMetaDataEnrichmentProcessor.STATIC_METADATA_INPUT_LABEL;
+
+public class StaticMetadataEnrichmentProcessorMigrationV2 implements IDataProcessorMigrator {
+  @Override
+  public ModelMigratorConfig config() {
+    return new ModelMigratorConfig(
+        StaticMetaDataEnrichmentProcessor.ID,
+        SpServiceTagPrefix.DATA_PROCESSOR,
+        1,
+        2
+    );
+  }
+
+  @Override
+  public MigrationResult<DataProcessorInvocation> migrate(DataProcessorInvocation element,
+                                                          IDataProcessorParameterExtractor extractor)
+      throws RuntimeException {
+    element.getStaticProperties()
+        .stream()
+        .filter(sp -> sp.getInternalName().equals(StaticMetaDataEnrichmentProcessor.STATIC_METADATA_INPUT))
+        .map(sp -> (CollectionStaticProperty) sp)
+        .forEach(sp -> {
+          addLabelAndDescriptionConfig((StaticPropertyGroup) sp.getStaticPropertyTemplate());
+          sp.getMembers().forEach(member -> {
+            addLabelAndDescriptionConfig((StaticPropertyGroup) member);
+          });
+        });
+    return MigrationResult.success(element);
+  }
+
+  private void addLabelAndDescriptionConfig(StaticPropertyGroup group) {
+    group.setHorizontalRendering(false);
+    group.getStaticProperties().addAll(
+        List.of(
+            makeFreeTextProperty(STATIC_METADATA_INPUT_LABEL),
+            makeFreeTextProperty(STATIC_METADATA_INPUT_DESCRIPTION)
+        )
+    );
+  }
+
+  private StaticProperty makeFreeTextProperty(String id) {
+    var sp = StaticProperties.stringFreeTextProperty(
+        Labels.withId(
+            id));
+    sp.setOptional(true);
+    sp.setValue("");
+    return sp;
+  }
+}

streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/staticmetadata/StaticMetaDataConfiguration.java

@@ -21,6 +21,8 @@
 public record StaticMetaDataConfiguration(
     String runtimeName,
     String value,
-    String dataType
+    String dataType,
+    String label,
+    String description
 ) {
 }

streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/staticmetadata/StaticMetaDataEnrichmentProcessor.java

@@ -33,6 +33,7 @@
 import org.apache.streampipes.model.schema.EventSchema;
 import org.apache.streampipes.model.schema.PropertyScope;
 import org.apache.streampipes.model.staticproperty.CollectionStaticProperty;
+import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
 import org.apache.streampipes.model.staticproperty.StaticProperty;
 import org.apache.streampipes.model.staticproperty.StaticPropertyGroup;
 import org.apache.streampipes.sdk.StaticProperties;
@@ -53,14 +54,16 @@
 
 public class StaticMetaDataEnrichmentProcessor
     implements IStreamPipesDataProcessor,
-               ResolvesContainerProvidedOutputStrategy<DataProcessorInvocation, ProcessingElementParameterExtractor> {
+    ResolvesContainerProvidedOutputStrategy<DataProcessorInvocation, ProcessingElementParameterExtractor> {
 
   public static final String ID = "org.apache.streampipes.processors.transformation.jvm.processor.staticmetadata";
 
-  protected static final String STATIC_METADATA_INPUT = "static-metadata-input";
+  public static final String STATIC_METADATA_INPUT = "static-metadata-input";
   protected static final String STATIC_METADATA_INPUT_RUNTIME_NAME = "static-metadata-input-runtime-name";
   protected static final String STATIC_METADATA_INPUT_VALUE = "static-metadata-input-value";
   protected static final String STATIC_METADATA_INPUT_DATATYPE = "static-metadata-input-datatype";
+  public static final String STATIC_METADATA_INPUT_LABEL = "static-metadata-input-label";
+  public static final String STATIC_METADATA_INPUT_DESCRIPTION = "static-metadata-input-description";
 
   protected static final String OPTION_BOOL = "Bool";
   protected static final String OPTION_STRING = "String";
@@ -75,45 +78,59 @@ public IDataProcessorConfiguration declareConfig() {
     return DataProcessorConfiguration.create(
         StaticMetaDataEnrichmentProcessor::new,
         ProcessingElementBuilder.create(
-                                    ID,
-                                    1
-                                )
-                                .category(
-                                    DataProcessorType.ENRICH)
-                                .withLocales(
-                                    Locales.EN)
-                                .withAssets(
-                                    ExtensionAssetType.DOCUMENTATION,
-                                    ExtensionAssetType.ICON
-                                )
-                                .requiredCollection(
-                                    Labels.withId(
-                                        STATIC_METADATA_INPUT),
-                                    StaticProperties.stringFreeTextProperty(
-                                        Labels.withId(
-                                            STATIC_METADATA_INPUT_RUNTIME_NAME)),
-                                    StaticProperties.stringFreeTextProperty(
-                                        Labels.withId(
-                                            STATIC_METADATA_INPUT_VALUE)),
-                                    StaticProperties.singleValueSelection(
-                                        Labels.withId(
-                                            STATIC_METADATA_INPUT_DATATYPE),
-                                        Options.from(
-                                            OPTION_BOOL,
-                                            OPTION_STRING,
-                                            OPTION_FLOAT,
-                                            OPTION_INTEGER
-                                        )
-                                    )
-                                )
-                                .requiredStream(
-                                    StreamRequirementsBuilder.any())
-                                .outputStrategy(
-                                    OutputStrategies.customTransformation())
-                                .build()
+                ID,
+                2
+            )
+            .category(
+                DataProcessorType.ENRICH)
+            .withLocales(
+                Locales.EN)
+            .withAssets(
+                ExtensionAssetType.DOCUMENTATION,
+                ExtensionAssetType.ICON
+            )
+            .requiredStaticProperty(
+                StaticProperties.collection(
+                    Labels.withId(
+                        STATIC_METADATA_INPUT),
+                    false,
+                    StaticProperties.stringFreeTextProperty(
+                        Labels.withId(
+                            STATIC_METADATA_INPUT_RUNTIME_NAME)),
+                    StaticProperties.stringFreeTextProperty(
+                        Labels.withId(
+                            STATIC_METADATA_INPUT_VALUE)),
+                    StaticProperties.singleValueSelection(
+                        Labels.withId(
+                            STATIC_METADATA_INPUT_DATATYPE),
+                        Options.from(
+                            OPTION_BOOL,
+                            OPTION_STRING,
+                            OPTION_FLOAT,
+                            OPTION_INTEGER
+                        )
+                    ),
+                    makeFreeTextInput(STATIC_METADATA_INPUT_LABEL, true),
+                    makeFreeTextInput(STATIC_METADATA_INPUT_DESCRIPTION, true)
+                )
+            )
+            .requiredStream(
+                StreamRequirementsBuilder.any())
+            .outputStrategy(
+                OutputStrategies.customTransformation())
+            .build()
     );
   }
 
+  private FreeTextStaticProperty makeFreeTextInput(String label,
+                                                   boolean optional) {
+    var sp = StaticProperties.stringFreeTextProperty(
+        Labels.withId(
+            label));
+    sp.setOptional(optional);
+    return sp;
+  }
+
   @Override
   public EventSchema resolveOutputStrategy(
       DataProcessorInvocation processingElement,
@@ -123,8 +140,8 @@ public EventSchema resolveOutputStrategy(
     var metaDataConfigurations = getMetaDataConfigurations(parameterExtractor);
 
     var eventSchema = processingElement.getInputStreams()
-                                       .get(0)
-                                       .getEventSchema();
+        .get(0)
+        .getEventSchema();
 
     addMetaDataConfigurationPropertiesToEventSchema(metaDataConfigurations, eventSchema);
 
@@ -185,7 +202,9 @@ private StaticMetaDataConfiguration getMetaDataConfiguration(StaticPropertyExtra
     var runtimeName = memberExtractor.textParameter(STATIC_METADATA_INPUT_RUNTIME_NAME);
     var value = memberExtractor.textParameter(STATIC_METADATA_INPUT_VALUE);
     var dataType = memberExtractor.selectedSingleValue(STATIC_METADATA_INPUT_DATATYPE, String.class);
-    return new StaticMetaDataConfiguration(runtimeName, value, dataType);
+    var label = memberExtractor.textParameter(STATIC_METADATA_INPUT_LABEL);
+    var description = memberExtractor.textParameter(STATIC_METADATA_INPUT_DESCRIPTION);
+    return new StaticMetaDataConfiguration(runtimeName, value, dataType, label, description);
   }
 
   protected Object castValueOfMetaDataConfiguration(StaticMetaDataConfiguration staticMetaDataConfiguration) {
@@ -214,7 +233,7 @@ private void addMetaDataConfigurationPropertiesToEventSchema(
     for (StaticMetaDataConfiguration metaDataConfiguration : metaDataConfigurations) {
       var metaDataEventProperty = getMetaDataEventProperty(metaDataConfiguration);
       eventSchema.getEventProperties()
-                 .add(metaDataEventProperty);
+          .add(metaDataEventProperty);
     }
   }
 
@@ -226,6 +245,8 @@ private EventPropertyPrimitive getMetaDataEventProperty(
             transformToStreamPipesDataType(metaDataConfiguration.dataType()),
             metaDataConfiguration.runtimeName()
         )
+        .label(metaDataConfiguration.label())
+        .description(metaDataConfiguration.description())
         .scope(PropertyScope.MEASUREMENT_PROPERTY)
         .build();
   }

streampipes-extensions/streampipes-processors-transformation-jvm/src/main/resources/org.apache.streampipes.processors.transformation.jvm.processor.staticmetadata/strings.en

@@ -30,3 +30,8 @@ static-metadata-input-value.description=This is used for the runtime value in th
 static-metadata-input-datatype.title=Data Type
 static-metadata-input-datatype.description=Select the data type of the value
 
+static-metadata-input-label.title=Label
+static-metadata-input-label.description=An optional short label which describes the field
+
+static-metadata-input-description.title=Description
+static-metadata-input-description.description=An optional description of the field

streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/staticmetadata/StaticMetaDataEnrichmentProcessorTest.java

@@ -104,15 +104,16 @@ public void castValueOfMetaDataConfigurationString() {
 
   @Test
   public void castValueOfMetaDataConfigurationDefault() {
-    StaticMetaDataConfiguration config = new StaticMetaDataConfiguration("runtimeName", "default", "default");
+    StaticMetaDataConfiguration config = new StaticMetaDataConfiguration(
+        "runtimeName", "default", "default", "", "");
 
     assertEquals("default", processor.castValueOfMetaDataConfiguration(config));
   }
 
   private static StaticMetaDataConfiguration getSampleMetaDataConfiguration(
       String value, String dataType
   ) {
-    return new StaticMetaDataConfiguration("runtimeName", value, dataType);
+    return new StaticMetaDataConfiguration("runtimeName", value, dataType, "", "");
   }
 
 }

streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java

@@ -33,6 +33,7 @@
 import org.apache.http.entity.ContentType;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 
 public class CustomTransformOutputSchemaGenerator extends OutputSchemaGenerator<CustomTransformOutputStrategy> {
 
@@ -79,7 +80,7 @@ private EventSchema makeRequest() {
   }
 
   private EventSchema handleResponse(Response httpResp) throws JsonSyntaxException, IOException {
-    String resp = httpResp.returnContent().asString();
+    String resp = httpResp.returnContent().asString(StandardCharsets.UTF_8);
 
     return JacksonSerializer
         .getObjectMapper()

ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html

@@ -20,7 +20,7 @@
     fxLayout="column"
     fxFlex="100"
     class="panel-outer"
-    [ngStyle]="{ margin: outerMargin, height: '100%' }"
+    [ngStyle]="{ margin: outerMargin }"
 >
     <div
         class="general-panel-header"
@@ -40,7 +40,8 @@
 
     <div
         class="general-panel"
-        [ngStyle]="{ padding: innerPadding, height: '100%' }"
+        fxFlex="100"
+        [ngStyle]="{ padding: innerPadding }"
     >
         <ng-content fxFlex="100"></ng-content>
     </div>

ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.scss

@@ -19,7 +19,7 @@
 .panel-outer {
     margin-top: 15px;
     margin-bottom: 15px;
-    border: 2px solid var(--color-bg-3);
+    border: 1px solid var(--color-bg-3);
 }
 
 .general-panel {
@@ -29,19 +29,19 @@
 .general-panel-header {
     margin-top: -1px;
     background: var(--color-bg-2);
-    border: 0px solid var(--color-bg-3);
-    border-bottom: 2px solid var(--color-bg-3);
+    border-top: 1px solid var(--color-bg-3);
+    border-bottom: 1px solid var(--color-bg-3);
     padding-left: 5px;
-    height: 50px;
-    min-height: 50px;
-    max-height: 50px;
+    height: 40px;
+    min-height: 40px;
+    max-height: 40px;
 }
 
 .content-box-title {
     margin-left: 5px;
     padding-left: 5px;
-    border-left: 4px solid var(--color-accent);
-    font-weight: 600;
+    border-left: 3px solid var(--color-accent);
+    font-weight: 500;
     font-size: 13pt;
     white-space: nowrap;
 }